diff --git a/.bazelrc b/.bazelrc new file mode 100644 index 00000000000..30c975b9e20 --- /dev/null +++ b/.bazelrc @@ -0,0 +1,108 @@ +# Optimizations used for TF Serving release builds. +build:release --copt=-mavx +build:release --copt=-msse4.2 + +# Options used to build with CUDA. +build:cuda --repo_env TF_NEED_CUDA=1 +build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain +build:cuda --@local_config_cuda//:enable_cuda + +# Options used to build with CUDA clang +build:cuda_clang --config=cuda +build:cuda_clang --copt=-Wno-gnu-offsetof-extensions +build:cuda_clang --copt=-Wno-error=unused-command-line-argument +build:cuda_clang --host_copt=-Wno-error=unused-command-line-argument +build:cuda_clang --repo_env TF_NEED_TENSORRT=0 +build:cuda_clang --action_env=TF_CUDA_CLANG="1" +build:cuda_clang --@local_config_cuda//:cuda_compiler=clang +build:cuda_clang --repo_env=TF_CUDA_COMPUTE_CAPABILITIES="sm_60,sm_70,sm_80,compute_90" + +build:cuda_clang --repo_env=HERMETIC_CUDA_VERSION="12.2.0" +build:cuda_clang --repo_env=HERMETIC_CUDNN_VERSION="8.9.4.25" +build:cuda_clang --action_env=GCC_HOST_COMPILER_PATH="/usr/bin/gcc-10" +build:cuda_clang --action_env=CLANG_CUDA_COMPILER_PATH="/usr/lib/llvm-17/bin/clang" + +build:kokoro --repo_env=CC="/usr/bin/gcc-10" +build:kokoro --action_env=CC="/usr/bin/gcc-10" + +# Options used to build with TPU support. +build:tpu --define=with_tpu_support=true --define=framework_shared_object=false +build:tpu --copt=-DLIBTPU_ON_GCE + +# Please note that MKL on MacOS or windows is still not supported. +# If you would like to use a local MKL instead of downloading, please set the +# environment variable "TF_MKL_ROOT" every time before build. +build:mkl --define=build_with_mkl=true --define=enable_mkl=true --define=build_with_openmp=false +build:mkl --define=tensorflow_mkldnn_contraction_kernel=0 + +# This config option is used to enable MKL-DNN open source library only, +# without depending on MKL binary version. +build:mkl_open_source_only --define=build_with_mkl_dnn_only=true +build:mkl_open_source_only --define=build_with_mkl=true --define=enable_mkl=true +build:mkl_open_source_only --define=tensorflow_mkldnn_contraction_kernel=0 + +# Config setting to build oneDNN with Compute Library for the Arm Architecture (ACL). +# This build is for the inference regime only. +build:mkl_aarch64 --define=build_with_mkl_aarch64=true --define=enable_mkl=true +build:mkl_aarch64 --define=tensorflow_mkldnn_contraction_kernel=0 +build:mkl_aarch64 --define=build_with_mkl_opensource=true +build:mkl_aarch64 --define=build_with_openmp=true +build:mkl_aarch64 --copt=-march=armv8.2-a +build:mkl_aarch64 --copt=-O3 + +build --define=build_with_onednn_v2=true +build --define=xnn_enable_avxvnni=false +build --define=xnn_enable_avxvnniint8=false +build --define=xnn_enable_avx256vnnigfni=false +build --define=xnn_enable_avx512amx=false +build --define=xnn_enable_avx512fp16=false + +# Processor native optimizations (depends on build host capabilities). +build:nativeopt --copt=-march=native +build:nativeopt --host_copt=-march=native +build:nativeopt --copt=-O3 + +build --keep_going +build --verbose_failures=true +build --spawn_strategy=standalone +build --genrule_strategy=standalone + +build --define=grpc_no_ares=true + +# Sets the default Apple platform to macOS. +build --apple_platform_type=macos + +build -c opt + +# LLVM, MLIR and TF requires C++17. +build --cxxopt=-std=c++17 +build --host_cxxopt=-std=c++17 + +# Adding "--cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0" creates parity with TF +# compilation options. It also addresses memory use due to +# copy-on-write semantics of std::strings of the older ABI. +build --cxxopt=-D_GLIBCXX_USE_CXX11_ABI=0 + +build --workspace_status_command=/proc/self/cwd/tools/gen_status_stamp.sh + +build --experimental_repo_remote_exec + +# TF now has `cc_shared_library` targets, so it needs the experimental flag +build --experimental_cc_shared_library + +# Yggdrasil Decision Forests (internal library of TensorFlow Decision Forests) +# uses TensorFlow for all IO operations. +build --define=use_tensorflow_io=1 + +# TensorFlow Decision Forests does not use Absl concurrency primitives on MacOs. +# Reason: TensorFlow/ABSL ODR violation (b/214189609) # copybara:strip +build:macos --define std_synchronization_primitives=1 + +# Taken from https://github.com/openxla/xla/blob/99559d7a4f7c55490f46385ad29a3cbf9c3911af/warnings.bazelrc#L6 +# We silence warnings for code in `external`. +build --per_file_copt=external/.*@-w +build --host_per_file_copt=external/.*@-w +build --copt=-Wno-macro-redefined # absl vs tsl logging clash +build --copt=-Wno-sign-compare # int as loop variable +build --copt=-Wno-deprecated-declarations +build --copt=-Wno-unused-but-set-variable # due to `ifdefs` in ml_dtypes diff --git a/.github/issue_template.md b/.github/issue_template.md new file mode 100644 index 00000000000..bf2500004fa --- /dev/null +++ b/.github/issue_template.md @@ -0,0 +1,55 @@ +Please go to Stack Overflow for help and support: + +https://stackoverflow.com/questions/tagged/tensorflow-serving + +If you open a GitHub issue, here is our policy: + +1. It must be a bug, a feature request, or a significant problem with + documentation (for small docs fixes please send a PR instead). +2. The form below must be filled out. + +**Here's why we have that policy**: TensorFlow developers respond to issues. +We want to focus on work that benefits the whole community, e.g., fixing bugs +and adding features. Support only helps individuals. GitHub also notifies +thousands of people when issues are filed. We want them to see you communicating +an interesting problem, rather than being redirected to Stack Overflow. + +----------------------- +## Feature Request +If this is a feature request, please fill out the following form in full: + +### Describe the problem the feature is intended to solve +A clear and concise description of what the problem is. Ex. I'm always +frustrated when [...] + +### Describe the solution +A clear and concise description of what you want to happen. + +### Describe alternatives you've considered +A clear and concise description of any alternative solutions or features you've +considered. + +### Additional context +Add any other context or screenshots about the feature request here. + +## Bug Report +If this is a bug report, please fill out the following form in full: + +### System information +- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: +- **TensorFlow Serving installed from (source or binary)**: +- **TensorFlow Serving version**: + +### Describe the problem +Describe the problem clearly here. Be sure to convey here why it's a bug in +TensorFlow Serving. + +### Exact Steps to Reproduce +Please include all steps necessary for someone to reproduce this issue on their +own machine. If not, skip this section. + +### Source code / logs +Include any logs or source code that would be helpful to diagnose the problem. +If including tracebacks, please include the full traceback. Large logs and files +should be attached. Try to provide a reproducible test case that is the bare +minimum necessary to generate the problem. diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml new file mode 100644 index 00000000000..e93efeb8251 --- /dev/null +++ b/.github/workflows/stale.yaml @@ -0,0 +1,66 @@ +# Copyright 2023 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +# This workflow alerts and then closes the stale issues/PRs after specific time +# You can adjust the behavior by modifying this file. +# For more information, see: +# https://github.com/actions/stale + +name: 'Close stale issues and PRs' +"on": + schedule: + - cron: "30 1 * * *" +permissions: + contents: read + issues: write + pull-requests: write +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: 'actions/stale@v7' + with: + # Comma separated list of labels that can be assigned to issues to exclude them from being marked as stale. + exempt-issue-labels: 'override-stale' + # Comma separated list of labels that can be assigned to PRs to exclude them from being marked as stale. + exempt-pr-labels: "override-stale" + # Limit the No. of API calls in one run default value is 30. + operations-per-run: 1000 + # Prevent to remove stale label when PRs or issues are updated. + remove-stale-when-updated: false + # comment on issue if not active for more then 7 days. + stale-issue-message: 'This issue has been marked stale because it has no recent activity since 7 days. It will be closed if no further activity occurs. Thank you.' + # comment on PR if not active for more then 14 days. + stale-pr-message: 'This PR has been marked stale because it has no recent activity since 14 days. It will be closed if no further activity occurs. Thank you.' + # comment on issue if stale for more then 7 days. + close-issue-message: This issue was closed due to lack of activity after being marked stale for past 7 days. + # comment on PR if stale for more then 14 days. + close-pr-message: This PR was closed due to lack of activity after being marked stale for past 14 days. + # Number of days of inactivity before an Issue Request becomes stale + days-before-issue-stale: 7 + # Number of days of inactivity before a stale Issue is closed + days-before-issue-close: 7 + # reason for closed the issue default value is not_planned + close-issue-reason: completed + # Number of days of inactivity before a stale PR is closed + days-before-pr-close: 14 + # Number of days of inactivity before an PR Request becomes stale + days-before-pr-stale: 14 + # Check for label to stale or close the issue/PR + any-of-labels: 'stat:awaiting response' + # override stale to stalled for PR + stale-pr-label: 'stale' + # override stale to stalled for Issue + stale-issue-label: "stale" diff --git a/.gitignore b/.gitignore index 150500ce279..fd22d60859d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ node_modules /bazel-out /bazel-serving /bazel-tensorflow +/bazel-tensorflow_serving /bazel-testlogs /bazel-tf /bazel-workspace diff --git a/.gitmodules b/.gitmodules index 8f873f6f781..e69de29bb2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +0,0 @@ -[submodule "tensorflow"] - path = tensorflow - url = https://github.com/tensorflow/tensorflow.git -[submodule "tf_models"] - path = tf_models - url = https://github.com/tensorflow/models.git diff --git a/README.md b/README.md index d4ddfd7516e..263fe7a9572 100644 --- a/README.md +++ b/README.md @@ -1,54 +1,121 @@ -#TensorFlow Serving - -[![Build Status](http://ci.tensorflow.org/buildStatus/icon?job=serving-master-cpu)](http://ci.tensorflow.org/job/serving-master-cpu) - -TensorFlow Serving is an open-source software library for serving -machine learning models. It deals with the *inference* aspect of machine -learning, taking models after *training* and managing their lifetimes, providing -clients with versioned access via a high-performance, reference-counted lookup -table. - -Multiple models, or indeed multiple versions of the same model, can be served -simultaneously. This flexibility facilitates canarying new versions, -non-atomically migrating clients to new models or versions, and A/B testing -experimental models. - -The primary use-case is high-performance production serving, but the same -serving infrastructure can also be used in bulk-processing (e.g. map-reduce) -jobs to pre-compute inference results or analyze model performance. In both -scenarios, GPUs can substantially increase inference throughput. TensorFlow -Serving comes with a scheduler that groups individual inference requests into -batches for joint execution on a GPU, with configurable latency controls. - -TensorFlow Serving has out-of-the-box support for TensorFlow models (naturally), -but at its core it manages arbitrary versioned items (*servables*) with -pass-through to their native APIs. In addition to trained TensorFlow models, -servables can include other assets needed for inference such as embeddings, -vocabularies and feature transformation configs, or even non-TensorFlow-based -machine learning models. - -The architecture is highly modular. You can use some parts individually (e.g. -batch scheduling) or use all the parts together. There are numerous plug-in -points; perhaps the most useful ways to extend the system are: -(a) [creating a new type of servable](tensorflow_serving/g3doc/custom_servable.md); -(b) [creating a custom source of servable versions](tensorflow_serving/g3doc/custom_source.md). +# TensorFlow Serving -**If you'd like to contribute to TensorFlow Serving, be sure to review the -[contribution guidelines](CONTRIBUTING.md).** +[![Ubuntu Build Status](https://storage.googleapis.com/tensorflow-serving-kokoro-build-badges-bucket/ubuntu.svg)](https://storage.googleapis.com/tensorflow-serving-kokoro-build-badges-bucket/ubuntu.html) +[![Ubuntu Build Status at TF HEAD](https://storage.googleapis.com/tensorflow-serving-kokoro-build-badges-bucket/ubuntu-tf-head.svg)](https://storage.googleapis.com/tensorflow-serving-kokoro-build-badges-bucket/ubuntu-tf-head.html) +![Docker CPU Nightly Build Status](https://storage.googleapis.com/tensorflow-serving-kokoro-build-badges-bucket/docker-cpu-nightly.svg) +![Docker GPU Nightly Build Status](https://storage.googleapis.com/tensorflow-serving-kokoro-build-badges-bucket/docker-gpu-nightly.svg) + +---- +TensorFlow Serving is a flexible, high-performance serving system for +machine learning models, designed for production environments. It deals with +the *inference* aspect of machine learning, taking models after *training* and +managing their lifetimes, providing clients with versioned access via +a high-performance, reference-counted lookup table. +TensorFlow Serving provides out-of-the-box integration with TensorFlow models, +but can be easily extended to serve other types of models and data. + +To note a few features: + +- Can serve multiple models, or multiple versions of the same model + simultaneously +- Exposes both gRPC as well as HTTP inference endpoints +- Allows deployment of new model versions without changing any client code +- Supports canarying new versions and A/B testing experimental models +- Adds minimal latency to inference time due to efficient, low-overhead + implementation +- Features a scheduler that groups individual inference requests into batches + for joint execution on GPU, with configurable latency controls +- Supports many *servables*: Tensorflow models, embeddings, vocabularies, + feature transformations and even non-Tensorflow-based machine learning + models + +## Serve a Tensorflow model in 60 seconds +```bash +# Download the TensorFlow Serving Docker image and repo +docker pull tensorflow/serving + +git clone https://github.com/tensorflow/serving +# Location of demo models +TESTDATA="$(pwd)/serving/tensorflow_serving/servables/tensorflow/testdata" + +# Start TensorFlow Serving container and open the REST API port +docker run -t --rm -p 8501:8501 \ + -v "$TESTDATA/saved_model_half_plus_two_cpu:/models/half_plus_two" \ + -e MODEL_NAME=half_plus_two \ + tensorflow/serving & + +# Query the model using the predict API +curl -d '{"instances": [1.0, 2.0, 5.0]}' \ + -X POST http://localhost:8501/v1/models/half_plus_two:predict + +# Returns => { "predictions": [2.5, 3.0, 4.5] } +``` + +## End-to-End Training & Serving Tutorial + +Refer to the official Tensorflow documentations site for [a complete tutorial to train and serve a Tensorflow Model](https://www.tensorflow.org/tfx/tutorials/serving/rest_simple). + + +## Documentation -**We use [GitHub issues](https://github.com/tensorflow/serving/issues) for -tracking requests and bugs. +### Set up -# Download and Setup +The easiest and most straight-forward way of using TensorFlow Serving is with +Docker images. We highly recommend this route unless you have specific needs +that are not addressed by running in a container. -See [install instructions](tensorflow_serving/g3doc/setup.md). +* [Install Tensorflow Serving using Docker](tensorflow_serving/g3doc/docker.md) + *(Recommended)* +* [Install Tensorflow Serving without Docker](tensorflow_serving/g3doc/setup.md) + *(Not Recommended)* +* [Build Tensorflow Serving from Source with Docker](tensorflow_serving/g3doc/building_with_docker.md) +* [Deploy Tensorflow Serving on Kubernetes](tensorflow_serving/g3doc/serving_kubernetes.md) -##Tutorials +### Use + +#### Export your Tensorflow model + +In order to serve a Tensorflow model, simply export a SavedModel from your +Tensorflow program. +[SavedModel](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md) +is a language-neutral, recoverable, hermetic serialization format that enables +higher-level systems and tools to produce, consume, and transform TensorFlow +models. + +Please refer to [Tensorflow documentation](https://www.tensorflow.org/guide/saved_model#save_and_restore_models) +for detailed instructions on how to export SavedModels. + +#### Configure and Use Tensorflow Serving + +* [Follow a tutorial on Serving Tensorflow models](tensorflow_serving/g3doc/serving_basic.md) +* [Configure Tensorflow Serving to make it fit your serving use case](tensorflow_serving/g3doc/serving_config.md) +* Read the [Performance Guide](tensorflow_serving/g3doc/performance.md) +and learn how to [use TensorBoard to profile and optimize inference requests](tensorflow_serving/g3doc/tensorboard.md) +* Read the [REST API Guide](tensorflow_serving/g3doc/api_rest.md) +or [gRPC API definition](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/apis) +* [Use SavedModel Warmup if initial inference requests are slow due to lazy initialization of graph](tensorflow_serving/g3doc/saved_model_warmup.md) +* [If encountering issues regarding model signatures, please read the SignatureDef documentation](tensorflow_serving/g3doc/signature_defs.md) +* If using a model with custom ops, [learn how to serve models with custom ops](tensorflow_serving/g3doc/custom_op.md) + +### Extend + +Tensorflow Serving's architecture is highly modular. You can use some parts +individually (e.g. batch scheduling) and/or extend it to serve new use cases. + +* [Ensure you are familiar with building Tensorflow Serving](tensorflow_serving/g3doc/building_with_docker.md) +* [Learn about Tensorflow Serving's architecture](tensorflow_serving/g3doc/architecture.md) +* [Explore the Tensorflow Serving C++ API reference](https://www.tensorflow.org/tfx/serving/api_docs/cc/) +* [Create a new type of Servable](tensorflow_serving/g3doc/custom_servable.md) +* [Create a custom Source of Servable versions](tensorflow_serving/g3doc/custom_source.md) + +## Contribute + + +**If you'd like to contribute to TensorFlow Serving, be sure to review the +[contribution guidelines](CONTRIBUTING.md).** -* [Basic tutorial](tensorflow_serving/g3doc/serving_basic.md) -* [Advanced tutorial](tensorflow_serving/g3doc/serving_advanced.md) -##For more information +## For more information -* [Serving architecture overview](tensorflow_serving/g3doc/architecture_overview.md) -* [TensorFlow website](http://tensorflow.org) +Please refer to the official [TensorFlow website](http://tensorflow.org) for +more information. diff --git a/RELEASE.md b/RELEASE.md index 2afa7bd3357..1f2f9235ead 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,1485 @@ + + +# Release 2.19.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.19.0. + +# Release 2.19.0-rc0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Update release notes for 2.18.0-rc0 (commit: a2253e5cb54de2f726e0b7d1756deb354f0ef6b0) +* cleanup: explicitly set use_java_stubby_library (commit: c22da48061f411b3387e4c3c995d710132a980de) +* cleanup: explicitly set use_java_stubby_library (commit: 5f55e76fbcf192d6026ddb2f284bfb16a921ec81) +* cleanup: explicitly set use_java_stubby_library (commit: b2be8a3ae3d868f8d3e00917a07aac5db5e0d5d1) +* Fix invalid argument to proto_library (commit: 36aa4a60e4fdd833f304136677cb9f97792e7aec) +* Enable serialization of predict response as tensor content. (commit: 1e165518394ca6461859a71284721eb1ca1b72e7) +* Prepare code for breaking change in Protobuf C++ API. (commit: 9a38b3aca84f245986a902c8c8d3fd0b0aebaf51) +* Add WaitResponses to PredictStreamedContext Method. (commit: a0cd2ee9834ad9c781309f3b12f45875dd624f05) +* Update release notes for 2.17.1 (commit: 8bb2af5f484af23600b8752be308860b432103c2) +* Add a new field in PredictRequest. (commit: 166dbe21f0c3ffe26920bb2fa16016f084ec97f3) +* Update release notes for 2.18.0 (commit: 313413006655bcab8ef24af074456e4d68d1c32b) +* Adds option to return stop tokens to PredictRequest. (commit: 2ece3e4c37401da13c710a989970d8536409e855) +* Remove obsolete anyinvocable flag (commit: ffc679630f29c57c1a048adf83325cae76802134) +* Move `tsl/platform/{cloud,default,windows}` to `xla/tsl/platform` (commit: 2c9e66b26e579258945bc3efe186a83e092485fd) +* Use hermetic python version. (commit: d0a16ccb98167ab182a8d4d75640e8124ed8c223) +* Allow HTTP Server to bind to a specific address, as opposed to listening on all addresses. (commit: 8d6cc18f16a68b2fbebcf021d601f3042d8f16af) +* Add more tracer (commit: 51f1cf6d2f46fdda9d209e27d70d33ac79fffec0) +* Add a note about net_http being open-sourced as a standalone library (github.com/google/net_http) (commit: 17186e6b91bc21a657ffd2ad62ba53676b15977f) +* Updating TensorFlow to latest passing continuous build. (commit: 04b7d4edf02f6c0104b6b73c5728aabc728035c0) +* Add aggregation key. (commit: 406764047d490dfd415296dbfd7979fa8cfb39fb) +* Update users of TSL headers and targets to new location in XLA (commit: 3087ed97c7e7c1fc4cac15fc1b48fcfa97a980a6) +* Remove usage of tensorflow::BlockingCounter. (commit: f86d984b0e744e16ed852b9a6b637430eee0000a) +* Update users of TSL headers and targets to new location in XLA (commit: 1e25043c495bd41c3f7933a6e6689d041cc25088) +* Force to call SavedMode::Run when output_filter is disabled. Note that the + output filter will still be applied at + `PostProcessPredictionResultWithoutOutputFilter`. We just don't prune the + graph in such cases. (commit: 39e33a7271b237cafad40484ab1cf2eb6fa6e532) +* Integrate BatchFunction op rewrite with serving stack (commit: 88ff3dd31ecde9ca8ba3dc21e194dce126202a48) +* add missing error_callback when load is cancelled (commit: e6e78cee6f671455f529e05de5283e5a4b9212f5) +* Add custom_logging_config so different loggers can introduce their own custom log processing logic. (commit: 4d5c006682f7373e6af9e3e8d0e0d092e5cbaf1a) +* Update version for 2.19.0-rc0 release. (#4077) (commit: 9a8564a72ecb5eff2ba79adf459fb973b5c60822) +* Added github.com/openxla/xla/pull/22572 into tensorflow.patch to resolve breakage. (commit: 72a33bd4befaf65a30104cd7574f48fbb9eaf6fc) +* Fix for gcc10 breakage. (commit: 984448fc724adf326af538182dcfbc177d6b15fd) +* Revert "Fix for gcc10 breakage." (commit: 69e5f6ba32fd612a331cffc034f64bfbde1d3083) +* Mark Tensorflow compatible with Protobuf v30+. (commit: 8bd5eb8996960ca199147ca8c1f0bf55cce3b526) +* Fix for gcc10 breakage, second attempt. (commit: 428fddee20009cf19e90431484736bd8ff6efd3f) +* Additional fixes for gcc10. (commit: 021555e3f13828db037bf1a05741517ce50efea2) + +# Release 2.18.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.18.1. + +# Release 2.18.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Update version for 2.18.0 release. (#2264) (commit: 5815bfdd1d1bbd9d0d3557576c98f13afc4d9016) +* This release is based on TF version 2.18.0. + +# Release 2.18.0-rc0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Extend GbmcChannel interface to implement redfish channel for TPUs (commit: 683cb64abb560324c9b1d391cdfe5b56ca1ee25a) +* Add tests to validate monitoring states. (commit: fab5c054d5c4dd18b69e21326367f0c5acae2028) +* Disable xnn_enable_avx256vnnigfni (commit: 19f9ccf9a3ddedc93812da7eac28554ebbc1f8dc) +* Reduce duplicate code using a test class (commit: 51cf3a796d87ed8726bf5525be6481b28de0ef94) +* Define an option to specify different IFRT client. (commit: aca5cfa285061815ab840274264fc6993cd620eb) +* Add release notes for tf-serving 2.17.0 (commit: b72a86e5768017b1699b2c463953d9a5f7db1583) +* avoid SetNumLoadThreads stall the server by forcing reset ThreadPool (commit: 6b9cf7c8777fd79868e73dc07663517993933be0) +* Add max_enqueued_batches option for model servers (commit: 7c99259e82cfdc4f12dbd5715acd6d17fc936b5d) +* Remove gpr_set_log_verbosity from grpc_client.cc (commit: 6e05a385d7f4591d46ae7b1d1a02244a5340a29b) +* Add option to stop retrying on permanent loading errors. (commit: 9ba72fa8a5df6e320caf207bd88673ad4c88e12e) +* Add the batch_padding_policy attribute the tensorflow serving api. (commit: ea02141a00089d77561db46aac0e2ca07bd44b2f) +* Improve handling of large JSON objects. (commit: 6cb013167d13f2ed3930aabb86dbc2c8c53f5adf) +* Silence warnings from external code (commit: 010d61a30f549423f61a3fa29ef0f2f0c8ed7f6c) +* Migration of the histogram header and cc code for TSL. Move tsl/lib/histogram to compiler/tsl/lib/histogram and update users. (commit: ab33df407e103b746aec8e165e31f4bc92ed388c) +* Add hermetic CUDA repository rule calls to TF serving project. (commit: 787c85f1a3f0268a243880418c97f37bed56762b) +* Update users of `status_test_util` to use the new location in `xla/tsl` (commit: 22b2b1e21793c9f7c583a1ee51cf8d73657fb0d2) +* Bump Bazel version from 6.4.0 to 6.5.0. (commit: 82e532fa3a3182560af6f23c38ddcb017c5e384f) +* provide an option to customize the sort order among servable names (commit: 32a85a8b42e6892e380bd4d54cf10b0c5734da4b) +* Remove cc_api_version stage 4: deletion where cc_api_version = 2 (commit: 7e0c1966627d9fa482acf1ef0ea983f4aa90f607) +* Remove cc_api_version stage 4: deletion where cc_api_version = 2 (commit: 48e0f56b8f84310596de1c97037d8d02053a9d14) +* This is a noop comment update for streaming inputs. (commit: cfac240ba956f29b0ae91008d1fe073f94c7ae84) +* Add a resource kind for number of LoRA models. (commit: 6b7ba27fd9dfef8d03fab076ea236e296370a3fe) +* Disable more warnings to make logs cleaner (commit: 4a830cadb604ed3d050b841e280a5b3486f86e4a) +* Add `bool return_single_response` field to `PredictStreamedOptions`. (commit: 648c9ee6489a3cf820aa1fcab82b821209e82af5) +* Use gcc-10 to avoid build issues while building XLA on CI (commit: 8bd1fda7e132a626921e458859db0e519deec451) +* Create separate `kokoro` config (commit: dbc7681fb6b89ed184dc0b41ddcfd59df0bd55b4) +* Remove top-level .bazelrc settings now that scripts use `--config=kokoro` (commit: f920b982ca7341eaf0b6456780d1268a3c8735be) +* Update Dockerfile.devel to build with gcc-10 (commit: f9c0262ecff0425f4647e6d52ab8f346a812e456) +* Move `tsl/lib/monitoring` to `xla/tsl/lib/monitoring` (commit: cb934df6ed2f1dd2b80e71611fe4db3f709dea4c) +* Delete 'enable_lazy_split', since the flag is not used anywhere. The code paths for the above flag being false are retained and true are eliminated. This will ensure that improving batching will be easier. (commit: 873993f9f4c8506194e3a130b0185f71db10bdc6) +* BUILD rule fix. (commit: d89b27235f94b245ae1822b5125f6c67e0b587db) +* Automated Code Change (commit: 4decd0ab78bb3ffb205baeccc64340c2a180ed01) +* Automated Code Change (commit: 0b05e865a05ca6a74344213ce74d0a43f1fbbc40) +* Fix build error (commit: d341c3406f5c4e66525f06fb9232a2ce64d7989b) +* Added capability to use XLA on a GPU. (commit: e5e795f518942a4c61b154a357bc4b16670d3f06) +* Update version for 2.18.0-rc0 release. (#2258) (commit: d6d402263bb9c9dec0151e5aebfa81e5ec015e40) +* Mark Tensorflow compatible with Protobuf v26+. (#2261) (commit: 424dba4101e3d28ac5cf9e65df5747676ef2a1e3) +* Update version for 2.18.0-rc0 release. (#2262) (commit: 67f4ee85350fb48ddda0bd7d1c1ebfd4601ed3e1) +* This release is based on TF version 2.18.0-rc2. + +# Release 2.17.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Create separate `kokoro` config (commit: 3af106649212377ef845a9d53fccddb00c10293f) +* Update version for 2.17.1 release. (#2266) (commit: 7b6021dd4cc6c1a815a84f160b77438c84818a66) +* This release is based on TF version 2.17.1. + +# Release 2.17.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Add RequestOptions and DeterministicMode options. (commit: a8b200b6363f4761478aa64e345b0822a32065b4) +* Remove usages of bridge fallback. (commit: 98570a6181910017d19990b4be7d9ccf9ae4d174) +* Provide a runtime option to lower bound the number of batch threads. (commit: 50b07e4ceac4f7c0e901f341b5d70b6a61dee736) +* Avoid GetChildren when using Specific servable versions (commit: 6fb94038e62fcc67627729e7f8416e0f146ec3fe) +* Add python clif target for prediction_log.proto. (commit: 39ba6232106cc8bdf99dfb54f5736ad88a17684a) +* Build with --xnn_enable_avx512amx=false (commit: f6c4219a564d8a37bda8ef612b2aca8c3953f304) +* Update comment in tfrt_saved_model_factory.h for wrong param name. (commit: 14ce91115cead955d9ce973866b48373a2ad52db) +* Upgraded libevent to 2.1.12. Fixed minor bug in EvHTTPServer. (commit: 2cda80ac6c30fe09e41a414669579d2d35460880) +* Introduce RequestRecorder in tfrt_servable so that implementation can record customized costs and metrics. (commit: 749007b9f94da6ab6ef3ed0dcfb3e19829f27298) +* Integrate TFRT+IFRT with tensorflow serving (commit: a8b64dd5e919efce56b52b2dbee39bb884f73296) +* Add core selector support for TFRT+IFRT serving on tensorflow serving (commit: 84a71a42ebdca0a518073ebb02eaac461b582316) +* Remove GPR_ASSERT . (commit: 2dca3af0cb435d00f7af90acbf0eb12db83a3cb8) +* Add timeout support when waiting on servables to load. (commit: 093d841041d3260d4e49ed2401494f0d8b9d1f19) +* Build with --xnn_enable_avx512fp16=false (commit: eeac086b6960923426aaf0eba615a5529bf8f95b) +* Support paging in TfrtSavedModelServable. (commit: 993a53c9a7bd37b70d2db52104bf63daa6b04582) +* Add max_enqueued_batches option for model servers. (commit: d914192fc890811e615be714a0e4769bf9b6dab2) +* Add max_enqueued_batches option for model servers. (commit: 67a2dcb2b9c057847616fb5ce54cb8545955d144) +* Update version for 2.17.0 release. (#2225) (commit: 68eda92e579da6d80d364fde1601504a415be817) +* Include patch files necessary for building at TF 2.17 (commit: 6311b72ea7efdc8c306de3c3bc808388468b7a3d) +* This release is based on TF version 2.17.0. + +# Release 2.16.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Update version for 2.16.1 release. (#2210) (commit: 0e6261315e2a8c529842929f5ceeb66b63264e7b) +* This release is based on TF version 2.16.1 + +# Release 2.15.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.15.1 + +# Release 2.15.0-rc0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Moves model server TFRT integration code oss (commit: 50ebab4ca601b5243b7aac674628954bef2d734b) +* Add an option to override to the size of GPU system (commit: 445a87ba217a9884bb567d9f9dc33511e08fd519) +* This cl is causing test failures and we are rolling it back. (commit: a39289b422714be4cf8a723d6f7d409f12c0f24c) +* Default signature_method_check to false (commit: 4711a8d4c9dd9d29aa404435e2297aa205914a81) +* Add an option to propagate current Context in periodic functions from AspiredVersionsManager. (commit: e4a8a87cd61bf0a65db948d615e69160e3070201) +* Refactor `Servable::PredictStreamed` so that implementations can support bidirectional streaming if needed (commit: a8c3ea682c57d031f74aae5cb6280e23b3c97b09) +* Create koltin proto library for the tensor flow protos. (commit: cae316414e198b47d86d7d30bc54f3dfd8a05a49) +* Create and use Kotlin proto targets for model.proto and predict.proto (commit: ea9529ea324d6bd76b15152eae7e06cda0b12c49) +* Add release notes for tf-serving 2.13.1 (commit: 45fae91c64f4fefc7e8d3077b2402a964244f0b6) +* Resubmit to move model server TFRT integration code oss (commit: eb5b3a5635e6af87013da86a6ec8c62f1097ec8f) +* Enable BF16 Automatic Mixed Precision (commit: 970c630723061e89e6eb45493c0883b0b97bd7eb) +* Follow expected format (commit: 60a3d7338f714f598071952ecf7e691ba70e3138) +* Remove upper_cost_threshold in TFRT serving (commit: 7f8d9d74bd3e336931526d3fd093aec2a2c66cc2) +* Build tensorflow_model_server with -rdynamic (commit: fc892403efd3d6329a80b11c6a585caec0d72b71) +* Add peak memory resource kind. (commit: 96e0661c2eead9d08e23a3b05c002d05b8cc3c68) +* Fix typo (commit: c0b35c742e51a60c83aee6e073f9488eec1b2326) +* Update warmup documentation (commit: 90148d72e5bfcf6a6977bb578988596c7a98a293) +* Implement Freeze() in pathways/tfrt serving. (commit: 0117fd401cf4063cbc06dc4e34a6a1ce7e3f04ba) +* This CL is a no-op (commit: b75349db95a18c8b906a8502d47a00f296f36c74) +* OSS remote_op_config_rewriter.proto (commit: ba473777a24142d34539505dd7eb85910156cdf0) +* Add release notes for tf-serving 2.14.0-rc0 (commit: 4d5ecfda28cdcb12f4f43a3d73e9386330c26a6f) +* Add flags for gpu multi-streaming support. (commit: 77cabded04612e0415181a790aa005ff0b3c232f) +* Add release notes for tf-serving 2.14.0-rc1 (commit: a3023de03f3064347ee7d14d297efc6fb724f3fa) +* Add 3 new resource kinds constants for GPU. (commit: 6b6dea341396cc2eb88d5a6c9ceac9c788486042) +* Adding flag allowing to turn off automatic TPU system initialization on startup. (commit: f83bc0c6de2b69965dcc0cc4723f21cf68d2a8a4) +* Add release notes for tf-serving 2.14.0 (commit: 60976ef6905e469de56473dd1925db58cdf1c5f9) +* Annotate which model is missing inputs. (commit: c99b18b98827400feb40daec1ce24e33473b7420) +* ebpf-transport-monitoring adding dependency on net_http. (commit: 152ef4ef087533e8608957cd1febff47c2058016) +* Add release notes for tf-serving 2.14.1 (commit: 83d970946861063977e70a0ac110be2651a84caa) +* OSS saved_model_config library, removes saved_model_config_stub/impl, moves GraphRewriter related API from session_bundle_util to graph_rewriter.h. (commit: 7356bbd6e4645b54b70013d9a81a583bdb3fc6d5) +* No-op. (commit: 9d02d895d8562a88974debb25821686218c293a7) +* Upgrading Bazel version from 6.1.0 to 6.4.0 (commit: 34521dc1d8ebce8c18a9acdbab49f0638ecb3398) +* Set xnn_enable_avxvnni=false in .bazelrc (commit: 4aed74931657cce1d12a9217e4ca928482843b4b) +* Add cuda-nvml-dev-11-8 to Dockerfile.gpu (commit: b2def7125180127e4a25212bc8fd7301633104eb) +* Revert problem with incorrect Dart build rules and targets. (commit: b6bccceabd24d449f4c77eb77d34c49b8132459a) +* Add cuda-nvml-dev-11-8 to Dockerfile.devel-gpu (and remove from Dockerfile.gpu) (commit: 028aac5bf94f21c60a74338cddc26a421b556141) +* OSS tfrt_http_api_handler*. (commit: 8ded4cead4fdc0ae6aee0c93991c02c713f1b0eb) +* Added FileAcl to tsl::FileSystem. (commit: d6c0917e5e7fcb76732c7eb172a69075ca2d1b08) +* Remove metadata size check in GetModelMetadata method in order to be consistent with other servable impl. (commit: a6355522da7f539483dc75ab6d995a18e10e9150) +* Replace the global registration with a registration class so that when we move server_init_internal to OSS we won't run into undetermined global registration sequence issue. (commit: 21d8f8816b5871a6bacf4f5ff7da0949d593bf88) +* Move TPU runner init stub to tensorflow serving OSS directory. (commit: 2b9e58cf4ad222adaa741d0cdac3b5a0d220b8fa) +* Add util function to verify if override resource have a subset of device kind of base resource. This is not used by OSS. (commit: 06ff18d38f8fdb955c880ed5200f81189504ae12) +* Add streaming options for predict request. (commit: 8ccd8a5219158294b8183c307809ab76ad6c0f0f) +* Define how tensors will be split for SPLIT streamed requests. (commit: b581572e3d9c1bb708ffa1314c2fd1eb1a143f8b) +* Add a client_id field for custom servables. (commit: eb578521a2a8c349a6602bbe98eb399bdcf20c30) +* Add option to configure the name of the input layer of remote model. (commit: f1e13413e2ad0f0aa8d25fbccaf3cd2abc8dfe0e) +* Added grpc reflection service to the serving binary. (commit: c140e0115a9c6d9cb28279acee0c80d56ed2982b) +* Add the option to enable GRPC health checking to model_server. This is useful for clients that want to use health checking with load balancing channels (if not we get errors on the client side). The current implementation is trivial, once we open our serving port we assume we we always be healthy but users may want to tweak this, specially if they need a mandated version, etc. (commit: a9a8e7bfe982f48a9156dedd48f61f537dea84a0) +* Automated Code Change (commit: f761fc7742d66129e7f8b26bba749d44d4d4b678) +* Update description of model versioning. (commit: d820234866ebb4f8f011f33339b3b9eb7f7333c3) +* Exported `FindMetaGraphDef` function. (commit: 0df0975055046ffc793557b48c68d9e2f16fe406) +* Automated Code Change (commit: 27923d3530cb963eee17d7f3670c9b27499827a0) +* Automated Code Change (commit: 704e2507aa7002d1008ea68f29296c91a6f10267) +* If accepting_requests_ is not set Terminate() returns without doing anything. (commit: c45fe1443b401f58d9440ca5d40c4e407137d042) +* Automated Code Change (commit: fce1804e06a784f9586f7114726c5dc66af1b135) +* Modify PredictStreamed to return a response or an error. (commit: 5b5d30fb9e4d57d3a6f03fc21fcd8e60750de5f9) +* Add support to use a MockServable in MockServerCore. (commit: 5b6e0b6b0806c9175274fb61c3c82a134de036f2) +* Fix OSS cpu build. (commit: 72acbaf554b3ce1c1e91dc86d9dcb859f95e9eda) +* Adds functionality to send TSL metrics over model_service RPC. (commit: 9564ef667dba57887954885328bed627e4c8ea4f) +* Add a method in tensorflow::serving::Servable to indicate whether a servable is critical. (commit: 5c0299e4b7934e5b84c534ed7f3d311fceec38b5) +* Upgrade to CUDA 12.2 and CuDNN 8.9.4 (commit: f82600afc9f22fac59ae10596c21887cbbf4594c) +* Fixes tensorflow_serving continuous build. (commit: a99fb9cbfe527d22f98eedd470a9a00ee0e35920) +* Add headers. (commit: fab72719273e8a6bd4cb1c9f22008269b1d98e83) +* Remove the criticality field in the BatchingSessionTask. (commit: b8663d08b38ad70397f1feb973d98a30c764ba4d) +* Move gpu docker build clang. (commit: 611c5a975681d4fe915274772482450ac6178506) +* Updated Dockerfile.devel-gpu to run setup.sources.sh from repo. (commit: d3102f00871e496a6276beeb546a6b9c46dc27e6) +* Add an interface for all Servables that support paging. (commit: e4716e5aa1b2dfd0b62fdf767de9dffd5d7b3304) +* Update cuda libraries to match TF (commit: 45446cf0f8b82f1bce51627f4005de16f1d597d0) +* Match libraries with Dockerfile.devel-gpu (commit: f6ef270ddf3e1f40064847e2dcf089806874e069) +* Update version for 2.15.0-rc0 release. (#2209) (commit: 73ba2b978678e3768c66d7fa90752da3978fd27b) +* Resolve breakages for 2.15 release. (commit: 318129247a068edbf59dd75e10dbfa2d1eb996dc) + +# Release 2.14.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.14.1 + +# Release 2.14.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.14.0 + +# Release 2.14.0-rc1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.14.0-rc1 + +# Release 2.14.0-rc0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Add Intel OpenMP for MKL build since it's required now (commit: 926b08d928d78cda9f371a5d14b2d551e8534631) +* Fixed the broken link in api_rest.md (commit: 146cda04e31ae38e2ae9b9563e639b8c29dd7dd4) +* Add a conditional any_invocable dependency to YDF to allow building with latest Tensorflow (commit: cb3698ef5c81db12cd67088c09eaa4a2dc027db3) +* Internal change. (commit: 62badd2558c792639778d2c517593efbbf26d172) +* Internal Code Change (commit: a1ae333bc7f0436ed0225d3aa1865512a945560f) +* Upgrading Bazel version from 5.3.0 to 6.1.0 (commit: 89a4bda0b9eff7eef1aeca853fdfd1bec5a3fda9) +* Reorder some status checks to break out of while loop immediately on error (commit: a87da7bc53b213ee1c4e14131682a0c31d73b456) +* Enable parallel model warm-up (commit: ba1246d95b9f7fe795bb5ffe449f3cd013552e9d) +* Enable parallel model warm-up (commit: 259f89b5664f4cec2965b819874c24262e3b8678) +* Updates Bazel version from 5.3.0 to 6.1.0 (commit: 9b986296238d1948c47ddfba6d1593dd925f85a6) +* Enable parallel model warm-up (commit: 79f9d2842223faf43db840b5def8e27274a38027) +* Do not register empty model name to warmup registry (commit: 1598cfeda97333b1711f434f016410109ad2dd8a) +* Print out model name for ModelConfig errors, which will make it easy to debug. (commit: 6420cc3381c5c1750635e539f81dde4a6d82a6cf) +* Internal change (commit: 291650abc818f7274c9b72cd410f9aa5fa5c3024) +* Pass criticality and queue option for low priority to support priority queue in shared hatch scheduler (commit: bf4aaf04feb563629cb514497f35e1f84609bb02) +* Add release notes for tf-serving 2.13.0-rc0 (commit: 6a56bd1eefdd46fbfdbd7c906b3fe22929aab3cb) +* Add release notes for tf-serving 2.12.2 (commit: 638fd783f387cc395f5771ec9e05b39b3814145c) +* Add release notes for tf-serving 2.13.0-rc1 (commit: d04bbfeed83801242c2f16c8673592c3ab895aec) +* Internal Code Change (commit: 72b83ed2237cea14bab34d551e01d9dd8fc5edc9) +* Move Servable interface to tensorflow serving. (commit: f196fa8a3c29a309899a8df2058a5692ce999253) +* Add release notes for tf-serving 2.13.0-rc2 (commit: 1ad5e7c89bc32c2fdb8b455d73b361b9accf0e49) +* Add release notes for tf-serving 2.13.0 (commit: 3d8e8c39be54ae88959cb5b9e3b06dc30aadd99d) +* Add RunOptions to Servable interface. (commit: 6a9d0fdafe326605cad1cae60dea0dd165bd2bb4) +* use hermetic python instead of system python for building and testing. (commit: ea05b0f1c274124704dfd3c0a440e9b273e5542a) +* Use OSS-compatible logging lib directly (commit: 3e8dd821e0c469f383e220da137d398168610255) +* Enable the saved model default input value feature for TFRT Predict API. (commit: 45bc2916a5bc5e71f7c46db16567479447f63d7e) +* Adding flag for automatic batch warmup. (commit: 736e1298a66b0976bf3a554492c74cb23db8904d) +* Internal Code Change (commit: 008c02c45e8a09b9a82d80b0962b5918e759f10e) +* Support streaming in request logger. (commit: 2cdb6b940378676e031d7b0c18f32a1a063c0f77) +* Use Servable for TFRT serving (commit: 3863ae35a69a1da616d94edcdc4f736cd046fddd) +* This is a noop (commit: 444257dc8cf3a9c69b3c8036821c2c9c1d73a670) +* Update version for 2.14.0-rc0 release. (commit: d07fc99ef40ebc044c27a04a20ff976ff9e2847c) +* Update Dockerfile.devel* with py3.9 installed. (#2178) (commit: 89e76e785ebe458264e6a40b76b8258e32377103) +* Update WORKSPACE to use python 3.9 (commit: 24b94ecab15e70afe2b546f5f7a093da6200569f) + +# Release 2.13.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.13.1 + +# Release 2.13.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.13.0 + +# Release 2.13.0-rc2 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.13.0-rc2 + +# Release 2.13.0-rc1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.13.0-rc1 + +# Release 2.13.0-rc0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Remove unused TF-DF define from bazelrc. (commit: dc0766022a49e8eca1ad9d7c7ce212a8ee3ea6d7) +* Remove usages of `tsl::Status::error_message`. (commit: 2b4ef1ae5ef3d24bdfa994dc08e840d635c66125) +* Add release notes for tf-serving 2.12.1 (commit: 19c3e66306722c44ddc7fc6dd2b4c59373640c95) +* Fix tests broken by the new defaults info in SignatureDef. (commit: 07b0882570a040aeb4a74a01ec5c81ecd6383afc) +* Fix remaining tests broken by the new defaults info in SignatureDef. (commit: 65b13b76083adfa7b694b5378e3f91a5088e62b8) +* Deprecate oss_or_google.bzl, move macros to serving.bzl (commit: a85461300e0e6a4b589e809bb7390ea4f17796bd) +* Revert "Fix tests broken by the new defaults info in SignatureDef." (commit: 663b375d8899e5937e67655f56ee6c888f4a084a) +* Update version for 2.13.0-rc0 release. (#2151) (commit: f449ae3dc99bfd19a3e9dbfa213c8553222382e2) + +# Release 2.12.2 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.12.1 + +# Release 2.12.1 + +* This is a re-release of 2.12.0 (that was marked as bad). Please use this instead. + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* Users of remote_predict_py should stay on v2.11 and avoid v2.12. + +## Bug Fixes and Other Changes + +* Update TF Serving pip package protobuf requirements to match [TF's](https://github.com/tensorflow/tensorflow/blob/0db597d0d758aba578783b5bf46c889700a45085/tensorflow/tools/pip_package/setup.py#L107). (commit: 24028778d11bf67992d481ff573de171c396119b) +* Update version for 2.12.1 release. (#2139) (commit: bd203faa888dd5ce90f21e3ee9af92dbc90b8a25) + +# Release 2.12.0 + +* **NOTE:** 2.12.0 has been identified as bad release. Please use 2.12.1 or later instead. + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* Users of remote_predict_py should stay on v2.11 and avoid v2.12. + +## Bug Fixes and Other Changes + +* Update TF Serving to Bazel 5.3.0, to match with TF. (commit: cda26f6065753167ac83e3b1aad7485d3d1d6db0) +* Update TF Text to v2.11.0. (commit: 1624fb20014921eac178318294e3b0c40d583d4e) +* Add pyclif_proto_library for get_model_metadata and session_service (commit: bedf391e8617cd3b1cf01ac5efd9e2fe8543a6ca) +* Raise the vlog level about aspired versions (commit: 73746fb3adeb29f8a1f20e154e8480397afd593d) +* Add PredictStreamed to PredictionLog. It represents a logged stream of PredictRequests and PredictResponses. (commit: 557f68a88d6be9a0f53beeca02a366359d787d4d) +* Update rules_pkg to 0.7.1. (commit: 2b5ad9a0d0424f285cc5e1a11eaeb5a8a0c89ad2) +* Track additional metadata for request logs. (commit: e0a3b5f990b9801d21c739dd27b8430c49353d8b) +* Replace usage of the tsl::Status constructor with a tsl::{error, errors}::Code. (commit: cfb9fb221e3375fe1dee144a70a9a2f4e28b01da) +* Replace usage of the tsl::Status constructor with a tsl::{error, errors}::Code. (commit: 19345a666becbe2df1d2d6096cae88b9013848ad) +* Update Dockerfile.devel* with py3.8 installed. (commit: 68d92ff3fdca0641f465cc3ba3858a619c8b82a6) +* Update TF Text to v2.12.0. (commit: dbe9339b436b2fa20705c8a444230848e771d65b) +* Stop depend on 'tensorflow-gpu' for tensorflow-serving GPU build on master branch. (commit: 85ff9c06021b47d487807a48a645d1c6ee9f654b) +* Upgrade cuda from 11.2 to 11.8. (commit: 20f51c91c0c19f1836508bec8ab7764d208f8f7f) +* Upgrade to decision-forests-1.3.0 and yggdrasil-decision-forests-1.4.0 (commit: 45d157458cbff136666700b6dee5fc7ccfe0dc70) +* Add tool to send predict (grpc) requests to ModelServer. (commit: 25c51251ed09ec80bf1d8380296649f9e1770e7b) +* Ignore remote_predict from :all for 2.12 release due to upstream TF breakage. (commit: 5830714e4f831e90b54f40e3f0467cac74caa009) + +# Release 2.11.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.11.1 + +# Release 2.11.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.11.0 + +# Release 2.11.0-rc2 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.11.0-rc2 + +# Release 2.11.0-rc1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.11.0-rc1 + +# Release 2.11.0-rc0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* No-op for public code. (commit: 20655b5904fdb4a810bfd8aef22db8becd4e80f3) +* Removed net_http/client/public/* now that all the users have been migrated to test_client/ (commit: e64a73afc23ea8f9538660cb8bc20f48d5682848) +* Fix the bug when accessing concat.tensor_data() outside its lifecycle (commit: cc3720091867c06615b8b3f50c1c2468a90e6b1c) +* BUILD cleanup (commit: 9e81e7b1bc8e1bd28c3324cf8feee717420f96f4) +* BUILD cleanup (commit: 8235ed86f9f07123e30ea8830a1454334bc86a2c) +* Update TF Text to v2.8.2. (commit: 31204d664d4ff985a526dccfe1f13ffde113deb6) +* Replace tensorflow/core/lib/core/blocking_counter in favor of the one from tensorflow/platform. (commit: 4ed7c5c30d0be43254a1177e8488675dc60d0384) +* Update TF Text to v2.9.0. (commit: e88506ab6709f2dc3a3bb1601ac4b84c23025611) +* Cleaning up BUILD files to remove "loose" headers. (commit: c2cd4c9629fbc868bd5e2cb186c7b081f22d709b) +* Remove the unnecessary dependency (commit: d976a37b4e9481de5ca371ae734645d925143088) +* Use `value` instead of `ValueOrDie`. (commit: b48e6e7cc02d8060cc3da5d363a90baf9324a2a0) +* Update TF Text to v2.9.0. (commit: a73f2925e367106bab0bfeca187fbaa1b3f36676) +* Replace `tensorflow::Status::OK()` with` tensorflow::OkStatus()`. (commit: 49ac8acb50291c21a0a72cfd9135aa2030e3ae88) +* Replace `tensorflow::Status::OK()` with` tensorflow::OkStatus()`. (commit: e8be1a742c3913cbbb6158dc1202c17130583219) +* Replace `tensorflow::Status::OK()` with` tensorflow::OkStatus()`. (commit: 88bd9d1638f6017b4af526a9b468641966d8972d) +* Add support for TensorFlow Decision Forests models. (commit: 4592081169068a0f059be71ac1b484d568f6e5d2) +* Remove special handling of tflite model when creating batching (commit: 2d39f8c8aa90ccfdc78faa51e4c8295832796a68) +* Improve batching test in following ways: (commit: 68d4d3962dc40fa6c1980df6d8df51338e44301e) +* Split proto helper function to its own util library. (commit: 01386268be6948429ea796fcd348beea4c7174d6) +* Created net_http/public folder for shared files and updated files in net_http and model_servers/http_server.cc to match. Also removed old directories from net_http/client to make way for new client API and implementation. (commit: 8086d33b6b74899dc8062ddbdba3fcc95e9af7e5) +* Updating a reference to Env post-refactor. (commit: 728474f92533be737958edc095abc0648a494452) +* Replace `tensorflow::Status::OK()` with` tensorflow::OkStatus()`. (commit: 9861f7fd742dc3ec0a5e2cf0314d68b799a11e73) +* Experimental support for per-model batching params. (commit: 94f2d6944f310069ebd527a52649fe3c001c38e3) +* tensorflow-serving-api python package requires Python >= 3.7 (support for prior versions has been dropped). (commit: 30ea18d66416e8cfd3873fba3c2482ec68184ec6) +* Replace `tensorflow::Status::OK()` with `tensorflow::{Status, OkStatus}()`. (commit: 6393b29242ab6648c723a51f64eb2e67ebf497db) +* Open up http_server_clients package group for easier 3P (commit: 66b199ba6e8fc81cb278988aa541e03987a1d27d) + +# Release 2.10.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.10.1. + +# Release 2.10.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.10.0. + +# Release 2.9.3 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.9.3 + +# Release 2.9.2 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.9.2 + +# Release 2.7.4 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.7.4 + +# Release 2.8.4 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.8.4. + +# Release 2.8.3 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.8.3. + +# Release 2.10.0-rc3 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.10.0-rc3 + +# Release 2.10.0-rc2 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.10.0-rc2 + +# Release 2.10.0-rc1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.10.0-rc1 + +# Release 2.10.0-rc0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* tfs:aarch64: add aarch64 mkl bazel config to enable onednn+acl backend (commit: 1285e41acc707ba0d18e8eaf8a42c6d5110e8af8) +* Match packages in devel and non-devel GPU with TF dockerfile. (commit: a8ffec79e0794650a4c0856c4122032e985296cc) +* Validate batching params when creating the wrapped (batching) session, (commit: 48ff72dcb6582e989452ba870c88a2bb710ea0c4) +* Merge and clean up implementations of `GetModelDiskSize` and `GetAllDescendants` in util.cc. (commit: 6da9c43c5f71abe361841fb3fd5eaad57fc847b1) +* Parallelize iteration over child files in `GetModelDiskSize` (commit: d09d2efe6e9b88ef0266e5982a3e732da14dc93b) +* Fix gpu docker build failure due to bad substitution (commit: 1d7cd5b2ba43c3d98f0c8bef6806c203b2c14592) +* Call run_returning_status instead of run (commit: 8f9085ac678755afea6bf0067fe40a32e37ce2fa) +* Fixing broken link for [ResNet in TensorFlow](https://www.tensorflow.org/api_docs/python/tf/keras/applications/resnet). (commit: b15210ca076b11eaa2dfd0ac2fb125be780c5d40) +* Update the TensorFlow BatchingSession metric monitoring class to be compatible with Google's internal monitoring tools. (commit: 05b944ad9367027a1082756a3069619f44955de1) +* Increase timeout for model tests. (commit: 677ba5a07813c4fb5a2ffb4567a7ec4a137eebe6) +* Use pb.h for topology.proto. (commit: 21fda280bc72bdbc4386c7b0d2ad4b97264921ad) + +# Release 2.7.3 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Update TF Text to v2.7.3 (commit: ee7892be7801a0e4ae9a6dd8b5f7bab06ae9c87c) +* This release is based on TF version 2.7.3 + +# Release 2.9.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.9.0 + +# Release 2.6.5 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Rollback incompatible C++17 changes. (commit: ba0fa72b61bc2c42388b815253ba72e6830f03cf) +* Roll forward with std::optional -> absl::optional. (commit: 009dac683bf84165b84702d177bb9a021ebef728) +* Replace STL algorithm call with a container method (performance-inefficient-algorithm). (commit: f5bc09998e0043ce72d34b14104379163048406c) +* Remove unused "using" decl. (commit: ffcc4a16c76c4fa1189edd9682fc486442a33e52) +* Move status_proto to public visible apis/ (it being used by public API protos) (commit: 7f894c79fce5e58758f3cb49e858a16e3602ae80) +* Move core/logging.proto -> apis/logging.proto (commit: 37c64b8820a923aafc1b5c8bf264fd5cce5224f3) +* Update TF Text to v2.5.0. (commit: 48e5a9e23a1e0b2951b77c3e8f9832193d9b1851) +* Adding python targets for config protos (commit: 757c3a6b6c8a03731dc73ff758f69a61aeddcf67) +* Remove experimental tags from uses of gRPC C++ callback API. (commit: b355023b034ca6ef72b507920d9de2a02e0f4a2a) +* Add new --use_alts_credentials flag, to enable building secure credentials using Google ALTS. (commit: ceba636bb7c5c98bde35d1818fd033b36300fffe) +* Enable HTTP PATCH support in evhttp_server (commit: 6cbc4a9eb419c8078c3a4e791381cda70dd8fc78) + +# Release 2.9.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Do not report successful loads that are cancelled, unload immediately instead. (commit: 923269955f5582cc26d0454992afa5c888a9377f) +* Add a lock to avoid race condition on memoized_resource_estimate_. (commit: 83959e364e7ff1234bf47a5d8139677d1bdb18c1) +* Update Resnet model used in K8S tutorial (commit: 6e76f6a9460bf2d37630f025fcfd3e06c4230fee) +* Prepare for development of new http client API. (commit: 78e94e99650deae956fe20dffa9932a72ec7d765) +* Integrate TPU initialization changes into TF Serving. (commit: 6549ef2136940cd98cfbb9ee0e29221d86101d16) +* Allow max_execution_batch_size to be actually used by allowing (commit: 48a699a2fd32404c4b19f55077a1fb29112a0afe) +* Delete batch_schedulers_ before thread_pool_name_ (commit: 2837812341e7c98be4717e5901286692a5dcc02a) +* Add missing NVIDIA repository key. (commit: c0998e13451b9b83c9bdf157dd3648b2272dac59) +* Bump minimum bazel version 5.1.1, to match with TF and root.workspace (commit: 8a02874cee6957e5817960613627a549bd80a6e9) +* Update to use C++17 (commit: 7166e2efc6b7e63c908515c6a53d0e4fe8fa0aae) +* Update tensorflow_model_server_test to depend on the pip installed tensorflow. (commit: 04b8676730a882cab61b7772097f2c23c0447ef9) +* This release is based on TF version 2.9.0 + +# Release 2.8.2 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Replace int64 with int64_t and uint64 with uint64_t. (commit: 21360c763767823b82768ce42c5c90c0c9012601) +* update to latest benchmark API changes (commit: 860e1013385289ad3f9eb4d854b55c23e7cb8087) +* This release is based on TF version 2.8.2 + +# Release 2.8.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Force-allow labels for the models which are not yet loaded. The feature is meant to be used for non-prod environments only as it may break the serving until relevant models are not loaded yet. (commit: 988bbce80038ac0b7141dcb4413124ba486344cf) +* Update CreateRPC API interface. (commit: 2e7ca90c18f310c542ed0dcde92d676db6454285) +* Add `--tensorflow_session_config_file` flag to tf serving model server to support custom SessionConfig (commit: 342a8088524c33f68e3eb4d66800f01a777ceb38) +* Add `--experimental_cc_shared_library` by default to all builds. (commit: 49b138fdd4c0fb7170736193063c6f03dfb4dba4) +* Add --num_request_iterations_for_warmup flag (fixes #1949) (commit: 2a55aec18cc1923ece84c7fcf701185306ef99b1) + +# Release 2.5.4 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.5.3 + +# Release 2.6.3 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.6.3 + +# Release 2.6.0 + +## Major Features and Improvements + +* Update TF Text to v2.5.0. (commit: 48e5a9e23a1e0b2951b77c3e8f9832193d9b1851) +* Add support for Google ALTS. (commit: ceba636bb7c5c98bde35d1818fd033b36300fffe) +* Enable HTTP PATCH support in HTTP/REST server (commit: 6cbc4a9eb419c8078c3a4e791381cda70dd8fc78) + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Enable tensor output filters with batching. (commit: 448dbe14624538ab76fd6aeb2a456344e7f41c78) +* Update tf.io import in warmup example doc. (commit: 6579d2d056530565cd6606a39c82b2f6c1d3799e) +* Resize tensors if the dimensions of the tflite and tensorflow inputs mismatch, even if the number of elements are the same (commit: 8293f44bd5c5ecc68636cd0d036234f891d29366) +* Add basic batch scheduler for tflite models to improve batch parallelism. (commit: 0ffd6080437ca8175b067be7cc00f5b3df9ea92a) +* Reserve Unavailable error to TF communication ops only. (commit: db9aca187affd0453627a1729916acfea98ae800) +* Add the flag thread_pool_factory_config_file to model server and fix a typo. (commit: efc445f416f8cb20606ca0d2aaf44c13fae7ea4c) + +## Thanks to our Contributors + +This release contains contributions from many people at Google. + +# Release 2.5.2 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.5.1 + +# Release 2.4.3 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.4.3 + +# Release 2.3.4 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* TensorFlow Serving using TensorFlow 2.3.4 + +# Release 2.4.1 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 2.4.1 + +# Release 2.4.0 + +## Major Features and Improvements + +* Update TF Text to v2.3.0. +* Upgrade to CUDA Version 11.0. +* Update CUDNN_VERSION to 8.0.4.30. +* Adds user guide for Remote Predict Op. +* Add support for serving regress/classify (native keras) TF2 models. + +## Breaking Changes + +## Bug Fixes and Other Changes + +* Adding /usr/local/cuda/extras/CUPTI/lib64 to LD_LIBRARY_PATH in order to unblock profiling (commit: 1270b8ce192225edcaafb00a50822216dd0b1de0) +* Improve error message when version directory is not found (commit: d687d3e8827c82f4f1b68337c67b2cbe6e4126e7) +* Migrate the remaining references of tf.app to compat.v1. (commit: 06fbf878a98c8bd4202e33bc1c097a6ce184d06e) +* Cleanup TraceMe idioms (commit: f22f802c73bfdd548f85dacffc24022b0d79dfc7) +* Adds LICENSE file to tensorflow-serving-api python package. (commit: 41188d482beb693d4e79e6934d25f1edd44321ac) +* Enable a way to 'forget' unloaded models in the ServableStateMonitor. (commit: 53c5a65e8158dc1a2a85a2394482cc6acc1736bc) +* Added abstract layer for remote predict op over different RPC protocols with template. (commit: c54ca7ec95928b6eec39f350140835ebbe3caeb0) +* Add an example which call the Remote Predict Op directly. (commit: d5b980f487996aa1f890a559eae968735dfebf5d) +* For batching session in TF serving model server, introduce options to enable large batch splitting. (commit: f84187e8d3e19a298656a661a888c0563c21910e) +* Add multi-inference support for TF2 models that use (commit: abb8d3b516a310ec4269cd6bf892644d5150485a) +* Use absl::optional instead of tensorflow::serving::optional. (commit: c809305a50412a2b47f2287c76ea0be1070aabd6) +* Use absl::optional instead of tensorflow::serving::optional. (commit: cf1cf93eac1896c3c482d17b440489edea110670) +* Remove tensorflow::serving::MakeCleanup and use tensorflow::gtl::MakeCleanup. (commit: 6ccb003eb45f4961128e5cc2edf5d8b61ef51111) +* Use absl::optional and remove tensorflow::serving::optional. (commit: e8e5222abbb39e84d1d4e5e9813626b2cc51daac) +* Deprecate tensorflow::CreateProfilerService() and update serving client. (commit: 98a55030e10a61ee0c3f6b8fc57e2cf63fc59719) +* Change the SASS & PTX we ship with TF (commit: 086929269b5f2c0f5d71c30accb79d74694c9ece) +* Adding custom op support. (commit: 892ea42864676b67cbccdfa0794a15d30e65a1b6) +* Upgrade to PY3 for tests. (commit: 02624a83f70060095df7c132fa46a7a09f9bff6a) +* Makes clear how to make a default config file for serving multiple models. (commit: 084eaeb15fdc87d83b8c19f558dc1f56bd3a024e) +* Use TraceMeEncode in BatchingSession's TraceMe. (commit: 78ff058501274aa37b6bbc18aec225604d4cda47) +* Export metrics for runtime latency for predict/classify/regress. (commit: c317582981cfc1550b27d9d73f71c6ca38e5c8c5) +* Refactor net_http/client to expose request/response functionality as a public API (not yet finalized) for usage testing ServerRequestInterface and HttpServerInterface instances. (commit: 0b951c807375f1f305280a96124d8b6d6e045bd2) +* In model warm-up path, re-write error code out-of-range (intended when reading EOF in a file) to ok. (commit: d9bde73569385b4ef3ef8e36d2c832a8ae9a92ad) +* fix Client Rest API endpoint (commit: b847bac5f2e1dc6a98f431b1fdf42ceebceceeb6) +* Support multiple SignatureDefs by key in TFLite models (commit: 2e14cd9dc2647127d7cb8c44ceab5dfcf6ac28c4) +* Add dedicated aliases field to ModelServerConfig. (commit: 718152dc386f9fa7b21ed36d9d85518e987d7bf5) +* Remove deprecated flag fail_if_no_model_versions_found from tensorflow serving binary (commit: 4b624628977a12b1757b9ddcd3312b3768de8231) +* Fix TraceMe instrumentation for the padding size. (commit: 0cb94cd79aacb965b3923d4a51b4091cf84d5e22) +* Add vlog to dump updated model label map (for debugging) each time the map is updated. (commit: ac10e74078123189dc1c8a3cd29d530b7c972782) +* Add python wrapper for remote predict op and clean the build and include files. (commit: d0daa1064ecdd56ecb5c0a8aca37c3e198cb313d) +* Add `portpicker` module required to run modelserver e2e tests. (commit: 82f8cc039d091916b8186dfa1ff4b6c006e7277c) +* changing "infintiy" to "really high value" (commit: c96474cfcca46b1216e52634efb68986cf8aa9b8) +* Minimal commandline client to trigger profiler on the modelserver. (commit: c0a5619a01e3af69459aa6396d614945370bbd02) +* Add signature name to RPOp. (commit: 84dfc8b66ff6c1a693766613034ddc3ff044a330) +* When RPC error occurs, the output tensors should still get allocated. (commit: 9113de22353350443bdd42c5d594ec653e57c0da) +* Fix BM_MobileNet benchmark (commit: af665627b8152d4c62d207a97c6e712cb2e9a120) +* Add JSPB BUILD targets for inference and example proto files. (commit: f1009eb0e6bdae2e35dbfb9f4ad7270e74705e2e) +* Fall back to legacy TFLite tensor naming when parsing signature defs in TFLiteSession. (commit: 3884187cb9253bb9baa240b2009cfc6d4847b9f9) + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +Adarshreddy Adelli, Lescurel + + +# Release 2.3.0 + +## Bug Fixes and Other Changes + +* Add a ThreadPoolFactory abstraction for returning inter- and intra- thread pools, and update PredictRequest handling logic to use the new abstraction. (commit: 8e3a00cd8ef00523227cbe1f694ab56454a880c3) +* Update Dockerfile.devel* with py3.6 installed. (commit: b3f46d44d07480266b28776caa13211339777bc5) +* Add more metrics for batching. (commit: f0bd9cf8b85710b638938361d356dbf15fda2e86) +* Rename method to clarify intent. (commit: 9feac12f2223124c7ecc85a687e1ee2b24e3f7ad) +* Plug ThreadPoolFactory into Classify request handling logic. (commit: 975f474a4ea9ef134439e266ec4a471741253ecf) +* Plug ThreadPoolFactory into Regress request handling logic. (commit: ff9ebf2db8bf7cbc7bb199bbb207409eae25d5cc) +* Plug ThreadPoolFactory into MultiInference request handling logic. (commit: 9a2db1da9b7e992d29ad4ccfcb125734d0cd760e) +* Add a tflite benchmark for Mobilenet v1 quant (commit: e26682237cf756eca2dc12c83e8d5d24f00c1261) +* Allow batch size of zero in row format JSON (commit: fee9d12070a76c1cf56bc8ae40f306a09dfd07b1) +* Add tests for zero-sized batch (commit: b064c1d3df03b0401c5ca61de0d5ab36cd5645a5) +* Support for MLMD(https://www.tensorflow.org/tfx/guide/mlmd) broadcast in TensorFlow serving. (commit: 4f8d3b789964d173f2d0bd87a42abfbd6a2b1e71) +* Fix docker based builds (fixes #1596) (commit: ca2e0032d1ead843398d7744e8c51ead28daf63c) +* Fix order dependency in batching_session_test. (commit: 58540f746c65516dc3fcda7751c6983050307409) +* Split BasicTest in machine_learning_metadata_test into multiple test methods without order dependency. (commit: 745c735e315941925e324cbebe78a1f09d5a7443) +* Revert pinning the version for "com_google_absl". (commit: ff9e950fa692c6f9387239bb9fa877975e8cf1c1) +* Minimize the diffs between mkl and non-mkl Dockerfiles (commit: e7830148e53acfec7d3af7dd512a7e825f75da2a) +* Pin "com_google_absl" at the same version(with same patch) with Tensorflow. (commit: f46b88af8af94be3c6497cc6c50a4e5c0625b2d5) +* Update TF Text to v2.2.0. (commit: f8ea95d906421ff9517b0027662546741c486edf) +* fix broken web link (commit: 0cb123f18df4032d8f22c1b2e19b4f41bd6c3da3) +* Test zero-sized batch with tensors of different shapes (commit: 1f7aebd906a70ba0fa04105ceee6227960b764f7) +* Test inconsistent batch size between zero and non-zero (commit: 91afd42dab8ce50f86bbf65065dce0c28163314b) +* Fix broken GetModelMetadata request processing (#1612) (commit: c1ce0752f1076bd6f92e1af5f73e3a3c552f4691) +* Adds support for SignatureDefs stored in metadata buffers to tflite sessions (commit: 4867fedbff8a33f499972268abe96618abcb81aa) +* Update ICU library to include knowledge of built-in data files. (commit: c32ebd5e9f09828c80413ca989b99e8544502c1a) +* Add support for version labels to REST API (Fixes #1555). (commit: 3df036223b66738de1b873e9b163230fb7661cb4) +* Update TF Text regression model to catch errors thrown from within ops. (commit: 425d596b9b0aef2bf3ea675c985f01e55f880a4e) +* Upgrade to CUDA Version 10.1. (commit: fd5a2a2508daf21ad174b4ec7b62501486137c01) +* Migrates profiler_client trace to the new api in tensorflow_model_server_test. (commit: 8d7d1d6bbc50756e73aed4b9eb5a2c8ff25cdc79) +* Update the testing model for TRT to fix the test. (commit: 28f812d8ce8f256e2d9256d6a98cd8f75f747842) +* Add release notes for TF Serving 2.2.0 (commit: 54475e6508889c13992aced1da12a372d997e4e3) +* Update bazel version requirement and version used in the docker images to match with TF (3.0.0). (commit: 56854d3fa27cce8c1f7816214f59e6e82c4bf5fc) +* Fixes instructions on sample commands to serve a model with docker. (commit: a5cd1caafacd7480f5d8d2dd164adce3410b024f) +* Change use_tflite_model to prefer_tflite_model to allow multi-tenancy of Tensorflow models with Tensorflow Lite models. (commit: 8589d8177bd300625b4c7596240150f8a8002d19) +* Introducing Arena usage to TensorFlow Serving's HTTP handlers. (commit: a33978ca4c29387845e9b51d5653b997d4b3f814) +* Fix tensorflow::errors:* calls, which use StrCat instead of StrFormat (commit: 2c0bcec68c040306e009b5a10d4bc80bc58fe0c5) +* Instrumentation for BatchingSession: (commit: 3ca9e89d1b6147706981467a84c6421c44d3794a) +* adjust error message for incorrect keys of instances object (commit: 83863b8fec26a8ea2d3957366173f9a52658b469) +* Update rules_pkg to latest (0.2.5) release. (commit: 932358ec7511e54ad9c93ea606cc677da2d1fcb2) +* In batching session, implement the support for 'enable_large_batch_splitting'. (commit: d7c6a65b816849cf2b84015a5b2972be7950dc89) +* Update version for 2.3.0-rc0 release. (commit: 3af330317628a713a6e318097c7cd6fa8571165d) +* Set cuda compute capabilities for `cuda` build config. (commit: 731a34f0b3f43a6f7a8da85655d3a4a5c72d066a) +* Update version for 2.3.0 release. (commit: 8b4c7095b9931442a77288624fdd1a207671eb4c) + +## Thanks to our Contributors + +This release contains contributions from many people at Google. + + +# Release 2.2.0 + +## Major Features and Improvements + +* Upgrade to CUDA Version 10.1. (commit: 3ab70a7811f63b994da076e2688ccc66feccee96) +* Update TF Text to v2.2.0. (commit: fd9842816eddb4782579eadd119156190d6d2fec) + +## Breaking Changes + +## Bug Fixes and Other Changes + +* This release is based on TensorFlow version 2.2.0 +* Add a SourceAdapter that adds a prefix to StoragePath. (commit: f337623da81521eefd8cdc2da1c4a450ecf1d028) +* Switch users of `tensorflow::Env::Now*()` to `EnvTime::Now*()`. (commit: 8a0895eb8127941b2e9dada20718dd28f3dbaee1) +* Remove SessionBundle support from Predictor. (commit: 2090d67f4e5e8ee5aa7faf8437bea096a438450a) +* Replace the error_codes.proto references in tf serving. (commit: ab475bf6c5e5e4b3b42ffa2aecf18b39fd481ad3) +* Adds [performance guide and documentation](tensorflow_serving/g3doc/tensorboard.md) for TensorBoard integration (commit: f1e4eb2259da90bb9c5fe028ba824ac18a436f67) +* Remove SessionBundleSourceAdapter as we load Session bundles via (commit: d50aa2b0b986b11368ddcf6b6eb20b9381af474c) +* Use SavedModelBundleSourceAdapterConfig instead of (commit: 8ed3ceea985529a350290cf782cb34c3c66827d4) +* Update minimum bazel version to 1.2.1. (commit: 1a36026198df5f7dec1e626ef9b112fecdd2916b) +* Drop support for beta gRPC APIs. (commit: 13d01fc64330ff883bd1553122d9fd114a5a7368) +* API spec for httpserver response-streaming (with flow-control). (commit: fd597f074ce127056515bc52ee3a3d4ff4b727bb) +* Change Python version to PY3. (commit: 7516746a311f96b57a60598feba40cbdd3989e73) +* Update Python tests in PY3. (commit: 0cf65d2105c191c00fba8918ba75fc955bbeace3) +* Upgrade bazel version for Dockerfiles. (commit: e507aa193b9f3520d40e3da5e4d2263280ff35e4) +* Change dockerfile for PY3. (commit: 7cbd06e8b7720b82b1d2dfae54c3a828d3a52eb4) +* Reduce contention in FastReadDynamicPtr by sharding the ReadPtrs, by default one per CPU. (commit: d3b374bc70348f2e5e22b7e9ebb191ee9d5b3268) +* Ensure that all outstanding ReadPtrs are destroyed before allowing a (commit: e41ee40826652b6aa5a3f071107074923d6ff6c7) +* Allow splitting fields from batched session metadata into individual sessions (commit: caf2a92ba07ca4d10515f0b018c920e9b325c6c8) +* Allow passing ThreadPoolOptions in various Session implementations. (commit: 2b6212cf0aa88b719ee00267f83c89d4f7599ef1) +* Update bazel version used in the docker images. (commit: 162f72949c6ecbe9e610182c923dec0aa5924cf2) +* Format error strings correctly in JSON response (Fixes #1600). (commit: 1ff4d31cd9a0a736162813c149139cce0ccaaa2c) +* Fix broken GetModelMetadata request processing (#1612) (commit: 55c40374b548b89e8de6d899ef2b0b355c0fa9e5) +* Support Python 3.7 in tensorflow-serving-api package (Fixes #1640) (commit: f775bb25e80a6c7b3c66842eb9085d44d9752ec2) +* Update ICU library to include knowledge of built-in data files. (commit: 774f2489384cf985c534298d1303474c268efe5c) +* Adds storage.googleapis.com as the primary download location for the ICU, and resets the sha256 to match this archive. (commit: 028d05095c4e302c06096e5ea32917718828ea47) + +# Release 2.1.0 + +## Major Features and Improvements +* Add integration with [TensorBoard profiler service](https://www.tensorflow.org/tensorboard). + +## Breaking Changes + +## Bug Fixes and Other Changes + +* Fix link for TFRecord in Saved Model Warmup documentation. (commit: 127a112a91bda3d7d3c3a56802632376bbe3e36e) +* Fix typo in http server log message. (commit: 509f6da062dc9b091ad6961a94740cf64e265c36) +* Be able to discard aspired-versions request from SourceRouter (commit: 10e4987502ee91fe74c6c179ed4ba52f17cc75b4) +* Use public tf.config APIs (commit: 87a4b2b28729bd269ab367742998b6f8426ea1b7) +* Fix copying of string tensor outputs by explicitly copying each (commit: 9544077bdb6eef9b20a0688a042155ee6dea011a) +* Migrate from std::string to tensorflow::tstring. (commit: e24571ac9ce390733f3b02188c7d740f08fff62d) + +# Release 2.0.0 + +## Major Features and Improvements +* Some Tensorflow Text ops have been added to ModelServer (specifically constrained_sequence_op, sentence_breaking_ops, unicode_script_tokenizer, whitespace_tokenizer, wordpiece_tokenizer) + +## Breaking Changes +* As previously announced[1](https://groups.google.com/a/tensorflow.org/forum/#!msg/announce/qXfsxr2sF-0/jHQ77dr3DAAJ)[2](https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md)[3](https://github.com/tensorflow/serving/releases/tag/1.15.0), Contrib ops will not be packaged with Tensorflow, and therefore will not be available in Tensorflow Serving. If serving with Tensorflow Serving >1.15, please ensure your models do not contain any tf.contrib ops. If you are critically dependent on custom ops, please review [this guide](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/custom_op.md) for instructions to statically build ops into the model server. +* After being [deprecated](https://developers.googleblog.com/2017/07/tensorflow-serving-10.html) for multiple years, as a part of tf.contrib deprecation, SessionBundle API will be removed starting from Tensorflow Serving 2.0 - if currently using SessionBundle, please migrate to SavedModel APIs. + +## Bug Fixes and Other Changes +* Add a section in the documentation for testing custom op manually. (commit: 1b65af1d7fee4fe79b4152f94d5ea422e8a79cca) +* Add ops delegate library to enable running TF ops. (commit: 14112359d16b3e1e275c2ba70b0e078ce4863783) +* Add command line tool to load TF Lite model for manual testing/debugging. (commit: 0b0254d4a90550b1d7228334187e624bf4b31c37) +* Fixes broken relative docs links (commit: 12813143b22616091388e7659d7f69cfcf518269) +* Cleaning up BUILD visibility for tf_pyclif_proto_library intermediate targets. (commit: 81ed5ef2307eea4c9396fd34f33673be072cdcf3) +* Remove unused load statements from BUILD files (commit: d0e01a3c56b280c6602d6c14e97ef60882d317aa) +* Manual tests for model server and including tf.Text in serving build. (commit: 142d0adb5e2975689d80d8fc608c9684e96de078) +* Remove tensorflow/contrib/session_bundle as dependency for Tensorflow Serving. (commit: 1bdd3499f1fe4d99b3c3024080560350d493e29b) + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +chaox + +# Release 1.15.0 + +## Major Features and Improvements + +## Breaking Changes +* As previously announced[1](https://groups.google.com/a/tensorflow.org/forum/#!msg/announce/qXfsxr2sF-0/jHQ77dr3DAAJ)[2](https://github.com/tensorflow/community/blob/master/rfcs/20180907-contrib-sunset.md), Contrib ops will not be packaged with Tensorflow, and therefore will not be available in Tensorflow Serving. If serving with Tensorflow Serving >1.15, please ensure your models do not contain any tf.contrib ops. If you are critically dependent on custom ops, please review [this guide](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/custom_op.md) for instructions to statically build ops into the model server. +* After being [deprecated](https://developers.googleblog.com/2017/07/tensorflow-serving-10.html) for multiple years, as a part of tf.contrib deprecation, SessionBundle API will be removed starting from Tensorflow Serving 2.0 - if currently using SessionBundle, please migrate to SavedModel APIs. + +## Upcoming Features +* Some Tensorflow Text ops will be added to Model Server starting from TF Serving 2.0.0 (specifically constrained_sequence_op, sentence_breaking_ops, unicode_script_tokenizer, whitespace_tokenizer, wordpiece_tokenizer). + +## Bug Fixes and Other Changes +* Add monitoring config (commit: 18db9a46168eadd4d3e28e9b0cdb27bd6a11add9) +* Fix docs (commit: 7fc2253157db1dff340d7b418a6cf5204db2ce09) +* Use master as references (commit: 08cb506672d4c2ef289f79eee545df26d6577b45) +* Fix docs (commit: 9cc986beb742c485a62637fd20e841288774585d) +* Remove hyphen from version numbers to conform with PIP. (commit: 4aa0cfc24098000163fdfe270c4eb205e98790b1) +* Fix ImportError: No module named builtins (commit: e35ffff3999be3f971fa1503c158f33d721228c8) +* Cleanup visibility specs. (commit: 8e3956cac1eec2213538d8d6c367398e2f883e70) +* Remove 'future' from setup.py (commit: 64a80dd955a384de776b6256f3abcaa28cf88e79) +* Install future>=0.17.1 during Dockerfile.devel-gpu build (commit: dc36c21df5117364a3390a8bfe1fd3bf7dc92cb7) +* Replace calls to deprecated googletest APIs SetUpTestCase/TearDownTestCase with SetUpTestSuite/TearDownTestSuite. (commit: 39bbeb70dec8054d8ad81a7aa0423ec7e1a07c2a) +* Add the option to allow assigning labels to unavailable models in open source model server. (commit: e6d91e72f7593be36dda933b3291c7ebbc646fa6) +* Adds polling for config file to model server (#1301) (commit: c3eeed4f245e43f6cf92329d251e2b9d6255d6e5) +* Adds util functions for getting min of two ResourceAllocations. (commit: ba6527c8d56a0752f6c87115071e4d1bf7941810) +* Cleanup usage of the protobuf_archive. See #19032 (commit: dca61db5137c416a454c6ef821ad0fac6d66dc91) +* Replace NumSchedulableCPUs() with MaxParallelism(). (commit: aa9dddb93576c814b97947d6386d400cf6c87679) +* Don't run model_servers:tensorflow_model_server_test under asan (commit: b5c24e3e3849978a551db3aae3854c8794d10124) +* Release notes for 1.14 (commit: dc986268756ef45a3ffca4b8578dfdc69e015d29) +* Fixing Docker link (commit: 3bd851d88cd2febcdec29a52bab1d7d225a3a54c) +* Update release notes for 1.14.0 release. (commit: 00b2980a4d6ca127b63409b3eae791f846d1031a) +* Add release notes for TF serving 1.12.3. (commit: 7226859e9dd0f45bade559ab12892d4e388a7c11) +* Remove unnecessary calls to `Tensor::flat` in the tensorflow regressor interface. (commit: 55d897ef71b1ba142defec67bcce8eba7d8f5236) +* Fix print syntax in sample code (commit: ecef0d2fea2af1d4653a41934649512aa6994fd0) +* Adds guide for serving with custom ops (commit: dae0b4dffb29efc647783d45c28c4db0282b4d51) +* Return more informative error message during warmup. (commit: 1126fcd5d179d7829f48471eca6ddbbce79e219e) +* Enables passing in the SessionMetadata to the TensorFlow Session through the SavedModel ingestion API. (commit: 9cf3ff32daaaa2bb941ba7d7b8f049f807e4288e) +* Modifies server configuration documentation (commit: ee4edd59ad5ea088f1a6616cc6de531f66f25c3d) +* Fixes bazel build errors. (commit: bc07ec9015cba820be7f1087153d216964bd1a0b) +* Add tf.distribute + Keras model save/load test in TF serving. (commit: 093734d002bd9de2a68d34160e23f35db196c080) +* Remove unused fields from MetaGraphDef proto message, stored in (commit: 1f8f2902b6465f239bb58af2b3fb27ba73b5c7c5) +* Fix typo (missing colon) in comment. (commit: 561cabbabe9d44da6b20fcf0eb9170859d3ea9fe) +* Makes ServerCore::Log(...) mockable. (commit: 4b00a803faea0b96c9cbce6fbe23dfaec93bfbd4) +* Uses VerifyFunctionInternal to replace VerifyResourceValidityInternal and VerifyValidityInternal. (commit: b7b3c33422bb5cf0813fdd6604742e7fa3841f84) +* Removed the net_http dependency on absl/base/internal/raw_logging. (commit: 4e12a193ad27fa31cb1e42e9a7fe7b5c08f74c52) +* Override TF defined Abseil version to a more recent version, (commit: 1c0291118f34ec7ba958a9cee594414f6531c0f3) +* Makes VerifyValidity, Normalize and IsNormalized method virtual. (commit: 071634e39f47cde52996c8bfd1ddda8abf4deef9) +* Example of creating tf_serving_warmup_requests (commit: 1623705e4205bc779109f8c4d1eadf6d3f24a549) +* Don't copy SignatureDef. (commit: 28d32a1e487666c8b324c74afb633006ba5cbf17) +* Update resnet_warmup.py example (commit: 00b49bd3f4bcb3b17d1fb61bf302aacccf80c83e) +* Update resnet_warmup.py example (commit: 263025f091dd60d415dd22e9667c0f37f11209ff) +* Instrument BatchingSession::Run with TraceMe (commit: 929ab172ec3553a9d563b13dccfb0926d8bf3724) +* Remove contrib ops from model server from tensorflow 2.0. (commit: e7c987d4b10ac751081c62595fcd18be7481e67a) +* Use C++14 by default. (commit: 41873601c73bcb91e403f9ddd70a168ae117ddb0) +* o Switch to using the half_plus_two model from TF to tensorflow_serving one. (commit: 3ba8a6d8ac31572548bbe7922e4152a6b92e626c) +* Add TfLiteSession class to run inference on TensorFlow Lite Model. (commit: f2407e2011b5fc6d255c0ea54181f9cdd1d691e5) +* Add ability to load+run TF Lite model in ModelServer. (commit: d16ceafa044932e2d9ef84bbe1a6ae5c6356252f) + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +Abolfazl Shahbazi, chaox, gison93, Minglotus-6, William D. Irons, ynqa + +# Release 1.12.3 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* This release is based on TF version 1.12.3. + +# Release 1.14.0 + +## Major Features and Improvements + +* Use MKL-DNN contraction kernels by default. (commit: a075ebe5eff56f3311d6e2cc2d23e4e82567596b) +* Add option to refuse to unload the last servable version. (commit: c8496b199cedf3e38a7ad0dc4c46db2b341b28e5) +* Add ability to disable periodic filesystem polling (#1295). (commit: 72450555c83ea5e6d18d05362192ad85613b23b1) + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Add `enforce_session_run_timeout` inside `Server::Options`. (commit: de030640ec6ed2cd504ee0ad9335fb93aebe51b5) +* Add -o option, to pass params to `docker` command. (commit: dd59021d3f807f23390afa8a2bc34a6f7029ed24) +* Stop using reader locks (tf_shared_lock) on the read path of FastReadDynamicPtr. (commit: f04e583a6a700a4943a57b6758b3e131b0865e97) +* Add saved model tags to logging metadata. These tags are used by (commit: 6320701645d5aeceac49a4f02cc629159559f143) +* Adds an option in `SessionBundleConfig` to repeat warmup replay n times per request. (commit: 15cd20263c8362f534afecbdf98b9d929eac70fd) +* Improve tpu server warm up (commit: 63d31a33b4f6faeb0764bb159d403f2b49061aed) +* Official PIP package releases are now tied to a specific version of TensorFlow (commit: 9514c37d22f0b728e2db9e8c6f28fb11ebde0fad) +* Bump the minimal Bazel version to 0.24.1 (commit: 96a716ca31f753b0c3efc1ef60779b77f5c60845) +* Add new device type for TPU. (commit: c74861d61131e2248a70d9c72317df8c49eb8f1a) +* Fix incorrect formatting of decimal numbers in JSON output (#1332) (commit: d7c3b3deacbabf763ed44fb6932535016852e90a) +* Fixed the gzip uncompression support in the HTTP server for large request bodies. (commit: fb7835c7cd95c5b6b163cb2abd6a8b9a1a283689) +* Add stack memory resource kind. (commit: e56e72b3e4b9a597832734208a3da455f6db1a04) +* Adds ModelServer test for loading SavedModel exported from Keras Sequential API (commit: 9578f3d10c786c6714b9a8b481dd74f454402477) +* Ignore SIGPIPE for libevent，prevent the SIGPIPE signal from being raised (#1257) (commit: 8d88a5b3c4ac502113c798a470111ca65f47b0c2) +* Fix #1367 (commit: 58af9011d72cbd062501c3f8066bf4d9eee04a7a) +* Update Serving_REST_simple.ipynb (commit: 3870ba59a764d859fc137a8363588c94906e0f5f) +* Updates README with link to architecture overview (commit: d233a82e0a569d5ccd23a0cbada8099644698dc6) +* Update example section to use Docker (commit: a5fc8bbc20f712fd6c4c148ff4d94a9231b79ceb) + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +G. Hussain Chinoy, Karthik Vadla, mkim301, yjhjstz + +# Release 1.13.0 + +## Major Features and Improvements + +* Support for TensorRT 5.0 (GPU docker image built against CUDA 10 and TensorRT 5.0) +* Support for listening gRPC over UNIX socket (commit: a25b0dad3984d3b154db1144df9d3b447b19aae6) +* New [GPU version of TensorFlow Serving API PIP package](https://pypi.org/project/tensorflow-serving-api-gpu/). This depends on the `tensorflow-gpu` instead of `tensorflow` PIP package, but is otherwise identical. (commit: 525c1af73ca543ce0165b3d22f0bbf21094fc443) +* [TF Serving end-to-end colab](tensorflow_serving/g3doc/tutorials/Serving_REST_simple.ipynb)! Training with Keras, serving with TF Serving and REST API (commit: 1ff8aadf20d75294aa4d496a807320603c6887c6) + +## Breaking Changes + +* No breaking changes. + +## Bug Fixes and Other Changes + +* Make error message for input size mismatch in `Predict` call even more actionable. (commit: 7237fb54c8d5898713e0bba7573add60cd19c25e) +* Document how to use the version policy to pin a specific version, or serve multiple versions, of a model. (commit: 2724bfee911f1d2294a9ceb705bbd09a2701c344) +* Document config reloading and model version labels. (commit: f4890afdc42f10f125cba64c3c2f2c01309ba2e2) +* Fix the compile error on ARM-32 in net_http/server. (commit: 5446fd973de228693c1652acd4922dc4b177f77a) +* Adds ModelSpec to SessionRunResponse. (commit: 58a22637ef5e3c50153eb42eff652137eb18c94a) +* Add MKL support (commit: 8f792532bea10d82fd3c3b126412d0546f54ae28) +* Fix default path of Prometheus metrics endpoint (commit: 9d05b0c17be47d3260ab58c2b9ac97e202699b96) +* Add monitoring metrics for saved model (export_dir) warm up latency. (commit: de0935b64ec972879ae623aa4f438282a4281dcc) +* Add more details/clarification to model version labels documentation. (commit: f9e6ac4d60a4044fc3b8c07719d0faaeae401dda) +* Split `--tensorflow_session_parallelism` flag into two new flags: `--tensorflow_intra_op_parallelism` and `--tensorflow_inter_op_parallelism` (commit: 71092e448c5432f4411f7333a02b274f0a3cdd3f) +* Update CPU Docker images to Ubuntu 18.04 (commit: 8023fba48c5b47a81fec25c17ba385a720650ef8) +* Upgrade to Bazel 0.20.0 (commit: fc0b75f2e325a187794bf437ff3227510d261afb) +* Update Python 2 scripts to be compatible with both Python 2 and 3 (commit: 846d443bb506f07242cd99347901f3ad5b7efe6a) + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +Daniel Shi, Karthik Vadla, lapolonio, robert, Shintaro Murakami, Siju, Tom Forbes, Ville TöRhöNen + +# Release 1.12.0 + +## Major Features and Improvements +* Add new REST API to [get model status](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/api_rest.md#model-status-api) from ModelServer (commit: 00e459f1604c40c073cbb9cb92d72cb6a88be9cd) +* Add new REST API to [get model metadata](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/api_rest.md#model-metadata-api) from ModelServer (fixes #1115) (commit: 97687024c3b7515d2f2979c35054f44c8f84d146) +* Support accepting gzipped REST API requests (fixes #1091) (commit: b94f6c89335782a7f175e8973c4f326375c55120) + +## Breaking Changes + +None + +## Bug Fixes and Other Changes + +* Update MKL build (commit: e11bd51540212242911dae00c8507e2852a5ad5a) +* Remove version pinning on pip packages (commit: 462072c2d78124c2769f820f7b63ee086de4e305) +* Update basic serving tutorials (commit: 33a4b052cedc39c21107bc99a090b59ca64ec568) +* Replacing legacy_init_op argument in SavedModelBuilder with main_op. (commit: 2fda31f905eefd2d108e9c84b8d7d55e4e482833) +* Add git hash for version metadata of model server and add tags for dev and nightly builds. (commit: 5c7740fc3d8d5c017643a8cc40a7202717b10dd6) +* Add error messages for specific cases when json for REST requests (commit: a17c89202e68bf19f369b9cbc97db7ced283b874) +* Python examples now run in a hermetic environment with all required dependencies (commit: 793fd90ee41ac34fa4c9261eef2d2c908dca9735) + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +Charles Verge, demfier, Kamidi Preetham, Lihang Li, naurril, vfdev, Yu Zheng + +# Release 1.11.1 + +## Bug Fixes and Other Changes + +* Fix version of model server binary (Fixes #1134) +* Range check floating point numbers correctly (Fixes #1136). +* Fix docker run script for same user and group name (Fixes #1137). +* Fix GPU build (Fixes #1150) + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +vfdev + +# Release 1.11.0 + +## Major Features and Improvements + +* Prometheus exporter for TF metrics (see https://github.com/tensorflow/serving/commit/021efbd3281aa815cab0b35eab6d6d25249c12d4 for details). + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Built against TensorFlow [1.11.0](https://github.com/tensorflow/tensorflow/releases/tag/v1.11.0) +* Accept integers for float/doubles in JSON REST API requests +* TF Serving API is now pre-built into Docker development images +* GPU Docker images are now built against cuDNN 7.2 +* Add `--max_num_load_retries` flag to ModelServer (fixes #1099) +* Add user-configured model version labels to the stand-alone ModelServer binary. +* Directly import tensor.proto.h (the transitive import will be removed from tensor.h soon) +* Building optimized TensorFlow Serving binaries is now easier (see [docs](https://github.com/tensorflow/serving/g3doc/setup.md]) for details) +* Adds columnar format support for input/output tensors in Predict REST API (fixes #1047) +* Development Dockerfiles now produce a more optimized ModelServer +* Fixed TensorFlow Serving API PyPi package overwriting TensorFlow package. + +## Thanks to our Contributors + +This release contains contributions from many people at Google, as well as: + +Feisan, joshua.horowitz, Prashanth Reddy Basani, tianyapiaozi, Vamsi Sripathi, Yu Zheng + +# Release 1.11.0-rc0 + +## Major Features and Improvements + +* Prometheus exporter for TF metrics (see https://github.com/tensorflow/serving/commit/021efbd3281aa815cab0b35eab6d6d25249c12d4 for details). +* Added new REST API to [get status of model(s)](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/api_rest.md#model-status-api) from ModelServer. + +## Breaking Changes + +* No breaking changes + +## Bug Fixes and Other Changes + +* Built against TensorFlow [1.11.0-rc0](https://github.com/tensorflow/tensorflow/releases/tag/v1.11.0-rc0). +* Directly import tensor.proto.h (the transitive import will be removed from tensor.h soon) +* Building optimized TensorFlow Serving binaries is now easier (see [docs](https://github.com/tensorflow/serving/g3doc/setup.md]) for details) +* Adds columnar format support for input/output tensors in Predict REST API (fixes #1047) +* Development Dockerfiles now produce a more optimized ModelServer +* Fixed TensorFlow Serving API PyPi package overwriting TensorFlow package. + +# Release 1.10.0 + +## Major Features and Improvements + +* No major features or improvements. + +## Breaking Changes + +* TensorFlow Serving API now uses gRPC's GA release. The beta gRPC API has been deprecated, and will be removed in a future version of TensorFlow Serving. Please update your gRPC client code ([sample](https://github.com/tensorflow/serving/commit/aa35cfdb24016f6d88f82c53d45c8ce9fa550499#diff-e7d756a12c65a8b5ac90229b23523023)) +* Docker images for GPU are built against NCCL 2.2, in following with Tensorflow 1.10. + +## Bug Fixes and Other Changes + +* Built against TensorFlow 1.10. +* Added GPU serving Docker image. +* Repo cloning and shell prompt in example readme. +* Updated Docker instructions. +* Updated min Bazel version (0.15.0). +* Convert TF_CHECK_OKs to TF_ASSERT_OK in some unit tests. +* Remove error suppression (.IgnoreError()) from BasicManager. +* Add new bazel_in_docker.sh tool for doing hermetic bazel builds. +* Fix erroneous formatting of numbers in REST API output that are larger than 6 digits. +* Add support for Python 3 while also compatible with Python 2.7 in mnist_saved_model.py. +* Fix an incorrect link to Dockerfile.devel-gpu. +* Add util for get model status. +* Adding support for secure channel to ModelServer. +* Add version output to model server binary. +* Change ServerRequestLogger::Update to only create new and delete old loggers if needed. +* Have the Model Server interpret specific hard-coded model version labels "stable" and "canary" as the smallest and largest version#, respectively. +* Add half_plus_two CPU and GPU models to test data. + # Release 0.4.0 Initial release of TensorFlow Serving. diff --git a/WORKSPACE b/WORKSPACE index 5a7e12f68ad..3981273a85a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,155 @@ workspace(name = "tf_serving") -# Please add all new TensorFlow Serving dependencies in workspace.bzl. -load('//tensorflow_serving:workspace.bzl', 'tf_serving_workspace') -tf_serving_workspace(__workspace_dir__) +# TODO(b/269515133): We temporarily remove remote_predict from our builds for +# 2.12 due to a breakage caused by +# github.com/tensorflow/tensorflow/commit/6147c03eb9af1e5d2ae155045b33e909ef96944e +# This will be removed in a subsequent release. +local_repository( + name = "ignore_remote_predict", + path = "tensorflow_serving/experimental/tensorflow/ops/remote_predict/", +) + +# ===== TensorFlow dependency ===== +# +# TensorFlow is imported here instead of in tf_serving_workspace() because +# existing automation scripts that bump the TF commit hash expect it here. +# +# To update TensorFlow to a new revision. +# 1. Update the 'git_commit' args below to include the new git hash. +# 2. Get the sha256 hash of the archive with a command such as... +# curl -L https://github.com/tensorflow/tensorflow/archive/.tar.gz | sha256sum +# and update the 'sha256' arg with the result. +# 3. Request the new archive to be mirrored on mirror.bazel.build for more +# reliable downloads. +load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") +tensorflow_http_archive( + name = "org_tensorflow", + sha256 = "e2dea5a1873f900a6af48ae16eccc20b275b5f3edd5b1a3b47da3feadfb513d9", + git_commit = "5fb3b1fefda9320202da184752a3366fbeddfeac", + patch = "//third_party/tensorflow:tensorflow.patch", +) + +# Import all of TensorFlow Serving's external dependencies. +# Downstream projects (projects importing TensorFlow Serving) need to +# duplicate all code below in their WORKSPACE file in order to also initialize +# those external dependencies. +load("//tensorflow_serving:workspace.bzl", "tf_serving_workspace") +tf_serving_workspace() + +# Check bazel version requirement, which is stricter than TensorFlow's. +load("@bazel_skylib//lib:versions.bzl", "versions") +versions.check("6.5.0") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "bazel_skylib", + sha256 = "74d544d96f4a5bb630d465ca8bbcfe231e3594e5aae57e1edbf17a6eb3ca2506", + urls = [ + "https://storage.googleapis.com/mirror.tensorflow.org/github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.3.0/bazel-skylib-1.3.0.tar.gz", + ], +) + +http_archive( + name = "rules_python", + sha256 = "84aec9e21cc56fbc7f1335035a71c850d1b9b5cc6ff497306f84cced9a769841", + strip_prefix = "rules_python-0.23.1", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.23.1/rules_python-0.23.1.tar.gz", +) + +load("@rules_python//python:repositories.bzl", "python_register_toolchains") + +load( + "@rules_python//python:repositories.bzl", + "py_repositories", + "python_register_toolchains", +) +py_repositories() + +load( + "@org_tensorflow//tensorflow/tools/toolchains/python:python_repo.bzl", + "python_repository", +) +python_repository(name = "python_version_repo") +load("@python_version_repo//:py_version.bzl", "HERMETIC_PYTHON_VERSION") + +python_register_toolchains( + name = "python", + ignore_root_user_error = True, + python_version = HERMETIC_PYTHON_VERSION, +) + +# Initialize TensorFlow's external dependencies. +load("@org_tensorflow//tensorflow:workspace3.bzl", "tf_workspace3") +tf_workspace3() +load("@org_tensorflow//tensorflow:workspace2.bzl", "tf_workspace2") +tf_workspace2() +load("@org_tensorflow//tensorflow:workspace1.bzl", "tf_workspace1") +tf_workspace1() +load("@org_tensorflow//tensorflow:workspace0.bzl", "tf_workspace0") +tf_workspace0() + +# Initialize bazel package rules' external dependencies. +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") +rules_pkg_dependencies() + +load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_proto_toolchains") + +rules_proto_dependencies() +rules_proto_toolchains() + +load( + "@local_xla//third_party/py:python_wheel.bzl", + "python_wheel_version_suffix_repository", +) + +python_wheel_version_suffix_repository(name = "tf_wheel_version_suffix") + +load( + "@local_xla//third_party/gpus/cuda/hermetic:cuda_json_init_repository.bzl", + "cuda_json_init_repository", +) + +cuda_json_init_repository() + +load( + "@cuda_redist_json//:distributions.bzl", + "CUDA_REDISTRIBUTIONS", + "CUDNN_REDISTRIBUTIONS", +) +load( + "@local_xla//third_party/gpus/cuda/hermetic:cuda_redist_init_repositories.bzl", + "cuda_redist_init_repositories", + "cudnn_redist_init_repository", +) + +cuda_redist_init_repositories( + cuda_redistributions = CUDA_REDISTRIBUTIONS, +) + +cudnn_redist_init_repository( + cudnn_redistributions = CUDNN_REDISTRIBUTIONS, +) + +load( + "@local_xla//third_party/gpus/cuda/hermetic:cuda_configure.bzl", + "cuda_configure", +) + +cuda_configure(name = "local_config_cuda") + +load( + "@local_xla//third_party/nccl/hermetic:nccl_redist_init_repository.bzl", + "nccl_redist_init_repository", +) + +nccl_redist_init_repository() + +load( + "@local_xla//third_party/nccl/hermetic:nccl_configure.bzl", + "nccl_configure", +) + +nccl_configure(name = "local_config_nccl") -# Specify the minimum required bazel version. -load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") -check_version("0.3.0") \ No newline at end of file diff --git a/tensorflow b/tensorflow deleted file mode 160000 index 83fe2a5b732..00000000000 --- a/tensorflow +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 83fe2a5b7328e1b754b53cbbcf9b313450a2f863 diff --git a/tensorflow_serving/BUILD b/tensorflow_serving/BUILD index 02b8aeec1b1..0d5ec358592 100644 --- a/tensorflow_serving/BUILD +++ b/tensorflow_serving/BUILD @@ -1,10 +1,8 @@ # Description: Tensorflow Serving. -package( - default_visibility = ["//tensorflow_serving:internal"], -) +package(default_visibility = ["//tensorflow_serving:internal"]) -licenses(["notice"]) # Apache 2.0 +licenses(["notice"]) exports_files(["LICENSE"]) @@ -13,6 +11,7 @@ exports_files(["LICENSE"]) package_group( name = "internal", packages = [ + "//learning/gemini/deployment/disaggregation/...", "//tensorflow_serving/...", ], ) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 893aa10a59d..3f4dc15a460 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -1,16 +1,24 @@ # Description: Tensorflow Serving APIs. +# buildifier: disable=out-of-order-load + +# Placeholder: load java_proto_library + +# Placeholder: load py_library +# Placeholder: load kt_jvm_proto_library +load("@org_tensorflow//tensorflow/core/platform:build_config.bzl", "tf_jspb_proto_library", "tf_pyclif_proto_library") +load("//tensorflow_serving:serving.bzl", "serving_go_grpc_library", "serving_proto_library", "serving_proto_library_py", "serving_tensorflow_proto_dep") + +# buildifier: enable=out-of-order-load + +# Placeholder: load go_proto_library + package( default_visibility = ["//visibility:public"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["layering_check"], ) -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) +licenses(["notice"]) filegroup( name = "all_files", @@ -23,17 +31,69 @@ filegroup( ), ) -load("//tensorflow_serving:serving.bzl", "serving_proto_library") -load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") +serving_proto_library( + name = "get_model_metadata_proto", + srcs = ["get_model_metadata.proto"], + deps = [ + ":model_proto", + "@com_google_protobuf//:cc_wkt_protos", + serving_tensorflow_proto_dep( + "@org_tensorflow//tensorflow/core:protos_all", + ), + ], +) + +serving_proto_library_py( + name = "get_model_metadata_proto_py_pb2", + srcs = ["get_model_metadata.proto"], + proto_library = "get_model_metadata_proto", + deps = [ + ":model_proto_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + +tf_pyclif_proto_library( + name = "get_model_metadata_pyclif", + proto_lib = ":get_model_metadata_proto", + proto_srcfile = "get_model_metadata.proto", +) + +serving_proto_library( + name = "input_proto", + srcs = ["input.proto"], + deps = [ + serving_tensorflow_proto_dep( + "@org_tensorflow//tensorflow/core:protos_all", + ), + ], +) + +serving_proto_library_py( + name = "input_proto_py_pb2", + srcs = ["input.proto"], + proto_library = "input_proto", + deps = [ + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + +tf_pyclif_proto_library( + name = "input_pyclif", + proto_lib = ":input_proto", + proto_srcfile = "input.proto", +) + +tf_jspb_proto_library( + name = "input_jspb_proto", + deps = [":input_proto"], +) serving_proto_library( name = "model_proto", srcs = ["model.proto"], - cc_api_version = 2, - go_api_version = 2, - java_api_version = 2, deps = [ - "@protobuf//:cc_wkt_protos", + "@com_google_protobuf//:cc_wkt_protos", ], ) @@ -44,15 +104,25 @@ serving_proto_library_py( deps = [], ) +tf_pyclif_proto_library( + name = "model_pyclif", + proto_lib = ":model_proto", + proto_srcfile = "model.proto", +) + +tf_jspb_proto_library( + name = "model_jspb_proto", + deps = [":model_proto"], +) + serving_proto_library( name = "predict_proto", srcs = ["predict.proto"], - cc_api_version = 2, - go_api_version = 2, - java_api_version = 2, deps = [ ":model_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + serving_tensorflow_proto_dep( + "@org_tensorflow//tensorflow/core:protos_all", + ), ], ) @@ -66,21 +136,336 @@ serving_proto_library_py( ], ) +tf_jspb_proto_library( + name = "predict_jspb_proto", + deps = [":predict_proto"], +) + +serving_proto_library( + name = "logging_proto", + srcs = ["logging.proto"], + deps = [ + ":model_proto", + "//tensorflow_serving/config:logging_config_proto", + ], +) + +serving_proto_library_py( + name = "logging_proto_py_pb2", + srcs = ["logging.proto"], + proto_library = "logging_proto", + deps = [ + ":model_proto_py_pb2", + "//tensorflow_serving/config:logging_config_proto_py_pb2", + ], +) + +serving_proto_library( + name = "prediction_log_proto", + srcs = ["prediction_log.proto"], + deps = [ + ":classification_proto", + ":inference_proto", + ":logging_proto", + ":predict_proto", + ":regression_proto", + ":session_service_proto", + ], +) + +serving_proto_library_py( + name = "prediction_log_proto_py_pb2", + srcs = ["prediction_log.proto"], + proto_library = "prediction_log_proto", + deps = [ + ":classification_proto_py_pb2", + ":inference_proto_py_pb2", + ":predict_proto_py_pb2", + ":regression_proto_py_pb2", + ":session_service_proto_py_pb2", + "//tensorflow_serving/apis:logging_proto_py_pb2", + ], +) + +tf_pyclif_proto_library( + name = "prediction_log_pyclif", + proto_lib = ":prediction_log_proto", + proto_srcfile = "prediction_log.proto", +) + serving_proto_library( name = "prediction_service_proto", srcs = ["prediction_service.proto"], has_services = 1, - cc_api_version = 2, cc_grpc_version = 1, - go_api_version = 2, - java_api_version = 2, deps = [ + ":classification_proto", + ":get_model_metadata_proto", + ":inference_proto", ":predict_proto", + ":regression_proto", ], ) py_library( name = "prediction_service_proto_py_pb2", - srcs = ["prediction_service_pb2.py"], - deps = [":predict_proto_py_pb2"], + srcs = [ + "prediction_service_pb2.py", + "prediction_service_pb2_grpc.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":classification_proto_py_pb2", + ":get_model_metadata_proto_py_pb2", + ":inference_proto_py_pb2", + ":predict_proto_py_pb2", + ":regression_proto_py_pb2", + ], +) + +tf_pyclif_proto_library( + name = "predict_pyclif", + proto_lib = ":predict_proto", + proto_srcfile = "predict.proto", +) + +serving_go_grpc_library( + name = "prediction_service_grpc", + srcs = [":prediction_service_proto"], + deps = [":prediction_service_go_proto"], +) + +serving_proto_library( + name = "status_proto", + srcs = ["status.proto"], + deps = [ + serving_tensorflow_proto_dep( + "@org_tensorflow//tensorflow/core:protos_all", + ), + ], +) + +serving_proto_library_py( + name = "status_proto_py_pb2", + srcs = ["status.proto"], + proto_library = "status_proto", + deps = [ + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + +serving_proto_library( + name = "model_management_proto", + srcs = ["model_management.proto"], + deps = [ + ":status_proto", + "//tensorflow_serving/config:model_server_config_proto", + ], +) + +serving_proto_library_py( + name = "model_management_proto_py_pb2", + srcs = ["model_management.proto"], + proto_library = "model_management_proto", + deps = [ + ":status_proto_py_pb2", + "//tensorflow_serving/config:model_server_config_proto_py_pb2", + ], +) + +serving_proto_library( + name = "get_model_status_proto", + srcs = ["get_model_status.proto"], + deps = [ + ":model_proto", + ":status_proto", + ], +) + +serving_proto_library_py( + name = "get_model_status_proto_py_pb2", + srcs = ["get_model_status.proto"], + proto_library = "get_model_status_proto", + deps = [ + ":model_proto_py_pb2", + ":status_proto_py_pb2", + ], +) + +serving_proto_library( + name = "model_service_proto", + srcs = ["model_service.proto"], + has_services = 1, + cc_grpc_version = 1, + deps = [ + ":get_model_status_proto", + ":model_management_proto", + ], +) + +py_library( + name = "model_service_proto_py_pb2", + srcs = [ + "model_service_pb2.py", + "model_service_pb2_grpc.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":get_model_status_proto_py_pb2", + ":model_management_proto_py_pb2", + ], +) + +serving_go_grpc_library( + name = "model_service_grpc", + srcs = [":model_service_proto"], + deps = [":model_service_go_proto"], +) + +serving_proto_library( + name = "classification_proto", + srcs = ["classification.proto"], + deps = [ + ":input_proto", + ":model_proto", + ], +) + +serving_proto_library_py( + name = "classification_proto_py_pb2", + srcs = ["classification.proto"], + proto_library = "classification_proto", + deps = [ + ":input_proto_py_pb2", + ":model_proto_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + +tf_pyclif_proto_library( + name = "classification_pyclif", + proto_lib = ":classification_proto", + proto_srcfile = "classification.proto", +) + +tf_jspb_proto_library( + name = "classification_jspb_proto", + deps = [":classification_proto"], +) + +serving_proto_library( + name = "inference_proto", + srcs = ["inference.proto"], + deps = [ + ":classification_proto", + ":input_proto", + ":model_proto", + ":regression_proto", + ], +) + +serving_proto_library_py( + name = "inference_proto_py_pb2", + srcs = ["inference.proto"], + proto_library = "inference_proto", + deps = [ + ":classification_proto_py_pb2", + ":input_proto_py_pb2", + ":model_proto_py_pb2", + ":regression_proto_py_pb2", + ], +) + +tf_pyclif_proto_library( + name = "inference_pyclif", + proto_lib = ":inference_proto", + proto_srcfile = "inference.proto", +) + +tf_jspb_proto_library( + name = "inference_jspb_proto", + deps = [":inference_proto"], +) + +serving_proto_library( + name = "regression_proto", + srcs = ["regression.proto"], + deps = [ + ":input_proto", + ":model_proto", + ], +) + +serving_proto_library_py( + name = "regression_proto_py_pb2", + srcs = ["regression.proto"], + proto_library = "regression_proto", + deps = [ + ":input_proto_py_pb2", + ":model_proto_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + +tf_jspb_proto_library( + name = "regression_jspb_proto", + deps = [":regression_proto"], +) + +serving_proto_library( + name = "session_service_proto", + srcs = ["session_service.proto"], + has_services = 1, + deps = [ + ":model_proto", + serving_tensorflow_proto_dep( + "@org_tensorflow//tensorflow/core:protos_all", + ), + ], +) + +tf_jspb_proto_library( + name = "session_service_jspb_proto", + visibility = ["//visibility:private"], + deps = [":session_service_proto"], +) + +serving_proto_library_py( + name = "session_service_proto_py_pb2", + srcs = ["session_service.proto"], + proto_library = "session_service_proto", + deps = [ + ":model_proto_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + +tf_pyclif_proto_library( + name = "session_service_pyclif", + proto_lib = ":session_service_proto", + proto_srcfile = "session_service.proto", +) + +tf_pyclif_proto_library( + name = "regression_pyclif", + proto_lib = ":regression_proto", + proto_srcfile = "regression.proto", +) + +cc_library( + name = "classifier", + hdrs = ["classifier.h"], + deps = [ + ":classification_cc_proto", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "regressor", + hdrs = ["regressor.h"], + deps = [ + ":regression_cc_proto", + "@org_tensorflow//tensorflow/core:lib", + ], ) diff --git a/tensorflow_serving/apis/classification.proto b/tensorflow_serving/apis/classification.proto new file mode 100644 index 00000000000..4de0b7dfc88 --- /dev/null +++ b/tensorflow_serving/apis/classification.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; + +package tensorflow.serving; + +import "tensorflow_serving/apis/input.proto"; +import "tensorflow_serving/apis/model.proto"; + +option cc_enable_arenas = true; + +// A single class. +message Class { + // Label or name of the class. + string label = 1; + // Score for this class (e.g., the probability the item belongs to this + // class). As per the proto3 default-value semantics, if the score is missing, + // it should be treated as 0. + float score = 2; +} + +// List of classes for a single item (tensorflow.Example). +message Classifications { + repeated Class classes = 1; +} + +// Contains one result per input example, in the same order as the input in +// ClassificationRequest. +message ClassificationResult { + repeated Classifications classifications = 1; +} + +// RPC Interfaces + +message ClassificationRequest { + // Model Specification. If version is not specified, will use the latest + // (numerical) version. + ModelSpec model_spec = 1; + + // Input data. + tensorflow.serving.Input input = 2; +} + +message ClassificationResponse { + // Effective Model Specification used for classification. + ModelSpec model_spec = 2; + + // Result of the classification. + ClassificationResult result = 1; +} diff --git a/tensorflow_serving/apis/classifier.h b/tensorflow_serving/apis/classifier.h new file mode 100644 index 00000000000..fbd5ef573f0 --- /dev/null +++ b/tensorflow_serving/apis/classifier.h @@ -0,0 +1,49 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ +#define TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/classification.pb.h" + +namespace tensorflow { +namespace serving { + +/// Model-type agnostic interface for performing classification. +/// +/// Specific implementations will exist for different model types +/// (e.g. TensorFlow SavedModel) that can convert the request into a model +/// specific input and know how to convert the output into a generic +/// ClassificationResult. +class ClassifierInterface { + public: + /// Given a ClassificationRequest, populates the ClassificationResult with the + /// result. + /// + /// @param request Input request specifying the model/signature to query + /// along with the data payload. + /// @param result The output classifications that will get populated. + /// @return A status object indicating success or failure. + virtual Status Classify(const ClassificationRequest& request, + ClassificationResult* result) = 0; + + virtual ~ClassifierInterface() = default; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ diff --git a/tensorflow_serving/apis/get_model_metadata.proto b/tensorflow_serving/apis/get_model_metadata.proto new file mode 100644 index 00000000000..5d765d8e39b --- /dev/null +++ b/tensorflow_serving/apis/get_model_metadata.proto @@ -0,0 +1,30 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +import "google/protobuf/any.proto"; +import "tensorflow/core/protobuf/meta_graph.proto"; +import "tensorflow_serving/apis/model.proto"; + +// Message returned for "signature_def" field. +message SignatureDefMap { + map signature_def = 1; +}; + +message GetModelMetadataRequest { + // Model Specification indicating which model we are querying for metadata. + // If version is not specified, will use the latest (numerical) version. + ModelSpec model_spec = 1; + // Metadata fields to get. Currently supported: "signature_def". + repeated string metadata_field = 2; +} + +message GetModelMetadataResponse { + // Model Specification indicating which model this metadata belongs to. + ModelSpec model_spec = 1; + // Map of metadata field name to metadata field. The options for metadata + // field name are listed in GetModelMetadataRequest. Currently supported: + // "signature_def". + map metadata = 2; +} diff --git a/tensorflow_serving/apis/get_model_status.proto b/tensorflow_serving/apis/get_model_status.proto new file mode 100644 index 00000000000..88812472a6c --- /dev/null +++ b/tensorflow_serving/apis/get_model_status.proto @@ -0,0 +1,68 @@ +syntax = "proto3"; + +package tensorflow.serving; + +import "tensorflow_serving/apis/model.proto"; +import "tensorflow_serving/apis/status.proto"; + +option cc_enable_arenas = true; + +// GetModelStatusRequest contains a ModelSpec indicating the model for which +// to get status. +message GetModelStatusRequest { + // Model Specification. If version is not specified, information about all + // versions of the model will be returned. If a version is specified, the + // status of only that version will be returned. + ModelSpec model_spec = 1; +} + +// Version number, state, and status for a single version of a model. +message ModelVersionStatus { + // Model version. + int64 version = 1; + + // States that map to ManagerState enum in + // tensorflow_serving/core/servable_state.h + enum State { + // Default value. + UNKNOWN = 0; + + // The manager is tracking this servable, but has not initiated any action + // pertaining to it. + START = 10; + + // The manager has decided to load this servable. In particular, checks + // around resource availability and other aspects have passed, and the + // manager is about to invoke the loader's Load() method. + LOADING = 20; + + // The manager has successfully loaded this servable and made it available + // for serving (i.e. GetServableHandle(id) will succeed). To avoid races, + // this state is not reported until *after* the servable is made + // available. + AVAILABLE = 30; + + // The manager has decided to make this servable unavailable, and unload + // it. To avoid races, this state is reported *before* the servable is + // made unavailable. + UNLOADING = 40; + + // This servable has reached the end of its journey in the manager. Either + // it loaded and ultimately unloaded successfully, or it hit an error at + // some point in its lifecycle. + END = 50; + } + + // Model state. + State state = 2; + + // Model status. + StatusProto status = 3; +} + +// Response for ModelStatusRequest on successful run. +message GetModelStatusResponse { + // Version number and status information for applicable model version(s). + repeated ModelVersionStatus model_version_status = 1 + [json_name = "model_version_status"]; +} diff --git a/tensorflow_serving/apis/inference.proto b/tensorflow_serving/apis/inference.proto new file mode 100644 index 00000000000..16e85cee5fb --- /dev/null +++ b/tensorflow_serving/apis/inference.proto @@ -0,0 +1,59 @@ +// This file contains messages for various machine learning inferences +// such as regression and classification. +// +// In many applications more than one type of inference is desired for a single +// input. For example, given meteorologic data an application may want to +// perform a classification to determine if we should expect rain, snow or sun +// and also perform a regression to predict the temperature. +// Sharing the single input data between two inference tasks can be accomplished +// using MultiInferenceRequest and MultiInferenceResponse. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/classification.proto"; +import "tensorflow_serving/apis/input.proto"; +import "tensorflow_serving/apis/model.proto"; +import "tensorflow_serving/apis/regression.proto"; + +package tensorflow.serving; + +// Inference request such as classification, regression, etc... +message InferenceTask { + // Model Specification. If version is not specified, will use the latest + // (numerical) version. + // All ModelSpecs in a MultiInferenceRequest must access the same model name. + ModelSpec model_spec = 1; + + // Signature's method_name. Should be one of the method names defined in + // third_party/tensorflow/python/saved_model/signature_constants.py. + // e.g. "tensorflow/serving/classify". + string method_name = 2; +} + +// Inference result, matches the type of request or is an error. +message InferenceResult { + ModelSpec model_spec = 1; + + oneof result { + ClassificationResult classification_result = 2; + RegressionResult regression_result = 3; + } +} + +// Inference request containing one or more requests. +message MultiInferenceRequest { + // Inference tasks. + repeated InferenceTask tasks = 1; + + // Input data. + Input input = 2; +} + +// Inference request containing one or more responses. +message MultiInferenceResponse { + // List of results; one for each InferenceTask in the request, returned in the + // same order as the request. + repeated InferenceResult results = 1; +} diff --git a/tensorflow_serving/apis/input.proto b/tensorflow_serving/apis/input.proto new file mode 100644 index 00000000000..e47ff43d744 --- /dev/null +++ b/tensorflow_serving/apis/input.proto @@ -0,0 +1,82 @@ +// Input used in serving APIs. Based on the tensorflow.Example family of +// feature representations. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow/core/example/example.proto"; + +package tensorflow.serving; + +// Specifies one or more fully independent input Examples. +// See examples at: +// https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto +message ExampleList { + repeated tensorflow.Example examples = 1; +} + +// Specifies one or more independent input Examples, with a common context +// Example. +// +// The common use case for context is to cleanly and optimally specify some +// features that are common across multiple examples. +// +// See example below with a search query as the context and multiple restaurants +// to perform some inference on. +// +// context: { +// features: { +// feature: { +// key : "query" +// value: { +// bytes_list: { +// value: [ "pizza" ] +// } +// } +// } +// } +// } +// examples: { +// features: { +// feature: { +// key : "cuisine" +// value: { +// bytes_list: { +// value: [ "Pizzeria" ] +// } +// } +// } +// } +// } +// examples: { +// features: { +// feature: { +// key : "cuisine" +// value: { +// bytes_list: { +// value: [ "Taqueria" ] +// } +// } +// } +// } +// } +// +// Implementations of ExampleListWithContext merge the context Example into each +// of the Examples. Note that feature keys must not be duplicated between the +// Examples and context Example, or the behavior is undefined. +// +// See also: +// tensorflow/core/example/example.proto +// https://developers.google.com/protocol-buffers/docs/proto3#maps +message ExampleListWithContext { + repeated tensorflow.Example examples = 1; + tensorflow.Example context = 2; +} + +message Input { + oneof kind { + ExampleList example_list = 1 [lazy = true]; + ExampleListWithContext example_list_with_context = 2 [lazy = true]; + } +} diff --git a/tensorflow_serving/apis/internal/BUILD b/tensorflow_serving/apis/internal/BUILD new file mode 100644 index 00000000000..0dd5582f545 --- /dev/null +++ b/tensorflow_serving/apis/internal/BUILD @@ -0,0 +1,23 @@ +# Internal implementation details of serving APIs. + +load("//tensorflow_serving:serving.bzl", "serving_proto_library") + +package( + default_visibility = [ + "//tensorflow_serving:internal", + ], + features = ["-layering_check"], +) + +licenses(["notice"]) + +serving_proto_library( + name = "serialized_input_proto", + srcs = ["serialized_input.proto"], + visibility = [ + "//tensorflow_serving:internal", + "@org_tensorflow//tensorflow_ranking/google:__pkg__", + ], + deps = [ + ], +) diff --git a/tensorflow_serving/apis/internal/serialized_input.proto b/tensorflow_serving/apis/internal/serialized_input.proto new file mode 100644 index 00000000000..862438e18e1 --- /dev/null +++ b/tensorflow_serving/apis/internal/serialized_input.proto @@ -0,0 +1,45 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Serialized counterparts of the messages in input.proto. These protos enable +// us to keep the original tensorflow.serving.Input's structure but with the +// tensorflow.Examples in their serialized form. When combined with lazy +// parsing, this improves performance by allowing us to skip a redundant +// deserialization/serialization loop. +// +// WARNING: These are internal implementation details and not part of the public +// API. + +syntax = "proto3"; + +package tensorflow.serving.internal; + +option cc_enable_arenas = true; + +message SerializedExampleList { + repeated bytes examples = 1; +} + +message SerializedExampleListWithContext { + repeated bytes examples = 1; + bytes context = 2; +} + +message SerializedInput { + oneof kind { + SerializedExampleList example_list = 1; + SerializedExampleListWithContext example_list_with_context = 2; + } +} diff --git a/tensorflow_serving/apis/logging.proto b/tensorflow_serving/apis/logging.proto new file mode 100644 index 00000000000..20d7fc6e307 --- /dev/null +++ b/tensorflow_serving/apis/logging.proto @@ -0,0 +1,22 @@ +syntax = "proto3"; + +package tensorflow.serving; + +import "tensorflow_serving/apis/model.proto"; +import "tensorflow_serving/config/logging_config.proto"; + +option cc_enable_arenas = true; + +// Metadata logged along with the request logs. +message LogMetadata { + ModelSpec model_spec = 1; + SamplingConfig sampling_config = 2; + // List of tags used to load the relevant MetaGraphDef from SavedModel. + repeated string saved_model_tags = 3; + int64 timestamp_secs = 4; // Seconds since epoch. + string dc = 5; // Datacenter where the request was logged. + string request_origin = 6; // Request origin identifier. + string request_criticality = 7; // Request QoS. + // An aggregation key that can be used for anonymization. + int64 safe_aggregation_privacy_key = 8; +} diff --git a/tensorflow_serving/apis/model.proto b/tensorflow_serving/apis/model.proto index 889303ff45d..afd95902beb 100644 --- a/tensorflow_serving/apis/model.proto +++ b/tensorflow_serving/apis/model.proto @@ -1,18 +1,36 @@ syntax = "proto3"; package tensorflow.serving; -option cc_enable_arenas = true; import "google/protobuf/wrappers.proto"; +option cc_enable_arenas = true; + // Metadata for an inference request such as the model name and version. message ModelSpec { // Required servable name. string name = 1; - // Optional version. If unspecified, will use the latest (numerical) version. - // Typically not needed unless coordinating across multiple models that were - // co-trained and/or have inter-dependencies on the versions used at inference - // time. - google.protobuf.Int64Value version = 2; + // Optional choice of which version of the model to use. + // + // Expected to be left unset in the common case. Should be specified when + // there is a strong version consistency requirement (e.g. when the model + // signature changes across versions and requests need to be + // version-specific). + // + // When left unspecified, the system will serve the best available version. + // This is typically the latest version, though during version transitions, + // notably when serving on a fleet of instances, may be either the previous or + // new version. + oneof version_choice { + // Use this specific version number. + google.protobuf.Int64Value version = 2; + + // Use the version associated with the given label. + string version_label = 4; + } + + // A named signature to evaluate. If unspecified, the default signature will + // be used. + string signature_name = 3; } diff --git a/tensorflow_serving/apis/model_management.proto b/tensorflow_serving/apis/model_management.proto new file mode 100644 index 00000000000..2eba09bca15 --- /dev/null +++ b/tensorflow_serving/apis/model_management.proto @@ -0,0 +1,25 @@ +syntax = "proto3"; + +package tensorflow.serving; + +import "tensorflow_serving/apis/status.proto"; +import "tensorflow_serving/config/model_server_config.proto"; + +option cc_enable_arenas = true; + +message ReloadConfigRequest { + ModelServerConfig config = 1; + repeated string metric_names = 2; +} + +message ReloadConfigResponse { + StatusProto status = 1; + repeated Metric metric = 2; +} + +message Metric { + string name = 1; + oneof value_increase { + int64 int64_value_increase = 2; + } +} diff --git a/tensorflow_serving/apis/model_service.proto b/tensorflow_serving/apis/model_service.proto new file mode 100644 index 00000000000..29a3b077512 --- /dev/null +++ b/tensorflow_serving/apis/model_service.proto @@ -0,0 +1,24 @@ +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/get_model_status.proto"; +import "tensorflow_serving/apis/model_management.proto"; + +package tensorflow.serving; + +// ModelService provides methods to query and update the state of the server, +// e.g. which models/versions are being served. +service ModelService { + // Gets status of model. If the ModelSpec in the request does not specify + // version, information about all versions of the model will be returned. If + // the ModelSpec in the request does specify a version, the status of only + // that version will be returned. + rpc GetModelStatus(GetModelStatusRequest) returns (GetModelStatusResponse); + + // Reloads the set of served models. The new config supersedes the old one, + // so if a model is omitted from the new config it will be unloaded and no + // longer served. + rpc HandleReloadConfigRequest(ReloadConfigRequest) + returns (ReloadConfigResponse); +} diff --git a/tensorflow_serving/apis/model_service_pb2.py b/tensorflow_serving/apis/model_service_pb2.py new file mode 100644 index 00000000000..76f1d606cb2 --- /dev/null +++ b/tensorflow_serving/apis/model_service_pb2.py @@ -0,0 +1,90 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorflow_serving/apis/model_service.proto +# To regenerate run +# python -m grpc.tools.protoc --python_out=. --grpc_python_out=. -I. tensorflow_serving/apis/model_service.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from tensorflow_serving.apis import get_model_status_pb2 as tensorflow__serving_dot_apis_dot_get__model__status__pb2 +from tensorflow_serving.apis import model_management_pb2 as tensorflow__serving_dot_apis_dot_model__management__pb2 + +DESCRIPTOR = _descriptor.FileDescriptor( + name='tensorflow_serving/apis/model_service.proto', + package='tensorflow.serving', + syntax='proto3', + serialized_pb=_b( + '\n+tensorflow_serving/apis/model_service.proto\x12\x12tensorflow.serving\x1a.tensorflow_serving/apis/get_model_status.proto\x1a.tensorflow_serving/apis/model_management.proto2\xe7\x01\n\x0cModelService\x12g\n\x0eGetModelStatus\x12).tensorflow.serving.GetModelStatusRequest\x1a*.tensorflow.serving.GetModelStatusResponse\x12n\n\x19HandleReloadConfigRequest\x12\'.tensorflow.serving.ReloadConfigRequest\x1a(.tensorflow.serving.ReloadConfigResponseB\x03\xf8\x01\x01\x62\x06proto3' + ), + dependencies=[ + tensorflow__serving_dot_apis_dot_get__model__status__pb2.DESCRIPTOR, + tensorflow__serving_dot_apis_dot_model__management__pb2.DESCRIPTOR, + ]) + +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) + +_MODELSERVICE = _descriptor.ServiceDescriptor( + name='ModelService', + full_name='tensorflow.serving.ModelService', + file=DESCRIPTOR, + index=0, + options=None, + serialized_start=164, + serialized_end=395, + methods=[ + _descriptor.MethodDescriptor( + name='GetModelStatus', + full_name='tensorflow.serving.ModelService.GetModelStatus', + index=0, + containing_service=None, + input_type=tensorflow__serving_dot_apis_dot_get__model__status__pb2. + _GETMODELSTATUSREQUEST, + output_type=tensorflow__serving_dot_apis_dot_get__model__status__pb2. + _GETMODELSTATUSRESPONSE, + options=None, + ), + _descriptor.MethodDescriptor( + name='HandleReloadConfigRequest', + full_name= + 'tensorflow.serving.ModelService.HandleReloadConfigRequest', + index=1, + containing_service=None, + input_type=tensorflow__serving_dot_apis_dot_model__management__pb2. + _RELOADCONFIGREQUEST, + output_type=tensorflow__serving_dot_apis_dot_model__management__pb2. + _RELOADCONFIGRESPONSE, + options=None, + ), + ]) +_sym_db.RegisterServiceDescriptor(_MODELSERVICE) + +DESCRIPTOR.services_by_name['ModelService'] = _MODELSERVICE + +# @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/apis/model_service_pb2_grpc.py b/tensorflow_serving/apis/model_service_pb2_grpc.py new file mode 100644 index 00000000000..44c578b8648 --- /dev/null +++ b/tensorflow_serving/apis/model_service_pb2_grpc.py @@ -0,0 +1,103 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorflow_serving/apis/model_service.proto +# To regenerate run +# python -m grpc.tools.protoc --python_out=. --grpc_python_out=. -I. tensorflow_serving/apis/model_service.proto + +import grpc + +from tensorflow_serving.apis import get_model_status_pb2 as tensorflow__serving_dot_apis_dot_get__model__status__pb2 +from tensorflow_serving.apis import model_management_pb2 as tensorflow__serving_dot_apis_dot_model__management__pb2 + + +class ModelServiceStub(object): + """ModelService provides methods to query and update the state of the server, + e.g. which models/versions are being served. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetModelStatus = channel.unary_unary( + '/tensorflow.serving.ModelService/GetModelStatus', + request_serializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusResponse.FromString, + ) + self.HandleReloadConfigRequest = channel.unary_unary( + '/tensorflow.serving.ModelService/HandleReloadConfigRequest', + request_serializer= + tensorflow__serving_dot_apis_dot_model__management__pb2. + ReloadConfigRequest.SerializeToString, + response_deserializer= + tensorflow__serving_dot_apis_dot_model__management__pb2. + ReloadConfigResponse.FromString, + ) + + +class ModelServiceServicer(object): + """ModelService provides methods to query and update the state of the server, + e.g. which models/versions are being served. + """ + + def GetModelStatus(self, request, context): + """Gets status of model. If the ModelSpec in the request does not specify + version, information about all versions of the model will be returned. If + the ModelSpec in the request does specify a version, the status of only + that version will be returned. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def HandleReloadConfigRequest(self, request, context): + """Reloads the set of served models. The new config supersedes the old one, + so if a model is omitted from the new config it will be unloaded and no + longer served. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ModelServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetModelStatus': + grpc.unary_unary_rpc_method_handler( + servicer.GetModelStatus, + request_deserializer= + tensorflow__serving_dot_apis_dot_get__model__status__pb2. + GetModelStatusRequest.FromString, + response_serializer= + tensorflow__serving_dot_apis_dot_get__model__status__pb2. + GetModelStatusResponse.SerializeToString, + ), + 'HandleReloadConfigRequest': + grpc.unary_unary_rpc_method_handler( + servicer.HandleReloadConfigRequest, + request_deserializer= + tensorflow__serving_dot_apis_dot_model__management__pb2. + ReloadConfigRequest.FromString, + response_serializer= + tensorflow__serving_dot_apis_dot_model__management__pb2. + ReloadConfigResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'tensorflow.serving.ModelService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/tensorflow_serving/apis/predict.proto b/tensorflow_serving/apis/predict.proto index 97011de793d..aaaaec3a2fe 100644 --- a/tensorflow_serving/apis/predict.proto +++ b/tensorflow_serving/apis/predict.proto @@ -1,38 +1,201 @@ syntax = "proto3"; package tensorflow.serving; -option cc_enable_arenas = true; import "tensorflow/core/framework/tensor.proto"; import "tensorflow_serving/apis/model.proto"; +option cc_enable_arenas = true; + // PredictRequest specifies which TensorFlow model to run, as well as // how inputs are mapped to tensors and how outputs are filtered before // returning to user. message PredictRequest { - // Model Specification. + // Model Specification. If version is not specified, will use the latest + // (numerical) version. ModelSpec model_spec = 1; // Input tensors. // Names of input tensor are alias names. The mapping from aliases to real - // input tensor names is expected to be stored as named generic signature - // under the key "inputs" in the model export. - // Each alias listed in a generic signature named "inputs" should be provided - // exactly once in order to run the prediction. + // input tensor names is stored in the SavedModel export as a prediction + // SignatureDef under the 'inputs' field. map inputs = 2; // Output filter. // Names specified are alias names. The mapping from aliases to real output - // tensor names is expected to be stored as named generic signature under - // the key "outputs" in the model export. + // tensor names is stored in the SavedModel export as a prediction + // SignatureDef under the 'outputs' field. // Only tensors specified here will be run/fetched and returned, with the // exception that when none is specified, all tensors specified in the // named signature will be run/fetched and returned. repeated string output_filter = 3; + + // Reserved field 4. + reserved 4; + + // Options for streaming requests to control how multiple requests/responses + // are handled within a single stream. + PredictStreamedOptions predict_streamed_options = 5; + + // Client identifier to group requests belonging to a specific entity. + // Example entities can be product ids, service names, user ids etc. + // Servers can use this to optimize placement, caching and colocation. + // TODO(b/329897437): Migrate to client_id in RequestOptions. + optional bytes client_id = 6; + + // Options for PredictRequest. + message RequestOptions { + // Client identifier to group requests belonging to a specific entity. + // Example entities can be product ids, service names, user ids etc. + // Servers can use this to optimize placement, caching and colocation. + optional bytes client_id = 1; + + // Deterministic mode for the request. When specified, model servers will + // reduce numeric instability based on different mode selections. + enum DeterministicMode { + DETERMINISTIC_MODE_UNSPECIFIED = 0; + + // Only supported in disaggregated serving. When set, the request will be + // pinned to a fixed decoder slot index that's deterministic across + // processes. + FIXED_DECODER_SLOT = 1; + } + + optional DeterministicMode deterministic_mode = 2; + + // Only supported in disaggregated serving. When set, additional arrays from + // prefill will be returned if available. + optional bool return_additional_arrays_from_prefill = 3; + + // Only supported in disaggregated serving. Returns these stop tokens in + // response if the model stops at them. The model may stop at other tokens, + // but will not return them in the response. + repeated int64 return_stoptokens = 4; + } + + optional RequestOptions request_options = 7; +} + +// Options only used for streaming requests that control how inputs/ouputs are +// handled in the stream. +message PredictStreamedOptions { + // Request state is used to control handling of individual streamed requests. + // + // SPLIT and END_SPLIT are used to handle splitting of requests. NONE is the + // default when the stream request is not split and used for a single-turn, + // single request. + // + // CANCEL is used for early termination of a stream. It can be sent anytime to + // cancel the stream. + // + // + // SPLIT is used when multiple streamed requests are used to generate a + // logical request. END_SPLIT should be called for the last split of the + // multi-turn request to start the processing of the current turn. NONE can + // not be interspersed with SPLIT and END_SPLIT messages. + // If another request is sent on the same stream after END_SPLIT, it can be + // either SPLIT or END_SPLIT to start accumulating input or trigger the next + // model turn respectively. + // + // Some examples with a mix of request states and the logical request. + // + // Example 1: + // NONE + // + // Single turn, single request. + // + // Example 2 : + // END_SPLIT + // + // Will be treated as a single logical input request for a single turn, + // similar to Example 1. + // + // Example 3: + // SPLIT + // SPLIT + // END_SPLIT + // + // Will be treated as a single logical input request for a single turn, + // similar to Example 1. + // + // Example 4: + // END_SPLIT + // END_SPLIT + // + // Will be treated as two logical turn requests (1. END_SPLIT 2. END_SPLIT) + // + // Example 5: + // SPLIT + // END_SPLIT + // SPLIT + // SPLIT + // END_SPLIT + // + // Will be treated as two logical turn requests (1. SPLIT, END_SPLIT 2. SPLIT, + // SPLIT, END_SPLIT) + // + // Incorrect Example 1: + // NONE + // END_SPLIT + // + // Invalid because NONE and END_SPLIT are interspersed. + // + // Incorrect Example 2: + // SPLIT + // SPLIT + // + // Invalid because END_SPLIT is never called. + // + // Incorrect Example 3: + // SPLIT + // NONE + // SPLIT + // END_SPLIT + // + // Invalid because NONE is interspersed with SPLIT/END_SPLIT. + enum RequestState { + NONE = 0; + SPLIT = 1; + END_SPLIT = 2; + CANCEL = 3; + } + + // Request state used to handle segmentation of requests. + // Only supported in disaggregated serving. + RequestState request_state = 1; + + // Input tensors split dimensions. + // Defines the dimension used to split input tensors specified + // in PredictRequest.inputs. The dimension will be used + // for concatenation of multiple SPLIT requests. + // + // For input tensor in PredictRequest.inputs that are not contained in this + // map, the tensors from the first SPLIT request will be used. + // + // For example, with an original input tensor of [[1, 2, 3, 4], [5, 6, 7, 8]]. + // + // For a split dimension of 0 and two requests (SPLIT and END_SPLIT), the + // input tensors for request 1 should be [1, 2, 3, 4] and request 2 should be + // be [5, 6, 7, 8]. + // + // For a split dimension of 1 and two requests (SPLIT and END_SPLIT), the + // input tensors for request 1 should be [[1, 2], [5, 6]] and request 2 should + // be [[3, 4], [7, 8]]. + // This field is ignored if request_state is NONE. + map split_dimensions = 2; + + // If true, there will be a single PredictResponse output. + // If false, output can be split into 1 or more PredictResponses. + // Value of this field should be the same for all requests in the stream. + // Only supported in disaggregated serving. + bool return_single_response = 3; } // Response for PredictRequest on successful run. message PredictResponse { + // Effective Model Specification used to process PredictRequest. + ModelSpec model_spec = 2; + // Output tensors. map outputs = 1; } diff --git a/tensorflow_serving/apis/prediction_log.proto b/tensorflow_serving/apis/prediction_log.proto new file mode 100644 index 00000000000..9deebc71264 --- /dev/null +++ b/tensorflow_serving/apis/prediction_log.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; + +package tensorflow.serving; + +import "tensorflow_serving/apis/classification.proto"; +import "tensorflow_serving/apis/inference.proto"; +import "tensorflow_serving/apis/logging.proto"; +import "tensorflow_serving/apis/predict.proto"; +import "tensorflow_serving/apis/regression.proto"; +import "tensorflow_serving/apis/session_service.proto"; + +option cc_enable_arenas = true; + +message ClassifyLog { + ClassificationRequest request = 1; + ClassificationResponse response = 2; +} + +message RegressLog { + RegressionRequest request = 1; + RegressionResponse response = 2; +} + +message PredictLog { + PredictRequest request = 1; + PredictResponse response = 2; +} + +message PredictStreamedLog { + repeated PredictRequest request = 1; + repeated PredictResponse response = 2; +} + +message MultiInferenceLog { + MultiInferenceRequest request = 1; + MultiInferenceResponse response = 2; +} + +message SessionRunLog { + SessionRunRequest request = 1; + SessionRunResponse response = 2; +} + +// Logged model inference request. +message PredictionLog { + LogMetadata log_metadata = 1; + oneof log_type { + ClassifyLog classify_log = 2; + RegressLog regress_log = 3; + PredictLog predict_log = 6; + PredictStreamedLog predict_streamed_log = 7; + MultiInferenceLog multi_inference_log = 4; + SessionRunLog session_run_log = 5; + } +} diff --git a/tensorflow_serving/apis/prediction_service.proto b/tensorflow_serving/apis/prediction_service.proto index 28bdd528a7a..44e655417fd 100644 --- a/tensorflow_serving/apis/prediction_service.proto +++ b/tensorflow_serving/apis/prediction_service.proto @@ -3,11 +3,29 @@ syntax = "proto3"; package tensorflow.serving; option cc_enable_arenas = true; +import "tensorflow_serving/apis/classification.proto"; +import "tensorflow_serving/apis/get_model_metadata.proto"; +import "tensorflow_serving/apis/inference.proto"; import "tensorflow_serving/apis/predict.proto"; +import "tensorflow_serving/apis/regression.proto"; +// open source marker; do not remove // PredictionService provides access to machine-learned models loaded by // model_servers. service PredictionService { + // Classify. + rpc Classify(ClassificationRequest) returns (ClassificationResponse); + + // Regress. + rpc Regress(RegressionRequest) returns (RegressionResponse); + // Predict -- provides access to loaded TensorFlow model. rpc Predict(PredictRequest) returns (PredictResponse); -} \ No newline at end of file + + // MultiInference API for multi-headed models. + rpc MultiInference(MultiInferenceRequest) returns (MultiInferenceResponse); + + // GetModelMetadata - provides access to metadata for loaded models. + rpc GetModelMetadata(GetModelMetadataRequest) + returns (GetModelMetadataResponse); +} diff --git a/tensorflow_serving/apis/prediction_service_pb2.py b/tensorflow_serving/apis/prediction_service_pb2.py index a2813d44fb1..e51a700f720 100644 --- a/tensorflow_serving/apis/prediction_service_pb2.py +++ b/tensorflow_serving/apis/prediction_service_pb2.py @@ -12,11 +12,17 @@ # See the License for the specific language governing permissions and # limitations under the License. # ============================================================================== +# +# To regenerate run +# python -m grpc.tools.protoc --python_out=. --grpc_python_out=. -I. tensorflow_serving/apis/prediction_service.proto + +### @@AUTOGENERATED SECTION STARTS HERE@@ + # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_serving/apis/prediction_service.proto import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -26,118 +32,89 @@ _sym_db = _symbol_database.Default() -from tensorflow_serving.apis import predict_pb2 as third__party_dot_tensorflow__serving_dot_apis_dot_predict__pb2 + +from tensorflow_serving.apis import classification_pb2 as tensorflow__serving_dot_apis_dot_classification__pb2 +from tensorflow_serving.apis import get_model_metadata_pb2 as tensorflow__serving_dot_apis_dot_get__model__metadata__pb2 +from tensorflow_serving.apis import inference_pb2 as tensorflow__serving_dot_apis_dot_inference__pb2 +from tensorflow_serving.apis import predict_pb2 as tensorflow__serving_dot_apis_dot_predict__pb2 +from tensorflow_serving.apis import regression_pb2 as tensorflow__serving_dot_apis_dot_regression__pb2 + DESCRIPTOR = _descriptor.FileDescriptor( - name='tensorflow_serving/apis/prediction_service.proto', - package='tensorflow.serving', - syntax='proto3', - serialized_pb=_b( - '\nhr{display:none;} - # TensorFlow Serving Batching Guide -[TOC] +* [Introduction](#introduction) +* [Simple Batching](#simple-batching) + * [BatchingSession](#batchingsession) + * [BasicBatchScheduler](#basicbatchscheduler) +* [Batch Scheduling Parameters and Tuning](#batch-scheduling-parameters-and-tuning) + * [Performance Tuning](#performance-tuning) +* [Servers with Multiple Models, Model Versions or Subtasks](#servers-with-multiple-models-model-versions-or-subtasks) +* [Mixed CPU/GPU/IO Workloads](#mixed-cpugpuio-workloads) ## Introduction -Batching individual model inference requests together can be important for -performance. In particular, batching is necessary to unlock the high throughput -promised by hardware accelerators such as GPUs. TensorFlow Serving provides a -library for batching requests and scheduling the batches. The library is not -tied to GPUs, per se, and can be used for any situation that benefits from -processing groups of small tasks in tandem (but this document assumes GPUs to -simplify exposition). Its code does not depend on the core TensorFlow Serving -code, so it can be used separately from a TensorFlow Serving deployment if -desired. It offers a specific TensorFlow Session API, as well as general APIs -that are not tied to TensorFlow. +While serving a TensorFlow model, batching individual model inference requests +together can be important for performance. In particular, batching is necessary +to unlock the high throughput promised by hardware accelerators such as GPUs. +This is a library for batching requests and scheduling the batches. The library +is not tied to GPUs, per se, and can be used for any situation that benefits +from processing groups of small tasks in tandem (but this document assumes GPUs +to simplify exposition). It offers a specific TensorFlow Session API, as well as +lower-level APIs that can be used to batch at other granularities. + +The library is currently split across two locations: +(1) core/kernels/batching_util (core API and implementation), and +(2) tensorflow_serving/batching (higher-level and experimental code). The library offers several alternative classes to choose from. The reason for the choices is that there are many reasonable ways to perform batching. No @@ -69,7 +74,7 @@ some number of times with a delay; again, see below. A final configuration parameter is `allowed_batch_sizes`. This parameter is optional. If unset, then batch sizes can vary freely between 1 and the maximum -allowed size, say 1024. Depending on your environment, having a large numbrer +allowed size, say 1024. Depending on your environment, having a large number of possible batch sizes may cause problems. The `allowed_batch_sizes` parameter lets you limit the batch sizes to a fixed set, say 128, 256, 512, 1024. `BatchingSession` adheres to this restriction by padding invalid-size batches @@ -91,30 +96,78 @@ invoked on a separate thread to process the batch. A good illustration of how to use this API is found in the implementation of `BatchingSession` in `batching_session.cc`. -The parameters that govern `BasicBatchScheduler` are: +## Batch Scheduling Parameters and Tuning + +The parameters that govern batch scheduling (e.g. in +`BasicBatchScheduler::Options`) are: + +* `max_batch_size`: The maximum size of any batch. This parameter governs the +throughput/latency tradeoff, and also avoids having batches that are so large +they exceed some resource constraint (e.g. GPU memory to hold a batch's data). + +* `batch_timeout_micros`: The maximum amount of time to wait before executing a +batch (even if it hasn't reached `max_batch_size`). Used to rein in tail +latency. (See `basic_batch_scheduler.h` for the exact latency contract.) + +* `num_batch_threads`: The degree of parallelism, i.e. the maximum number of +batches processed concurrently. + +* `max_enqueued_batches`: The number of batches worth of tasks that can be +enqueued to the scheduler. Used to bound queueing delay, by turning away +requests that would take a long time to get to, rather than building up a large +backlog. + +### Performance Tuning + +The best values to use for the batch scheduling parameters depend on your model, +system and environment, as well as your throughput and latency goals. Choosing +good values is best done via experiments. Here are some guidelines that may be +helpful in selecting values to experiment with. + +#### Overall Guidelines + +First of all, while experimenting you should temporarily set +`max_enqueued_batches` to a really high value. Later, for your production setup, set it as +follows: If you are performing online serving, depending on the policy used to +(re-)route requests to server instances, consider setting `max_enqueued_batches` +equal to `num_batch_threads` to minimize queueing delay at a given server while +keeping it busy. For bulk processing jobs, set `max_enqueued_batches` to a +generous value, but low enough to avoid out-of-memory crashes. + +Second, if for system architecture reasons you need to constrain the set of +possible batch sizes (e.g. just 100, 200 or 400, rather than any value between 1 +and 400): If you are using `BatchingSession` you can set the +`allowed_batch_sizes` parameter. Otherwise, you can arrange for your callback +code to pad the batches with dummy elements. + +#### CPU-only: One Approach + +If your system is CPU-only (no GPU), then consider starting with the following +values: `num_batch_threads` equal to the number of CPU cores; `max_batch_size` +to a really high value; `batch_timeout_micros` to 0. Then experiment with +`batch_timeout_micros` values in the 1-10 millisecond (1000-10000 microsecond) +range, while keeping in mind that 0 may be the optimal value. -* `max_batch_size`: The maximum size of any batch. The largest value supported -by the hardware should be used; it will likely be model-dependent. +#### GPU: One Approach -* `batch_timeout_micros`: A way to bound request latency. The scheduler will -avoid delaying a task too long by processing an underfull batch, if needed. -(See `basic_batch_scheduler.h` for the exact latency contract.) A value slightly -above zero, e.g. 1 millisecond, tends to smooth out batch sizes when the request -rate is low, thus keeping tail latency low while still maintaining high -throughput. The best value to use is of course a function of your workload and -system. +If your model uses a GPU device for part or all of your its inference work, +consider the following approach: -* `num_batch_threads`: The number of threads used to process batches. +1. Set `num_batch_threads` to the number of CPU cores. -* `max_enqueued_batches`: The number of batches worth of tasks the scheduler is -willing to enqueue. For online serving with bounded latency (and the option to -route request to other server instances), you may want to set this equal to -`num_batch_threads`. For bulk processing jobs and throughput-oriented -benchmarks, you may want to set it much higher. +2. Temporarily set `batch_timeout_micros` to a really high value while you tune +`max_batch_size` to achieve the desired balance between throughput and average +latency. Consider values in the hundreds or thousands. -(If you need to limit the possible batch sizes, as in `BatchingSession`'s -`allowed_batch_sizes` parameter, you can have your callback code pad the -batches.) +3. For online serving, tune `batch_timeout_micros` to rein in tail latency. The +idea is that batches normally get filled to `max_batch_size`, but occasionally +when there is a lapse in incoming requests, to avoid introducing a latency spike +it makes sense to process whatever's in the queue even if it represents an +underfull batch. The best value for `batch_timeout_micros` is typically a few +milliseconds, and depends on your context and goals. Zero is a value to +consider; it works well for some workloads. (For bulk processing jobs, choose a +large value, perhaps a few seconds, to ensure good throughput but not wait too +long for the final (and likely underfull) batch.) ## Servers with Multiple Models, Model Versions or Subtasks @@ -123,7 +176,8 @@ multiple versions of a model offered concurrently). In another scenario, a single request gets broken down into sub-requests involving multiple distinct servables (e.g. a recommender system might have a triggering model that decides whether to formulate a recommendation, followed by a model that selects the -actual recommendation). +actual recommendation). A third scenario is bucketizing sequence model requests +to batch together requests of similar length, to minimize padding. Generally speaking, using a separate batch scheduler for each kind of request or sub-task does not work well if they share a common underlying compute diff --git a/tensorflow_serving/batching/basic_batch_scheduler.h b/tensorflow_serving/batching/basic_batch_scheduler.h deleted file mode 100644 index 2029502dc4e..00000000000 --- a/tensorflow_serving/batching/basic_batch_scheduler.h +++ /dev/null @@ -1,263 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ -#define TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ - -#include -#include -#include -#include -#include - -#include "tensorflow_serving/batching/shared_batch_scheduler.h" - -namespace tensorflow { -namespace serving { - -// A BatchScheduler implementation geared toward handling a single request type -// running on a specific set of hardware resources. A typical scenario is one in -// which all requests invoke the same machine-learned model on one GPU. -// -// If there are, say, two GPUs and two models each bound to one of the GPUs, one -// could use two BasicBatchScheduler instances to schedule the two model/GPU -// combinations independently. If multiple models must share a given GPU or -// other hardware resource, consider using SharedBatchScheduler instead. -// -// -// PARAMETERS AND BEHAVIOR: -// -// BasicBatchScheduler runs a fixed pool of threads, which it uses to process -// batches of tasks. It enforces a maximum batch size, and enqueues a bounded -// number of tasks. If the queue is nearly empty, such that a full batch cannot -// be formed, when a thread becomes free, it anyway schedules a batch -// immediately if a task has been in the queue for longer than a given timeout -// parameter. If the timeout parameter is set to 0, then the batch threads will -// always be kept busy (unless there are zero tasks waiting to be processed). -// -// For online serving, it is recommended to set the maximum number of enqueued -// batches worth of tasks equal to the number of batch threads, which allows -// enqueuing of enough tasks s.t. if every thread becomes available it can be -// kept busy, but no more. For bulk processing jobs and throughput-oriented -// benchmarks, you may want to set it much higher. -// -// When Schedule() is called, if the queue is full the call will fail with an -// UNAVAILABLE error (after which the client may retry again later). If the call -// succeeds, the maximum time the task will spend in the queue before being -// placed in a batch and assigned to a thread for processing, is the greater of: -// - the maximum time to process ceil(max_enqueued_batches/num_batch_threads) -// (1 in the recommended configuration) batches of previously-submitted tasks -// - the configured timeout parameter (which can be 0, as mentioned above) -// -// Unlike StreamingBatchScheduler, when BasicBatchScheduler assigns a batch to a -// thread, it closes the batch. The process-batch callback may assume that every -// batch it receives is closed at the outset. -// -// -// RECOMMENDED USE-CASES: -// -// BasicBatchScheduler is suitable for use-cases that feature a single kind of -// request (e.g. a server performing inference with a single machine-learned -// model, possibly evolving over time), with loose versioning semantics. -// Concretely, the following conditions should hold: -// -// A. All requests batched onto a given resource (e.g. a hardware accelerator, -// or a pool accelerators) are of the same type. For example, they all -// invoke the same machine-learned model. -// -// These variations are permitted: -// - The model may reside in a single servable, or it may be spread across -// multiple servables that are used in unison (e.g. a vocabulary lookup -// table servable and a tensorflow session servable). -// - The model's servable(s) may be static, or they may evolve over time -// (successive servable versions). -// - Zero or more of the servables are used in the request thread; the rest -// are used in the batch thread. In our running example, the vocabulary -// lookups and tensorflow runs may both be performed in the batch thread, -// or alternatively the vocabulary lookup may occur in the request thread -// with only the tensorflow run performed in the batch thread. -// -// In contrast, BasicBatchScheduler is not a good fit if the server -// hosts multiple distinct models running on a pool accelerators, with each -// request specifying which model it wants to use. BasicBatchScheduler -// has no facility to time-multiplex the batch threads across multiple -// models in a principled way. More basically, it cannot ensure that a given -// batch doesn't contain a mixture of requests for different models. -// -// B. Requests do not specify a particular version of the servable(s) that must -// be used. Instead, each request is content to use the "latest" version. -// -// BasicBatchScheduler does not constrain which requests get grouped -// together into a batch, so using this scheduler there is no way to achieve -// cohesion of versioned requests to version-specific batches. -// -// C. No servable version coordination needs to be performed between the -// request threads and the batch threads. Often, servables are only used in -// the batch threads, in which case this condition trivially holds. If -// servables are used in both threads, then the use-case must tolerate -// version skew across the servables used in the two kinds of threads. -// -// -// EXAMPLE USE-CASE FLOW: -// -// For such use-cases, request processing via BasicBatchScheduler generally -// follows this flow (given for illustration; variations are possible): -// 1. Optionally perform some pre-processing on each request in the request -// threads. -// 2. Route the requests to the batch scheduler, as batching::Task objects. -// (Since all requests are of the same type and are not versioned, the -// scheduler is free to group them into batches arbitrarily.) -// 3. Merge the requests into a single batched representation B. -// 4. Obtain handles to the servable(s) needed to process B. The simplest -// approach is to obtain the latest version of each servable. Alternatively, -// if cross-servable consistency is required (e.g. the vocabulary lookup -// table's version number must match that of the tensorflow session), -// identify an appropriate version number and obtain the servable handles -// accordingly. -// 5. Process B using the obtained servable handles, and split the result into -// individual per-request units. -// 6. Perform any post-processing in the batch thread and/or request thread. -// -template -class BasicBatchScheduler : public BatchScheduler { - public: - // TODO(b/25089730): Tune defaults based on best practices as they develop. - // (Keep them mirrored to the ones in SharedBatchScheduler::QueueOptions and - // SharedBatchScheduler::Options.) - struct Options { - // The maximum size of each batch. - // - // The scheduler may form batches of any size between 1 and this number - // (inclusive). If there is a need to quantize the batch sizes, i.e. only - // submit batches whose size is in a small set of allowed sizes, that can be - // done by adding padding in the process-batch callback. - int max_batch_size = 1000; - - // If a task has been enqueued for this amount of time (in microseconds), - // and a thread is available, the scheduler will immediately form a batch - // from enqueued tasks and assign the batch to the thread for processing, - // even if the batch's size is below 'max_batch_size'. - // - // This parameter offers a way to bound queue latency, so that a task isn't - // stuck in the queue indefinitely waiting for enough tasks to arrive to - // make a full batch. (The latency bound is given in the class documentation - // above.) - // - // The goal is to smooth out batch sizes under low request rates, and thus - // avoid latency spikes. The default value of 1 millisecond was determined - // via benchmarking. You may need to adjust it to suit your workload and - // environment. - int64 batch_timeout_micros = 1 * 1000 /* 1 millisecond */; - - // The name to use for the pool of batch threads. - string thread_pool_name = {"batch_threads"}; - - // The number of threads to use to process batches. - // Must be >= 1, and should be tuned carefully. - int num_batch_threads = 1; - - // The maximum allowable number of enqueued (accepted by Schedule() but - // not yet being processed on a batch thread) tasks in terms of batches. - // If this limit is reached, Schedule() will return an UNAVAILABLE error. - // See the class documentation above for guidelines on how to tune this - // parameter. - int max_enqueued_batches = 1; - - // The following options are typically only overridden by test code. - - // The environment to use. - Env* env = Env::Default(); - }; - static Status Create(const Options& options, - std::function>)> - process_batch_callback, - std::unique_ptr* scheduler); - - ~BasicBatchScheduler() override = default; - - Status Schedule(std::unique_ptr* task) override; - size_t NumEnqueuedTasks() const override; - size_t SchedulingCapacity() const override; - - private: - explicit BasicBatchScheduler( - std::unique_ptr> shared_scheduler_queue); - - // This class is merely a thin wrapper around a SharedBatchScheduler with a - // single queue. - std::unique_ptr> shared_scheduler_queue_; - - TF_DISALLOW_COPY_AND_ASSIGN(BasicBatchScheduler); -}; - -////////// -// Implementation details follow. API users need not read. - -template -Status BasicBatchScheduler::Create( - const Options& options, - std::function>)> - process_batch_callback, - std::unique_ptr* scheduler) { - typename SharedBatchScheduler::Options shared_scheduler_options; - shared_scheduler_options.thread_pool_name = options.thread_pool_name; - shared_scheduler_options.num_batch_threads = options.num_batch_threads; - shared_scheduler_options.env = options.env; - std::shared_ptr> shared_scheduler; - TF_RETURN_IF_ERROR(SharedBatchScheduler::Create( - shared_scheduler_options, &shared_scheduler)); - - typename SharedBatchScheduler::QueueOptions - shared_scheduler_queue_options; - shared_scheduler_queue_options.max_batch_size = options.max_batch_size; - shared_scheduler_queue_options.batch_timeout_micros = - options.batch_timeout_micros; - shared_scheduler_queue_options.max_enqueued_batches = - options.max_enqueued_batches; - std::unique_ptr> shared_scheduler_queue; - TF_RETURN_IF_ERROR(shared_scheduler->AddQueue(shared_scheduler_queue_options, - process_batch_callback, - &shared_scheduler_queue)); - - scheduler->reset( - new BasicBatchScheduler(std::move(shared_scheduler_queue))); - return Status::OK(); -} - -template -Status BasicBatchScheduler::Schedule( - std::unique_ptr* task) { - return shared_scheduler_queue_->Schedule(task); -} - -template -size_t BasicBatchScheduler::NumEnqueuedTasks() const { - return shared_scheduler_queue_->NumEnqueuedTasks(); -} - -template -size_t BasicBatchScheduler::SchedulingCapacity() const { - return shared_scheduler_queue_->SchedulingCapacity(); -} - -template -BasicBatchScheduler::BasicBatchScheduler( - std::unique_ptr> shared_scheduler_queue) - : shared_scheduler_queue_(std::move(shared_scheduler_queue)) {} - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc b/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc deleted file mode 100644 index 9ee939fd670..00000000000 --- a/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc +++ /dev/null @@ -1,442 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Benchmarks for performance (throughput and latency) of BasicBatchScheduler -// under various rates of task injection. -// -// Run with: -// bazel run -c opt --dynamic_mode=off \ -// tensorflow_serving/batching:basic_batch_scheduler_benchmark -- -// --benchmarks=. --benchmark_use_picoseconds -// For a longer run time and more consistent results for the throughput -// benchmark, consider a min time e.g.: --benchmark_min_time=60.0 - -#include "tensorflow/core/lib/histogram/histogram.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/test_benchmark.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" - -namespace tensorflow { -namespace serving { -namespace { - -using ::tensorflow::histogram::Histogram; - -// An abstract class for injecting load into a system at a specific rate. -class LoadInjector { - public: - virtual ~LoadInjector() = default; - - // Run 'injector' 'num_injection' times, with average inter-injection spacing - // as 'average_injection_interval_micros' (in microseconds). - virtual void InjectLoad(std::function injector, int num_injections, - int64 average_injection_interval_micros) const = 0; -}; - -// A load injector that uses uniform inter-injection spacing, i.e. each pair of -// injections is separated in time by 'average_injection_interval_micros' (as -// best as possible). -class UniformLoadInjector : public LoadInjector { - public: - UniformLoadInjector() = default; - ~UniformLoadInjector() override = default; - - void InjectLoad(std::function injector, int num_injections, - int64 average_injection_interval_micros) const override; - - private: - TF_DISALLOW_COPY_AND_ASSIGN(UniformLoadInjector); -}; - -void UniformLoadInjector::InjectLoad( - std::function injector, const int num_injections, - const int64 average_injection_interval_micros) const { - int num_injections_performed = 0; - const int64 start_time_micros = Env::Default()->NowMicros(); - while (num_injections_performed < num_injections) { - // Inject. - injector(); - ++num_injections_performed; - - // Wait until it's time for the next injection. - const int64 next_injection_time_micros = - start_time_micros + - (num_injections_performed * average_injection_interval_micros); - int64 now_micros = Env::Default()->NowMicros(); - while (now_micros < next_injection_time_micros) { - const int64 kSleepThresholdMicros = 1000; - if (next_injection_time_micros - now_micros >= kSleepThresholdMicros) { - Env::Default()->SleepForMicroseconds(1 /* minimum time */); - } - now_micros = Env::Default()->NowMicros(); - } - } -} - -class BenchmarkBatchTask : public BatchTask { - public: - BenchmarkBatchTask(); - - BenchmarkBatchTask(const BenchmarkBatchTask&) = delete; - BenchmarkBatchTask& operator=(const BenchmarkBatchTask&) = delete; - - ~BenchmarkBatchTask() override = default; - - size_t size() const override { return 1; } - - uint64 start_time_micros() const { return start_time_micros_; } - - private: - // The time at which the task was created, in microseconds. - const uint64 start_time_micros_; -}; - -BenchmarkBatchTask::BenchmarkBatchTask() - : start_time_micros_(Env::Default()->NowMicros()) {} - -// The state and logic associated with a throughput benchmark, which injects a -// large number of tasks into a batch scheduler and measures the total time to -// process all the tasks. -class ThroughputBenchmark { - public: - explicit ThroughputBenchmark( - const BasicBatchScheduler::Options& - scheduler_options); - - ThroughputBenchmark(const ThroughputBenchmark&) = delete; - ThroughputBenchmark& operator=(const ThroughputBenchmark&) = delete; - - // Perform the benchmark run, based on the parameters supplied to the ctor. - void RunBenchmark(int iters); - - private: - // Resets all mutable state, including the scheduler. - void ResetState(); - - // Processes a batch of tasks. (Invoked by 'scheduler_' on one of its batch - // threads.) - void ProcessBatch(std::unique_ptr> batch); - - // Parameters for the BasicBatchScheduler being benchmarked. - const BasicBatchScheduler::Options scheduler_options_; - - // The BasicBatchScheduler being benchmarked. - std::unique_ptr> scheduler_; -}; - -ThroughputBenchmark::ThroughputBenchmark( - const BasicBatchScheduler::Options& scheduler_options) - : scheduler_options_(scheduler_options) {} - -void ThroughputBenchmark::RunBenchmark(int iters) { - CHECK_GE(iters, 1); - - testing::StopTiming(); - ResetState(); - - // Have each iteration issue a reasonably large number of tasks, to ensure our - // measurements reflect steady-state behavior. - const int kNumTasksPerIteration = 100 * 1000; - - testing::ItemsProcessed(iters * kNumTasksPerIteration); - testing::UseRealTime(); - testing::StartTiming(); - - // Schedule 'num_iterations_*kNumTasksPerIteration' tasks. - for (int i = 0; i < iters; ++i) { - for (int j = 0; j < kNumTasksPerIteration; ++j) { - auto task = std::unique_ptr(new BenchmarkBatchTask); - TF_CHECK_OK(scheduler_->Schedule(&task)); - } - } - - // Wait for the scheduler to process all tasks. - scheduler_.reset(); - testing::StopTiming(); -} - -void ThroughputBenchmark::ResetState() { - auto process_batch_callback = - [this](std::unique_ptr> batch) { - ProcessBatch(std::move(batch)); - }; - TF_CHECK_OK(BasicBatchScheduler::Create( - scheduler_options_, process_batch_callback, &scheduler_)); -} - -void ThroughputBenchmark::ProcessBatch( - std::unique_ptr> batch) { - // No-op. -} - -// The state and logic associated with a latency benchmark, which injects tasks -// into a batch scheduler at a controlled rate and measures the distribution of -// task completion latencies. -// -// Reports the measurements to std::cout (not LOG(INFO)), like the throughput -// measurements. -class LatencyBenchmark { - public: - LatencyBenchmark( - const BasicBatchScheduler::Options& scheduler_options, - int64 task_injection_interval_micros, int batch_cpu_cost); - - LatencyBenchmark(const LatencyBenchmark&) = delete; - LatencyBenchmark& operator=(const LatencyBenchmark&) = delete; - - // Perform the benchmark run, based on the parameters supplied to the ctor. - void RunBenchmark(); - - private: - // Resets all mutable state, including the scheduler and latency measurements. - void ResetState() LOCKS_EXCLUDED(mu_); - - // Processes a batch of tasks. (Invoked by 'scheduler_' on one of its batch - // threads.) - void ProcessBatch(std::unique_ptr> batch); - - // Performs one batch's dummy CPU work. - void PerformBatchCpuWork() const; - - // Parameters for the BasicBatchScheduler being benchmarked. - const BasicBatchScheduler::Options scheduler_options_; - - // The time interval between successively injected tasks, in microseconds. - // A large interval corresponds to a slow rate of task injection, and vice- - // versa. - const int64 task_injection_interval_micros_; - - // The amount of work to do while processing one batch of tasks. (The cost is - // independent of the number of tasks in the batch.) - const int batch_cpu_cost_; - - // The BasicBatchScheduler being benchmarked. - std::unique_ptr> scheduler_; - - mutable mutex mu_; - - // A histogram of the task latencies, i.e. queue time plus processing time, in - // milliseconds. - Histogram task_latency_millis_histogram_ GUARDED_BY(mu_); - - // A histogram of the batch sizes. - Histogram batch_size_histogram_ GUARDED_BY(mu_); -}; - -LatencyBenchmark::LatencyBenchmark( - const BasicBatchScheduler::Options& scheduler_options, - int64 task_injection_interval_micros, int batch_cpu_cost) - : scheduler_options_(scheduler_options), - task_injection_interval_micros_(task_injection_interval_micros), - batch_cpu_cost_(batch_cpu_cost) {} - -void LatencyBenchmark::RunBenchmark() { - ResetState(); - - // Arrange to inject tasks at the specified rate, for a fixed total time - // duration. - const int kTimeDurationMicros = 100 * 1000 * 1000 /* 100 seconds */; - const int kNumTasks = kTimeDurationMicros / task_injection_interval_micros_; - CHECK_GE(kNumTasks, 100000) - << "Not enough tasks to report meaningful 99.9% latency"; - - const int64 start_time_micros = Env::Default()->NowMicros(); - - // Inject the tasks. - UniformLoadInjector injector; - injector.InjectLoad( - [this] { - auto task = std::unique_ptr(new BenchmarkBatchTask); - TF_CHECK_OK(scheduler_->Schedule(&task)); - }, - kNumTasks, task_injection_interval_micros_); - - // Be sure we were able to more-or-less match our target injection rate. - const int64 target_injection_time_micros = - kNumTasks * task_injection_interval_micros_; - const int64 actual_injection_time_micros = - Env::Default()->NowMicros() - start_time_micros; - if (actual_injection_time_micros > 1.1 * target_injection_time_micros) { - LOG(FATAL) << "Unable to inject tasks at the requested rate"; - } - - // Wait for the scheduler to process all injected tasks. - scheduler_.reset(); - - // Be sure the scheduler was able to process the tasks at close to the - // injection rate. If not, our latency measurements will be dominated by queue - // waiting time - const int64 actual_processing_time_micros = - Env::Default()->NowMicros() - start_time_micros; - if (actual_processing_time_micros > 1.01 * actual_injection_time_micros) { - LOG(FATAL) << "Unable to keep up with task injection rate"; - } - - // Report benchmark measurements. - { - mutex_lock l(mu_); - std::cout << "\t" - << "99.9% latency: " - << task_latency_millis_histogram_.Percentile(99.9) << "ms" - << "\t" - << "99% batch size: " << batch_size_histogram_.Percentile(99) - << std::endl; - } -} - -void LatencyBenchmark::ResetState() { - auto process_batch_callback = - [this](std::unique_ptr> batch) { - ProcessBatch(std::move(batch)); - }; - TF_CHECK_OK(BasicBatchScheduler::Create( - scheduler_options_, process_batch_callback, &scheduler_)); - - { - mutex_lock l(mu_); - task_latency_millis_histogram_.Clear(); - batch_size_histogram_.Clear(); - } -} - -void LatencyBenchmark::ProcessBatch( - std::unique_ptr> batch) { - PerformBatchCpuWork(); - const uint64 batch_completion_time = Env::Default()->NowMicros(); - - { - mutex_lock l(mu_); - batch_size_histogram_.Add(batch->num_tasks()); - } - - for (int i = 0; i < batch->num_tasks(); ++i) { - const BenchmarkBatchTask& task = batch->task(i); - - const uint64 task_latency_micros = - batch_completion_time - task.start_time_micros(); - - { - mutex_lock l(mu_); - task_latency_millis_histogram_.Add(task_latency_micros / 1000.0); - } - } -} - -void LatencyBenchmark::PerformBatchCpuWork() const { - int dummy = 1; - for (int i = 0; i < batch_cpu_cost_; ++i) { - dummy += dummy * 2; - } - CHECK_NE(dummy, 0); -} - -static void RunThroughputBenchmark(int iters, int64 batch_timeout_micros, - int num_batch_threads) { - BasicBatchScheduler::Options scheduler_options; - const int kMaxBatchSize = 100; - scheduler_options.max_batch_size = kMaxBatchSize; - scheduler_options.batch_timeout_micros = batch_timeout_micros; - scheduler_options.num_batch_threads = num_batch_threads; - scheduler_options.max_enqueued_batches = INT_MAX; // Unbounded queue. - ThroughputBenchmark benchmark(scheduler_options); - benchmark.RunBenchmark(iters); -} - -static void ThroughputBM_ZeroTimeout(int iters, int num_batch_threads) { - RunThroughputBenchmark(iters, 0 /* 0 ms timeout */, num_batch_threads); -} -BENCHMARK(ThroughputBM_ZeroTimeout) - ->Arg(1) - ->Arg(2) - ->Arg(4) - ->Arg(8) - ->Arg(16) - ->Arg(32) - ->Arg(64); - -static void ThroughputBM_SmallTimeout(int iters, int num_batch_threads) { - RunThroughputBenchmark(iters, 1 * 1000 /* 1 ms timeout */, num_batch_threads); -} -BENCHMARK(ThroughputBM_SmallTimeout) - ->Arg(1) - ->Arg(2) - ->Arg(4) - ->Arg(8) - ->Arg(16) - ->Arg(32) - ->Arg(64); - -static void ThroughputBM_LargeTimeout(int iters, int num_batch_threads) { - RunThroughputBenchmark(iters, 50 * 1000 /* 50 ms timeout */, - num_batch_threads); -} -BENCHMARK(ThroughputBM_LargeTimeout) - ->Arg(1) - ->Arg(2) - ->Arg(4) - ->Arg(8) - ->Arg(16) - ->Arg(32) - ->Arg(64); - -static void RunLatencyBenchmark(int64 task_injection_interval_micros, - int64 batch_timeout_micros) { - BasicBatchScheduler::Options scheduler_options; - const int kMaxBatchSize = 100; - scheduler_options.max_batch_size = kMaxBatchSize; - scheduler_options.batch_timeout_micros = batch_timeout_micros; - const int kNumBatchThreads = 2; - scheduler_options.num_batch_threads = kNumBatchThreads; - scheduler_options.max_enqueued_batches = INT_MAX; // Unbounded queue. - const int kBatchCpuCost = 10 * 1000 * 1000; - LatencyBenchmark benchmark(scheduler_options, task_injection_interval_micros, - kBatchCpuCost); - benchmark.RunBenchmark(); -} - -static void RunLatencyBenchmarks() { - for (const int64 batch_timeout_micros : {0, 1 * 1000, 2 * 1000, 5 * 1000}) { - for (const int64 task_injection_interval_micros : {1000, 50, 20}) { - std::cout << "Latency benchmark w/ batch timeout " - << batch_timeout_micros / 1000.0 << "ms" - << "; " - << "task injection rate " - << 1000000.0 / task_injection_interval_micros << "/sec" - << "\t..."; - RunLatencyBenchmark(task_injection_interval_micros, batch_timeout_micros); - } - std::cout << std::endl; - } -} - -} // namespace -} // namespace serving -} // namespace tensorflow - -int main(int argc, char** argv) { - tensorflow::port::InitMain(argv[0], &argc, &argv); - std::setprecision(5); - - // Run latency benchmarks (outside of tensorflow benchmark framework). - tensorflow::serving::RunLatencyBenchmarks(); - - // Run throughput benchmarks (via tensorflow benchmark framework). - tensorflow::testing::RunBenchmarks(); - - return 0; -} diff --git a/tensorflow_serving/batching/basic_batch_scheduler_test.cc b/tensorflow_serving/batching/basic_batch_scheduler_test.cc deleted file mode 100644 index 5c5ef65ca94..00000000000 --- a/tensorflow_serving/batching/basic_batch_scheduler_test.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/batching/basic_batch_scheduler.h" - -#include - -#include -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/batching/batch_scheduler.h" - -namespace tensorflow { -namespace serving { -namespace { - -class FakeTask : public BatchTask { - public: - explicit FakeTask(size_t size) : size_(size) {} - - ~FakeTask() override = default; - - size_t size() const override { return size_; } - - private: - const size_t size_; - - TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); -}; - -// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' -// on that task. Returns the resulting status. -Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { - std::unique_ptr task(new FakeTask(task_size)); - Status status = scheduler->Schedule(&task); - // Schedule() should have consumed 'task' iff it returned Status::OK. - CHECK_EQ(status.ok(), task == nullptr); - return status; -} - -// Since BasicBatchScheduler is implemented as a thin wrapper around -// SharedBatchScheduler, we only do some basic testing. More comprehensive -// testing is done in shared_batch_scheduler_test.cc. - -TEST(BasicBatchSchedulerTest, Basic) { - bool callback_called = false; - auto callback = [&callback_called](std::unique_ptr> batch) { - callback_called = true; - ASSERT_TRUE(batch->IsClosed()); - ASSERT_EQ(2, batch->num_tasks()); - EXPECT_EQ(3, batch->task(0).size()); - EXPECT_EQ(5, batch->task(1).size()); - }; - { - BasicBatchScheduler::Options options; - options.max_batch_size = 10; - options.batch_timeout_micros = 100 * 1000; // 100 milliseconds - options.num_batch_threads = 1; - options.max_enqueued_batches = 3; - std::unique_ptr> scheduler; - TF_ASSERT_OK( - BasicBatchScheduler::Create(options, callback, &scheduler)); - EXPECT_EQ(0, scheduler->NumEnqueuedTasks()); - EXPECT_EQ(3 * 10, scheduler->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(3, scheduler.get())); - EXPECT_EQ(1, scheduler->NumEnqueuedTasks()); - EXPECT_EQ((3 * 10) - 3, scheduler->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(5, scheduler.get())); - EXPECT_EQ(2, scheduler->NumEnqueuedTasks()); - EXPECT_EQ((3 * 10) - (3 + 5), scheduler->SchedulingCapacity()); - } - EXPECT_TRUE(callback_called); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/batching/batch_scheduler.h b/tensorflow_serving/batching/batch_scheduler.h deleted file mode 100644 index 0780c5bc4d6..00000000000 --- a/tensorflow_serving/batching/batch_scheduler.h +++ /dev/null @@ -1,259 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Abstractions for processing small tasks in a batched fashion, to reduce -// processing times and costs that can be amortized across multiple tasks. -// -// The core class is BatchScheduler, which groups tasks into batches. -// -// BatchScheduler encapsulates logic for aggregating multiple tasks into a -// batch, and kicking off processing of a batch on a thread pool it manages. -// -// This file defines an abstract BatchScheduler class. - -#ifndef TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ -#define TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ - -#include -#include -#include -#include -#include -#include - -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace serving { - -// The abstract superclass for a unit of work to be done as part of a batch. -// -// An implementing subclass typically contains (or points to): -// (a) input data; -// (b) a thread-safe completion signal (e.g. a Notification); -// (c) a place to store the outcome (success, or some error), upon completion; -// (d) a place to store the output data, upon success. -// -// Items (b), (c) and (d) are typically non-owned pointers to data homed -// elsewhere, because a task's ownership gets transferred to a BatchScheduler -// (see below) and it may be deleted as soon as it is done executing. -class BatchTask { - public: - virtual ~BatchTask() = default; - - // Returns the size of the task, in terms of how much it contributes to the - // size of a batch. (A batch's size is the sum of its task sizes.) - virtual size_t size() const = 0; -}; - -// A thread-safe collection of BatchTasks, to be executed together in some -// fashion. -// -// At a given time, a batch is either "open" or "closed": an open batch can -// accept new tasks; a closed one cannot. A batch is monotonic: initially it is -// open and tasks can be added to it; then it is closed and its set of tasks -// remains fixed for the remainder of its life. A closed batch cannot be re- -// opened. Tasks can never be removed from a batch. -// -// Type parameter TaskType must be a subclass of BatchTask. -template -class Batch { - public: - Batch() = default; - ~Batch(); // Blocks until the batch is closed. - - // Appends 'task' to the batch. After calling AddTask(), the newly-added task - // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1). - // Dies if the batch is closed. - void AddTask(std::unique_ptr task); - - // Returns the number of tasks in the batch. - int num_tasks() const; - - // Returns true iff the batch contains 0 tasks. - bool empty() const; - - // Returns a reference to the ith task (in terms of insertion order). - const TaskType& task(int i) const; - - // Returns a pointer to the ith task (in terms of insertion order). - TaskType* mutable_task(int i); - - // Returns the sum of the task sizes. - size_t size() const; - - // Returns true iff the batch is currently closed. - bool IsClosed() const; - - // Blocks until the batch is closed. - void WaitUntilClosed() const; - - // Marks the batch as closed. Dies if called more than once. - void Close(); - - private: - mutable mutex mu_; - - // The tasks in the batch. - std::vector> tasks_ GUARDED_BY(mu_); - - // The sum of the sizes of the tasks in 'tasks_'. - size_t size_ GUARDED_BY(mu_) = 0; - - // Whether the batch has been closed. - Notification closed_; - - TF_DISALLOW_COPY_AND_ASSIGN(Batch); -}; - -// An abstract batch scheduler class. Collects individual tasks into batches, -// and processes each batch on a pool of "batch threads" that it manages. The -// actual logic for processing a batch is accomplished via a callback. -// -// Type parameter TaskType must be a subclass of BatchTask. -template -class BatchScheduler { - public: - virtual ~BatchScheduler() = default; - - // Submits a task to be processed as part of a batch. - // - // Ownership of '*task' is transferred to the callee iff the method returns - // Status::OK. In that case, '*task' is left as nullptr. Otherwise, '*task' is - // left as-is. - // - // If no batch processing capacity is available to process this task at the - // present time, and any task queue maintained by the implementing subclass is - // full, this method returns an UNAVAILABLE error code. The client may retry - // later. - // - // Other problems, such as the task size being larger than the maximum batch - // size, yield other, permanent error types. - // - // In all cases, this method returns "quickly" without blocking for any - // substantial amount of time. If the method returns Status::OK, the task is - // processed asynchronously, and any errors that occur during the processing - // of the batch that includes the task can be reported to 'task'. - virtual Status Schedule(std::unique_ptr* task) = 0; - - // Returns the number of tasks that have been scheduled (i.e. accepted by - // Schedule()), but have yet to be handed to a thread for execution as part of - // a batch. Note that this returns the number of tasks, not the aggregate task - // size (so if there is one task of size 3 and one task of size 5, this method - // returns 2 rather than 8). - virtual size_t NumEnqueuedTasks() const = 0; - - // Returns a guaranteed number of size 1 tasks that can be Schedule()d without - // getting an UNAVAILABLE error. In a typical implementation, returns the - // available space on a queue. - // - // There are two important caveats: - // 1. The guarantee does not extend to varying-size tasks due to possible - // internal fragmentation of batches. - // 2. The guarantee only holds in a single-thread environment or critical - // section, i.e. if an intervening thread cannot call Schedule(). - // - // This method is useful for monitoring, or for guaranteeing a future slot in - // the schedule (but being mindful about the caveats listed above). - virtual size_t SchedulingCapacity() const = 0; -}; - -////////// -// Implementation details follow. API users need not read. - -template -Batch::~Batch() { - WaitUntilClosed(); -} - -template -void Batch::AddTask(std::unique_ptr task) { - DCHECK(!IsClosed()); - { - mutex_lock l(mu_); - size_ += task->size(); - tasks_.push_back(std::move(task)); - } -} - -template -int Batch::num_tasks() const { - { - mutex_lock l(mu_); - return tasks_.size(); - } -} - -template -bool Batch::empty() const { - { - mutex_lock l(mu_); - return tasks_.empty(); - } -} - -template -const TaskType& Batch::task(int i) const { - DCHECK_GE(i, 0); - { - mutex_lock l(mu_); - DCHECK_LT(i, tasks_.size()); - return *tasks_[i].get(); - } -} - -template -TaskType* Batch::mutable_task(int i) { - DCHECK_GE(i, 0); - { - mutex_lock l(mu_); - DCHECK_LT(i, tasks_.size()); - return tasks_[i].get(); - } -} - -template -size_t Batch::size() const { - { - mutex_lock l(mu_); - return size_; - } -} - -template -bool Batch::IsClosed() const { - return const_cast(&closed_)->HasBeenNotified(); -} - -template -void Batch::WaitUntilClosed() const { - const_cast(&closed_)->WaitForNotification(); -} - -template -void Batch::Close() { - closed_.Notify(); -} - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/batch_scheduler_retrier.h b/tensorflow_serving/batching/batch_scheduler_retrier.h index 4c2a197edad..bca1594cbd4 100644 --- a/tensorflow_serving/batching/batch_scheduler_retrier.h +++ b/tensorflow_serving/batching/batch_scheduler_retrier.h @@ -21,10 +21,10 @@ limitations under the License. #include #include +#include "tensorflow/core/kernels/batching_util/batch_scheduler.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/batching/batch_scheduler.h" namespace tensorflow { namespace serving { @@ -39,10 +39,10 @@ class BatchSchedulerRetrier : public BatchScheduler { struct Options { // The maximum amount of time to spend retrying 'wrapped_->Schedule()' // calls, in microseconds. - int64 max_time_micros = 10 * 1000 /* 10 milliseconds */; + int64_t max_time_micros = 10 * 1000 /* 10 milliseconds */; // The amount of time to pause between retry attempts, in microseconds. - int64 retry_delay_micros = 100; + int64_t retry_delay_micros = 100; // The environment to use for time and sleeping. Env* env = Env::Default(); @@ -57,6 +57,8 @@ class BatchSchedulerRetrier : public BatchScheduler { size_t NumEnqueuedTasks() const override; size_t SchedulingCapacity() const override; + size_t max_task_size() const override { return wrapped_->max_task_size(); } + private: BatchSchedulerRetrier(const Options& options, std::unique_ptr> wrapped); @@ -84,7 +86,7 @@ Status BatchSchedulerRetrier::Create( options.retry_delay_micros); } result->reset(new BatchSchedulerRetrier(options, std::move(wrapped))); - return Status::OK(); + return Status(); } template @@ -92,7 +94,7 @@ Status BatchSchedulerRetrier::Schedule( std::unique_ptr* task) { Status status; - const uint64 start_time_micros = options_.env->NowMicros(); + const uint64_t start_time_micros = options_.env->NowMicros(); for (;;) { status = wrapped_->Schedule(task); if (status.code() != error::UNAVAILABLE) { diff --git a/tensorflow_serving/batching/batch_scheduler_retrier_test.cc b/tensorflow_serving/batching/batch_scheduler_retrier_test.cc index 84afd80b377..79ca63ee2ef 100644 --- a/tensorflow_serving/batching/batch_scheduler_retrier_test.cc +++ b/tensorflow_serving/batching/batch_scheduler_retrier_test.cc @@ -16,15 +16,17 @@ limitations under the License. #include "tensorflow_serving/batching/batch_scheduler_retrier.h" #include +#include +#include #include -#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/kernels/batching_util/fake_clock_env.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" namespace tensorflow { namespace serving { @@ -47,7 +49,7 @@ class BrokenScheduler : public BatchScheduler { BrokenScheduler() = default; ~BrokenScheduler() override = default; - Status Schedule(std::unique_ptr* task) override { + absl::Status Schedule(std::unique_ptr* task) override { ++num_submit_calls_; return errors::Unknown("BrokenScheduler faithfully failing"); } @@ -58,6 +60,8 @@ class BrokenScheduler : public BatchScheduler { int num_submit_calls() const { return num_submit_calls_; } + size_t max_task_size() const override { return 1000; } + private: int num_submit_calls_ = 0; @@ -72,11 +76,11 @@ class StubbornScheduler : public BatchScheduler { : num_attempts_to_succeed_(num_attempts_to_succeed) {} ~StubbornScheduler() override = default; - Status Schedule(std::unique_ptr* task) override { + absl::Status Schedule(std::unique_ptr* task) override { ++num_attempts_; if (num_attempts_ >= num_attempts_to_succeed_) { std::unique_ptr consumed_task = std::move(*task); - return Status::OK(); + return absl::OkStatus(); } else { return errors::Unavailable( "StubbornScheduler faithfully being stubborn; this is attempt ", @@ -90,6 +94,8 @@ class StubbornScheduler : public BatchScheduler { return std::numeric_limits::max(); } + size_t max_task_size() const override { return 1000; } + int num_attempts() const { return num_attempts_; } private: @@ -105,6 +111,7 @@ TEST(BatchSchedulerRetrierTest, ConstMethodsForwardToWrappedScheduler) { std::unique_ptr> retrier; TF_CHECK_OK(BatchSchedulerRetrier::Create( options, std::move(broken_scheduler), &retrier)); + EXPECT_EQ(1000, retrier->max_task_size()); EXPECT_EQ(7, retrier->NumEnqueuedTasks()); EXPECT_EQ(42, retrier->SchedulingCapacity()); } @@ -117,7 +124,7 @@ TEST(BatchSchedulerRetrierTest, PermanentFailure) { TF_CHECK_OK(BatchSchedulerRetrier::Create( options, std::move(broken_scheduler), &retrier)); auto task = std::unique_ptr(new FakeTask); - Status status = retrier->Schedule(&task); + absl::Status status = retrier->Schedule(&task); ASSERT_FALSE(status.ok()); EXPECT_EQ(error::UNKNOWN, status.code()); EXPECT_FALSE(task == nullptr); @@ -147,7 +154,7 @@ TEST(BatchSchedulerRetrierTest, MaxTime) { {}, "RunRetrier", [&retrier, &expect_success, &done]() { auto task = std::unique_ptr(new FakeTask); - Status status = retrier->Schedule(&task); + absl::Status status = retrier->Schedule(&task); EXPECT_EQ(expect_success, status.ok()); if (!status.ok()) { EXPECT_EQ(error::UNAVAILABLE, status.code()); @@ -189,7 +196,7 @@ TEST(BatchSchedulerRetrierTest, RetryDelay) { {}, "RunRetrier", [&retrier, &done]() { auto task = std::unique_ptr(new FakeTask); - Status status = retrier->Schedule(&task); + absl::Status status = retrier->Schedule(&task); TF_EXPECT_OK(status); done.Notify(); })); diff --git a/tensorflow_serving/batching/batch_scheduler_test.cc b/tensorflow_serving/batching/batch_scheduler_test.cc deleted file mode 100644 index b5188a8c6b5..00000000000 --- a/tensorflow_serving/batching/batch_scheduler_test.cc +++ /dev/null @@ -1,112 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/batching/batch_scheduler.h" - -#include -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/macros.h" - -namespace tensorflow { -namespace serving { -namespace { - -class FakeTask : public BatchTask { - public: - explicit FakeTask(size_t size) : size_(size) {} - - ~FakeTask() override = default; - - size_t size() const override { return size_; } - - private: - const size_t size_; - - TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); -}; - -TEST(BatchTest, Basic) { - Batch batch; - - EXPECT_EQ(0, batch.num_tasks()); - EXPECT_TRUE(batch.empty()); - EXPECT_EQ(0, batch.size()); - EXPECT_FALSE(batch.IsClosed()); - - auto task0 = new FakeTask(3); - batch.AddTask(std::unique_ptr(task0)); - - EXPECT_EQ(1, batch.num_tasks()); - EXPECT_FALSE(batch.empty()); - EXPECT_EQ(task0->size(), batch.size()); - EXPECT_EQ(task0->size(), batch.task(0).size()); - EXPECT_FALSE(batch.IsClosed()); - - auto task1 = new FakeTask(7); - batch.AddTask(std::unique_ptr(task1)); - - EXPECT_EQ(2, batch.num_tasks()); - EXPECT_FALSE(batch.empty()); - EXPECT_EQ(task0->size() + task1->size(), batch.size()); - EXPECT_EQ(task1->size(), batch.task(1).size()); - EXPECT_EQ(task1->size(), batch.mutable_task(1)->size()); - EXPECT_FALSE(batch.IsClosed()); - - batch.Close(); - EXPECT_TRUE(batch.IsClosed()); - - EXPECT_EQ(2, batch.num_tasks()); - EXPECT_FALSE(batch.empty()); - EXPECT_EQ(task0->size() + task1->size(), batch.size()); - EXPECT_EQ(task0->size(), batch.task(0).size()); - EXPECT_EQ(task1->size(), batch.task(1).size()); -} - -TEST(BatchTest, WaitUntilClosed) { - Batch batch; - batch.AddTask(std::unique_ptr(new FakeTask(3))); - EXPECT_FALSE(batch.IsClosed()); - - std::unique_ptr close_thread( - Env::Default()->StartThread(ThreadOptions(), "test", [&batch]() { - Env::Default()->SleepForMicroseconds(100); - batch.Close(); - })); - batch.WaitUntilClosed(); - EXPECT_TRUE(batch.IsClosed()); -} - -TEST(BatchTest, DeletionBlocksUntilClosed) { - Batch* batch = new Batch; - batch->AddTask(std::unique_ptr(new FakeTask(3))); - EXPECT_FALSE(batch->IsClosed()); - - Notification do_delete, deleted; - std::unique_ptr delete_thread(Env::Default()->StartThread( - ThreadOptions(), "test", [&batch, &do_delete, &deleted]() { - do_delete.WaitForNotification(); - delete batch; - deleted.Notify(); - })); - do_delete.Notify(); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(deleted.HasBeenNotified()); - batch->Close(); - deleted.WaitForNotification(); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/batching/batching_options.h b/tensorflow_serving/batching/batching_options.h new file mode 100644 index 00000000000..6b0a28389d2 --- /dev/null +++ b/tensorflow_serving/batching/batching_options.h @@ -0,0 +1,83 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_BATCHING_BATCHING_OPTIONS_H_ +#define TENSORFLOW_SERVING_BATCHING_BATCHING_OPTIONS_H_ + +#include + +namespace tensorflow { +namespace serving { + +// Batching options. +struct BatchingOptions { + // If set, restricts the allowed tensor batch sizes. + // + // When the batch scheduler forms a batch of size N, the batch size is rounded + // up to the smallest value M in 'allowed_batch_sizes' s.t. M >= N. The + // tensors submitted to the "Run()" call are padded with M-N repeats of one of + // the first N entries (i.e. a guaranteed valid entry). The last M-N entries + // of the output tensors are ignored. + // + // This option is useful when the underlying platform has some per-batch-size + // overhead, to limit the number of distinct batch sizes that can occur. It + // may be sensible to use an exponential sequence e.g. [8, 16, 32, ..., + // max_batch_size], a linear one e.g. [100, 200, 300, ..., max_batch_size], or + // perhaps a hybrid e.g. [8, 16, 32, 64, 100, 200, 300, ..., max_batch_size]. + // + // IMPORTANT: The entries must be in increasing order. + // + // IMPORTANT: The final entry in 'allowed_batch_sizes' must equal the maximum + // batch size parameter supplied to the batch scheduler. + // + // If left empty, no rounding/padding is performed. + std::vector allowed_batch_sizes; + + // If set to true, padding is performed for tensors of the same name + // but with unequal dimensions (modulo zeroth dimension), so that + // all tensors of the same name have equal dim sizes. + // For each tensor its first element is used as padding value. + // + // For example: + // given input tensors of shapes [1, 500, 101], [2, 300, 101], [1, 400, 101] + // they will be padded to shapes [1, 500, 101], [2, 500, 101], [1, 500, 101]. + // Padding is not performed in zeroth dimension. + // + // Supported tensor datatypes: + // DT_FLOAT, DT_DOUBLE, DT_INT8, DT_UINT8, DT_INT16, + // DT_UINT16, DT_INT32, DT_INT64, DT_COMPLEX64, DT_COMPLEX128, + // DT_STRING, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, + // DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE. + // + // Supported ranks: from 1 to 6. + // + // This option is useful when using recurrent models(like LSTMs) with serving. + // These models typically accept variable-length inputs and when + // training them typical strategy is to save sequence lengths for decoding + // and pad those variable-length dims to maximum in batch. + // So, this option is used to achieve similar behavior + // when serving with batching, it is assumed that sequence lengths + // have already been saved. + // + // If tensors with the same name have different shapes + // (modulo zeroth dimension) and this option is set to false, + // then error Status will be returned. + bool pad_variable_length_inputs = false; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_BATCHING_BATCHING_OPTIONS_H_ diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index e2deee54713..b3fd640fe8c 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -17,20 +17,144 @@ limitations under the License. #include +#include +#include +#include +#include +#include +#include +#include + +#include "absl/container/fixed_array.h" +#include "tensorflow/core/framework/cost_graph.pb.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_util.h" +#include "tensorflow/core/kernels/batching_util/input_split_metadata.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/gtl/cleanup.h" +#include "tensorflow/core/lib/monitoring/counter.h" +#include "tensorflow/core/lib/monitoring/percentile_sampler.h" +#include "tensorflow/core/lib/monitoring/sampler.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/threadpool_options.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/profiler/lib/traceme.h" +#include "tensorflow/core/profiler/lib/traceme_encode.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow_serving/batching/batching_util.h" +#include "tensorflow_serving/batching/incremental_barrier.h" +#include "tensorflow_serving/batching/threadsafe_status.h" #include "tensorflow_serving/servables/tensorflow/serving_session.h" -#include "tensorflow_serving/util/cleanup.h" +#include "tensorflow_serving/util/hash.h" namespace tensorflow { namespace serving { +namespace { + +auto* queuing_latency = monitoring::Sampler<1>::New( + {"/tensorflow/serving/batching_session/queuing_latency", + "Distribution of wall time spent (in microseconds) in queuing", + "thread_pool_name"}, + // Scale of 100, power of 1.2 with bucket count 52 (~1 second). + monitoring::Buckets::Exponential(100, 1.2, 52)); + +auto* wrapped_run_count = monitoring::Counter<0>::New( + "/tensorflow/serving/batching_session/wrapped_run_count", + "Total count of run calls on the wrapped session"); + +string TensorSignatureDebugString(const TensorSignature& signature) { + return strings::StrCat("{input_tensors: <", + absl::StrJoin(signature.input_tensors, ", "), + ">, output_tensors: <", + absl::StrJoin(signature.output_tensors, ", "), ">}"); +} + +struct HashTensorSignature { + uint64_t operator()(const TensorSignature& signature) const { + uint64_t hash = 0xDECAFCAFFE /* seed */; + for (const string& input_tensor : signature.input_tensors) { + hash = HashCombine(hash, std::hash()(input_tensor)); + } + for (const string& output_tensor : signature.output_tensors) { + hash = HashCombine(hash, std::hash()(output_tensor)); + } + return hash; + } +}; + +struct EqTensorSignature { + bool operator()(const TensorSignature& lhs, + const TensorSignature& rhs) const { + return lhs.input_tensors == rhs.input_tensors && + lhs.output_tensors == rhs.output_tensors; + } +}; + +// Constructs a TensorSignature from a Run() call's 'inputs' and +// 'output_tensor_names' arguments. +TensorSignature TensorSignatureFromRunArgs( + const std::vector>& inputs, + const std::vector& output_tensor_names) { + TensorSignature signature; + for (const auto& entry : inputs) { + const string& tensor_name = entry.first; + signature.input_tensors.insert(tensor_name); + } + for (const string& output_tensor_name : output_tensor_names) { + signature.output_tensors.insert(output_tensor_name); + } + return signature; +} + +// Constructs vector of one task input from one BatchingSessionTask. +const std::vector>& GetTaskInput( + const BatchingSessionTask& batching_session_task) { + if (batching_session_task.is_partial) { + return *batching_session_task.owned_split_inputs; + } + return *batching_session_task.inputs; +} + +// Constructs vector of all task inputs from Batch of BatchingSessionTasks. +// Input for each task is a vector of pairs (tensor_name, tensor_value). +std::vector>> GetTaskInputsVector( + const Batch& batch) { + std::vector>> all_task_inputs; + all_task_inputs.reserve(batch.num_tasks()); + for (int i = 0; i < batch.num_tasks(); ++i) { + all_task_inputs.push_back(GetTaskInput(batch.task(i))); + } + return all_task_inputs; +} + +} // namespace + +TensorSignature TensorSignatureFromSignatureDef( + const SignatureDef& signature_def) { + return TensorSignatureFromSignatureDefs({signature_def}); +} + +TensorSignature TensorSignatureFromSignatureDefs( + const std::vector& signature_defs) { + TensorSignature tensor_signature; + for (const SignatureDef& signature_def : signature_defs) { + for (const auto& entry : signature_def.inputs()) { + const TensorInfo& tensor_info = entry.second; + tensor_signature.input_tensors.insert(tensor_info.name()); + } + for (const auto& entry : signature_def.outputs()) { + const TensorInfo& tensor_info = entry.second; + tensor_signature.output_tensors.insert(tensor_info.name()); + } + } + return tensor_signature; +} + // A session that performs batching on top of a wrapped session. See the // documentation in batching_session.h for details and constraints. class BatchingSession : public ServingSession { @@ -38,228 +162,436 @@ class BatchingSession : public ServingSession { // Constructs a BatchingSession. Arguments: // - 'options' contains parameters. See batching_session.h. // - 'wrapped' is the session to wrap with batching. - // - 'batch_scheduler_creator' constructs a batch scheduler given a process- - // batch callback. See batching_session.h for example usage. - static Status Create( + // - 'signatures_with_scheduler_creators' specifies the set of supported + // signatures, and for each one supplies a lambda to construct a batch + // scheduler given a process-batch callback. See batching_session.h for + // example usage. + static absl::Status Create( + const BatchingSessionOptions& options, std::unique_ptr wrapped, + const std::vector& + signatures_with_scheduler_creators, + const std::string& thread_pool_name, + std::unique_ptr* result); + + // Same as above but allows for specification of a default scheduler creator + // which enables requests that don't match an exact signature to also + // have batching. + static absl::Status Create( const BatchingSessionOptions& options, std::unique_ptr wrapped, - std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, + BatchingSessionSchedulerCreator default_creator, + const std::string& thread_pool_name, std::unique_ptr* result); ~BatchingSession() override = default; - Status Run(const std::vector>& inputs, - const std::vector& output_tensor_names, - const std::vector& target_node_names, - std::vector* outputs) override; + absl::Status Run(const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs) override; + + // RunOptions handling: + // Since multiple of these Run() calls get backed into a single call to the + // underlying Session's Run(), we select an arbitrary 'run_options' (typically + // they are the same across calls). The exception is the timeout; we take the + // largest value (after subtracting time spent in the batching queue). + // + // RunMetadata: + // We copy the batched call's RunMetadata to each non-batched call's output. + // When input of a call is processed in multiple batches as opposed to one + // (i.e., `enable_large_batch_splitting` is true for batch scheduler), + // `RunMetadata.CostGraphDef.AggregatedCost` is the sum of all splits of the + // corresponding input and as correct as if the input is not split (again + // assuming all individual tasks in a batch have equal cost, which is the + // assumption before splitting is introduced), the rest of fields in + // `RunMetadata` are copied from the processing result of first split. + absl::Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, + RunMetadata* run_metadata) override; + + // Similar to the function above, but takes an additional + // 'thread_pool_options' to pass to the underlying Session's Run(). We select + // an arbitrary 'thread_pool_options' (typically they are the same across + // calls). + absl::Status Run( + const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, RunMetadata* run_metadata, + const thread::ThreadPoolOptions& thread_pool_options) override; + + absl::Status ListDevices(std::vector* response) override; private: - explicit BatchingSession(const BatchingSessionOptions& options); + explicit BatchingSession(const BatchingSessionOptions& options, + const std::string& thread_pool_name); + + // Helper fucntion to run the session. + absl::Status InternalRun( + const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, RunMetadata* run_metadata, + absl::optional thread_pool_options); // Computes the size of an input tensor list for batching purposes, by // analyzing the 0th dimension size of each of the tensors. All tensors in the // list must have the same 0th dimension size to be batchable. If the sizes // are not all identical, returns an error. - Status ComputeInputSize(const std::vector>& inputs, - size_t* size) const; - - // Returns the smallest entry in 'options_.allowed_batch_sizes' that is - // greater than or equal to 'batch_size'. If 'options_.allowed_batch_sizes' is - // empty, simply returns 'batch_size'. - int RoundToLowestAllowedBatchSize(int batch_size) const; + absl::Status ComputeInputSize( + const std::vector>& inputs, size_t* size) const; // Merges the input tensors in a batch, via concatenation of correspondingly- - // named tensors, and extracts the output tensor names. Assumes 'batch' is - // non-empty. Returns an error if there are any mismatches among the tasks in - // the batch that violate the constraints for batchability. - Status MergeInputTensors( - const Batch& batch, - std::vector>* merged_inputs, - std::vector* output_tensor_names); + // named tensors. Puts the merged inputs in the order they are in in the + // signature. Assumes 'batch' is non-empty. Returns an error if there are any + // mismatches among the tasks in the batch that violate the constraints for + // batchability. + absl::Status MergeInputTensors( + const TensorSignature& signature, const Batch& batch, + std::vector>* merged_inputs); // Splits the output of a batched call to 'wrapped_->Run()' into individual - // task outputs. - Status SplitOutputTensors(const std::vector& combined_outputs, - Batch* batch); + // task outputs. Assumes the output tensor order matches the signature. + absl::Status SplitOutputTensors(const TensorSignature& signature, + const std::vector& combined_outputs, + Batch* batch); + + // Splits RunMetadata parts (e.g. costgraph attribution) into individual task + // outputs. + absl::Status SplitRunMetadata(RunMetadata* batch_metadata, + Batch* batch); - // Processes one batch. Called by 'batch_scheduler_' in a batch thread. - void ProcessBatch(std::unique_ptr> batch); + // Processes one batch of Run() calls with 'signature'. Called by + // 'batch_scheduler_' in a batch thread. + void ProcessBatch(const TensorSignature& signature, + std::unique_ptr> batch); const BatchingSessionOptions options_; + // The name of the thread pool of the underlying batch scheduler. It is used + // for monitoring purpose, and can be empty if not known. + const std::string thread_pool_name_; std::unique_ptr wrapped_; - std::unique_ptr> batch_scheduler_; + std::unordered_map>, + HashTensorSignature, EqTensorSignature> + batch_schedulers_; + + // If set, default_scheduler_creator_ is used when the input signature does + // not match any existing signature defined during model load. This helps + // when the user uses either a combination of signatures or filter certain + // output tensors. + absl::optional default_scheduler_creator_; + absl::Mutex mu_; + std::unordered_map>, + HashTensorSignature, EqTensorSignature> + custom_signature_batch_schedulers_ ABSL_GUARDED_BY(mu_); TF_DISALLOW_COPY_AND_ASSIGN(BatchingSession); }; -Status BatchingSession::Create( +absl::Status BatchingSession::Create( const BatchingSessionOptions& options, std::unique_ptr wrapped, - std::function< - Status(std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, + BatchingSessionSchedulerCreator default_creator, + const std::string& thread_pool_name, std::unique_ptr* result) { - auto batching_session = - std::unique_ptr(new BatchingSession(options)); + auto status = BatchingSession::Create(options, std::move(wrapped), + signatures_with_scheduler_creators, + thread_pool_name, result); + result->get()->default_scheduler_creator_ = default_creator; + return status; +} + +absl::Status BatchingSession::Create( + const BatchingSessionOptions& options, std::unique_ptr wrapped, + const std::vector& + signatures_with_scheduler_creators, + const std::string& thread_pool_name, + std::unique_ptr* result) { + auto batching_session = std::unique_ptr( + new BatchingSession(options, thread_pool_name)); BatchingSession* raw_batching_session = batching_session.get(); batching_session->wrapped_ = std::move(wrapped); - std::unique_ptr> batch_scheduler; - TF_RETURN_IF_ERROR(batch_scheduler_creator( - [raw_batching_session]( - std::unique_ptr> batch) { - raw_batching_session->ProcessBatch(std::move(batch)); - }, - &batch_scheduler)); - batching_session->batch_scheduler_ = std::move(batch_scheduler); + + for (const auto& entry : signatures_with_scheduler_creators) { + const TensorSignature& signature = entry.signature; + const BatchingSessionSchedulerCreator& scheduler_creator = + entry.scheduler_creator; + + std::unique_ptr> batch_scheduler; + TF_RETURN_IF_ERROR(scheduler_creator( + [signature, raw_batching_session]( + std::unique_ptr> batch) { + raw_batching_session->ProcessBatch(signature, std::move(batch)); + }, + &batch_scheduler)); + batching_session->batch_schedulers_[signature] = std::move(batch_scheduler); + } + *result = std::move(batching_session); - return Status::OK(); + return absl::OkStatus(); } -Status BatchingSession::Run( +absl::Status BatchingSession::Run( const std::vector>& inputs, const std::vector& output_tensor_names, const std::vector& target_node_names, std::vector* outputs) { + RunMetadata run_metadata; + return Run(RunOptions(), inputs, output_tensor_names, target_node_names, + outputs, &run_metadata); +} + +absl::Status BatchingSession::Run( + const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, std::vector* outputs, + RunMetadata* run_metadata) { + return InternalRun(run_options, inputs, output_tensor_names, + target_node_names, outputs, run_metadata, absl::nullopt); +} + +absl::Status BatchingSession::Run( + const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, std::vector* outputs, + RunMetadata* run_metadata, + const thread::ThreadPoolOptions& thread_pool_options) { + return InternalRun(run_options, inputs, output_tensor_names, + target_node_names, outputs, run_metadata, + thread_pool_options); +} + +absl::Status BatchingSession::InternalRun( + const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, std::vector* outputs, + RunMetadata* run_metadata, + absl::optional thread_pool_options) { if (!target_node_names.empty()) { return errors::PermissionDenied( "BatchingSession does not support target nodes"); } + tsl::profiler::TraceMe trace_me([this] { + return tsl::profiler::TraceMeEncode( + "BatchingSessionRun", + {{"thread_pool_name", thread_pool_name_}, {"_r", 1} /*root_event*/}); + }); + const TensorSignature signature = + TensorSignatureFromRunArgs(inputs, output_tensor_names); + auto batch_scheduler_it = batch_schedulers_.find(signature); + if (batch_scheduler_it == batch_schedulers_.end()) { + if (default_scheduler_creator_.has_value()) { + absl::MutexLock l(&mu_); + batch_scheduler_it = custom_signature_batch_schedulers_.find(signature); + if (batch_scheduler_it == custom_signature_batch_schedulers_.end()) { + std::unique_ptr> batch_scheduler; + TF_RETURN_IF_ERROR(default_scheduler_creator_.value()( + [&, signature](std::unique_ptr> batch) { + ProcessBatch(signature, std::move(batch)); + }, + &batch_scheduler)); + custom_signature_batch_schedulers_[signature] = + std::move(batch_scheduler); + batch_scheduler_it = custom_signature_batch_schedulers_.find(signature); + } + } else { + // We have a Run() call that doesn't match one of our batching signatures. + // Run it in-line. + LOG_EVERY_N_SEC(WARNING, 120) + << "Request doesn't match any declared signature and no default " + "scheduler creator specified. Bypassing " + "batcher. Request signature is: " + << TensorSignatureDebugString(signature); + + // Because the wrapped session may not provide an implementation for + // thread_pool_options, we need to invoke different Run() functions + // depending on whether thread_pool_options is specified. + if (thread_pool_options) { + return wrapped_->Run(run_options, inputs, output_tensor_names, + target_node_names, outputs, run_metadata, + thread_pool_options.value()); + } else { + return wrapped_->Run(run_options, inputs, output_tensor_names, + target_node_names, outputs, run_metadata); + } + } + } + BatchScheduler* batch_scheduler = + batch_scheduler_it->second.get(); + + outputs->clear(); + Notification done; - Status status; + absl::Status status; auto task = std::unique_ptr(new BatchingSessionTask); + task->enqueue_time_micros = EnvTime::NowMicros(); + task->run_options = run_options; TF_RETURN_IF_ERROR(ComputeInputSize(inputs, &task->zeroth_dim_size)); task->inputs = &inputs; task->output_tensor_names = &output_tensor_names; task->done = &done; task->status = &status; task->outputs = outputs; + task->run_metadata = run_metadata; + task->thread_pool_options = thread_pool_options; + task->thread_safe_status = std::make_shared(); + task->shared_outputs = std::make_shared>>(); + task->split_run_metadatas = absl::make_unique>(); - TF_RETURN_IF_ERROR(batch_scheduler_->Schedule(&task)); + TF_RETURN_IF_ERROR(batch_scheduler->Schedule(&task)); done.WaitForNotification(); return status; } -BatchingSession::BatchingSession(const BatchingSessionOptions& options) - : options_(options) {} +absl::Status BatchingSession::ListDevices( + std::vector* response) { + return wrapped_->ListDevices(response); +} -Status BatchingSession::ComputeInputSize( - const std::vector>& inputs, size_t* size) const { - if (inputs.size() == 0) { - return errors::InvalidArgument( - "Batching session Run() must have at least one input tensor"); - } +BatchingSession::BatchingSession(const BatchingSessionOptions& options, + const std::string& thread_pool_name) + : options_(options), thread_pool_name_(thread_pool_name) {} - bool first = true; +absl::Status BatchingSession::ComputeInputSize( + const std::vector>& inputs, size_t* size) const { + TF_RETURN_IF_ERROR(::tensorflow::serving::ComputeTensorBatchSize( + inputs, size, + [](const std::pair& tensor) { + return tensor.second.shape().dims(); + }, + [](const std::pair& tensor, size_t dim) { + return tensor.second.shape().dim_size(dim); + })); for (const auto& entry : inputs) { const Tensor& tensor = entry.second; - - if (tensor.shape().dims() == 0) { - return errors::InvalidArgument( - "Batching session Run() input tensors must have at least one " - "dimension"); - } - const size_t this_size = tensor.shape().dim_size(0); - - if (first) { - *size = this_size; - first = false; - } else { - if (this_size != *size) { - return errors::InvalidArgument( - "Batching session Run() input tensors must have equal " - "0th-dimension size"); - } - } - } - return Status::OK(); -} - -int BatchingSession::RoundToLowestAllowedBatchSize(int batch_size) const { - if (options_.allowed_batch_sizes.empty()) { - return batch_size; - } - for (int allowed_size : options_.allowed_batch_sizes) { - if (allowed_size >= batch_size) { - return allowed_size; - } + RecordInputBatchSize(tensor.shape().dim_size(0)); } - LOG(ERROR) << "Maximum batch size greater than largest allowed size; " - "ignoring allowed sizes constraint"; - return batch_size; + return absl::OkStatus(); } -Status BatchingSession::MergeInputTensors( - const Batch& batch, - std::vector>* merged_inputs, - std::vector* output_tensor_names) { +absl::Status BatchingSession::MergeInputTensors( + const TensorSignature& signature, const Batch& batch, + std::vector>* merged_inputs) { DCHECK_GE(batch.num_tasks(), 1); if (batch.num_tasks() < 1) { return errors::Internal("Batch size expected to be positive; was ", batch.num_tasks()); } - *output_tensor_names = *batch.task(0).output_tensor_names; - std::vector input_tensor_names; - for (const auto& input : *batch.task(0).inputs) { - const string& tensor_name = input.first; - input_tensor_names.push_back(tensor_name); - } - - // Fast-path for a singleton batch with no padding. - if (batch.num_tasks() == 1 && options_.allowed_batch_sizes.empty()) { - *merged_inputs = *batch.task(0).inputs; - return Status::OK(); - } - const int padding_size = - RoundToLowestAllowedBatchSize(batch.size()) - batch.size(); + const int lowest_allowed_batch_size = + RoundToLowestAllowedBatchSize(options_.allowed_batch_sizes, batch.size()); + const int padding_size = lowest_allowed_batch_size - batch.size(); + tsl::profiler::TraceMe trace_me([lowest_allowed_batch_size, padding_size]() { + return tsl::profiler::TraceMeEncode( + "MergeInputTensors", + {{"batch_size_after_padding", lowest_allowed_batch_size}, + {"padding_amount", padding_size}}); + }); + RecordPaddingSize(padding_size, + lowest_allowed_batch_size); + RecordProcessedBatchSize(lowest_allowed_batch_size); - for (int input_tensor_idx = 0; input_tensor_idx < input_tensor_names.size(); - ++input_tensor_idx) { - const string& input_tensor_name = input_tensor_names[input_tensor_idx]; + // For each input tensor name, a vector of tensors from the individual tasks. + std::map> tensors_to_merge; + // For each input tensor name a vector of maximum dimension sizes + // among tensors from individual tasks. + absl::optional>> max_dim_sizes; + if (options_.pad_variable_length_inputs) { + std::vector>> all_task_inputs = + GetTaskInputsVector(batch); + max_dim_sizes = CalculateMaxDimSizes(all_task_inputs); + } + // Populate 'tensors_to_merge'. + for (int i = 0; i < batch.num_tasks(); ++i) { + const std::vector>& task_inputs = + GetTaskInput(batch.task(i)); + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; - std::vector tensors_to_merge; - for (int task_idx = 0; task_idx < batch.num_tasks(); ++task_idx) { - const std::vector>& task_inputs = - *batch.task(task_idx).inputs; - if (task_inputs.size() != input_tensor_names.size() || - task_inputs[input_tensor_idx].first != input_tensor_name) { - return errors::InvalidArgument( - "Batching session Run() calls must supply the same input tensors"); + std::vector& tensor_vec = tensors_to_merge[tensor_name]; + Tensor optionally_padded_tensor; + if (options_.pad_variable_length_inputs) { + TF_RETURN_IF_ERROR(AddPadding(tensor, (*max_dim_sizes)[tensor_name], + &optionally_padded_tensor)); + } else { + optionally_padded_tensor = tensor; + // Check whether tensors with the same name have equal dims + // (except zeroth dim) when padding is turned off. + if (i > 0) { // added at least one task to tensors_to_merge + TensorShape reference_shape = + tensors_to_merge[tensor_name][0].shape(); + if (!AreShapesEqualExceptZeroDim(tensor.shape(), reference_shape)) { + return errors::FailedPrecondition( + "Tensors with name '" + tensor_name + + "' from different tasks have different shapes and padding is " + "turned off. Set pad_variable_length_inputs to true, or ensure " + "that all tensors with the same name have equal dimensions " + "starting with the first dim."); + } + } } - if (input_tensor_idx == 0) { - if (*batch.task(task_idx).output_tensor_names != *output_tensor_names) { - return errors::InvalidArgument( - "Batching session Run() calls must supply the same output " - "tensors"); + tensor_vec.push_back(std::move(optionally_padded_tensor)); + if (i == batch.num_tasks() - 1 && padding_size > 0) { + // This is the last task. Insert padding. + // + // Use the first row of this task's tensor as the padding data. (We know + // it represents a valid input tensor row, so it should always be safe + // to use for padding.) + // + // Slice() operates on the 0th dimension, which is the batch dimension. + // It avoids a deep copy, which is a nice efficiency bonus. + const Tensor padding_tensor = tensor_vec.back().Slice(0, 1); + for (int i = 0; i < padding_size; ++i) { + tensor_vec.push_back(padding_tensor); } } - tensors_to_merge.push_back(task_inputs[input_tensor_idx].second); } - if (padding_size > 0) { - // Use the first row of the first task's input tensor for padding. - // (We know it exists, and represents a valid input tensor row, so it - // should always be safe to use for padding.) - const Tensor& first_task_tensor = - (*batch.task(0).inputs)[input_tensor_idx].second; - // Slice() operates on the 0th dimension, which is the batch dimension. It - // avoids a deep copy, which is a nice efficiency bonus. - const Tensor padding_tensor = first_task_tensor.Slice(0, 1); - for (int i = 0; i < padding_size; ++i) { - tensors_to_merge.push_back(padding_tensor); - } + } + + // Merge the tensors. + DCHECK_EQ(signature.input_tensors.size(), tensors_to_merge.size()); + if (tensors_to_merge.size() != signature.input_tensors.size()) { + return errors::Internal( + "One or more tasks does not conform to batch signature"); + } + for (const string& tensor_name : signature.input_tensors) { + auto tensors = tensors_to_merge.find(tensor_name); + DCHECK(tensors != tensors_to_merge.end()); + if (tensors == tensors_to_merge.end()) { + return errors::Internal( + "One or more tasks does not conform to batch signature"); + } + Tensor concated; + const absl::Status concat_status = + tensor::Concat(tensors->second, &concated); + DCHECK(concat_status.ok()) << concat_status.ToString(); + if (!concat_status.ok()) { + return errors::Internal("Tensor concat operation failed: ", + concat_status.ToString()); } - merged_inputs->push_back( - {input_tensor_name, tensor::Concat(tensors_to_merge)}); + merged_inputs->push_back({tensor_name, std::move(concated)}); } - return Status::OK(); + return absl::OkStatus(); } -Status BatchingSession::SplitOutputTensors( +absl::Status BatchingSession::SplitOutputTensors( + const TensorSignature& signature, const std::vector& combined_outputs, Batch* batch) { DCHECK_GE(batch->num_tasks(), 1); @@ -268,23 +600,32 @@ Status BatchingSession::SplitOutputTensors( batch->num_tasks()); } - // Fast-path for a singleton batch with no padding. - if (batch->num_tasks() == 1 && options_.allowed_batch_sizes.empty()) { - *batch->mutable_task(0)->outputs = combined_outputs; - return Status::OK(); - } - - std::vector task_sizes_plus_optional_padding; + std::vector task_sizes_plus_optional_padding; + task_sizes_plus_optional_padding.reserve(batch->num_tasks()); for (int i = 0; i < batch->num_tasks(); ++i) { task_sizes_plus_optional_padding.push_back(batch->task(i).zeroth_dim_size); } - const int padding_size = - RoundToLowestAllowedBatchSize(batch->size()) - batch->size(); + const int padding_size = RoundToLowestAllowedBatchSize( + options_.allowed_batch_sizes, batch->size()) - + batch->size(); if (padding_size > 0) { task_sizes_plus_optional_padding.push_back(padding_size); } - for (const Tensor& tensor : combined_outputs) { + // For each output tensor name, a divided-up tensor with one entry per task. + std::map> split_tensors; + + // Populate 'split_tensors'. + DCHECK_EQ(signature.output_tensors.size(), combined_outputs.size()); + if (combined_outputs.size() != signature.output_tensors.size()) { + return errors::Internal("Wrong number of batched output tensors"); + } + const std::vector output_tensors(signature.output_tensors.begin(), + signature.output_tensors.end()); + for (int i = 0; i < output_tensors.size(); ++i) { + const string& tensor_name = output_tensors[i]; + const Tensor& tensor = combined_outputs[i]; + if (tensor.shape().dims() == 0) { return errors::FailedPrecondition( "Batched output tensor has 0 dimensions"); @@ -295,8 +636,14 @@ Status BatchingSession::SplitOutputTensors( "0th dimension sizes of the input tensors"); } - std::vector split_tensor = - tensor::Split(tensor, task_sizes_plus_optional_padding); + std::vector split_tensor; + const absl::Status split_status = + tensor::Split(tensor, task_sizes_plus_optional_padding, &split_tensor); + DCHECK(split_status.ok()) << split_status.ToString(); + if (!split_status.ok()) { + return errors::Internal("Tensor split operation failed: ", + split_status.ToString()); + } DCHECK_EQ(split_tensor.size(), task_sizes_plus_optional_padding.size()); if (split_tensor.size() != task_sizes_plus_optional_padding.size()) { return errors::Internal( @@ -304,18 +651,69 @@ Status BatchingSession::SplitOutputTensors( split_tensor.size(), " splits; expected ", task_sizes_plus_optional_padding.size()); } + split_tensors[tensor_name] = std::move(split_tensor); + } - for (int i = 0; i < batch->num_tasks(); ++i) { - BatchingSessionTask* task = batch->mutable_task(i); - task->outputs->push_back(split_tensor[i]); + for (int i = 0; i < batch->num_tasks(); ++i) { + BatchingSessionTask* task = batch->mutable_task(i); + for (const string& tensor_name : *task->output_tensor_names) { + auto split_tensor = split_tensors.find(tensor_name); + DCHECK(split_tensor != split_tensors.end()); + if (split_tensor == split_tensors.end()) { + return errors::Internal("Task does not conform to batch signature"); + } + + if (task->is_partial) { + std::vector& tensor_vector = + (*task->shared_outputs)[task->split_index]; + tensor_vector.push_back(std::move(split_tensor->second[i])); + } else { + task->outputs->push_back(std::move(split_tensor->second[i])); + } } - // (Ignore a possible final split_tensor entry containing the padding.) } + // (Ignore a possible final split_tensors entry containing the padding.) - return Status::OK(); + return absl::OkStatus(); +} + +absl::Status BatchingSession::SplitRunMetadata( + RunMetadata* batch_metadata, Batch* batch) { + if (batch->num_tasks() > 0) { + if (batch_metadata->has_cost_graph()) { + // Scale the batch aggregated to reflect the cost of an individual request + // in the batch; this assumes all requests in a batch have an equal cost. + for (size_t i = 0; i < batch_metadata->cost_graph().cost_size(); ++i) { + CostGraphDef_AggregatedCost* cost = + batch_metadata->mutable_cost_graph()->mutable_cost(i); + const float agg_cost = cost->cost(); + cost->set_cost(agg_cost / static_cast(batch->num_tasks())); + } + } + + for (size_t i = 0; i < batch->num_tasks(); ++i) { + BatchingSessionTask* batching_session_task = batch->mutable_task(i); + if (batching_session_task->is_partial) { + // If 'is_partial', 'split_run_metadatas' is not nullptr and points + // to a vector of size + // 'batching_session_task->output_tensor_names->size'. + (*batching_session_task + ->split_run_metadatas)[batching_session_task->split_index] = + *batch_metadata; + } else { + RunMetadata* run_metadata = batching_session_task->run_metadata; + if (run_metadata != nullptr) { + *run_metadata = *batch_metadata; + } + } + } + } + + return absl::OkStatus(); } void BatchingSession::ProcessBatch( + const TensorSignature& signature, std::unique_ptr> batch) { // As a possible performance optimization, consider overlapping the tensor // concatenation with waiting for the batch to close (i.e. do the @@ -326,59 +724,323 @@ void BatchingSession::ProcessBatch( return; } - Status status; + const uint64_t dequeue_time_micros = EnvTime::NowMicros(); // Regardless of the outcome, we need to propagate the status to the // individual tasks and signal that they are done. We use MakeCleanup() to // ensure that this happens no matter how we exit the method below. - auto finally = MakeCleanup([&status, &batch] { + absl::Status status; + auto finally = gtl::MakeCleanup([&status, &batch] { for (int i = 0; i < batch->num_tasks(); ++i) { - *batch->mutable_task(i)->status = status; - batch->mutable_task(i)->done->Notify(); + BatchingSessionTask* task = batch->mutable_task(i); + if (task->is_partial) { + task->thread_safe_status->Update(status); + task->done_callback(); + } else { + *batch->mutable_task(i)->status = status; + batch->mutable_task(i)->done->Notify(); + } } }); + // Make sure we have at least one task that hasn't exceeded its timeout from + // queue time alone, and find the latest task deadline which we'll use for the + // overall batch. + bool all_tasks_timeout_exceeded = true; + uint64_t batch_deadline_micros = 0; + for (int i = 0; i < batch->num_tasks(); ++i) { + const BatchingSessionTask& task = batch->task(i); + // If the caller doesn't populate RunOptions, the timeout is 0 by default. + // Interpret that as "no timeout" i.e. infinity. + const int64_t task_timeout_micros = + task.run_options.timeout_in_ms() <= 0 + ? INT_MAX + : task.run_options.timeout_in_ms() * 1000; + const uint64_t task_deadline_micros = + task.enqueue_time_micros + task_timeout_micros; + if (task_deadline_micros > dequeue_time_micros) { + all_tasks_timeout_exceeded = false; + if (task_deadline_micros > batch_deadline_micros) { + batch_deadline_micros = task_deadline_micros; + } + } + queuing_latency->GetCell(thread_pool_name_) + ->Add(dequeue_time_micros - task.enqueue_time_micros); + } + if (all_tasks_timeout_exceeded) { + status = absl::Status( + static_cast(absl::StatusCode::kResourceExhausted), + "Run() timeout exceeded while waiting in batching queue"); + return; + } + + RunOptions run_options = batch->task(0).run_options; + if (batch_deadline_micros == INT_MAX) { + run_options.set_timeout_in_ms(0); + } else { + run_options.set_timeout_in_ms( + (batch_deadline_micros - dequeue_time_micros) / 1000); + } + std::vector> merged_inputs; - std::vector output_tensor_names; - status = MergeInputTensors(*batch, &merged_inputs, &output_tensor_names); + status = MergeInputTensors(signature, *batch, &merged_inputs); if (!status.ok()) { return; } + absl::optional thread_pool_options = + batch->task(0).thread_pool_options; + + const std::vector output_tensor_names( + signature.output_tensors.begin(), signature.output_tensors.end()); std::vector combined_outputs; - status = wrapped_->Run(merged_inputs, output_tensor_names, - {} /* target node names */, &combined_outputs); + RunMetadata run_metadata; + // Because the wrapped session may not provide an implementation for + // thread_pool_options, we need to invoke different Run() functions depending + // on whether thread_pool_options is specified. + if (thread_pool_options) { + status = wrapped_->Run(run_options, merged_inputs, output_tensor_names, + {} /* target node names */, &combined_outputs, + &run_metadata, thread_pool_options.value()); + } else { + status = wrapped_->Run(run_options, merged_inputs, output_tensor_names, + {} /* target node names */, &combined_outputs, + &run_metadata); + } + wrapped_run_count->GetCell()->IncrementBy(1); + status.Update(SplitRunMetadata(&run_metadata, batch.get())); + if (!status.ok()) { return; } - status = SplitOutputTensors(combined_outputs, batch.get()); + status = SplitOutputTensors(signature, combined_outputs, batch.get()); +} + +// TODO(b/158393551): +// Share implementation between `SplitInputTask` here and +// `BatchResource::SplitInputTask` by refactoring and unifying the naming or +// type differences of data members. +absl::Status SplitInputTask( + std::unique_ptr* input_task_ptr, + int open_batch_remaining_slot, int max_batch_size, + std::vector>* output_tasks) { + BatchingSessionTask& input_task = *(*input_task_ptr); + const int64_t input_task_size = input_task.size(); + + DCHECK_GT(input_task_size, 0); + + // `split_task_done_callback` runs only after all split tasks are complete. + std::function split_task_done_callback = + [done_notification = input_task.done, + shared_outputs = input_task.shared_outputs, + shared_status = input_task.thread_safe_status, + num_output = input_task.output_tensor_names->size(), + outputs = input_task.outputs, status = input_task.status, + run_metadata = input_task.run_metadata, + split_run_metadatas = input_task.split_run_metadatas]() { + auto finally = gtl::MakeCleanup([&] { + *status = shared_status->status(); + done_notification->Notify(); + }); + + // Some slices of tasks encounter errors, return early without + // processing per-split result. + if (!shared_status->status().ok()) { + return; + } + + for (int i = 0; i < num_output; ++i) { + Tensor output_tensor; + + // Concat i-th tensor from each split into i-th tensor of output. + std::vector to_concatenate; + to_concatenate.reserve(shared_outputs->size()); + for (int j = 0; j < shared_outputs->size(); ++j) { + to_concatenate.push_back(std::move((*shared_outputs)[j][i])); + } + const auto concat_status = + tensor::Concat(to_concatenate, &output_tensor); + if (!concat_status.ok()) { + shared_status->Update(concat_status); + return; + } + + outputs->push_back(std::move(output_tensor)); + } + + // `cost_dimension_map` aggregates costs from all splits for each + // dimension. + absl::flat_hash_map cost_dimension_map; + for (const auto& split : *split_run_metadatas) { + if (split.has_cost_graph()) { + for (const auto& cost : split.cost_graph().cost()) { + cost_dimension_map[cost.dimension()] += cost.cost(); + } + } + } + + *run_metadata = (*split_run_metadatas)[0]; + std::vector cost_dimensions; + for (const auto& cost_and_dimension : + run_metadata->cost_graph().cost()) { + cost_dimensions.push_back(cost_and_dimension.dimension()); + } + run_metadata->mutable_cost_graph()->clear_cost(); + for (const auto& dimension : cost_dimensions) { + const auto iter = cost_dimension_map.find(dimension); + if (iter != cost_dimension_map.end()) { + auto graph_cost = run_metadata->mutable_cost_graph()->add_cost(); + graph_cost->set_dimension(iter->first); + graph_cost->set_cost(iter->second); + } + } + }; + IncrementalBarrier barrier(split_task_done_callback); + + const internal::InputSplitMetadata input_split_metadata( + input_task_size, open_batch_remaining_slot, max_batch_size); + + // Creates an array of int64_t from an array of int, since `tensor::Split` + // requires an array of int64. + const absl::FixedArray output_task_sizes( + input_split_metadata.task_sizes().begin(), + input_split_metadata.task_sizes().end()); + const int num_batches = output_task_sizes.size(); + + input_task.shared_outputs->resize(num_batches); + + for (int i = 0; i < num_batches; ++i) { + (*input_task.shared_outputs)[i].reserve( + input_task.output_tensor_names->size()); + } + + input_task.split_run_metadatas->resize(num_batches); + + output_tasks->reserve(num_batches); + for (int i = 0; i < num_batches; i++) { + auto task = absl::make_unique(); + task->enqueue_time_micros = input_task.enqueue_time_micros; + task->run_options = input_task.run_options; + task->zeroth_dim_size = output_task_sizes[i]; + // `task->owned_input` will be initialized separately out of this for-loop. + task->output_tensor_names = input_task.output_tensor_names; + + task->owned_split_inputs = + absl::make_unique>>(); + task->split_index = i; + task->shared_outputs = input_task.shared_outputs; + task->thread_safe_status = input_task.thread_safe_status; + task->is_partial = true; + task->done_callback = barrier.Inc(); + task->thread_pool_options = input_task.thread_pool_options; + + task->split_run_metadatas = input_task.split_run_metadatas; + + output_tasks->push_back(std::move(task)); + } + + const int num_input_tensors = input_task.inputs->size(); + + // Splits each input tensor according to `output_task_sizes`, and + // initializes input of `output_tasks` with split results. + for (int i = 0; i < num_input_tensors; ++i) { + std::vector split_tensors; + const string& tensor_name = (*input_task.inputs)[i].first; + const Tensor& input_tensor = (*input_task.inputs)[i].second; + // TODO(b/158393551): + // Figure out the optimal implementation of Split, by using + // 'Tensor::Slice' and eliminating unnecessary memcpy as much as possible. + const absl::Status split_status = + tensor::Split(input_tensor, output_task_sizes, &split_tensors); + if (!split_status.ok()) { + return errors::Internal( + "When splitting input, Tensor split operation failed: ", + split_status.ToString()); + } + if (split_tensors.size() != output_task_sizes.size()) { + return errors::Internal( + "When splitting input, tensor split operation did not work as " + "expected; got ", + split_tensors.size(), " splits; expected ", output_task_sizes.size()); + } + for (int j = 0; j < output_tasks->size(); ++j) { + BatchingSessionTask& output_task = *((*output_tasks)[j]); + output_task.owned_split_inputs->push_back( + std::make_pair(tensor_name, split_tensors[j])); + } + } + return absl::OkStatus(); +} + +absl::Status CreateBatchingSession( + const BatchingSessionOptions& options, + const std::vector& + signatures_with_scheduler_creators, + BatchingSessionSchedulerCreator default_creator, + std::unique_ptr session, + std::unique_ptr* batching_session) { + std::unique_ptr internal_batching_session; + TF_RETURN_IF_ERROR(BatchingSession::Create( + options, std::move(session), signatures_with_scheduler_creators, + default_creator, /*thread_pool_name=*/"", &internal_batching_session)); + *batching_session = std::move(internal_batching_session); + return absl::OkStatus(); } -Status CreateBatchingSession( +absl::Status CreateBatchingSession( const BatchingSessionOptions& options, - std::function< - Status(std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, std::unique_ptr session, std::unique_ptr* batching_session) { std::unique_ptr internal_batching_session; - TF_RETURN_IF_ERROR(BatchingSession::Create(options, std::move(session), - batch_scheduler_creator, - &internal_batching_session)); + TF_RETURN_IF_ERROR(BatchingSession::Create( + options, std::move(session), signatures_with_scheduler_creators, + /*thread_pool_name=*/"", &internal_batching_session)); *batching_session = std::move(internal_batching_session); - return Status::OK(); + return absl::OkStatus(); } -Status CreateBasicBatchingSession( +absl::Status CreateBasicBatchingSession( const BasicBatchScheduler::Options& schedule_options, const BatchingSessionOptions& batching_session_options, - std::unique_ptr session, + const TensorSignature& signature, std::unique_ptr session, std::unique_ptr* batching_session) { - if (!batching_session_options.allowed_batch_sizes.empty()) { - if (batching_session_options.allowed_batch_sizes.back() != - schedule_options.max_batch_size) { + const auto& allowed_batch_sizes = + batching_session_options.allowed_batch_sizes; + if (!allowed_batch_sizes.empty()) { + if (schedule_options.enable_large_batch_splitting) { + const int max_allowed_batch_size = allowed_batch_sizes.back(); + int32 last_size = 0; + for (size_t i = 0; i < allowed_batch_sizes.size(); ++i) { + const int32 size = allowed_batch_sizes.at(i); + if (i > 0 && size <= last_size) { + return errors::InvalidArgument( + "allowed_batch_sizes entries must be monotonically increasing"); + } + last_size = size; + } + if (max_allowed_batch_size > schedule_options.max_batch_size) { + return errors::InvalidArgument( + "Last entry in allowed_batch_sizes must be less than or equal to " + "max_batch_size; last " + "entry was ", + max_allowed_batch_size, "; expected ", + schedule_options.max_batch_size); + } + if (schedule_options.max_execution_batch_size != max_allowed_batch_size) { + return errors::InvalidArgument( + "Last entry in allowed_batch_sizes must be equal to " + "max_execution_batch_size; last " + "entry was ", + max_allowed_batch_size, "; expected ", + schedule_options.max_execution_batch_size); + } + } else if (allowed_batch_sizes.back() != schedule_options.max_batch_size) { + // TODO(b/161641195): + // Validate `allowed_batch_sizes` increase monotonically for non + // large_batch_splitting case. return errors::InvalidArgument( "Last entry in allowed_batch_sizes must match max_batch_size; last " "entry was ", @@ -387,19 +1049,27 @@ Status CreateBasicBatchingSession( } } - auto scheduler_creator = [schedule_options]( - std::function>)> - process_batch_callback, - std::unique_ptr>* batch_scheduler) { - std::unique_ptr> - basic_batch_scheduler; - TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( - schedule_options, process_batch_callback, &basic_batch_scheduler)); - *batch_scheduler = std::move(basic_batch_scheduler); - return Status::OK(); - }; - return CreateBatchingSession(batching_session_options, scheduler_creator, - std::move(session), batching_session); + auto scheduler_creator = + [schedule_options]( + std::function>)> + process_batch_callback, + std::unique_ptr>* + batch_scheduler) { + std::unique_ptr> + basic_batch_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + schedule_options, process_batch_callback, &basic_batch_scheduler)); + *batch_scheduler = std::move(basic_batch_scheduler); + return absl::OkStatus(); + }; + + std::unique_ptr internal_batching_session; + TF_RETURN_IF_ERROR(BatchingSession::Create( + batching_session_options, std::move(session), + {{signature, scheduler_creator}}, schedule_options.thread_pool_name, + &internal_batching_session)); + *batching_session = std::move(internal_batching_session); + return absl::OkStatus(); } } // namespace serving diff --git a/tensorflow_serving/batching/batching_session.h b/tensorflow_serving/batching/batching_session.h index 576e664675c..5420ab6ff01 100644 --- a/tensorflow_serving/batching/batching_session.h +++ b/tensorflow_serving/batching/batching_session.h @@ -26,9 +26,15 @@ limitations under the License. #include #include +#include "absl/types/optional.h" +#include "tensorflow/core/kernels/batching_util/basic_batch_scheduler.h" +#include "tensorflow/core/kernels/batching_util/batch_scheduler.h" +#include "tensorflow/core/platform/threadpool_options.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" #include "tensorflow/core/public/session.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" -#include "tensorflow_serving/batching/batch_scheduler.h" +#include "tensorflow_serving/batching/batching_options.h" +#include "tensorflow_serving/batching/threadsafe_status.h" namespace tensorflow { namespace serving { @@ -37,52 +43,70 @@ namespace serving { // scheduler template parameters, e.g. BasicBatchScheduler. struct BatchingSessionTask; -// Options for batching tensorflow Sessions; see the Create*() functions below. -struct BatchingSessionOptions { - // If set, restricts the allowed tensor batch sizes. - // - // When the batch scheduler forms a batch of size N, the batch size is rounded - // up to the smallest value M in 'allowed_batch_sizes' s.t. M >= N. The - // tensors submitted to the underlying Session are padded with M-N repeats of - // one of the first N entries (i.e. a guaranteed valid entry). The last M-N - // entries of the output tensors are ignored. - // - // This option is useful when the underlying platform has some per-batch-size - // overhead, to limit the number of distinct batch sizes that can occur. It - // may be sensible to use an exponential sequence e.g. [8, 16, 32, ..., - // max_batch_size], a linear one e.g. [100, 200, 300, ..., max_batch_size], or - // perhaps a hybrid e.g. [8, 16, 32, 64, 100, 200, 300, ..., max_batch_size]. - // - // IMPORTANT: The entries must be in increasing order. - // - // IMPORTANT: The final entry in 'allowed_batch_sizes' must equal the maximum - // batch size parameter supplied to the batch scheduler. - // - // If left empty, no rounding/padding is performed. - std::vector allowed_batch_sizes; +// A function to construct a batch scheduler for BatchingSessionTasks from a +// process-batch callback. +using BatchingSessionSchedulerCreator = std::function>)>, + std::unique_ptr>*)>; + +// The signature associated with a Session::Run() call, in terms of input and +// output tensor names (with the order in which the tensors are listed factored +// out). (Note that 'target_node_names' are not supported in batching sessions.) +struct TensorSignature { + std::set input_tensors; + std::set output_tensors; +}; + +// Constructs a TensorSignature for a given SignatureDef. +TensorSignature TensorSignatureFromSignatureDef( + const SignatureDef& signature_def); + +// Constructs a TensorSignature for a given set of SignatureDefs. The resulting +// TensorSignature represents the Session::Run() arguments that would be used +// when issuing a single Run() call that exercises the signature defs jointly. +// +// For example, say there's a graph that takes 'input' and transforms it into +// 'predicted_label' and 'confidence_score'. Suppose SignatureDef 1 requests +// only 'predicted_label' as output, and SignatureDef 2 requests only +// 'confidence_score'. A joint TensorSignature would feed 'input' and receive +// both 'predicted_label' and 'confidence_score' as output, in a single Run() +// invocation. +TensorSignature TensorSignatureFromSignatureDefs( + const std::vector& signature_defs); + +// A signature paired with a lambda to create a batch scheduler for Run() calls +// matching the signature. +struct SignatureWithBatchingSessionSchedulerCreator { + TensorSignature signature; + BatchingSessionSchedulerCreator scheduler_creator; }; +// Options for batching tensorflow Sessions; see the Create*() functions below. +using BatchingSessionOptions = BatchingOptions; + // Wraps a session in a new session that automatically batches Run() calls. -// In addition to a session to wrap, takes a function that constructs a batch -// scheduler given a process-batch callback. +// Uses one batcher for each distinct Run() signature supported. In addition to +// a session to wrap, takes a list of signature/BatchingSessionSchedulerCreator +// pairs. (The number of supported signatures is typically small, and often just +// a single one.) // -// The wrapped session only supports Run(), with the following restrictions: -// - 'target_node_names' must be empty -// - 'inputs' must not be empty -// - all calls supply the same input tensor names in 'inputs' -// - all calls supply the same output tensor names in 'output_tensor_names' +// The wrapped session only batches Run() calls that conform to one of the +// specified signatures and leave 'target_node_names' empty. Other Run() calls +// are executed in-line without batching, and may harm performance. (Extra- +// signature Run() support is intended primarily for debugging and diagnostics.) // -// It is assumed that the outermost (0th) dimension of each input and output -// tensor is the batch-size dimension. All input tensors must have the same 0th- -// dimension size B; the produced output tensors are also assumed to have 0th- -// dimension size B. +// For batched calls, it is assumed that the outermost (0th) dimension of each +// input and output tensor is the batch-size dimension. All input tensors must +// have the same 0th-dimension size B; the produced output tensors are also +// assumed to have 0th-dimension size B. // // IMPORTANT: Each call to Session::Run() is synchronous, and blocks waiting for -// other Run() calls to merge with to form a large batch. Consequently, to -// achieve good throughput we recommend setting the number of client threads -// that call Session::Run() equal to about twice the maximum batch size. +// other Run() calls with the same signature to merge with to form a large +// batch. Consequently, to achieve good throughput we recommend setting the +// number of client threads that call Session::Run() equal to about twice the +// sum over all signatures of the maximum batch size. // -// Example usage: +// Example usage, for the common case of a single signature: // // BatchingSessionOptions options = ...; // auto scheduler_creator = [schedule_options, retry_options]( @@ -99,27 +123,44 @@ struct BatchingSessionOptions { // return Status::OK(); // }; // std::unique_ptr batching_session; -// TF_CHECK_OK(CreateBatchingSession(options, scheduler_creator, +// TF_CHECK_OK(CreateBatchingSession(options, {{signature, scheduler_creator}}, // std::move(session), &batching_session)); // Status CreateBatchingSession( const BatchingSessionOptions& options, - std::function< - Status(std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, + std::unique_ptr session, + std::unique_ptr* batching_session); + +// Same as above but allows for a default scheduler creator for which signatures +// that don't match a supplied value during run time can still use batching. +Status CreateBatchingSession( + const BatchingSessionOptions& options, + const std::vector& + signatures_with_scheduler_creators, + BatchingSessionSchedulerCreator default_creator, std::unique_ptr session, std::unique_ptr* batching_session); // A convenience for using CreateBatchingSession() to create a -// BasicBatchScheduler. +// BasicBatchScheduler for a single signature. Status CreateBasicBatchingSession( const typename BasicBatchScheduler::Options& schedule_options, const BatchingSessionOptions& batching_session_options, - std::unique_ptr session, + const TensorSignature& signature, std::unique_ptr session, std::unique_ptr* batching_session); +// The default implementation of +// `BasicBatchScheduler::Options.split_input_task_func` if corresponding batch +// scheduler for a batching session sets +// `BasicBatchScheduler::Options.enable_large_batch_splitting` to true. +Status SplitInputTask( + std::unique_ptr* input_task_ptr, + int open_batch_remaining_slot, int max_batch_size, + std::vector>* output_tasks); + ////////// // Implementation details follow. API users need not read. @@ -127,15 +168,48 @@ struct BatchingSessionTask : public BatchTask { ~BatchingSessionTask() override = default; size_t size() const override { return zeroth_dim_size; } + // For monitoring purpose. + static std::string Name() { return "batching_session"; } + // Fields populated when a task is received. + uint64_t enqueue_time_micros; + RunOptions run_options; size_t zeroth_dim_size; const std::vector>* inputs; const std::vector* output_tensor_names; - // Fields populated when a task is processed (as part of a batch). + // Fields populated when a task is processed (as part of a batch), and + // returned by BatchingSession when a task is complete. Notification* done; Status* status; std::vector* outputs; + RunMetadata* run_metadata; + absl::optional thread_pool_options; + + // Fields populated when a task is processed (as part of a batch), and + // substantially used in the intermediate stage if a task is a slice of + // input task (i.e., is_partial=true). + bool is_partial = false; + // 'owned_split_inputs' stores pairs of tensor names and input tensors + // if 'is_partial' = true. + std::unique_ptr>> owned_split_inputs; + // The index of this split, along the 0-th dimension of input from op + // invocation. + int split_index = 0; + std::function done_callback; + typedef std::vector> TensorMatrix; + // For shared_ptr objects, ownership shared by: + // 1) each split of task (to fill one row in this matrix) + // and + // 2) callback that runs to merge output of individual splits for an op + // invocation, after all splits complete. + // Two-dimensional tensor matrix, + std::shared_ptr shared_outputs; + // 'status' records error (could be from any split) if at least one split + // returns error, OK otherwise. + std::shared_ptr thread_safe_status; + // 'split_run_metadatas' records `run_metadata` of each split. + std::shared_ptr> split_run_metadatas; }; } // namespace serving diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index 08d79f170bf..6c4bfc6918c 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -15,17 +15,30 @@ limitations under the License. #include "tensorflow_serving/batching/batching_session.h" +#include +#include +#include +#include +#include + #include -#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "absl/synchronization/notification.h" +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/tag_constants.h" +#include "tensorflow/core/framework/cost_graph.pb.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/monitoring/sampler.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/thread_annotations.h" +#include "tensorflow/core/platform/threadpool_options.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/public/session_options.h" #include "tensorflow_serving/servables/tensorflow/serving_session.h" #include "tensorflow_serving/test_util/test_util.h" @@ -34,6 +47,9 @@ namespace tensorflow { namespace serving { namespace { +using ::testing::HasSubstr; +using ::testing::UnorderedElementsAre; + // A wrapper around a Session that captures the batch size. class BatchSizeCapturingSession : public ServingSession { public: @@ -41,22 +57,63 @@ class BatchSizeCapturingSession : public ServingSession { : wrapped_(std::move(wrapped)) {} ~BatchSizeCapturingSession() override = default; - Status Run(const std::vector>& inputs, - const std::vector& output_tensor_names, - const std::vector& target_node_names, - std::vector* outputs) override { - latest_batch_size_ = inputs[0].second.shape().dim_size(0); - return wrapped_->Run(inputs, output_tensor_names, target_node_names, - outputs); + absl::Status Run(const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs) override { + RunMetadata run_metadata; + return Run(RunOptions(), inputs, output_tensor_names, target_node_names, + outputs, &run_metadata); + } + + absl::Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, + RunMetadata* run_metadata) override { + return Run(run_options, inputs, output_tensor_names, target_node_names, + outputs, run_metadata, thread::ThreadPoolOptions()); } - int latest_batch_size() const { return latest_batch_size_; } + absl::Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, RunMetadata* run_metadata, + const thread::ThreadPoolOptions& thread_pool_options) + override TF_LOCKS_EXCLUDED(latest_batch_size_mu_) { + { + mutex_lock l(latest_batch_size_mu_); + latest_batch_size_ = inputs[0].second.shape().dim_size(0); + } + absl::Status status = wrapped_->Run( + run_options, inputs, output_tensor_names, target_node_names, outputs, + run_metadata, thread_pool_options); + *(run_metadata->mutable_cost_graph()) = cost_graph_; + return status; + } + + absl::Status ListDevices(std::vector* response) override { + return wrapped_->ListDevices(response); + } + + int latest_batch_size() const TF_LOCKS_EXCLUDED(latest_batch_size_mu_) { + mutex_lock l(latest_batch_size_mu_); + return latest_batch_size_; + } + + CostGraphDef* mutable_cost_graph() { return &cost_graph_; } private: std::unique_ptr wrapped_; + mutable mutex latest_batch_size_mu_; // The size of the batch most recently submitted to Run(). - int latest_batch_size_ = -1; + int latest_batch_size_ TF_GUARDED_BY(latest_batch_size_mu_) = -1; + + // Cost graph associated with the latest call to Run(). + CostGraphDef cost_graph_; TF_DISALLOW_COPY_AND_ASSIGN(BatchSizeCapturingSession); }; @@ -64,27 +121,49 @@ class BatchSizeCapturingSession : public ServingSession { // Creates a (non-batching) session with the half-plus-two model loaded. std::unique_ptr CreateHalfPlusTwoSession() { tensorflow::SessionOptions session_options; - const string export_dir = test_util::ContribTestSrcDirPath( - "session_bundle/example/half_plus_two/00000123"); - SessionBundle session_bundle; - TF_CHECK_OK( - LoadSessionBundleFromPath(session_options, export_dir, &session_bundle)); - return std::move(session_bundle.session); + tensorflow::RunOptions run_options; + const string export_dir = test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two/00000123"); + SavedModelBundle bundle; + TF_CHECK_OK(LoadSavedModel(session_options, run_options, export_dir, + {kSavedModelTagServe}, &bundle)); + return std::move(bundle.session); } -// Test that a session handles a single request for the half-plus-two model -// properly. The request has two input floats (size=2 for batching purposes). -void TestSingleRequest(float input_0, float input_1, Session* session) { - Tensor input = test::AsTensor({input_0, input_1}, {2}); - // Half plus two: each output should be input / 2 + 2. - Tensor expected_output = - test::AsTensor({input_0 / 2 + 2, input_1 / 2 + 2}, {2}); +std::unique_ptr CreateMatrixHalfPlusTwoSession() { + tensorflow::SessionOptions session_options; + tensorflow::RunOptions run_options; + const string export_dir = + test_util::TestSrcDirPath("batching/testdata/matrix_half_plus_two/1"); + SavedModelBundle bundle; + TF_CHECK_OK(LoadSavedModel(session_options, run_options, export_dir, + {kSavedModelTagServe}, &bundle)); + return std::move(bundle.session); +} - const std::vector> inputs = {{"x", input}}; - std::vector outputs; - TF_ASSERT_OK(session->Run(inputs, {"y"}, {} /* target nodes */, &outputs)); - ASSERT_EQ(1, outputs.size()); - test::ExpectTensorEqual(expected_output, outputs[0]); +void TestRequest(const std::vector& x_values, TensorShape x_shape, + const std::vector& y_values, TensorShape y_shape, + Session* session, + test_util::CountingThreadPool* inter_op_threadpool = nullptr, + test_util::CountingThreadPool* intra_op_threadpool = nullptr) { + Tensor input = test::AsTensor(x_values, x_shape); + Tensor expected_output = test::AsTensor(y_values, y_shape); + + RunMetadata run_metadata; + thread::ThreadPoolOptions thread_pool_options; + thread_pool_options.inter_op_threadpool = inter_op_threadpool; + thread_pool_options.intra_op_threadpool = intra_op_threadpool; + std::vector output; + TF_ASSERT_OK(session->Run(RunOptions(), {{"x", input}}, {"y"}, + {} /* target nodes */, &output, &run_metadata, + thread_pool_options)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); + + // The intra_op_threadpool doesn't have anything scheduled. + if (inter_op_threadpool != nullptr) { + ASSERT_GE(inter_op_threadpool->NumScheduled(), 1); + } } // Invoke Run() with the supplied arguments, and expect a particular error. @@ -93,117 +172,564 @@ void ExpectError(const string& error_message, const std::vector& output_tensor_names, Session* session) { std::vector outputs; - Status status = session->Run(inputs, output_tensor_names, - {} /* target nodes */, &outputs); + absl::Status status = session->Run(inputs, output_tensor_names, + {} /* target nodes */, &outputs); ASSERT_FALSE(status.ok()); - EXPECT_EQ(error_message, status.error_message()); + EXPECT_EQ(error_message, status.message()); +} + +// Creates a SignatureDef from a TensorSignature. +SignatureDef CreateSignatureDef(const TensorSignature& tensor_signature) { + SignatureDef signature_def; + for (const string& input_tensor : tensor_signature.input_tensors) { + TensorInfo input; + input.set_name(input_tensor); + (*signature_def.mutable_inputs())[input_tensor] = input; + } + for (const string& output_tensor : tensor_signature.output_tensors) { + TensorInfo output; + output.set_name(output_tensor); + (*signature_def.mutable_outputs())[output_tensor] = output; + } + return signature_def; +} + +int GetPercentileTotal(string label) { + auto* collection_registry = monitoring::CollectionRegistry::Default(); + monitoring::CollectionRegistry::CollectMetricsOptions options; + const std::unique_ptr collected_metrics = + collection_registry->CollectMetrics(options); + int total_samples = 0; + const auto& point_set_map = collected_metrics->point_set_map; + if (point_set_map.find(label) == point_set_map.end()) return 0; + const monitoring::PointSet& lps = *point_set_map.at(label); + for (int i = 0; i < lps.points.size(); ++i) { + total_samples += lps.points[i]->histogram_value.sum(); + } + return static_cast(total_samples); +} + +bool CheckDescriptor(string label, const string& description, + const std::vector& labels) { + auto* collection_registry = monitoring::CollectionRegistry::Default(); + monitoring::CollectionRegistry::CollectMetricsOptions options; + const std::unique_ptr collected_metrics = + collection_registry->CollectMetrics(options); + const auto& metric_descriptor_map = collected_metrics->metric_descriptor_map; + if (metric_descriptor_map.find(label) == metric_descriptor_map.end()) { + return false; + } + const monitoring::MetricDescriptor& desc = *metric_descriptor_map.at(label); + if (desc.description != description) return false; + if (labels.size() != desc.label_names.size()) return false; + for (int i = 0; i < labels.size(); ++i) { + if (labels[i] != desc.label_names[i]) return false; + } + return true; +} + +TEST(BatchingSessionSignatureTest, TensorSignatureFromSignatureDef) { + const SignatureDef signature_def = + CreateSignatureDef({{"x0", "x1"}, {"y0", "y1"}}); + const TensorSignature tensor_signature = + TensorSignatureFromSignatureDef(signature_def); + EXPECT_THAT(tensor_signature.input_tensors, UnorderedElementsAre("x0", "x1")); + EXPECT_THAT(tensor_signature.output_tensors, + UnorderedElementsAre("y0", "y1")); +} + +TEST(BatchingSessionSignatureTest, TensorSignatureFromSignatureDefs) { + const SignatureDef signature_def_0 = + CreateSignatureDef({{"x0", "x1"}, {"y0", "y1"}}); + const SignatureDef signature_def_1 = + CreateSignatureDef({{"x1", "x2"}, {"y1", "y3"}}); + const TensorSignature tensor_signature = + TensorSignatureFromSignatureDefs({signature_def_0, signature_def_1}); + EXPECT_THAT(tensor_signature.input_tensors, + UnorderedElementsAre("x0", "x1", "x2")); + EXPECT_THAT(tensor_signature.output_tensors, + UnorderedElementsAre("y0", "y1", "y3")); } -TEST(BatchingSessionTest, Basic) { +class BatchingSessionTest : public ::testing::TestWithParam { + public: + BatchingSessionTest() {} + + bool enable_large_batch_splitting() const { return GetParam(); } + + std::function* input_task, + int first_output_task_size, int max_batch_size, + std::vector>* output_tasks)> + get_split_input_task_func() const { + if (enable_large_batch_splitting()) { + return SplitInputTask; + } + return nullptr; + } + + // If 'enable_large_batch_splitting' is true, annotate `input_options` with + // parameters for splitting large batches. + BasicBatchScheduler::Options annotate_options( + const BasicBatchScheduler::Options input_options) { + BasicBatchScheduler::Options output_options = + input_options; + output_options.enable_large_batch_splitting = + enable_large_batch_splitting(); + if (enable_large_batch_splitting()) { + output_options.split_input_task_func = get_split_input_task_func(); + // Bump up the max batch size, and set execution batch size to the max + // size we actually want -- this will allow us to exercise large batch + // splits (they trigger when execution_batch_size < max_batch_size). + output_options.max_execution_batch_size = input_options.max_batch_size; + output_options.max_batch_size = input_options.max_batch_size * 2; + } + return output_options; + } +}; + +TEST_P(BatchingSessionTest, Basic) { BasicBatchScheduler::Options schedule_options; schedule_options.max_batch_size = 4; // fits two 2-unit tasks schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); // Asynchronously send two requests whose total size is 4. The two requests in // conjunction should trigger a batch to be processed. std::unique_ptr first_request_thread(Env::Default()->StartThread( ThreadOptions(), "first_request_thread", [&batching_session] { - TestSingleRequest(100.0f, 42.0f, batching_session.get()); + TestRequest({100.0f, 42.0f}, {2}, {52.0f, 23.0f}, {2}, + batching_session.get()); })); std::unique_ptr second_request_thread(Env::Default()->StartThread( ThreadOptions(), "second_request_thread", [&batching_session] { - TestSingleRequest(71.5f, 18.3f, batching_session.get()); + TestRequest({71.5f, 18.3f}, {2}, {37.75f, 11.15f}, {2}, + batching_session.get()); })); } -TEST(BatchingSessionTest, SingletonBatch) { +TEST_P(BatchingSessionTest, BatchingWithPadding) { BasicBatchScheduler::Options schedule_options; - schedule_options.max_batch_size = 4; // fits two 2-unit tasks - schedule_options.batch_timeout_micros = 0; + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = 1e6; schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; + batching_session_options.pad_variable_length_inputs = true; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); - TestSingleRequest(100.0f, 42.0f, batching_session.get()); + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateMatrixHalfPlusTwoSession(), &batching_session)); + // two requests form a batch and first input gets padded with zeros to match + // [1, 3, 3] shape that is accepted by the model. + // if padding doesn't work, test will fail. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session] { + TestRequest({1, 2, 3, 4}, {1, 2, 2}, + {2.5, 3, 2.5, 3.5, 4, 2.5, 2.5, 2.5, 2.5}, {1, 3, 3}, + batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + TestRequest({5, 6, 7, 8, 9, 10, 11, 12, 13}, {1, 3, 3}, + {4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5}, {1, 3, 3}, + batching_session.get()); + })); } -TEST(BatchingSessionTest, RequestWithIncompatibleInputTensorSizes) { +TEST_P(BatchingSessionTest, BatchingWithLargeBatch) { BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 3; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 2; + schedule_options = annotate_options(schedule_options); + schedule_options.max_execution_batch_size = 2; std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); + if (enable_large_batch_splitting()) { + // `max_execution_batch_size` is 2, so input of second request will be + // split for processing. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session] { + TestRequest({5, 6, 7, 8}, {1, 2, 2}, {4.5, 5, 5.5, 6}, {1, 2, 2}, + batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + TestRequest({5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16}, {3, 2, 2}, + {4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5, 9, 9.5, 10}, + {3, 2, 2}, batching_session.get()); + })); + } else { + Tensor input1 = test::AsTensor({5, 6, 7, 8}, {1, 2, 2}); + Tensor expected_output1 = + test::AsTensor({4.5, 5, 5.5, 6}, {1, 2, 2}); + std::vector output1; + Notification notify; + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request", [&] { + auto status = + batching_session->Run({{"x", input1}}, {"y"}, {}, &output1); + EXPECT_TRUE(status.ok()); + test::ExpectTensorEqual(expected_output1, output1[0]); + })); - ExpectError( - "Batching session Run() input tensors must have equal 0th-dimension size", - {{"input_0", test::AsTensor({3}, {1})}, - {"input_1", test::AsTensor({5, 7}, {2})}}, - {"output"}, batching_session.get()); + // `max_batch_size` is 3, so input2 (of size 4) will be invalidated. + Tensor input2 = test::AsTensor( + {5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20}, {4, 2, 2}); + std::vector output2; + std::unique_ptr second_request_thread( + Env::Default()->StartThread(ThreadOptions(), "second_request", [&] { + auto status = + batching_session->Run({{"x", input2}}, {"y"}, {}, &output2); + EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.message(), + HasSubstr("Task size 4 is larger than " + "maximum input batch size 3")); + })); + } } -TEST(BatchingSessionTest, RequestsWithDifferentInputTensors) { +TEST_P(BatchingSessionTest, BatchHandlesSplitError) { + if (!enable_large_batch_splitting()) { + return; + } + BasicBatchScheduler::Options schedule_options; - schedule_options.max_batch_size = 2; // fits two 1-unit tasks - schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + schedule_options.max_batch_size = 3; + schedule_options.batch_timeout_micros = INT_MAX; // set a large time out + schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + schedule_options.max_execution_batch_size = 2; std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); + + string expected_error_msg = + "Tensors with name 'x' from different tasks have different shapes and " + "padding is turned off. Set pad_variable_length_inputs to true, or " + "ensure that all tensors with the same name have equal dimensions " + "starting with the first dim."; - std::unique_ptr thread_1(Env::Default()->StartThread( - ThreadOptions(), "thread_1", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same input tensors", - {{"input", test::AsTensor({3}, {1})}}, {"output"}, - batching_session.get()); + // `max_batch_size` is 3 and `max_execution_batch_size` is 2, so inputs of + // first thread will span over two tasks, causing errors in both batch tasks. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", + [&batching_session, &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor({1, 2, 3}, {3, 1, 1})}}, {"y"}, + batching_session.get()); })); - std::unique_ptr thread_2(Env::Default()->StartThread( - ThreadOptions(), "thread_2", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same input tensors", - {{"input", test::AsTensor({3}, {1})}, - {"another_input", test::AsTensor({3}, {1})}}, - {"output"}, batching_session.get()); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", + [&batching_session, &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor({1, 2}, {1, 2})}}, {"y"}, + batching_session.get()); })); } -TEST(BatchingSessionTest, RequestsWithDifferentOutputTensors) { +TEST_P(BatchingSessionTest, BatchingLazySplit) { + if (!enable_large_batch_splitting()) { + return; + } + BasicBatchScheduler::Options schedule_options; - schedule_options.max_batch_size = 2; // fits two 1-unit tasks - schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = INT_MAX; // set a large time out + schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + schedule_options.max_execution_batch_size = 1; std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); + + // `max_batch_size` is 2 and `max_execution_batch_size` is 1, so inputs + // will be split and process. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session] { + TestRequest({5, 6, 7, 8}, {1, 2, 2}, {4.5, 5, 5.5, 6}, {1, 2, 2}, + batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + TestRequest({1, 2, 3, 4}, {1, 2, 2}, {2.5, 3, 3.5, 4.0}, {1, 2, 2}, + batching_session.get()); + })); +} + +TEST(BatchingSessionTest, BatchingWithPaddingAndCost) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 1; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + batching_session_options.pad_variable_length_inputs = true; + std::unique_ptr batch_size_capturing_session( + new BatchSizeCapturingSession(CreateHalfPlusTwoSession())); + auto batch_size_capturing_session_raw = batch_size_capturing_session.get(); + + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + std::move(batch_size_capturing_session), &batching_session)); + + CostGraphDef* cg = batch_size_capturing_session_raw->mutable_cost_graph(); + CostGraphDef_AggregatedCost* ag = cg->add_cost(); + ag->set_cost(7.0); + ag = cg->add_cost(); + ag->set_dimension("named-cost"); + ag->set_cost(1.0); + + // two requests form a batch and first input gets padded with zeros to match + // [1, 3, 3] shape that is accepted by the model. + // if padding doesn't work, test will fail. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session] { + Tensor input = test::AsTensor({1, 2, 3, 4}, {1, 2, 2}); + Tensor expected_output = test::AsTensor( + {2.5, 3, 2.5, 3.5, 4, 2.5, 2.5, 2.5, 2.5}, {1, 3, 3}); + std::vector output; + RunMetadata run_metadata; + TF_ASSERT_OK(batching_session->Run({}, {{"x", input}}, {"y"}, {}, + &output, &run_metadata)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); + const CostGraphDef& cgs = run_metadata.cost_graph(); + EXPECT_EQ(2, cgs.cost_size()); + EXPECT_NEAR(3.5, cgs.cost(0).cost(), 0.001); + EXPECT_NEAR(0.5, cgs.cost(1).cost(), 0.001); + EXPECT_EQ("named-cost", cgs.cost(1).dimension()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + Tensor input = + test::AsTensor({5, 6, 7, 8, 9, 10, 11, 12, 13}, {1, 3, 3}); + Tensor expected_output = test::AsTensor( + {4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5}, {1, 3, 3}); + std::vector output; + RunMetadata run_metadata; + TF_ASSERT_OK(batching_session->Run({}, {{"x", input}}, {"y"}, {}, + &output, &run_metadata)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); + const CostGraphDef& cgs = run_metadata.cost_graph(); + EXPECT_EQ(2, cgs.cost_size()); + EXPECT_NEAR(3.5, cgs.cost(0).cost(), 0.001); + EXPECT_NEAR(0.5, cgs.cost(1).cost(), 0.001); + EXPECT_EQ("named-cost", cgs.cost(1).dimension()); + })); +} + +TEST_P(BatchingSessionTest, BatchingWithCost) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 3; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 2; + schedule_options = annotate_options(schedule_options); + schedule_options.max_execution_batch_size = 2; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + std::unique_ptr batch_size_capturing_session( + new BatchSizeCapturingSession(CreateHalfPlusTwoSession())); + auto batch_size_capturing_session_raw = batch_size_capturing_session.get(); + + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + std::move(batch_size_capturing_session), &batching_session)); + + CostGraphDef* cg = batch_size_capturing_session_raw->mutable_cost_graph(); + CostGraphDef_AggregatedCost* ag = cg->add_cost(); + ag->set_cost(7.0); + ag = cg->add_cost(); + ag->set_dimension("named-cost"); + ag->set_cost(1.0); - std::unique_ptr thread_1(Env::Default()->StartThread( - ThreadOptions(), "thread_1", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same output tensors", - {{"input", test::AsTensor({3}, {1})}}, {"output"}, - batching_session.get()); + // two requests form a batch and first input gets padded with zeros to match + // [1, 3, 3] shape that is accepted by the model. + // if padding doesn't work, test will fail. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session] { + Tensor input = test::AsTensor({1, 2, 3, 4, 5, 6}, {1, 2, 3}); + Tensor expected_output = + test::AsTensor({2.5, 3, 3.5, 4, 4.5, 5}, {1, 2, 3}); + std::vector output; + RunMetadata run_metadata; + TF_ASSERT_OK(batching_session->Run({}, {{"x", input}}, {"y"}, {}, + &output, &run_metadata)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); + const CostGraphDef& cgs = run_metadata.cost_graph(); + EXPECT_EQ(2, cgs.cost_size()); + EXPECT_NEAR(3.5, cgs.cost(0).cost(), 0.001); + EXPECT_NEAR(0.5, cgs.cost(1).cost(), 0.001); + EXPECT_EQ("named-cost", cgs.cost(1).dimension()); })); - std::unique_ptr thread_2(Env::Default()->StartThread( - ThreadOptions(), "thread_2", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same output tensors", - {{"input", test::AsTensor({3}, {1})}}, - {"output", "another_output"}, batching_session.get()); + if (enable_large_batch_splitting()) { + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + Tensor input = + test::AsTensor({5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, + 17, 18, 19, 20, 21, 22}, + {3, 2, 3}); + Tensor expected_output = + test::AsTensor({4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5, 9, + 9.5, 10, 10.5, 11, 11.5, 12, 12.5, 13}, + {3, 2, 3}); + std::vector output; + RunMetadata run_metadata; + TF_ASSERT_OK(batching_session->Run({}, {{"x", input}}, {"y"}, {}, + &output, &run_metadata)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); + const CostGraphDef& cgs = run_metadata.cost_graph(); + EXPECT_EQ(2, cgs.cost_size()); + EXPECT_NEAR(10.5, cgs.cost(0).cost(), 0.001); + EXPECT_NEAR(1.5, cgs.cost(1).cost(), 0.001); + EXPECT_EQ("named-cost", cgs.cost(1).dimension()); + })); + } else { + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + Tensor input = test::AsTensor({5, 6, 7, 8, 9, 10}, {1, 2, 3}); + Tensor expected_output = + test::AsTensor({4.5, 5, 5.5, 6, 6.5, 7}, {1, 2, 3}); + std::vector output; + RunMetadata run_metadata; + TF_ASSERT_OK(batching_session->Run({}, {{"x", input}}, {"y"}, {}, + &output, &run_metadata)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); + const CostGraphDef& cgs = run_metadata.cost_graph(); + EXPECT_EQ(2, cgs.cost_size()); + EXPECT_NEAR(3.5, cgs.cost(0).cost(), 0.001); + EXPECT_NEAR(0.5, cgs.cost(1).cost(), 0.001); + EXPECT_EQ("named-cost", cgs.cost(1).dimension()); + })); + } +} + +TEST_P(BatchingSessionTest, UnequalTensorShapesWithPaddingTurnedOff) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + batching_session_options.pad_variable_length_inputs = false; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateMatrixHalfPlusTwoSession(), &batching_session)); + string expected_error_msg = + "Tensors with name 'x' from different tasks have different shapes and " + "padding is turned off. Set pad_variable_length_inputs to true, or " + "ensure that all tensors with the same name have equal dimensions " + "starting with the first dim."; + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", + [&batching_session, &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor({1, 2, 3, 4}, {1, 2, 2})}}, + {"y"}, batching_session.get()); })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", + [&batching_session, &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor( + {5, 6, 7, 8, 9, 10, 11, 12, 13}, {1, 3, 3})}}, + {"y"}, batching_session.get()); + })); +} + +TEST_P(BatchingSessionTest, SingletonBatch) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 4; // fits two 2-unit tasks + schedule_options.batch_timeout_micros = 0; + schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); + TestRequest({100.0f, 42.0f}, {2}, {52.0f, 23.0f}, {2}, + batching_session.get()); +} + +TEST_P(BatchingSessionTest, RequestThatDoesntMatchSignatureGetsRunAnyway) { + BasicBatchScheduler::Options schedule_options; + // Set the batching parameters s.t. if the request is batched the test will + // timeout. + schedule_options.max_batch_size = 100; + schedule_options.batch_timeout_micros = INT_MAX; + schedule_options = annotate_options(schedule_options); + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x2"}, {"y3"}}, + CreateHalfPlusTwoSession(), &batching_session)); + // Issue a request using x/y, which doesn't match the x2/y3 signature. + TestRequest({100.0f, 42.0f}, {2}, {52.0f, 23.0f}, {2}, + batching_session.get()); +} + +TEST_P(BatchingSessionTest, RequestWithIncompatibleInputTensorSizes) { + BasicBatchScheduler::Options schedule_options; + schedule_options = annotate_options(schedule_options); + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + + int32 start_input_value = GetPercentileTotal( + "/tensorflow/serving/batching_session/input_batch_size"); + int32 start_process_value = GetPercentileTotal( + "/tensorflow/serving/batching_session/processed_batch_size"); + int32 start_pad_value = + GetPercentileTotal("/tensorflow/serving/batching_session/padding_size"); + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, + {{"input_0", "input_1"}, {"output"}}, CreateHalfPlusTwoSession(), + &batching_session)); + + ExpectError("Batching Run() input tensors must have equal 0th-dimension size", + {{"input_0", test::AsTensor({3}, {1})}, + {"input_1", test::AsTensor({5, 7}, {2})}}, + {"output"}, batching_session.get()); + + // We expect no change. + EXPECT_EQ(start_input_value, + GetPercentileTotal( + "/tensorflow/serving/batching_session/input_batch_size")); + EXPECT_EQ(start_process_value, + GetPercentileTotal( + "/tensorflow/serving/batching_session/processed_batch_size")); + EXPECT_EQ( + start_pad_value, + GetPercentileTotal("/tensorflow/serving/batching_session/padding_size")); } -TEST(BatchingSessionTest, AllowedBatchSizes_NoPaddingNeeded) { +TEST_P(BatchingSessionTest, AllowedBatchSizesNoPaddingNeeded) { + int32 start_input_value = GetPercentileTotal( + "/tensorflow/serving/batching_session/input_batch_size"); + int32 start_process_value = GetPercentileTotal( + "/tensorflow/serving/batching_session/processed_batch_size"); + int32 start_pad_value = + GetPercentileTotal("/tensorflow/serving/batching_session/padding_size"); // Arrange to capture the batch size. std::unique_ptr batch_size_capturing_session( new BatchSizeCapturingSession(CreateHalfPlusTwoSession())); @@ -213,19 +739,39 @@ TEST(BatchingSessionTest, AllowedBatchSizes_NoPaddingNeeded) { schedule_options.max_batch_size = 4; schedule_options.batch_timeout_micros = 0; schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); BatchingSessionOptions batching_session_options; batching_session_options.allowed_batch_sizes = {2, 4}; std::unique_ptr batching_session; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, + schedule_options, batching_session_options, {{"x"}, {"y"}}, std::move(batch_size_capturing_session), &batching_session)); - TestSingleRequest(100.0f, 42.0f, batching_session.get()); + TestRequest({100.0f, 42.0f}, {2}, {52.0f, 23.0f}, {2}, + batching_session.get()); // It should not add any padding, i.e. leave the batch size at 2. EXPECT_EQ(2, batch_size_capturing_session_raw->latest_batch_size()); + + // We expect no pad, 2 inputs, and a batch process of 2. + EXPECT_EQ(start_input_value + 2, + GetPercentileTotal( + "/tensorflow/serving/batching_session/input_batch_size")); + EXPECT_EQ(start_process_value + 2, + GetPercentileTotal( + "/tensorflow/serving/batching_session/processed_batch_size")); + EXPECT_EQ( + start_pad_value, + GetPercentileTotal("/tensorflow/serving/batching_session/padding_size")); } -TEST(BatchingSessionTest, AllowedBatchSizesRequirePadding) { +TEST_P(BatchingSessionTest, AllowedBatchSizesRequirePadding) { + int32 start_input_value = GetPercentileTotal( + "/tensorflow/serving/batching_session/input_batch_size"); + int32 start_process_value = GetPercentileTotal( + "/tensorflow/serving/batching_session/processed_batch_size"); + int32 start_pad_value = + GetPercentileTotal("/tensorflow/serving/batching_session/padding_size"); + // Arrange to capture the batch size. std::unique_ptr batch_size_capturing_session( new BatchSizeCapturingSession(CreateHalfPlusTwoSession())); @@ -235,43 +781,328 @@ TEST(BatchingSessionTest, AllowedBatchSizesRequirePadding) { schedule_options.max_batch_size = 4; schedule_options.batch_timeout_micros = 0; schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); BatchingSessionOptions batching_session_options; batching_session_options.allowed_batch_sizes = {1, 3, 4}; std::unique_ptr batching_session; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, + schedule_options, batching_session_options, {{"x"}, {"y"}}, std::move(batch_size_capturing_session), &batching_session)); - TestSingleRequest(100.0f, 42.0f, batching_session.get()); + TestRequest({100.0f, 42.0f}, {2}, {52.0f, 23.0f}, {2}, + batching_session.get()); // It should pad the batch size from 2 to 3. EXPECT_EQ(3, batch_size_capturing_session_raw->latest_batch_size()); + + // We expect 1 pad, 2 inputs, and a batch process of 3. + EXPECT_EQ(start_input_value + 2, + GetPercentileTotal( + "/tensorflow/serving/batching_session/input_batch_size")); + EXPECT_EQ(start_process_value + 3, + GetPercentileTotal( + "/tensorflow/serving/batching_session/processed_batch_size")); + EXPECT_EQ( + start_pad_value + 1, + GetPercentileTotal("/tensorflow/serving/batching_session/padding_size")); + EXPECT_TRUE( + CheckDescriptor("/tensorflow/serving/batching_session/padding_size", + "Tracks the padding size distribution on batches.", + {"execution_batch_size"})); + EXPECT_TRUE( + CheckDescriptor("/tensorflow/serving/batching_session/input_batch_size", + "Tracks the batch size distribution on the inputs.", {})); + EXPECT_TRUE(CheckDescriptor( + "/tensorflow/serving/batching_session/processed_batch_size", + "Tracks the batch size distribution on processing.", {})); } -TEST(BatchingSessionTest, UnsortedAllowedBatchSizesRejected) { +TEST_P(BatchingSessionTest, UnsortedAllowedBatchSizesRejected) { BasicBatchScheduler::Options schedule_options; schedule_options.max_batch_size = 4; + schedule_options = annotate_options(schedule_options); BatchingSessionOptions batching_session_options; batching_session_options.allowed_batch_sizes = {4, 2}; // Not sorted. std::unique_ptr batching_session; - EXPECT_FALSE( - CreateBasicBatchingSession(schedule_options, batching_session_options, - CreateHalfPlusTwoSession(), &batching_session) - .ok()); + EXPECT_FALSE(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session) + .ok()); } -TEST(BatchingSessionTest, - FinalAllowedBatchSizeDifferingFromMaxBatchSizeRejected) { +TEST_P(BatchingSessionTest, + FinalAllowedBatchSizeLargerThanMaxBatchSizeRejected) { BasicBatchScheduler::Options schedule_options; schedule_options.max_batch_size = 4; + schedule_options = annotate_options(schedule_options); BatchingSessionOptions batching_session_options; batching_session_options.allowed_batch_sizes = {2, 8}; // Final entry != 4. std::unique_ptr batching_session; - EXPECT_FALSE( - CreateBasicBatchingSession(schedule_options, batching_session_options, - CreateHalfPlusTwoSession(), &batching_session) - .ok()); + auto status = CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session); + EXPECT_EQ(status.code(), error::INVALID_ARGUMENT); + EXPECT_THAT(status.message(), HasSubstr(enable_large_batch_splitting() + ? "max_execution_batch_size" + : "max_batch_size")); } +TEST_P(BatchingSessionTest, DifferentOrderForInputAndOutputTensors) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 6; // fits three 2-unit tasks + schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + BatchingSessionOptions batching_session_options; + std::unique_ptr batching_session; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x", "x2"}, {"y", "y3"}}, + CreateHalfPlusTwoSession(), &batching_session)); + + const Tensor input0 = test::AsTensor({8.0f, 6.0f}, {2}); + const Tensor expected_output0 = test::AsTensor({6.0f, 5.0f}, {2}); + const Tensor input1 = test::AsTensor({100.0f, 42.0f}, {2}); + const Tensor expected_output1 = test::AsTensor({53.0f, 24.0f}, {2}); + + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x", input0}, {"x2", input1}}, + {"y", "y3"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(2, outputs.size()); + test::ExpectTensorEqual(expected_output0, outputs[0]); + test::ExpectTensorEqual(expected_output1, outputs[1]); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, + {"y3", "y"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(2, outputs.size()); + test::ExpectTensorEqual(expected_output1, outputs[0]); + test::ExpectTensorEqual(expected_output0, outputs[1]); + })); + std::unique_ptr third_request_thread( + Env::Default()->StartThread(ThreadOptions(), "third_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, + {"y", "y3"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(2, outputs.size()); + test::ExpectTensorEqual(expected_output0, outputs[0]); + test::ExpectTensorEqual(expected_output1, outputs[1]); + })); +} + +TEST_P(BatchingSessionTest, MultipleSignatures) { + std::vector*> schedulers; + auto create_scheduler = + [&schedulers, this]( + std::function>)> + process_batch_callback, + std::unique_ptr>* scheduler) { + BasicBatchScheduler::Options options; + options.max_batch_size = 4; // fits two 2-unit tasks + options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + options.num_batch_threads = 1; + options = annotate_options(options); + std::unique_ptr> + basic_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + options, process_batch_callback, &basic_scheduler)); + schedulers.push_back(basic_scheduler.get()); + *scheduler = std::move(basic_scheduler); + return absl::OkStatus(); + }; + BatchingSessionOptions batching_session_options; + std::unique_ptr batching_session; + TF_CHECK_OK(CreateBatchingSession(batching_session_options, + {{{{"x"}, {"y"}}, create_scheduler}, + {{{"x2"}, {"y3"}}, create_scheduler}}, + CreateHalfPlusTwoSession(), + &batching_session)); + ASSERT_EQ(2, schedulers.size()); + + // Create lambdas for 2-unit inference requests to each signature. + auto run_signature0_request = [&batching_session] { + Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + Tensor expected_output = test::AsTensor({52.0f, 23.0f}, {2}); + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x", input}}, {"y"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(1, outputs.size()); + test::ExpectTensorEqual(expected_output, outputs[0]); + }; + auto run_signature1_request = [&batching_session] { + Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + Tensor expected_output = test::AsTensor({53.0f, 24.0f}, {2}); + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input}}, {"y3"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(1, outputs.size()); + test::ExpectTensorEqual(expected_output, outputs[0]); + }; + + // Enqueue one request for each signature. Both should block because neither + // batching queue will be full yet. + std::unique_ptr signature0_thread(Env::Default()->StartThread( + ThreadOptions(), "signature0_thread", [&] { run_signature0_request(); })); + std::unique_ptr signature1_thread(Env::Default()->StartThread( + ThreadOptions(), "signature1_thread", [&] { run_signature1_request(); })); + while (schedulers[0]->NumEnqueuedTasks() != 1 && + schedulers[1]->NumEnqueuedTasks() != 1) { + Env::Default()->SleepForMicroseconds(100); + } + + // Enqueue a second request for each signature. This should fill both queues + // and unblock all the processing. + run_signature0_request(); + EXPECT_EQ(0, schedulers[0]->NumEnqueuedTasks()); + run_signature1_request(); + EXPECT_EQ(0, schedulers[1]->NumEnqueuedTasks()); +} + +TEST_P(BatchingSessionTest, EnqueuedLongerThanTimeout) { + BatchScheduler* scheduler = nullptr; + auto create_scheduler = + [&scheduler, this]( + std::function>)> + process_batch_callback, + std::unique_ptr>* new_scheduler) { + BasicBatchScheduler::Options options; + options.max_batch_size = 4; // fits two 2-unit tasks + options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + options.num_batch_threads = 1; + options = annotate_options(options); + std::unique_ptr> + basic_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + options, process_batch_callback, &basic_scheduler)); + scheduler = basic_scheduler.get(); + *new_scheduler = std::move(basic_scheduler); + return absl::OkStatus(); + }; + BatchingSessionOptions batching_session_options; + std::unique_ptr batching_session; + TF_CHECK_OK(CreateBatchingSession( + batching_session_options, {{{{"x"}, {"y"}}, create_scheduler}}, + CreateHalfPlusTwoSession(), &batching_session)); + ASSERT_FALSE(scheduler == nullptr); + + // Enqueue a request with a timeout specified via RunOptions. + Notification request_returned; + auto issue_request = [&batching_session, &request_returned] { + Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + RunOptions run_options; + run_options.set_timeout_in_ms(1); + std::vector outputs; + RunMetadata run_metadata; + const absl::Status status = + batching_session->Run(run_options, {{"x", input}}, {"y"} /* outputs */, + {} /* target nodes */, &outputs, &run_metadata); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(error::RESOURCE_EXHAUSTED, status.code()); + EXPECT_THAT( + status.message(), + HasSubstr("Run() timeout exceeded while waiting in batching queue")); + request_returned.Notify(); + }; + std::unique_ptr request_thread(Env::Default()->StartThread( + ThreadOptions(), "request_thread", [&] { issue_request(); })); + while (scheduler->NumEnqueuedTasks() != 1) { + Env::Default()->SleepForMicroseconds(100); + } + // Sleep for longer than the request's timeout, so that when it does finally + // get dequeued for batch processing it has already exceeded its timeout. + Env::Default()->SleepForMicroseconds(10 * 1000); + // Tear down the batcher, so that it schedules the pending batch. + batching_session = nullptr; + request_returned.WaitForNotification(); +} + +TEST_P(BatchingSessionTest, ThreadPoolOptions) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 3; + schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + schedule_options.max_execution_batch_size = 1; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); + + test_util::CountingThreadPool inter_op_threadpool(Env::Default(), "InterOp", + /*num_threads=*/1); + test_util::CountingThreadPool intra_op_threadpool(Env::Default(), "IntraOp", + /*num_threads=*/1); + + // Asynchronously send two requests whose total size is 4. + // They form two batches in both non-split and input-split mode. + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + TestRequest({100.0f, 42.0f}, {2}, {52.0f, 23.0f}, {2}, + batching_session.get(), &inter_op_threadpool, + &intra_op_threadpool); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + TestRequest({71.5f, 18.3f}, {2}, {37.75f, 11.15f}, {2}, + batching_session.get(), &inter_op_threadpool, + &intra_op_threadpool); + })); +} + +TEST_P(BatchingSessionTest, SubsetOutputTensors) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 6; // fits three 2-unit tasks + schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + schedule_options.num_batch_threads = 1; + schedule_options = annotate_options(schedule_options); + BatchingSessionOptions batching_session_options; + std::unique_ptr batching_session; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x", "x2"}, {"y", "y3"}}, + CreateHalfPlusTwoSession(), &batching_session)); + + const Tensor input0 = test::AsTensor({8.0f, 6.0f}, {2}); + const Tensor expected_output0 = test::AsTensor({6.0f, 5.0f}, {2}); + const Tensor input1 = test::AsTensor({100.0f, 42.0f}, {2}); + const Tensor expected_output1 = test::AsTensor({53.0f, 24.0f}, {2}); + + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x", input0}, {"x2", input1}}, + {"y"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(1, outputs.size()); + test::ExpectTensorEqual(expected_output0, outputs[0]); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, + {"y3"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(1, outputs.size()); + test::ExpectTensorEqual(expected_output1, outputs[0]); + })); + std::unique_ptr third_request_thread( + Env::Default()->StartThread(ThreadOptions(), "third_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, + {"y"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(1, outputs.size()); + test::ExpectTensorEqual(expected_output0, outputs[0]); + })); +} + +INSTANTIATE_TEST_SUITE_P(Parameter, BatchingSessionTest, ::testing::Bool()); + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/batching/batching_util.cc b/tensorflow_serving/batching/batching_util.cc new file mode 100644 index 00000000000..bd6dad36b1f --- /dev/null +++ b/tensorflow_serving/batching/batching_util.cc @@ -0,0 +1,244 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/batching_util.h" + +#include +#include +#include +#include + +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/env_time.h" + +namespace tensorflow { +namespace serving { + +// Padding for one dimension of a tensor. +// pad_before is a number of values to add before elements in one dimension, +// pad_after is number of objects to add after. +// NOTE: fields are named this way because of Eigen::TensorMap::pad method. +// It requires padding to be an array of elements that have fields +// "first" and "second". +struct OneDimPadding { + int64_t first; // pad before + int64_t second; // pad after +}; + +// Constructs array of paddings, where: +// paddings[i].first - number of objects to add before elements in dimension i +// of given tensor, +// paddings[i].second - number of objects to add after elements in dimension i. +// This padding signature is used when performing internal padding with Eigen. +// +// When building paddings it is assumed that tensor needs to be padded +// after each dimension, so its shape matches max_dim_sizes, +// First entry of max_dim_sizes, which is maximum size of zeroth dimension, +// is ignored, because we don't perform padding in that dimension. +template +Eigen::array CreatePadding( + Tensor tensor, absl::Span max_dim_sizes) { + Eigen::array padding; + for (unsigned int i = 0; i < max_dim_sizes.size(); ++i) { + if (i > 0 && max_dim_sizes[i] - tensor.dim_size(i) > 0) { + padding[i] = {0, max_dim_sizes[i] - tensor.dim_size(i)}; + } else { + padding[i] = {0, 0}; + } + } + return padding; +} + +// Functor, which performs padding of given input tensor +// using specified padding signature. +// For example, given tensor of shape [1, 2, 3] and padding signature +// [[0, 0], [0, 2], [2, 2]] +// functor produces padded_tensor of shape [1, 4, 7]. +template +struct PadTensor { + absl::Status operator()(Tensor input, + const Eigen::array& padding, + Tensor* output) { + TensorShape output_shape; + for (int d = 0; d < num_dims; ++d) { + // Pad before existing elements. + const int32 before_d = padding[d].first; + // Pad after existing elements. + const int32 after_d = padding[d].second; + output_shape.AddDim(before_d + input.dim_size(d) + after_d); + } + if (output_shape.num_elements() == input.NumElements()) { + bool result = output->CopyFrom(input, output_shape); + if (!result) { + return errors::Internal("Couldn't create output."); + } + return absl::OkStatus(); + } + if (input.NumElements() < 1) { + return errors::InvalidArgument( + "Got empty tensor in batch of non-empty tensors."); + } + *output = Tensor(input.dtype(), output_shape); + typename TTypes::Tensor inputs = input.tensor(); + T pad_value(input.flat()(0)); // using existing values in padding + output->tensor() = inputs.pad(padding, pad_value); + return absl::OkStatus(); + } +}; + +// Invokes padding procedure for specific tensor ranks. +// Only ranks from 1 to 6 are supported (like in PadOp). +template +absl::Status PadTensorOfSpecificType(const Tensor& tensor, + absl::Span max_dim_sizes, + Tensor* output_tensor) { + int num_dims = tensor.dims(); + switch (num_dims) { + case 1: { + Eigen::array padding; + padding = CreatePadding<1>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 2: { + Eigen::array padding; + padding = CreatePadding<2>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 3: { + Eigen::array padding; + padding = CreatePadding<3>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 4: { + Eigen::array padding; + padding = CreatePadding<4>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 5: { + Eigen::array padding; + padding = CreatePadding<5>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 6: { + Eigen::array padding; + padding = CreatePadding<6>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + default: + // only ranks from 1 to 6 are supported + // (like in tensorflow/core/kernels/pad_op.cc) + return errors::InvalidArgument( + "Only tensors with rank from 1 to 6 can be padded."); + } +} + +std::map> CalculateMaxDimSizes( + const std::vector>>& batch) { + std::map> max_dim_sizes; + // Populate 'max_dim_sizes' + // init + const std::vector>& task_inputs = batch[0]; + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; + max_dim_sizes[tensor_name] = std::vector(tensor.dims(), 0); + } + // fill + for (int i = 0; i < batch.size(); ++i) { + const std::vector>& task_inputs = batch[i]; + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; + + std::vector& max_dim_sizes_for_one_input = + max_dim_sizes[tensor_name]; + for (int j = 0; j < tensor.dims(); ++j) { + const int old_max_size = max_dim_sizes_for_one_input[j]; + if (tensor.shape().dim_size(j) > old_max_size) { + max_dim_sizes_for_one_input[j] = tensor.shape().dim_size(j); + } + } + } + } + return max_dim_sizes; +} + +absl::Status AddPadding(const Tensor& tensor, + absl::Span max_dim_sizes, + Tensor* padded_tensor) { + const DataType input_dtype = tensor.dtype(); + absl::Status padding_status; +#define CASE(type) \ + case DataTypeToEnum::value: { \ + padding_status = \ + PadTensorOfSpecificType(tensor, max_dim_sizes, padded_tensor); \ + break; \ + } + switch (input_dtype) { + TF_CALL_ALL_TYPES(CASE); + TF_CALL_QUANTIZED_TYPES(CASE); + // quantized types macro doesn't include these types + TF_CALL_quint16(CASE); + TF_CALL_qint16(CASE); + default: + padding_status = errors::InvalidArgument("Unsupported type"); + } +#undef CASE + return padding_status; +} + +int RoundToLowestAllowedBatchSize(absl::Span allowed_batch_sizes, + int batch_size) { + if (allowed_batch_sizes.empty()) { + return batch_size; + } + for (int allowed_size : allowed_batch_sizes) { + if (allowed_size >= batch_size) { + return allowed_size; + } + } + // `allowed_batch_sizes` is guaranteed to be not empty and sorted in + // ascending order. + LOG(WARNING) << "Input batch size " << batch_size + << " is greater than largest allowed size " + << *allowed_batch_sizes.rbegin() + << " ignoring allowed sizes constraint."; + return batch_size; +} + +bool AreShapesEqualExceptZeroDim(const TensorShape& shape1, + const TensorShape& shape2) { + if (shape1.dims() != shape2.dims()) { + return false; + } + for (int i = 1; i < shape1.dims(); ++i) { + if (shape1.dim_size(i) != shape2.dim_size(i)) { + return false; + } + } + return true; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/batching_util.h b/tensorflow_serving/batching/batching_util.h new file mode 100644 index 00000000000..246d4cec6fa --- /dev/null +++ b/tensorflow_serving/batching/batching_util.h @@ -0,0 +1,157 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ +#define TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ + +#include +#include +#include + +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/monitoring/sampler.h" + +namespace tensorflow { +namespace serving { + +// For batch of inputs calculates maximum dim sizes across all tensors +// with the same name. +// These dim sizes are used later to calculate padding amount for each tensor. +// For example, for batch containing three tasks with the following inputs +// (instead of tensors there are their shapes): +// +// task1: {'tensor_a': [100, 500, 300], 'tensor_b': [100]} +// task2: {'tensor_a': [100, 200, 123], 'tensor_b': [100]} +// task3: {'tensor_a': [200, 100, 400], 'tensor_b': [200]} +// +// the following map will be generated: +// {'tensor_a': [200, 500, 400], 'tensor_b': [200]} +std::map> CalculateMaxDimSizes( + const std::vector>>& batch); + +// Pads tensor so that its shape becomes as specified in max_dim_sizes, +// except for zeroth dimension, which is left as is. +// First entry in max_dim_sizes is ignored. +// First element of a tensor is used as padding value. +// If tensor is empty, an error will be returned. +// +// For example given input tensor with shape [1, 2, 3, 4] and max_dim_sizes +// [1, 2, 5, 8] function produces padded_tensor of shape +// [1, 2, 5, 8], padded with tensor[0][0][0][0] element. +// +// Supported tensor datatypes: +// DT_FLOAT, DT_DOUBLE, DT_INT8, DT_UINT8, DT_INT16, +// DT_UINT16, DT_INT32, DT_INT64, DT_COMPLEX64, DT_COMPLEX128, +// DT_STRING, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, +// DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE. +// +// Supported tensor ranks: from 1 to 6. + +Status AddPadding(const Tensor& tensor, absl::Span max_dim_sizes, + Tensor* padded_tensor); + +// Returns the smallest entry in `allowed_batch_sizes` that is greater than or +// equal to `batch_size`. If `allowed_batch_sizes` is empty, simply returns +// `batch_size`. +int RoundToLowestAllowedBatchSize(absl::Span allowed_batch_sizes, + int batch_size); + +// Returns true iff all dims of shape1 are equal to dims of shape2 starting with +// the first (not zeroth) dimension. +// For example, for shapes [1, 2, 3] and [4, 2, 3] the result is true. +bool AreShapesEqualExceptZeroDim(const TensorShape& shape1, + const TensorShape& shape2); + +// Returns the first dimension size (batching dimension) of each tensor in +// `inputs`. If their first dimension sizes don't match, returns an error. +template +Status ComputeTensorBatchSize(TensorList inputs, size_t* size, DimFunc dim_func, + DimSizeFunc dim_size_func) { + if (inputs.empty()) { + return errors::InvalidArgument( + "Batching Run() must have at least one input tensor"); + } + + bool first = true; + for (const auto& tensor : inputs) { + if (dim_func(tensor) == 0) { + return errors::InvalidArgument( + "Batching Run() input tensors must have at least one " + "dimension"); + } + const size_t this_size = dim_size_func(tensor, 0); + + if (first) { + *size = this_size; + first = false; + } else { + if (this_size != *size) { + return errors::InvalidArgument( + "Batching Run() input tensors must have equal " + "0th-dimension size"); + } + } + } + return Status(); +} + +/***************** Below utilities are for monitoring purpose *****************/ + +// For all metrics: consider adding breakdowns based on model name or status if +// needed. Note that model name is not available as a session property or on any +// of the inputs currently. +template +void RecordPaddingSize(int32 padding_size, int32 execution_batch_size) { + static const std::string batching_task_name = BatchingTask::Name(); + static auto* cell = tensorflow::monitoring::Sampler<1>::New( + {absl::StrCat("/tensorflow/serving/", batching_task_name, + "/padding_size"), + "Tracks the padding size distribution on batches.", + "execution_batch_size"}, + // Exponential buckets [1*2^0, ..., 1*2^13, DBL_MAX]. + monitoring::Buckets::Exponential(1, 2, 14)); + cell->GetCell(absl::StrCat(execution_batch_size)) + ->Add(static_cast(padding_size)); +} + +template +void RecordInputBatchSize(int32 batch_size) { + static const std::string batching_task_name = BatchingTask::Name(); + static auto* cell = tensorflow::monitoring::Sampler<0>::New( + {absl::StrCat("/tensorflow/serving/", batching_task_name, + "/input_batch_size"), + "Tracks the batch size distribution on the inputs."}, + // Exponential buckets [1*2^0, ..., 1*2^13, DBL_MAX]. + monitoring::Buckets::Exponential(1, 2, 14)); + cell->GetCell()->Add(static_cast(batch_size)); +} + +template +void RecordProcessedBatchSize(int32 batch_size) { + static const std::string batching_task_name = BatchingTask::Name(); + static auto* cell = tensorflow::monitoring::Sampler<0>::New( + {absl::StrCat("/tensorflow/serving/", batching_task_name, + "/processed_batch_size"), + "Tracks the batch size distribution on processing."}, + // Exponential buckets [1*2^0, ..., 1*2^13, DBL_MAX]. + monitoring::Buckets::Exponential(1, 2, 14)); + cell->GetCell()->Add(static_cast(batch_size)); +} + +} // namespace serving +} // namespace tensorflow +#endif // TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ diff --git a/tensorflow_serving/batching/batching_util_test.cc b/tensorflow_serving/batching/batching_util_test.cc new file mode 100644 index 00000000000..73da395782d --- /dev/null +++ b/tensorflow_serving/batching/batching_util_test.cc @@ -0,0 +1,102 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/batching_util.h" + +#include +#include +#include +#include + +#include +#include +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::ElementsAre; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +// Creates vector of pairs (tensor_name, tensor_value), where tensors +// have shapes as specified in shapes. +// Tensor with shape shapes[i] has tensor_name "x" + std::to_string(i). +std::vector> CreateInputsWithTensorShapes( + const std::vector& shapes) { + std::vector> inputs; + for (int i = 0; i < shapes.size(); ++i) { + inputs.push_back({"x" + std::to_string(i), Tensor(DT_FLOAT, shapes[i])}); + } + return inputs; +} + +TEST(BatchingUtilTest, CalculateMaxDimSizes) { + const std::vector shapes1{{10, 20, 30}, {10, 100}}; + std::vector> inputs1 = + CreateInputsWithTensorShapes(shapes1); + const std::vector shapes2{{20, 50, 15}, {20, 101}}; + std::vector> inputs2 = + CreateInputsWithTensorShapes(shapes2); + std::vector>> batch{inputs1, inputs2}; + std::map> max_dim_sizes = + CalculateMaxDimSizes(batch); + EXPECT_THAT(max_dim_sizes, + UnorderedElementsAre(Pair("x0", ElementsAre(20, 50, 30)), + Pair("x1", ElementsAre(20, 101)))); +} + +TEST(BatchingUtilTest, AddPadding) { + const std::vector max_dim_sizes{20, 100, 200}; + const std::vector types{ + DT_FLOAT, DT_DOUBLE, DT_INT32, DT_UINT8, DT_INT16, + DT_UINT16, DT_INT8, DT_STRING, DT_BOOL, DT_COMPLEX64, + DT_COMPLEX128, DT_INT64, DT_QINT8, DT_QUINT8, DT_QINT16, + DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE}; + absl::Status padding_status; + for (DataType type : types) { + Tensor tensor(type, {10, 20, 30}); +#define INIT_TYPE(T) \ + if (type == DataTypeToEnum::value) { \ + tensor.flat().setConstant(T()); \ + } + TF_CALL_ALL_TYPES(INIT_TYPE); + TF_CALL_QUANTIZED_TYPES(INIT_TYPE); + // quantized types macro doesn't include these types + TF_CALL_quint16(INIT_TYPE); + TF_CALL_qint16(INIT_TYPE); +#undef INIT_TYPE + Tensor padded_tensor; + padding_status = AddPadding(tensor, max_dim_sizes, &padded_tensor); + ASSERT_EQ(absl::OkStatus(), padding_status); + EXPECT_EQ(TensorShape({10, 100, 200}), padded_tensor.shape()); + } +} + +TEST(BatchingUtilTest, AddPaddingTensorWithUnsupportedRank) { + const std::vector max_dim_sizes{1, 1, 1, 1, 1, 1, 1}; + const Tensor tensor(DT_FLOAT, {1, 1, 1, 1, 1, 1, 1}); + Tensor padded_tensor; + ASSERT_EQ(errors::InvalidArgument( + "Only tensors with rank from 1 to 6 can be padded."), + AddPadding(tensor, max_dim_sizes, &padded_tensor)); +} +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/incremental_barrier.cc b/tensorflow_serving/batching/incremental_barrier.cc new file mode 100644 index 00000000000..cdc623b14f1 --- /dev/null +++ b/tensorflow_serving/batching/incremental_barrier.cc @@ -0,0 +1,67 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/incremental_barrier.h" + +#include +#include +#include + +#include "absl/functional/bind_front.h" +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { +namespace serving { + +class InternalIncrementalBarrier { + public: + explicit InternalIncrementalBarrier(IncrementalBarrier::DoneCallback callback) + : left_(1), done_callback_(std::move(callback)) {} + + void operator()() { + DCHECK_GE(left_.load(std::memory_order_relaxed), 0); + + if (left_.fetch_sub(1, std::memory_order_acq_rel) - 1 == 0) { + IncrementalBarrier::DoneCallback done_callback = + std::move(done_callback_); + delete this; + done_callback(); + } + } + + IncrementalBarrier::BarrierCallback Inc() { + left_.fetch_add(1, std::memory_order_acq_rel); + + // std::bind_front is only available ever since C++20. + return absl::bind_front(&InternalIncrementalBarrier::operator(), this); + } + + private: + std::atomic left_; + IncrementalBarrier::DoneCallback done_callback_; +}; + +IncrementalBarrier::IncrementalBarrier(DoneCallback done_callback) + : internal_barrier_( + new InternalIncrementalBarrier(std::move(done_callback))) {} + +IncrementalBarrier::~IncrementalBarrier() { (*internal_barrier_)(); } + +IncrementalBarrier::BarrierCallback IncrementalBarrier::Inc() { + return internal_barrier_->Inc(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/incremental_barrier.h b/tensorflow_serving/batching/incremental_barrier.h new file mode 100644 index 00000000000..e7c09dc333b --- /dev/null +++ b/tensorflow_serving/batching/incremental_barrier.h @@ -0,0 +1,83 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_INCREMENTAL_BARRIER_H_ +#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_INCREMENTAL_BARRIER_H_ + +#include +#include + +namespace tensorflow { +namespace serving { + +class InternalIncrementalBarrier; + +// BarrierClosure (see +// https://github.com/chromium/chromium/blob/master/base/barrier_closure.h) +// executes a callback after it has been invoked |num_closures| times. +// Plus, `BarrierClosure` is a continuation-passing style abstraction and self- +// deleting. + +// IncrementalBarrier is a convenience class to be used in place of a barrier +// closure, which is particularly helpful (e.g. simplify code) because callers +// don't need to calculate the |num_closures| beforehand. +// +// Example Usage: +// void MakeCalls() { +// typedef std::function Callback; +// typedef std::function OtherCallback; +// Callback done_callback = ... +// OtherCallback cb1 = ... +// OtherCallback cb2 = ... +// std::thread threads[2]; +// { +// IncrementalBarrier barrier(done_callback); +// threads[0] = std::thread(cb1(barrier.Inc()); +// threads[1] = std::thread(cb2(barrier.Inc()); +// ... at this moment, `barrier` is incremented twice, and then +// destructed.... +// } +// threads[0].join(); +// threads[1].join(); +// } +// +// `done_callback` will be called when both conditions are true: +// 1) after `barrier` is destructed. +// 2) Each `BarrierCallback` returned by `Inc` is called. +// This class is thread-safe. +class IncrementalBarrier { + public: + typedef std::function DoneCallback; + typedef std::function BarrierCallback; + explicit IncrementalBarrier(DoneCallback callback); + + ~IncrementalBarrier(); + + // Returns a BarrierCallback (std::function) that individual task call to + // signal its completeness. + // The returned BarrierCallback outlives this `IncrementalBarrier` instance. + // Furthermore, each task should eventually call the returned function, or + // else done_callback wouldn't be called. + BarrierCallback Inc(); + + private: + // self-deleting, thereby not owned by 'IncrementalBarrier'. + InternalIncrementalBarrier* internal_barrier_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_INCREMENTAL_BARRIER_H_ diff --git a/tensorflow_serving/batching/incremental_barrier_test.cc b/tensorflow_serving/batching/incremental_barrier_test.cc new file mode 100644 index 00000000000..b58d38aec16 --- /dev/null +++ b/tensorflow_serving/batching/incremental_barrier_test.cc @@ -0,0 +1,135 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/incremental_barrier.h" + +#include + +#include "absl/functional/bind_front.h" +#include "absl/time/time.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/platform.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/platform/test_benchmark.h" +#include "tensorflow/core/platform/thread_annotations.h" +#include "tensorflow/core/platform/threadpool.h" + +namespace tensorflow { +namespace serving { +namespace { + +// A thread-safe counter class. +class Counter { + public: + void Increment() TF_LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + ++count_; + } + + int GetCount() TF_LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + return count_; + } + + private: + mutex mu_; + int count_ = 0; +}; + +TEST(IncrementalBarrierTest, RunInstantlyWhenZeroClosure) { + Counter counter; + EXPECT_EQ(counter.GetCount(), 0); + { + IncrementalBarrier::DoneCallback done_callback = + absl::bind_front(&Counter::Increment, &counter); + IncrementalBarrier barrier(done_callback); + EXPECT_EQ(counter.GetCount(), 0); + } + EXPECT_EQ(counter.GetCount(), 1); +} + +TEST(IncrementalBarrierTest, RunAfterNumClosuresOneNowTwoLater) { + Counter counter; + + IncrementalBarrier::BarrierCallback bc1, bc2; + { + IncrementalBarrier::DoneCallback done_callback = + absl::bind_front(&Counter::Increment, &counter); + IncrementalBarrier barrier(done_callback); + + CHECK_EQ(counter.GetCount(), 0); + + bc1 = barrier.Inc(); + bc2 = barrier.Inc(); + + IncrementalBarrier::BarrierCallback bc3 = barrier.Inc(); + bc3(); + + CHECK_EQ(counter.GetCount(), 0); + } + + CHECK_EQ(counter.GetCount(), 0); + bc1(); + CHECK_EQ(counter.GetCount(), 0); + bc2(); + CHECK_EQ(counter.GetCount(), 1); +} + +TEST(IncrementalBarrierTest, RunAfterNumClosuresConcurrency) { + const int num_closure = 100, num_thread = 2; + std::atomic schedule_count{0}; + Counter counter; + + { + IncrementalBarrier::DoneCallback done_callback = + absl::bind_front(&Counter::Increment, &counter); + IncrementalBarrier barrier(done_callback); + + CHECK_EQ(counter.GetCount(), 0); + + tensorflow::thread::ThreadPool pool(tensorflow::Env::Default(), + "BarrierClosure", num_thread); + for (int i = 0; i < num_closure; ++i) { + pool.Schedule([&barrier, &schedule_count]() { + schedule_count.fetch_add(1); + IncrementalBarrier::BarrierCallback bc = barrier.Inc(); + + Env::Default()->SleepForMicroseconds(100); + bc(); + }); + } + + CHECK_EQ(counter.GetCount(), 0); + } + + CHECK_EQ(schedule_count.load(std::memory_order_relaxed), 100); + CHECK_EQ(counter.GetCount(), 1); +} + +#if defined(PLATFORM_GOOGLE) +void BM_FunctionInc(benchmark::State& state) { + IncrementalBarrier barrier([] {}); + for (auto _ : state) { + barrier.Inc()(); + } +} + +BENCHMARK(BM_FunctionInc); +#endif // PLATFORM_GOOGLE + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/shared_batch_scheduler.h b/tensorflow_serving/batching/shared_batch_scheduler.h deleted file mode 100644 index 4be8aa12a96..00000000000 --- a/tensorflow_serving/batching/shared_batch_scheduler.h +++ /dev/null @@ -1,698 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ -#define TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ - -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/batching/batch_scheduler.h" -#include "tensorflow_serving/util/periodic_function.h" - -namespace tensorflow { -namespace serving { -namespace internal { -template -class Queue; -} // namespace internal -} // namespace serving -} // namespace tensorflow - -namespace tensorflow { -namespace serving { - -// A batch scheduler for server instances that service multiple request types -// (e.g. multiple machine-learned models, or multiple versions of a model served -// concurrently), or even multiple distinct tasks for a given request. The -// scheduler multiplexes batches of different kinds of tasks onto a fixed-size -// thread pool (each batch contains tasks of a single type), in a carefully -// controlled manner. A common configuration is to set the number of threads -// equal to the number of hardware accelerator units, in which case the -// scheduler takes care of multiplexing the task types onto the shared hardware, -// in a manner that is both fair and efficient. -// -// Semantically, SharedBatchScheduler behaves like having N instances of -// BasicBatchScheduler (see basic_batch_scheduler.h), one per task type. The -// difference is that under the covers there is a single shared thread pool, -// instead of N independent ones, with their sharing deliberately coordinated. -// -// SharedBatchScheduler does not implement the BatchScheduler API; rather, it -// presents an abstraction of "queues", where each queue coresponds to one type -// of task. Tasks submitted to a given queue are placed in their own batches, -// and cannot be mixed with other tasks. Queues can be added and deleted -// dynamically, to accommodate e.g. versions of a model being brought up and -// down over the lifetime of a server. -// -// The batch thread pool round-robins through the queues, running one batch -// from a queue and then moving to the next queue. Each queue behaves like a -// BasicBatchScheduler instance, in the sense that it has maximum batch size and -// timeout parameters, which govern when a batch is eligible to be processed. -// -// Each queue is independently configured with a maximum size (in terms of the -// maximum number of batches worth of enqueued tasks). For online serving, it is -// recommended that the queue sizes be configured such that the sum of the sizes -// of the active queues roughly equal the number of batch threads. (The idea is -// that if all threads become available at roughly the same time, there will be -// enough enqueued work for them to take on, but no more.) -// -// If queue sizes are configured in the manner suggested above, the maximum time -// a task can spend in a queue before being placed in a batch and assigned to a -// thread for processing, is the greater of: -// - the maximum time to process one batch of tasks from any active queue -// - the configured timeout parameter for the task's queue (which can be 0) -// -// For bulk processing jobs and throughput-oriented benchmarks, you may want to -// set the maximum queue size to a large value. -// -// TODO(b/26539183): Support queue servicing policies other than round-robin. -// E.g. let each queue specify a "share" (an int >= 1), so e.g. with queues A -// and B having shares 1 and 2 respectively, the servicing pattern is ABBABB... -// -template -class SharedBatchScheduler - : public std::enable_shared_from_this> { - public: - // TODO(b/25089730): Tune defaults based on best practices as they develop. - struct Options { - // The name to use for the pool of batch threads. - string thread_pool_name = {"batch_threads"}; - - // The number of threads to use to process batches. - // Must be >= 1, and should be tuned carefully. - int num_batch_threads = 1; - - // The environment to use. - // (Typically only overridden by test code.) - Env* env = Env::Default(); - }; - // Ownership is shared between the caller of Create() and any queues created - // via AddQueue(). - static Status Create( - const Options& options, - std::shared_ptr>* scheduler); - - ~SharedBatchScheduler(); - - // Adds a queue to which tasks may be submitted. The returned queue implements - // the BatchScheduler API. Each queue has its own set of scheduling options, - // and its own callback to process batches of tasks submitted to the queue. - // - // The returned queue's destructor blocks until all tasks submitted to it have - // been processed. - struct QueueOptions { - // The maximum size of each batch. - // - // The scheduler may form batches of any size between 1 and this number - // (inclusive). If there is a need to quantize the batch sizes, i.e. only - // submit batches whose size is in a small set of allowed sizes, that can be - // done by adding padding in the process-batch callback. - int max_batch_size = 1000; - - // If a task has been enqueued for this amount of time (in microseconds), - // and a thread is available, the scheduler will immediately form a batch - // from enqueued tasks and assign the batch to the thread for processing, - // even if the batch's size is below 'max_batch_size'. - // - // This parameter offers a way to bound queue latency, so that a task isn't - // stuck in the queue indefinitely waiting for enough tasks to arrive to - // make a full batch. (The latency bound is given in the class documentation - // above.) - // - // The goal is to smooth out batch sizes under low request rates, and thus - // avoid latency spikes. The default value of 1 millisecond was determined - // via benchmarking. You may need to adjust it to suit your workload and - // environment. - int64 batch_timeout_micros = 1 * 1000 /* 1 millisecond */; - - // The maximum allowable number of enqueued (accepted by Schedule() but - // not yet being processed on a batch thread) tasks in terms of batches. - // If this limit is reached, Schedule() will return an UNAVAILABLE error. - // See the class documentation above for guidelines on how to tune this - // parameter. - int max_enqueued_batches = 1; - }; - Status AddQueue(const QueueOptions& options, - std::function>)> - process_batch_callback, - std::unique_ptr>* queue); - - private: - explicit SharedBatchScheduler(const Options& options); - - // The code executed in 'batch_threads_'. Obtains a batch to process from the - // queue pointed to by 'next_queue_to_schedule_', and processes it. If that - // queue declines to provide a batch to process, moves onto the next queue. If - // no queues provide a batch to process, just sleeps briefly and exits. - void ThreadLogic(); - - const Options options_; - - mutex mu_; - - // A list of queues. (We use std::list instead of std::vector to ensure that - // iterators are not invalidated by adding/removing elements. It also offers - // efficient removal of elements from the middle.) - using QueueList = std::list>>; - - // All "active" queues, i.e. ones that either: - // - have not been removed, or - // - have been removed but are not yet empty. - QueueList queues_ GUARDED_BY(mu_); - - // An iterator over 'queues_', pointing to the queue from which the next - // available batch thread should grab work. - typename QueueList::iterator next_queue_to_schedule_ GUARDED_BY(mu_); - - // Used by idle batch threads to wait for work to enter the system. Notified - // whenever a batch becomes schedulable. - condition_variable schedulable_batch_cv_; - - // Threads that process batches obtained from the queues. - std::vector> batch_threads_; - - TF_DISALLOW_COPY_AND_ASSIGN(SharedBatchScheduler); -}; - -////////// -// Implementation details follow. API users need not read. - -namespace internal { - -// A task queue for SharedBatchScheduler. Accepts tasks and accumulates them -// into batches, and dispenses those batches to be processed via a "pull" -// interface. The queue's behavior is governed by maximum batch size, timeout -// and maximum queue length parameters; see their documentation in -// SharedBatchScheduler. -// -// The queue is implemented as a deque of batches, with these invariants: -// - The number of batches is between 1 and 'options_.max_enqueued_batches'. -// - The back-most batch is open; the rest are closed. -// -// Submitted tasks are added to the open batch. If that batch doesn't have room -// but the queue isn't full, then that batch is closed and a new open batch is -// started. -// -// Batch pull requests are handled by dequeuing the front-most batch if it is -// closed. If the front-most batch is open (i.e. the queue contains only one -// batch) and has reached the timeout, it is immediately closed and returned; -// otherwise no batch is returned for the request. -template -class Queue { - public: - using ProcessBatchCallback = - std::function>)>; - using SchedulableBatchCallback = std::function; - Queue(const typename SharedBatchScheduler::QueueOptions& options, - Env* env, ProcessBatchCallback process_batch_callback, - SchedulableBatchCallback schdulable_batch_callback); - - // Illegal to destruct unless the queue is empty. - ~Queue(); - - // Submits a task to the queue, with the same semantics as - // BatchScheduler::Schedule(). - Status Schedule(std::unique_ptr* task); - - // Returns the number of enqueued tasks, with the same semantics as - // BatchScheduler::NumEnqueuedTasks(). - size_t NumEnqueuedTasks() const; - - // Returns the queue capacity, with the same semantics as - // BatchScheduler::SchedulingCapacity(). - size_t SchedulingCapacity() const; - - // Called by a thread that is ready to process a batch, to request one from - // this queue. Either returns a batch that is ready to be processed, or - // nullptr if the queue declines to schedule a batch at this time. If it - // returns a batch, the batch is guaranteed to be closed. - std::unique_ptr> ScheduleBatch(); - - // Processes a batch that has been returned earlier by ScheduleBatch(). - void ProcessBatch(std::unique_ptr> batch); - - // Determines whether the queue is empty, i.e. has no tasks waiting or being - // processed. - bool IsEmpty() const; - - // Marks the queue closed, and waits until it is empty. - void CloseAndWaitUntilEmpty(); - - bool closed() const { - mutex_lock l(mu_); - return closed_; - } - - private: - // Same as IsEmpty(), but assumes the caller already holds a lock on 'mu_'. - bool IsEmptyInternal() const EXCLUSIVE_LOCKS_REQUIRED(mu_); - - // Closes the open batch residing at the back of 'batches_', and inserts a - // fresh open batch behind it. - void StartNewBatch() EXCLUSIVE_LOCKS_REQUIRED(mu_); - - // Determines whether the open batch residing at the back of 'batches_' is - // currently schedulable. - bool IsOpenBatchSchedulable() const EXCLUSIVE_LOCKS_REQUIRED(mu_); - - const typename SharedBatchScheduler::QueueOptions options_; - - // The environment to use. - Env* env_; - - // A callback invoked to processes a batch of work units. Always invoked from - // a batch thread. - ProcessBatchCallback process_batch_callback_; - - // A callback invoked to notify the scheduler that a new batch has become - // schedulable. - SchedulableBatchCallback schedulable_batch_callback_; - - mutable mutex mu_; - - // Whether this queue can accept new tasks. This variable is monotonic: it - // starts as false, and then at some point gets set to true and remains true - // for the duration of this object's life. - bool closed_ GUARDED_BY(mu_) = false; - - // The enqueued batches. See the invariants in the class comments above. - std::deque>> batches_ GUARDED_BY(mu_); - - // The time at which the first task was added to the open (back-most) batch - // in 'batches_'. Valid iff that batch contains at least one task. - uint64 open_batch_start_time_micros_ GUARDED_BY(mu_); - - // Whether this queue contains a batch that is eligible to be scheduled. Used - // to keep track of when to call 'schedulable_batch_callback_'. - bool schedulable_batch_ GUARDED_BY(mu_) = false; - - // The number of batches currently being processed by batch threads. - // Incremented in ScheduleBatch() and decremented in ProcessBatch(). - int num_batches_being_processed_ GUARDED_BY(mu_) = 0; - - // Used by CloseAndWaitUntilEmpty() to wait until the queue is empty, for the - // case in which the queue is not empty when CloseAndWaitUntilEmpty() starts. - // When ProcessBatch() dequeues the last batch and makes the queue empty, if - // 'empty_notification_' is non-null it calls 'empty_notification_->Notify()'. - Notification* empty_notification_ GUARDED_BY(mu_) = nullptr; - - TF_DISALLOW_COPY_AND_ASSIGN(Queue); -}; - -// A RAII-style object that points to a Queue and implements -// the BatchScheduler API. To be handed out to clients who call AddQueue(). -template -class QueueHandle : public BatchScheduler { - public: - QueueHandle(std::shared_ptr> scheduler, - Queue* queue); - ~QueueHandle() override; - - Status Schedule(std::unique_ptr* task) override; - size_t NumEnqueuedTasks() const override; - size_t SchedulingCapacity() const override; - - private: - // The scheduler that owns 'queue_'. - std::shared_ptr> scheduler_; - - // The queue this handle wraps. Owned by 'scheduler_', which keeps it alive at - // least until this class's destructor closes it. - Queue* queue_; - - TF_DISALLOW_COPY_AND_ASSIGN(QueueHandle); -}; - -} // namespace internal - -template -Status SharedBatchScheduler::Create( - const Options& options, - std::shared_ptr>* scheduler) { - if (options.num_batch_threads < 1) { - return errors::InvalidArgument("num_batch_threads must be positive; was ", - options.num_batch_threads); - } - scheduler->reset(new SharedBatchScheduler(options)); - return Status::OK(); -} - -template -SharedBatchScheduler::~SharedBatchScheduler() { - // Wait until the batch threads finish clearing out and deleting the closed - // queues. - for (;;) { - { - mutex_lock l(mu_); - if (queues_.empty()) { - break; - } - } - const int64 kSleepTimeMicros = 100; - options_.env->SleepForMicroseconds(kSleepTimeMicros); - } - // Delete the batch threads before allowing state the threads may access (e.g. - // 'mu_') to be deleted. - batch_threads_.clear(); -} - -template -Status SharedBatchScheduler::AddQueue( - const QueueOptions& options, - std::function>)> - process_batch_callback, - std::unique_ptr>* queue) { - if (options.max_batch_size <= 0) { - return errors::InvalidArgument("max_batch_size must be positive; was ", - options.max_batch_size); - } - if (options.batch_timeout_micros < 0) { - return errors::InvalidArgument( - "batch_timeout_micros must be non-negative; was ", - options.batch_timeout_micros); - } - if (options.max_enqueued_batches < 0) { - return errors::InvalidArgument( - "max_enqueued_batches must be non-negative; was ", - options.max_enqueued_batches); - } - - auto schedulable_batch_callback = [this] { - mutex_lock l(mu_); - schedulable_batch_cv_.notify_one(); - }; - auto internal_queue = - std::unique_ptr>(new internal::Queue( - options, options_.env, process_batch_callback, - schedulable_batch_callback)); - auto handle = std::unique_ptr>( - new internal::QueueHandle(this->shared_from_this(), - internal_queue.get())); - { - mutex_lock l(mu_); - queues_.push_back(std::move(internal_queue)); - if (next_queue_to_schedule_ == queues_.end()) { - next_queue_to_schedule_ = queues_.begin(); - } - } - *queue = std::move(handle); - return Status::OK(); -} - -template -SharedBatchScheduler::SharedBatchScheduler(const Options& options) - : options_(options), next_queue_to_schedule_(queues_.end()) { - // Kick off the batch threads. - PeriodicFunction::Options periodic_fn_options; - periodic_fn_options.thread_name_prefix = - strings::StrCat(options.thread_pool_name, "_"); - for (int i = 0; i < options.num_batch_threads; ++i) { - std::unique_ptr thread(new PeriodicFunction( - [this] { this->ThreadLogic(); }, - 0 /* function invocation interval time */, periodic_fn_options)); - batch_threads_.push_back(std::move(thread)); - } -} - -template -void SharedBatchScheduler::ThreadLogic() { - // A batch to process next (or nullptr if no work to do). - std::unique_ptr> batch_to_process; - // The queue with which 'batch_to_process' is associated. - internal::Queue* queue_for_batch = nullptr; - { - mutex_lock l(mu_); - - const int num_queues = queues_.size(); - for (int num_queues_tried = 0; - batch_to_process == nullptr && num_queues_tried < num_queues; - ++num_queues_tried) { - DCHECK(next_queue_to_schedule_ != queues_.end()); - - // If a closed queue responds to ScheduleBatch() with nullptr, the queue - // will never yield any further batches so we can drop it. To avoid a - // race, we take a snapshot of the queue's closedness state *before* - // calling ScheduleBatch(). - const bool queue_closed = (*next_queue_to_schedule_)->closed(); - - // Ask '*next_queue_to_schedule_' if it wants us to process a batch. - batch_to_process = (*next_queue_to_schedule_)->ScheduleBatch(); - if (batch_to_process != nullptr) { - queue_for_batch = next_queue_to_schedule_->get(); - } - - // Advance 'next_queue_to_schedule_'. - if (queue_closed && (*next_queue_to_schedule_)->IsEmpty() && - batch_to_process == nullptr) { - // We've encountered a closed queue with no work to do. Drop it. - DCHECK_NE(queue_for_batch, next_queue_to_schedule_->get()); - next_queue_to_schedule_ = queues_.erase(next_queue_to_schedule_); - } else { - ++next_queue_to_schedule_; - } - if (next_queue_to_schedule_ == queues_.end() && !queues_.empty()) { - // We've hit the end. Wrap to the first queue. - next_queue_to_schedule_ = queues_.begin(); - } - } - - if (batch_to_process == nullptr) { - // We couldn't find any work to do. Wait until a new batch becomes - // schedulable, or some time has elapsed, before checking again. - const int64 kTimeoutMillis = 1; // The smallest accepted granule of time. - WaitForMilliseconds(&l, &schedulable_batch_cv_, kTimeoutMillis); - return; - } - } - - queue_for_batch->ProcessBatch(std::move(batch_to_process)); -} - -namespace internal { - -template -Queue::Queue( - const typename SharedBatchScheduler::QueueOptions& options, - Env* env, ProcessBatchCallback process_batch_callback, - SchedulableBatchCallback schedulable_batch_callback) - : options_(options), - env_(env), - process_batch_callback_(process_batch_callback), - schedulable_batch_callback_(schedulable_batch_callback) { - // Create an initial, open batch. - batches_.emplace_back(new Batch); -} - -template -Queue::~Queue() { - mutex_lock l(mu_); - DCHECK(IsEmptyInternal()); - - // Close the (empty) open batch, so its destructor doesn't block. - batches_.back()->Close(); -} - -template -Status Queue::Schedule(std::unique_ptr* task) { - if ((*task)->size() > options_.max_batch_size) { - return errors::InvalidArgument("Task size ", (*task)->size(), - " is larger than maximum batch size ", - options_.max_batch_size); - } - - bool notify_of_schedulable_batch = false; - { - mutex_lock l(mu_); - - DCHECK(!closed_); - - if (batches_.back()->size() + (*task)->size() > options_.max_batch_size) { - if (batches_.size() >= options_.max_enqueued_batches) { - return errors::Unavailable( - "The batch scheduling queue to which this task was submitted is " - "full"); - } - StartNewBatch(); - } - if (batches_.back()->empty()) { - open_batch_start_time_micros_ = env_->NowMicros(); - } - batches_.back()->AddTask(std::move(*task)); - - if (!schedulable_batch_) { - if (batches_.size() > 1 || IsOpenBatchSchedulable()) { - schedulable_batch_ = true; - notify_of_schedulable_batch = true; - } - } - } - - if (notify_of_schedulable_batch) { - schedulable_batch_callback_(); - } - - return Status::OK(); -} - -template -size_t Queue::NumEnqueuedTasks() const { - mutex_lock l(mu_); - size_t num_enqueued_tasks = 0; - for (const auto& batch : batches_) { - num_enqueued_tasks += batch->num_tasks(); - } - return num_enqueued_tasks; -} - -template -size_t Queue::SchedulingCapacity() const { - mutex_lock l(mu_); - const int num_new_batches_schedulable = - options_.max_enqueued_batches - batches_.size(); - const int open_batch_capacity = - options_.max_batch_size - batches_.back()->size(); - return (num_new_batches_schedulable * options_.max_batch_size) + - open_batch_capacity; -} - -template -std::unique_ptr> Queue::ScheduleBatch() { - // The batch to schedule, which we may populate below. (If left as nullptr, - // that means we are electing not to schedule a batch at this time.) - std::unique_ptr> batch_to_schedule; - - { - mutex_lock l(mu_); - - // Consider closing the open batch at this time, to schedule it. - if (batches_.size() == 1 && IsOpenBatchSchedulable()) { - StartNewBatch(); - } - - if (batches_.size() >= 2) { - // There is at least one closed batch that is ready to be scheduled. - ++num_batches_being_processed_; - batch_to_schedule = std::move(batches_.front()); - batches_.pop_front(); - } else { - schedulable_batch_ = false; - } - } - - return batch_to_schedule; -} - -template -void Queue::ProcessBatch(std::unique_ptr> batch) { - process_batch_callback_(std::move(batch)); - - { - mutex_lock l(mu_); - --num_batches_being_processed_; - if (empty_notification_ != nullptr && IsEmptyInternal()) { - empty_notification_->Notify(); - } - } -} - -template -bool Queue::IsEmpty() const { - mutex_lock l(mu_); - return IsEmptyInternal(); -} - -template -void Queue::CloseAndWaitUntilEmpty() { - Notification empty; - { - mutex_lock l(mu_); - closed_ = true; - if (IsEmptyInternal()) { - empty.Notify(); - } else { - // Arrange for ProcessBatch() to notify when the queue becomes empty. - empty_notification_ = ∅ - } - } - empty.WaitForNotification(); -} - -template -bool Queue::IsEmptyInternal() const { - return num_batches_being_processed_ == 0 && batches_.size() == 1 && - batches_.back()->empty(); -} - -template -void Queue::StartNewBatch() { - batches_.back()->Close(); - batches_.emplace_back(new Batch); -} - -template -bool Queue::IsOpenBatchSchedulable() const { - Batch* open_batch = batches_.back().get(); - if (open_batch->empty()) { - return false; - } - return closed_ || open_batch->size() >= options_.max_batch_size || - env_->NowMicros() >= - open_batch_start_time_micros_ + options_.batch_timeout_micros; -} - -template -QueueHandle::QueueHandle( - std::shared_ptr> scheduler, - Queue* queue) - : scheduler_(scheduler), queue_(queue) {} - -template -QueueHandle::~QueueHandle() { - queue_->CloseAndWaitUntilEmpty(); -} - -template -Status QueueHandle::Schedule(std::unique_ptr* task) { - return queue_->Schedule(task); -} - -template -size_t QueueHandle::NumEnqueuedTasks() const { - return queue_->NumEnqueuedTasks(); -} - -template -size_t QueueHandle::SchedulingCapacity() const { - return queue_->SchedulingCapacity(); -} - -} // namespace internal - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/shared_batch_scheduler_test.cc b/tensorflow_serving/batching/shared_batch_scheduler_test.cc deleted file mode 100644 index be236818906..00000000000 --- a/tensorflow_serving/batching/shared_batch_scheduler_test.cc +++ /dev/null @@ -1,593 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/batching/shared_batch_scheduler.h" - -#include -#include -#include "tensorflow/core/lib/core/error_codes.pb.h" -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" - -using ::testing::ElementsAre; -using ::testing::IsEmpty; -using ::testing::UnorderedElementsAre; - -namespace tensorflow { -namespace serving { -namespace { - -class FakeTask : public BatchTask { - public: - explicit FakeTask(size_t size) : size_(size) {} - - ~FakeTask() override = default; - - size_t size() const override { return size_; } - - private: - const size_t size_; - - TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); -}; - -// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on -// that task. Returns the resulting status. -Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { - std::unique_ptr task(new FakeTask(task_size)); - Status status = scheduler->Schedule(&task); - // Schedule() should have consumed 'task' iff it returned Status::OK. - CHECK_EQ(status.ok(), task == nullptr); - return status; -} - -// Creates a thread that waits on 'start' and then advances the fake clock in -// 'env' in a loop until 'stop' is notified. Useful for allowing objects that -// use the clock to be destroyed. -std::unique_ptr CreateFakeClockAdvancerThread( - test_util::FakeClockEnv* env, Notification* start, Notification* stop) { - return std::unique_ptr( - Env::Default()->StartThread({}, "FakeClockAdvancerThread", - [env, start, stop] { - start->WaitForNotification(); - while (!stop->HasBeenNotified()) { - env->AdvanceByMicroseconds(10); - Env::Default()->SleepForMicroseconds(10); - } - })); -} - -TEST(SharedBatchSchedulerTest, Basic) { - for (int num_batch_threads : {1, 2, 3}) { - for (const bool delete_scheduler_early : {false, true}) { - for (const bool delete_queue_1_early : {false, true}) { - bool queue_0_callback_called = false; - auto queue_0_callback = - [&queue_0_callback_called](std::unique_ptr> batch) { - queue_0_callback_called = true; - ASSERT_TRUE(batch->IsClosed()); - ASSERT_EQ(3, batch->num_tasks()); - EXPECT_EQ(1, batch->task(0).size()); - EXPECT_EQ(3, batch->task(1).size()); - EXPECT_EQ(5, batch->task(2).size()); - }; - bool queue_1_callback_called = false; - auto queue_1_callback = - [&queue_1_callback_called](std::unique_ptr> batch) { - queue_1_callback_called = true; - ASSERT_TRUE(batch->IsClosed()); - ASSERT_EQ(2, batch->num_tasks()); - EXPECT_EQ(2, batch->task(0).size()); - EXPECT_EQ(4, batch->task(1).size()); - }; - { - SharedBatchScheduler::Options options; - options.num_batch_threads = num_batch_threads; - std::shared_ptr> scheduler; - TF_ASSERT_OK( - SharedBatchScheduler::Create(options, &scheduler)); - - // Create two queues. - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 10 * 1000 * 1000; // 10 seconds - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue_0; - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); - std::unique_ptr> queue_1; - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); - - if (delete_scheduler_early) { - // Delete our copy of the scheduler. The queues should keep it alive - // under the covers. - scheduler = nullptr; - } - - // Submit tasks to the two queues, and (optionally) remove the queues. - TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); - TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); - TF_ASSERT_OK(ScheduleTask(3, queue_0.get())); - TF_ASSERT_OK(ScheduleTask(4, queue_1.get())); - if (delete_queue_1_early) { - queue_1 = nullptr; - } - TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); - } - EXPECT_TRUE(queue_0_callback_called); - EXPECT_TRUE(queue_1_callback_called); - } - } - } -} - -TEST(SharedBatchSchedulerTest, ObeyBatchSizeConstraint) { - // Set up a callback that captures the batches' task sizes. - mutex mu; - std::vector> callback_data; - auto callback = [&mu, - &callback_data](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - std::vector batch_data; - for (int i = 0; i < batch->num_tasks(); ++i) { - batch_data.push_back(batch->mutable_task(i)->size()); - } - { - mutex_lock l(mu); - callback_data.push_back(batch_data); - } - }; - - // Run a batch scheduler and inject some tasks. - { - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 10 * 1000 * 1000; // 10 seconds - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // First batch. - TF_ASSERT_OK(ScheduleTask(3, queue.get())); - TF_ASSERT_OK(ScheduleTask(5, queue.get())); - - // Second batch (due to size overage). - TF_ASSERT_OK(ScheduleTask(3 /* (3+5) + 3 > 10 */, queue.get())); - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - TF_ASSERT_OK(ScheduleTask(6, queue.get())); - - // (Empty third batch, since the second batch exactly hit the size limit, - // which should never get sent to the callback.) - } - - // Expect a certain grouping of the tasks into batches. - EXPECT_THAT(callback_data, - UnorderedElementsAre(ElementsAre(3, 5), ElementsAre(3, 1, 6))); -} - -TEST(SharedBatchSchedulerTest, ObeysTimeout) { - // Set up a fake clock, which only advances when we explicitly tell it to. - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - Notification first_batch_processed, second_batch_processed, - third_batch_processed; - auto callback = - [&first_batch_processed, &second_batch_processed, - &third_batch_processed](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - if (batch->size() == 1) { - first_batch_processed.Notify(); - } else if (batch->size() == 2) { - second_batch_processed.Notify(); - } else if (batch->size() == 3) { - third_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 4; - queue_options.batch_timeout_micros = 10; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // Create an underfull batch, and ensure that it gets processed when the - // clock hits the timeout. - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - env.AdvanceByMicroseconds(9); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(first_batch_processed.HasBeenNotified()); - env.AdvanceByMicroseconds(1); - first_batch_processed.WaitForNotification(); - - // Start creating a batch, while leaving the clock well below the timeout. - // Then submit a new task that overflows into the next batch, causing - // the original batch to close. - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(second_batch_processed.HasBeenNotified()); - TF_ASSERT_OK(ScheduleTask(3, queue.get())); - second_batch_processed.WaitForNotification(); - - // Allow the third batch to hit its timeout, and ensure it gets closed at - // the right time. - env.AdvanceByMicroseconds(9); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(third_batch_processed.HasBeenNotified()); - env.AdvanceByMicroseconds(1); - third_batch_processed.WaitForNotification(); - - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(SharedBatchSchedulerTest, ObeysTimeoutWithRealClock) { - Notification first_batch_processed, second_batch_processed; - auto callback = [&first_batch_processed, &second_batch_processed]( - std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - if (batch->size() == 1) { - first_batch_processed.Notify(); - } else if (batch->size() == 2) { - second_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 100 * 1000; // 100 milliseconds - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // Submit a single task that doesn't fill up the batch. - // Ensure that it gets processed due to the timeout. - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - first_batch_processed.WaitForNotification(); - - // Do it again. - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - second_batch_processed.WaitForNotification(); -} - -TEST(SharedBatchSchedulerTest, - WithZeroTimeoutBatchesScheduledAsSoonAsThreadIsAvailable) { - // Set up a fake clock, and never advance the time. - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - Notification first_batch_processed, second_batch_processed; - auto callback = [&first_batch_processed, &second_batch_processed]( - std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - if (batch->size() == 1) { - first_batch_processed.Notify(); - } else if (batch->size() == 2) { - second_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - // Set a large batch size, so that we don't hit the batch size limit. - queue_options.max_batch_size = 100; - // Process a batch as soon as a thread is available. - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - first_batch_processed.WaitForNotification(); - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - second_batch_processed.WaitForNotification(); - - // Shut everything down. - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(SharedBatchSchedulerTest, Fairness) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - Notification queue_0_first_batch_scheduled, queue_0_first_batch_proceed, - queue_0_second_batch_scheduled; - auto queue_0_callback = [&queue_0_first_batch_scheduled, - &queue_0_first_batch_proceed, - &queue_0_second_batch_scheduled]( - std::unique_ptr> batch) { - if (!queue_0_first_batch_scheduled.HasBeenNotified()) { - queue_0_first_batch_scheduled.Notify(); - queue_0_first_batch_proceed.WaitForNotification(); - } else if (!queue_0_second_batch_scheduled.HasBeenNotified()) { - queue_0_second_batch_scheduled.Notify(); - } - }; - - Notification queue_1_first_batch_scheduled, queue_1_first_batch_proceed; - auto queue_1_callback = - [&queue_1_first_batch_scheduled, - &queue_1_first_batch_proceed](std::unique_ptr> batch) { - queue_1_first_batch_scheduled.Notify(); - queue_1_first_batch_proceed.WaitForNotification(); - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 1; - queue_options.max_enqueued_batches = 100 /* give plenty of room */; - std::vector>> queues(2); - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_0_callback, &queues[0])); - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_1_callback, &queues[1])); - - // Enqueue a batch-filling task to queue 0, and wait for it to get - // scheduled. - TF_ASSERT_OK(ScheduleTask(10, queues[0].get())); - env.AdvanceByMicroseconds(1); - queue_0_first_batch_scheduled.WaitForNotification(); - - // Enqueue two more batch-filling tasks to queue 0. - TF_ASSERT_OK(ScheduleTask(10, queues[0].get())); - TF_ASSERT_OK(ScheduleTask(10, queues[0].get())); - - // Enqueue one task to queue 1, and then advance the clock so it becomes - // eligible for scheduling due to the timeout. Ensure that the queue 1 batch - // gets scheduled before the next queue 0 one. - TF_ASSERT_OK(ScheduleTask(1, queues[1].get())); - env.AdvanceByMicroseconds(1); - queue_0_first_batch_proceed.Notify(); - queue_1_first_batch_scheduled.WaitForNotification(); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(queue_0_second_batch_scheduled.HasBeenNotified()); - - // Shut everything down. - queue_1_first_batch_proceed.Notify(); - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(SharedBatchSchedulerTest, ConstMethods) { - for (const int max_enqueued_batches : {1, 2, 5}) { - Notification processing, proceed; - auto callback = [&processing, - &proceed](std::unique_ptr> batch) { - if (!processing.HasBeenNotified()) { - processing.Notify(); - } - proceed.WaitForNotification(); - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 2; - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = max_enqueued_batches; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - EXPECT_EQ(0, queue->NumEnqueuedTasks()); - EXPECT_EQ(max_enqueued_batches * 2, queue->SchedulingCapacity()); - - // Get one batch going on the thread, and keep the thread blocked until - // we're done testing the maximum queue length. - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - processing.WaitForNotification(); - EXPECT_EQ(0, queue->NumEnqueuedTasks()); - - // We should be able to enqueue 'max_enqueued_batches'*2 tasks without - // issue. - for (int i = 0; i < max_enqueued_batches; ++i) { - EXPECT_EQ(i * 2, queue->NumEnqueuedTasks()); - EXPECT_EQ((max_enqueued_batches - i) * 2, queue->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - EXPECT_EQ((i * 2) + 1, queue->NumEnqueuedTasks()); - EXPECT_EQ((max_enqueued_batches - i) * 2 - 1, - queue->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - } - EXPECT_EQ(max_enqueued_batches * 2, queue->NumEnqueuedTasks()); - EXPECT_EQ(0, queue->SchedulingCapacity()); - - // Attempting to enqueue one more task should yield an UNAVAILABLE error. - Status status = ScheduleTask(1, queue.get()); - ASSERT_FALSE(status.ok()); - EXPECT_EQ(error::UNAVAILABLE, status.code()); - EXPECT_EQ(max_enqueued_batches * 2, queue->NumEnqueuedTasks()); - EXPECT_EQ(0, queue->SchedulingCapacity()); - - proceed.Notify(); - } -} - -TEST(SharedBatchSchedulerTest, OneFullQueueDoesntBlockOtherQueues) { - Notification queue_0_processing, queue_0_proceed; - auto queue_0_callback = [&queue_0_processing, &queue_0_proceed]( - std::unique_ptr> batch) { - if (!queue_0_processing.HasBeenNotified()) { - queue_0_processing.Notify(); - queue_0_proceed.WaitForNotification(); - } - }; - - Notification queue_1_first_batch_processed, queue_1_second_batch_processed, - queue_1_third_batch_processed; - auto queue_1_callback = - [&queue_1_first_batch_processed, &queue_1_second_batch_processed, - &queue_1_third_batch_processed](std::unique_ptr> batch) { - if (batch->size() == 1) { - queue_1_first_batch_processed.Notify(); - } else if (batch->size() == 2) { - queue_1_second_batch_processed.Notify(); - } else if (batch->size() == 3) { - queue_1_third_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue_0; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); - std::unique_ptr> queue_1; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); - - // Clog up queue 0. - TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); - queue_0_processing.WaitForNotification(); - Status queue_0_status; - do { - queue_0_status = ScheduleTask(1, queue_0.get()); - } while (queue_0_status.ok()); - EXPECT_EQ(error::UNAVAILABLE, queue_0_status.code()); - - // Ensure that queue 1 still behaves normally, and lets us process tasks. - TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); - queue_1_first_batch_processed.WaitForNotification(); - TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); - queue_1_second_batch_processed.WaitForNotification(); - TF_ASSERT_OK(ScheduleTask(3, queue_1.get())); - queue_1_third_batch_processed.WaitForNotification(); - - // Let poor queue 0 drain. - queue_0_proceed.Notify(); -} - -TEST(SharedBatchSchedulerTest, QueueDestructorBlocksUntilAllTasksProcessed) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - int current_batch = 0; - Notification first_callback_started; - const int kMaxEnqueuedBatches = 3; - std::vector callback_proceed(kMaxEnqueuedBatches); - auto callback = - [¤t_batch, &first_callback_started, - &callback_proceed](std::unique_ptr> batch) { - if (current_batch == 0) { - first_callback_started.Notify(); - } - callback_proceed[current_batch].WaitForNotification(); - ++current_batch; - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // Clog up the queue. - int num_enqueued_batches = 0; - TF_ASSERT_OK(ScheduleTask(10, queue.get())); - ++num_enqueued_batches; - env.AdvanceByMicroseconds(1); - first_callback_started.WaitForNotification(); - for (int i = 0; i < 2; ++i) { - TF_ASSERT_OK(ScheduleTask(10, queue.get())); - ++num_enqueued_batches; - } - EXPECT_EQ(kMaxEnqueuedBatches, num_enqueued_batches); - EXPECT_EQ(error::UNAVAILABLE, ScheduleTask(10, queue.get()).code()); - - // Destroy the queue. The destructor should block until all tasks have been - // processed. - Notification destroy_queue_thread_started, queue_destroyed; - std::unique_ptr destroy_queue_thread(Env::Default()->StartThread( - {}, "DestroyQueueThread", - [&queue, &destroy_queue_thread_started, &queue_destroyed] { - destroy_queue_thread_started.Notify(); - queue = nullptr; - queue_destroyed.Notify(); - })); - destroy_queue_thread_started.WaitForNotification(); - for (int i = 0; i < num_enqueued_batches; ++i) { - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(queue_destroyed.HasBeenNotified()); - callback_proceed[i].Notify(); - } - - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/batching/streaming_batch_scheduler.cc b/tensorflow_serving/batching/streaming_batch_scheduler.cc index 8cb0a45e843..6e2060a3d29 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler.cc +++ b/tensorflow_serving/batching/streaming_batch_scheduler.cc @@ -15,6 +15,11 @@ limitations under the License. #include "tensorflow_serving/batching/streaming_batch_scheduler.h" +#include +#include + +#include "absl/types/optional.h" + namespace tensorflow { namespace serving { @@ -23,21 +28,21 @@ namespace internal { // SingleTaskScheduler SingleTaskScheduler::SingleTaskScheduler(Env* env, string thread_name, - uint64 no_tasks_wait_time_micros) + uint64_t no_tasks_wait_time_micros) : env_(env), - thread_name_(thread_name), + thread_name_(std::move(thread_name)), no_tasks_wait_time_micros_(no_tasks_wait_time_micros) {} SingleTaskScheduler::~SingleTaskScheduler() { stop_.Notify(); } -void SingleTaskScheduler::Schedule(uint64 time_micros, +void SingleTaskScheduler::Schedule(uint64_t time_micros, std::function closure) { DCHECK_GE(time_micros, last_task_time_); last_task_time_ = time_micros; { mutex_lock l(mu_); - updated_task_ = {time_micros, closure}; + updated_task_ = {time_micros, std::move(closure)}; } if (thread_ == nullptr) { @@ -48,11 +53,11 @@ void SingleTaskScheduler::Schedule(uint64 time_micros, } void SingleTaskScheduler::ThreadLogic() { - optional current_task = nullopt; + absl::optional current_task = absl::nullopt; for (;;) { // Sleep until the time specified in the current task, if any. if (current_task) { - const uint64 now = env_->NowMicros(); + const uint64_t now = env_->NowMicros(); if (current_task->time_micros > now) { env_->SleepForMicroseconds(current_task->time_micros - now); } @@ -63,7 +68,7 @@ void SingleTaskScheduler::ThreadLogic() { mutex_lock l(mu_); if (updated_task_) { current_task = updated_task_; - updated_task_ = nullopt; + updated_task_ = absl::nullopt; // We've got an updated task. Start over. continue; } @@ -73,7 +78,7 @@ void SingleTaskScheduler::ThreadLogic() { // nothing to do, so sleep for a spell. if (current_task) { current_task->closure(); - current_task = nullopt; + current_task = absl::nullopt; } else { if (stop_.HasBeenNotified()) { return; diff --git a/tensorflow_serving/batching/streaming_batch_scheduler.h b/tensorflow_serving/batching/streaming_batch_scheduler.h index f9c58e7c443..02fedd91112 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler.h +++ b/tensorflow_serving/batching/streaming_batch_scheduler.h @@ -23,19 +23,19 @@ limitations under the License. #include #include +#include "tensorflow/core/kernels/batching_util/batch_scheduler.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/batching/batch_scheduler.h" #include "tensorflow_serving/batching/batch_scheduler_retrier.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -135,14 +135,14 @@ class StreamingBatchScheduler : public BatchScheduler { // // A negative value means that no timeout will be enforced. This setting is // useful in some test code. - int64 batch_timeout_micros = 10 * 1000 /* 10 milliseconds */; + int64_t batch_timeout_micros = 0; // The name to use for the pool of batch threads. string thread_pool_name = "batch_threads"; // The number of threads to use to process batches. // Must be >= 1, and should be tuned carefully. - int num_batch_threads = 1; + int num_batch_threads = port::MaxParallelism(); // The following options are typically only overridden by test code. @@ -151,7 +151,7 @@ class StreamingBatchScheduler : public BatchScheduler { // How long SingleTaskScheduler should wait if there are no scheduled tasks, // in microseconds. - uint64 no_tasks_wait_time_micros = 1000; // 1 millisecond + uint64_t no_tasks_wait_time_micros = 1000; // 1 millisecond }; static Status Create( const Options& options, @@ -170,6 +170,8 @@ class StreamingBatchScheduler : public BatchScheduler { // immediately (there is no queueing). size_t SchedulingCapacity() const override; + size_t max_task_size() const override { return options_.max_batch_size; } + private: StreamingBatchScheduler(const Options& options, std::function>)> @@ -181,13 +183,13 @@ class StreamingBatchScheduler : public BatchScheduler { // Closes 'open_batch_' (unless it equals nullptr), and replaces it with a // fresh open batch. Schedules the new batch on 'batch_threads_'. - void StartNewBatch() EXCLUSIVE_LOCKS_REQUIRED(mu_); + void StartNewBatch() TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Takes a snapshot of 'open_batch_num_', and schedules an event with // 'batch_closer_' to close it at time 'close_time_micros' if it is still open // at that time. - void ScheduleCloseOfCurrentOpenBatch(uint64 close_time_micros) - EXCLUSIVE_LOCKS_REQUIRED(mu_); + void ScheduleCloseOfCurrentOpenBatch(uint64_t close_time_micros) + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); const Options options_; @@ -203,21 +205,22 @@ class StreamingBatchScheduler : public BatchScheduler { // The batch that is currently open and into which new tasks can be added. // Not owned here; owned by the batch thread pool. - Batch* open_batch_ GUARDED_BY(mu_) = nullptr; + Batch* open_batch_ TF_GUARDED_BY(mu_) = nullptr; // The sequence number of 'open_batch_'. Incremented each time 'open_batch_' // is assigned to a new (non-null) batch object. - int64 open_batch_num_ GUARDED_BY(mu_) = 0; + int64_t open_batch_num_ TF_GUARDED_BY(mu_) = 0; // The number of batches "in progress", i.e. batches that have been started // but for which the process-batch callback hasn't finished. Note that this // counter is somewhat conservative (i.e. might be an overestimate), because // it gets decremented after the callback finishes and there could be races. - int num_batches_in_progress_ GUARDED_BY(mu_) = 0; + int num_batches_in_progress_ TF_GUARDED_BY(mu_) = 0; // A background task we use to schedule batches to close when they hit their // timeout. - std::unique_ptr batch_closer_ GUARDED_BY(mu_); + std::unique_ptr batch_closer_ + TF_GUARDED_BY(mu_); TF_DISALLOW_COPY_AND_ASSIGN(StreamingBatchScheduler); }; @@ -243,7 +246,7 @@ namespace internal { class SingleTaskScheduler { public: SingleTaskScheduler(Env* env, string thread_name, - uint64 no_tasks_wait_time_micros); + uint64_t no_tasks_wait_time_micros); // Blocks until the currently-set closure (if any) runs. ~SingleTaskScheduler(); @@ -253,7 +256,7 @@ class SingleTaskScheduler { // cancels any closures provided in them (if they haven't already been run). // // IMPORTANT: 'time_micros' must be monotonically non-decreasing across calls. - void Schedule(uint64 time_micros, std::function closure); + void Schedule(uint64_t time_micros, std::function closure); private: // The code executed in 'thread_'. Looks for updated tasks, and executes them @@ -268,16 +271,16 @@ class SingleTaskScheduler { // The arguments to Schedule(). struct Task { - uint64 time_micros; + uint64_t time_micros; std::function closure; }; // A newly-scheduled task hasn't yet been picked up by 'thread_'. - optional updated_task_ GUARDED_BY(mu_); + absl::optional updated_task_ TF_GUARDED_BY(mu_); // The time parameter passed in the most recent Schedule() invocation. // Used to enforce monotonicity. - uint64 last_task_time_ = 0; + uint64_t last_task_time_ = 0; // A notification for stopping the thread, during destruction. Notification stop_; @@ -289,7 +292,7 @@ class SingleTaskScheduler { std::unique_ptr thread_; // How long to wait if there are no scheduled tasks, in microseconds. - const uint64 no_tasks_wait_time_micros_; + const uint64_t no_tasks_wait_time_micros_; TF_DISALLOW_COPY_AND_ASSIGN(SingleTaskScheduler); }; @@ -312,7 +315,7 @@ Status StreamingBatchScheduler::Create( } scheduler->reset( new StreamingBatchScheduler(options, process_batch_callback)); - return Status::OK(); + return Status(); } template @@ -360,7 +363,7 @@ Status StreamingBatchScheduler::Schedule( // If we are about to add the first task to a batch, schedule the batch to // be closed after the timeout. if (options_.batch_timeout_micros > 0 && open_batch_->empty()) { - const uint64 batch_deadline = + const uint64_t batch_deadline = options_.env->NowMicros() + options_.batch_timeout_micros; ScheduleCloseOfCurrentOpenBatch(batch_deadline); } @@ -373,7 +376,7 @@ Status StreamingBatchScheduler::Schedule( } } - return Status::OK(); + return Status(); } template @@ -430,13 +433,13 @@ void StreamingBatchScheduler::StartNewBatch() { template void StreamingBatchScheduler::ScheduleCloseOfCurrentOpenBatch( - uint64 close_time_micros) { + uint64_t close_time_micros) { if (batch_closer_ == nullptr) { batch_closer_.reset(new internal::SingleTaskScheduler( options_.env, "batch_closer", options_.no_tasks_wait_time_micros)); } - const int64 batch_num_to_close = open_batch_num_; + const int64_t batch_num_to_close = open_batch_num_; batch_closer_->Schedule(close_time_micros, [this, batch_num_to_close] { { mutex_lock l(this->mu_); @@ -461,7 +464,7 @@ Status CreateRetryingStreamingBatchScheduler( TF_RETURN_IF_ERROR(BatchSchedulerRetrier::Create( retry_options, std::move(streaming_scheduler), &retrier)); *scheduler = std::move(retrier); - return Status::OK(); + return Status(); } } // namespace serving diff --git a/tensorflow_serving/batching/streaming_batch_scheduler_test.cc b/tensorflow_serving/batching/streaming_batch_scheduler_test.cc index fefacd8a472..8f0c25ee902 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler_test.cc +++ b/tensorflow_serving/batching/streaming_batch_scheduler_test.cc @@ -15,14 +15,15 @@ limitations under the License. #include "tensorflow_serving/batching/streaming_batch_scheduler.h" +#include #include #include #include -#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/kernels/batching_util/fake_clock_env.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" using ::testing::ElementsAre; using ::testing::IsEmpty; @@ -48,9 +49,10 @@ class FakeTask : public BatchTask { // Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on // that task. Returns the resulting status. -Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { +absl::Status ScheduleTask(size_t task_size, + BatchScheduler* scheduler) { std::unique_ptr task(new FakeTask(task_size)); - Status status = scheduler->Schedule(&task); + absl::Status status = scheduler->Schedule(&task); // Schedule() should have consumed 'task' iff it returned Status::OK. CHECK_EQ(status.ok(), task == nullptr); return status; @@ -277,6 +279,8 @@ TEST(StreamingBatchSchedulerTest, ConstMethods) { TF_ASSERT_OK(StreamingBatchScheduler::Create(options, callback, &scheduler)); + EXPECT_EQ(2, scheduler->max_task_size()); + // Submit 'num_threads' full batches, to make the scheduling threads "full". // (At all times, the queue length should show as 0, since // StreamingBatchScheduler never enqueues tasks.) @@ -293,7 +297,7 @@ TEST(StreamingBatchSchedulerTest, ConstMethods) { // Make another Schedule() call while the threads are full, which should // yield an UNAVAILABLE error. - Status status = ScheduleTask(1, scheduler.get()); + absl::Status status = ScheduleTask(1, scheduler.get()); EXPECT_FALSE(status.ok()); EXPECT_EQ(error::UNAVAILABLE, status.code()); EXPECT_EQ(0, scheduler->NumEnqueuedTasks()); diff --git a/tensorflow_serving/batching/test_util/BUILD b/tensorflow_serving/batching/test_util/BUILD index 35802960058..57b585122fe 100644 --- a/tensorflow_serving/batching/test_util/BUILD +++ b/tensorflow_serving/batching/test_util/BUILD @@ -1,16 +1,13 @@ # Description: Tensorflow Serving batching test utilities. +# Placeholder: load py_binary + package( default_visibility = ["//tensorflow_serving:internal"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) +licenses(["notice"]) filegroup( name = "all_files", @@ -29,8 +26,8 @@ cc_library( hdrs = ["puppet_batch_scheduler.h"], visibility = ["//visibility:private"], deps = [ - "//tensorflow_serving/batching:batch_scheduler", "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core/kernels/batching_util:batch_scheduler", ], ) @@ -46,3 +43,11 @@ cc_test( "@org_tensorflow//tensorflow/core:test", ], ) + +# script that generates saved_model for matrix_half_plus_two model. +py_binary( + name = "matrix_half_plus_two_saved_model", + srcs = ["matrix_half_plus_two_saved_model.py"], + python_version = "PY3", + srcs_version = "PY3", +) diff --git a/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py b/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py new file mode 100644 index 00000000000..d111459009c --- /dev/null +++ b/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py @@ -0,0 +1,58 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import tensorflow.compat.v1 as tf +FLAGS = tf.app.flags.FLAGS + +tf.app.flags.DEFINE_string("output_dir", "/tmp/matrix_half_plus_two/1", + "The directory where to write SavedModel files.") + + +def _generate_saved_model_for_matrix_half_plus_two(export_dir): + """Creates SavedModel for half plus two model that accepts batches of + 3*3 matrices. + The model divides all elements in each matrix by 2 and adds 2 to them. + So, for one input matrix [[1, 2, 3], [4, 5, 6], [7, 8, 9]] + the result will be [[2.5, 3, 3.5], [4, 4.5, 5], [5.5, 6, 6.5]]. + Args: + export_dir: The directory where to write SavedModel files. + """ + builder = tf.saved_model.builder.SavedModelBuilder(export_dir) + with tf.Session() as session: + x = tf.placeholder(tf.float32, shape=[None, 3, 3], name="x") + a = tf.constant(0.5) + b = tf.constant(2.0) + y = tf.add(tf.multiply(a, x), b, name="y") + predict_signature_def = ( + tf.saved_model.signature_def_utils.predict_signature_def({ + "x": x + }, {"y": y})) + signature_def_map = { + tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + predict_signature_def + } + session.run(tf.global_variables_initializer()) + builder.add_meta_graph_and_variables( + session, [tf.saved_model.tag_constants.SERVING], + signature_def_map=signature_def_map) + builder.save() + + +def main(_): + _generate_saved_model_for_matrix_half_plus_two(FLAGS.output_dir) + + +if __name__ == "__main__": + tf.compat.v1.app.run() diff --git a/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h b/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h index 5cb70d96766..1a639993204 100644 --- a/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h +++ b/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h @@ -24,7 +24,7 @@ limitations under the License. #include #include -#include "tensorflow_serving/batching/batch_scheduler.h" +#include "tensorflow/core/kernels/batching_util/batch_scheduler.h" namespace tensorflow { namespace serving { @@ -59,6 +59,10 @@ class PuppetBatchScheduler : public BatchScheduler { // Processes all enqueued tasks. void ProcessAllTasks(); + size_t max_task_size() const override { + return std::numeric_limits::max(); + } + private: std::function>)> process_batch_callback_; @@ -81,7 +85,7 @@ template Status PuppetBatchScheduler::Schedule( std::unique_ptr* task) { queue_.push(std::move(*task)); - return Status::OK(); + return Status(); } template diff --git a/tensorflow_serving/batching/test_util/puppet_batch_scheduler_test.cc b/tensorflow_serving/batching/test_util/puppet_batch_scheduler_test.cc index 440fdc612c0..e8eb12a9a06 100644 --- a/tensorflow_serving/batching/test_util/puppet_batch_scheduler_test.cc +++ b/tensorflow_serving/batching/test_util/puppet_batch_scheduler_test.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow_serving/batching/test_util/puppet_batch_scheduler.h" +#include + #include #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" diff --git a/tensorflow_serving/batching/testdata/BUILD b/tensorflow_serving/batching/testdata/BUILD new file mode 100644 index 00000000000..5cecf582ef8 --- /dev/null +++ b/tensorflow_serving/batching/testdata/BUILD @@ -0,0 +1,15 @@ +# Description: Tensorflow Serving batching test data. + +package( + default_visibility = ["//tensorflow_serving:internal"], + features = ["-layering_check"], +) + +licenses(["notice"]) + +filegroup( + name = "matrix_half_plus_two", + srcs = glob( + ["matrix_half_plus_two/**/*"], + ), +) diff --git a/tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb b/tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb new file mode 100644 index 00000000000..8651e45a044 Binary files /dev/null and b/tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb differ diff --git a/tensorflow_serving/batching/tfrt_saved_model_with_batching.cc b/tensorflow_serving/batching/tfrt_saved_model_with_batching.cc new file mode 100644 index 00000000000..0a5349d5e0c --- /dev/null +++ b/tensorflow_serving/batching/tfrt_saved_model_with_batching.cc @@ -0,0 +1,523 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/tfrt_saved_model_with_batching.h" + +#include "absl/container/flat_hash_map.h" +#include "absl/container/inlined_vector.h" +#include "tensorflow/core/framework/tensor_util.h" +#include "tensorflow/core/kernels/batching_util/batch_scheduler.h" +#include "tensorflow/core/lib/gtl/cleanup.h" +#include "tensorflow/core/lib/monitoring/sampler.h" +#include "tensorflow_serving/batching/batching_util.h" +#include "tensorflow_serving/batching/incremental_barrier.h" + +namespace tensorflow { +namespace serving { + +using tfrt::FunctionMetadata; +using tfrt::SavedModel; + +namespace { + +auto *queuing_latency = monitoring::Sampler<0>::New( + {"/tensorflow/serving/saved_model_with_batching/queuing_latency", + "Distribution of wall time spent (in microseconds) in queuing"}, + // Scale of 100, power of 1.2 with bucket count 52 (~1 second). + monitoring::Buckets::Exponential(100, 1.2, 52)); + +// Batching implementation of SavedModel. +class SavedModelWithBatching : public tfrt::SavedModel { + public: + static absl::Status Create( + const SavedModelBatchingOptions &options, + const std::vector + &func_name_with_batching_scheduler_creator, + std::unique_ptr saved_model, + std::unique_ptr *result); + + // Public ctor because of absl::make_unique. It's okay given the class is + // not publicly visible. + SavedModelWithBatching(const SavedModelBatchingOptions &options, + std::unique_ptr saved_model); + + const tensorflow::MetaGraphDef &GetMetaGraphDef() const override { + return wrapped_->GetMetaGraphDef(); + } + + std::vector GetFunctionNames() const override { + return wrapped_->GetFunctionNames(); + } + + absl::optional GetFunctionMetadata( + absl::string_view func_name) const override { + return wrapped_->GetFunctionMetadata(func_name); + } + + // The semantics of Run() is identical to its parent, it internally blocks, + // batches multiple Run() and splits the result once the batch finishes and + // unblocks. + absl::Status Run(const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, absl::Span inputs, + std::vector *outputs) override; + + absl::Status RunMultipleSignatures( + const RunOptions &run_options, absl::Span names, + absl::Span> multi_inputs, + std::vector> *multi_outputs) override { + // TODO(b/191149783): Implement batching support for + // RunMultipleSignatures(). + return wrapped_->RunMultipleSignatures(run_options, names, multi_inputs, + multi_outputs); + } + + absl::Status RunByTensorNames( + const RunOptions &run_options, + absl::Span> inputs, + absl::Span output_tensor_names, + absl::Span target_node_names, + std::vector *outputs) { + // TODO(b/191149783): Implement batching support for RunByTensorNames(). + return wrapped_->RunByTensorNames(run_options, inputs, output_tensor_names, + target_node_names, outputs); + } + + private: + // Batches tensors in `batch` and invokes Run() with underlying `wrapped_`. + void ProcessBatch(absl::string_view func_name, + std::unique_ptr> batch); + + // Batches tensors in `batch` and stores the result in `batch_inputs`. + absl::Status BatchInputTensors(absl::string_view func_name, + const Batch &batch, + std::vector *batch_inputs); + + // For each tensor in `combined_outputs`, splits it according to `batch` and + // stores the result in corresponding BatchingTask. + absl::Status SplitOutputTensors(std::vector combined_outputs, + Batch *batch); + + const SavedModelBatchingOptions options_; + + // Underlying SavedModel. + std::unique_ptr wrapped_; + absl::flat_hash_map>> + batch_schedulers_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelWithBatching); +}; + +SavedModelWithBatching::SavedModelWithBatching( + const SavedModelBatchingOptions &options, + std::unique_ptr saved_model) + : tfrt::SavedModel(&saved_model->runtime()), + options_(options), + wrapped_(std::move(saved_model)) {} + +absl::Status SavedModelWithBatching::Create( + const SavedModelBatchingOptions &options, + const std::vector + &func_name_with_batching_scheduler_creators, + std::unique_ptr saved_model, + std::unique_ptr *result) { + if (saved_model == nullptr) { + return errors::FailedPrecondition("saved_model must not be null."); + } + + SavedModel *raw_saved_model = saved_model.get(); + std::unique_ptr saved_model_with_batching = + absl::make_unique(options, + std::move(saved_model)); + SavedModelWithBatching *raw_saved_model_with_batching = + saved_model_with_batching.get(); + + for (const auto &entry : func_name_with_batching_scheduler_creators) { + if (!raw_saved_model->GetFunctionMetadata(entry.func_name)) { + LOG(WARNING) << "Function " << entry.func_name + << " is not found in the model. "; + continue; + } + + auto insert_result = saved_model_with_batching->batch_schedulers_.emplace( + std::string(entry.func_name), /*scheduler=*/nullptr); + if (!insert_result.second) { + return errors::FailedPrecondition( + absl::StrCat("Specified multiple batch schedulers for function ", + entry.func_name)); + } + + const std::string &func_name = insert_result.first->first; + TF_RETURN_IF_ERROR(entry.scheduler_creator( + [func_name, raw_saved_model_with_batching]( + std::unique_ptr> batch) { + raw_saved_model_with_batching->ProcessBatch(func_name, + std::move(batch)); + }, + &insert_result.first->second)); + if (insert_result.first->second == nullptr) { + return errors::FailedPrecondition(absl::StrCat( + "Failed to create batch scheduler for function ", entry.func_name)); + } + } + *result = std::move(saved_model_with_batching); + return absl::Status(); +} + +absl::Status SavedModelWithBatching::Run( + const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, absl::Span inputs, + std::vector *outputs) { + if (outputs == nullptr) { + return errors::FailedPrecondition("outputs must not be null"); + } + auto it = batch_schedulers_.find(func_name); + if (it == batch_schedulers_.end()) { + // Batch scheduler not found for this function, run it with underlying + // SavedModel in-line. + static uint64_t last_log_message_secs = 0; + // Not thread safe, but that's how batching_session is doing as well. + // TODO(b/168220822): It probably matters, what if last_log_message_secs + // mistakenly becomes too large? + uint64_t now_secs = EnvTime::NowSeconds(); + if (now_secs - last_log_message_secs >= 120) { + LOG(WARNING) << "Request doesn't match any declared function. Bypassing " + "batcher. Request function is: " + << func_name; + last_log_message_secs = now_secs; + } + return wrapped_->Run(run_options, func_name, inputs, outputs); + } + outputs->clear(); + + Notification done; + absl::Status status; + auto task = absl::make_unique(); + TF_RETURN_IF_ERROR(ComputeTensorBatchSize( + inputs, &task->zeroth_dim_size, + [](const Tensor &tensor) { return tensor.dims(); }, + [](const Tensor &tensor, size_t dim) { return tensor.dim_size(dim); })); + RecordInputBatchSize(task->zeroth_dim_size); + + task->host_context = GetHostContext(); + task->tfrt_inputs = inputs; + task->tfrt_outputs = outputs; + task->done = &done; + task->status = &status; + task->run_options = run_options; + task->enqueue_time_micros = EnvTime::NowMicros(); + + TF_RETURN_IF_ERROR(it->second->Schedule(&task)); + done.WaitForNotification(); + return status; +} + +// TODO(b/168220822): Once tfrt supports tensor split/pad/concat utilities and +// removes llvm dependency, refactors this function accordingly (return type may +// change). +std::vector> CalculateMaxDimSizes( + const Batch &batch) { + std::vector> max_dim_sizes; + for (int batch_idx = 0; batch_idx < batch.num_tasks(); ++batch_idx) { + const auto inputs = batch.task(batch_idx).tfrt_inputs; + for (int tensor_idx = 0; tensor_idx < inputs.size(); ++tensor_idx) { + const Tensor &tensor = inputs[tensor_idx]; + const TensorShape &shape = tensor.shape(); + const int rank = shape.dims(); + + absl::InlinedVector dims; + dims.reserve(rank); + for (auto dim : shape) { + dims.push_back(dim.size); + } + + if (batch_idx == 0) { + max_dim_sizes.push_back(std::move(dims)); + } else { + for (int rank_idx = 0; rank_idx < rank; ++rank_idx) { + int &cur_max_size = max_dim_sizes[tensor_idx][rank_idx]; + cur_max_size = std::max(cur_max_size, dims[rank_idx]); + } + } + } + } + return max_dim_sizes; +} + +absl::Status SavedModelWithBatching::BatchInputTensors( + absl::string_view func_name, const Batch &batch, + std::vector *batch_inputs) { + if (batch.num_tasks() < 1) { + return errors::Internal("Batch size expected to be positive; was ", + batch.num_tasks()); + } + const int original_batch_size = batch.size(); + const int target_batch_size = RoundToLowestAllowedBatchSize( + options_.allowed_batch_sizes, original_batch_size); + const int padding_size = target_batch_size - original_batch_size; + RecordPaddingSize(padding_size, target_batch_size); + RecordProcessedBatchSize(target_batch_size); + + std::vector> max_dim_sizes; + if (options_.pad_variable_length_inputs) { + max_dim_sizes = CalculateMaxDimSizes(batch); + } + + // TODO(b/168220822): Padding logic below operates on tfrt inputs. It's pretty + // much a duplicate of batching session Padding logic. Rewrite once TFRT + // tensor supports necessary utilities. + std::vector> tensors_to_merge( + batch.task(0).tfrt_inputs.size(), std::vector()); + for (int batch_idx = 0; batch_idx < batch.num_tasks(); ++batch_idx) { + auto inputs = batch.task(batch_idx).tfrt_inputs; + + for (int tensor_idx = 0; tensor_idx < inputs.size(); ++tensor_idx) { + Tensor tensor = inputs[tensor_idx]; + std::vector &tensor_vec = tensors_to_merge[tensor_idx]; + + Tensor optionally_padded_tensor; + if (options_.pad_variable_length_inputs) { + TF_RETURN_IF_ERROR(AddPadding(tensor, max_dim_sizes[tensor_idx], + &optionally_padded_tensor)); + } else { + optionally_padded_tensor = tensor; + if (batch_idx > 0) { + TensorShape reference_shape = tensors_to_merge[tensor_idx][0].shape(); + + if (!AreShapesEqualExceptZeroDim(tensor.shape(), reference_shape)) { + return errors::FailedPrecondition( + " Tensors in a single batch have different shapes other than" + " first dimension and padding is turned off."); + } + } + } + tensor_vec.push_back(std::move(optionally_padded_tensor)); + + if (batch_idx == batch.num_tasks() - 1 && padding_size > 0) { + const Tensor padding_tensor = tensor_vec.back().Slice(0, 1); + for (int i = 0; i < padding_size; ++i) { + tensor_vec.push_back(padding_tensor); + } + } + } + } + + for (const auto &tensors : tensors_to_merge) { + Tensor concated; + TF_RETURN_IF_ERROR(tensor::Concat(tensors, &concated)); + batch_inputs->push_back(concated); + } + + return absl::Status(); +} + +void SavedModelWithBatching::ProcessBatch( + absl::string_view func_name, + std::unique_ptr> batch) { + batch->WaitUntilClosed(); + + if (batch->empty()) return; + absl::Status status = absl::Status(); + auto cleanup = gtl::MakeCleanup([&status, &batch] { + for (int batch_idx = 0; batch_idx < batch->num_tasks(); ++batch_idx) { + SavedModelBatchingTask *task = batch->mutable_task(batch_idx); + if (task->partial_status != nullptr) { + task->partial_status->Update(status); + task->done_callback(); + } else { + *(task->status) = status; + task->done->Notify(); + } + } + }); + + const uint64_t dequeue_time_micros = EnvTime::NowMicros(); + + bool all_tasks_timeout_exceeded = true; + absl::optional batch_deadline; + for (int batch_idx = 0; batch_idx < batch->num_tasks(); ++batch_idx) { + const SavedModelBatchingTask &task = batch->task(batch_idx); + if (!task.run_options.deadline.has_value() || + absl::ToChronoTime(absl::Now()) < task.run_options.deadline.value()) { + all_tasks_timeout_exceeded = false; + if (task.run_options.deadline.has_value() && + (!batch_deadline.has_value() || + batch_deadline.value() < task.run_options.deadline.value())) { + batch_deadline = task.run_options.deadline; + } + } + queuing_latency->GetCell()->Add(dequeue_time_micros - + task.enqueue_time_micros); + } + + if (all_tasks_timeout_exceeded) { + status = absl::Status( + static_cast(absl::StatusCode::kResourceExhausted), + "Run() timeout exceeded while waiting in batching queue"); + return; + } + + tfrt::SavedModel::RunOptions batch_run_options; + batch_run_options.deadline = batch_deadline; + std::vector batch_inputs; + status = BatchInputTensors(func_name, *batch, &batch_inputs); + if (!status.ok()) return; + + std::vector combined_outputs; + status = wrapped_->Run(batch_run_options, func_name, batch_inputs, + &combined_outputs); + if (!status.ok()) return; + status = SplitOutputTensors(std::move(combined_outputs), batch.get()); +} + +absl::Status SavedModelWithBatching::SplitOutputTensors( + std::vector combined_outputs, + Batch *batch) { + std::vector split_batch_sizes; + split_batch_sizes.reserve(batch->num_tasks()); + for (int batch_idx = 0; batch_idx < batch->num_tasks(); ++batch_idx) { + split_batch_sizes.push_back(batch->task(batch_idx).size()); + } + const int64_t no_padded_batch_size = batch->size(); + const int64_t padded_batch_size = RoundToLowestAllowedBatchSize( + options_.allowed_batch_sizes, no_padded_batch_size); + + const int64_t padding_size = padded_batch_size - no_padded_batch_size; + if (padding_size > 0) { + split_batch_sizes.push_back(padding_size); + } + + for (const auto &combined_tensor : combined_outputs) { + std::vector split_tensors; + TF_RETURN_IF_ERROR( + tensor::Split(combined_tensor, split_batch_sizes, &split_tensors)); + + for (int batch_idx = 0; batch_idx < batch->num_tasks(); ++batch_idx) { + SavedModelBatchingTask *task = batch->mutable_task(batch_idx); + task->tfrt_outputs->push_back(split_tensors.at(batch_idx)); + } + } + return absl::Status(); +} + +} // namespace + +absl::Status CreateSavedModelWithBatching( + const SavedModelBatchingOptions &options, + const std::vector + &func_name_with_batching_scheduler_creator, + std::unique_ptr saved_model, + std::unique_ptr *saved_model_with_batching) { + return SavedModelWithBatching::Create( + options, func_name_with_batching_scheduler_creator, + std::move(saved_model), saved_model_with_batching); +} + +absl::Status SplitSavedModelInputTask( + std::unique_ptr *input_task_ptr, + int open_batch_remaining_slot, int max_batch_size, + std::vector> *output_tasks) { + SavedModelBatchingTask *input_task = input_task_ptr->get(); + + // TODO(b/168220822): Also split RunMetadata once TFRT supports it. + + // Each inner vector will be passed to a partial task thus needs to be + // unique_ptr. shared_ptr because std::function is not compatible with + // capture by move. + auto split_output = + std::make_shared>>>(); + auto partial_status = std::make_shared(); + + auto split_task_done_callback = [split_output, partial_status, + status = input_task->status, + output = input_task->tfrt_outputs, + done_notification = input_task->done]() { + auto cleanup = gtl::MakeCleanup( + [done_notification]() { done_notification->Notify(); }); + + // Fail early if any partial task fails. + if (!partial_status->status().ok()) { + *status = partial_status->status(); + return; + } + + int output_size = split_output->size(); + int tensor_size = (*split_output)[0]->size(); + for (int tensor_idx = 0; tensor_idx < tensor_size; ++tensor_idx) { + Tensor output_tensor; + std::vector to_concatenate; + to_concatenate.reserve(output_size); + for (int output_idx = 0; output_idx < output_size; ++output_idx) { + to_concatenate.push_back( + std::move((*(*split_output)[output_idx])[tensor_idx])); + } + const auto concat_status = tensor::Concat(to_concatenate, &output_tensor); + if (!concat_status.ok()) { + *status = concat_status; + return; + } + output->push_back(output_tensor); + } + *status = absl::Status(); + }; + + // The Callback will be run only after all partial tasks finished. + IncrementalBarrier barrier(std::move(split_task_done_callback)); + std::vector output_task_sizes; + + if (open_batch_remaining_slot > 0) { + output_task_sizes.push_back(open_batch_remaining_slot); + split_output->emplace_back(absl::make_unique>()); + } + + for (int left_task_size = input_task->size() - open_batch_remaining_slot; + left_task_size > 0; left_task_size -= max_batch_size) { + int next_task_size = std::min(left_task_size, max_batch_size); + output_task_sizes.push_back(next_task_size); + split_output->emplace_back(absl::make_unique>()); + } + + const int output_task_num = output_task_sizes.size(); + + // Construct partial tasks. + output_tasks->reserve(output_task_num); + for (int i = 0; i < output_task_num; ++i) { + auto task = absl::make_unique(); + task->zeroth_dim_size = output_task_sizes[i]; + task->run_options = input_task->run_options; + task->tfrt_outputs = (*split_output)[i].get(); + task->done_callback = barrier.Inc(); + task->partial_status = partial_status.get(); + output_tasks->push_back(std::move(task)); + } + + for (const Tensor &input : input_task->tfrt_inputs) { + std::vector split_tensors; + TF_RETURN_IF_ERROR(tensor::Split(input, output_task_sizes, &split_tensors)); + for (int output_idx = 0; output_idx < output_task_num; ++output_idx) { + auto &output_task = (*output_tasks)[output_idx]; + output_task->tfrt_partial_inputs.push_back(split_tensors[output_idx]); + } + } + + for (auto &task : *output_tasks) { + task->tfrt_inputs = task->tfrt_partial_inputs; + } + + return absl::Status(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/tfrt_saved_model_with_batching.h b/tensorflow_serving/batching/tfrt_saved_model_with_batching.h new file mode 100644 index 00000000000..d9ffef9df97 --- /dev/null +++ b/tensorflow_serving/batching/tfrt_saved_model_with_batching.h @@ -0,0 +1,98 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Batching interface on top of TFRT SavedModel. Subject to change since TFRT +// SavedModel API is temporary and experimental. +#ifndef TENSORFLOW_SERVING_BATCHING_TFRT_SAVED_MODEL_WITH_BATCHING_H_ +#define TENSORFLOW_SERVING_BATCHING_TFRT_SAVED_MODEL_WITH_BATCHING_H_ + +#include +#include +#include + +#include "tensorflow/core/tfrt/saved_model/saved_model.h" +#include "tensorflow_serving/batching/batching_options.h" +#include "tensorflow_serving/batching/batching_session.h" + +namespace tensorflow { +namespace serving { + +// The batch scheduler task type used for SavedModel batching, for use in batch +// scheduler template parameters, e.g. +// BasicBatchScheduler. +struct SavedModelBatchingTask; + +// A function to construct a batch scheduler for SavedModelBatchingTasks from a +// process-batch callback. +using SavedModelBatchingSchedulerCreator = std::function>)>, + std::unique_ptr> *)>; + +// A function name paired with a lambda to create a batch scheduler for Run() +// calls matching the function name. +struct FuncNameWithBatchingSchedulerCreator { + absl::string_view func_name; + SavedModelBatchingSchedulerCreator scheduler_creator; +}; + +using SavedModelBatchingOptions = BatchingOptions; + +// Creates `saved_model_with_batching` that batches requests per function +// internally, where the batch scheduler for each function is created according +// to `func_name_with_batching_scheduler_creator`. `saved_model` is the +// underlying core to run inference logic and must not be null. Upon successful +// return, `saved_model_with_batching` should be used in the same way as a +// normal SavedModel. Run() call is still synchronized, and all the batching +// logic is transparent to the caller. +// Also note that the first dimension of all tensors passed to Run() must be +// batching dimension. +Status CreateSavedModelWithBatching( + const SavedModelBatchingOptions &options, + const std::vector + &func_name_with_batching_scheduler_creator, + std::unique_ptr saved_model, + std::unique_ptr *saved_model_with_batching); + +struct SavedModelBatchingTask : public BatchingSessionTask { + // For monitoring purpose. + static std::string Name() { return "tfrt_saved_model_with_batching"; } + + tfrt::HostContext *host_context; + absl::Span tfrt_inputs; + std::vector *tfrt_outputs; + tfrt::SavedModel::RunOptions run_options; + + // If fields below are used, this is a partial task by splitting a large batch + // task. + std::vector tfrt_partial_inputs; + + // Status shared by all partial tasks by splitting a large batch task. The + // original task succedds only if all partial tasks succeed. + ThreadSafeStatus *partial_status = nullptr; +}; + +// The default implementation of +// `BasicBatchScheduler::Options.split_input_task_func` if corresponding batch +// scheduler for a batching session sets +// `BasicBatchScheduler::Options.enable_large_batch_splitting` to true. +Status SplitSavedModelInputTask( + std::unique_ptr *input_task_ptr, + int open_batch_remaining_slot, int max_batch_size, + std::vector> *output_tasks); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_BATCHING_TFRT_SAVED_MODEL_WITH_BATCHING_H_ diff --git a/tensorflow_serving/batching/tfrt_saved_model_with_batching_test.cc b/tensorflow_serving/batching/tfrt_saved_model_with_batching_test.cc new file mode 100644 index 00000000000..f3c321c61cb --- /dev/null +++ b/tensorflow_serving/batching/tfrt_saved_model_with_batching_test.cc @@ -0,0 +1,699 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/tfrt_saved_model_with_batching.h" + +#include +#include "absl/functional/bind_front.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/kernels/batching_util/basic_batch_scheduler.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/tfrt/utils/tensor_util.h" +#include "tensorflow_serving/batching/batching_util.h" +#include "tensorflow_serving/servables/tensorflow/test_util/mock_tfrt_saved_model.h" + +namespace tensorflow { +namespace serving { + +namespace { + +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::Invoke; +using ::testing::Return; + +constexpr char kFunctionOne[] = "func1"; +constexpr char kFunctionTwo[] = "func2"; +constexpr char kUnknownFunction[] = "unknown_func"; +const tfrt::internal::Signature signature; + +// TODO(b/168220822): Consider declaring a TensorMatcher for more +// exhaustive error messages. +MATCHER_P(MatchesTensor, p, "") { + const Tensor &x = arg; + const Tensor &y = *p; + const float *Tx = x.unaligned_flat().data(); + const float *Ty = y.unaligned_flat().data(); + auto size = x.NumElements(); + for (decltype(size) i = 0; i < size; ++i) { + if (Tx[i] != Ty[i]) return false; + } + return true; +} + +MATCHER_P2(TFStatusIs, error_code, partial_error_message, "") { + return arg.code() == error_code && + absl::StrContains(arg.message(), partial_error_message); +} + +absl::Status CreateDefaultBasicBatchScheduler( + const BasicBatchScheduler::Options &options, + std::function>)> + process_batch_callback, + std::unique_ptr> *batch_scheduler) { + std::unique_ptr> + basic_batch_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + options, process_batch_callback, &basic_batch_scheduler)); + *batch_scheduler = std::move(basic_batch_scheduler); + return absl::Status(); +} + +class SavedModelWithBatchingTest : public ::testing::Test { + protected: + SavedModelWithBatchingTest() = default; + + std::unique_ptr InitializeMockSavedModel() { + auto wrapped_saved_model = absl::make_unique(); + wrapped_saved_model_ = wrapped_saved_model.get(); + ON_CALL(*wrapped_saved_model_, GetFunctionMetadata(_)) + .WillByDefault(Return(tfrt::FunctionMetadata(&signature))); + return wrapped_saved_model; + } + + void Initialize( + const BasicBatchScheduler::Options + &scheduler_options = + BasicBatchScheduler::Options(), + const SavedModelBatchingOptions &options = SavedModelBatchingOptions()) { + std::unique_ptr wrapped_saved_model = + InitializeMockSavedModel(); + auto scheduler_creator = + absl::bind_front(&CreateDefaultBasicBatchScheduler, scheduler_options); + + std::vector creators = { + {kFunctionOne, scheduler_creator}, {kFunctionTwo, scheduler_creator}}; + TF_CHECK_OK(CreateSavedModelWithBatching(options, creators, + std::move(wrapped_saved_model), + &saved_model_with_batching_)); + } + + Tensor MakeTensor(const std::vector &tensor_vec, + const TensorShape &shape) { + return test::AsTensor(tensor_vec, shape); + } + + std::vector MakeTensors( + const std::vector, TensorShape>> &tensors) { + std::vector inputs; + inputs.reserve(tensors.size()); + for (const auto &entry : tensors) { + inputs.push_back(MakeTensor(entry.first, entry.second)); + } + return inputs; + } + + std::vector> MakeTensorsBatch( + const std::vector, TensorShape>>> + &tensor_batch) { + std::vector> result; + for (const auto &tensors : tensor_batch) { + result.push_back(MakeTensors(tensors)); + } + return result; + } + + std::unique_ptr saved_model_with_batching_; + test_util::MockSavedModel *wrapped_saved_model_; +}; + +SavedModelBatchingOptions BuildSavedModelBatchingOptions( + bool pad_variable_length_inputs, std::vector allowed_batch_sizes) { + SavedModelBatchingOptions options; + options.pad_variable_length_inputs = pad_variable_length_inputs; + options.allowed_batch_sizes = std::move(allowed_batch_sizes); + return options; +} + +// Builds BasicBatchingScheduler options. Only tunnable parameter is +// `max_batch_size`, as we fully use it to control batching behavior. +BasicBatchScheduler::Options BuildSchedulerOptions( + int max_batch_size) { + BasicBatchScheduler::Options options; + options.max_batch_size = max_batch_size; + options.batch_timeout_micros = 1000 * 1000 * 1000; // 1000s. + options.num_batch_threads = 1; + return options; +} + +// Expands the `tensor_vec` along 0th dimension by `dim0_size` times. +std::vector ExpandTensor(const std::vector &tensor_vec, + int64_t dim0_size) { + std::vector result; + for (int i = 0; i < dim0_size; ++i) { + result.insert(result.end(), tensor_vec.begin(), tensor_vec.end()); + } + return result; +} + +// Tests that creation of SavedModelWithBatching returns an appropriate error if +// the passed underlying SavedMode is NULL. +TEST_F(SavedModelWithBatchingTest, NullWrappedSavedModel) { + const string error = "must not be null"; + EXPECT_THAT( + CreateSavedModelWithBatching(SavedModelBatchingOptions(), {}, nullptr, + &saved_model_with_batching_), + TFStatusIs(error::FAILED_PRECONDITION, error)); +} + +// Tests that creation of SavedModelWithBatching returns an appropriate error if +// multiple scheduler creators are specified for each function. +TEST_F(SavedModelWithBatchingTest, MultipleBatchSchedulersForOneFunction) { + std::unique_ptr wrapped_saved_model = + InitializeMockSavedModel(); + auto scheduler_creator = + absl::bind_front(&CreateDefaultBasicBatchScheduler, + BasicBatchScheduler::Options()); + + const string error = "multiple batch schedulers"; + std::vector creators = { + {kFunctionOne, scheduler_creator}, {kFunctionOne, scheduler_creator}}; + EXPECT_THAT(CreateSavedModelWithBatching( + SavedModelBatchingOptions(), creators, + std::move(wrapped_saved_model), &saved_model_with_batching_), + TFStatusIs(error::FAILED_PRECONDITION, error)); +} + +// Tests that creation of SavedModelWithBatching returns an appropriate error if +// a scheduler failed being created. +TEST_F(SavedModelWithBatchingTest, FailedCreatingBatchScheduler) { + std::unique_ptr wrapped_saved_model = + InitializeMockSavedModel(); + auto scheduler_creator = + [](std::function>)> + process_batch_callback, + std::unique_ptr> + *batch_scheduler) { return absl::Status(); }; + + const string error = "Failed to create batch scheduler"; + std::vector creators = { + {kFunctionOne, scheduler_creator}}; + EXPECT_THAT(CreateSavedModelWithBatching( + SavedModelBatchingOptions(), creators, + std::move(wrapped_saved_model), &saved_model_with_batching_), + TFStatusIs(error::FAILED_PRECONDITION, error)); +} + +// Tests that when Run() is invoked with a function without a scheduler, it +// delegates to the underlying wrapped SavedModel directly. +TEST_F(SavedModelWithBatchingTest, FunctionNameNotFound) { + Initialize(BuildSchedulerOptions(/*max_batch_size=*/3)); + std::vector input_tensor_vec1 = {1, 2, 3}; + std::vector input_tensor_vec2 = {2, 3, 4}; + TensorShape input_shape = {1, 3}; + std::vector inputs = MakeTensors( + {{input_tensor_vec1, input_shape}, {input_tensor_vec2, input_shape}}); + + std::vector output_tensor_vec = {2, 2, 3}; + TensorShape output_shape = {1, 3}; + std::vector expected_outputs = + MakeTensors({{output_tensor_vec, output_shape}}); + std::vector outputs; + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kUnknownFunction, ::testing::An>(), _)) + .WillOnce(Invoke([&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, + absl::Span inputs, + std::vector *outputs) { + outputs->push_back(MakeTensor(output_tensor_vec, output_shape)); + return absl::Status(); + })); + tfrt::SavedModel::RunOptions run_options; + // If a corresponding scheduler is found, Run should block forever since + // maximum batch size isn't reached. + TF_ASSERT_OK(saved_model_with_batching_->Run(run_options, kUnknownFunction, + inputs, &outputs)); + EXPECT_THAT(outputs, ElementsAre(MatchesTensor(&expected_outputs[0]))); +} + +// Tests Basic batching behavior without any padding. +TEST_F(SavedModelWithBatchingTest, BatchingWithoutPadding) { + Initialize(BuildSchedulerOptions(/*max_batch_size=*/3)); + + std::vector input_tensor1_vec1 = {1, 2, 3}; + std::vector input_tensor1_vec2 = {1, 3, 4}; + std::vector input_tensor2_vec1 = {1, 2, 3, 1, 2, 3}; + std::vector input_tensor2_vec2 = {1, 3, 4, 1, 3, 4}; + + TensorShape input1_shape = {1, 3}; + TensorShape input2_shape = {2, 3}; + TensorShape combined_shape = {3, 3}; + + auto inputs = MakeTensorsBatch({ + {{input_tensor1_vec1, input1_shape}, {input_tensor1_vec2, input1_shape}}, + {{input_tensor2_vec1, input2_shape}, {input_tensor2_vec2, input2_shape}}, + }); + + std::vector combined_inputs = + MakeTensors({{ExpandTensor(input_tensor1_vec1, 3), combined_shape}, + {ExpandTensor(input_tensor1_vec2, 3), combined_shape}}); + + std::vector output_tensor1_vec = {1, 5, 5, 5}; + std::vector output_tensor2_vec = {1, 5, 5, 5, 1, 5, 5, 5}; + TensorShape output1_shape = {1, 4}; + TensorShape output2_shape = {2, 4}; + + auto expected_outputs = + MakeTensorsBatch({{{output_tensor1_vec, output1_shape}}, + {{output_tensor2_vec, output2_shape}}}); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionOne, ::testing::An>(), _)) + .WillOnce(Invoke([&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, + absl::Span inputs, + std::vector *outputs) { + absl::Span span(inputs); + EXPECT_THAT(span, ElementsAre(MatchesTensor(&combined_inputs[0]), + MatchesTensor(&combined_inputs[1]))); + + // Output is concatenation of `output_tensor1_vec` and + // `output_tensor2_vec`, in one of the two orders. + outputs->push_back(MakeTensor(ExpandTensor(output_tensor1_vec, 3), + /*shape=*/{3, 4})); + return absl::Status(); + })); + + tfrt::SavedModel::RunOptions run_options; + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[0], &outputs)); + EXPECT_THAT(outputs, + ElementsAre(MatchesTensor(&expected_outputs[0][0]))); + })); + + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[1], &outputs)); + EXPECT_THAT(outputs, + ElementsAre(MatchesTensor(&expected_outputs[1][0]))); + })); +} + +// Tests the batching behavior when padding is required both to extend each +// tensor's dimension size and to pad dummy tensors to fit target batch size. +TEST_F(SavedModelWithBatchingTest, BatchingWithPadding) { + // Need to pad 2 dummy tensors. + int batch_size = 5; + Initialize( + BuildSchedulerOptions(/*max_batch_size=*/3), + BuildSavedModelBatchingOptions(/*pad_variable_length_inputs=*/true, + /*allowed_batch_sizes=*/{batch_size})); + + std::vector input_tensor1_vec1 = {1, 2, 1, 3}; + std::vector input_tensor1_vec2 = {1, 3, 5, 1, 3, 4}; + std::vector input_tensor2_vec1 = {1, 2, 3}; + std::vector input_tensor2_vec2 = {1, 3, 4, 5}; + // Need to extend 1st dimension. + TensorShape input1_shape1 = {2, 2}; + TensorShape input1_shape2 = {2, 3}; + TensorShape input2_shape1 = {1, 3}; + TensorShape input2_shape2 = {1, 4}; + + auto inputs = MakeTensorsBatch({{{input_tensor1_vec1, input1_shape1}, + {input_tensor1_vec2, input1_shape2}}, + {{input_tensor2_vec1, input2_shape1}, + {input_tensor2_vec2, input2_shape2}}}); + + std::vector output_tensor1_vec = {1, 5, 5, 1, 5, 5}; + std::vector output_tensor2_vec = {1, 5, 5}; + TensorShape output1_shape = {2, 3}; + TensorShape output2_shape = {1, 3}; + + auto expected_outputs = + MakeTensorsBatch({{{output_tensor1_vec, output1_shape}}, + {{output_tensor2_vec, output2_shape}}}); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionOne, ::testing::An>(), _)) + .WillOnce(Invoke([&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, + absl::Span inputs, + std::vector *outputs) { + // First input tensor is of shape (5, 3) + EXPECT_EQ(15, inputs[0].NumElements()); + // Second input tensor is of shape (5, 4) + EXPECT_EQ(20, inputs[1].NumElements()); + + // Output is concatenation of `output_tensor1_vec` and + // `output_tensor2_vec`, in one of the two orders. + outputs->push_back(MakeTensor(ExpandTensor({1, 5, 5}, batch_size), + /*shape=*/{batch_size, 3})); + return absl::Status(); + })); + + tfrt::SavedModel::RunOptions run_options; + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[0], &outputs)); + EXPECT_THAT(outputs, + ElementsAre(MatchesTensor(&expected_outputs[0][0]))); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[1], &outputs)); + EXPECT_THAT(outputs, + ElementsAre(MatchesTensor(&expected_outputs[1][0]))); + })); +} + +// Tests that batching tensors with variable length dimension size (except for +// batching dimension) returns an appropriate error when padding is turned off. +TEST_F(SavedModelWithBatchingTest, UnequalShapesWhenPaddingIsTurnedOff) { + Initialize(BuildSchedulerOptions(/*max_batch_size=*/2)); + + std::vector input_tensor1_vec = {1, 2, 3}; + std::vector input_tensor2_vec = {1, 2, 3, 4}; + + TensorShape input1_shape = {1, 3}; + TensorShape input2_shape = {1, 4}; + + auto inputs = MakeTensorsBatch({{{input_tensor1_vec, input1_shape}}, + {{input_tensor2_vec, input2_shape}}}); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionOne, ::testing::An>(), _)) + .Times(0); + + tfrt::SavedModel::RunOptions run_options; + const string error = "different shapes other than first dimension"; + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + EXPECT_THAT(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[0], &outputs), + TFStatusIs(error::FAILED_PRECONDITION, error)); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + EXPECT_THAT(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[1], &outputs), + TFStatusIs(error::FAILED_PRECONDITION, error)); + })); +} + +// Tests that processing batch returns an appropriate error if all tasks in the +// batch has a past deadline. +TEST_F(SavedModelWithBatchingTest, AllTasksExceededDeadline) { + Initialize(BuildSchedulerOptions(/*max_batch_size=*/2)); + + std::vector input_tensor1_vec = {1, 2, 3}; + std::vector input_tensor2_vec = {1, 2, 4}; + + TensorShape input_shape = {1, 3}; + + auto inputs = MakeTensorsBatch( + {{{input_tensor1_vec, input_shape}}, {{input_tensor2_vec, input_shape}}}); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionOne, ::testing::An>(), _)) + .Times(0); + + tfrt::SavedModel::RunOptions run_options; + run_options.deadline = absl::ToChronoTime(absl::Now()); + const string error = "timeout exceeded"; + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + EXPECT_THAT(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[0], &outputs), + TFStatusIs(error::RESOURCE_EXHAUSTED, error)); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + EXPECT_THAT(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[1], &outputs), + TFStatusIs(error::RESOURCE_EXHAUSTED, error)); + })); +} + +// Tests that distinct functions should be batched independently. +TEST_F(SavedModelWithBatchingTest, MultipleFunctions) { + Initialize(BuildSchedulerOptions(/*max_batch_size=*/3)); + + std::vector input_tensor1_vec = {1, 3, 4}; + std::vector input_tensor2_vec = {2, 4, 5}; + std::vector input_tensor3_vec = {1, 3, 4, 1, 3, 4}; + std::vector input_tensor4_vec = {2, 4, 5, 2, 4, 5}; + + TensorShape input_shape1 = {1, 3}; + TensorShape input_shape2 = {2, 3}; + TensorShape combined_shape = {3, 3}; + + std::vector> inputs = + MakeTensorsBatch({{{input_tensor1_vec, input_shape1}}, + {{input_tensor2_vec, input_shape1}}, + {{input_tensor3_vec, input_shape2}}, + {{input_tensor4_vec, input_shape2}}}); + + std::vector combined_inputs1 = + MakeTensors({{ExpandTensor(input_tensor1_vec, 3), combined_shape}}); + std::vector combined_inputs2 = + MakeTensors({{ExpandTensor(input_tensor2_vec, 3), combined_shape}}); + + std::vector output_tensor1_vec = {1, 5, 5, 5}; + std::vector output_tensor2_vec = {1, 6, 6, 6}; + std::vector output_tensor3_vec = {1, 5, 5, 5, 1, 5, 5, 5}; + std::vector output_tensor4_vec = {1, 6, 6, 6, 1, 6, 6, 6}; + + TensorShape output1_shape = {1, 4}; + TensorShape output2_shape = {2, 4}; + + std::vector> expected_outputs = MakeTensorsBatch({ + {{output_tensor1_vec, output1_shape}}, + {{output_tensor2_vec, output1_shape}}, + {{output_tensor3_vec, output2_shape}}, + {{output_tensor4_vec, output2_shape}}, + }); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionOne, ::testing::An>(), _)) + .WillOnce(Invoke([&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, + absl::Span inputs, + std::vector *outputs) { + absl::Span span(inputs); + EXPECT_THAT(span, ElementsAre(MatchesTensor(&combined_inputs1[0]))); + + // Output is concatenation of `output_tensor1_vec` and + // `output_tensor2_vec`, in one of the two orders. + outputs->push_back( + MakeTensor(ExpandTensor(output_tensor1_vec, 3), {3, 4})); + return absl::Status(); + })); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionTwo, ::testing::An>(), _)) + .WillOnce(Invoke([&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, + absl::Span inputs, + std::vector *outputs) { + absl::Span span(inputs); + EXPECT_THAT(span, ElementsAre(MatchesTensor(&combined_inputs2[0]))); + + // Output is concatenation of `output_tensor3_vec` and + // `output_tensor4_vec`, in one of the two orders. + outputs->push_back( + MakeTensor(ExpandTensor(output_tensor2_vec, 3), {3, 4})); + return absl::Status(); + })); + + std::vector> threads; + for (int i = 0; i < 4; ++i) { + threads.emplace_back(std::unique_ptr(Env::Default()->StartThread( + ThreadOptions(), absl::StrCat("request_thread_", i), + [this, i, &inputs, &expected_outputs] { + std::vector outputs; + TF_ASSERT_OK(saved_model_with_batching_->Run( + tfrt::SavedModel::RunOptions(), + i == 0 || i == 2 ? kFunctionOne : kFunctionTwo, inputs[i], + &outputs)); + EXPECT_THAT(outputs, + ElementsAre(MatchesTensor(&expected_outputs[i][0]))); + }))); + } +} + +// Tests that when a large batch needs to be splitted, tensors are splitted and +// partial outputs are eventually merged appropriately. +TEST_F(SavedModelWithBatchingTest, SplitInputBasic) { + const int batch_size = 3; + BasicBatchScheduler::Options options = + BuildSchedulerOptions(6); + options.enable_large_batch_splitting = true; + options.max_execution_batch_size = batch_size; + options.split_input_task_func = SplitSavedModelInputTask; + Initialize(options, BuildSavedModelBatchingOptions( + /*pad_variable_length_inputs=*/true, + /*allowed_batch_sizes=*/{batch_size})); + + std::vector input_tensor1_vec1 = {1, 2, 3, 4, 5, 6}; + std::vector input_tensor1_vec2 = {1, 3, 4, 5, 6, 7, 8, 9}; + std::vector input_tensor2_vec1 = {1, 2, 1, 3}; + std::vector input_tensor2_vec2 = {1, 3, 5, 1, 3, 4, 5, 6}; + + TensorShape input1_shape1 = {2, 3}; + TensorShape input1_shape2 = {2, 4}; + TensorShape input2_shape1 = {4, 1}; + TensorShape input2_shape2 = {4, 2}; + + auto inputs = MakeTensorsBatch({{{input_tensor1_vec1, input1_shape1}, + {input_tensor1_vec2, input1_shape2}}, + {{input_tensor2_vec1, input2_shape1}, + {input_tensor2_vec2, input2_shape2}}}); + + std::vector output_tensor1_vec = {1, 5, 5, 1, 5, 5}; + std::vector output_tensor2_vec = {1, 5, 5, 1, 5, 5, 1, 5, 5, 1, 5, 5}; + TensorShape output1_shape = {2, 3}; + TensorShape output2_shape = {4, 3}; + + auto expected_outputs = + MakeTensorsBatch({{{output_tensor1_vec, output1_shape}}, + {{output_tensor2_vec, output2_shape}}}); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionOne, ::testing::An>(), _)) + .Times(2) + .WillRepeatedly(Invoke( + [&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, absl::Span inputs, + std::vector *outputs) { + // Second input (batch size 4) should be split into two tasks, one + // with batch size 1 and batches with the first input, one with + // batch size 3 which should stay itself. We only verify the first + // dimension is 3 because we don't know which batch comes first. + EXPECT_EQ(3, inputs[0].dim_size(0)); + // Second input tensor is of shape (3, 2) + EXPECT_EQ(3, inputs[1].dim_size(0)); + + // Output is concatenation of `output_tensor1_vec` and + // `output_tensor2_vec`, in one of the two orders. + outputs->push_back(MakeTensor(ExpandTensor({1, 5, 5}, batch_size), + /*shape=*/{batch_size, 3})); + return absl::Status(); + })); + + tfrt::SavedModel::RunOptions run_options; + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[0], &outputs)); + EXPECT_THAT(outputs, + ElementsAre(MatchesTensor(&expected_outputs[0][0]))); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[1], &outputs)); + EXPECT_THAT(outputs, + ElementsAre(MatchesTensor(&expected_outputs[1][0]))); + })); +} + +TEST_F(SavedModelWithBatchingTest, PartialTaskFails) { + const int batch_size = 3; + BasicBatchScheduler::Options options = + BuildSchedulerOptions(6); + options.enable_large_batch_splitting = true; + options.max_execution_batch_size = batch_size; + options.split_input_task_func = SplitSavedModelInputTask; + Initialize(options, BuildSavedModelBatchingOptions( + /*pad_variable_length_inputs=*/true, + /*allowed_batch_sizes=*/{batch_size})); + + std::vector input_tensor1_vec1 = {1, 2, 3, 4, 5, 6}; + std::vector input_tensor1_vec2 = {1, 3, 4, 5, 6, 7, 8, 9}; + std::vector input_tensor2_vec1 = {1, 2, 1, 3}; + std::vector input_tensor2_vec2 = {1, 3, 5, 1, 3, 4, 5, 6}; + + TensorShape input1_shape1 = {2, 3}; + TensorShape input1_shape2 = {2, 4}; + TensorShape input2_shape1 = {4, 1}; + TensorShape input2_shape2 = {4, 2}; + + auto inputs = MakeTensorsBatch({{{input_tensor1_vec1, input1_shape1}, + {input_tensor1_vec2, input1_shape2}}, + {{input_tensor2_vec1, input2_shape1}, + {input_tensor2_vec2, input2_shape2}}}); + + EXPECT_CALL( + *wrapped_saved_model_, + Run(_, kFunctionOne, ::testing::An>(), _)) + .Times(2) + // Fail One of the two partial tasks. + .WillOnce(Invoke([&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, + absl::Span inputs, + std::vector *outputs) { + return errors::Internal("Error"); + })) + .WillOnce(Invoke([&](const tfrt::SavedModel::RunOptions &run_options, + absl::string_view func_name, + absl::Span inputs, + std::vector *outputs) { + // Output is concatenation of `output_tensor1_vec` and + // `output_tensor2_vec`, in one of the two orders. + outputs->push_back(MakeTensor(ExpandTensor({1, 5, 5}, batch_size), + /*shape=*/{batch_size, 3})); + return absl::Status(); + })); + + tfrt::SavedModel::RunOptions run_options; + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + // First input may or may not succeed, because it is only in one of the + // two batches and only one batch fails. + const absl::Status ignore_result = saved_model_with_batching_->Run( + run_options, kFunctionOne, inputs[0], &outputs); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + // Second input must fail, since it is splitted into two partial tasks + // which are in different batches. Thus failure of any one of the two + // batches will fail this the second input. + EXPECT_THAT(saved_model_with_batching_->Run(run_options, kFunctionOne, + inputs[1], &outputs), + TFStatusIs(error::INTERNAL, "Error")); + })); +} + +} // namespace + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/threadsafe_status.cc b/tensorflow_serving/batching/threadsafe_status.cc new file mode 100644 index 00000000000..7ad15e0b7e9 --- /dev/null +++ b/tensorflow_serving/batching/threadsafe_status.cc @@ -0,0 +1,55 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/threadsafe_status.h" + +#include + +#include "absl/base/thread_annotations.h" +#include "absl/status/status.h" +#include "absl/synchronization/mutex.h" +#include "tensorflow/core/platform/mutex.h" + +namespace tensorflow { +namespace serving { +const absl::Status& ThreadSafeStatus::status() const& { + tf_shared_lock lock(mutex_); + return status_; +} + +absl::Status ThreadSafeStatus::status() && { + tf_shared_lock lock(mutex_); + return std::move(status_); +} + +void ThreadSafeStatus::Update(const absl::Status& new_status) { + if (new_status.ok()) { + return; + } + + mutex_lock lock(mutex_); + status_.Update(new_status); +} + +void ThreadSafeStatus::Update(absl::Status&& new_status) { + if (new_status.ok()) { + return; + } + + mutex_lock lock(mutex_); + status_.Update(std::forward(new_status)); +} +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/threadsafe_status.h b/tensorflow_serving/batching/threadsafe_status.h new file mode 100644 index 00000000000..5f87fb90072 --- /dev/null +++ b/tensorflow_serving/batching/threadsafe_status.h @@ -0,0 +1,62 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_THREADSAFE_STATUS_H_ +#define TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_THREADSAFE_STATUS_H_ + +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/platform/thread_annotations.h" + +namespace tensorflow { +namespace serving { +// TODO(b/161829688): +// Possibly merge ThreadSafeStatus implementation in TF and TF serving, when +// TF exposes ThreadSafeStatus in its public API (i.e., an release candidate). +// Wrapper class to allow both lock-free construction and concurrent updates on +// a 'status'. +// +// Example Usage: +// std::thread threads[2]; +// ThreadSafeStatus thread_safe_status; +// threads[0] = std::thread([&]() { +// status.Update(errors::Internal("internal error")); +// }); +// threads[1] = std::thread([&]() { +// status.Update(errors::InvalidArgument("invalid argument")); +// }); +// threads[0].Join(); +// threads[1].Join(); +// +// NOTE: +// When updated in a multi-threading setup, only the first error is retained. +class ThreadSafeStatus { + public: + const Status& status() const& TF_LOCKS_EXCLUDED(mutex_); + Status status() && TF_LOCKS_EXCLUDED(mutex_); + + // Retains the first error status: replaces the current status with + // `new_status` if `new_status` is not OK and the previous status is OK. + void Update(const Status& new_status) TF_LOCKS_EXCLUDED(mutex_); + void Update(Status&& new_status) TF_LOCKS_EXCLUDED(mutex_); + + private: + mutable mutex mutex_; + Status status_ TF_GUARDED_BY(mutex_); +}; +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_CORE_KERNELS_BATCHING_UTIL_THREADSAFE_STATUS_H_ diff --git a/tensorflow_serving/batching/threadsafe_status_test.cc b/tensorflow_serving/batching/threadsafe_status_test.cc new file mode 100644 index 00000000000..d3ca0c9ab13 --- /dev/null +++ b/tensorflow_serving/batching/threadsafe_status_test.cc @@ -0,0 +1,53 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/threadsafe_status.h" + +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" + +namespace tensorflow { +namespace serving { +namespace { + +TEST(ThreadSafeStatus, DefaultOk) { + ThreadSafeStatus status; + TF_EXPECT_OK(status.status()); +} + +TEST(ThreadSafeStatus, Update) { + ThreadSafeStatus status; + TF_EXPECT_OK(status.status()); + + status.Update(errors::FailedPrecondition("original error")); + EXPECT_EQ(status.status().code(), error::FAILED_PRECONDITION); + + status.Update(absl::OkStatus()); + EXPECT_EQ(status.status().code(), error::FAILED_PRECONDITION); + + status.Update(errors::Internal("new error")); + EXPECT_EQ(status.status().code(), error::FAILED_PRECONDITION); +} + +TEST(ThreadSafeStatus, Move) { + ThreadSafeStatus status; + TF_EXPECT_OK(std::move(status).status()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index fdb757d46e3..2cc8d6b5218 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -1,16 +1,14 @@ # Description: Tensorflow Serving configs. +load("//tensorflow_serving:serving.bzl", "serving_proto_library", "serving_proto_library_py") +# Placeholder: load go_proto_library + package( default_visibility = ["//visibility:public"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) +licenses(["notice"]) filegroup( name = "all_files", @@ -23,13 +21,93 @@ filegroup( ), ) -load("//tensorflow_serving:serving.bzl", "serving_proto_library") +serving_proto_library( + name = "file_system_storage_path_source_proto", + srcs = ["file_system_storage_path_source.proto"], +) + +serving_proto_library_py( + name = "file_system_storage_path_source_proto_py_pb2", + srcs = ["file_system_storage_path_source.proto"], + proto_library = "file_system_storage_path_source_proto", +) serving_proto_library( name = "model_server_config_proto", srcs = ["model_server_config.proto"], - cc_api_version = 2, deps = [ - "@protobuf//:cc_wkt_protos", + ":file_system_storage_path_source_proto", + ":logging_config_proto", + "@com_google_protobuf//:cc_wkt_protos", + ], +) + +serving_proto_library_py( + name = "model_server_config_proto_py_pb2", + srcs = ["model_server_config.proto"], + proto_library = "model_server_config_proto", + deps = [ + "file_system_storage_path_source_proto_py_pb2", + ":logging_config_proto_py_pb2", + ], +) + +serving_proto_library( + name = "platform_config_proto", + srcs = ["platform_config.proto"], + deps = [ + "@com_google_protobuf//:cc_wkt_protos", + ], +) + +serving_proto_library_py( + name = "platform_config_py_pb2", + proto_library = "platform_config_proto", +) + +serving_proto_library( + name = "log_collector_config_proto", + srcs = ["log_collector_config.proto"], + deps = [ + ], +) + +serving_proto_library_py( + name = "log_collector_config_proto_py_pb2", + srcs = ["log_collector_config.proto"], + proto_library = "log_collector_config_proto", + deps = [ + ], +) + +serving_proto_library( + name = "logging_config_proto", + srcs = ["logging_config.proto"], + deps = [ + ":log_collector_config_proto", + "@com_google_protobuf//:cc_wkt_protos", + ], +) + +serving_proto_library_py( + name = "logging_config_proto_py_pb2", + srcs = ["logging_config.proto"], + proto_library = "logging_config_proto", + deps = [ + ":log_collector_config_proto_py_pb2", + ], +) + +serving_proto_library( + name = "monitoring_config_proto", + srcs = ["monitoring_config.proto"], + deps = [ + ], +) + +serving_proto_library( + name = "ssl_config_proto", + srcs = ["ssl_config.proto"], + deps = [ ], ) diff --git a/tensorflow_serving/config/file_system_storage_path_source.proto b/tensorflow_serving/config/file_system_storage_path_source.proto new file mode 100644 index 00000000000..6924fe44926 --- /dev/null +++ b/tensorflow_serving/config/file_system_storage_path_source.proto @@ -0,0 +1,83 @@ +syntax = "proto3"; + +package tensorflow.serving; + +// Config proto for FileSystemStoragePathSource. +message FileSystemStoragePathSourceConfig { + // A policy that dictates which version(s) of a servable should be served. + message ServableVersionPolicy { + // Serve the latest versions (i.e. the ones with the highest version + // numbers), among those found on disk. + // + // This is the default policy, with the default number of versions as 1. + message Latest { + // Number of latest versions to serve. (The default is 1.) + uint32 num_versions = 1; + } + + // Serve all versions found on disk. + message All {} + + // Serve a specific version (or set of versions). + // + // This policy is useful for rolling back to a specific version, or for + // canarying a specific version while still serving a separate stable + // version. + message Specific { + // The version numbers to serve. + repeated int64 versions = 1; + } + + oneof policy_choice { + Latest latest = 100; + All all = 101; + Specific specific = 102; + } + } + + // A servable name and base path to look for versions of the servable. + message ServableToMonitor { + // The servable name to supply in aspired-versions callback calls. Child + // paths of 'base_path' are considered to be versions of this servable. + string servable_name = 1; + + // The path to monitor, i.e. look for child paths of the form base_path/123. + string base_path = 2; + + // The policy to determines the number of versions of the servable to be + // served at the same time. + tensorflow.serving.FileSystemStoragePathSourceConfig.ServableVersionPolicy + servable_version_policy = 4; + + reserved 3; // Legacy version_policy definition. + } + + // The servables to monitor for new versions, and aspire. + repeated ServableToMonitor servables = 5; + + // How long to wait between file-system polling to look for children of + // 'base_path', in seconds. + // + // If set to zero, filesystem will be polled exactly once. If set to a + // negative value (for testing use only), polling will be entirely disabled. + int64 file_system_poll_wait_seconds = 3; + + // If true, then FileSystemStoragePathSource::Create() and ::UpdateConfig() + // fail if, for any configured servables, the file system doesn't currently + // contain at least one version under the base path. + // (Otherwise, it will emit a warning and keep pinging the file system to + // check for a version to appear later.) + // DEPRECATED: Use 'servable_versions_always_present' instead, which includes + // this behavior. + // TODO(b/30898016): Remove 2019-10-31 or later. + bool fail_if_zero_versions_at_startup = 4 [deprecated = true]; + + // If true, the servable is always expected to exist on the underlying + // filesystem. FileSystemStoragePathSource::Create() and ::UpdateConfig() will + // fail if, for any configured servables, the file system doesn't currently + // contain at least one version under the base path. In addition, if a polling + // loop find the base path empty, it will not unload existing servables. + bool servable_versions_always_present = 6; + + reserved 1, 2; +} diff --git a/tensorflow_serving/config/log_collector_config.proto b/tensorflow_serving/config/log_collector_config.proto new file mode 100644 index 00000000000..4ce01d34a29 --- /dev/null +++ b/tensorflow_serving/config/log_collector_config.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +message LogCollectorConfig { + // Identifies the type of the LogCollector we will use to collect these logs. + string type = 1; + + // The prefix to use for the filenames of the logs. + string filename_prefix = 2; +} diff --git a/tensorflow_serving/config/logging_config.proto b/tensorflow_serving/config/logging_config.proto new file mode 100644 index 00000000000..cce49e8e5e7 --- /dev/null +++ b/tensorflow_serving/config/logging_config.proto @@ -0,0 +1,32 @@ +syntax = "proto3"; + +package tensorflow.serving; + +import "google/protobuf/any.proto"; +import "tensorflow_serving/config/log_collector_config.proto"; + +option cc_enable_arenas = true; + +message SamplingConfig { + // Requests will be logged uniformly at random with this probability. + // Valid range: [0, 1.0]. + double sampling_rate = 1; + + // Attributes of requests that can be optionally sampled. + // Note: Enabling more attributes will increase logging storage requirements. + enum Attributes { + ATTR_DEFAULT = 0x0; + ATTR_REQUEST_ORIGIN = 0x1; + ATTR_REQUEST_CRITICALITY = 0x2; + } + // Bitwise OR of above attributes + int32 attributes = 2; +} + +// Configuration for logging query/responses. +message LoggingConfig { + LogCollectorConfig log_collector_config = 1; + SamplingConfig sampling_config = 2; + // Additional logging config that can be processed by the logger. + google.protobuf.Any custom_logging_config = 3; +} diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index d25275c831b..cadc2b6e60f 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -1,9 +1,20 @@ syntax = "proto3"; package tensorflow.serving; -option cc_enable_arenas = true; import "google/protobuf/any.proto"; +import "tensorflow_serving/config/file_system_storage_path_source.proto"; +import "tensorflow_serving/config/logging_config.proto"; + +option cc_enable_arenas = true; + +// The type of model. +// TODO(b/31336131): DEPRECATED. +enum ModelType { + MODEL_TYPE_UNSPECIFIED = 0 [deprecated = true]; + TENSORFLOW = 1 [deprecated = true]; + OTHER = 2 [deprecated = true]; +} // Common configuration for loading a model being served. message ModelConfig { @@ -13,10 +24,48 @@ message ModelConfig { // Base path to the model, excluding the version directory. // E.g> for a model at /foo/bar/my_model/123, where 123 is the version, the // base path is /foo/bar/my_model. + // + // (This can be changed once a model is in serving, *if* the underlying data + // remains the same. Otherwise there are no guarantees about whether the old + // or new data will be used for model versions currently loaded.) string base_path = 2; + // Type of model. + // TODO(b/31336131): DEPRECATED. Please use 'model_platform' instead. + ModelType model_type = 3 [deprecated = true]; + // Type of model (e.g. "tensorflow"). - string model_type = 3; + // + // (This cannot be changed once a model is in serving.) + string model_platform = 4; + + reserved 5, 9; + + // Version policy for the model indicating which version(s) of the model to + // load and make available for serving simultaneously. + // The default option is to serve only the latest version of the model. + // + // (This can be changed once a model is in serving.) + FileSystemStoragePathSourceConfig.ServableVersionPolicy model_version_policy = + 7; + + // String labels to associate with versions of the model, allowing inference + // queries to refer to versions by label instead of number. Multiple labels + // can map to the same version, but not vice-versa. + // + // An envisioned use-case for these labels is canarying tentative versions. + // For example, one can assign labels "stable" and "canary" to two specific + // versions. Perhaps initially "stable" is assigned to version 0 and "canary" + // to version 1. Once version 1 passes canary, one can shift the "stable" + // label to refer to version 1 (at that point both labels map to the same + // version -- version 1 -- which is fine). Later once version 2 is ready to + // canary one can move the "canary" label to version 2. And so on. + map version_labels = 8; + + // Configures logging requests and responses, to the model. + // + // (This can be changed once a model is in serving.) + LoggingConfig logging_config = 6; } // Static list of models to be loaded for serving. @@ -26,12 +75,11 @@ message ModelConfigList { // ModelServer config. message ModelServerConfig { - // ModelServer takes either a static file-based model config list or an Any - // proto representing custom model config that is loaded dynamically at + // proto representing custom model config that is fetched dynamically at // runtime (through network RPC, custom service, etc.). oneof config { ModelConfigList model_config_list = 1; - google.protobuf.Any dynamic_model_config = 2; + google.protobuf.Any custom_model_config = 2; } -} \ No newline at end of file +} diff --git a/tensorflow_serving/config/monitoring_config.proto b/tensorflow_serving/config/monitoring_config.proto new file mode 100644 index 00000000000..9da3700de46 --- /dev/null +++ b/tensorflow_serving/config/monitoring_config.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +// Configuration for Prometheus monitoring. +message PrometheusConfig { + // Whether to expose Prometheus metrics. + bool enable = 1; + + // The endpoint to expose Prometheus metrics. + // If not specified, PrometheusExporter::kPrometheusPath value is used. + string path = 2; +} + +// Configuration for monitoring. +message MonitoringConfig { + PrometheusConfig prometheus_config = 1; +} diff --git a/tensorflow_serving/config/platform_config.proto b/tensorflow_serving/config/platform_config.proto new file mode 100644 index 00000000000..4e506b38833 --- /dev/null +++ b/tensorflow_serving/config/platform_config.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +import "google/protobuf/any.proto"; + +// Configuration for a servable platform e.g. tensorflow or other ML systems. +message PlatformConfig { + // The config proto for a SourceAdapter in the StoragePathSourceAdapter + // registry. + google.protobuf.Any source_adapter_config = 1; +}; + +message PlatformConfigMap { + // A map from a platform name to a platform config. The platform name is used + // in ModelConfig.model_platform. + map platform_configs = 1; +}; diff --git a/tensorflow_serving/config/ssl_config.proto b/tensorflow_serving/config/ssl_config.proto new file mode 100644 index 00000000000..0e51cd61a20 --- /dev/null +++ b/tensorflow_serving/config/ssl_config.proto @@ -0,0 +1,16 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +// Configuration for a secure gRPC channel +message SSLConfig { + // private server key for SSL + string server_key = 1; + // public server certificate + string server_cert = 2; + // custom certificate authority + string custom_ca = 3; + // valid client certificate required ? + bool client_verify = 4; +}; diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index b7f16be192e..fdb8769172b 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -4,15 +4,10 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) +licenses(["notice"]) filegroup( name = "all_files", @@ -34,7 +29,6 @@ cc_library( deps = [ "//tensorflow_serving/util:hash", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -56,7 +50,6 @@ cc_library( deps = [ ":servable_id", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -79,10 +72,9 @@ cc_library( ], deps = [ ":source", - "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/resources:resources_cc_proto", "//tensorflow_serving/util:any_ptr", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -95,11 +87,12 @@ cc_library( deps = [ ":loader", ":source_adapter", + "//tensorflow_serving/resources:resource_util", "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/util:any_ptr", - "//tensorflow_serving/util:optional", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:variant", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -152,9 +145,12 @@ cc_library( "//visibility:public", ], deps = [ + ":loader", ":servable_data", ":source", + ":storage_path", ":target", + "//tensorflow_serving/util:class_registration", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -194,6 +190,7 @@ cc_test( ":servable_data", ":source_router", ":storage_path", + ":target", "//tensorflow_serving/core/test_util:mock_storage_path_target", "//tensorflow_serving/core/test_util:test_main", "@org_tensorflow//tensorflow/core:lib", @@ -210,7 +207,6 @@ cc_library( deps = [ ":source_router", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -230,6 +226,60 @@ cc_test( ], ) +cc_library( + name = "dynamic_source_router", + hdrs = ["dynamic_source_router.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":source_router", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "dynamic_source_router_test", + srcs = ["dynamic_source_router_test.cc"], + deps = [ + ":dynamic_source_router", + ":servable_data", + ":source", + ":storage_path", + ":target", + "//tensorflow_serving/core/test_util:mock_storage_path_target", + "//tensorflow_serving/core/test_util:test_main", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", + ], +) + +cc_library( + name = "prefix_storage_path_source_adapter", + srcs = ["prefix_storage_path_source_adapter.cc"], + hdrs = ["prefix_storage_path_source_adapter.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":source_adapter", + ":storage_path", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "prefix_storage_path_source_adapter_test", + srcs = ["prefix_storage_path_source_adapter_test.cc"], + deps = [ + ":prefix_storage_path_source_adapter", + ":servable_data", + ":storage_path", + "//tensorflow_serving/core/test_util:test_main", + ], +) + cc_library( name = "storage_path", hdrs = ["storage_path.h"], @@ -274,7 +324,7 @@ cc_library( deps = [ ":loader", "//tensorflow_serving/util:any_ptr", - "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:lib", ], ) @@ -287,7 +337,7 @@ cc_library( deps = [ ":servable_handle", ":servable_id", - "//tensorflow_serving/util:optional", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -297,6 +347,7 @@ cc_test( srcs = ["manager_test.cc"], deps = [ ":manager", + "//tensorflow_serving/core/test_util:servable_handle_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:any_ptr", "@org_tensorflow//tensorflow/core:lib", @@ -340,7 +391,7 @@ cc_test( srcs = ["aspired_versions_manager_builder_test.cc"], deps = [ ":aspired_versions_manager_builder", - ":eager_load_policy", + ":availability_preserving_policy", ":servable_data", ":servable_handle", ":servable_state_monitor", @@ -349,7 +400,6 @@ cc_test( "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/core/test_util:fake_storage_path_source_adapter", - "//tensorflow_serving/core/test_util:source_adapter_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:event_bus", ], @@ -367,7 +417,8 @@ cc_library( ":servable_id", ":servable_state", "//tensorflow_serving/util:event_bus", - "//tensorflow_serving/util:optional", + "@com_google_absl//absl/time", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -377,9 +428,14 @@ cc_test( size = "small", srcs = ["servable_state_monitor_test.cc"], deps = [ + ":manager", + ":servable_id", + ":servable_state", ":servable_state_monitor", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", + "//tensorflow_serving/util:event_bus", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core/kernels/batching_util:fake_clock_env", ], ) @@ -390,9 +446,10 @@ cc_library( deps = [ ":loader", ":servable_id", - "//tensorflow_serving/util:optional", + "//tensorflow_serving/util:retrier", + "@com_google_absl//absl/status", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -424,14 +481,16 @@ cc_library( ":servable_handle", ":servable_id", ":servable_state", + ":source", "//tensorflow_serving/resources:resource_tracker", - "//tensorflow_serving/util:cleanup", "//tensorflow_serving/util:event_bus", "//tensorflow_serving/util:executor", "//tensorflow_serving/util:fast_read_dynamic_ptr", "//tensorflow_serving/util:hash", "//tensorflow_serving/util:inline_executor", + "//tensorflow_serving/util:retrier", "//tensorflow_serving/util:threadpool_executor", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -445,11 +504,15 @@ cc_test( ":servable_state_monitor", "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:fake_loader", + "//tensorflow_serving/core/test_util:manager_test_util", "//tensorflow_serving/core/test_util:mock_loader", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:any_ptr", "//tensorflow_serving/util:event_bus", "//tensorflow_serving/util:threadpool_executor", + "@com_google_absl//absl/status", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", @@ -467,6 +530,7 @@ cc_library( ":aspired_version_policy", ":basic_manager", ":loader", + ":loader_harness", ":manager", ":servable_data", ":servable_handle", @@ -475,9 +539,10 @@ cc_library( ":source", ":target", "//tensorflow_serving/util:event_bus", - "//tensorflow_serving/util:optional", - "//tensorflow_serving/util:periodic_function", + "//tensorflow_serving/util:observer", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core/kernels/batching_util:periodic_function_dynamic", ], ) @@ -486,8 +551,9 @@ cc_test( size = "small", srcs = ["aspired_versions_manager_test.cc"], deps = [ + ":aspired_version_policy", ":aspired_versions_manager", - ":eager_load_policy", + ":availability_preserving_policy", ":servable_state_monitor", "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:fake_loader", @@ -496,6 +562,7 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:any_ptr", "//tensorflow_serving/util:event_bus", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", @@ -508,14 +575,13 @@ cc_test( deps = [ ":aspired_version_policy", ":aspired_versions_manager", - ":eager_load_policy", + ":availability_preserving_policy", ":loader", ":manager", ":servable_data", ":servable_handle", ":simple_loader", "//tensorflow_serving/core/test_util:manager_test_util", - "//tensorflow_serving/util:periodic_function", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", @@ -545,7 +611,6 @@ cc_test( deps = [ ":static_manager", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", ], ) @@ -553,6 +618,9 @@ cc_library( name = "caching_manager", srcs = ["caching_manager.cc"], hdrs = ["caching_manager.h"], + visibility = [ + "//visibility:public", + ], deps = [ ":basic_manager", ":loader", @@ -560,7 +628,8 @@ cc_library( ":servable_data", ":servable_handle", ":servable_id", - "//tensorflow_serving/util:optional", + ":source_adapter", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -576,10 +645,10 @@ cc_test( ":servable_state", ":servable_state_monitor", ":simple_loader", + "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/core/test_util:manager_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:event_bus", - "//tensorflow_serving/util:optional", "//tensorflow_serving/util:threadpool_executor", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", @@ -588,6 +657,7 @@ cc_test( cc_library( name = "aspired_version_policy", + srcs = ["aspired_version_policy.cc"], hdrs = ["aspired_version_policy.h"], visibility = [ "//visibility:public", @@ -595,55 +665,214 @@ cc_library( deps = [ ":loader_harness", ":servable_id", - "//tensorflow_serving/util:optional", + "@com_google_absl//absl/types:optional", "@org_tensorflow//tensorflow/core:lib", ], ) +cc_test( + name = "aspired_version_policy_test", + srcs = ["aspired_version_policy_test.cc"], + deps = [ + ":aspired_version_policy", + ":loader_harness", + ":servable_id", + "//tensorflow_serving/core/test_util:test_main", + "@com_google_absl//absl/types:optional", + ], +) + cc_library( - name = "eager_load_policy", - srcs = ["eager_load_policy.cc"], - hdrs = ["eager_load_policy.h"], + name = "availability_preserving_policy", + srcs = ["availability_preserving_policy.cc"], + hdrs = ["availability_preserving_policy.h"], visibility = [ "//visibility:public", ], deps = [ ":aspired_version_policy", ":loader_harness", - "//tensorflow_serving/util:optional", + "@com_google_absl//absl/types:optional", ], ) cc_test( - name = "eager_load_policy_test", - srcs = ["eager_load_policy_test.cc"], + name = "availability_preserving_policy_test", + srcs = ["availability_preserving_policy_test.cc"], deps = [ - ":eager_load_policy", + ":availability_preserving_policy", ":servable_id", "//tensorflow_serving/core/test_util:test_main", ], ) cc_library( - name = "eager_unload_policy", - srcs = ["eager_unload_policy.cc"], - hdrs = ["eager_unload_policy.h"], + name = "resource_preserving_policy", + srcs = ["resource_preserving_policy.cc"], + hdrs = ["resource_preserving_policy.h"], visibility = [ "//visibility:public", ], deps = [ ":aspired_version_policy", ":loader_harness", - "//tensorflow_serving/util:optional", + "@com_google_absl//absl/types:optional", ], ) cc_test( - name = "eager_unload_policy_test", - srcs = ["eager_unload_policy_test.cc"], + name = "resource_preserving_policy_test", + srcs = ["resource_preserving_policy_test.cc"], deps = [ - ":eager_unload_policy", + ":resource_preserving_policy", ":servable_id", "//tensorflow_serving/core/test_util:test_main", ], ) + +cc_library( + name = "load_servables_fast", + srcs = ["load_servables_fast.cc"], + hdrs = ["load_servables_fast.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":aspired_versions_manager", + ":loader", + ":manager", + ":servable_state", + ":servable_state_monitor", + ":source", + ":target", + "@com_google_absl//absl/types:optional", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "log_collector", + srcs = ["log_collector.cc"], + hdrs = ["log_collector.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//tensorflow_serving/config:log_collector_config_cc_proto", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "log_collector_test", + srcs = ["log_collector_test.cc"], + deps = [ + ":log_collector", + "//tensorflow_serving/core/test_util:test_main", + ], +) + +cc_library( + name = "request_logger", + srcs = ["request_logger.cc"], + hdrs = ["request_logger.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":log_collector", + ":stream_logger", + "//tensorflow_serving/apis:logging_cc_proto", + "//tensorflow_serving/apis:model_cc_proto", + "//tensorflow_serving/config:logging_config_cc_proto", + "@com_google_protobuf//:protobuf", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + ], +) + +cc_test( + name = "request_logger_test", + size = "small", + srcs = ["request_logger_test.cc"], + deps = [ + ":log_collector", + ":request_logger", + "//tensorflow_serving/apis:logging_cc_proto", + "//tensorflow_serving/apis:model_cc_proto", + "//tensorflow_serving/apis:predict_cc_proto", + "//tensorflow_serving/config:logging_config_cc_proto", + "//tensorflow_serving/core/test_util:mock_log_collector", + "//tensorflow_serving/core/test_util:mock_prediction_stream_logger", + "//tensorflow_serving/core/test_util:mock_request_logger", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@com_google_protobuf//:protobuf", + "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", + ], +) + +cc_library( + name = "server_request_logger", + srcs = ["server_request_logger.cc"], + hdrs = ["server_request_logger.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":request_logger", + ":stream_logger", + "//tensorflow_serving/apis:logging_cc_proto", + "//tensorflow_serving/apis:model_cc_proto", + "//tensorflow_serving/config:logging_config_cc_proto", + "//tensorflow_serving/util:fast_read_dynamic_ptr", + "@com_google_absl//absl/base:nullability", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/synchronization", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "server_request_logger_test", + size = "small", + srcs = ["server_request_logger_test.cc"], + deps = [ + ":request_logger", + ":server_request_logger", + ":stream_logger", + "//tensorflow_serving/apis:logging_cc_proto", + "//tensorflow_serving/apis:model_cc_proto", + "//tensorflow_serving/apis:predict_cc_proto", + "//tensorflow_serving/config:log_collector_config_cc_proto", + "//tensorflow_serving/config:logging_config_cc_proto", + "//tensorflow_serving/core/test_util:fake_log_collector", + "//tensorflow_serving/core/test_util:mock_prediction_stream_logger", + "//tensorflow_serving/core/test_util:mock_request_logger", + "//tensorflow_serving/core/test_util:test_main", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/synchronization", + "@com_google_protobuf//:cc_wkt_protos", + "@local_xla//xla/tsl/platform:status", + "@local_xla//xla/tsl/platform:status_matchers", + "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", + ], +) + +cc_library( + name = "stream_logger", + hdrs = ["stream_logger.h"], + deps = [ + "//tensorflow_serving/apis:logging_cc_proto", + "@com_google_absl//absl/status", + "@com_google_protobuf//:protobuf", + "@org_tensorflow//tensorflow/core:lib", + ], +) diff --git a/tensorflow_serving/core/eager_load_policy.cc b/tensorflow_serving/core/aspired_version_policy.cc similarity index 61% rename from tensorflow_serving/core/eager_load_policy.cc rename to tensorflow_serving/core/aspired_version_policy.cc index b4ff9f1ac97..6abd695cd9d 100644 --- a/tensorflow_serving/core/eager_load_policy.cc +++ b/tensorflow_serving/core/aspired_version_policy.cc @@ -1,4 +1,4 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. +/* Copyright 2017 Google Inc. All Rights Reserved. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -13,28 +13,25 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/aspired_version_policy.h" + +#include namespace tensorflow { namespace serving { -optional EagerLoadPolicy::GetNextAction( - const std::vector& all_versions) const { - // If there is a new aspired version, load it. +absl::optional AspiredVersionPolicy::GetHighestAspiredNewServableId( + const std::vector& all_versions) { + absl::optional highest_version_id; for (const auto& version : all_versions) { if (version.is_aspired && version.state == LoaderHarness::State::kNew) { - return {{Action::kLoad, version.id}}; - } - } - - // If there is no new aspired version, but a not-aspired version, unload the - // latter. - for (const auto& version : all_versions) { - if (!version.is_aspired && version.state == LoaderHarness::State::kReady) { - return {{Action::kUnload, version.id}}; + if (!highest_version_id || + version.id.version > highest_version_id.value().version) { + highest_version_id = version.id; + } } } - return {}; + return highest_version_id; } } // namespace serving diff --git a/tensorflow_serving/core/aspired_version_policy.h b/tensorflow_serving/core/aspired_version_policy.h index 83013fe31be..c0e40ee6bf8 100644 --- a/tensorflow_serving/core/aspired_version_policy.h +++ b/tensorflow_serving/core/aspired_version_policy.h @@ -19,64 +19,72 @@ limitations under the License. #include #include +#include "absl/types/optional.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/loader_harness.h" #include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/util/optional.h" - namespace tensorflow { namespace serving { -// A snapshot of a servable's state and aspiredness. +/// A snapshot of a servable's state and aspiredness. struct AspiredServableStateSnapshot final { ServableId id; LoaderHarness::State state; bool is_aspired; }; -// An interface for the policy to be applied for transitioning servable versions -// in a servable stream. -// -// Policies should be entirely stateless and idempotent. Asking the same policy -// multiple times for the next action, for an identical vector of -// AspiredServableStateSnapshots, should return the same result. -// -// If additional state is required to implement a Policy, such state shall be -// shared via AspiredServableStateSnapshots. Depending on the kind of state, the -// most likely candidates for originating or tracking state are Sources or the -// Harness and Manager. +/// An interface for the policy to be applied for transitioning servable +/// versions in a servable stream. +/// +/// Policies should be entirely stateless and idempotent. Asking the same policy +/// multiple times for the next action, for an identical vector of +/// AspiredServableStateSnapshots, should return the same result. +/// +/// If additional state is required to implement a Policy, such state shall be +/// shared via AspiredServableStateSnapshots. Depending on the kind of state, +/// the most likely candidates for originating or tracking state are Sources or +/// the Harness and Manager. class AspiredVersionPolicy { public: - // The different actions that could be recommended by a policy. + /// The different actions that could be recommended by a policy. enum class Action : int { - // Call load on the servable. + /// Call load on the servable. kLoad, - // Call unload on the servable. + /// Call unload on the servable. kUnload, }; virtual ~AspiredVersionPolicy() = default; - // Action and the id of the servable associated with it. + /// Action and the id of the servable associated with it. struct ServableAction final { Action action; ServableId id; string DebugString() const { - return strings::StrCat("{ action: ", static_cast(action), " id: ", - id.DebugString(), " }"); + return strings::StrCat("{ action: ", static_cast(action), + " id: ", id.DebugString(), " }"); } }; - // Takes in a vector of state snapshots of all versions of a servable stream - // and returns an action to be performed for a particular servable version, - // depending only on the states of all the versions. - // - // If no action is to be performed, we don't return an action, meaning - // that the servable stream is up to date. - virtual optional GetNextAction( + /// Takes in a vector of state snapshots of all versions of a servable stream + /// and returns an action to be performed for a particular servable version, + /// depending only on the states of all the versions. + /// + /// If no action is to be performed, we don't return an action, meaning + /// that the servable stream is up to date. + virtual absl::optional GetNextAction( const std::vector& all_versions) const = 0; + + protected: + /// Returns the aspired ServableId with the highest version that matches + /// kNew state, if any exists. + static absl::optional GetHighestAspiredNewServableId( + const std::vector& all_versions); + + private: + friend class AspiredVersionPolicyTest; }; inline bool operator==(const AspiredVersionPolicy::ServableAction& lhs, diff --git a/tensorflow_serving/core/aspired_version_policy_test.cc b/tensorflow_serving/core/aspired_version_policy_test.cc new file mode 100644 index 00000000000..5cc9d368141 --- /dev/null +++ b/tensorflow_serving/core/aspired_version_policy_test.cc @@ -0,0 +1,64 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow_serving/core/aspired_version_policy.h" + +#include + +#include +#include "absl/types/optional.h" +#include "tensorflow_serving/core/loader_harness.h" +#include "tensorflow_serving/core/servable_id.h" + +namespace tensorflow { +namespace serving { + +class AspiredVersionPolicyTest : public ::testing::Test { + public: + // Expose the protected AspiredVersionPolicy::GetHighestAspiredNewServableId() + // method. + absl::optional GetHighestAspiredNewServableId( + const std::vector& all_versions) { + return AspiredVersionPolicy::GetHighestAspiredNewServableId(all_versions); + } +}; + +TEST_F(AspiredVersionPolicyTest, NoHighestAspiredNewServableId) { + // No new aspired versions. + std::vector versions; + versions.push_back({{"test", 10}, LoaderHarness::State::kNew, false}); + versions.push_back({{"test", 9}, LoaderHarness::State::kUnloading, true}); + absl::optional highest_aspired_new = + GetHighestAspiredNewServableId(versions); + ASSERT_FALSE(highest_aspired_new); +} + +TEST_F(AspiredVersionPolicyTest, HighestAspiredNewServableId) { + // Three new aspired versions and two other versions. + std::vector versions; + versions.push_back({{"test", 10}, LoaderHarness::State::kNew, false}); + versions.push_back({{"test", 9}, LoaderHarness::State::kUnloading, true}); + versions.push_back({{"test", 1}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + absl::optional highest_aspired_new = + GetHighestAspiredNewServableId(versions); + ASSERT_TRUE(highest_aspired_new); + EXPECT_EQ("test", highest_aspired_new.value().name); + EXPECT_EQ(5, highest_aspired_new.value().version); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index e50efd21ea5..11084eb8588 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -19,12 +19,15 @@ limitations under the License. #include #include #include +#include +#include #include #include #include #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/context.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow_serving/core/loader_harness.h" @@ -50,8 +53,13 @@ struct Aspired { // Note that this returns a strict weak ordering. struct CompareActions { public: - bool operator()(const optional& lhs, - const optional& rhs) { + explicit CompareActions( + AspiredVersionsManager::CustomSortActionsFn custom_sort_actions) + : custom_sort_actions_(custom_sort_actions) {} + + bool operator()( + const absl::optional& lhs, + const absl::optional& rhs) { if (!lhs) { return false; } @@ -59,6 +67,9 @@ struct CompareActions { return true; } // By this point, we are sure the optionals have values. + if (custom_sort_actions_) { + return custom_sort_actions_(lhs.value(), rhs.value()); + } return OrderActions(lhs.value(), rhs.value()).action != rhs.value().action; } @@ -76,12 +87,13 @@ struct CompareActions { return lhs; } } + AspiredVersionsManager::CustomSortActionsFn custom_sort_actions_; }; // Validates whether all entries in 'versions' pertain to the servable named // 'servable_name'. -Status ValidateAspiredVersions( - const StringPiece servable_name, +absl::Status ValidateAspiredVersions( + const absl::string_view servable_name, const std::vector>>& versions) { for (const auto& version : versions) { if (servable_name != version.id().name) { @@ -90,24 +102,35 @@ Status ValidateAspiredVersions( " doesn't match name in servable version: ", version.id().name)); } } - return Status::OK(); + return absl::OkStatus(); } // Returns the set of version numbers in 'versions'. -std::set GetVersionNumbers( +std::set GetVersionNumbers( const std::vector>>& versions) { - std::set version_numbers; + std::set version_numbers; for (const auto& version : versions) { version_numbers.insert(version.id().version); } return version_numbers; } +// Creates a debug string for a given vector of servable versions. +string ServableVersionsDebugString( + const std::vector>>& versions) { + std::vector version_strings; + version_strings.reserve(versions.size()); + for (const ServableData>& version : versions) { + version_strings.push_back(version.id().DebugString()); + } + return absl::StrJoin(version_strings, ", "); +} + } // namespace namespace internal { -// AspiredVersionManager's implementation of the Target API. +// AspiredVersionsManager's implementation of the Target API. class AspiredVersionsManagerTargetImpl final : public TargetBase> { public: @@ -118,7 +141,7 @@ class AspiredVersionsManagerTargetImpl final protected: void SetAspiredVersions( - const StringPiece servable_name, + const absl::string_view servable_name, std::vector>> versions) override { parent_->EnqueueAspiredVersionsRequest(servable_name, std::move(versions)); } @@ -132,7 +155,7 @@ class AspiredVersionsManagerTargetImpl final } // namespace internal -Status AspiredVersionsManager::Create( +absl::Status AspiredVersionsManager::Create( Options options, std::unique_ptr* manager) { if (options.aspired_version_policy == nullptr) { return errors::InvalidArgument( @@ -141,39 +164,70 @@ Status AspiredVersionsManager::Create( } BasicManager::Options basic_manager_options; basic_manager_options.resource_tracker = std::move(options.resource_tracker); - basic_manager_options.num_load_unload_threads = - options.num_load_unload_threads; + basic_manager_options.num_load_threads = options.num_load_threads; + basic_manager_options.num_unload_threads = options.num_unload_threads; basic_manager_options.max_num_load_retries = options.max_num_load_retries; basic_manager_options.load_retry_interval_micros = options.load_retry_interval_micros; + basic_manager_options.flush_filesystem_caches = + options.flush_filesystem_caches; + basic_manager_options.env = options.env; basic_manager_options.servable_event_bus = options.servable_event_bus; + basic_manager_options.pre_load_hook = std::move(options.pre_load_hook); + if (options.should_retry_model_load) { + basic_manager_options.should_retry_model_load = + std::move(options.should_retry_model_load); + } std::unique_ptr basic_manager; TF_RETURN_IF_ERROR( BasicManager::Create(std::move(basic_manager_options), &basic_manager)); manager->reset(new AspiredVersionsManager( options.manage_state_interval_micros, options.env, - std::move(options.aspired_version_policy), std::move(basic_manager))); - return Status::OK(); + std::move(options.aspired_version_policy), + std::move(options.custom_sort_actions), std::move(basic_manager), + options.with_current_context)); + (manager->get())->enable_reload_servables_with_error_ = + options.enable_reload_servables_with_error; + return absl::OkStatus(); } AspiredVersionsManager::AspiredVersionsManager( - int64 manage_state_interval_micros, Env* env, + int64_t manage_state_interval_micros, Env* env, std::unique_ptr aspired_version_policy, - std::unique_ptr basic_manager) + CustomSortActionsFn custom_sort_actions, + std::unique_ptr basic_manager, bool with_current_context) : aspired_version_policy_(std::move(aspired_version_policy)), + custom_sort_actions_(std::move(custom_sort_actions)), target_impl_(new internal::AspiredVersionsManagerTargetImpl(this)), basic_manager_(std::move(basic_manager)) { + set_num_load_threads_observer_.reset( + new Observer([this](const uint32 num_load_threads) { + this->SetNumLoadThreads(num_load_threads); + })); if (manage_state_interval_micros > 0) { PeriodicFunction::Options pf_options; pf_options.env = env; pf_options.thread_name_prefix = "AspiredVersionsManager_ManageState_Thread"; - manage_state_thread_.reset(new PeriodicFunction( - [this]() { - this->HandlePendingAspiredVersionsRequests(); - this->InvokePolicyAndExecuteAction(); - }, - manage_state_interval_micros)); + if (with_current_context) { + tensorflow::Context context(tensorflow::ContextKind::kThread); + manage_state_thread_.reset(new PeriodicFunction( + [this, context = std::move(context)]() { + tensorflow::WithContext wc(context); + this->FlushServables(); + this->HandlePendingAspiredVersionsRequests(); + this->InvokePolicyAndExecuteAction(); + }, + manage_state_interval_micros)); + } else { + manage_state_thread_.reset(new PeriodicFunction( + [this]() { + this->FlushServables(); + this->HandlePendingAspiredVersionsRequests(); + this->InvokePolicyAndExecuteAction(); + }, + manage_state_interval_micros)); + } } } @@ -191,7 +245,7 @@ std::vector AspiredVersionsManager::ListAvailableServableIds() return basic_manager_->ListAvailableServableIds(); } -Status AspiredVersionsManager::GetUntypedServableHandle( +absl::Status AspiredVersionsManager::GetUntypedServableHandle( const ServableRequest& request, std::unique_ptr* const untyped_handle) { return basic_manager_->GetUntypedServableHandle(request, untyped_handle); @@ -208,43 +262,53 @@ AspiredVersionsManager::GetAspiredVersionsCallback() { } void AspiredVersionsManager::EnqueueAspiredVersionsRequest( - const StringPiece servable_name, + const absl::string_view servable_name, std::vector>> versions) { - const Status validation_status = + const absl::Status validation_status = ValidateAspiredVersions(servable_name, versions); - DCHECK(validation_status.ok()) << validation_status.error_message(); + DCHECK(validation_status.ok()) << validation_status.message(); if (!validation_status.ok()) { - LOG(ERROR) << validation_status.error_message(); + LOG(ERROR) << validation_status.message(); return; } { mutex_lock l(pending_aspired_versions_requests_mu_); - pending_aspired_versions_requests_[servable_name.ToString()] = + VLOG(2) << "Enqueueing aspired versions request: " << servable_name << ": " + << ServableVersionsDebugString(versions); + pending_aspired_versions_requests_[string(servable_name)] = std::move(versions); } } void AspiredVersionsManager::ProcessAspiredVersionsRequest( - const StringPiece servable_name, + const absl::string_view servable_name, std::vector>> versions) { - const std::set next_aspired_versions = GetVersionNumbers(versions); + VLOG(2) << "Processing aspired versions request: " << servable_name << ": " + << ServableVersionsDebugString(versions); + + const std::set next_aspired_versions = GetVersionNumbers(versions); // We gather all the servables with the servable_name and // 1. Add the current aspired version numbers to a set, // 2. Set the aspired bool to false for all current servable harnesses which // are not aspired. - std::set current_aspired_versions; + std::set current_aspired_versions; + std::set current_aspired_versions_with_error; const std::vector> state_snapshots = basic_manager_->GetManagedServableStateSnapshots( - servable_name.ToString()); - for (const ServableStateSnapshot state_snapshot : state_snapshots) { + string(servable_name)); + for (const ServableStateSnapshot& state_snapshot : state_snapshots) { if (state_snapshot.additional_state->is_aspired) { current_aspired_versions.insert(state_snapshot.id.version); + if (state_snapshot.state == LoaderHarness::State::kError) { + current_aspired_versions_with_error.insert(state_snapshot.id.version); + } } // If this version is not part of the aspired versions. - if (std::find(next_aspired_versions.begin(), next_aspired_versions.end(), - state_snapshot.id.version) == next_aspired_versions.end()) { + if (next_aspired_versions.find(state_snapshot.id.version) == + next_aspired_versions.end()) { + VLOG(1) << "Setting is_aspired=false for " << state_snapshot.id; basic_manager_->GetAdditionalServableState(state_snapshot.id) ->is_aspired = false; basic_manager_->CancelLoadServableRetry(state_snapshot.id); @@ -254,7 +318,7 @@ void AspiredVersionsManager::ProcessAspiredVersionsRequest( // We do a set_difference (A - B), on the next aspired versions and the // current aspired versions to find the version numbers which need to be // added the harness map. - std::set additions; + std::set additions; std::set_difference( next_aspired_versions.begin(), next_aspired_versions.end(), current_aspired_versions.begin(), current_aspired_versions.end(), @@ -263,29 +327,50 @@ void AspiredVersionsManager::ProcessAspiredVersionsRequest( // We go through the aspired_servable_versions, pull out the versions which // need to be added and add them to the harness map. for (auto& version : versions) { + bool should_add = false; + const auto& version_id = version.id(); + if (additions.find(version.id().version) != additions.end()) { + should_add = true; + } + if (enable_reload_servables_with_error_ && + current_aspired_versions_with_error.find(version.id().version) != + current_aspired_versions_with_error.end()) { + ServableId id; + id.name = std::string(servable_name); + id.version = version_id.version; + const absl::Status manage_status = + basic_manager_->StopManagingServable(id); + DCHECK(manage_status.ok()) << manage_status.message(); + if (!manage_status.ok()) { + LOG(ERROR) << "Internal error: Unable to clear errored servable " + << version_id.DebugString() + << " from 'basic_manager_': " << manage_status.message(); + } + should_add = true; + } + // if this aspired version is not already present in the map. - if (std::find(additions.begin(), additions.end(), version.id().version) != - additions.end()) { - const Status manage_status = + if (should_add) { + const absl::Status manage_status = basic_manager_->ManageServableWithAdditionalState( std::move(version), std::unique_ptr(new Aspired{true})); - DCHECK(manage_status.ok()) << manage_status.error_message(); + DCHECK(manage_status.ok()) << manage_status.message(); if (!manage_status.ok()) { LOG(ERROR) << "Internal error: Unable to transfer servable " - << version.id().DebugString() - << " to 'basic_manager_': " << manage_status.error_message(); + << version_id.DebugString() + << " to 'basic_manager_': " << manage_status.message(); } } } } bool AspiredVersionsManager::ContainsAnyReaspiredVersions( - const StringPiece servable_name, + const absl::string_view servable_name, const std::vector>>& versions) const { const std::vector> state_snapshots = basic_manager_->GetManagedServableStateSnapshots( - servable_name.ToString()); - const std::set version_numbers = GetVersionNumbers(versions); + string(servable_name)); + const std::set version_numbers = GetVersionNumbers(versions); for (const ServableStateSnapshot& state_snapshot : state_snapshots) { if (!state_snapshot.additional_state->is_aspired && version_numbers.find(state_snapshot.id.version) != @@ -298,27 +383,30 @@ bool AspiredVersionsManager::ContainsAnyReaspiredVersions( // We collect the version policy actions for each servable stream first. Then // we sort them based on the global policy and pick the first one. -optional +absl::optional AspiredVersionsManager::GetNextAction() { - std::vector> actions; - std::vector aspired_state_snapshots; + std::vector> actions; for (const string& servable_name : basic_manager_->GetManagedServableNames()) { - aspired_state_snapshots.clear(); + std::vector aspired_state_snapshots; for (const ServableStateSnapshot& state_snapshot : basic_manager_->GetManagedServableStateSnapshots( servable_name)) { aspired_state_snapshots.push_back( {state_snapshot.id, state_snapshot.state, state_snapshot.additional_state->is_aspired}); - actions.emplace_back( - aspired_version_policy_->GetNextAction(aspired_state_snapshots)); } + actions.emplace_back( + aspired_version_policy_->GetNextAction(aspired_state_snapshots)); } - std::sort(actions.begin(), actions.end(), CompareActions()); - const optional next_action = - !actions.empty() ? actions[0] : nullopt; + std::sort(actions.begin(), actions.end(), + CompareActions(custom_sort_actions_)); + const absl::optional next_action = + !actions.empty() ? actions[0] : absl::nullopt; + if (next_action) { + VLOG(1) << "Taking action: " << next_action->DebugString(); + } return next_action; } @@ -326,24 +414,54 @@ void AspiredVersionsManager::PerformAction( const AspiredVersionPolicy::ServableAction action) { switch (action.action) { case AspiredVersionPolicy::Action::kLoad: { - basic_manager_->LoadServable(action.id, [action](const Status& status) { - if (!status.ok()) { - LOG(ERROR) << "Servable " << action.id.DebugString() - << " cannot be loaded: " << status; - } - }); + basic_manager_->LoadServable( + action.id, [action](const absl::Status& status) { + if (!status.ok()) { + LOG(ERROR) << "Servable " << action.id.DebugString() + << " cannot be loaded: " << status; + } + }); } break; case AspiredVersionPolicy::Action::kUnload: { - basic_manager_->UnloadServable(action.id, [action](const Status& status) { - if (!status.ok()) { - LOG(ERROR) << "Servable " << action.id.DebugString() - << " cannot be unloaded: " << status; - } - }); + basic_manager_->UnloadServable( + action.id, [action](const absl::Status& status) { + if (!status.ok()) { + LOG(ERROR) << "Servable " << action.id.DebugString() + << " cannot be unloaded: " << status; + } + }); } break; } } +void AspiredVersionsManager::FlushServables() { + mutex_lock l(basic_manager_read_modify_write_mu_); + for (const string& servable_name : + basic_manager_->GetManagedServableNames()) { + for (const ServableStateSnapshot& state_snapshot : + basic_manager_->GetManagedServableStateSnapshots( + servable_name)) { + if ((state_snapshot.state == LoaderHarness::State::kNew || + state_snapshot.state == LoaderHarness::State::kDisabled || + state_snapshot.state == LoaderHarness::State::kError) && + !state_snapshot.additional_state->is_aspired) { + const absl::Status status = + basic_manager_->StopManagingServable(state_snapshot.id); + if (status.ok()) { + VLOG(1) << "Removed " << state_snapshot.id << "from BasicManager"; + } else { + // This scenario is likely a bug, perhaps a race (either here in + // AspiredVersionsManager, or in BasicManager). We'll wind up retrying + // StopManagingServable() on the next FlushServables() call, so just + // log the error and move on for now. + LOG(ERROR) << "Error removing " << state_snapshot.id + << "from BasicManager: " << status << " will retry later"; + } + } + } + } +} + void AspiredVersionsManager::HandlePendingAspiredVersionsRequests() { mutex_lock l(basic_manager_read_modify_write_mu_); mutex_lock l2(pending_aspired_versions_requests_mu_); @@ -361,6 +479,9 @@ void AspiredVersionsManager::HandlePendingAspiredVersionsRequests() { if (ContainsAnyReaspiredVersions(servable_name, versions)) { // Sit on it for now. We'll check again later. ++it; + VLOG(1) << "Postponing processing of aspired versions request due to " + "re-aspired version(s) among: " + << ServableVersionsDebugString(versions); } else { ProcessAspiredVersionsRequest(servable_name, std::move(versions)); it = pending_aspired_versions_requests_.erase(it); @@ -371,7 +492,7 @@ void AspiredVersionsManager::HandlePendingAspiredVersionsRequests() { void AspiredVersionsManager::InvokePolicyAndExecuteAction() { mutex_lock l(basic_manager_read_modify_write_mu_); - const optional next_action = + const absl::optional next_action = GetNextAction(); if (!next_action) { return; @@ -381,5 +502,13 @@ void AspiredVersionsManager::InvokePolicyAndExecuteAction() { PerformAction(*next_action); } +void AspiredVersionsManager::SetNumLoadThreads(const uint32 num_load_threads) { + basic_manager_->SetNumLoadThreads(num_load_threads); +} + +uint32 AspiredVersionsManager::num_load_threads() const { + return basic_manager_->num_load_threads(); +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index f78e2300933..e72bc6cdce1 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -21,6 +21,8 @@ limitations under the License. #include #include +#include "absl/types/optional.h" +#include "tensorflow/core/kernels/batching_util/periodic_function.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/hash/hash.h" @@ -32,79 +34,133 @@ limitations under the License. #include "tensorflow_serving/core/basic_manager.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_id.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/target.h" -#include "tensorflow_serving/util/optional.h" -#include "tensorflow_serving/util/periodic_function.h" +#include "tensorflow_serving/util/event_bus.h" +#include "tensorflow_serving/util/observer.h" namespace tensorflow { namespace serving { +class AspiredVersionsManager; + namespace internal { + class AspiredVersionsManagerTargetImpl; + +uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager); + +// Returns the Notifier function of the manager's Observer, which forwards +// SetNumLoadThreads(). This indirection is to prevent callers from using +// SetNumLoadThreads() on a deleted manager. +std::function SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager); + } // namespace internal namespace test_util { class AspiredVersionsManagerTestAccess; } // namespace test_util -// A manager that implements the Target API which uses aspired-versions -// callbacks to dictate which servable versions to load. This manager also uses -// that API to infer which ones to unload: If a given servable version is -// currently loaded, and is omitted from an aspired-versions callback invocation -// pertaining to its servable stream, this manager interprets that omission as -// an implicit instruction to unload the version. See below for details. -// -// (The implicit-unload semantics facilitates stateless Source implementations, -// whereby a given iteration of the Source's logic simply decides which versions -// of a servable ought to be loaded, without needing to know what it has decided -// in the past.) -// -// This manager makes transitions between versions of a servable stream using a -// configured AspiredVersionPolicy. The manager prefers unloading before loading -// to free up resources in the server when deciding among transitions suggested -// by the policy. +/// A manager that implements the Target&lt;Loader> API which uses +/// aspired-versions callbacks to dictate which servable versions to load. This +/// manager also uses that API to infer which ones to unload: If a given +/// servable version is currently loaded, and is omitted from an +/// aspired-versions callback invocation pertaining to its servable stream, this +/// manager interprets that omission as an implicit instruction to unload the +/// version. See below for details. +/// +/// (The implicit-unload semantics facilitates stateless Source implementations, +/// whereby a given iteration of the Source's logic simply decides which +/// versions of a servable ought to be loaded, without needing to know what it +/// has decided in the past.) +/// +/// This manager makes transitions between versions of a servable stream using a +/// configured AspiredVersionPolicy. The manager prefers unloading before +/// loading to free up resources in the server when deciding among transitions +/// suggested by the policy. class AspiredVersionsManager : public Manager, public Target> { public: + using PreLoadHook = BasicManager::PreLoadHook; + + using CustomSortActionsFn = + std::function; + + /// Config options and pluggable objects that will be used by the + /// AspiredVersionsManager. struct Options { - // The resource tracker to use while managing servable resources. Optional. - // If left as nullptr, we do not validate servable resource usage. + /// The resource tracker to use while managing servable resources. Optional. + /// If left as nullptr, we do not validate servable resource usage. std::unique_ptr resource_tracker; - // The periodicity, in microseconds, of the thread which manages the state - // of the servables. Default: 100 milliseconds. If this is set less than or - // equal to 0, we don't run this thread at all. - int64 manage_state_interval_micros = 100 * 1000; + /// The periodicity, in microseconds, of the thread which manages the state + /// of the servables. Default: 100 milliseconds. If this is set less than or + /// equal to 0, we don't run this thread at all. + int64_t manage_state_interval_micros = 100 * 1000; - // EventBus to publish servable state changes. This is optional, if unset, - // we don't publish. + /// EventBus to publish servable state changes. This is optional, if unset, + /// we don't publish. EventBus* servable_event_bus = nullptr; - // The AspiredVersionPolicy to use for the manager. Must be non-null. + /// The AspiredVersionPolicy to use for the manager. Must be non-null. std::unique_ptr aspired_version_policy; - // The number of threads in the thread-pool used to load and unload - // servables. - // - // If set as 0, we don't use a thread-pool, and the {Load,Unload}Servable() - // methods block. - uint32 num_load_unload_threads = 0; - - // Maximum number of times we retry loading a servable, after the first - // failure, before we give up. + /// Given a list of ServableAction, each ServableAction representing the + /// chosen version for that servable, this provides a custom sort order on + /// which action to take first. Useful when certain servable needs to be + /// loaded or unloaded before some other servable + CustomSortActionsFn custom_sort_actions; + + /// The number of threads in the thread-pool used to load servables. + /// + /// If set as 0, we don't use a thread-pool, and servable loads are + /// performed serially in the manager's main work loop. + uint32 num_load_threads = 0; + + /// The number of threads in the thread-pool used to unload servables. + /// + /// If set as 0, we don't use a thread-pool, and servable unloads are + /// performed serially in the manager's main work loop. + uint32 num_unload_threads = 0; + + /// Maximum number of times we retry loading a servable, after the first + /// failure, before we give up. uint32 max_num_load_retries = 5; - // The interval, in microseconds, between each servable load retry. If set - // negative, we don't wait. - // Default: 1 minute. - int64 load_retry_interval_micros = 1LL * 60 * 1000 * 1000; + /// The interval, in microseconds, between each servable load retry. If set + /// negative, we don't wait. + /// Default: 1 minute. + int64_t load_retry_interval_micros = 1LL * 60 * 1000 * 1000; + + // Defines how we want to retry when model loading fails. + std::function should_retry_model_load; + + // If true, and there are not multiple load threads, filesystem caches will + // be flushed after each servable is loaded. (Cache flush is skipped when + // multiple load threads are active, in order to avoid setting back a + // concurrent load on another thread.) + bool flush_filesystem_caches = false; - // The environment to use for starting threads in the thread-pool or for - // sleeping. + /// The environment to use for starting threads in the thread-pool or for + /// sleeping. Env* env = Env::Default(); + + /// Callback to be called just before a servable is to be loaded. This will + /// called on the same manager load thread which starts the load. + PreLoadHook pre_load_hook; + + // For servables which end with LoaderHarness::State::kError, enable + // future attempts at reload to progress. + bool enable_reload_servables_with_error = false; + + // If true, the AspiredVersionsManager will propagate its current context to + // the newly created periodic functions. + bool with_current_context = false; }; static Status Create(Options options, std::unique_ptr* manager); @@ -112,9 +168,11 @@ class AspiredVersionsManager : public Manager, std::vector ListAvailableServableIds() const override; - // Returns a callback to set the list of aspired versions for a particular - // servable stream, using Loaders. AspiredVersionsManager's semantics with - // respect to this callback are as follows: + /// \brief Returns a callback to set the list of aspired versions for a + /// particular servable stream, using Loaders. + // + // AspiredVersionsManager's semantics with respect to this callback are as + // follows: // // 1. OMITTING A VERSION INSTRUCTS THE MANAGER TO UNLOAD IT // @@ -170,11 +228,17 @@ class AspiredVersionsManager : public Manager, private: friend class internal::AspiredVersionsManagerTargetImpl; friend class test_util::AspiredVersionsManagerTestAccess; + friend class ServerCore; + friend uint32 internal::GetManagerNumLoadThreads( + AspiredVersionsManager* manager); + friend std::function internal::SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager); AspiredVersionsManager( - int64 manage_state_interval_micros, Env* env, + int64_t manage_state_interval_micros, Env* env, std::unique_ptr aspired_version_policy, - std::unique_ptr basic_manager); + CustomSortActionsFn custom_sort_actions, + std::unique_ptr basic_manager, bool with_current_context); Status GetUntypedServableHandle( const ServableRequest& request, @@ -188,7 +252,7 @@ class AspiredVersionsManager : public Manager, void EnqueueAspiredVersionsRequest( const StringPiece servable_name, std::vector>> versions) - LOCKS_EXCLUDED(pending_aspired_versions_requests_mu_); + TF_LOCKS_EXCLUDED(pending_aspired_versions_requests_mu_); // Processes an aspired-versions request. It assumes the request doesn't // re-aspire any servables currently marked as not aspired in @@ -196,37 +260,51 @@ class AspiredVersionsManager : public Manager, void ProcessAspiredVersionsRequest( const StringPiece servable_name, std::vector>> versions) - EXCLUSIVE_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); + TF_EXCLUSIVE_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); // Determines whether an aspired-versions request contains any versions that // are currently being managed in 'basic_manager_' with is_aspired==false. bool ContainsAnyReaspiredVersions( const StringPiece servable_name, const std::vector>>& versions) const - SHARED_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); + TF_SHARED_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); // Performs the action on the harness. void PerformAction(const AspiredVersionPolicy::ServableAction action) - EXCLUSIVE_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); + TF_EXCLUSIVE_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); // Goes through the harness map and calls the configured servable_policy with // the state snapshots to get a list of suggested actions. The actions are // then ordered and finally the topmost one is performed. - optional GetNextAction() - EXCLUSIVE_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); + absl::optional GetNextAction() + TF_EXCLUSIVE_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); + + // Checks for servables that are not aspired and at some final state and tells + // 'basic_manager_' to forget about them. This method is intended to be + // invoked periodically, interleaved with InvokePolicyAndExecuteAction() and + // HandlePendingAspiredVersionsRequests(). + void FlushServables() TF_LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_); // Handles enqueued aspired-versions requests. This method is intended to be // invoked periodically, interleaved with InvokePolicyAndExecuteAction(). void HandlePendingAspiredVersionsRequests() - LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_, - pending_aspired_versions_requests_mu_); + TF_LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_, + pending_aspired_versions_requests_mu_); // Invokes the aspired-version policy and executes any returned policy action. // This method is intended to be invoked periodically. void InvokePolicyAndExecuteAction() - LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_); + TF_LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_); + + // Sets the number of load threads. + // + // This may block all new load requests, or temporarily allow more threads to + // start, before it returns. See BasicManager::SetNumLoadThreads for details + void SetNumLoadThreads(uint32 num_load_threads); + uint32 num_load_threads() const; std::unique_ptr aspired_version_policy_; + CustomSortActionsFn custom_sort_actions_; // Aspired-versions requests pending to be processed, keyed by servable name. // @@ -239,7 +317,7 @@ class AspiredVersionsManager : public Manager, using AspiredVersionsMap = std::map>>>; AspiredVersionsMap pending_aspired_versions_requests_ - GUARDED_BY(pending_aspired_versions_requests_mu_); + TF_GUARDED_BY(pending_aspired_versions_requests_mu_); mutable mutex pending_aspired_versions_requests_mu_; // To lock basic_manager_ to perform atomic read/modify/write operations on @@ -256,6 +334,14 @@ class AspiredVersionsManager : public Manager, // This is where the servables "live" while they are being managed. std::unique_ptr basic_manager_; + // An observer object that forwards to SetNumLoadThreads(), if not detached. + // This is declared last here so that it is deleted before basic_manager_. + std::unique_ptr> set_num_load_threads_observer_; + + // For servables which end with LoaderHarness::State::kError, enable + // future attempts at reload to progress. + bool enable_reload_servables_with_error_ = false; + TF_DISALLOW_COPY_AND_ASSIGN(AspiredVersionsManager); }; diff --git a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc index 85ffbe37cb1..d6bb568e823 100644 --- a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc +++ b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc @@ -16,7 +16,7 @@ limitations under the License. // Run with: // bazel run -c opt --dynamic_mode=off \ // tensorflow_serving/core:aspired_versions_manager_benchmark -- -// --benchmarks=. --benchmark_use_picoseconds +// --benchmarks=. // For a longer run time and more consistent results, consider a min time // e.g.: --benchmark_min_time=60.0 @@ -24,8 +24,10 @@ limitations under the License. #include #include #include +#include #include +#include "tensorflow/core/kernels/batching_util/periodic_function.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/threadpool.h" @@ -41,14 +43,13 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/aspired_version_policy.h" #include "tensorflow_serving/core/aspired_versions_manager.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/manager.h" #include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/simple_loader.h" #include "tensorflow_serving/core/test_util/manager_test_util.h" -#include "tensorflow_serving/util/periodic_function.h" namespace tensorflow { namespace serving { @@ -74,12 +75,12 @@ class BenchmarkState { AspiredVersionsManager::Options options; // Do policy thread won't be run automatically. options.manage_state_interval_micros = -1; - options.aspired_version_policy.reset(new EagerLoadPolicy()); + options.aspired_version_policy.reset(new AvailabilityPreservingPolicy()); TF_CHECK_OK(AspiredVersionsManager::Create(std::move(options), &manager_)); } // Actually perform iters reads on the fast read ptr. - void RunBenchmark(int iters, int num_threads); + void RunBenchmark(::testing::benchmark::State& state, int num_threads); private: void SetUp(); @@ -93,10 +94,10 @@ class BenchmarkState { void RunUpdate(); // Starts serving this loader version. - void StartServing(int64 loader_version); + void StartServing(int64_t loader_version); // Gets the latest version of the loader available for serving. - int64 GetLatestVersion(bool do_work); + int64_t GetLatestVersion(bool do_work); // To avoid having the benchmark timing include time spent scheduling threads, // we use this notification to notify when the read threads should begin. @@ -119,14 +120,14 @@ class BenchmarkState { bool do_work_; }; -void BenchmarkState::StartServing(const int64 loader_version) { - std::unique_ptr loader(new SimpleLoader( - [loader_version](std::unique_ptr* const servable) { - servable->reset(new int64); +void BenchmarkState::StartServing(const int64_t loader_version) { + std::unique_ptr loader(new SimpleLoader( + [loader_version](std::unique_ptr* const servable) { + servable->reset(new int64_t); **servable = loader_version; - return Status::OK(); + return OkStatus(); }, - SimpleLoader::EstimateNoResources())); + SimpleLoader::EstimateNoResources())); std::vector>> versions; versions.push_back({{kServableName, loader_version}, std::move(loader)}); manager_->GetAspiredVersionsCallback()(kServableName, std::move(versions)); @@ -144,8 +145,8 @@ void BenchmarkState::StartServing(const int64 loader_version) { CHECK_EQ(1, manager_->ListAvailableServableIds().size()); } -int64 BenchmarkState::GetLatestVersion(const bool do_work) { - ServableHandle handle; +int64_t BenchmarkState::GetLatestVersion(const bool do_work) { + ServableHandle handle; const Status status = manager_->GetServableHandle( ServableRequest::Latest(kServableName), &handle); TF_CHECK_OK(status) << status; @@ -165,10 +166,7 @@ int64 BenchmarkState::GetLatestVersion(const bool do_work) { void BenchmarkState::RunUpdate() { StartServing(GetLatestVersion(false) + 1); } void BenchmarkState::SetUp() { - testing::StopTiming(); - StartServing(0); - if (interval_micros_ > 0) { PeriodicFunction::Options pf_options; pf_options.thread_name_prefix = @@ -176,87 +174,107 @@ void BenchmarkState::SetUp() { update_thread_.reset(new PeriodicFunction([this] { RunUpdate(); }, interval_micros_, pf_options)); } - - testing::StartTiming(); } void BenchmarkState::TearDown() { - testing::StopTiming(); - // Destruct the update thread which blocks until it exits. update_thread_.reset(); - - testing::StartTiming(); } void BenchmarkState::RunReads(int iters) { - for (int i = 0; i < iters; i++) { + for (int i = 0; i < iters; ++i) { // Prevents the compiler from optimizing this away. CHECK_GE(GetLatestVersion(do_work_), 0); } } -void BenchmarkState::RunBenchmark(int iters, int num_threads) { +void BenchmarkState::RunBenchmark(::testing::benchmark::State& state, + int num_threads) { SetUp(); - testing::StopTiming(); - - // The benchmarking system by default uses cpu time to calculate items per - // second, which would include time spent by all the threads on the cpu. - // Instead of that we use real-time here so that we can see items/s increasing - // with increasing threads, which is easier to understand. - testing::UseRealTime(); - testing::ItemsProcessed(num_threads * iters); - - std::unique_ptr pool(new thread::ThreadPool( - Env::Default(), "RunBenchmarkReadThread", num_threads)); - for (int thread_index = 0; thread_index < num_threads; ++thread_index) { - std::function run_reads_fn = [&]() { - // Wait until all_read_threads_scheduled_ has been notified. - all_read_threads_scheduled_.WaitForNotification(); - RunReads(iters); - }; - pool->Schedule(run_reads_fn); + // To be compatible with the Google benchmark framework, the tensorflow new + // benchmark API requires that each benchmark routine has exactly one. + // `for (auto s : state)` benchmark loop (in all threads). + // Therefore we cannot have multiple threads executing the same for-each loop. + // We need to introduce a new parameter for the fixed number of iteration in + // each thread. + + // Pick a reasonably large value. + const int kSubIters = 500; + + // The benchmark timing loop. Timer automatically starts/stops. + // In each iteration, we spin up a thread-pool and execute kSubIters in each + // thread. + for (auto s : state) { + // Exclude the scheduling setup time. + state.PauseTiming(); + std::unique_ptr pool(new thread::ThreadPool( + Env::Default(), "RunBenchmarkReadThread", num_threads)); + for (int thread_index = 0; thread_index < num_threads; ++thread_index) { + std::function run_reads_fn = [&]() { + // Wait until all_read_threads_scheduled_ has been notified. + all_read_threads_scheduled_.WaitForNotification(); + RunReads(kSubIters); + }; + pool->Schedule(run_reads_fn); + } + state.ResumeTiming(); + if (!all_read_threads_scheduled_.HasBeenNotified()) + all_read_threads_scheduled_.Notify(); + + // Note that destructing the threadpool blocks on completion of all + // scheduled execution. This is intentional as we want all threads to + // complete iters iterations. It also means that the timing may be off + // (work done == iters * num_threads) and includes time scheduling work on + // the threads. + pool.reset(); } - testing::StartTiming(); - all_read_threads_scheduled_.Notify(); - - // Note that destructing the threadpool blocks on completion of all scheduled - // execution. This is intentional as we want all threads to complete iters - // iterations. It also means that the timing may be off (work done == iters * - // num_threads) and includes time scheduling work on the threads. - pool.reset(); + state.SetItemsProcessed(num_threads * kSubIters * state.iterations()); TearDown(); } -static void BenchmarkReadsAndUpdates(int iters, int num_threads, - int interval_micros, bool do_work) { - BenchmarkState state(interval_micros, do_work); - state.RunBenchmark(iters, num_threads); +void BenchmarkReadsAndUpdates(::testing::benchmark::State& state, + int num_threads, int interval_micros, + bool do_work) { + BenchmarkState bm_state(interval_micros, do_work); + bm_state.RunBenchmark(state, num_threads); } -static void BM_Work_NoUpdates_Reads(int iters, int num_threads) { +void BM_Work_NoUpdates_Reads(::testing::benchmark::State& state) { + const int num_threads = state.range(0); + // No updates. 0 interval_micros signals not to update at all. - BenchmarkReadsAndUpdates(iters, num_threads, 0, true); + BenchmarkReadsAndUpdates(state, num_threads, 0, true); } -static void BM_Work_FrequentUpdates_Reads(int iters, int num_threads) { +void BM_Work_FrequentUpdates_Reads(::testing::benchmark::State& state) { + const int num_threads = state.range(0); + // Frequent updates: 1000 micros == 1 millisecond or 1000qps of updates - BenchmarkReadsAndUpdates(iters, num_threads, 1000, true); + BenchmarkReadsAndUpdates(state, num_threads, 1000, true); } -static void BM_NoWork_NoUpdates_Reads(int iters, int num_threads) { +void BM_NoWork_NoUpdates_Reads(::testing::benchmark::State& state) { + const int num_threads = state.range(0); + // No updates. 0 interval_micros signals not to update at all. - BenchmarkReadsAndUpdates(iters, num_threads, 0, false); + BenchmarkReadsAndUpdates(state, num_threads, 0, false); } -static void BM_NoWork_FrequentUpdates_Reads(int iters, int num_threads) { +void BM_NoWork_FrequentUpdates_Reads(::testing::benchmark::State& state) { + const int num_threads = state.range(0); + // Frequent updates: 1000 micros == 1 millisecond or 1000qps of updates - BenchmarkReadsAndUpdates(iters, num_threads, 1000, false); + BenchmarkReadsAndUpdates(state, num_threads, 1000, false); } +// The benchmarking system by default uses cpu time to calculate items per +// second, which would include time spent by all the threads on the cpu. +// Instead of that we use real-time here so that we can see items/s increasing +// with increasing threads, which is easier to understand. BENCHMARK(BM_Work_NoUpdates_Reads) + ->UseRealTime() ->Arg(1) ->Arg(2) ->Arg(4) @@ -266,6 +284,7 @@ BENCHMARK(BM_Work_NoUpdates_Reads) ->Arg(64); BENCHMARK(BM_Work_FrequentUpdates_Reads) + ->UseRealTime() ->Arg(1) ->Arg(2) ->Arg(4) @@ -275,6 +294,7 @@ BENCHMARK(BM_Work_FrequentUpdates_Reads) ->Arg(64); BENCHMARK(BM_NoWork_NoUpdates_Reads) + ->UseRealTime() ->Arg(1) ->Arg(2) ->Arg(4) @@ -284,6 +304,7 @@ BENCHMARK(BM_NoWork_NoUpdates_Reads) ->Arg(64); BENCHMARK(BM_NoWork_FrequentUpdates_Reads) + ->UseRealTime() ->Arg(1) ->Arg(2) ->Arg(4) @@ -292,9 +313,7 @@ BENCHMARK(BM_NoWork_FrequentUpdates_Reads) ->Arg(32) ->Arg(64); -static void BM_GetServableHandle(const int iters) { - testing::StopTiming(); - +void BM_GetServableHandle(::testing::benchmark::State& state) { // Number of different servable streams. constexpr int kNumServableStreams = 10; // Number of versions of a particular servable stream. @@ -304,7 +323,7 @@ static void BM_GetServableHandle(const int iters) { AspiredVersionsManager::Options options; // Do policy thread won't be run automatically. options.manage_state_interval_micros = -1; - options.aspired_version_policy.reset(new EagerLoadPolicy()); + options.aspired_version_policy.reset(new AvailabilityPreservingPolicy()); std::unique_ptr manager; TF_CHECK_OK(AspiredVersionsManager::Create(std::move(options), &manager)); auto aspired_versions_callback = manager->GetAspiredVersionsCallback(); @@ -312,13 +331,13 @@ static void BM_GetServableHandle(const int iters) { const string servable_name = strings::StrCat(kServableName, i); std::vector>> versions; for (int j = 0; j < kNumServableVersions; ++j) { - std::unique_ptr loader(new SimpleLoader( - [j](std::unique_ptr* const servable) { - servable->reset(new int64); + std::unique_ptr loader(new SimpleLoader( + [j](std::unique_ptr* const servable) { + servable->reset(new int64_t); **servable = j; - return Status::OK(); + return OkStatus(); }, - SimpleLoader::EstimateNoResources())); + SimpleLoader::EstimateNoResources())); versions.push_back({{servable_name, j}, std::move(loader)}); } @@ -339,32 +358,32 @@ static void BM_GetServableHandle(const int iters) { // Ratio of requests which are asking for the latest servable as opposed to a // specific version. constexpr float kLatestRatio = 0.8; - static const std::vector& requests = []() { - std::unique_ptr> requests( - new std::vector()); + static const std::vector* requests = []() { + std::vector* requests(new std::vector()); random::PhiloxRandom philox(testing::RandomSeed()); random::SimplePhilox random(&philox); for (int i = 0; i < kNumRequests; ++i) { const string name = strings::StrCat(kServableName, random.Uniform(kNumServableStreams)); if (random.RandFloat() > kLatestRatio) { - const int64 version = random.Uniform(kNumServableVersions); + const int64_t version = random.Uniform(kNumServableVersions); requests->push_back(ServableRequest::Specific(name, version)); } else { requests->push_back(ServableRequest::Latest(name)); } } - return *requests.release(); + return requests; }(); - ServableHandle handle; - testing::ItemsProcessed(iters); - testing::StartTiming(); - for (int i = 0; i < iters; ++i) { + ServableHandle handle; + int i = 0; + for (auto s : state) { const Status status = - manager->GetServableHandle(requests[i % kNumRequests], &handle); + manager->GetServableHandle(requests->at(i % kNumRequests), &handle); TF_CHECK_OK(status) << status; + ++i; } + state.SetItemsProcessed(state.iterations()); } BENCHMARK(BM_GetServableHandle); diff --git a/tensorflow_serving/core/aspired_versions_manager_builder.cc b/tensorflow_serving/core/aspired_versions_manager_builder.cc index 2bd1540ebeb..f84f0daa065 100644 --- a/tensorflow_serving/core/aspired_versions_manager_builder.cc +++ b/tensorflow_serving/core/aspired_versions_manager_builder.cc @@ -15,6 +15,9 @@ limitations under the License. #include "tensorflow_serving/core/aspired_versions_manager_builder.h" +#include +#include + #include "tensorflow_serving/core/manager_wrapper.h" namespace tensorflow { @@ -27,7 +30,7 @@ Status AspiredVersionsManagerBuilder::Create( &aspired_versions_manager)); builder->reset( new AspiredVersionsManagerBuilder(std::move(aspired_versions_manager))); - return Status::OK(); + return absl::OkStatus(); } AspiredVersionsManagerBuilder::AspiredVersionsManagerBuilder( diff --git a/tensorflow_serving/core/aspired_versions_manager_builder.h b/tensorflow_serving/core/aspired_versions_manager_builder.h index 9f8be093ea9..e2ab9c9cdb9 100644 --- a/tensorflow_serving/core/aspired_versions_manager_builder.h +++ b/tensorflow_serving/core/aspired_versions_manager_builder.h @@ -28,31 +28,34 @@ limitations under the License. namespace tensorflow { namespace serving { -// Builds an AspiredVersionsManager with options and sources connected to it. -// It takes over the ownership of the sources and the returned manager handles -// the destruction of itself and its dependencies. Both single sources and -// source/source-adapter chains are accepted, i.e. you can use sources that -// directly supply loaders (Source) or composites that -// consist of Source + some chain of SourceAdapter, ..., -// SourceAdapter<..., std::unique_ptr>. The builder connects the chain -// for you. -// -// Usage: -// ... -// AspiredVersionsManagerBuilder::Options options = ManagerOptions(); -// std::unique_ptr builder; -// TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( -// std::move(options), &builder)); -// builder->AddSource(std::move(some_source)); -// builder->AddSourceChain( -// std::move(source), std::move(source_adapter1), -// std::move(source_adapter2)); -// std::unique_ptr manager = builder->Build(); -// ... -// -// NOTE: A builder can only be used to build a single AspiredVersionsManager. -// -// This class is not thread-safe. +// TODO(b/64163389): revisit the escaped HTML characters in c2devsite toolchain + +/// Builds an AspiredVersionsManager with options and sources connected to it. +/// It takes over the ownership of the sources and the returned manager handles +/// the destruction of itself and its dependencies. Both single sources and +/// source/source-adapter chains are accepted, i.e. you can use sources that +/// directly supply loaders (Source&lt;std::unique_ptr&lt;Loader>>) or +/// composites that consist of Source&lt;S> + some chain of +/// SourceAdapter&lt;S, ...>, ..., SourceAdapter&lt;..., +/// std::unique_ptr&lt;Loader>>. The builder connects the chain for you. +/// +/// Usage: +/// +/// ... +/// AspiredVersionsManagerBuilder::Options options = ManagerOptions(); +/// std::unique_ptr<AspiredVersionsManagerBuilder> builder; +/// TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( +/// std::move(options), &builder)); +/// builder->AddSource(std::move(some_source)); +/// builder->AddSourceChain( +/// std::move(source), std::move(source_adapter1), +/// std::move(source_adapter2)); +/// std::unique_ptr<Manager> manager = builder->Build(); +/// ... +/// +/// NOTE: A builder can only be used to build a single AspiredVersionsManager. +/// +/// This class is not thread-safe. class AspiredVersionsManagerBuilder { public: using Options = AspiredVersionsManager::Options; @@ -61,30 +64,30 @@ class AspiredVersionsManagerBuilder { ~AspiredVersionsManagerBuilder() = default; - // Connects the source to the AspiredVersionsManager being built and takes - // over its ownership. - // - // REQUIRES: Template type S be convertible to - // Source>. + /// Connects the source to the AspiredVersionsManager being built and takes + /// over its ownership. + /// + /// REQUIRES: Template type S be convertible to + /// Source&lt;std::unique_ptr&lt;Loader>>. template void AddSource(std::unique_ptr source); - // Connects a chain comprising a source and a chain of source adapters, s.t. - // the final adapter in the chain emits Loaders for the manager. The final - // adapter is connected to the manager. We take ownership of the whole chain. - // - // REQUIRES: At least one source adapter. - // - // Usage: - // builder->AddSourceChain( - // std::move(source), std::move(source_adapter1), - // std::move(source_adapter2)); + /// Connects a chain comprising a source and a chain of source adapters, s.t. + /// the final adapter in the chain emits Loaders for the manager. The final + /// adapter is connected to the manager. We take ownership of the whole chain. + /// + /// REQUIRES: At least one source adapter. + /// + /// Usage: + /// builder->AddSourceChain( + /// std::move(source), std::move(source_adapter1), + /// std::move(source_adapter2)); template void AddSourceChain(std::unique_ptr source, std::unique_ptr first_source_adapter, std::unique_ptr... remaining_source_adapters); - // Builds the AspiredVersionsManager and returns it as the Manager interface. + /// Builds the AspiredVersionsManager and returns it as the Manager interface. std::unique_ptr Build(); private: diff --git a/tensorflow_serving/core/aspired_versions_manager_builder_test.cc b/tensorflow_serving/core/aspired_versions_manager_builder_test.cc index c9da9dd712c..322718a21af 100644 --- a/tensorflow_serving/core/aspired_versions_manager_builder_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_builder_test.cc @@ -15,10 +15,15 @@ limitations under the License. #include "tensorflow_serving/core/aspired_versions_manager_builder.h" +#include +#include +#include +#include + #include #include #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state_monitor.h" @@ -27,7 +32,6 @@ limitations under the License. #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/core/test_util/fake_storage_path_source_adapter.h" -#include "tensorflow_serving/core/test_util/source_adapter_test_util.h" #include "tensorflow_serving/util/event_bus.h" namespace tensorflow { @@ -46,7 +50,8 @@ class AspiredVersionsManagerBuilderTest : public ::testing::Test { servable_state_monitor_(servable_event_bus_.get()) { AspiredVersionsManagerBuilder::Options manager_options; manager_options.servable_event_bus = servable_event_bus_.get(); - manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + manager_options.aspired_version_policy.reset( + new AvailabilityPreservingPolicy()); TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( std::move(manager_options), &builder_)); } diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 7464965f212..44801bfe8ac 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -17,15 +17,23 @@ limitations under the License. #include #include +#include +#include +#include +#include +#include +#include #include #include -#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "absl/types/optional.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" +#include "tensorflow_serving/core/aspired_version_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader.h" @@ -38,50 +46,61 @@ namespace tensorflow { namespace serving { namespace { +using test_util::FakeLoader; +using test_util::WaitUntilServableManagerStateIsOneOf; using ::testing::_; using ::testing::Invoke; using ::testing::InvokeWithoutArgs; using ::testing::NiceMock; using ::testing::Return; +using ::testing::UnorderedElementsAre; using ::testing::UnorderedElementsAreArray; -using test_util::FakeLoader; -using test_util::WaitUntilServableManagerStateIsOneOf; constexpr char kServableName[] = "kServableName"; constexpr char kServableName2[] = "kServableName2"; constexpr int kNumVersionsPerServable = 2; constexpr int kNumTotalVersions = 4; -class AspiredVersionsManagerTest : public ::testing::TestWithParam { +// Creates an aspired-versions entry with 'id' and a FakeLoader whose servable +// is id.version. +ServableData> CreateAspiredVersion( + const ServableId& id) { + std::unique_ptr loader(new FakeLoader(id.version)); + return CreateServableData(id, std::move(loader)); +} + +// We parameterize this test with the number of load & unload threads. (Zero +// means use an in-line executor instead of a thread pool.) +struct ThreadPoolSizes { + uint64_t num_load_threads; + uint64_t num_unload_threads; +}; +class AspiredVersionsManagerTest + : public ::testing::TestWithParam> { protected: AspiredVersionsManagerTest() : servable_event_bus_(EventBus::CreateEventBus()), - servable_state_monitor_(servable_event_bus_.get()) { + servable_state_monitor_(servable_event_bus_.get()), + thread_pool_sizes_(std::get<0>(GetParam())), + enable_reload_servables_with_error_(std::get<1>(GetParam())) { AspiredVersionsManager::Options manager_options; - // We parameterize this test to either run with or without a - // thread-pool. - num_load_unload_threads_ = GetParam(); - manager_options.num_load_unload_threads = num_load_unload_threads_; + manager_options.num_load_threads = thread_pool_sizes_.num_load_threads; + manager_options.num_unload_threads = thread_pool_sizes_.num_unload_threads; // The state manager thread won't be run automatically. manager_options.manage_state_interval_micros = -1; manager_options.env = Env::Default(); - manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + manager_options.aspired_version_policy.reset( + new AvailabilityPreservingPolicy()); manager_options.servable_event_bus = servable_event_bus_.get(); max_num_load_retries_ = 1; manager_options.max_num_load_retries = max_num_load_retries_; manager_options.load_retry_interval_micros = 0; + manager_options.enable_reload_servables_with_error = + enable_reload_servables_with_error_; TF_CHECK_OK( AspiredVersionsManager::Create(std::move(manager_options), &manager_)); } - // Creates an aspired-versions entry with 'id' and a FakeLoader whose servable - // is id.version. - ServableData> CreateAspiredVersion( - const ServableId& id) { - std::unique_ptr loader(new FakeLoader(id.version)); - return CreateServableData(id, std::move(loader)); - } - // Creates an aspired-versions entry with 'id' and an error (and no loader). ServableData> CreateErroneousAspiredVersion( const ServableId& id) { @@ -124,6 +143,11 @@ class AspiredVersionsManagerTest : public ::testing::TestWithParam { } } + void FlushServables() { + test_util::AspiredVersionsManagerTestAccess(manager_.get()) + .FlushServables(); + } + void HandlePendingAspiredVersionsRequests() { test_util::AspiredVersionsManagerTestAccess(manager_.get()) .HandlePendingAspiredVersionsRequests(); @@ -136,17 +160,31 @@ class AspiredVersionsManagerTest : public ::testing::TestWithParam { std::shared_ptr> servable_event_bus_; ServableStateMonitor servable_state_monitor_; - uint32 num_load_unload_threads_; + ThreadPoolSizes thread_pool_sizes_; uint32 max_num_load_retries_; + bool enable_reload_servables_with_error_; std::unique_ptr manager_; }; -INSTANTIATE_TEST_CASE_P(WithOrWithoutThreadPool, AspiredVersionsManagerTest, - ::testing::Values(0 /* WithoutThreadPool */, 4)); +INSTANTIATE_TEST_CASE_P( + WithOrWithoutThreadPools, AspiredVersionsManagerTest, + ::testing::Values( + // without load or unload threadpools + std::make_tuple(ThreadPoolSizes{0, 0}, false), + // with just a load threadpool + std::make_tuple(ThreadPoolSizes{2, 0}, false), + // with just an unload threadpool + std::make_tuple(ThreadPoolSizes{0, 2}, false), + // with load and unload threadpools + std::make_tuple(ThreadPoolSizes{4, 4}, false), + // without load or unload threadpools and retries of failed loads + std::make_tuple(ThreadPoolSizes{0, 0}, true), + // with load and unload threadpools and retries of failed loads + std::make_tuple(ThreadPoolSizes{4, 4}, true))); TEST_P(AspiredVersionsManagerTest, ServableHandleNotFoundMissingLoaderName) { - ServableHandle handle; - const Status status = manager_->GetServableHandle( + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle( ServableRequest::Latest(strings::StrCat(kServableName, "missing")), &handle); ASSERT_FALSE(status.ok()) << status; @@ -155,9 +193,9 @@ TEST_P(AspiredVersionsManagerTest, ServableHandleNotFoundMissingLoaderName) { TEST_P(AspiredVersionsManagerTest, ServableHandleNotFoundMissingVersion) { // This version is missing. - const int64 missing_version = 100; - ServableHandle handle; - const Status status = manager_->GetServableHandle( + const int64_t missing_version = 100; + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle( ServableRequest::Specific(kServableName, missing_version), &handle); ASSERT_FALSE(status.ok()) << status; EXPECT_EQ(error::NOT_FOUND, status.code()); @@ -167,7 +205,7 @@ TEST_P(AspiredVersionsManagerTest, ServableHandleInvalidArgument) { // The servable is supposed to be an int type and we ask for a float type, // thus causing an invalid argument error. ServableHandle handle; - const Status status = manager_->GetServableHandle( + const absl::Status status = manager_->GetServableHandle( ServableRequest::Latest(kServableName), &handle); ASSERT_FALSE(status.ok()) << status; EXPECT_EQ(error::INVALID_ARGUMENT, status.code()); @@ -180,12 +218,16 @@ TEST_P(AspiredVersionsManagerTest, ServableHandleLatest) { manager_->GetAspiredVersionsCallback()(kServableName, std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); - ServableHandle handle; - const Status status = manager_->GetServableHandle( + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle( ServableRequest::Latest(kServableName), &handle); TF_ASSERT_OK(status); EXPECT_EQ(kNumVersionsPerServable + 1, *handle); @@ -206,18 +248,80 @@ TEST_P(AspiredVersionsManagerTest, ServableHandleLatestVersionIsZero) { WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); - ServableHandle handle; - const Status status = manager_->GetServableHandle( + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle( ServableRequest::Latest(kServableName3), &handle); TF_ASSERT_OK(status); EXPECT_EQ(0, *handle); EXPECT_EQ(id, handle.id()); } +TEST_P(AspiredVersionsManagerTest, ReloadAspiredError) { + const char kServableName[] = "kAspiredError"; + auto callback_fn = manager_->GetAspiredVersionsCallback(); + // First, load a working Servable under version 1. + { + std::vector>> aspired_versions; + const ServableId id = {kServableName, 1}; + aspired_versions.push_back(CreateAspiredVersion(id)); + callback_fn(kServableName, std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle( + ServableRequest::Latest(kServableName), &handle); + TF_ASSERT_OK(status); + EXPECT_EQ(1, *handle); + EXPECT_EQ(id, handle.id()); + } + // Having a failing servable load for version 2. + { + std::vector>> aspired_versions; + const ServableId id = {kServableName, 2}; + aspired_versions.push_back(CreateErroneousAspiredVersion(id)); + callback_fn(kServableName, std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + ServableHandle handle; + absl::Status status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 2), &handle); + EXPECT_FALSE(status.ok()) << status; + } + // Attempt to reload servable for version 2. + { + std::vector>> aspired_versions; + const ServableId id = {kServableName, 2}; + aspired_versions.push_back(CreateAspiredVersion(id)); + callback_fn(kServableName, std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + if (enable_reload_servables_with_error_) { + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, + {ServableState::ManagerState::kAvailable}); + ServableHandle handle; + absl::Status status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 2), &handle); + TF_ASSERT_OK(status) << status; + } else { + // Sleep for 1ms. There's nothing to wait on as the state will not change. + Env::Default()->SleepForMicroseconds(1000 /* 1 ms */); + ServableHandle handle; + absl::Status status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 2), &handle); + EXPECT_FALSE(status.ok()) << status; + } + } +} + TEST_P(AspiredVersionsManagerTest, ServableHandleSpecificVersion) { - ServableHandle handle; + ServableHandle handle; const ServableId id = {kServableName2, 0}; - const Status status = + const absl::Status status = manager_->GetServableHandle(ServableRequest::FromId(id), &handle); TF_ASSERT_OK(status); EXPECT_EQ(0, *handle); @@ -248,6 +352,12 @@ TEST_P(AspiredVersionsManagerTest, ListAvailableServableIds) { } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); + + manager_->GetAspiredVersionsCallback()(kServableName, {}); + HandlePendingAspiredVersionsRequests(); + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 0}, {ServableState::ManagerState::kEnd}); @@ -264,8 +374,8 @@ TEST_P(AspiredVersionsManagerTest, ListAvailableServableIds) { TEST_P(AspiredVersionsManagerTest, GetAvailableServableHandles) { // Scoped to destruct handles at the end of it. { - const std::map> handles_before = - manager_->GetAvailableServableHandles(); + const std::map> handles_before = + manager_->GetAvailableServableHandles(); ASSERT_EQ(kNumVersionsPerServable * 2, handles_before.size()); const std::vector expected_ids_before = {{kServableName, 0}, @@ -295,16 +405,21 @@ TEST_P(AspiredVersionsManagerTest, GetAvailableServableHandles) { } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); + + manager_->GetAspiredVersionsCallback()(kServableName, {}); + HandlePendingAspiredVersionsRequests(); + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 0}, {ServableState::ManagerState::kEnd}); WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 1}, {ServableState::ManagerState::kEnd}); - { - const std::map> handles_after = - manager_->GetAvailableServableHandles(); + const std::map> handles_after = + manager_->GetAvailableServableHandles(); ASSERT_EQ(kNumVersionsPerServable, handles_after.size()); const std::vector expected_ids_after = {{kServableName2, 0}, @@ -327,8 +442,8 @@ TEST_P(AspiredVersionsManagerTest, AspiredRemovedFull) { // Scoped so that the handle is destructed at the end, and the harness is // destructed when we run the manager looping thread. { - ServableHandle handle; - const Status status = manager_->GetServableHandle( + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle( ServableRequest::Latest(kServableName), &handle); TF_ASSERT_OK(status); EXPECT_EQ(1, *handle); @@ -347,26 +462,19 @@ TEST_P(AspiredVersionsManagerTest, AspiredRemovedFull) { WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 1}, {ServableState::ManagerState::kEnd}); + FlushServables(); const int num_fake_loaders_after = FakeLoader::num_fake_loaders(); EXPECT_EQ(kNumVersionsPerServable, num_fake_loaders_before - num_fake_loaders_after); - ServableHandle missing_handle; - const Status missing_status = manager_->GetServableHandle( + ServableHandle missing_handle; + const absl::Status missing_status = manager_->GetServableHandle( ServableRequest::Latest(kServableName), &missing_handle); ASSERT_FALSE(missing_status.ok()); EXPECT_EQ(error::NOT_FOUND, missing_status.code()); } TEST_P(AspiredVersionsManagerTest, AspiredRemovedPartial) { - { - ServableHandle handle; - const Status status = manager_->GetServableHandle( - ServableRequest::Specific(kServableName, 1), &handle); - TF_ASSERT_OK(status); - EXPECT_EQ(1, *handle); - } - std::vector>> aspired_versions; aspired_versions.push_back(CreateAspiredVersion({kServableName, 0})); manager_->GetAspiredVersionsCallback()(kServableName, @@ -378,17 +486,84 @@ TEST_P(AspiredVersionsManagerTest, AspiredRemovedPartial) { {kServableName, 1}, {ServableState::ManagerState::kEnd}); - ServableHandle missing_handle; - const Status missing_status = manager_->GetServableHandle( - ServableRequest::Specific(kServableName, 1), &missing_handle); - ASSERT_FALSE(missing_status.ok()); - EXPECT_EQ(error::NOT_FOUND, missing_status.code()); + // Version 0 should remain available in the manager. + ServableHandle v0_handle; + const absl::Status v0_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 0), &v0_handle); + TF_ASSERT_OK(v0_status); + EXPECT_EQ(0, *v0_handle); + + // Version 1 should no longer be available. + ServableHandle v1_handle; + const absl::Status v1_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 1), &v1_handle); + ASSERT_FALSE(v1_status.ok()); + EXPECT_EQ(error::NOT_FOUND, v1_status.code()); +} + +TEST_P(AspiredVersionsManagerTest, RevertToSmallerVersionNumber) { + // Initially, versions 0 and 1 of kServableName are loaded. + std::set initial_versions; + for (const ServableId& id : manager_->ListAvailableServableIds()) { + if (id.name == kServableName) { + initial_versions.insert(id.version); + } + } + ASSERT_THAT(initial_versions, UnorderedElementsAre(0, 1)); + + // Unload version 0, s.t. only version 1 is loaded. + std::vector>> initial_aspired_versions; + initial_aspired_versions.push_back(CreateAspiredVersion({kServableName, 1})); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(initial_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + FlushServables(); + + // Now, switch to version 0 (dropping version 1). + std::vector>> new_aspired_versions; + new_aspired_versions.push_back(CreateAspiredVersion({kServableName, 0})); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(new_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + Notification done_transitioning; + std::unique_ptr transition_servables( + Env::Default()->StartThread({}, "TransitionServables", [&]() { + while (!done_transitioning.HasBeenNotified()) { + InvokePolicyAndExecuteAction(); + Env::Default()->SleepForMicroseconds(1000 /* 1 ms */); + } + })); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, {kServableName, 0}, + {ServableState::ManagerState::kAvailable}); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 1}, + {ServableState::ManagerState::kEnd}); + done_transitioning.Notify(); + + // Version 0 should be available. + ServableHandle v0_handle; + const absl::Status v0_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 0), &v0_handle); + TF_ASSERT_OK(v0_status); + EXPECT_EQ(0, *v0_handle); + + // Version 1 should not be available. + ServableHandle v1_handle; + const absl::Status v1_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 1), &v1_handle); + ASSERT_FALSE(v1_status.ok()); + EXPECT_EQ(error::NOT_FOUND, v1_status.code()); } TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateLoad) { const ServableId id = {kServableName, 2}; - ServableHandle not_found_handle; - const Status not_found_status = manager_->GetServableHandle( + ServableHandle not_found_handle; + const absl::Status not_found_status = manager_->GetServableHandle( ServableRequest::FromId(id), ¬_found_handle); ASSERT_FALSE(not_found_status.ok()) << not_found_status; EXPECT_EQ(error::NOT_FOUND, not_found_status.code()); @@ -399,18 +574,22 @@ TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateLoad) { std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - ServableHandle not_ready_handle; - const Status not_ready_status = manager_->GetServableHandle( + ServableHandle not_ready_handle; + const absl::Status not_ready_status = manager_->GetServableHandle( ServableRequest::FromId(id), ¬_ready_handle); ASSERT_FALSE(not_ready_status.ok()) << not_ready_status; EXPECT_EQ(error::NOT_FOUND, not_ready_status.code()); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); - ServableHandle handle; - const Status status = + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle(ServableRequest::FromId(id), &handle); TF_ASSERT_OK(status); EXPECT_EQ(2, *handle); @@ -418,8 +597,8 @@ TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateLoad) { TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateUnload) { { - ServableHandle handle; - const Status status = manager_->GetServableHandle( + ServableHandle handle; + const absl::Status status = manager_->GetServableHandle( ServableRequest::Specific(kServableName, 0), &handle); TF_ASSERT_OK(status); EXPECT_EQ(0, *handle); @@ -438,8 +617,8 @@ TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateUnload) { {kServableName, 1}, {ServableState::ManagerState::kEnd}); - ServableHandle not_found_handle; - const Status not_found_status = manager_->GetServableHandle( + ServableHandle not_found_handle; + const absl::Status not_found_status = manager_->GetServableHandle( ServableRequest::Specific(kServableName, 0), ¬_found_handle); ASSERT_FALSE(not_found_status.ok()) << not_found_status; EXPECT_EQ(error::NOT_FOUND, not_found_status.code()); @@ -448,15 +627,15 @@ TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateUnload) { // The manager prefers unloading over loading when deciding between different // servable actions. This behaviour is tested here. TEST_P(AspiredVersionsManagerTest, ManagerPrefersUnloadOverLoad) { - ServableHandle not_found_2_handle; - Status not_found_2_status = manager_->GetServableHandle( + ServableHandle not_found_2_handle; + absl::Status not_found_2_status = manager_->GetServableHandle( ServableRequest::Specific(kServableName2, 2), ¬_found_2_handle); ASSERT_FALSE(not_found_2_status.ok()) << not_found_2_status; EXPECT_EQ(error::NOT_FOUND, not_found_2_status.code()); { - ServableHandle found_0_handle; - const Status found_0_status = manager_->GetServableHandle( + ServableHandle found_0_handle; + const absl::Status found_0_status = manager_->GetServableHandle( ServableRequest::Specific(kServableName, 0), &found_0_handle); TF_ASSERT_OK(found_0_status); EXPECT_EQ(0, *found_0_handle); @@ -465,14 +644,14 @@ TEST_P(AspiredVersionsManagerTest, ManagerPrefersUnloadOverLoad) { // We want to unload version 0 of the first servable stream and load version 2 // of the second stream. struct { - StringPiece name; + absl::string_view name; int start; int end; } servable_aspired_list[2] = {{kServableName, 1, 1}, {kServableName2, 0, 2}}; for (const auto& servable_aspired : servable_aspired_list) { std::vector>> aspired_versions; for (int i = servable_aspired.start; i <= servable_aspired.end; ++i) { - const ServableId id = {servable_aspired.name.ToString(), i}; + const ServableId id = {string(servable_aspired.name), i}; aspired_versions.push_back(CreateAspiredVersion(id)); } manager_->GetAspiredVersionsCallback()(servable_aspired.name, @@ -487,8 +666,8 @@ TEST_P(AspiredVersionsManagerTest, ManagerPrefersUnloadOverLoad) { {kServableName, 0}, {ServableState::ManagerState::kEnd}); - ServableHandle not_found_0_handle; - const Status not_found_0_status = manager_->GetServableHandle( + ServableHandle not_found_0_handle; + const absl::Status not_found_0_status = manager_->GetServableHandle( ServableRequest::Specific(kServableName, 0), ¬_found_0_handle); ASSERT_FALSE(not_found_0_status.ok()) << not_found_0_status; EXPECT_EQ(error::NOT_FOUND, not_found_2_status.code()); @@ -504,13 +683,101 @@ TEST_P(AspiredVersionsManagerTest, ManagerPrefersUnloadOverLoad) { servable_state_monitor_, {kServableName2, 2}, {ServableState::ManagerState::kAvailable}); - ServableHandle found_2_handle; - const Status found_2_status = manager_->GetServableHandle( + ServableHandle found_2_handle; + const absl::Status found_2_status = manager_->GetServableHandle( ServableRequest::Specific(kServableName2, 2), &found_2_handle); TF_ASSERT_OK(found_2_status); EXPECT_EQ(2, *found_2_handle); } +TEST_P(AspiredVersionsManagerTest, CustomSortActions) { + test_util::AspiredVersionsManagerTestAccess(manager_.get()) + .SetCustomSortActions( + [](const AspiredVersionPolicy::ServableAction& lhs, + const AspiredVersionPolicy::ServableAction& rhs) -> bool { + // Prefer kServableName2 over anything else; note the impl needs to + // be a valid strict-weak ordering + bool lhs_is_servable_2 = lhs.id.name == kServableName2; + bool rhs_is_servable_2 = rhs.id.name == kServableName2; + if (lhs_is_servable_2 != rhs_is_servable_2) { + return lhs_is_servable_2; + } + return false; + }); + + { + ServableHandle not_found_2_handle; + absl::Status not_found_2_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName2, 2), ¬_found_2_handle); + ASSERT_FALSE(not_found_2_status.ok()) << not_found_2_status; + EXPECT_EQ(error::NOT_FOUND, not_found_2_status.code()); + } + + { + ServableHandle found_0_handle; + absl::Status found_0_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 0), &found_0_handle); + TF_ASSERT_OK(found_0_status); + EXPECT_EQ(0, *found_0_handle); + } + + // We want to unload version 0 of the first servable stream and load version 2 + // of the second stream. + struct { + absl::string_view name; + int start; + int end; + } servable_aspired_list[2] = {{kServableName, 1, 1}, {kServableName2, 0, 2}}; + for (const auto& servable_aspired : servable_aspired_list) { + std::vector>> aspired_versions; + for (int i = servable_aspired.start; i <= servable_aspired.end; ++i) { + const ServableId id = {string(servable_aspired.name), i}; + aspired_versions.push_back(CreateAspiredVersion(id)); + } + manager_->GetAspiredVersionsCallback()(servable_aspired.name, + std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + } + + // By default, the manager prefers to unload a servable before loading a + // servable, see ManagerPrefersUnloadOverLoad test case above; but here our + // custom sort order prefers kServableName2 + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, {kServableName2, 2}, + {ServableState::ManagerState::kAvailable}); + + { + ServableHandle found_0_handle; + absl::Status found_0_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 0), &found_0_handle); + TF_ASSERT_OK(found_0_status); + EXPECT_EQ(0, *found_0_handle); + } + + { + ServableHandle found_2_handle; + const absl::Status found_2_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName2, 2), &found_2_handle); + TF_ASSERT_OK(found_2_status); + EXPECT_EQ(2, *found_2_handle); + } + + // Now it would unload the first + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + + { + ServableHandle not_found_0_handle; + const absl::Status not_found_0_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 0), ¬_found_0_handle); + ASSERT_FALSE(not_found_0_status.ok()) << not_found_0_status; + EXPECT_EQ(error::NOT_FOUND, not_found_0_status.code()); + } +} + // Test to ensure the manager doesn't try to load or serve an incoming erroneous // aspired-version entry. TEST_P(AspiredVersionsManagerTest, ErroneousAspiredVersion) { @@ -520,8 +787,8 @@ TEST_P(AspiredVersionsManagerTest, ErroneousAspiredVersion) { std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - ServableHandle handle; - Status status = manager_->GetServableHandle( + ServableHandle handle; + absl::Status status = manager_->GetServableHandle( ServableRequest::Specific(kServableName, 3), &handle); EXPECT_FALSE(status.ok()) << status; @@ -535,9 +802,9 @@ TEST_P(AspiredVersionsManagerTest, ErroneousAspiredVersion) { // Test to ensure that the deletion of a loader/servable occurs in a manager // thread, and not a request thread. TEST_P(AspiredVersionsManagerTest, DestructOnNonServingThread) { - std::unique_ptr> latest_handle( - new ServableHandle()); - const Status status = manager_->GetServableHandle( + std::unique_ptr> latest_handle( + new ServableHandle()); + const absl::Status status = manager_->GetServableHandle( ServableRequest::Latest(kServableName), latest_handle.get()); TF_ASSERT_OK(status); EXPECT_EQ(1, **latest_handle); @@ -555,9 +822,10 @@ TEST_P(AspiredVersionsManagerTest, DestructOnNonServingThread) { WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, {kServableName, 0}, {ServableState::ManagerState::kEnd}); + FlushServables(); // The servable has been deleted in this thread if there is no - // thread-pool for load/unload. - if (num_load_unload_threads_ == 0) { + // thread-pool for unload. + if (thread_pool_sizes_.num_unload_threads == 0) { EXPECT_TRUE(FakeLoader::was_deleted_in_this_thread()); } done_unload_servable.Notify(); @@ -604,11 +872,15 @@ TEST_P(AspiredVersionsManagerTest, EventBusErrorOnLoad) { HandlePendingAspiredVersionsRequests(); const ServableState start_state = {id, ServableState::ManagerState::kStart, - Status::OK()}; + absl::OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(start_state)); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); @@ -628,27 +900,33 @@ TEST_P(AspiredVersionsManagerTest, EventBusServableLifecycle) { HandlePendingAspiredVersionsRequests(); const ServableState start_state = {id, ServableState::ManagerState::kStart, - Status::OK()}; + absl::OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(start_state)); Notification load_called; Notification load_continue; - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) .WillOnce(InvokeWithoutArgs([&]() { load_called.Notify(); load_continue.WaitForNotification(); - return Status::OK(); + return absl::OkStatus(); })); - std::unique_ptr load_thread( - Env::Default()->StartThread(ThreadOptions(), "LoadThread", - [&]() { InvokePolicyAndExecuteAction(); })); + std::unique_ptr load_unload_thread( + Env::Default()->StartThread(ThreadOptions(), "LoadUnloadThread", [&]() { + // Unload version 0 and load the new aspired version. Version 1 may or + // may not be unloaded (depending on whether load/unload thread pools + // are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } + })); load_called.WaitForNotification(); const ServableState loading_state = { - id, ServableState::ManagerState::kLoading, Status::OK()}; + id, ServableState::ManagerState::kLoading, absl::OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(loading_state)); @@ -657,7 +935,7 @@ TEST_P(AspiredVersionsManagerTest, EventBusServableLifecycle) { servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); const ServableState available_state = { - id, ServableState::ManagerState::kAvailable, Status::OK()}; + id, ServableState::ManagerState::kAvailable, absl::OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(available_state)); @@ -666,23 +944,23 @@ TEST_P(AspiredVersionsManagerTest, EventBusServableLifecycle) { Notification unload_called; Notification unload_continue; - EXPECT_CALL(*loader, Unload()) - .WillOnce(Invoke([&]() { - unload_called.Notify(); - unload_continue.WaitForNotification(); - })); + EXPECT_CALL(*loader, Unload()).WillOnce(Invoke([&]() { + unload_called.Notify(); + unload_continue.WaitForNotification(); + })); std::unique_ptr unload_thread( Env::Default()->StartThread(ThreadOptions(), "UnloadThread", [&]() { - for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { - InvokePolicyAndExecuteAction(); - } + // Call InvokePolicyAndExecuteAction() twice to unload version 1 and the + // new version, in case version 1 has not been unloaded previously. + InvokePolicyAndExecuteAction(); + InvokePolicyAndExecuteAction(); })); unload_called.WaitForNotification(); const ServableState unloading_state = { - id, ServableState::ManagerState::kUnloading, Status::OK()}; + id, ServableState::ManagerState::kUnloading, absl::OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(unloading_state)); @@ -691,7 +969,7 @@ TEST_P(AspiredVersionsManagerTest, EventBusServableLifecycle) { {ServableState::ManagerState::kEnd}); const ServableState end_state = { - {kServableName, 7}, ServableState::ManagerState::kEnd, Status::OK()}; + {kServableName, 7}, ServableState::ManagerState::kEnd, absl::OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(end_state)); } @@ -702,7 +980,7 @@ TEST_P(AspiredVersionsManagerTest, NoEventBus) { // The state manager thread won't be run automatically. options.manage_state_interval_micros = -1; options.env = Env::Default(); - options.aspired_version_policy.reset(new EagerLoadPolicy()); + options.aspired_version_policy.reset(new AvailabilityPreservingPolicy()); std::unique_ptr aspired_versions_manager; TF_ASSERT_OK(AspiredVersionsManager::Create(std::move(options), &aspired_versions_manager)); @@ -718,26 +996,29 @@ TEST_P(AspiredVersionsManagerTest, NoEventBus) { TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallySucceeds) { CHECK_GE(max_num_load_retries_, 1); - + const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock; // We succeed on the last load, before the manager gives up. - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) .WillOnce(Return(errors::Internal("Error on load."))) - .WillOnce(Return(Status::OK())); + .WillOnce(Return(absl::OkStatus())); - const ServableId id = {kServableName, 7}; std::vector>> aspired_versions; aspired_versions.push_back({id, std::unique_ptr(loader)}); manager_->GetAspiredVersionsCallback()(kServableName, std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); const ServableState available_state = { - id, ServableState::ManagerState::kAvailable, Status::OK()}; + id, ServableState::ManagerState::kAvailable, absl::OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(available_state)); } @@ -755,7 +1036,11 @@ TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallyFails) { std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); @@ -765,6 +1050,75 @@ TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallyFails) { EqualsServableState(error_state)); } +// Tests the interaction between AspiredVersionsManager and the +// AvailabilityPreservingPolicy. +// Specifically, we want to make sure that the manager will not try to unload +// all serving versions that are no longer aspired if the new aspired version +// was not able to start serving. +TEST_P(AspiredVersionsManagerTest, AspireErrorDontUnload) { + const std::vector expected_before = {{kServableName, 0}, + {kServableName, 1}, + {kServableName2, 0}, + {kServableName2, 1}}; + EXPECT_THAT(manager_->ListAvailableServableIds(), + UnorderedElementsAreArray(expected_before)); + + // Set stream kServableName to have servable 7. + // This causes 0 & 1 to be set to not aspired and 7 to be loaded, but 7 errors + // on load, so never moves to a loaded state. + { + std::vector>> aspired_versions; + const ServableId id = {kServableName, 7}; + std::unique_ptr loader( + new FakeLoader(7, errors::Internal("An error."))); + aspired_versions.push_back({id, std::move(loader)}); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Will unload version 0. + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + + // Will try to load version 7 and fail. + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + } + + // For kServableName, version 0 has been unloaded. For kServableName2, both + // versions should still be loaded. + const std::vector expected_after_first_load = { + {kServableName, 1}, {kServableName2, 0}, {kServableName2, 1}}; + EXPECT_THAT(manager_->ListAvailableServableIds(), + UnorderedElementsAreArray(expected_after_first_load)); + + // Now successfully loading a new version should allow the older versions to + // be unloaded. + { + std::vector>> aspired_versions; + const ServableId id = {kServableName, 8}; + std::unique_ptr loader(new FakeLoader(8)); + aspired_versions.push_back({id, std::move(loader)}); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Will try to load version 8 and succeed. + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + + // Will unload version 1. + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 1}, + {ServableState::ManagerState::kEnd}); + } +} + TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { // This test exercises a scenario in which a servable has been unaspired, and // while it is still being managed (e.g. loading, serving or unloading) it @@ -777,15 +1131,25 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { std::vector>> first_aspired_versions; test_util::MockLoader* first_loader = new NiceMock(); first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); - EXPECT_CALL(*first_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*first_loader, LoadWithMetadata(Loader::Metadata{id})) + .WillOnce(Return(absl::OkStatus())); manager_->GetAspiredVersionsCallback()(kServableName, std::move(first_aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); - // Pin 'first_loader' in the manager by holding a handle to its servable. + // Wait for verion 0 to be unloaded and the new aspired version to be loaded. + // If we don't wait, the first_loader_handle below may be obtained before + // the loading or unloading finishes, which may block the loading or + // unloading. + InvokePolicyAndExecuteAction(); + InvokePolicyAndExecuteAction(); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + + // Pin 'first_loader' in the manager by holding a handle to its servable. int servable = 42; EXPECT_CALL(*first_loader, servable()).WillOnce(InvokeWithoutArgs([&]() { return AnyPtr{&servable}; @@ -808,11 +1172,15 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { manager_->GetAspiredVersionsCallback()(kServableName, std::move(empty_aspired_versions)); HandlePendingAspiredVersionsRequests(); + // The following thread will block trying to unload the first loader, while we // hold the handle. std::unique_ptr unload_thread( - Env::Default()->StartThread(ThreadOptions(), "UnloadThread", - [&]() { InvokePolicyAndExecuteAction(); })); + Env::Default()->StartThread(ThreadOptions(), "UnloadThread", [&]() { + // Unload version 1 and the newly un-aspired version. + InvokePolicyAndExecuteAction(); + InvokePolicyAndExecuteAction(); + })); // Re-aspire the servable with a fresh loader. std::vector>> second_aspired_versions; @@ -820,10 +1188,11 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { second_aspired_versions.push_back( {id, std::unique_ptr(second_loader)}); Notification second_load_called; - EXPECT_CALL(*second_loader, Load(_)).WillOnce(InvokeWithoutArgs([&]() { - second_load_called.Notify(); - return Status::OK(); - })); + EXPECT_CALL(*second_loader, LoadWithMetadata(Loader::Metadata{id})) + .WillOnce(InvokeWithoutArgs([&]() { + second_load_called.Notify(); + return absl::OkStatus(); + })); manager_->GetAspiredVersionsCallback()(kServableName, std::move(second_aspired_versions)); @@ -832,6 +1201,7 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { std::unique_ptr reaspire_thread( Env::Default()->StartThread(ThreadOptions(), "ReaspireThread", [&]() { while (!second_load_called.HasBeenNotified()) { + FlushServables(); HandlePendingAspiredVersionsRequests(); InvokePolicyAndExecuteAction(); Env::Default()->SleepForMicroseconds(1000 /* 1 ms */); @@ -848,6 +1218,170 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { second_load_called.WaitForNotification(); } +TEST_P(AspiredVersionsManagerTest, + UnaspireFailedServableThenImmediatelyReaspire) { + // Like UnaspireThenImmediatelyReaspire, but covers the case in which the + // servable fails to load the first time it is aspired. + + const ServableId id = {kServableName, 7}; + + std::vector>> first_aspired_versions; + test_util::MockLoader* first_loader = new NiceMock(); + first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); + EXPECT_CALL(*first_loader, LoadWithMetadata(Loader::Metadata{id})) + .WillRepeatedly(Return(absl::Status( + static_cast(absl::StatusCode::kUnknown), + "first load failing"))); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(first_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + + // Now, we'll un-aspire the servable, and then re-aspire it with a new loader. + // The manager should wait until it is able to flush the first loader, then + // bring up the second loader. + + std::vector>> empty_aspired_versions; + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(empty_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Re-aspire the servable with a fresh loader. + std::vector>> second_aspired_versions; + test_util::MockLoader* second_loader = new NiceMock(); + second_aspired_versions.push_back( + {id, std::unique_ptr(second_loader)}); + Notification second_load_called; + EXPECT_CALL(*second_loader, LoadWithMetadata(Loader::Metadata{id})) + .WillOnce(InvokeWithoutArgs([&]() { + second_load_called.Notify(); + return absl::OkStatus(); + })); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(second_aspired_versions)); + + // Run the manager's background logic in a loop, but sans FlushServables(). + // Nothing should happen for now because the first loader isn't flushed. + std::unique_ptr reaspire_thread( + Env::Default()->StartThread(ThreadOptions(), "ReaspireThread", [&]() { + while (!second_load_called.HasBeenNotified()) { + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + Env::Default()->SleepForMicroseconds(1000 /* 1 ms */); + } + })); + Env::Default()->SleepForMicroseconds(50 * 1000 /* 50 ms */); + EXPECT_FALSE(second_load_called.HasBeenNotified()); + + // Flush the first loader. The manager should finally bring up the second + // loader. + FlushServables(); + second_load_called.WaitForNotification(); +} + +TEST_P(AspiredVersionsManagerTest, UnaspireNewServableThenImmediatelyReaspire) { + // Like UnaspireThenImmediatelyReaspire, but covers the case in which the + // servable is in state kNew when it gets unaspired. + // (Regression test for b/27766674.) + + const ServableId id = {kServableName, 7}; + + std::vector>> first_aspired_versions; + test_util::MockLoader* first_loader = new NiceMock(); + EXPECT_CALL(*first_loader, LoadWithMetadata(Loader::Metadata{id})).Times(0); + first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(first_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + // (We *don't* call InvokePolicyAndExecuteAction(), thus causing the servable + // to remain in state kNew.) + + // Now, we'll un-aspire the servable, and then re-aspire it with a new loader. + // The manager should get rid of the first loader, then bring up the second + // one. + + std::vector>> empty_aspired_versions; + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(empty_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Re-aspire the servable with a fresh loader. + std::vector>> second_aspired_versions; + test_util::MockLoader* second_loader = new NiceMock(); + second_aspired_versions.push_back( + {id, std::unique_ptr(second_loader)}); + Notification second_load_called; + EXPECT_CALL(*second_loader, LoadWithMetadata(Loader::Metadata{id})) + .WillOnce(InvokeWithoutArgs([&]() { + second_load_called.Notify(); + return absl::OkStatus(); + })); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(second_aspired_versions)); + // The first HandlePendingAspiredVersionsRequests() call will do nothing, + // because the first loader remains in the manager (with state kNew). + HandlePendingAspiredVersionsRequests(); + // FlushServables() should remove the first loader, thus clearing the way for + // a subsequent HandlePendingAspiredVersionsRequests() call to accept the + // second loader. + FlushServables(); + HandlePendingAspiredVersionsRequests(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } + second_load_called.WaitForNotification(); +} + +class MockAspiredVersionPolicy : public AspiredVersionPolicy { + public: + MOCK_METHOD(absl::optional, GetNextAction, + (const std::vector&), + (const, override)); +}; + +TEST(AspiredVersionsManagerTest, CallPolicyWithAllVersions) { + std::unique_ptr manager; + AspiredVersionsManager::Options manager_options; + MockAspiredVersionPolicy* policy = new MockAspiredVersionPolicy; + // The state manager thread won't be run automatically. + manager_options.manage_state_interval_micros = -1; + manager_options.aspired_version_policy = + std::unique_ptr(policy); + TF_CHECK_OK( + AspiredVersionsManager::Create(std::move(manager_options), &manager)); + std::set servables; + std::vector>> aspired_versions; + for (int i = 0; i < kNumVersionsPerServable; ++i) { + const ServableId id = {kServableName, i}; + aspired_versions.push_back(CreateAspiredVersion(id)); + servables.insert(id); + } + manager->GetAspiredVersionsCallback()(kServableName, + std::move(aspired_versions)); + test_util::AspiredVersionsManagerTestAccess(manager.get()) + .HandlePendingAspiredVersionsRequests(); + + std::vector all_versions; + EXPECT_CALL(*policy, GetNextAction(_)) + .WillOnce(Invoke( + [&all_versions]( + const std::vector& snapshots) { + all_versions = snapshots; + return absl::nullopt; + })); + test_util::AspiredVersionsManagerTestAccess(manager.get()) + .InvokePolicyAndExecuteAction(); + EXPECT_EQ(kNumVersionsPerServable, all_versions.size()); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/availability_preserving_policy.cc b/tensorflow_serving/core/availability_preserving_policy.cc new file mode 100644 index 00000000000..2841be7258a --- /dev/null +++ b/tensorflow_serving/core/availability_preserving_policy.cc @@ -0,0 +1,91 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/availability_preserving_policy.h" + +#include +#include + +#include "absl/types/optional.h" +#include "tensorflow_serving/core/loader_harness.h" + +namespace tensorflow { +namespace serving { + +namespace { + +// Returns the ServableId with the lowest version, if any exists. +absl::optional GetLowestServableId( + const std::vector& all_versions) { + const auto& iterator = + std::min_element(all_versions.begin(), all_versions.end(), + [](const AspiredServableStateSnapshot& a, + const AspiredServableStateSnapshot& b) { + return a.id.version < b.id.version; + }); + if (iterator == all_versions.end()) { + return absl::nullopt; + } else { + return iterator->id; + } +} + +} // namespace + +absl::optional +AvailabilityPreservingPolicy::GetNextAction( + const std::vector& all_versions) const { + // We first try to unload non-aspired versions (if any). + bool has_aspired = false; + bool has_aspired_serving = false; + std::vector unaspired_serving_versions; + for (const auto& version : all_versions) { + if (version.is_aspired) { + has_aspired = true; + if (version.state == LoaderHarness::State::kReady) { + has_aspired_serving = true; + } + } else if (version.state == LoaderHarness::State::kReady) { + unaspired_serving_versions.push_back(version); + } + } + + // If there is no aspired version, there is at least one aspired version + // that is ready, or there are more than one un-aspired versions that are + // ready, unload the lowest non-aspired version. + if (!has_aspired || has_aspired_serving || + unaspired_serving_versions.size() > 1) { + absl::optional version_to_unload = + GetLowestServableId(unaspired_serving_versions); + if (version_to_unload) { + return {{Action::kUnload, version_to_unload.value()}}; + } + } + + // If there is at least one new aspired version, load the one with the + // highest version number. + absl::optional highest_new_aspired_version_id = + GetHighestAspiredNewServableId(all_versions); + if (highest_new_aspired_version_id) { + VLOG(1) << "AvailabilityPreservingPolicy requesting to load servable " + << highest_new_aspired_version_id.value(); + return {{Action::kLoad, highest_new_aspired_version_id.value()}}; + } + + return absl::nullopt; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/availability_preserving_policy.h b/tensorflow_serving/core/availability_preserving_policy.h new file mode 100644 index 00000000000..497dea76d95 --- /dev/null +++ b/tensorflow_serving/core/availability_preserving_policy.h @@ -0,0 +1,49 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_AVAILABILITY_PRESERVING_POLICY_H_ +#define TENSORFLOW_SERVING_CORE_AVAILABILITY_PRESERVING_POLICY_H_ + +#include + +#include "tensorflow_serving/core/aspired_version_policy.h" +#include "tensorflow_serving/core/loader_harness.h" + +namespace tensorflow { +namespace serving { + +// AspiredVersionPolicy that provides servable availability with the trade-off +// of temporary increased resource consumption while newly-aspired versions load +// followed by newly-un-aspired versions unloading. At the same time, it tries +// to minimize the resource usage caused by loading more versions than needed to +// maintain availability. +// +// Here is a detailed description of how this policy works: +// First, if there are any unaspired loaded versions, we unload the smallest +// such version, *unless* that is the only loaded version (to avoid compromising +// availability). +// Second, if there are no non-aspired versions we are permitted to unload, we +// load the aspired new version with the highest version number. +class AvailabilityPreservingPolicy final : public AspiredVersionPolicy { + public: + absl::optional GetNextAction( + const std::vector& all_versions) + const override; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_AVAILABILITY_PRESERVING_POLICY_H_ diff --git a/tensorflow_serving/core/availability_preserving_policy_test.cc b/tensorflow_serving/core/availability_preserving_policy_test.cc new file mode 100644 index 00000000000..73191078495 --- /dev/null +++ b/tensorflow_serving/core/availability_preserving_policy_test.cc @@ -0,0 +1,164 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/availability_preserving_policy.h" + +#include + +#include +#include "tensorflow_serving/core/servable_id.h" + +namespace tensorflow { +namespace serving { +namespace { + +// None unloadable (ready+unaspired); multiple loadable (new+aspired). Loads the +// highest loadable version. +TEST(AvailabilityPreservingPolicyTest, LoadsNewAspired) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); + + AvailabilityPreservingPolicy policy; + auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(5, action->id.version); +} + +// Both unloadable and loadable versions present. Unloading doesn't compromise +// availability. Opts to unload. +TEST(AvailabilityPreservingPolicyTest, UnloadsNonAspiredFirst) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 4}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); + + AvailabilityPreservingPolicy policy; + auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(3, action->id.version); +} + +// One unloadable. Nothing aspired so the goal is to unload all versions and +// lose availability. Unloads. +TEST(AvailabilityPreservingPolicyTest, UnloadsFirstNonAspiredWhenNoAspired) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kDisabled, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kError, false}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(2, action->id.version); +} + +// None unloadable or loadable. Takes no action. +TEST(AvailabilityPreservingPolicyTest, ReturnsNoActionWhenNone) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kLoading, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 5}, LoaderHarness::State::kDisabled, false}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +// One unloadable; none loadable. Unloading would compromise availability. Takes +// no action. +TEST(AvailabilityPreservingPolicyTest, DoesNotUnloadWhenOtherNotReady) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +// One unloadable; none loadable. Unloading would compromise availability. Takes +// no action. +TEST(AvailabilityPreservingPolicyTest, DoesNotUnloadWhenOtherInError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +// One unloadable; none loadable. Unloading doesn't compromise availability. +// Unloads. +TEST(AvailabilityPreservingPolicyTest, UnloadIfOtherReadyEvenIfLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(1, action->id.version); +} + +// One unloadable; none loadable. Unloading doesn't compromise availability. +// Unloads. +TEST(AvailabilityPreservingPolicyTest, UnloadIfOtherReadyEvenIfError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(1, action->id.version); +} + +// Multiple unloadable; none loadable. Availability is achieved despite no +// aspired versions being loaded, because one or more non-aspired versions are +// loaded. Unloading one non-aspired version doesn't compromise availability. +// Unloads the lowest such version. +TEST(AvailabilityPreservingPolicyTest, UnloadIfNoAspiredVersionsReady) { + std::vector versions; + versions.push_back({{"test", 2}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kLoading, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(1, action->id.version); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 7515c5abe1f..7426698ae3d 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow_serving/core/basic_manager.h" #include +#include #include #include #include @@ -23,29 +24,52 @@ limitations under the License. #include #include +#include "absl/status/status.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/source.h" -#include "tensorflow_serving/util/cleanup.h" +#include "tensorflow_serving/resources/resource_tracker.h" +#include "tensorflow_serving/util/event_bus.h" #include "tensorflow_serving/util/hash.h" #include "tensorflow_serving/util/inline_executor.h" +#include "tensorflow_serving/util/retrier.h" #include "tensorflow_serving/util/threadpool_executor.h" namespace tensorflow { namespace serving { +namespace { + +std::unique_ptr CreateExecutor(Env* const env, + const uint32 num_threads, + const string& threadpool_name) { + std::unique_ptr executor; + if (num_threads == 0) { + executor.reset(new InlineExecutor()); + } else { + executor.reset(new ThreadPoolExecutor(env, threadpool_name, num_threads)); + } + return executor; +} + +} // namespace + struct BasicManager::ServingMap::EqRequest { bool operator()(const ServableRequest& lhs, const ServableRequest& rhs) const { if (lhs.version != rhs.version) { return false; } - // Even if there is a small probability that version checking can eliminate - // string checking, we should do that since O(string_equality) >> - // O(version_equality) + if (lhs.auto_version_policy != rhs.auto_version_policy) { + return false; + } + // Even if there is a small probability that version & policy checking can + // eliminate string checking, we should do that since O(string_equality) >> + // O(version_equality) + O(policy_equality). if (lhs.name != rhs.name) { return false; } @@ -54,7 +78,7 @@ struct BasicManager::ServingMap::EqRequest { }; struct BasicManager::ServingMap::HashRequest { - uint64 operator()(const ServableRequest& request) const { + uint64_t operator()(const ServableRequest& request) const { // Hash codes for many common types are remarkably bad, often clustering // around the same values of the low and/or high bits for linear // sequences of inputs such as 1, 2, 3; or addresses of consecutively @@ -65,12 +89,17 @@ struct BasicManager::ServingMap::HashRequest { // make the high bits contain more entropy from the entire hash code. // It's based on Fibonacci hashing from Knuth's Art of Computer // Programming volume 3, section 6.4. - const uint64 version_hash = [&]() -> uint64 { + const uint64_t version_hash = [&]() -> uint64_t { if (request.version) { - return std::hash()(request.version.value()) * + return std::hash()(request.version.value()) * 0x9E3779B97F4A7C13; // (sqrt(5) - 1)/2 as a binary fraction. } else { - return 0xDECAFCAFFE; + switch (request.auto_version_policy) { + case ServableRequest::AutoVersionPolicy::kEarliest: + return 0x01234CAFFE; + case ServableRequest::AutoVersionPolicy::kLatest: + return 0xDECAFCAFFE; + } } }(); // Using version_hash as the seed here to combine the hashes. @@ -98,7 +127,7 @@ std::vector BasicManager::ServingMap::ListAvailableServableIds() return ids; } -Status BasicManager::ServingMap::GetUntypedServableHandle( +absl::Status BasicManager::ServingMap::GetUntypedServableHandle( const ServableRequest& request, std::unique_ptr* const untyped_handle) { std::shared_ptr handles_map = handles_map_.get(); @@ -115,7 +144,7 @@ Status BasicManager::ServingMap::GetUntypedServableHandle( // previous map is freed, when we are doing handles_map updates. untyped_handle->reset(new SharedPtrHandle( harness.id(), std::shared_ptr(handles_map, harness.loader()))); - return Status::OK(); + return absl::OkStatus(); } std::map> @@ -124,8 +153,8 @@ BasicManager::ServingMap::GetAvailableUntypedServableHandles() const { std::shared_ptr handles_map = handles_map_.get(); for (const auto& handle : *handles_map) { const ServableRequest& request = handle.first; - // If the entry is the one for the latest request, skip it. We would already - // get it from the entry which has the specific request. + // If the entry is one of the auto-versioned request ones, skip it. We would + // already get it from the entry which has the specific request. if (!request.version) { continue; } @@ -163,12 +192,23 @@ void BasicManager::ServingMap::Update(const ManagedMap& managed_map) { } std::unique_ptr new_handles_map(new HandlesMap()); + auto prev_iter = sorted_available_map.end(); for (auto iter = sorted_available_map.begin(); iter != sorted_available_map.end(); ++iter) { std::shared_ptr harness = iter->second; new_handles_map->emplace(ServableRequest::FromId(harness->id()), harness); - // If this is the last harness in the stream, add it again to the - // handles_map, marking it as the latest for that stream. + + // If this is the first harness in the stream for a given servable name, add + // it again to the handles_map, marking it as the earliest for that stream. + if (prev_iter == sorted_available_map.end() || + prev_iter->second->id().name != harness->id().name) { + const ServableRequest earliest_request = + ServableRequest::Earliest(harness->id().name); + new_handles_map->emplace(earliest_request, harness); + } + + // If this is the last harness in the stream for a given servable name, add + // it again to the handles_map, marking it as the latest for that stream. const auto next_iter = std::next(iter); if (next_iter == sorted_available_map.end() || next_iter->second->id().name != harness->id().name) { @@ -176,6 +216,8 @@ void BasicManager::ServingMap::Update(const ManagedMap& managed_map) { ServableRequest::Latest(harness->id().name); new_handles_map->emplace(latest_request, harness); } + + prev_iter = iter; } // This blocks until the last handle given out by the old handles map is @@ -183,75 +225,94 @@ void BasicManager::ServingMap::Update(const ManagedMap& managed_map) { handles_map_.Update(std::move(new_handles_map)); } -Status BasicManager::Create(Options options, - std::unique_ptr* manager) { - std::unique_ptr load_unload_executor; - if (options.num_load_unload_threads == 0) { - LOG(INFO) << "Using InlineExecutor for BasicManager."; - load_unload_executor.reset(new InlineExecutor()); - } else { - LOG(INFO) << "Using ThreadPoolExecutor for BasicManager with " - "num_load_unload_threads: " - << options.num_load_unload_threads; - load_unload_executor.reset(new ThreadPoolExecutor( - options.env, "BasicManager_LoadUnload_ThreadPool", - options.num_load_unload_threads)); - } - - LoaderHarness::Options harness_options; - harness_options.max_num_load_retries = options.max_num_load_retries; - harness_options.load_retry_interval_micros = - options.load_retry_interval_micros; - manager->reset(new BasicManager(std::move(load_unload_executor), - std::move(options.resource_tracker), - options.servable_event_bus, harness_options)); - return Status::OK(); -} - -BasicManager::BasicManager(std::unique_ptr load_unload_executor, - std::unique_ptr resource_tracker, - EventBus* servable_event_bus, - const LoaderHarness::Options& harness_options) - : harness_options_(harness_options), - servable_event_bus_(servable_event_bus) { - load_unload_executor_ = std::move(load_unload_executor); +absl::Status BasicManager::Create(Options options, + std::unique_ptr* manager) { + manager->reset(new BasicManager( + options.env, options.num_load_threads, options.num_unload_threads, + options.max_num_load_retries, std::move(options.should_retry_model_load), + options.load_retry_interval_micros, options.flush_filesystem_caches, + std::move(options.resource_tracker), options.servable_event_bus, + std::move(options.pre_load_hook))); + return absl::OkStatus(); +} + +BasicManager::BasicManager( + Env* const env, const uint32 num_load_threads, + const uint32 num_unload_threads, uint32 max_num_load_retries, + std::function should_retry_model_load, + int64_t load_retry_interval_micros, bool flush_filesystem_caches, + std::unique_ptr resource_tracker, + EventBus* servable_event_bus, + std::function pre_load_hook) + : servable_event_bus_(servable_event_bus), + should_retry_model_load_(std::move(should_retry_model_load)), + env_(env), + num_load_threads_(num_load_threads), + flush_filesystem_caches_(flush_filesystem_caches), + pre_load_hook_(std::move(pre_load_hook)) { + harness_options_.max_num_load_retries = max_num_load_retries; + harness_options_.load_retry_interval_micros = load_retry_interval_micros; + harness_options_.error_callback = [this](const ServableId& id, + const absl::Status& error) { + PublishOnEventBus({id, ServableState::ManagerState::kEnd, error}); + }; + + { + mutex_lock l(load_executor_mu_); + load_executor_ = + CreateExecutor(env_, num_load_threads, "BasicManager_Load_ThreadPool"); + } + unload_executor_ = CreateExecutor(env_, num_unload_threads, + "BasicManager_Unload_ThreadPool"); resource_tracker_ = std::move(resource_tracker); } BasicManager::~BasicManager() { - // Reset the executor first to finish all pending loads/unloads. - load_unload_executor_.reset(); - UnloadAllServables(); + // Reset the executors first to finish all pending loads/unloads. + { + mutex_lock l(load_executor_mu_); + load_executor_.reset(); + } + unload_executor_.reset(); + + const absl::Status unload_status = UnloadAllServables(); + if (!unload_status.ok()) { + LOG(ERROR) << "Error unloading all servables in BasicManager destructor: " + << unload_status; + } } -void BasicManager::UnloadAllServables() { +absl::Status BasicManager::UnloadAllServables() { LOG(INFO) << "Unload all remaining servables in the manager."; + absl::Status status = absl::OkStatus(); { mutex_lock l(mu_); for (auto it = managed_map_.begin(); it != managed_map_.end(); ++it) { LoaderHarness* const harness = it->second.get(); if (harness->state() == LoaderHarness::State::kReady) { - harness->UnloadRequested(); - harness->StartQuiescing(); - harness->DoneQuiescing(); - harness->Unload(); + status.Update(harness->UnloadRequested()); + status.Update(harness->StartQuiescing()); + status.Update(harness->DoneQuiescing()); + status.Update(harness->Unload()); } if (harness->state() == LoaderHarness::State::kQuiescing) { - harness->DoneQuiescing(); - harness->Unload(); + status.Update(harness->DoneQuiescing()); + status.Update(harness->Unload()); } if (harness->state() == LoaderHarness::State::kQuiesced) { - harness->Unload(); + status.Update(harness->Unload()); } } } + + return status; } std::vector BasicManager::ListAvailableServableIds() const { return serving_map_.ListAvailableServableIds(); } -Status BasicManager::GetUntypedServableHandle( +absl::Status BasicManager::GetUntypedServableHandle( const ServableRequest& request, std::unique_ptr* const untyped_handle) { return serving_map_.GetUntypedServableHandle(request, untyped_handle); @@ -279,22 +340,13 @@ BasicManager::ManagedMap::iterator BasicManager::FindHarnessInMap( return managed_map_.end(); } -void BasicManager::DeleteHarness(const ServableId& id) { - const auto it = FindHarnessInMap(id); - DCHECK(it != managed_map_.end()); - if (it == managed_map_.end()) { - LOG(ERROR) << "Request to delete harness for " << id - << ", but no such harness found in managed_map_"; - return; - } - managed_map_.erase(it); -} - -Status BasicManager::ManageServableInternal( +absl::Status BasicManager::ManageServableInternal( ServableData> servable, std::function(const ServableId&, std::unique_ptr)> harness_creator) { + VLOG(1) << "Request to start managing servable " << servable.id(); + mutex_lock l(mu_); const auto iter = BasicManager::FindHarnessInMap(servable.id()); @@ -311,20 +363,21 @@ Status BasicManager::ManageServableInternal( std::shared_ptr harness = harness_creator(servable.id(), std::move(loader)); + if (should_retry_model_load_) { + harness->set_should_retry(should_retry_model_load_); + } if (!servable.status().ok()) { harness->Error(servable.status()); - PublishOnEventBus( - {harness->id(), ServableState::ManagerState::kEnd, harness->status()}); } else { PublishOnEventBus({harness->id(), ServableState::ManagerState::kStart, harness->status()}); - managed_map_.emplace(servable.id().name, harness); } + managed_map_.emplace(servable.id().name, harness); - return Status::OK(); + return absl::OkStatus(); } -Status BasicManager::ManageServable( +absl::Status BasicManager::ManageServable( ServableData> servable) { return ManageServableInternal( std::move(servable), @@ -334,8 +387,33 @@ Status BasicManager::ManageServable( }); } -Status BasicManager::GetHealthyHarness(const ServableId& id, - LoaderHarness** harness) { +absl::Status BasicManager::StopManagingServable(const ServableId& id) { + VLOG(1) << "Request to stop managing servable " << id; + mutex_lock l(mu_); + const auto it = FindHarnessInMap(id); + if (it == managed_map_.end()) { + LOG(ERROR) << "Request to delete harness for " << id + << ", but no such harness found in managed_map_"; + return errors::FailedPrecondition("This servable is not being managed: ", + id.DebugString()); + } + const auto state = it->second->state(); + if (state != LoaderHarness::State::kNew && + state != LoaderHarness::State::kError && + state != LoaderHarness::State::kDisabled) { + LOG(ERROR) << "Request to delete harness for " << id + << ", but it is not in a new or end state. State: " << state; + return errors::FailedPrecondition( + "This servable is not in a new or end state and we cannot stop " + "managing it: ", + id.DebugString(), " ", LoaderHarness::StateDebugString(state)); + } + managed_map_.erase(it); + return absl::OkStatus(); +} + +absl::Status BasicManager::GetHealthyHarness(const ServableId& id, + LoaderHarness** harness) { // Look up the request servable's harness. auto iter = FindHarnessInMap(id); if (iter == managed_map_.end()) { @@ -345,7 +423,7 @@ Status BasicManager::GetHealthyHarness(const ServableId& id, } TF_RETURN_IF_ERROR(iter->second->status()); *harness = iter->second.get(); - return Status::OK(); + return absl::OkStatus(); } std::vector BasicManager::GetLoadersCurrentlyUsingResources() @@ -407,30 +485,45 @@ std::vector BasicManager::GetManagedServableNames() const { return servable_names; } -Status BasicManager::ExecuteLoad(LoaderHarness* harness) { +absl::Status BasicManager::ExecuteLoad(LoaderHarness* harness) { PublishOnEventBus({harness->id(), ServableState::ManagerState::kLoading, harness->status()}); - const auto load_status = harness->Load(ResourceAllocation()); + // We save the id of the harness so that we can publish it after Load(). (We + // can't query harness again after Load() as it may be deleted by another + // thread that called StopManagingServable().) + const ServableId id = harness->id(); - { - mutex_lock l(mu_); + if (pre_load_hook_) { + pre_load_hook_(id); + } - if (!load_status.ok()) { - PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, - harness->status()}); - DeleteHarness(harness->id()); - return load_status; + // We don't hold the lock while calling Load() as it may block. + const absl::Status status = harness->Load(); + + // Whether the load succeeded or failed, flush filesystem caches if there is + // only one load thread. + if (flush_filesystem_caches_ && num_load_threads() <= 1) { + const absl::Status flush_status = Env::Default()->FlushFileSystemCaches(); + if (!flush_status.ok()) { + LOG(WARNING) << "flushing filesystem caches failed: " << flush_status; } + } + TF_RETURN_IF_ERROR(status); + + { + mutex_lock l(mu_); UpdateServingMap(); - PublishOnEventBus({harness->id(), ServableState::ManagerState::kAvailable, - harness->status()}); } - return Status::OK(); + + PublishOnEventBus( + {id, ServableState::ManagerState::kAvailable, absl::OkStatus()}); + return absl::OkStatus(); } void BasicManager::LoadServable(const ServableId& id, const DoneCallback done_callback) { + VLOG(1) << "Request to load servable " << id; LoadOrUnloadRequest request; request.kind = LoadOrUnloadRequest::Kind::kLoad; request.servable_id = id; @@ -440,47 +533,46 @@ void BasicManager::LoadServable(const ServableId& id, void BasicManager::CancelLoadServableRetry(const ServableId& id) { mutex_lock l(mu_); LoaderHarness* harness; - const Status status = GetHealthyHarness(id, &harness); + const absl::Status status = GetHealthyHarness(id, &harness); if (!status.ok()) { return; } - harness->set_cancel_load_retry(true); + harness->set_should_retry([](absl::Status status) { return false; }); } -Status BasicManager::ExecuteUnload(LoaderHarness* harness) { +absl::Status BasicManager::ExecuteUnload(LoaderHarness* harness) { + // We save the id of the harness so that we can publish it after Unload(). (We + // can't query harness again after Unload() as it may be deleted by another + // thread that called StopManagingServable().) + const ServableId id = harness->id(); + { // StartQuiescing() would have been already called. mutex_lock l(mu_); - PublishOnEventBus({harness->id(), ServableState::ManagerState::kUnloading, - harness->status()}); + PublishOnEventBus( + {id, ServableState::ManagerState::kUnloading, harness->status()}); UpdateServingMap(); - harness->DoneQuiescing(); + TF_RETURN_IF_ERROR(harness->DoneQuiescing()); } - harness->Unload(); - - { - mutex_lock l(mu_); - auto iter = FindHarnessInMap(harness->id()); - PublishOnEventBus({iter->second->id(), ServableState::ManagerState::kEnd, - iter->second->status()}); - // This erase will lead to the LoaderHarness being deleted. - managed_map_.erase(iter); - } - return Status::OK(); + // We don't hold the lock while calling Unload() as it may block. + TF_RETURN_IF_ERROR(harness->Unload()); + PublishOnEventBus({id, ServableState::ManagerState::kEnd, absl::OkStatus()}); + return absl::OkStatus(); } void BasicManager::UnloadServable(const ServableId& id, const DoneCallback done_callback) { + VLOG(1) << "Request to unload servable " << id; LoadOrUnloadRequest request; request.kind = LoadOrUnloadRequest::Kind::kUnload; request.servable_id = id; LoadOrUnloadServable(request, done_callback); } -Status BasicManager::ExecuteLoadOrUnload(const LoadOrUnloadRequest& request, - LoaderHarness* harness) { - Status execution_status; +absl::Status BasicManager::ExecuteLoadOrUnload( + const LoadOrUnloadRequest& request, LoaderHarness* harness) { + absl::Status execution_status; switch (request.kind) { case LoadOrUnloadRequest::Kind::kLoad: execution_status = ExecuteLoad(harness); @@ -500,9 +592,44 @@ Status BasicManager::ExecuteLoadOrUnload(const LoadOrUnloadRequest& request, return execution_status; } +void BasicManager::SetNumLoadThreads(const uint32 num_load_threads) { + // ThreadPoolExecutor destructor, implicitly calling ThreadPool destructor, + // waits for all scheduled work to finish. Should we wait within + // `load_executor_mu_` or outside? + // + // When changing `num_load_threads_` from M to N, the effective number of + // threads changes like this: + // - From M to 0 then to N, if destruct within lock; or + // - From M to (up to) M+N then to N, if destruct outside lock. + // The former is more intuitive when M and N are small (e.g. client intention + // is inline or single threaded loading), while the latter makes more sense + // when they are large. + const uint32 old_num_threads = num_load_threads_.load(); + if (old_num_threads < 2 || num_load_threads < 2) { // destruct within lock + mutex_lock l(load_executor_mu_); + load_executor_.reset(); + num_load_threads_.store(num_load_threads); + load_executor_ = + CreateExecutor(env_, num_load_threads, "BasicManager_Load_ThreadPool"); + } else { // destruct outside lock + std::unique_ptr old_executor; + { + mutex_lock l(load_executor_mu_); + old_executor = std::move(load_executor_); + num_load_threads_.store(num_load_threads); + load_executor_ = CreateExecutor(env_, num_load_threads, + "BasicManager_Load_ThreadPool"); + } + } +} + +uint32 BasicManager::num_load_threads() const { + return num_load_threads_.load(); +} + void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, DoneCallback done_callback) { - const Status status = [&]() { + const absl::Status status = [&]() { mutex_lock l(mu_); LoaderHarness* harness; TF_RETURN_IF_ERROR(GetHealthyHarness(request.servable_id, &harness)); @@ -516,21 +643,34 @@ void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, TF_RETURN_IF_ERROR(harness->UnloadRequested()); break; } - return Status::OK(); + return absl::OkStatus(); }(); if (!status.ok()) { done_callback(status); return; } - load_unload_executor_->Schedule([this, request, done_callback]() { - HandleLoadOrUnloadRequest(request, done_callback); - }); + + switch (request.kind) { + case LoadOrUnloadRequest::Kind::kLoad: { + mutex_lock l(load_executor_mu_); + load_executor_->Schedule([this, request, done_callback]() { + HandleLoadOrUnloadRequest(request, done_callback); + }); + break; + } + case LoadOrUnloadRequest::Kind::kUnload: { + unload_executor_->Schedule([this, request, done_callback]() { + HandleLoadOrUnloadRequest(request, done_callback); + }); + break; + } + } } void BasicManager::HandleLoadOrUnloadRequest(const LoadOrUnloadRequest& request, DoneCallback done_callback) { // Decision phase. - Status decision_status; + absl::Status decision_status; LoaderHarness* harness; { // We serialize the decision phases of the requests. We will make a decision @@ -545,70 +685,44 @@ void BasicManager::HandleLoadOrUnloadRequest(const LoadOrUnloadRequest& request, } // Execution phase. - const Status execution_status = ExecuteLoadOrUnload(request, harness); + const absl::Status execution_status = ExecuteLoadOrUnload(request, harness); done_callback(execution_status); } -Status BasicManager::ApproveLoadOrUnload(const LoadOrUnloadRequest& request, - LoaderHarness** harness) { +absl::Status BasicManager::ApproveLoadOrUnload( + const LoadOrUnloadRequest& request, LoaderHarness** harness) { mutex_lock l(mu_); TF_RETURN_IF_ERROR(GetHealthyHarness(request.servable_id, harness)); - Status approval_status; switch (request.kind) { case LoadOrUnloadRequest::Kind::kLoad: { - approval_status = ApproveLoad(*harness, &l); + TF_RETURN_IF_ERROR(ApproveLoad(*harness, &l)); break; } case LoadOrUnloadRequest::Kind::kUnload: { - approval_status = ApproveUnload(*harness); + TF_RETURN_IF_ERROR(ApproveUnload(*harness)); break; } } - if (approval_status.ok()) { - ++num_ongoing_load_unload_executions_; - } + ++num_ongoing_load_unload_executions_; - return approval_status; + return absl::OkStatus(); } -Status BasicManager::ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) { +absl::Status BasicManager::ApproveLoad(LoaderHarness* harness, + mutex_lock* mu_lock) { if (resource_tracker_ != nullptr) { // Attempt to reserve resources for the load. - while (true) { - resource_tracker_->RecomputeUsedResources( - GetLoadersCurrentlyUsingResources()); - bool resources_reserved; - TF_RETURN_IF_ERROR(resource_tracker_->ReserveResources( - *harness->loader(), &resources_reserved)); - if (resources_reserved) { - // Woohoo! We got our resources. - LOG(INFO) << "Successfully reserved resources to load servable " - << harness->id().DebugString(); - break; - } - - // We weren't able to reserve the resources. See if there are any ongoing - // load/unload executions that may be temporarily tying up resources. - if (num_ongoing_load_unload_executions_ == 0) { - // There are no ongoing load/unloads, so we really are out of resources - // for this servable. - LOG(WARNING) << "Unable to reserve resources to load servable " - << harness->id().DebugString(); - const Status error = errors::ResourceExhausted( - "Insufficient resources to load servable ", - harness->id().DebugString()); - harness->Error(error); - PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, - harness->status()}); - DeleteHarness(harness->id()); - return error; - } else { - // Wait until at least one load/unload request finishes, then retry. - num_ongoing_load_unload_executions_cv_.wait(*mu_lock); - } + const absl::Status resource_reservation_status = + ReserveResources(harness, mu_lock); + if (!resource_reservation_status.ok()) { + LOG(WARNING) << resource_reservation_status; + harness->Error(resource_reservation_status); + PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, + resource_reservation_status}); + return resource_reservation_status; } } @@ -617,18 +731,65 @@ Status BasicManager::ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) { // GetLoadersCurrentlyUsingResources(). TF_RETURN_IF_ERROR(harness->LoadApproved()); - return Status::OK(); + return absl::OkStatus(); } -Status BasicManager::ApproveUnload(LoaderHarness* harness) { +absl::Status BasicManager::ApproveUnload(LoaderHarness* harness) { // Transition to state kQuiescing inside the decision phase, to prevent any // concurrent unload requests from executing. - // - // StartQuiescing() returns an error status if the harness is not in a state - // to be quiesced. TF_RETURN_IF_ERROR(harness->StartQuiescing()); - return Status::OK(); + return absl::OkStatus(); +} + +absl::Status BasicManager::ReserveResources(LoaderHarness* harness, + mutex_lock* mu_lock) { + while (true) { + TF_RETURN_IF_ERROR(resource_tracker_->RecomputeUsedResources( + GetLoadersCurrentlyUsingResources())); + bool resources_reserved; + // We retry reserving resources because it may involve transiently failing + // operations like file-reads. + const absl::Status reserve_resources_status = Retry( + strings::StrCat("Reserving resources for servable: ", + harness->id().DebugString()), + harness_options_.max_num_load_retries, + harness_options_.load_retry_interval_micros, + [&]() TF_EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return resource_tracker_->ReserveResources(*harness->loader(), + &resources_reserved); + }, + [&](absl::Status status) { return harness->should_retry(status); }); + if (!reserve_resources_status.ok()) { + return absl::Status( + reserve_resources_status.code(), + strings::StrCat( + "Error while attempting to reserve resources to load servable ", + harness->id().DebugString(), ": ", + reserve_resources_status.message())); + } + if (resources_reserved) { + // Woohoo! We got our resources. + LOG(INFO) << "Successfully reserved resources to load servable " + << harness->id().DebugString(); + return absl::OkStatus(); + } + + // We weren't able to reserve the resources. See if there are any + // ongoing load/unload executions that may be temporarily tying up + // resources. + if (num_ongoing_load_unload_executions_ == 0) { + // There are no ongoing load/unloads, so we really are out of + // resources for this servable. + return errors::ResourceExhausted( + "Insufficient resources to load servable ", + harness->id().DebugString()); + } else { + // Wait until at least one load/unload request finishes, then retry. + VLOG(1) << "Waiting for another load/unload request to finish"; + num_ongoing_load_unload_executions_cv_.wait(*mu_lock); + } + } } void BasicManager::PublishOnEventBus(const ServableState& state) { diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index 0b1bf63b40c..54a2fc84d2e 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -16,11 +16,13 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_CORE_BASIC_MANAGER_H_ #define TENSORFLOW_SERVING_CORE_BASIC_MANAGER_H_ +#include #include #include #include #include +#include "absl/types/optional.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/hash/hash.h" @@ -39,73 +41,92 @@ limitations under the License. #include "tensorflow_serving/util/event_bus.h" #include "tensorflow_serving/util/executor.h" #include "tensorflow_serving/util/fast_read_dynamic_ptr.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { -// Helps manage the lifecycle of servables including loading, serving and -// unloading them. The manager accepts servables in the form of Loaders. -// -// We start managing a servable through one of the ManageServable* methods. You -// can go on to load the servable after this by calling LoadServable. Loading -// will also make the servable available to serve. Once you decide to unload it, -// you can call UnloadServable on it, which will make it unavailable to serve, -// then unload the servable and delete it. After unload, or after hitting an -// error, the servable is no longer managed by the manager. -// -// BasicManager tracks the resources (e.g. RAM) used by loaded servables, and -// only allows loading new servables that fit within the overall resource pool. -// -// BasicManager can be configured to use a thread-pool to do it's load and -// unloads. This makes the {Load,Unload}Servable() methods schedule the -// load/unloads rather than executing them synchronously. If there are more -// pending load/unloads than threads in the thread pool, they are processed in -// FIFO order. -// -// In the presence of loaders that over-estimate their servables' resource needs -// and/or only bind their servables' resources to device instances, load/unload -// concurrency can be reduced below the thread-pool size. That is because we may -// have to wait for one servable's load/unload to finish to pin down the -// resource availability for loading another servable. -// -// REQUIRES: -// 1. Order of method calls - -// ManageServable*() -> LoadServable() -> UnloadServable(). -// 2. Do not schedule concurrent load and unloads of the same servable. -// 3. Do not call load or unload multiple times on the same servable. -// -// This class is thread-safe. -// -// Example usage: -// -// const ServableId id = {kServableName, 0}; -// std::unique_ptr loader = ...; -// ... -// BasicManager manager; -// TF_CHECK_OK(manager.ManageServable( -// CreateServableData(id, std::move(loader)))); -// TF_CHECK_OK(manager.LoadServable(id)); -// -// ... -// TF_CHECK_OK(manager.GetServableHandle( -// ServableRequest::Latest(kServableName), &handle)); -// ... -// -// TF_CHECK_OK(manager.UnloadServable(id)); +namespace test_util { +class BasicManagerTestAccess; +} // namespace test_util + +/// Helps manage the lifecycle of servables including loading, serving and +/// unloading them. The manager accepts servables in the form of Loaders. +/// +/// We start managing a servable through one of the ManageServable* methods. You +/// can go on to load the servable after this by calling LoadServable(). Loading +/// will also make the servable available to serve. Once you decide to unload +/// it, you can call UnloadServable() on it, which will make it unavailable to +/// serve, then unload the servable. +/// +/// Servables are retained until StopManagingServable() is called. This allows a +/// higher level manager with more information to decide when it's safe to +/// forget about a servable. +/// +/// BasicManager tracks the resources (e.g. RAM) used by loaded servables, and +/// only allows loading new servables that fit within the overall resource pool. +/// +/// BasicManager can be configured to use a thread-pool to do it's load and +/// unloads. This makes the LoadServable() and UnloadServable() methods schedule +/// the load/unloads rather than executing them synchronously. If there are more +/// pending load/unloads than threads in the thread pool, they are processed in +/// FIFO order. +/// +/// In the presence of loaders that over-estimate their servables' resource +/// needs and/or only bind their servables' resources to device instances, +/// load/unload concurrency can be reduced below the thread-pool size. That is +/// because we may have to wait for one servable's load/unload to finish to pin +/// down the resource availability for loading another servable. +/// +/// REQUIRES: +/// 1. Order of method calls - +/// ManageServable() (and variants) -> LoadServable() -> UnloadServable() -> +/// StopManagingServable(). +/// 2. Do not schedule concurrent load and unloads of the same servable. +/// 3. Do not call load or unload multiple times on the same servable. +/// +/// This class is thread-safe. +/// +/// Example usage: +/// +/// const ServableId id = {kServableName, 0}; +/// std::unique_ptr<Loader> loader = ...; +/// ... +/// BasicManager manager; +/// TF_CHECK_OK(manager.ManageServable( +/// CreateServableData(id, std::move(loader)))); +/// TF_CHECK_OK(manager.LoadServable(id)); +/// +/// ... +/// TF_CHECK_OK(manager.GetServableHandle( +/// ServableRequest::Latest(kServableName), &handle)); +/// ... +/// +/// TF_CHECK_OK(manager.UnloadServable(id)); +/// TF_CHECK_OK(manager.StopManagingServable(id)); class BasicManager : public Manager { public: + // Type of the callback to be called just before a servable is to be loaded. + using PreLoadHook = std::function; + + /// Config options and pluggable objects that will be used by the + /// BasicManager. struct Options { // The resource tracker to use while managing servable resources. Optional. // If left as nullptr, we do not validate servable resource usage. std::unique_ptr resource_tracker; - // The number of threads in the thread-pool used to load and unload - // servables. + // The number of threads in the thread-pool used to load servables. + // + // If set as 0, we don't use a thread-pool, and LoadServable() blocks. + uint32 num_load_threads = 0; + + // The number of threads in the thread-pool used to unload servables. // - // If set as 0, we don't use a thread-pool, and the {Load,Unload}Servable() - // methods block. - uint32 num_load_unload_threads = 0; + // If set as 0, we don't use a thread-pool, and UnloadServable() blocks. + uint32 num_unload_threads = 0; + + // Defines how we want to retry when model loading fails. + std::function should_retry_model_load; // EventBus to publish servable state changes. This is optional, if unset, // we don't publish. @@ -120,15 +141,25 @@ class BasicManager : public Manager { // The interval, in microseconds, between each servable load retry. If set // negative, we don't wait. // Default: 1 minute. - int64 load_retry_interval_micros = 1LL * 60 * 1000 * 1000; + int64_t load_retry_interval_micros = 1LL * 60 * 1000 * 1000; + + // If true, and there are not multiple load threads, filesystem caches will + // be flushed after each servable is loaded. (Cache flush is skipped when + // multiple load threads are active, in order to avoid setting back a + // concurrent load on another thread.) + bool flush_filesystem_caches = false; // The environment to use for starting threads in the thread-pool. Env* env = Env::Default(); + + // Callback to be called just before a servable is to be loaded. This will + // called on the same manager load thread which starts the load. + PreLoadHook pre_load_hook; }; static Status Create(Options options, std::unique_ptr* manager); - // If configured to use a load/unload thread-pool, waits until all scheduled - // loads and unloads have finished and then destroys the set of threads. + /// If configured to use a load/unload thread-pool, waits until all scheduled + /// loads and unloads have finished and then destroys the set of threads. ~BasicManager() override; std::vector ListAvailableServableIds() const override; @@ -140,102 +171,114 @@ class BasicManager : public Manager { std::map> GetAvailableUntypedServableHandles() const override; - // Starts managing the servable. - // - // Returns an error if given a servable that is already being managed. - // - // If 'servable' is in an error state, this method does *not* return an error. - // Instead, the manager accepts the servable, puts it in state kError (with a - // notification sent to the event bus), and then immediately stops managing - // it. This behavior facilitates uniform handling of errors that occur in - // sources (e.g. invalid file path to servable data) and ones that occur in - // the manager (e.g. insufficient resources to load servable). + /// Starts managing the servable. + /// + /// Returns an error if given a servable that is already being managed. + /// + /// If *servable* is in an error state, this method does **not** return an + /// error. Instead, the manager accepts the servable, puts it in state kError + /// (with a notification sent to the event bus), and then immediately stops + /// managing it. This behavior facilitates uniform handling of errors that + /// occur in sources (e.g. invalid file path to servable data) and ones that + /// occur in the manager (e.g. insufficient resources to load servable). Status ManageServable(ServableData> servable); - // Similar to the above method, but callers, usually other managers built on - // top of this one, can associate additional state with the servable. - // Additional state may be ACL or lifetime metadata for that servable. The - // ownership of the state is transferred to this class. + /// Similar to the above method, but callers, usually other managers built on + /// top of this one, can associate additional state with the servable. + /// Additional state may be ACL or lifetime metadata for that servable. The + /// ownership of the state is transferred to this class. template Status ManageServableWithAdditionalState( ServableData> servable, std::unique_ptr additional_state); - // Returns the names of all the servables managed by this manager. The names - // will be duplicate-free and not in any particular order. + /// Tells the manager to stop managing this servable. Requires that the + /// servable is currently being managed and that its state is one of {kNew, + /// kError, kDisabled}. + Status StopManagingServable(const ServableId& id); + + /// @return the names of all the servables managed by this manager. The names + /// will be duplicate-free and not in any particular order. std::vector GetManagedServableNames() const; - // Returns the state snapshots of all the servables of a particular stream, - // managed by this manager. - // - // T is the additional-state type, if any. + /// @return the state snapshots of all the servables of a particular stream, + /// managed by this manager. + /// + /// T is the additional-state type, if any. template std::vector> GetManagedServableStateSnapshots( const string& servable_name) const; - // Returns the state snapshot of a particular servable-id managed by this - // manager if available. - // - // REQUIRES: This manager should have been managing this servable already, - // else we return nullopt. + /// @return the state snapshot of a particular servable-id managed by this + /// manager if available. + /// + /// REQUIRES: This manager should have been managing this servable already, + /// else we return nullopt. template - optional> GetManagedServableStateSnapshot( + absl::optional> GetManagedServableStateSnapshot( const ServableId& id); - // Returns the additional state for the servable. Returns nullptr if there is - // no additional state setup or if there is a type mismatch between what was - // setup and what is being asked for. - // - // REQUIRES: This manager should have been managing this servable already, - // else we return nullptr. + /// @return the additional state for the servable. Returns nullptr if there is + /// no additional state setup or if there is a type mismatch between what was + /// setup and what is being asked for. + /// + /// REQUIRES: This manager should have been managing this servable already, + /// else we return nullptr. template T* GetAdditionalServableState(const ServableId& id); - // Callback called at the end of {Load,Unload}Servable(). We pass in the - // status of the operation to the callback. + /// Callback called at the end of {Load,Unload}Servable(). We pass in the + /// status of the operation to the callback. using DoneCallback = std::function; - // Loads the servable with this id, and updates the serving map too. Calls - // 'done_callback' with ok iff the servable was loaded successfully, else - // returns an error status. - // - // If using a thread-pool, this method transitions the servable harness to - // kLoading state, schedules the load and returns, otherwise it - // completes the load before returning. - // - // REQUIRES: This manager should have been managing this servable already, for - // it to be loaded, else we call 'done_callback' with an error status. Do not - // call this multiple times on the same servable. Only one of those will - // succeed and the rest will fail with an error status. + /// Loads the servable with this id, and updates the serving map too. Calls + /// *done_callback* with ok iff the servable was loaded successfully, else + /// returns an error status. + /// + /// If using a thread-pool, this method transitions the servable harness to + /// kLoading state, schedules the load and returns, otherwise it + /// completes the load before returning. + /// + /// REQUIRES: This manager should have been managing this servable already, + /// for it to be loaded, else we call *done_callback* with an error status. Do + /// not call this multiple times on the same servable. Only one of those will + /// succeed and the rest will fail with an error status. void LoadServable(const ServableId& id, DoneCallback done_callback); - // Cancels retrying the servable load during LoadServable(). Does nothing if - // the servable isn't managed. - // - // If the retries are cancelled, the servable goes into a state dependent on - // the last Load() called on it. If the last Load() was successful, it will be - // in state kReady, else in kError. + /// Cancels retrying the servable load during LoadServable() by replacing the + /// LoaderHarness::should_retry with a function that always returns false. + /// Does nothing if the servable isn't managed. + /// + /// If the retries are cancelled, the servable goes into a state dependent on + /// the last Load() called on it. If the last Load() was successful, it will + /// be in state kReady, else in kError. void CancelLoadServableRetry(const ServableId& id); - // Unloads the servable with this id, and updates the serving map too. Calls - // 'done_callback' with ok iff the servable was unloaded successfully, else - // returns an error status. - // - // If using a thread-pool, this method transitions the servable harness to - // kQuiescing state, schedules the unload and returns, otherwise it completes - // the unload before returning. - // - // REQUIRES: This manager should have been managing this servable already, for - // it to be unloaded, else calls 'done_callback' with an error status. Do not - // call this multiple times on the same servable. Only one of those will - // succeed and the rest will fail with an error status. + /// Unloads the servable with this id, and updates the serving map too. Calls + /// *done_callback* with ok iff the servable was unloaded successfully, else + /// returns an error status. + /// + /// If using a thread-pool, this method transitions the servable harness to + /// kQuiescing state, schedules the unload and returns, otherwise it completes + /// the unload before returning. + /// + /// REQUIRES: This manager should have loaded and made this servable + /// available, for it to be unloaded, else calls *done_callback* with an error + /// status. Do not call this multiple times on the same servable. Only one of + /// those will succeed and the rest will fail with an error status. void UnloadServable(const ServableId& id, DoneCallback done_callback); private: - BasicManager(std::unique_ptr load_unload_executor, + friend class AspiredVersionsManager; + friend class test_util::BasicManagerTestAccess; + + BasicManager(Env* env, uint32 num_load_threads, uint32 num_unload_threads, + uint32 max_num_load_retries, + std::function should_retry_model_load, + int64_t load_retry_interval_micros, bool flush_filesystem_caches, std::unique_ptr resource_tracker, EventBus* servable_event_bus, - const LoaderHarness::Options& harness_options); + PreLoadHook pre_load_hook); // Starts managing the servable. // @@ -255,13 +298,13 @@ class BasicManager : public Manager { // status if a corresponding harness was found, else an error status. Status GetHealthyHarness(const ServableId& servable_id, LoaderHarness** harness) - EXCLUSIVE_LOCKS_REQUIRED(mu_); + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Obtains a pointer to every managed loader that is currently holding // resources, i.e. whose state is one of kApprovedForLoading, kLoading, // kReady, kUnloadRequested, kQuiescing, kQuiesced or kUnloading. std::vector GetLoadersCurrentlyUsingResources() const - EXCLUSIVE_LOCKS_REQUIRED(mu_); + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // A load or unload request for a particular servable. Facilitates code // sharing across the two cases. @@ -273,13 +316,13 @@ class BasicManager : public Manager { // A unification of LoadServable() and UnloadServable(). void LoadOrUnloadServable(const LoadOrUnloadRequest& request, - DoneCallback done_callback) LOCKS_EXCLUDED(mu_); + DoneCallback done_callback) TF_LOCKS_EXCLUDED(mu_); // The synchronous logic for handling a load/unload request, including both // the decision and execution phases. This is the method run in the executor. void HandleLoadOrUnloadRequest(const LoadOrUnloadRequest& request, DoneCallback done_callback) - LOCKS_EXCLUDED(mu_); + TF_LOCKS_EXCLUDED(mu_); // The decision phase of whether to approve a load/unload request. Delegates // to one of ApproveLoad() or ApproveUnload() -- see those methods' comments @@ -294,7 +337,7 @@ class BasicManager : public Manager { // precludes concurrent execution of another request that could delete the // harness.) Status ApproveLoadOrUnload(const LoadOrUnloadRequest& request, - LoaderHarness** harness) LOCKS_EXCLUDED(mu_); + LoaderHarness** harness) TF_LOCKS_EXCLUDED(mu_); // The decision phase of whether to approve a load request. // @@ -302,81 +345,94 @@ class BasicManager : public Manager { // other things, that prevents a subsequent load request from proceeding // concurrently. // - // If it fails, removes 'harness' from 'managed_map_' (which causes 'harness' - // to be deleted). - // // Argument 'mu_lock' is a lock held on 'mu_'. It is released temporarily via // 'num_ongoing_load_unload_executions_cv_'. Status ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) - EXCLUSIVE_LOCKS_REQUIRED(mu_); + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // The decision phase of whether to approve an unload request. If it succeeds, // places the servable into state kQuiescing. Among other things, that // prevents a subsequent unload request from proceeding concurrently. - Status ApproveUnload(LoaderHarness* harness) EXCLUSIVE_LOCKS_REQUIRED(mu_); + Status ApproveUnload(LoaderHarness* harness) TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); + + // Attempts to reserve the resources required to load the servable in + // 'harness'. Does not make any state transitions on 'harness' -- merely + // reserves the resources in 'resource_tracker_' (upon success) or returns an + // error. + // + // Argument 'mu_lock' is a lock held on 'mu_'. It is released temporarily via + // 'num_ongoing_load_unload_executions_cv_'. + Status ReserveResources(LoaderHarness* harness, mutex_lock* mu_lock) + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // The execution phase of loading/unloading a servable. Delegates to either // ExecuteLoad() or ExecuteUnload(). // // Upon completion (and regardless of the outcome), signals exit of the // execution phase by decrementing 'num_ongoing_load_unload_executions_'. - // - // If it fails, removes 'harness' from 'managed_map_' (which causes 'harness' - // to be deleted). Status ExecuteLoadOrUnload(const LoadOrUnloadRequest& request, LoaderHarness* harness); // The execution phase of loading a servable. - Status ExecuteLoad(LoaderHarness* harness) LOCKS_EXCLUDED(mu_); + Status ExecuteLoad(LoaderHarness* harness) TF_LOCKS_EXCLUDED(mu_); // The execution phase of loading a unservable. - Status ExecuteUnload(LoaderHarness* harness) LOCKS_EXCLUDED(mu_); + Status ExecuteUnload(LoaderHarness* harness) TF_LOCKS_EXCLUDED(mu_); // Unloads all the managed servables. - void UnloadAllServables() LOCKS_EXCLUDED(mu_); + Status UnloadAllServables() TF_LOCKS_EXCLUDED(mu_); // Updates the serving map by copying servables from the managed map, which // are ready to be served. - void UpdateServingMap() EXCLUSIVE_LOCKS_REQUIRED(mu_); + void UpdateServingMap() TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); + + // Sets the number of load threads. + // + // When either existing or target num_load_threads means single thread, we + // block all new load requests while the old thread pool is destructed, a new + // one is created and then swapped with the old one. Note that destructing + // the old thread pool blocks until all threads are done, so it could block + // for a long time. + // + // When both existing and target num_load_threads are multi-threaded, this + // call still blocks until the old thread pool is destructed, but other loads + // can happen concurrently, potentially increasing the number of running load + // threads, up to the sum of existing and target num_load_threads. + void SetNumLoadThreads(uint32 num_load_threads) + TF_LOCKS_EXCLUDED(load_executor_mu_); + uint32 num_load_threads() const; - struct HashString { - uint64 operator()(const string& str) const { return Hash64(str); } - }; // Keys are the servable names. // Values are the harnesses for each servable version. The values when // fetched, are unordered. using ManagedMap = - std::unordered_multimap, - HashString>; + std::unordered_multimap>; // Fetches the harness with this id from the harness_map_. Returns // harness_map_.end(), if the harness is not found. ManagedMap::iterator FindHarnessInMap(const ServableId& id) - EXCLUSIVE_LOCKS_REQUIRED(mu_); - - // Removes the harness associated with 'id' from 'managed_map_' and deletes - // the harness. - // - // If no matching harness is found, DCHECK-fails and logs an error. - void DeleteHarness(const ServableId& id) EXCLUSIVE_LOCKS_REQUIRED(mu_); + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Publishes the state on the event bus, if an event bus was part of the // options, if not we ignore it. void PublishOnEventBus(const ServableState& state); - const LoaderHarness::Options harness_options_; + LoaderHarness::Options harness_options_; // The event bus to which to publish servable state change events, or nullptr // if no bus has been configured. EventBus* servable_event_bus_; + // Defines how we want to retry when model loading fails. + std::function should_retry_model_load_; + // Used to protect access to 'managed_map_', 'resource_tracker_' and other // core state elements. mutable mutex mu_; // ManagedMap contains all the servables managed by this manager, in different // states. - ManagedMap managed_map_ GUARDED_BY(mu_); + ManagedMap managed_map_ TF_GUARDED_BY(mu_); // ServingMap contains all the servables which are ready to be served, which // is a subset of those in the managed map. @@ -438,23 +494,37 @@ class BasicManager : public Manager { // serially, which guarantees that request i’s decision phase can complete // before considering request i+1's so there’s no starvation. - // The executor used for executing load and unload of servables. - std::unique_ptr load_unload_executor_; + Env* const env_; + + // The number of load threads. Can be changed after instantiation of the + // manager via SetNumLoadThreads(). + std::atomic num_load_threads_; + // Whether to flush filesystem caches (if num_load_threads_ == 1) + const bool flush_filesystem_caches_ = false; + // The executor (and associated mutex) used for executing loads of servables. + mutable mutex load_executor_mu_; + std::unique_ptr load_executor_ TF_GUARDED_BY(load_executor_mu_); + + // The executor used for executing unloads of servables. (Unlike for loads, + // the unload executor is fixed for the lifetime of the manager.) + std::unique_ptr unload_executor_; // Used to serialize the decision phases of the load/unload requests. mutable mutex load_unload_decision_phase_mu_; // A module that keeps track of available, used and reserved servable // resources (e.g. RAM). - std::unique_ptr resource_tracker_ GUARDED_BY(mu_); + std::unique_ptr resource_tracker_ TF_GUARDED_BY(mu_); // The number of load/unload requests currently in their execution phase. - int num_ongoing_load_unload_executions_ GUARDED_BY(mu_) = 0; + int num_ongoing_load_unload_executions_ TF_GUARDED_BY(mu_) = 0; // Used to wake up threads that are waiting for 'num_ongoing_executions' to // decrease. condition_variable num_ongoing_load_unload_executions_cv_; + PreLoadHook pre_load_hook_; + TF_DISALLOW_COPY_AND_ASSIGN(BasicManager); }; @@ -493,13 +563,13 @@ BasicManager::GetManagedServableStateSnapshots( } template -optional> +absl::optional> BasicManager::GetManagedServableStateSnapshot(const ServableId& id) { mutex_lock l(mu_); auto iter = FindHarnessInMap(id); if (iter == managed_map_.end()) { - return nullopt; + return absl::nullopt; } return iter->second->loader_state_snapshot(); } diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 85ab636809a..c6f698439d6 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -16,18 +16,30 @@ limitations under the License. #include "tensorflow_serving/core/basic_manager.h" #include +#include #include +#include +#include +#include +#include +#include #include #include -#include "tensorflow/core/lib/core/blocking_counter.h" -#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "absl/status/status.h" +#include "absl/synchronization/barrier.h" +#include "absl/synchronization/notification.h" +#include "absl/types/optional.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/errors.h" +#include "tensorflow/core/platform/null_file_system.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader.h" +#include "tensorflow_serving/core/test_util/manager_test_util.h" #include "tensorflow_serving/core/test_util/mock_loader.h" #include "tensorflow_serving/util/any_ptr.h" #include "tensorflow_serving/util/event_bus.h" @@ -37,18 +49,19 @@ namespace tensorflow { namespace serving { namespace { +using test_util::FakeLoader; +using test_util::WaitUntilServableManagerStateIsOneOf; using ::testing::_; using ::testing::AnyOf; using ::testing::HasSubstr; using ::testing::InSequence; using ::testing::Invoke; using ::testing::InvokeWithoutArgs; +using ::testing::MockFunction; using ::testing::NiceMock; using ::testing::Return; using ::testing::UnorderedElementsAre; using ::testing::UnorderedElementsAreArray; -using test_util::FakeLoader; -using test_util::WaitUntilServableManagerStateIsOneOf; constexpr char kServableName[] = "kServableName"; constexpr char kServableName2[] = "kServableName2"; @@ -58,22 +71,36 @@ constexpr int kNumVersionsPerServable = 2; constexpr int kNumThreads = 10; +MATCHER_P(EqualsServableState, servable_state, servable_state.DebugString()) { + if (arg == servable_state) { + return true; + } + *result_listener << arg.DebugString(); + return false; +} + // Creates a ServableData around a FakeLoader. ServableData> CreateServable( - const ServableId& id, const Status load_status = Status::OK()) { + const ServableId& id, const Status load_status = OkStatus()) { std::unique_ptr loader(new FakeLoader(id.version, load_status)); return CreateServableData(id, std::move(loader)); } -// We parameterize this test to either run with or without a thread-pool. -class BasicManagerTest : public ::testing::TestWithParam { +// We parameterize this test with the number of load & unload threads. (Zero +// means use an in-line executor instead of a thread pool.) +struct ThreadPoolSizes { + uint64_t num_load_threads; + uint64_t num_unload_threads; +}; +class BasicManagerTest : public ::testing::TestWithParam { protected: BasicManagerTest() - : num_load_unload_threads_(GetParam()), + : thread_pool_sizes_(GetParam()), servable_event_bus_(EventBus::CreateEventBus()), servable_state_monitor_(servable_event_bus_.get()) { BasicManager::Options options; - options.num_load_unload_threads = num_load_unload_threads_; + options.num_load_threads = thread_pool_sizes_.num_load_threads; + options.num_unload_threads = thread_pool_sizes_.num_unload_threads; options.servable_event_bus = servable_event_bus_.get(); options.max_num_load_retries = 10; options.load_retry_interval_micros = 0; @@ -87,7 +114,7 @@ class BasicManagerTest : public ::testing::TestWithParam { for (const char* servable_name : {kServableName, kServableName2}) { for (int i = 1; i <= kNumVersionsPerServable; ++i) { const ServableId id = {servable_name, i}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); loaded_servables.insert(id); @@ -100,18 +127,22 @@ class BasicManagerTest : public ::testing::TestWithParam { } } - int num_load_unload_threads_; + ThreadPoolSizes thread_pool_sizes_; std::shared_ptr> servable_event_bus_; ServableStateMonitor servable_state_monitor_; std::unique_ptr basic_manager_; }; -INSTANTIATE_TEST_CASE_P(WithOrWithoutThreadPool, BasicManagerTest, - ::testing::Values(0 /* WithoutThreadPool */, - kNumThreads)); +INSTANTIATE_TEST_CASE_P( + WithOrWithoutThreadPools, BasicManagerTest, + ::testing::Values( + ThreadPoolSizes{0, 0} /* without load or unload threadpools */, + ThreadPoolSizes{2, 0} /* with just a load threadpool */, + ThreadPoolSizes{0, 2} /* with just an unload threadpool */, + ThreadPoolSizes{4, 4} /* with load and unload threadpools */)); TEST_P(BasicManagerTest, ServableHandleNotFoundMissingLoaderName) { - ServableHandle handle; + ServableHandle handle; const Status status = basic_manager_->GetServableHandle( ServableRequest::Latest(strings::StrCat(kServableName, "missing")), &handle); @@ -121,23 +152,32 @@ TEST_P(BasicManagerTest, ServableHandleNotFoundMissingLoaderName) { TEST_P(BasicManagerTest, ServableHandleNotFoundMissingVersion) { // This version is missing. - const int64 missing_version = 100; - ServableHandle handle; + const int64_t missing_version = 100; + ServableHandle handle; const Status status = basic_manager_->GetServableHandle( ServableRequest::Specific(kServableName, missing_version), &handle); ASSERT_FALSE(status.ok()) << status; EXPECT_EQ(error::NOT_FOUND, status.code()); } +TEST_P(BasicManagerTest, ServableHandleEarliest) { + ASSERT_GT(kNumVersionsPerServable, 1); + ServableHandle handle; + const Status status = basic_manager_->GetServableHandle( + ServableRequest::Earliest(kServableName), &handle); + TF_ASSERT_OK(status); + EXPECT_EQ(1, *handle); +} + TEST_P(BasicManagerTest, ServableHandleLatest) { const ServableId id = {kServableName, kNumVersionsPerServable + 1}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); - ServableHandle handle; + ServableHandle handle; const Status status = basic_manager_->GetServableHandle( ServableRequest::Latest(kServableName), &handle); TF_ASSERT_OK(status); @@ -153,13 +193,13 @@ TEST_P(BasicManagerTest, AlreadyManagedError) { // Tests the case where the latest version of a servable available is 0. TEST_P(BasicManagerTest, ServableHandleLatestVersionIsZero) { const ServableId id = {kServableName3, 1}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); - ServableHandle handle; + ServableHandle handle; const Status status = basic_manager_->GetServableHandle( ServableRequest::Latest(kServableName3), &handle); TF_ASSERT_OK(status); @@ -167,8 +207,65 @@ TEST_P(BasicManagerTest, ServableHandleLatestVersionIsZero) { EXPECT_EQ(id, handle.id()); } +TEST_P(BasicManagerTest, StopManagingUnknownId) { + const ServableId id = {kServableName3, 1}; + EXPECT_FALSE(basic_manager_->StopManagingServable(id).ok()); +} + +TEST_P(BasicManagerTest, StopManagingActiveServable) { + const ServableId id = {kServableName3, 1}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); + basic_manager_->LoadServable( + id, [](const Status& status) { TF_EXPECT_OK(status); }); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + EXPECT_FALSE(basic_manager_->StopManagingServable(id).ok()); +} + +TEST_P(BasicManagerTest, StopManagingDisabledServable) { + const ServableId id = {kServableName3, 1}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); + basic_manager_->LoadServable( + id, [](const Status& status) { TF_EXPECT_OK(status); }); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + basic_manager_->UnloadServable( + id, [](const Status& status) { TF_EXPECT_OK(status); }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + const absl::optional> snapshot = + basic_manager_->GetManagedServableStateSnapshot(id); + EXPECT_EQ(LoaderHarness::State::kDisabled, snapshot->state); + const ServableState expected_state = {id, ServableState::ManagerState::kEnd, + OkStatus()}; + EXPECT_THAT(*servable_state_monitor_.GetState(id), + EqualsServableState(expected_state)); + + TF_ASSERT_OK(basic_manager_->StopManagingServable(id)); + EXPECT_FALSE(basic_manager_->GetManagedServableStateSnapshot(id)); +} + +TEST_P(BasicManagerTest, DontStopManagingOnError) { + const ServableId id = {kServableName, 7}; + const Status error_status = errors::Internal("An error."); + std::unique_ptr loader(new FakeLoader(7, error_status)); + TF_ASSERT_OK(basic_manager_->ManageServable({id, std::move(loader)})); + basic_manager_->LoadServable(id, [error_status](const Status& status) { + EXPECT_EQ(error_status, status); + }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + const absl::optional> snapshot = + basic_manager_->GetManagedServableStateSnapshot(id); + EXPECT_EQ(LoaderHarness::State::kError, snapshot->state); + const ServableState expected_error_state = { + id, ServableState::ManagerState::kEnd, error_status}; + EXPECT_THAT(*servable_state_monitor_.GetState(id), + EqualsServableState(expected_error_state)); +} + TEST_P(BasicManagerTest, ServableHandleSpecificVersion) { - ServableHandle handle; + ServableHandle handle; const ServableId id = {kServableName2, 1}; const Status status = basic_manager_->GetServableHandle(ServableRequest::FromId(id), &handle); @@ -183,25 +280,26 @@ TEST_P(BasicManagerTest, UpdateServingMapServableHandleLatest) { // Using kServableName3 which doesn't have any servables loaded in the // manager, as opposed to kServableName which already has 2 loaded. const ServableId id0 = {kServableName3, 0}; - // Servable is int64 with value 0. - basic_manager_->ManageServable(CreateServable(id0)); + // Servable is int64_t with value 0. + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id0))); basic_manager_->LoadServable( id0, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id0, {ServableState::ManagerState::kAvailable}); test_util::MockLoader* notify_to_unload = new NiceMock; - // Don't make it const otherwise servable types will mismatch: const int64 vs - // int64. - int64 servable = 1; + // Don't make it const otherwise servable types will mismatch: const int64_t + // vs int64. + int64_t servable = 1; ON_CALL(*notify_to_unload, servable()) .WillByDefault(Return(AnyPtr(&servable))); ON_CALL(*notify_to_unload, EstimateResources(_)) - .WillByDefault(Return(Status::OK())); - ON_CALL(*notify_to_unload, Load(_)).WillByDefault(Return(Status::OK())); + .WillByDefault(Return(OkStatus())); + ON_CALL(*notify_to_unload, LoadWithMetadata(Loader::Metadata{id0})) + .WillByDefault(Return(OkStatus())); const ServableId id1 = {kServableName3, 1}; - basic_manager_->ManageServable( - {id1, std::unique_ptr(notify_to_unload)}); + TF_ASSERT_OK(basic_manager_->ManageServable( + {id1, std::unique_ptr(notify_to_unload)})); basic_manager_->LoadServable( id1, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -210,7 +308,7 @@ TEST_P(BasicManagerTest, UpdateServingMapServableHandleLatest) { // We have loaded both versions 0 and 1 of kServableName3, so the latest // handle should be that of v1. { - ServableHandle handle; + ServableHandle handle; const Status status = basic_manager_->GetServableHandle( ServableRequest::Latest(kServableName3), &handle); TF_ASSERT_OK(status); @@ -220,13 +318,13 @@ TEST_P(BasicManagerTest, UpdateServingMapServableHandleLatest) { // We will now try to unload v1, but we only allow it to move out from kReady // state, and not complete the unload. Also, after it moves out from kReady, // the serving map is also updated, so v0 would be the latest. - Notification unload_started; - Notification finish_unload; + absl::Notification unload_started; + absl::Notification finish_unload; EXPECT_CALL(*notify_to_unload, Unload()).WillOnce(Invoke([&]() { unload_started.Notify(); finish_unload.WaitForNotification(); })); - Notification unload_finished; + absl::Notification unload_finished; std::unique_ptr unload_last_servable( Env::Default()->StartThread({}, "UnloadLastServable", [&]() { basic_manager_->UnloadServable(id1, [&](const Status& status) { @@ -238,7 +336,7 @@ TEST_P(BasicManagerTest, UpdateServingMapServableHandleLatest) { // Servable map should just have {kServableName3, 0} at this point. { - ServableHandle handle; + ServableHandle handle; const Status status = basic_manager_->GetServableHandle( ServableRequest::Latest(kServableName3), &handle); TF_EXPECT_OK(status); @@ -264,7 +362,8 @@ TEST_P(BasicManagerTest, ListAvailableServableIds) { const ServableId id = {kServableName, 7}; std::unique_ptr loader( new FakeLoader(7, errors::Internal("An error."))); - basic_manager_->ManageServable(CreateServableData(id, std::move(loader))); + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(id, std::move(loader)))); basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_EQ(errors::Internal("An error."), status); }); @@ -290,8 +389,8 @@ TEST_P(BasicManagerTest, ListAvailableServableIds) { TEST_P(BasicManagerTest, GetAvailableServableHandles) { // Scoped to destruct handles at the end of it. { - const std::map> handles_before = - basic_manager_->GetAvailableServableHandles(); + const std::map> handles_before = + basic_manager_->GetAvailableServableHandles(); ASSERT_EQ(kNumVersionsPerServable * 2, handles_before.size()); const std::vector expected_ids_before = {{kServableName, 1}, @@ -310,7 +409,8 @@ TEST_P(BasicManagerTest, GetAvailableServableHandles) { const ServableId id = {kServableName, 7}; std::unique_ptr loader( new FakeLoader(7, errors::Internal("An error."))); - basic_manager_->ManageServable(CreateServableData(id, std::move(loader))); + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(id, std::move(loader)))); basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_EQ(errors::Internal("An error."), status); }); @@ -328,8 +428,8 @@ TEST_P(BasicManagerTest, GetAvailableServableHandles) { {ServableState::ManagerState::kEnd}); { - const std::map> handles_after = - basic_manager_->GetAvailableServableHandles(); + const std::map> handles_after = + basic_manager_->GetAvailableServableHandles(); ASSERT_EQ(kNumVersionsPerServable, handles_after.size()); const std::vector expected_ids_after = {{kServableName2, 1}, @@ -366,7 +466,7 @@ TEST_P(BasicManagerTest, GetManagedServableStateSnapshot) { // Check servable state snapshot corresponding to a servable-id that is in // ready state. const ServableId id_ready = {kServableName, 1}; - const optional> actual_ready_snapshot = + const absl::optional> actual_ready_snapshot = basic_manager_->GetManagedServableStateSnapshot(id_ready); EXPECT_TRUE(actual_ready_snapshot); const ServableStateSnapshot<> expected_ready_snapshot = { @@ -380,10 +480,10 @@ TEST_P(BasicManagerTest, GetManagedServableStateSnapshot) { } TEST_P(BasicManagerTest, GetManagedServableStateSnapshotsWithAdditionalState) { - basic_manager_->ManageServableWithAdditionalState( - CreateServable({kServableName3, 0}), std::unique_ptr(new int(0))); - basic_manager_->ManageServableWithAdditionalState( - CreateServable({kServableName3, 1}), std::unique_ptr(new int(1))); + TF_CHECK_OK(basic_manager_->ManageServableWithAdditionalState( + CreateServable({kServableName3, 0}), std::unique_ptr(new int(0)))); + TF_CHECK_OK(basic_manager_->ManageServableWithAdditionalState( + CreateServable({kServableName3, 1}), std::unique_ptr(new int(1)))); const std::vector> expected = { {{kServableName3, 0}, LoaderHarness::State::kNew, {0}}, {{kServableName3, 1}, LoaderHarness::State::kNew, {1}}}; @@ -394,18 +494,25 @@ TEST_P(BasicManagerTest, GetManagedServableStateSnapshotsWithAdditionalState) { TEST_P(BasicManagerTest, MultipleManageCallsUsesFirstServable) { const ServableId id = {kServableName, 1}; - std::unique_ptr first_loader( + + // Servable 'id' is already managed, so further ManageServable() calls should + // fail (and not affect the status of the already-managed servable). + std::unique_ptr first_ignored_loader( new FakeLoader(1, errors::Internal("An error."))); + EXPECT_FALSE(basic_manager_ + ->ManageServable( + CreateServableData(id, std::move(first_ignored_loader))) + .ok()); - basic_manager_->ManageServable( - CreateServableData(id, std::move(first_loader))); - // Different servable returned. - std::unique_ptr second_loader( + // Same thing, but this time using a loader for a different servable version. + std::unique_ptr second_ignored_loader( new FakeLoader(2, errors::Internal("An error."))); - basic_manager_->ManageServable( - CreateServableData(id, std::move(second_loader))); + EXPECT_FALSE(basic_manager_ + ->ManageServable( + CreateServableData(id, std::move(second_ignored_loader))) + .ok()); - ServableHandle handle; + ServableHandle handle; TF_ASSERT_OK(basic_manager_->GetServableHandle( ServableRequest::Specific(kServableName, 1), &handle)); EXPECT_EQ(1, *handle); @@ -415,10 +522,10 @@ TEST_P(BasicManagerTest, MultipleManageCallsUsesFirstServable) { // erroneous servable. TEST_P(BasicManagerTest, ErroneousServable) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable( - ServableData>(id, errors::Unknown("error"))); + TF_ASSERT_OK(basic_manager_->ManageServable( + ServableData>(id, errors::Unknown("error")))); - ServableHandle handle; + ServableHandle handle; Status status = basic_manager_->GetServableHandle( ServableRequest::Specific(kServableName, 3), &handle); EXPECT_FALSE(status.ok()) << status; @@ -434,21 +541,21 @@ TEST_P(BasicManagerTest, ErroneousServable) { // thread, and not a request thread. TEST_P(BasicManagerTest, DestructOnNonServingThread) { const ServableId id = {kServableName, 7}; - basic_manager_->ManageServable( - CreateServableData(id, std::unique_ptr(new FakeLoader(7)))); + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(id, std::unique_ptr(new FakeLoader(7))))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); - std::unique_ptr> latest_handle( - new ServableHandle()); + std::unique_ptr> latest_handle( + new ServableHandle()); const Status status = basic_manager_->GetServableHandle( ServableRequest::Latest(kServableName), latest_handle.get()); TF_ASSERT_OK(status); EXPECT_EQ(7, **latest_handle); - Notification done_unload_servable; + absl::Notification done_unload_servable; std::unique_ptr unload_servable( Env::Default()->StartThread({}, "UnloadServable", [&]() { // Unload the servable. @@ -456,9 +563,10 @@ TEST_P(BasicManagerTest, DestructOnNonServingThread) { id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); + TF_ASSERT_OK(basic_manager_->StopManagingServable(id)); // The servable has been deleted in this thread if there is no // thread-pool for load/unload. - if (num_load_unload_threads_ == 0) { + if (thread_pool_sizes_.num_load_threads == 0) { EXPECT_TRUE(FakeLoader::was_deleted_in_this_thread()); } done_unload_servable.Notify(); @@ -474,8 +582,8 @@ TEST_P(BasicManagerTest, DestructOnNonServingThread) { TEST_P(BasicManagerTest, AdditionalState) { const ServableId id = {kServableName, 3}; std::unique_ptr state(new int(1)); - basic_manager_->ManageServableWithAdditionalState(CreateServable(id), - std::move(state)); + TF_CHECK_OK(basic_manager_->ManageServableWithAdditionalState( + CreateServable(id), std::move(state))); EXPECT_EQ(1, *basic_manager_->GetAdditionalServableState(id)); EXPECT_EQ(nullptr, basic_manager_->GetAdditionalServableState(id)); @@ -483,7 +591,7 @@ TEST_P(BasicManagerTest, AdditionalState) { TEST_P(BasicManagerTest, NoAdditionalState) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); // Will return nullptr when there is no metadata set. EXPECT_EQ(nullptr, basic_manager_->GetAdditionalServableState(id)); @@ -495,13 +603,13 @@ TEST_P(BasicManagerTest, OutOfOrderLoadServable) { basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::NOT_FOUND, status.code()); - EXPECT_THAT(status.error_message(), HasSubstr("is not being managed")); + EXPECT_THAT(status.message(), HasSubstr("is not being managed")); }); } TEST_P(BasicManagerTest, MultipleLoadServables) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -509,14 +617,13 @@ TEST_P(BasicManagerTest, MultipleLoadServables) { basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); - EXPECT_THAT(status.error_message(), - HasSubstr("cannot be transitioned to load-requested")); + EXPECT_THAT(status.message(), HasSubstr("Duplicate load request")); }); } TEST_P(BasicManagerTest, MultipleUnloadServables) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -527,8 +634,9 @@ TEST_P(BasicManagerTest, MultipleUnloadServables) { {ServableState::ManagerState::kEnd}); basic_manager_->UnloadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); - EXPECT_EQ(error::NOT_FOUND, status.code()); - EXPECT_THAT(status.error_message(), HasSubstr("is not being managed")); + EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); + EXPECT_THAT(status.message(), + HasSubstr("unload already requested/ongoing")); }); } @@ -537,33 +645,24 @@ TEST_P(BasicManagerTest, UnloadWithoutManage) { basic_manager_->UnloadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::NOT_FOUND, status.code()); - EXPECT_THAT(status.error_message(), HasSubstr("is not being managed")); + EXPECT_THAT(status.message(), HasSubstr("is not being managed")); }); } TEST_P(BasicManagerTest, UnloadWithoutLoad) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->UnloadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); - EXPECT_THAT(status.error_message(), - HasSubstr("cannot be transitioned to unload-requested")); + EXPECT_THAT(status.message(), HasSubstr("Servable not loaded")); }); } -MATCHER_P(EqualsServableState, servable_state, servable_state.DebugString()) { - if (arg == servable_state) { - return true; - } - *result_listener << arg.DebugString(); - return false; -} - TEST_P(BasicManagerTest, EventBusErroneousVersion) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable( - ServableData>(id, errors::Unknown("error"))); + TF_ASSERT_OK(basic_manager_->ManageServable( + ServableData>(id, errors::Unknown("error")))); const ServableState expected_published_state = { id, ServableState::ManagerState::kEnd, errors::Unknown("error")}; @@ -575,10 +674,10 @@ TEST_P(BasicManagerTest, EventBusErrorOnLoad) { const ServableId id = {kServableName, 7}; std::unique_ptr loader( new FakeLoader(7, errors::Internal("Error on load."))); - basic_manager_->ManageServable({id, std::move(loader)}); + TF_ASSERT_OK(basic_manager_->ManageServable({id, std::move(loader)})); const ServableState start_state = {id, ServableState::ManagerState::kStart, - Status::OK()}; + OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(start_state)); @@ -595,20 +694,21 @@ TEST_P(BasicManagerTest, EventBusErrorOnLoad) { TEST_P(BasicManagerTest, EventBusServableLifecycle) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_ASSERT_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); const ServableState start_state = {id, ServableState::ManagerState::kStart, - Status::OK()}; + OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(start_state)); - Notification load_called; - Notification load_continue; - EXPECT_CALL(*loader, Load(_)) + absl::Notification load_called; + absl::Notification load_continue; + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) .WillOnce(InvokeWithoutArgs([&]() { load_called.Notify(); load_continue.WaitForNotification(); - return Status::OK(); + return OkStatus(); })); std::unique_ptr load_thread( @@ -619,7 +719,7 @@ TEST_P(BasicManagerTest, EventBusServableLifecycle) { load_called.WaitForNotification(); const ServableState loading_state = { - id, ServableState::ManagerState::kLoading, Status::OK()}; + id, ServableState::ManagerState::kLoading, OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(loading_state)); @@ -628,17 +728,16 @@ TEST_P(BasicManagerTest, EventBusServableLifecycle) { servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); const ServableState available_state = { - id, ServableState::ManagerState::kAvailable, Status::OK()}; + id, ServableState::ManagerState::kAvailable, OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(available_state)); - Notification unload_called; - Notification unload_continue; - EXPECT_CALL(*loader, Unload()) - .WillOnce(Invoke([&]() { - unload_called.Notify(); - unload_continue.WaitForNotification(); - })); + absl::Notification unload_called; + absl::Notification unload_continue; + EXPECT_CALL(*loader, Unload()).WillOnce(Invoke([&]() { + unload_called.Notify(); + unload_continue.WaitForNotification(); + })); // Scoped to ensure UnloadServable() is scheduled. std::unique_ptr unload_thread( Env::Default()->StartThread(ThreadOptions(), "UnloadThread", [&]() { @@ -648,7 +747,7 @@ TEST_P(BasicManagerTest, EventBusServableLifecycle) { unload_called.WaitForNotification(); const ServableState unloading_state = { - id, ServableState::ManagerState::kUnloading, Status::OK()}; + id, ServableState::ManagerState::kUnloading, OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(unloading_state)); @@ -657,7 +756,7 @@ TEST_P(BasicManagerTest, EventBusServableLifecycle) { {ServableState::ManagerState::kEnd}); const ServableState end_state = {id, ServableState::ManagerState::kEnd, - Status::OK()}; + OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(end_state)); } @@ -666,7 +765,7 @@ TEST_P(BasicManagerTest, EventBusServableLifecycle) { TEST_P(BasicManagerTest, NoEventBus) { BasicManager::Options options; // Single threaded execution. - options.num_load_unload_threads = 0; + options.num_load_threads = 0; // No event bus. options.servable_event_bus = nullptr; std::unique_ptr manager; @@ -674,7 +773,7 @@ TEST_P(BasicManagerTest, NoEventBus) { const ServableId id = {kServableName, 7}; std::unique_ptr loader(new FakeLoader(7)); - manager->ManageServable({id, std::move(loader)}); + TF_ASSERT_OK(manager->ManageServable({id, std::move(loader)})); manager->LoadServable(id, [](const Status& status) { TF_ASSERT_OK(status); }); manager->UnloadServable(id, [](const Status& status) { TF_ASSERT_OK(status); }); @@ -689,8 +788,8 @@ TEST_P(BasicManagerTest, LoadsThenUnloads) { for (int i = 0; i < 20; ++i) { const ServableId id = {kServableName3, i}; servables.insert(id); - load_executor.Schedule([this, id, &servables]() { - basic_manager_->ManageServable(CreateServable(id)); + load_executor.Schedule([this, id]() { + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); }); @@ -724,8 +823,8 @@ TEST_P(BasicManagerTest, InterleavedLoadsAndUnloads) { for (int i = 0; i < 20; ++i) { executor.Schedule([this, i]() { const ServableId id = {kServableName3, i}; - basic_manager_->ManageServable(CreateServable(id)); - Notification load_done; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); + absl::Notification load_done; basic_manager_->LoadServable(id, [&load_done](const Status& status) { TF_ASSERT_OK(status); load_done.Notify(); @@ -737,6 +836,200 @@ TEST_P(BasicManagerTest, InterleavedLoadsAndUnloads) { } } +class SetNumLoadThreadsBasicManagerTest : public ::testing::Test { + protected: + SetNumLoadThreadsBasicManagerTest() { + BasicManager::Options options; + options.num_load_threads = 0; + options.max_num_load_retries = 10; + options.load_retry_interval_micros = 0; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager_)); + } + + std::unique_ptr basic_manager_; +}; + +TEST_F(SetNumLoadThreadsBasicManagerTest, ThreadPoolSwapped) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + manager_test_access.SetNumLoadThreads(2); + EXPECT_EQ(2, manager_test_access.num_load_threads()); + + const auto load_done_fn = [&](const Status& status) { + TF_ASSERT_OK(status); + // Tests whether the threadpools are actually swapped in + // SetNumLoadThreads(). + static thread_local int per_thread_load_ctr = 0; + ++per_thread_load_ctr; + EXPECT_EQ(1, per_thread_load_ctr); + }; + + const ServableId id0 = {kServableName3, 0}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id0))); + basic_manager_->LoadServable(id0, load_done_fn); + + manager_test_access.SetNumLoadThreads(0); + EXPECT_EQ(0, manager_test_access.num_load_threads()); + + const ServableId id1 = {kServableName3, 1}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id1))); + basic_manager_->LoadServable(id1, load_done_fn); + + // Force the manager to finish before deleting the notifications. + basic_manager_.reset(); +} + +TEST_F(SetNumLoadThreadsBasicManagerTest, ThreadPoolsNotAliveSimultaneously) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + manager_test_access.SetNumLoadThreads(1); + EXPECT_EQ(1, manager_test_access.num_load_threads()); + + std::set data_race_set; + const auto data_race_fn = [&](const Status& status) { + // This line will cause a data race if both the loads happen simultaneously + // on different threads. This will be caught by the ThreadSanitizer, causing + // the test to fail. + data_race_set.insert("string"); + }; + + const ServableId id0 = {kServableName3, 0}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id0))); + absl::Notification notify_for_setting; + absl::Notification continue_load; + basic_manager_->LoadServable(id0, [&](const Status& status) { + notify_for_setting.Notify(); + continue_load.WaitForNotification(); + data_race_fn(status); + }); + + { + ThreadPoolExecutor executor(Env::Default(), "SetNumLoadThreads", + kNumThreads); + executor.Schedule([&]() { + notify_for_setting.WaitForNotification(); + manager_test_access.SetNumLoadThreads(1); + EXPECT_EQ(1, manager_test_access.num_load_threads()); + }); + + executor.Schedule([&]() { + const ServableId id1 = {kServableName3, 1}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id1))); + continue_load.Notify(); + basic_manager_->LoadServable( + id1, [&](const Status& status) { data_race_fn(status); }); + }); + } + + // Force the manager to finish before deleting the notifications. + basic_manager_.reset(); +} + +// Tests whether the fast-load scenario works. In the fast-load scenario we try +// to load a bunch of servables as fast as possible using a lot of threads. +TEST_F(SetNumLoadThreadsBasicManagerTest, FastLoad) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + const uint32 prev_num_load_threads = manager_test_access.num_load_threads(); + manager_test_access.SetNumLoadThreads(32); + EXPECT_EQ(32, manager_test_access.num_load_threads()); + + { + ThreadPoolExecutor executor(Env::Default(), "FirstThreadPoolLoads", + kNumThreads); + for (int i = 0; i < 20; ++i) { + executor.Schedule([this, i]() { + const ServableId id = {kServableName3, i}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); + basic_manager_->LoadServable( + id, [](const Status& status) { TF_ASSERT_OK(status); }); + // We don't wait for load to be done here because we want to test that + // SetNumLoadThreads() waits properly till all queued loads are + // finished. If a queued load hasn't been finished the corresponding + // UnloadServable() will fail. + }); + } + } + + manager_test_access.SetNumLoadThreads(prev_num_load_threads); + EXPECT_EQ(prev_num_load_threads, manager_test_access.num_load_threads()); + + { + ThreadPoolExecutor executor(Env::Default(), "Unloads", kNumThreads); + for (int i = 0; i < 20; ++i) { + executor.Schedule([this, i]() { + const ServableId id = {kServableName3, i}; + basic_manager_->UnloadServable( + id, [](const Status& status) { TF_ASSERT_OK(status); }); + }); + } + } +} + +// This filesystem detects a call to FlushCaches(), which is triggered by the +// BasicManager's call to Env::Default()->FlushFileSystemCaches() after loading +// a servable. +class FlushDetectingFileSystem : public NullFileSystem { + public: + void FlushCaches() override { flushed = true; } + static std::atomic flushed; +}; + +std::atomic FlushDetectingFileSystem::flushed; + +REGISTER_FILE_SYSTEM("flush", FlushDetectingFileSystem); + +// This test loads servables with BasicManager::Options::flush_filesystem_caches +// true or false, and verifies that filesystem caches were flushed (or not +// flushed) as expected. +class FlushFileSystemCachesTest : public ::testing::TestWithParam { + protected: + FlushFileSystemCachesTest() : flush_filesystem_caches_(GetParam()) { + BasicManager::Options options; + options.flush_filesystem_caches = flush_filesystem_caches_; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager_)); + } + + std::unique_ptr basic_manager_; + bool flush_filesystem_caches_; +}; + +TEST_P(FlushFileSystemCachesTest, Load) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + // The number of load threads is initially zero, so filesystems should be + // flushed if flush_filesystem_caches_ is true. + FlushDetectingFileSystem::flushed.store(false); + const ServableId id0 = {kServableName3, 0}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id0))); + basic_manager_->LoadServable(id0, [&](const Status& status) { + TF_ASSERT_OK(status); + EXPECT_EQ(flush_filesystem_caches_, + FlushDetectingFileSystem::flushed.load()); + }); + // Load another servable with two load threads. Filesystem caches should not + // be flushed. + manager_test_access.SetNumLoadThreads(2); + FlushDetectingFileSystem::flushed.store(false); + const ServableId id1 = {kServableName3, 1}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id1))); + basic_manager_->LoadServable(id1, [&](const Status& status) { + TF_ASSERT_OK(status); + EXPECT_FALSE(FlushDetectingFileSystem::flushed.load()); + }); + // Now move to a single load thread and load a third servable. Filesystem + // caches should once again be flushed if flush_filesystem_caches_ is true. + manager_test_access.SetNumLoadThreads(1); + FlushDetectingFileSystem::flushed.store(false); + const ServableId id2 = {kServableName3, 2}; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id2))); + basic_manager_->LoadServable(id2, [&](const Status& status) { + TF_ASSERT_OK(status); + EXPECT_EQ(flush_filesystem_caches_, + FlushDetectingFileSystem::flushed.load()); + }); + basic_manager_.reset(); +} + +INSTANTIATE_TEST_CASE_P(WithOrWithoutFlush, FlushFileSystemCachesTest, + ::testing::Bool()); + TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { const ServableId id = {kServableName3, 0}; mutex status_mu; @@ -746,7 +1039,8 @@ TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { kNumThreads); for (int i = 0; i < 4; ++i) { load_executor.Schedule([this, id, i, &statuses, &status_mu]() { - basic_manager_->ManageServable(CreateServable(id)); + // (Suppress a possible "this servable is already managed" error.) + basic_manager_->ManageServable(CreateServable(id)).IgnoreError(); basic_manager_->LoadServable( id, [i, &statuses, &status_mu](const Status& status) { mutex_lock l(status_mu); @@ -765,8 +1059,7 @@ TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { mutex_lock l(status_mu); if (!statuses[i].ok()) { EXPECT_EQ(error::FAILED_PRECONDITION, statuses[i].code()); - EXPECT_THAT(statuses[i].error_message(), - HasSubstr("cannot be transitioned to load-requested")); + EXPECT_THAT(statuses[i].message(), HasSubstr("Duplicate load request")); } else { ++num_status_ok; } @@ -776,7 +1069,7 @@ TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { TEST_P(BasicManagerTest, ConcurrentUnloadsOnlyOneSucceeds) { const ServableId id = {kServableName3, 0}; - basic_manager_->ManageServable(CreateServable(id)); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); // At this point, all loads may not have completed, so we wait for them. @@ -811,11 +1104,10 @@ TEST_P(BasicManagerTest, ConcurrentUnloadsOnlyOneSucceeds) { ASSERT_THAT(statuses[i].code(), AnyOf(error::NOT_FOUND, error::FAILED_PRECONDITION)); if (statuses[i].code() == error::NOT_FOUND) { - EXPECT_THAT(statuses[i].error_message(), - HasSubstr("not being managed")); + EXPECT_THAT(statuses[i].message(), HasSubstr("not being managed")); } else { - EXPECT_THAT(statuses[i].error_message(), - HasSubstr("cannot be transitioned to unload-requested")); + EXPECT_THAT(statuses[i].message(), + HasSubstr("unload already requested/ongoing")); } } else { ++num_status_ok; @@ -827,10 +1119,11 @@ TEST_P(BasicManagerTest, ConcurrentUnloadsOnlyOneSucceeds) { TEST_P(BasicManagerTest, RetryOnLoadErrorFinallySucceeds) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); - EXPECT_CALL(*loader, Load(_)) + TF_ASSERT_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) .WillOnce(Return(errors::Internal("Load error."))) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(OkStatus())); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); } @@ -838,8 +1131,9 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorFinallySucceeds) { TEST_P(BasicManagerTest, RetryOnLoadErrorFinallyFails) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); - EXPECT_CALL(*loader, Load(_)) + TF_ASSERT_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) .WillRepeatedly(Return(errors::Internal("Load error."))); basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_EQ(errors::Internal("Load error."), status); @@ -850,17 +1144,18 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorFinallyFails) { TEST_P(BasicManagerTest, RetryOnLoadErrorCancelledLoad) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_ASSERT_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); - Notification load_called; - Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + absl::Notification load_called; + absl::Notification load_should_return; + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { load_called.Notify(); load_should_return.WaitForNotification(); return errors::Internal("Load error."); })) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(OkStatus())); std::unique_ptr load_thread( Env::Default()->StartThread(ThreadOptions(), "LoadServable", [&]() { basic_manager_->LoadServable(id, [](const Status& status) { @@ -877,17 +1172,18 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorCancelledLoad) { TEST_P(BasicManagerTest, LoadAfterCancelledLoad) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_ASSERT_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); - Notification load_called; - Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + absl::Notification load_called; + absl::Notification load_should_return; + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { load_called.Notify(); load_should_return.WaitForNotification(); return errors::Internal("Load error."); })) - .WillRepeatedly(Return(Status::OK())); + .WillRepeatedly(Return(OkStatus())); std::unique_ptr load_thread( Env::Default()->StartThread(ThreadOptions(), "LoadServable", [&]() { @@ -905,6 +1201,35 @@ TEST_P(BasicManagerTest, LoadAfterCancelledLoad) { id, [](const Status& status) { EXPECT_FALSE(status.ok()) << status; }); } +TEST(NonParameterizedBasicManagerTest, PreLoadHook) { + BasicManager::Options options; + // Single threaded execution. + options.num_load_threads = 0; + // No event bus. + options.servable_event_bus = nullptr; + MockFunction mock_pre_load_hook; + options.pre_load_hook = mock_pre_load_hook.AsStdFunction(); + std::unique_ptr manager; + TF_ASSERT_OK(BasicManager::Create(std::move(options), &manager)); + + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock(); + TF_ASSERT_OK(manager->ManageServable({id, std::unique_ptr(loader)})); + + bool pre_load_hook_called = false; + EXPECT_CALL(mock_pre_load_hook, Call(id)).WillOnce(InvokeWithoutArgs([&]() { + pre_load_hook_called = true; + })); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) + .WillOnce(InvokeWithoutArgs([&]() { + EXPECT_TRUE(pre_load_hook_called); + return OkStatus(); + })); + manager->LoadServable(id, [](const Status& status) { TF_ASSERT_OK(status); }); + manager->UnloadServable(id, + [](const Status& status) { TF_ASSERT_OK(status); }); +} + // Creates a ResourceAllocation proto with 'quantity' units of RAM. ResourceAllocation CreateResourceQuantity(const int quantity) { ResourceAllocation allocation; @@ -935,8 +1260,9 @@ class ResourceConstrainedBasicManagerTest : public ::testing::Test { // Seed the manager with ten resource units. options.resource_tracker = CreateSimpleResourceTracker(10); options.servable_event_bus = servable_event_bus_.get(); - // Allow up to two load/unload requests to be processed concurrently. - options.num_load_unload_threads = 2; + // Allow up to two loads and two unloads to be processed concurrently. + options.num_load_threads = 2; + options.num_unload_threads = 2; // We don't want retries. options.max_num_load_retries = 0; TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager_)); @@ -951,18 +1277,17 @@ class ResourceConstrainedBasicManagerTest : public ::testing::Test { // resource units, i.e. half of the total system resources. class BarrierLoader : public Loader { public: - explicit BarrierLoader(BlockingCounter* counter) : counter_(counter) {} + explicit BarrierLoader(absl::Barrier* barrier) : barrier_(barrier) {} ~BarrierLoader() override = default; Status EstimateResources(ResourceAllocation* estimate) const override { *estimate = CreateResourceQuantity(5); - return Status::OK(); + return OkStatus(); } - Status Load(const ResourceAllocation& available_resources) override { - counter_->DecrementCount(); - counter_->Wait(); - return Status::OK(); + Status Load() override { + barrier_->Block(); + return OkStatus(); } void Unload() override {} @@ -970,7 +1295,7 @@ class BarrierLoader : public Loader { AnyPtr servable() override { return AnyPtr(); } private: - BlockingCounter* const counter_; + absl::Barrier* const barrier_; TF_DISALLOW_COPY_AND_ASSIGN(BarrierLoader); }; @@ -980,11 +1305,12 @@ TEST_F(ResourceConstrainedBasicManagerTest, ConcurrentLoads) { // concurrently (i.e. the manager should not serialize them needlessly). // BarrierLoader verifies that the Load() calls occur concurrently. int kNumLoaders = 2; - BlockingCounter barrier(kNumLoaders); + absl::Barrier barrier(kNumLoaders); for (int i = 0; i < kNumLoaders; ++i) { std::unique_ptr loader(new BarrierLoader(&barrier)); const ServableId id = {"barrier", i}; - basic_manager_->ManageServable(CreateServableData(id, std::move(loader))); + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(id, std::move(loader)))); basic_manager_->LoadServable( id, [](const Status& status) { TF_EXPECT_OK(status); }); } @@ -1000,12 +1326,13 @@ TEST_F(ResourceConstrainedBasicManagerTest, InsufficientResources) { ON_CALL(*hogging_loader, EstimateResources(_)) .WillByDefault(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(10 /* = total system resources */); - return Status::OK(); + return OkStatus(); })); - EXPECT_CALL(*hogging_loader, Load(_)).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable( - CreateServableData(hogging_id, std::unique_ptr(hogging_loader))); - Notification hogging_loaded; + EXPECT_CALL(*hogging_loader, LoadWithMetadata(Loader::Metadata{hogging_id})) + .WillOnce(Return(OkStatus())); + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(hogging_id, std::unique_ptr(hogging_loader)))); + absl::Notification hogging_loaded; basic_manager_->LoadServable(hogging_id, [&hogging_loaded](const Status& status) { TF_EXPECT_OK(status); @@ -1019,11 +1346,11 @@ TEST_F(ResourceConstrainedBasicManagerTest, InsufficientResources) { ON_CALL(*rejected_loader, EstimateResources(_)) .WillByDefault(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(1); - return Status::OK(); + return OkStatus(); })); - basic_manager_->ManageServable(CreateServableData( - rejected_id, std::unique_ptr(rejected_loader))); - Notification rejection_received; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServableData( + rejected_id, std::unique_ptr(rejected_loader)))); + absl::Notification rejection_received; Status rejected_status; basic_manager_->LoadServable( rejected_id, @@ -1038,6 +1365,11 @@ TEST_F(ResourceConstrainedBasicManagerTest, InsufficientResources) { rejected_id, ServableState::ManagerState::kEnd, rejected_status}; EXPECT_THAT(*servable_state_monitor_.GetState(rejected_id), EqualsServableState(expected_error_state)); + + // Make sure we're still managing the rejected servable. + const absl::optional> snapshot = + basic_manager_->GetManagedServableStateSnapshot(rejected_id); + EXPECT_EQ(LoaderHarness::State::kError, snapshot->state); } TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { @@ -1047,13 +1379,13 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { ON_CALL(*failing_loader, EstimateResources(_)) .WillByDefault(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(10); - return Status::OK(); + return OkStatus(); })); - EXPECT_CALL(*failing_loader, Load(_)) + EXPECT_CALL(*failing_loader, LoadWithMetadata(Loader::Metadata{failing_id})) .WillOnce(Return(errors::Unknown("Load failure"))); - basic_manager_->ManageServable( - CreateServableData(failing_id, std::unique_ptr(failing_loader))); - Notification failing_failed; + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(failing_id, std::unique_ptr(failing_loader)))); + absl::Notification failing_failed; basic_manager_->LoadServable(failing_id, [&failing_failed](const Status& status) { EXPECT_FALSE(status.ok()); @@ -1070,11 +1402,13 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { ON_CALL(*succeeding_loader, EstimateResources(_)) .WillByDefault(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(10); - return Status::OK(); + return OkStatus(); })); - EXPECT_CALL(*succeeding_loader, Load(_)).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + EXPECT_CALL(*succeeding_loader, + LoadWithMetadata(Loader::Metadata{succeeding_id})) + .WillOnce(Return(OkStatus())); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); basic_manager_->LoadServable( succeeding_id, [](const Status& status) { TF_EXPECT_OK(status); }); } @@ -1090,20 +1424,22 @@ TEST_F(ResourceConstrainedBasicManagerTest, EXPECT_CALL(*overestimating_loader, EstimateResources(_)) .WillOnce(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(10); - return Status::OK(); + return OkStatus(); })) .RetiresOnSaturation(); - EXPECT_CALL(*overestimating_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*overestimating_loader, + LoadWithMetadata(Loader::Metadata{overestimating_id})) + .WillOnce(Return(OkStatus())); EXPECT_CALL(*overestimating_loader, EstimateResources(_)) .WillOnce(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(5 /* lower estimate after load */); - return Status::OK(); + return OkStatus(); })) .RetiresOnSaturation(); } - basic_manager_->ManageServable(CreateServableData( - overestimating_id, std::unique_ptr(overestimating_loader))); - Notification overestimating_loaded; + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServableData( + overestimating_id, std::unique_ptr(overestimating_loader)))); + absl::Notification overestimating_loaded; basic_manager_->LoadServable(overestimating_id, [&overestimating_loaded](const Status& status) { TF_EXPECT_OK(status); @@ -1120,11 +1456,13 @@ TEST_F(ResourceConstrainedBasicManagerTest, ON_CALL(*succeeding_loader, EstimateResources(_)) .WillByDefault(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(5); - return Status::OK(); + return OkStatus(); })); - EXPECT_CALL(*succeeding_loader, Load(_)).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + EXPECT_CALL(*succeeding_loader, + LoadWithMetadata(Loader::Metadata{succeeding_id})) + .WillOnce(Return(OkStatus())); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); basic_manager_->LoadServable( succeeding_id, [](const Status& status) { TF_EXPECT_OK(status); }); } @@ -1135,20 +1473,22 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedAfterUnload) { ON_CALL(*unloading_loader, EstimateResources(_)) .WillByDefault(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(10); - return Status::OK(); + return OkStatus(); })); - Notification load_done; - EXPECT_CALL(*unloading_loader, Load(_)).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - unloading_id, std::unique_ptr(unloading_loader))); + absl::Notification load_done; + EXPECT_CALL(*unloading_loader, + LoadWithMetadata(Loader::Metadata{unloading_id})) + .WillOnce(Return(OkStatus())); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServableData( + unloading_id, std::unique_ptr(unloading_loader)))); basic_manager_->LoadServable(unloading_id, [&load_done](const Status& status) { TF_EXPECT_OK(status); load_done.Notify(); }); load_done.WaitForNotification(); - Notification unload_started; - Notification finish_unload; + absl::Notification unload_started; + absl::Notification finish_unload; EXPECT_CALL(*unloading_loader, Unload()) .WillOnce(Invoke([&unload_started, &finish_unload] { unload_started.Notify(); @@ -1168,15 +1508,17 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedAfterUnload) { .WillOnce(Invoke([&finish_unload](ResourceAllocation* estimate) { finish_unload.Notify(); *estimate = CreateResourceQuantity(10); - return Status::OK(); + return OkStatus(); })) .WillOnce(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(10); - return Status::OK(); + return OkStatus(); })); - EXPECT_CALL(*succeeding_loader, Load(_)).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + EXPECT_CALL(*succeeding_loader, + LoadWithMetadata(Loader::Metadata{succeeding_id})) + .WillOnce(Return(OkStatus())); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); basic_manager_->LoadServable( succeeding_id, [](const Status& status) { TF_EXPECT_OK(status); }); @@ -1188,20 +1530,21 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { // A first loader that gets rejected due to insufficient resources. const ServableId denied_id = {"denied", 0}; test_util::MockLoader* denied_loader = new NiceMock; - Notification denied_estimate_started; - Notification finish_denied_estimate; + absl::Notification denied_estimate_started; + absl::Notification finish_denied_estimate; EXPECT_CALL(*denied_loader, EstimateResources(_)) .WillOnce(Invoke([&denied_estimate_started, &finish_denied_estimate](ResourceAllocation* estimate) { denied_estimate_started.Notify(); finish_denied_estimate.WaitForNotification(); *estimate = CreateResourceQuantity(11 /* more than the system total */); - return Status::OK(); + return OkStatus(); })); // Load won't be called because resources are not enough to load it. - EXPECT_CALL(*denied_loader, Load(_)).Times(0); - basic_manager_->ManageServable( - CreateServableData(denied_id, std::unique_ptr(denied_loader))); + EXPECT_CALL(*denied_loader, LoadWithMetadata(Loader::Metadata{denied_id})) + .Times(0); + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(denied_id, std::unique_ptr(denied_loader)))); // A second loader that succeeds. const ServableId succeeding_id = {"succeeding", 0}; @@ -1210,10 +1553,10 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { ON_CALL(*succeeding_loader, EstimateResources(_)) .WillByDefault(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(10); - return Status::OK(); + return OkStatus(); })); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + TF_ASSERT_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); Status denied_load_status; // Place the first servable into a load request decision phase. @@ -1226,13 +1569,13 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { denied_estimate_started.WaitForNotification(); // The second servable's Load() call shouldn't occur until after the first // servable's load request exits its decision phase. - EXPECT_CALL(*succeeding_loader, Load(_)) - .WillOnce(Invoke([&finish_denied_estimate]( - const ResourceAllocation& available_resources) { + EXPECT_CALL(*succeeding_loader, + LoadWithMetadata(Loader::Metadata{succeeding_id})) + .WillOnce(InvokeWithoutArgs([&finish_denied_estimate]() { // Ensure that the first servable's load request has been given // permission to exit its decision phase. EXPECT_TRUE(finish_denied_estimate.HasBeenNotified()); - return Status::OK(); + return OkStatus(); })); // Scoping ensures that the thread is run by the end of this scope. @@ -1260,6 +1603,138 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { EqualsServableState(expected_error_state)); } +TEST_F(ResourceConstrainedBasicManagerTest, EventBusErrorOnEstimateResources) { + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock; + EXPECT_CALL(*loader, EstimateResources(_)) + .WillOnce(Return(errors::Internal("Error on estimate resources."))); + TF_ASSERT_OK(basic_manager_->ManageServable( + CreateServableData(id, std::unique_ptr(loader)))); + basic_manager_->LoadServable( + id, [](const Status& status) { EXPECT_FALSE(status.ok()); }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + const ServableState error_state = { + id, ServableState::ManagerState::kEnd, + errors::Internal(strings::StrCat( + "Error while attempting to reserve resources to load servable ", + id.DebugString(), ": Error on estimate resources."))}; + EXPECT_THAT(*servable_state_monitor_.GetState(id), + EqualsServableState(error_state)); +} + +TEST(EstimateResourcesRetriedTest, Succeeds) { + std::shared_ptr> servable_event_bus = + EventBus::CreateEventBus(); + ServableStateMonitor servable_state_monitor(servable_event_bus.get()); + + BasicManager::Options options; + // Seed the manager with ten resource units. + options.resource_tracker = CreateSimpleResourceTracker(10); + options.servable_event_bus = servable_event_bus.get(); + options.num_load_threads = 0; + options.num_unload_threads = 0; + + options.max_num_load_retries = 1; + options.load_retry_interval_micros = 0; + + std::unique_ptr basic_manager; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager)); + + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock; + EXPECT_CALL(*loader, EstimateResources(_)) + .WillOnce(Return(errors::Internal("Error on estimate resources."))) + .WillOnce(Return(OkStatus())); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{id})) + .WillRepeatedly(Return(OkStatus())); + TF_ASSERT_OK(basic_manager->ManageServable( + CreateServableData(id, std::unique_ptr(loader)))); + basic_manager->LoadServable( + id, [](const Status& status) { EXPECT_TRUE(status.ok()); }); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor, id, {ServableState::ManagerState::kAvailable}); + const ServableState available_state = { + id, ServableState::ManagerState::kAvailable, OkStatus()}; + EXPECT_THAT(*servable_state_monitor.GetState(id), + EqualsServableState(available_state)); +} + +TEST(EstimateResourcesRetriedTest, Fails) { + std::shared_ptr> servable_event_bus = + EventBus::CreateEventBus(); + ServableStateMonitor servable_state_monitor(servable_event_bus.get()); + + BasicManager::Options options; + // Seed the manager with ten resource units. + options.resource_tracker = CreateSimpleResourceTracker(10); + options.servable_event_bus = servable_event_bus.get(); + options.num_load_threads = 0; + options.num_unload_threads = 0; + + options.max_num_load_retries = 1; + options.load_retry_interval_micros = 0; + + std::unique_ptr basic_manager; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager)); + + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock; + EXPECT_CALL(*loader, EstimateResources(_)) + .WillOnce(Return(errors::Internal("Error on estimate resources."))) + .WillOnce(Return(errors::Internal("Error on estimate resources."))) + .WillRepeatedly(Return(OkStatus())); + TF_ASSERT_OK(basic_manager->ManageServable( + CreateServableData(id, std::unique_ptr(loader)))); + basic_manager->LoadServable( + id, [](const Status& status) { EXPECT_FALSE(status.ok()); }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor, id, + {ServableState::ManagerState::kEnd}); + EXPECT_FALSE(servable_state_monitor.GetState(id)->health.ok()); +} + +TEST(EstimateResourcesRetriedTest, NonRetriableError) { + std::shared_ptr> servable_event_bus = + EventBus::CreateEventBus(); + ServableStateMonitor servable_state_monitor(servable_event_bus.get()); + + BasicManager::Options options; + // Seed the manager with ten resource units. + options.resource_tracker = CreateSimpleResourceTracker(10); + options.servable_event_bus = servable_event_bus.get(); + options.num_load_threads = 0; + options.num_unload_threads = 0; + options.should_retry_model_load = + ([](absl::Status status) { return !absl::IsInvalidArgument(status); }); + + options.max_num_load_retries = 10; + options.load_retry_interval_micros = 100000000; + + std::unique_ptr basic_manager; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager)); + + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock; + EXPECT_CALL(*loader, LoadWithMetadata(_)) + .WillOnce(Return(errors::InvalidArgument("Non-retriable error."))) + .WillRepeatedly(Return(absl::OkStatus())); + TF_ASSERT_OK(basic_manager->ManageServable( + CreateServableData(id, std::unique_ptr(loader)))); + basic_manager->LoadServable( + id, [](const auto& status) { EXPECT_FALSE(status.ok()); }); + + // Make sure the final state is kEnd. + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor, id, + {ServableState::ManagerState::kEnd, + ServableState::ManagerState::kAvailable}); + const auto final_state = servable_state_monitor.GetState(id); + ASSERT_TRUE(final_state.has_value()); + EXPECT_EQ(final_state->manager_state, ServableState::ManagerState::kEnd); + EXPECT_FALSE(final_state->health.ok()); + EXPECT_EQ(final_state->health.message(), "Non-retriable error."); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/caching_manager.cc b/tensorflow_serving/core/caching_manager.cc index b1ee51f8dba..91a55d404bf 100644 --- a/tensorflow_serving/core/caching_manager.cc +++ b/tensorflow_serving/core/caching_manager.cc @@ -15,8 +15,12 @@ limitations under the License. #include "tensorflow_serving/core/caching_manager.h" +#include +#include #include +#include +#include "absl/types/optional.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" @@ -24,7 +28,6 @@ limitations under the License. #include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -35,11 +38,12 @@ Status CachingManager::Create( // Set up basic manager options from the caching manager options. BasicManager::Options basic_manager_options; basic_manager_options.resource_tracker = std::move(options.resource_tracker); - basic_manager_options.num_load_unload_threads = - options.num_load_unload_threads; + basic_manager_options.num_load_threads = options.num_load_threads; + basic_manager_options.num_unload_threads = options.num_unload_threads; basic_manager_options.max_num_load_retries = options.max_num_load_retries; basic_manager_options.load_retry_interval_micros = options.load_retry_interval_micros; + basic_manager_options.env = options.env; basic_manager_options.servable_event_bus = options.servable_event_bus; // Create a basic manager and use it to construct the caching manager. @@ -49,7 +53,7 @@ Status CachingManager::Create( caching_manager->reset( new CachingManager(std::move(loader_factory), std::move(basic_manager))); - return Status::OK(); + return OkStatus(); } CachingManager::CachingManager(std::unique_ptr loader_factory, @@ -68,8 +72,10 @@ Status CachingManager::GetUntypedServableHandle( } // Since there is no explicit version in the request, get the latest from the // loader-factory. - const int64 latest_version = loader_factory_->GetLatestVersion(request.name); - return GetUntypedServableHandleForId({request.name, latest_version}, handle); + const int64_t policy_dictated_version = loader_factory_->GetServableVersion( + request.name, request.auto_version_policy); + return GetUntypedServableHandleForId({request.name, policy_dictated_version}, + handle); } Status CachingManager::GetUntypedServableHandleForId( @@ -86,8 +92,8 @@ Status CachingManager::GetUntypedServableHandleForId( } // Build the servable data corresponding to the servable-id. - std::unique_ptr>> loader_data; - TF_RETURN_IF_ERROR(loader_factory_->CreateLoader(servable_id, &loader_data)); + ServableData> loader_data = + loader_factory_->CreateLoader(servable_id); // Load the servable corresponding to the servable-id. For multiple concurrent // requests enforces that exactly one thread performs the load operation with @@ -101,8 +107,8 @@ Status CachingManager::GetUntypedServableHandleForId( } Status CachingManager::LoadServable( - std::unique_ptr>> loader_data) { - const ServableId servable_id = loader_data->id(); + ServableData> loader_data) { + const ServableId servable_id = loader_data.id(); std::shared_ptr servable_id_mu; { @@ -121,7 +127,7 @@ Status CachingManager::LoadServable( // Retrieve the state of the servable from the wrapped basic-manager. The // servable should already be managed by the basic-manager. - const optional> snapshot = + const absl::optional> snapshot = basic_manager_->GetManagedServableStateSnapshot(servable_id); if (snapshot) { // The servable is already being managed by 'basic_manager_'. Hence it @@ -145,11 +151,11 @@ Status CachingManager::LoadServable( // automatically available in the caching-manager as well (via the basic // manager). const Status manage_status = - basic_manager_->ManageServable(std::move(*loader_data)); + basic_manager_->ManageServable(std::move(loader_data)); if (!manage_status.ok()) { const string error_msg = strings::StrCat( "Internal error: unable to transfer servable to 'basic_manager_': ", - manage_status.error_message()); + manage_status.message()); DCHECK(false) << error_msg; return errors::Internal(error_msg); } @@ -166,7 +172,7 @@ Status CachingManager::LoadServable( } servable_id_mu.reset(); MaybeEraseLoadMutexMapEntry(servable_id); - return Status::OK(); + return OkStatus(); } void CachingManager::MaybeEraseLoadMutexMapEntry( @@ -180,7 +186,7 @@ void CachingManager::MaybeEraseLoadMutexMapEntry( } } -int64 CachingManager::GetLoadMutexMapSize() const { +int64_t CachingManager::GetLoadMutexMapSize() const { mutex_lock l(load_mutex_map_mu_); return load_mutex_map_.size(); } @@ -194,5 +200,28 @@ std::vector CachingManager::ListAvailableServableIds() const { return basic_manager_->ListAvailableServableIds(); } +PathPrefixLoaderFactory::PathPrefixLoaderFactory( + const string& path_prefix, + std::unique_ptr adapter) + : path_prefix_(path_prefix), adapter_(std::move(adapter)) {} + +ServableData> PathPrefixLoaderFactory::CreateLoader( + const ServableId& id) { + if (id.version != 0) { + return ServableData>( + id, + errors::FailedPrecondition("PathPrefixLoaderFactory only supports " + "single-version servables at version 0")); + } + const StoragePath servable_path = io::JoinPath(path_prefix_, id.name); + return adapter_->AdaptOneVersion({id, servable_path}); +} + +int64_t PathPrefixLoaderFactory::GetServableVersion( + const string& servable_name, + ServableRequest::AutoVersionPolicy policy) const { + return 0; +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/caching_manager.h b/tensorflow_serving/core/caching_manager.h index 38cea83f524..4a8e79980ee 100644 --- a/tensorflow_serving/core/caching_manager.h +++ b/tensorflow_serving/core/caching_manager.h @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow_serving/core/basic_manager.h" #include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/source_adapter.h" namespace tensorflow { namespace serving { @@ -31,29 +32,34 @@ namespace test_util { class CachingManagerTestAccess; } // namespace test_util -// A manager that manages and loads servables on-demand. Upon receiving the -// request for a servable name and optional version, the manager checks if it -// already has the requested servable loaded. If not, it initiates the load -// operation and then serves the request. -// -// The manager blocks on the load operation and returns the handle when the -// servable has been loaded, or upon error. +/// A manager that manages and loads servables on-demand. Upon receiving the +/// request for a servable name and optional version, the manager checks if it +/// already has the requested servable loaded. If not, it initiates the load +/// operation and then serves the request. +/// +/// The manager blocks on the load operation and returns the handle when the +/// servable has been loaded, or upon error. // // TODO(b/25449742): Add support for evictions of loaded servables from the // caching-manager. class CachingManager : public Manager { public: + /// Config options and pluggable objects that will be used by the + /// CachingManager. struct Options { // The resource tracker to use while managing servable resources. Optional. // If left as nullptr, we do not validate servable resource usage. std::unique_ptr resource_tracker; - // The number of threads in the thread-pool used to load and unload - // servables. + // The number of threads in the thread-pool used to load servables. + // + // If set as 0, we don't use a thread-pool, and LoadServable() blocks. + uint32 num_load_threads = 0; + + // The number of threads in the thread-pool used to unload servables. // - // If set as 0, we don't use a thread-pool, and the {Load,Unload}Servable() - // methods block. - uint32 num_load_unload_threads = 0; + // If set as 0, we don't use a thread-pool. + uint32 num_unload_threads = 0; // EventBus to publish servable state changes. This is optional, if unset, // we don't publish. @@ -66,27 +72,29 @@ class CachingManager : public Manager { // The interval, in microseconds, between each servable load retry. If set // negative, we don't wait. // Default: 1 minute. - int64 load_retry_interval_micros = 1LL * 60 * 1000 * 1000; + int64_t load_retry_interval_micros = 1LL * 60 * 1000 * 1000; // The environment to use for starting threads in the thread-pool. Env* env = Env::Default(); }; - // An abstraction for a loader-factory to map from a servable request to the - // corresponding loader. + /// An abstraction for a loader-factory to map from a servable request to the + /// corresponding loader. class LoaderFactory { public: virtual ~LoaderFactory() = default; - // Creates servable data consisting of the loader corresponding to the - // servable-id. - virtual Status CreateLoader( - const ServableId& servable_id, - std::unique_ptr>>* - loader_data) = 0; - - // Returns the latest version corresponding to the servable name. - virtual int64 GetLatestVersion(const string& servable_name) const = 0; + /// Creates servable data consisting of the loader corresponding to the + /// servable-id. Any errors can be reported by embedding them in the + /// returned ServableData item. + virtual ServableData> CreateLoader( + const ServableId& servable_id) = 0; + + /// Returns a version corresponding to the servable name, for the given + /// policy. + virtual int64_t GetServableVersion( + const string& servable_name, + ServableRequest::AutoVersionPolicy policy) const = 0; }; static Status Create(Options options, @@ -126,12 +134,11 @@ class CachingManager : public Manager { // exactly one thread performs the load operation using the wrapped // basic-manager. All other requests block until the load completes and then // trivially succeed. - Status LoadServable( - std::unique_ptr>> loader_data) - LOCKS_EXCLUDED(load_mutex_map_mu_); + Status LoadServable(ServableData> loader_data) + TF_LOCKS_EXCLUDED(load_mutex_map_mu_); // Returns the size of the load_mutex_map_. - int64 GetLoadMutexMapSize() const LOCKS_EXCLUDED(load_mutex_map_mu_); + int64_t GetLoadMutexMapSize() const TF_LOCKS_EXCLUDED(load_mutex_map_mu_); // Erases the entry from the map corresponding to the servable-id if there is // only one remaining reference to the mutex. @@ -149,11 +156,37 @@ class CachingManager : public Manager { // a shared_ptr to allow for reference counting and consequent garbage // collection. std::map> load_mutex_map_ - GUARDED_BY(load_mutex_map_mu_); + TF_GUARDED_BY(load_mutex_map_mu_); TF_DISALLOW_COPY_AND_ASSIGN(CachingManager); }; +/// A simple LoaderFactory that looks for a servable at a path formed by +/// concatenating a fixed path prefix with the servable's name. It assumes that +/// a given servable only has one version, namely version 0. +class PathPrefixLoaderFactory : public CachingManager::LoaderFactory { + public: + PathPrefixLoaderFactory(const string& path_prefix, + std::unique_ptr adapter); + ~PathPrefixLoaderFactory() override = default; + + ServableData> CreateLoader( + const ServableId& id) override; + + int64_t GetServableVersion( + const string& servable_name, + ServableRequest::AutoVersionPolicy policy) const override; + + private: + // The prefix of the path to the servables. + const string path_prefix_; + + // An adapter for creating a loader from a given path. + const std::unique_ptr adapter_; + + TF_DISALLOW_COPY_AND_ASSIGN(PathPrefixLoaderFactory); +}; + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/caching_manager_test.cc b/tensorflow_serving/core/caching_manager_test.cc index a3ac3e6121a..87c8ff285d1 100644 --- a/tensorflow_serving/core/caching_manager_test.cc +++ b/tensorflow_serving/core/caching_manager_test.cc @@ -15,7 +15,10 @@ limitations under the License. #include "tensorflow_serving/core/caching_manager.h" +#include +#include #include +#include #include #include @@ -30,74 +33,89 @@ limitations under the License. #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/simple_loader.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/core/test_util/manager_test_util.h" #include "tensorflow_serving/util/event_bus.h" -#include "tensorflow_serving/util/optional.h" #include "tensorflow_serving/util/threadpool_executor.h" namespace tensorflow { namespace serving { namespace { -using ::testing::UnorderedElementsAre; +using ::testing::HasSubstr; using ::testing::UnorderedElementsAreArray; // A simple loader-factory that concatenates requested servable name and // version. class StringLoaderFactory : public CachingManager::LoaderFactory { public: - explicit StringLoaderFactory(const int64 starting_version) + explicit StringLoaderFactory(const int64_t starting_version) : latest_version_(starting_version) {} ~StringLoaderFactory() override = default; - Status CreateLoader(const ServableId& id, - std::unique_ptr>>* - loaded_data) override { + ServableData> CreateLoader( + const ServableId& id) override { + // Update state to indicate a new loader was created. + { + mutex_lock l(mu_); + num_loaders_dispensed_++; + } + auto servable_creator = [&](std::unique_ptr* servable) { servable->reset(new string); **servable = strings::StrCat(id.name, "-", id.version); - return Status::OK(); + return OkStatus(); }; std::unique_ptr loader; loader.reset(new SimpleLoader( servable_creator, SimpleLoader::EstimateNoResources())); - loaded_data->reset( - new ServableData>(id, std::move(loader))); - // Update state to indicate a new loader was created. + return ServableData>(id, std::move(loader)); + } + + // Returns the earliest/latest version corresponding to the servable name. + int64_t GetServableVersion( + const string& request_name, + ServableRequest::AutoVersionPolicy policy) const override { mutex_lock l(mu_); - num_loaders_dispensed_++; - return Status::OK(); + switch (policy) { + case ServableRequest::AutoVersionPolicy::kEarliest: + return earliest_version_; + case ServableRequest::AutoVersionPolicy::kLatest: + return latest_version_; + } } - // Returns the latest version corresponding to the servable name. - int64 GetLatestVersion(const string& request_name) const override { - // Increment the current latest version until a maximum of 42. + // Update the earliest available version. + void set_earliest_version(int64_t version) { mutex_lock l(mu_); - return latest_version_; + earliest_version_ = version; } // Update the latest available version. - void set_latest_version(int64 version) { + void set_latest_version(int64_t version) { mutex_lock l(mu_); latest_version_ = version; } // Returns the number of loaders created by the loader-factory. - int64 num_loaders_dispensed() const { + int64_t num_loaders_dispensed() const { mutex_lock l(mu_); return num_loaders_dispensed_; } private: - // Used to protect updates to the latest_version_. + // Used to protect updates to 'earliest_version_' and 'latest_version_'. mutable mutex mu_; + // The current earliest version. + int64_t earliest_version_ TF_GUARDED_BY(mu_) = 0; + // The current latest version. - int64 latest_version_ GUARDED_BY(mu_) = 0; + int64_t latest_version_ TF_GUARDED_BY(mu_) = 0; // Tracks the number of loaders dispensed by the loader-factory. - int64 num_loaders_dispensed_ GUARDED_BY(mu_) = 0; + int64_t num_loaders_dispensed_ TF_GUARDED_BY(mu_) = 0; TF_DISALLOW_COPY_AND_ASSIGN(StringLoaderFactory); }; @@ -109,22 +127,21 @@ class ErrorLoaderFactory : public CachingManager::LoaderFactory { ErrorLoaderFactory() = default; ~ErrorLoaderFactory() override = default; - Status CreateLoader(const ServableId& id, - std::unique_ptr>>* - loaded_data) override { + ServableData> CreateLoader( + const ServableId& id) override { auto servable_creator = [&](std::unique_ptr* servable) { return errors::Unknown("error loader-factory"); }; std::unique_ptr loader; loader.reset(new SimpleLoader( servable_creator, SimpleLoader::EstimateNoResources())); - loaded_data->reset( - new ServableData>(id, std::move(loader))); - return Status::OK(); + return ServableData>(id, std::move(loader)); } - int64 GetLatestVersion(const string& request_name) const override { - // A simple "latest" interpretation that always returns version 42. + int64_t GetServableVersion( + const string& request_name, + ServableRequest::AutoVersionPolicy policy) const override { + // A simple policy interpretation that always returns version 42. return 42; } @@ -138,7 +155,13 @@ constexpr char kServableName2[] = "kServableName2"; constexpr int kNumThreads = 10; -class CachingManagerTest : public ::testing::TestWithParam { +// We parameterize this test with the number of load & unload threads. (Zero +// means use an in-line executor instead of a thread pool.) +struct ThreadPoolSizes { + uint64_t num_load_threads; + uint64_t num_unload_threads; +}; +class CachingManagerTest : public ::testing::TestWithParam { protected: CachingManagerTest() : servable_event_bus_(EventBus::CreateEventBus()), @@ -146,7 +169,8 @@ class CachingManagerTest : public ::testing::TestWithParam { CachingManager::Options options; options.env = Env::Default(); options.servable_event_bus = servable_event_bus_.get(); - options.num_load_unload_threads = GetParam(); + options.num_load_threads = GetParam().num_load_threads; + options.num_unload_threads = GetParam().num_unload_threads; options.max_num_load_retries = 1; options.load_retry_interval_micros = 0; @@ -165,7 +189,8 @@ class CachingManagerTest : public ::testing::TestWithParam { CachingManager::Options options; options.env = Env::Default(); options.servable_event_bus = servable_event_bus_.get(); - options.num_load_unload_threads = GetParam(); + options.num_load_threads = GetParam().num_load_threads; + options.num_unload_threads = GetParam().num_unload_threads; options.max_num_load_retries = 1; options.load_retry_interval_micros = 0; @@ -180,7 +205,7 @@ class CachingManagerTest : public ::testing::TestWithParam { // Helper function to return the size of the load-mutex map from the // caching-manager. - int64 GetLoadMutexMapSize() { + int64_t GetLoadMutexMapSize() { return test_util::CachingManagerTestAccess(manager_.get()) .GetLoadMutexMapSize(); } @@ -191,8 +216,11 @@ class CachingManagerTest : public ::testing::TestWithParam { StringLoaderFactory* string_loader_factory_; }; -INSTANTIATE_TEST_CASE_P(WithOrWithoutThreadPool, CachingManagerTest, - ::testing::Values(0 /* WithoutThreadPool */, 4)); +INSTANTIATE_TEST_CASE_P( + WithOrWithoutThreadPools, CachingManagerTest, + ::testing::Values( + ThreadPoolSizes{0, 0} /* without load or unload threadpools */, + ThreadPoolSizes{4, 4} /* with load and unload threadpools */)); /////////////////////////////////////////////////////////////////////////////// // Servable handles. @@ -230,6 +258,18 @@ TEST_P(CachingManagerTest, ServableHandleMultipleRequests) { } } +// Tests functionality when the version corresponding to the "earliest" needs to +// be newly managed and loaded by the manager. +TEST_P(CachingManagerTest, ServableHandleSingleRequestEarliest) { + string_loader_factory_->set_earliest_version(30); + ServableHandle handle; + TF_ASSERT_OK(manager_->GetServableHandle( + ServableRequest::Earliest({kServableName}), &handle)); + EXPECT_EQ("kServableName-30", *handle); + const ServableId id = {kServableName, 30}; + EXPECT_EQ(id, handle.id()); +} + // Tests functionality when the version corresponding to the "latest" needs to // be newly managed and loaded by the manager. TEST_P(CachingManagerTest, ServableHandleSingleRequestLatest) { @@ -242,6 +282,36 @@ TEST_P(CachingManagerTest, ServableHandleSingleRequestLatest) { EXPECT_EQ(id, handle.id()); } +// Tests functionality when the version corresponding to the "earliest" is +// already managed and loaded by the caching-manager. +TEST_P(CachingManagerTest, ServableHandleMultipleRequestsEarliest) { + const ServableId id = {kServableName, 42}; + { + // Make an explicit request for version 42. + ServableHandle handle; + TF_ASSERT_OK( + manager_->GetServableHandle(ServableRequest::FromId(id), &handle)); + EXPECT_EQ("kServableName-42", *handle); + EXPECT_EQ(id, handle.id()); + // We expect a new loader to be created for this request. + EXPECT_EQ(1, string_loader_factory_->num_loaders_dispensed()); + // Update the earliest available version. + string_loader_factory_->set_earliest_version(42); + } + { + // Now request for the earliest. The returned handle should have an id + // corresponding to version 42. + ServableHandle handle; + TF_ASSERT_OK(manager_->GetServableHandle( + ServableRequest::Earliest({kServableName}), &handle)); + EXPECT_EQ("kServableName-42", *handle); + EXPECT_EQ(id, handle.id()); + // We do not expect a new loader to be created for this request, since it is + // identical to the previous request. + EXPECT_EQ(1, string_loader_factory_->num_loaders_dispensed()); + } +} + // Tests functionality when the version corresponding to the "latest" is // already managed and loaded by the caching-manager. TEST_P(CachingManagerTest, ServableHandleMultipleRequestsLatest) { @@ -404,7 +474,7 @@ TEST_P(CachingManagerTest, EventBusSingleRequest) { // Check that the state published on the event-bus matches produced by the // loader-factory for a successful request. const ServableState expected_published_state = { - id, ServableState::ManagerState::kAvailable, Status::OK()}; + id, ServableState::ManagerState::kAvailable, OkStatus()}; EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(expected_published_state)); } @@ -449,7 +519,7 @@ TEST_P(CachingManagerTest, ConcurrentDisjointRequests) { // Check that all requests returned with an ok status. for (int i = 0; i < 4; i++) { mutex_lock l(status_mu); - EXPECT_EQ(Status::OK(), statuses[i]); + EXPECT_EQ(OkStatus(), statuses[i]); } // Check that the available servable handles now includes all requested // servables. @@ -492,7 +562,7 @@ TEST_P(CachingManagerTest, ConcurrentIntersectingRequests) { // Check that all requests returned with an ok status. for (int i = 0; i < 8; i++) { mutex_lock l(status_mu); - EXPECT_EQ(Status::OK(), statuses[i]); + EXPECT_EQ(OkStatus(), statuses[i]); } // Check that the available servable handles now includes all requested // servables. @@ -512,6 +582,37 @@ TEST_P(CachingManagerTest, ConcurrentIntersectingRequests) { /////////////////////////////////////////////////////////////////////////////// +TEST(PathPrefixLoaderFactoryTest, Basic) { + auto adapter = std::unique_ptr( + new test_util::FakeLoaderSourceAdapter("suffix")); + PathPrefixLoaderFactory factory("prefix", std::move(adapter)); + + ServableData> loader_data = + factory.CreateLoader({"servable_name", 0}); + TF_ASSERT_OK(loader_data.status()); + std::unique_ptr loader = loader_data.ConsumeDataOrDie(); + TF_ASSERT_OK(loader->Load()); + EXPECT_EQ("prefix/servable_name/suffix", *loader->servable().get()); + + EXPECT_EQ(0, factory.GetServableVersion( + "blah", ServableRequest::AutoVersionPolicy::kEarliest)); + EXPECT_EQ(0, factory.GetServableVersion( + "blah", ServableRequest::AutoVersionPolicy::kLatest)); +} + +TEST(PathPrefixLoaderFactoryTest, VersionOtherThanZeroYieldsError) { + auto adapter = std::unique_ptr( + new test_util::FakeLoaderSourceAdapter("suffix")); + PathPrefixLoaderFactory factory("prefix", std::move(adapter)); + + ServableData> loader_data = + factory.CreateLoader({"servable_name", 42}); + ASSERT_FALSE(loader_data.status().ok()); + EXPECT_THAT(loader_data.status().ToString(), + HasSubstr("PathPrefixLoaderFactory only supports single-version " + "servables at version 0")); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/dynamic_source_router.h b/tensorflow_serving/core/dynamic_source_router.h new file mode 100644 index 00000000000..ea1fd5552bf --- /dev/null +++ b/tensorflow_serving/core/dynamic_source_router.h @@ -0,0 +1,146 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_DYNAMIC_SOURCE_ROUTER_H_ +#define TENSORFLOW_SERVING_CORE_DYNAMIC_SOURCE_ROUTER_H_ + +#include +#include +#include + +#include "tensorflow/core/platform/macros.h" +#include "tensorflow_serving/core/source_router.h" + +namespace tensorflow { +namespace serving { + +// A SourceRouter with a fixed set of N output ports, and a dynamically +// reconfigurable map from servable name to port. The route map can only +// reference the first N-1 ports; the Nth port is reserved for servables not +// found in the route map. +template +class DynamicSourceRouter final : public SourceRouter { + public: + // A servable name -> output port map. + using Routes = std::map; + + // Creates a DynamicSourceRouter with 'num_output_ports' output ports and an + // (initial) route map given by 'routes'. + static Status Create(int num_output_ports, const Routes& routes, + std::unique_ptr>* result); + ~DynamicSourceRouter() override; + + // Gets the current route map. + Routes GetRoutes() const; + + // Sets the route map to 'routes'. + Status UpdateRoutes(const Routes& routes); + + protected: + int num_output_ports() const override { return num_output_ports_; } + + int Route(const StringPiece servable_name, + const std::vector>& versions) override; + + private: + DynamicSourceRouter(int num_output_ports, const Routes& routes); + + // Returns an error if 'routes' is invalid, given 'num_output_ports'. + static Status ValidateRoutes(int num_output_ports, const Routes& routes); + + const int num_output_ports_; + + mutable mutex routes_mu_; + Routes routes_ TF_GUARDED_BY(routes_mu_); + + TF_DISALLOW_COPY_AND_ASSIGN(DynamicSourceRouter); +}; + +////////// +// Implementation details follow. API users need not read. + +template +Status DynamicSourceRouter::Create( + int num_output_ports, const Routes& routes, + std::unique_ptr>* result) { + TF_RETURN_IF_ERROR(ValidateRoutes(num_output_ports, routes)); + result->reset(new DynamicSourceRouter(num_output_ports, routes)); + return Status(); +} + +template +DynamicSourceRouter::~DynamicSourceRouter() { + TargetBase::Detach(); +} + +template +typename DynamicSourceRouter::Routes DynamicSourceRouter::GetRoutes() + const { + mutex_lock l(routes_mu_); + return routes_; +} + +template +Status DynamicSourceRouter::UpdateRoutes(const Routes& routes) { + TF_RETURN_IF_ERROR(ValidateRoutes(num_output_ports_, routes)); + { + mutex_lock l(routes_mu_); + routes_ = routes; + } + return Status(); +} + +template +int DynamicSourceRouter::Route( + const StringPiece servable_name, + const std::vector>& versions) { + mutex_lock l(routes_mu_); + auto it = routes_.find(string(servable_name)); + if (it == routes_.end()) { + LOG(INFO) << "Routing servable(s) from stream " << servable_name + << " to default output port " << num_output_ports_ - 1; + return num_output_ports_ - 1; + } else { + return it->second; + } +} + +template +DynamicSourceRouter::DynamicSourceRouter(int num_output_ports, + const Routes& routes) + : num_output_ports_(num_output_ports), routes_(routes) {} + +template +Status DynamicSourceRouter::ValidateRoutes(int num_output_ports, + const Routes& routes) { + for (const auto& entry : routes) { + const int port = entry.second; + if (port < 0 || port >= num_output_ports) { + return errors::InvalidArgument( + strings::StrCat("Port number out of range: ", port)); + } + if (port == num_output_ports - 1) { + return errors::InvalidArgument( + "Last port cannot be used in route map, since it's reserved for the " + "default route"); + } + } + return Status(); +} + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_DYNAMIC_SOURCE_ROUTER_H_ diff --git a/tensorflow_serving/core/dynamic_source_router_test.cc b/tensorflow_serving/core/dynamic_source_router_test.cc new file mode 100644 index 00000000000..73468c871da --- /dev/null +++ b/tensorflow_serving/core/dynamic_source_router_test.cc @@ -0,0 +1,148 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/dynamic_source_router.h" + +#include +#include +#include +#include + +#include +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/servable_data.h" +#include "tensorflow_serving/core/source.h" +#include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/core/target.h" +#include "tensorflow_serving/core/test_util/mock_storage_path_target.h" + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::StrictMock; + +namespace tensorflow { +namespace serving { +namespace { + +TEST(DynamicSourceRouterTest, InvalidRouteMap) { + std::unique_ptr> router; + // Negative output port. + EXPECT_FALSE( + DynamicSourceRouter::Create(2, {{"foo", -1}}, &router).ok()); + // Out of range output port. + EXPECT_FALSE( + DynamicSourceRouter::Create(2, {{"foo", 2}}, &router).ok()); + // Using reserved (last) output port. + EXPECT_FALSE( + DynamicSourceRouter::Create(2, {{"foo", 1}}, &router).ok()); +} + +TEST(DynamicSourceRouterTest, ReconfigureToInvalidRouteMap) { + std::unique_ptr> router; + TF_ASSERT_OK(DynamicSourceRouter::Create(2, {{"foo", 0}}, &router)); + // Negative output port. + EXPECT_FALSE(router->UpdateRoutes({{"foo", -1}}).ok()); + // Out of range output port. + EXPECT_FALSE(router->UpdateRoutes({{"foo", 2}}).ok()); + // Using reserved (last) output port. + EXPECT_FALSE(router->UpdateRoutes({{"foo", 1}}).ok()); +} + +TEST(DynamicSourceRouterTest, Basic) { + std::unique_ptr> router; + DynamicSourceRouter::Routes routes = {{"foo", 0}, {"bar", 1}}; + TF_ASSERT_OK(DynamicSourceRouter::Create(4, routes, &router)); + EXPECT_EQ(routes, router->GetRoutes()); + std::vector*> output_ports = router->GetOutputPorts(); + ASSERT_EQ(4, output_ports.size()); + std::vector> targets; + for (int i = 0; i < output_ports.size(); ++i) { + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(output_ports[i], target.get()); + targets.push_back(std::move(target)); + } + + // "foo" goes to port 0. + EXPECT_CALL(*targets[0], SetAspiredVersions( + Eq("foo"), ElementsAre(ServableData( + {"foo", 7}, "data")))); + router->SetAspiredVersions("foo", + {ServableData({"foo", 7}, "data")}); + + // "bar" goes to port 1. + EXPECT_CALL(*targets[1], SetAspiredVersions( + Eq("bar"), ElementsAre(ServableData( + {"bar", 7}, "data")))); + router->SetAspiredVersions("bar", + {ServableData({"bar", 7}, "data")}); + + // Servable whose name doesn't match any route goes to the last port (port 3). + EXPECT_CALL(*targets[3], + SetAspiredVersions(Eq("not_foo_or_bar"), + ElementsAre(ServableData( + {"not_foo_or_bar", 7}, "data")))); + router->SetAspiredVersions( + "not_foo_or_bar", + {ServableData({"not_foo_or_bar", 7}, "data")}); +} + +TEST(DynamicSourceRouterTest, Reconfigure) { + std::unique_ptr> router; + TF_ASSERT_OK(DynamicSourceRouter::Create(2, {{"foo", 0}}, &router)); + std::vector*> output_ports = router->GetOutputPorts(); + ASSERT_EQ(2, output_ports.size()); + std::vector> targets; + for (int i = 0; i < output_ports.size(); ++i) { + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(output_ports[i], target.get()); + targets.push_back(std::move(target)); + } + + // Initially, "foo" goes to port 0 and "bar" goes to the default (port 1). + EXPECT_CALL(*targets[0], SetAspiredVersions( + Eq("foo"), ElementsAre(ServableData( + {"foo", 7}, "data")))); + router->SetAspiredVersions("foo", + {ServableData({"foo", 7}, "data")}); + EXPECT_CALL(*targets[1], SetAspiredVersions( + Eq("bar"), ElementsAre(ServableData( + {"bar", 7}, "data")))); + router->SetAspiredVersions("bar", + {ServableData({"bar", 7}, "data")}); + + TF_ASSERT_OK(router->UpdateRoutes({{"bar", 0}})); + + // Now, the routes of "foo" and "bar" should be swapped. + EXPECT_CALL(*targets[1], SetAspiredVersions( + Eq("foo"), ElementsAre(ServableData( + {"foo", 7}, "data")))); + router->SetAspiredVersions("foo", + {ServableData({"foo", 7}, "data")}); + EXPECT_CALL(*targets[0], SetAspiredVersions( + Eq("bar"), ElementsAre(ServableData( + {"bar", 7}, "data")))); + router->SetAspiredVersions("bar", + {ServableData({"bar", 7}, "data")}); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/eager_load_policy.h b/tensorflow_serving/core/eager_load_policy.h deleted file mode 100644 index 40986cd5d3e..00000000000 --- a/tensorflow_serving/core/eager_load_policy.h +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ -#define TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ - -#include - -#include "tensorflow_serving/core/aspired_version_policy.h" -#include "tensorflow_serving/core/loader_harness.h" -#include "tensorflow_serving/util/optional.h" - -namespace tensorflow { -namespace serving { - -// AspiredVersionPolicy that loads any aspired versions of a servable before -// unloading any no-longer-aspired versions. -// -// This policy provides servable availability with the trade-off of temporary -// increased resource consumption while the new version loads followed by the -// old versions unloading. -class EagerLoadPolicy final : public AspiredVersionPolicy { - public: - optional GetNextAction( - const std::vector& all_versions) - const override; -}; - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ diff --git a/tensorflow_serving/core/eager_load_policy_test.cc b/tensorflow_serving/core/eager_load_policy_test.cc deleted file mode 100644 index 1e84dc67e11..00000000000 --- a/tensorflow_serving/core/eager_load_policy_test.cc +++ /dev/null @@ -1,75 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/core/eager_load_policy.h" - -#include -#include "tensorflow_serving/core/servable_id.h" - -namespace tensorflow { -namespace serving { -namespace { - -// Test that the first new and aspired version is loaded first. -TEST(EagerLoadPolicy, LoadsFirstAspired) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 4}, LoaderHarness::State::kNew, true}); - versions.push_back({{"test", 5}, LoaderHarness::State::kReady, false}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); - EXPECT_EQ(4, action->id.version); -} - -// Test that the first non-aspired version is unloaded when there are none to -// load. -TEST(EagerLoadPolicy, UnLoadsFirstNonAspiredWhenNoneToLoad) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 4}, LoaderHarness::State::kDisabled, false}); - versions.push_back({{"test", 5}, LoaderHarness::State::kReady, false}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); - EXPECT_EQ(3, action->id.version); -} - -// Test that no action is returned (empty optional) when there are no versions -// needing loading or unloading. -TEST(EagerLoadPolicy, ReturnsNoActionWhenNone) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kLoading, true}); - versions.push_back({{"test", 4}, LoaderHarness::State::kUnloading, false}); - versions.push_back({{"test", 5}, LoaderHarness::State::kDisabled, false}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/core/eager_unload_policy.cc b/tensorflow_serving/core/eager_unload_policy.cc deleted file mode 100644 index 702e7fc7fdc..00000000000 --- a/tensorflow_serving/core/eager_unload_policy.cc +++ /dev/null @@ -1,44 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/core/eager_unload_policy.h" - -namespace tensorflow { -namespace serving { - -optional EagerUnloadPolicy::GetNextAction( - const std::vector& all_versions) const { - // First iterate over all_versions and find any in kReady that are no longer - // aspired. Unload the first if any. - for (const auto& version : all_versions) { - if (version.state == LoaderHarness::State::kReady && !version.is_aspired) { - return AspiredVersionPolicy::ServableAction( - {Action::kUnload, version.id}); - } - } - - // Second and only if no action was found earlier, iterate over all - // versions and find any in kNew that are aspired. Load the first if any. - for (const auto& version : all_versions) { - if (version.state == LoaderHarness::State::kNew && version.is_aspired) { - return AspiredVersionPolicy::ServableAction({Action::kLoad, version.id}); - } - } - - return nullopt; -} - -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/core/load_servables_fast.cc b/tensorflow_serving/core/load_servables_fast.cc new file mode 100644 index 00000000000..03e118f0db0 --- /dev/null +++ b/tensorflow_serving/core/load_servables_fast.cc @@ -0,0 +1,125 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/load_servables_fast.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "absl/types/optional.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow_serving/core/servable_state.h" +#include "tensorflow_serving/core/source.h" +#include "tensorflow_serving/core/target.h" + +namespace tensorflow { +namespace serving { + +namespace internal { + +uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager) { + return manager->num_load_threads(); +} + +std::function SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager) { + return manager->set_num_load_threads_observer_->Notifier(); +} + +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, + const std::function& wait_until_loaded_fn, + const uint32 num_threads) { + const uint32 prev_num_load_threads = GetManagerNumLoadThreads(manager); + std::function set_manager_num_load_threads = + SetManagerNumLoadThreadsNotifier(manager); + set_manager_num_load_threads(num_threads); + for (Source>* source : sources) { + ConnectSourceToTarget(source, manager); + } + const Status status = wait_until_loaded_fn(); + set_manager_num_load_threads(prev_num_load_threads); + return status; +} + +} // namespace internal + +Status ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + ServableStateMonitor* servable_state_monitor, + const std::vector& initial_servables, + const uint32 num_threads) { + return ConnectSourcesWithFastInitialLoad(manager, {source}, + servable_state_monitor, + initial_servables, num_threads); +} + +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, + ServableStateMonitor* servable_state_monitor, + const std::vector& initial_servables, + const uint32 num_threads) { + return internal::ConnectSourcesWithFastInitialLoad( + manager, sources, + [&]() { + std::map states_reached; + const bool all_servables_available = + servable_state_monitor->WaitUntilServablesReachState( + initial_servables, ServableState::ManagerState::kAvailable, + &states_reached); + if (!all_servables_available) { + const int num_unavailable_servables = std::count_if( + states_reached.begin(), states_reached.end(), + [](const std::pair& + id_and_state) { + return id_and_state.second != + ServableState::ManagerState::kAvailable; + }); + string message = + strings::StrCat(num_unavailable_servables, + " servable(s) did not become available: {"); + for (const auto& id_and_state : states_reached) { + if (id_and_state.second != + ServableState::ManagerState::kAvailable) { + absl::optional maybe_state = + servable_state_monitor->GetState(id_and_state.first); + const string error_msg = + maybe_state && !maybe_state.value().health.ok() + ? " due to error: " + + maybe_state.value().health.ToString() + : ""; + strings::StrAppend(&message, "{", + id_and_state.first.DebugString(), error_msg, + "}, "); + } + } + strings::StrAppend(&message, "}"); + return errors::Unknown(message); + } + return OkStatus(); + }, + num_threads); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/load_servables_fast.h b/tensorflow_serving/core/load_servables_fast.h new file mode 100644 index 00000000000..cf2c706b6fd --- /dev/null +++ b/tensorflow_serving/core/load_servables_fast.h @@ -0,0 +1,70 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ +#define TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ + +#include +#include + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/cpu_info.h" +#include "tensorflow_serving/core/aspired_versions_manager.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/servable_state_monitor.h" + +namespace tensorflow { +namespace serving { + +// Connects 'source' to 'manager', and speeds up loading of the servables +// matching 'initial_servables'. The speeding up is accomplished by boosting the +// number of threads used for loading until the initial servables have been +// loaded, and then resetting it to the manager's originally configured value. +Status ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + ServableStateMonitor* servable_state_monitor, + const std::vector& initial_servables, + uint32 num_threads = 4 * port::NumSchedulableCPUs()); + +// Like ConnectSourceWithFastInitialLoad(), but with multiple sources. +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, + ServableStateMonitor* servable_state_monitor, + const std::vector& initial_servables, + uint32 num_threads = 4 * port::NumSchedulableCPUs()); + +//// +// Implementation detail. API readers may skip. +/// + +namespace internal { + +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, + const std::function& wait_until_loaded_fn, uint32 num_threads); + +uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager); +std::function SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager); + +} // namespace internal + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ diff --git a/tensorflow_serving/core/loader.h b/tensorflow_serving/core/loader.h index 4de200af228..883f2176b3e 100644 --- a/tensorflow_serving/core/loader.h +++ b/tensorflow_serving/core/loader.h @@ -18,6 +18,7 @@ limitations under the License. #include +#include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/core/source.h" #include "tensorflow_serving/resources/resources.pb.h" @@ -26,120 +27,137 @@ limitations under the License. namespace tensorflow { namespace serving { -// A standardized abstraction for an object that manages the lifecycle of a -// servable, including loading and unloading it. Servables are arbitrary objects -// that serve algorithms or data that often, though not necessarily, use a -// machine-learned model. -// -// A Loader for a servable object represents one instance of a stream of -// servable versions, all sharing a common name (e.g. "my_servable") and -// increasing version numbers, typically representing updated model parameters -// learned from fresh training data. -// -// A Loader should start in an unloaded state, meaning that no work has been -// done to prepare to perform operations. A typical instance that has not yet -// been loaded contains merely a pointer to a location from which its data can -// be loaded (e.g. a file-system path or network location). Construction and -// destruction of instances should be fairly cheap. Expensive initialization -// operations should be done in Load(). -// -// Subclasses may optionally store a pointer to the Source that originated it, -// for accessing state shared across multiple servable objects in a given -// servable stream. -// -// Implementations need to ensure that the methods they expose are thread-safe, -// or carefully document and/or coordinate their thread-safety properties with -// their clients to ensure correctness. -// Servables do not need to worry about concurrent execution of Load()/Unload() -// as the caller will ensure that does not happen. +/// A standardized abstraction for an object that manages the lifecycle of a +/// servable, including loading and unloading it. Servables are arbitrary +/// objects that serve algorithms or data that often, though not necessarily, +/// use a machine-learned model. +/// +/// A Loader for a servable object represents one instance of a stream of +/// servable versions, all sharing a common name (e.g. "my_servable") and +/// increasing version numbers, typically representing updated model parameters +/// learned from fresh training data. +/// +/// A Loader should start in an unloaded state, meaning that no work has been +/// done to prepare to perform operations. A typical instance that has not yet +/// been loaded contains merely a pointer to a location from which its data can +/// be loaded (e.g. a file-system path or network location). Construction and +/// destruction of instances should be fairly cheap. Expensive initialization +/// operations should be done in Load(). +/// +/// Subclasses may optionally store a pointer to the Source that originated it, +/// for accessing state shared across multiple servable objects in a given +/// servable stream. +/// +/// Implementations need to ensure that the methods they expose are thread-safe, +/// or carefully document and/or coordinate their thread-safety properties with +/// their clients to ensure correctness. +/// Servables do not need to worry about concurrent execution of Load()/Unload() +/// as the caller will ensure that does not happen. class Loader { public: - // The destructor will never be called on a Loader whose servable is currently - // loaded, i.e. between (successful) calls to Load() and Unload(). + /// The destructor will never be called on a Loader whose servable is + /// currently loaded, i.e. between (successful) calls to Load() and Unload(). virtual ~Loader() = default; - // Returns an estimate of the resources the servable will consume once loaded. - // If the servable has already been loaded, returns an estimate of the actual - // resource usage. - // - // IMPORTANT: This method's implementation must obey following requirements, - // which enable the serving system to reason correctly about which servables - // can be loaded safely: - // 1. The estimate must represent an upper bound on the actual value. - // 2. Prior to load, the estimate may include resources that are not bound - // to any specific device instance, e.g. RAM on one of the two GPUs. - // 3. While loaded, for any devices with multiple instances (e.g. two GPUs), - // the estimate must specify the instance to which each resource is bound. - // 4. The estimate must be monotonically non-increasing, i.e. it cannot - // increase over time. + /// Estimates the resources a servable will use. + /// + /// IMPORTANT: This method's implementation must obey following requirements, + /// which enable the serving system to reason correctly about which servables + /// can be loaded safely: + /// 1. The estimate must represent an upper bound on the actual value. + /// 2. Prior to load, the estimate may include resources that are not bound + /// to any specific device instance, e.g. RAM on one of the two GPUs. + /// 3. While loaded, for any devices with multiple instances (e.g. two GPUs), + /// the estimate must specify the instance to which each resource is + /// bound. + /// 4. The estimate must be monotonically non-increasing, i.e. it cannot + /// increase over time. Reasons to have it potentially decrease over time + // include: (a) replace conservative estimate with actual measurement + // once loaded in memory; (b) load process consumes extra transient + // memory that is not used in steady-state after the load completes. + /// + /// @return an estimate of the resources the servable will consume once + /// loaded. If the servable has already been loaded, returns an estimate of + /// the actual resource usage. virtual Status EstimateResources(ResourceAllocation* estimate) const = 0; - // Fetches any data that needs to be loaded before using the servable returned - // by servable(). May use no more resources than the estimate reported by - // EstimateResources(). If that estimate included unbound resources (e.g. 2GB - // of GPU RAM, but not specifying which of two GPU devices to use), then the - // binding of resources to specific device instances must take into account - // the availability on each instance, given by 'available_resources'. - virtual Status Load(const ResourceAllocation& available_resources) = 0; - - // Frees any resources allocated during Load() (except perhaps for resources - // shared across servables that are still needed for other active ones). - // The loader does not need to return to the "new" state (i.e. Load() cannot - // be called after Unload()). + /// Fetches any data that needs to be loaded before using the servable + /// returned by servable(). May use no more resources than the estimate + /// reported by EstimateResources(). + /// + /// If implementing Load(), you don't have to override LoadWithMetadata(). + virtual Status Load() { + return errors::Unimplemented("Load isn't implemented."); + } + + /// The metadata consists of the ServableId. + struct Metadata { + ServableId servable_id; + }; + /// Similar to the above method, but takes Metadata as a param, which + /// may be used by the loader implementation appropriately. + /// + /// If you're overriding LoadWithMetadata(), because you can use the metadata + /// appropriately, you can skip overriding Load(). + virtual Status LoadWithMetadata(const Metadata& metadata) { return Load(); } + + /// Frees any resources allocated during Load() (except perhaps for resources + /// shared across servables that are still needed for other active ones). + /// The loader does not need to return to the "new" state (i.e. Load() cannot + /// be called after Unload()). virtual void Unload() = 0; - // Returns an opaque interface to the underlying servable object. - // The caller should know the precise type of the interface in order to make - // actual use of it. For example: - // - // CustomLoader implementation: - // - // class CustomLoader : public Loader { - // public: - // ... - // Status Load() override { - // servable_ = ...; - // } - // - // AnyPtr servable() override { return servable_; } - // - // private: - // CustomServable* servable_ = nullptr; - // }; - // - // Serving user request: - // - // ServableHandle handle = ... - // CustomServable* servable = handle.get(); - // servable->... - // - // If servable() is called after successful Load() and before Unload(), it - // returns a valid, non-null AnyPtr object. If called before a successful - // Load() call or after Unload(), it returns null AnyPtr. + /// Returns an opaque interface to the underlying servable object. + /// The caller should know the precise type of the interface in order to make + /// actual use of it. For example: + /// + /// CustomLoader implementation: + /// + /// class CustomLoader : public Loader { + /// public: + /// ... + /// Status Load() override { + /// servable_ = ...; + /// } + /// + /// AnyPtr servable() override { return servable_; } + /// + /// private: + /// CustomServable* servable_ = nullptr; + /// }; + /// + /// Serving user request: + /// + /// ServableHandle<CustomServable> handle = ... + /// CustomServable* servable = handle.get(); + /// servable->... + /// + /// If servable() is called after successful Load() and before Unload(), it + /// returns a valid, non-null AnyPtr object. If called before a successful + /// Load() call or after Unload(), it returns null AnyPtr. virtual AnyPtr servable() = 0; }; -// A Loader that is oblivious to resources. Its Load() method does not take -// an 'available_resources' argument. Its EstimateResources() method returns 0, -// thus effectively disabling resource-based safety checks in the serving -// system. -// -// Loaders that are experimental, or run in environments that do not need the -// resource safety checks, can subclass ResourceUnsafeLoader instead of Loader. +inline bool operator==(const Loader::Metadata& a, const Loader::Metadata& b) { + return a.servable_id == b.servable_id; +} + +inline bool operator!=(const Loader::Metadata& a, const Loader::Metadata& b) { + return !(a == b); +} + +/// A Loader that is oblivious to resources. Its EstimateResources() method +/// returns 0, thus effectively disabling resource-based safety checks in the +/// serving system. +/// +/// Loaders that are experimental, or run in environments that do not need the +/// resource safety checks, can subclass ResourceUnsafeLoader instead of Loader. class ResourceUnsafeLoader : public Loader { public: Status EstimateResources(ResourceAllocation* estimate) const final { estimate->Clear(); - return Status::OK(); + return Status(); } - - Status Load(const ResourceAllocation& available_resources) final { - return Load(); - } - - private: - // Subclasses implement this overload, which lacks 'available_resources'. - virtual Status Load() = 0; }; // A source that emits Loader unique pointers. diff --git a/tensorflow_serving/core/loader_harness.cc b/tensorflow_serving/core/loader_harness.cc index 68b4dcf12e4..e3bf3fd95b6 100644 --- a/tensorflow_serving/core/loader_harness.cc +++ b/tensorflow_serving/core/loader_harness.cc @@ -15,10 +15,15 @@ limitations under the License. #include "tensorflow_serving/core/loader_harness.h" -#include +#include +#include +#include +#include "absl/status/status.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/env.h" +#include "tensorflow_serving/util/retrier.h" namespace tensorflow { namespace serving { @@ -29,7 +34,10 @@ LoaderHarness::LoaderHarness(const ServableId& id, : id_(id), loader_(std::move(loader)), additional_state_(nullptr), - options_(options) {} + options_(options), + should_retry_([&](absl::Status status) { return true; }) { + VLOG(1) << "Starting to manage servable version " << id_; +} LoaderHarness::~LoaderHarness() { mutex_lock l(mu_); @@ -47,78 +55,57 @@ Status LoaderHarness::LoadRequested() { mutex_lock l(mu_); if (state_ != State::kNew) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), - " cannot be transitioned to load-requested. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kNew)); + return errors::FailedPrecondition("Duplicate load request"); } state_ = State::kLoadRequested; - return Status::OK(); + VLOG(1) << "Load requested for servable version " << id_; + + return OkStatus(); } Status LoaderHarness::LoadApproved() { mutex_lock l(mu_); - - if (state_ != State::kLoadRequested) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), - " cannot be approved for loading. In state: ", StateDebugString(state_), - " instead of state: ", StateDebugString(State::kLoadRequested)); - } - state_ = State::kLoadApproved; - VLOG(1) << "Approving load for servable version " << id_; - - return Status::OK(); + TF_RETURN_IF_ERROR( + TransitionState(State::kLoadRequested, State::kLoadApproved)); + LOG(INFO) << "Approving load for servable version " << id_; + return OkStatus(); } -Status LoaderHarness::Load(const ResourceAllocation& available_resources) { +Status LoaderHarness::Load() { { mutex_lock l(mu_); - if (state_ != State::kLoadApproved) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), " cannot be loaded. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kLoadApproved)); - } - state_ = State::kLoading; - VLOG(1) << "Loading servable version " << id_; + TF_RETURN_IF_ERROR(TransitionState(State::kLoadApproved, State::kLoading)); + LOG(INFO) << "Loading servable version " << id_; } - const Status status = [&]() { - Status load_status; - int num_tries = 0; - do { - if (num_tries > 0) { - if (cancel_load_retry()) { - LOG(INFO) << "Load retry cancelled for servable: " << id_; - break; - } - Env::Default()->SleepForMicroseconds( - options_.load_retry_interval_micros); - LOG(INFO) << "Retrying load on servable version: " << id_ - << " retry: " << num_tries; + const Status status = Retry( + strings::StrCat("Loading servable: ", id_.DebugString()), + options_.max_num_load_retries, options_.load_retry_interval_micros, + [&]() { return loader_->LoadWithMetadata({id_}); }, + [&](absl::Status status) { return should_retry(status); }); + + if (status.ok()) { + if (!should_retry(absl::UnknownError(""))) { + // Using UnknownError to check if the load is cancelled. If so, it means + // Servable is going to be unloaded very soon, we report a failure here so + // that we do not accidentally report that the servable is available. + TF_RETURN_IF_ERROR(UnloadDueToCancelledLoad()); + absl::Status s = + errors::Cancelled(strings::StrCat("Loading of servable cancelled")); + if (options_.error_callback) { + // Invokes BasicManager::PublishOnEventBus(kEnd). + options_.error_callback(id_, s); } - load_status = loader_->Load(available_resources); - if (!load_status.ok()) { - LOG(ERROR) << "Servable: " << id_ << " load failure: " << load_status; - } - ++num_tries; - } while (!cancel_load_retry() && !load_status.ok() && - (num_tries - 1) < options_.max_num_load_retries); - - return load_status; - }(); - - { - mutex_lock l(mu_); - DCHECK_EQ(State::kLoading, state_); - if (status.ok()) { - state_ = State::kReady; - VLOG(1) << "Successfully loaded servable version " << id_; - } else { - ErrorInternal(status); + return s; } + { + mutex_lock l(mu_); + TF_RETURN_IF_ERROR(TransitionState(State::kLoading, State::kReady)); + LOG(INFO) << "Successfully loaded servable version " << id_; + } + } else { + mutex_lock l(mu_); + ErrorInternal(status); } return status; @@ -126,78 +113,94 @@ Status LoaderHarness::Load(const ResourceAllocation& available_resources) { Status LoaderHarness::UnloadRequested() { mutex_lock l(mu_); - if (state_ != State::kReady) { return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), - " cannot be transitioned to unload-requested. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kReady)); + "Servable not loaded, or unload already requested/ongoing"); } state_ = State::kUnloadRequested; - return Status::OK(); -} - -void LoaderHarness::set_cancel_load_retry(const bool value) { - mutex_lock l(mu_); - cancel_load_retry_ = value; + return OkStatus(); } -bool LoaderHarness::cancel_load_retry() { - mutex_lock l(mu_); - return cancel_load_retry_; -} - -void LoaderHarness::Unload() { +Status LoaderHarness::UnloadInternal(State from_state) { { mutex_lock l(mu_); - DCHECK_EQ(state_, State::kQuiesced); - state_ = State::kUnloading; - VLOG(1) << "Unloading servable version " << id_; + TF_RETURN_IF_ERROR(TransitionState(from_state, State::kUnloading)); + LOG(INFO) << "Unloading just-loaded servable version " << id_; } loader_->Unload(); { mutex_lock l(mu_); - DCHECK_EQ(state_, State::kUnloading); - state_ = State::kDisabled; - VLOG(1) << "Done unloading servable version " << id_; + TF_RETURN_IF_ERROR(TransitionState(State::kUnloading, State::kDisabled)); + LOG(INFO) << "Done unloading servable version " << id_; } + return OkStatus(); +} + +Status LoaderHarness::UnloadDueToCancelledLoad() { + return UnloadInternal(State::kLoading); +} + +void LoaderHarness::set_should_retry( + std::function should_retry) { + mutex_lock l(mu_); + should_retry_ = std::move(should_retry); +} + +bool LoaderHarness::should_retry(absl::Status status) { + mutex_lock l(mu_); + return should_retry_(status); } +Status LoaderHarness::Unload() { return UnloadInternal(State::kQuiesced); } + Status LoaderHarness::StartQuiescing() { mutex_lock l(mu_); - if (state_ != State::kUnloadRequested) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), " cannot be quiesced. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kUnloadRequested)); - } - state_ = State::kQuiescing; - VLOG(1) << "Quiescing servable version " << id_; - return Status::OK(); + TF_RETURN_IF_ERROR( + TransitionState(State::kUnloadRequested, State::kQuiescing)); + LOG(INFO) << "Quiescing servable version " << id_; + return OkStatus(); } -void LoaderHarness::DoneQuiescing() { +Status LoaderHarness::DoneQuiescing() { mutex_lock l(mu_); - DCHECK_EQ(state_, State::kQuiescing); - state_ = State::kQuiesced; - VLOG(1) << "Done quiescing servable version " << id_; + TF_RETURN_IF_ERROR(TransitionState(State::kQuiescing, State::kQuiesced)); + LOG(INFO) << "Done quiescing servable version " << id_; + return OkStatus(); } -void LoaderHarness::ErrorInternal(const Status status) { +void LoaderHarness::ErrorInternal(const Status& status) { state_ = State::kError; status_ = status; - VLOG(1) << "Encountered an error for servable version " << id_ << ": " - << status_; + if (options_.error_callback) { + options_.error_callback(id(), status); + } + LOG(INFO) << "Encountered an error for servable version " << id_ << ": " + << status_; } -void LoaderHarness::Error(const Status status) { +void LoaderHarness::Error(const Status& status) { mutex_lock l(mu_); ErrorInternal(status); } +Status LoaderHarness::TransitionState(const State from, const State to) { + if (state_ != from) { + const Status error = errors::Internal( + "Illegal request to transition from state ", StateDebugString(state_), + " to ", StateDebugString(to)); +#ifndef NDEBUG + LOG(FATAL) << error; // Crash OK +#else + ErrorInternal(error); +#endif + return error; + } + state_ = to; + return OkStatus(); +} + Status LoaderHarness::status() const { mutex_lock l(mu_); return status_; diff --git a/tensorflow_serving/core/loader_harness.h b/tensorflow_serving/core/loader_harness.h index 07209d79213..2329534c052 100644 --- a/tensorflow_serving/core/loader_harness.h +++ b/tensorflow_serving/core/loader_harness.h @@ -18,6 +18,8 @@ limitations under the License. #include +#include "absl/status/status.h" +#include "absl/types/optional.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" @@ -25,86 +27,93 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { +// START_SKIP_DOXYGEN // Forward-declaration for the use in LoaderHarness. template struct ServableStateSnapshot; - -// LoaderHarness is a widget the Manager uses to hold on to and talk to a Loader -// while it owns it. It tracks the overall state of a Servable such that Manager -// can determine which state transitions to make at what times. -// -// A manager implementation can also add some additional state with each -// harness. For example, a manager could put ACL or lifecycle metadata here. The -// ownership is maintained by the harness. -// -// This class is thread-safe. +// END_SKIP_DOXYGEN + +/// LoaderHarness is a widget the Manager uses to hold on to and talk to a +/// Loader while it owns it. It tracks the overall state of a Servable such that +/// Manager can determine which state transitions to make at what times. +/// +/// A manager implementation can also add some additional state with each +/// harness. For example, a manager could put ACL or lifecycle metadata here. +/// The ownership is maintained by the harness. +/// +/// This class is thread-safe. class LoaderHarness final { public: - // State of the underlying servable, from the perspective of the LoaderHarness - // and for the purpose of communication between it and a Manager. Not - // equivalent to the semantic servable states in servable_state.h. - // - // Valid transitions: - // kNew-->kLoading-->kReady-->kQuiescing-->kQuiesced-->kUnloading-->kDisabled - // as well as: any_state-->kError. + /// State of the underlying servable, from the perspective of the + /// LoaderHarness and for the purpose of communication between it and a + /// Manager. Not equivalent to the semantic servable states in + /// servable_state.h. + /// + /// Valid transitions: + /// kNew-->kLoading-->kReady-->kQuiescing-->kQuiesced-->kUnloading-->kDisabled + /// as well as: any_state-->kError. enum class State { - // Initial state. + /// Initial state. kNew, - // The manager has been requested to load this servable. + /// The manager has been requested to load this servable. kLoadRequested, - // The servable has been approved for loading, e.g. resources have been set - // aside for it. + /// The servable has been approved for loading, e.g. resources have been set + /// aside for it. kLoadApproved, - // 'loader_->Load()' has been called. + /// 'loader_->Load()' has been called. kLoading, - // 'loader_->Load()' has succeeded. + /// 'loader_->Load()' has succeeded. kReady, - // The manager has been requested to unload this servable. + /// The manager has been requested to unload this servable. kUnloadRequested, - // The servable is going to be made unavailable for serving. + /// The servable is going to be made unavailable for serving. kQuiescing, - // The servable has been made unavailable for serving. + /// The servable has been made unavailable for serving. kQuiesced, - // 'loader_->Unload()' has been called. + /// 'loader_->Unload()' has been called. kUnloading, - // 'loader_->Unload()' has finished. + /// 'loader_->Unload()' has finished. kDisabled, - // An error has occurred, either during 'loader_->Load()' or outside of the - // harness (and was reported to the harness via a call to Error()). + /// An error has occurred, either during 'loader_->Load()' or outside of the + /// harness (and was reported to the harness via a call to Error()). kError }; + /// Options to configure a LoaderHarness. struct Options { Options() {} - // Maximum number of times we retry loading a servable, after the first - // failure, before we give up. + /// Maximum number of times we retry loading a servable, after the first + /// failure, before we give up. uint32 max_num_load_retries = 0; - // The interval, in microseconds, between each servable load retry. - uint64 load_retry_interval_micros = 0; + /// The interval, in microseconds, between each servable load retry. + uint64_t load_retry_interval_micros = 0; + + /// An (optional) function to call upon transitioning to state kError. + std::function + error_callback; }; LoaderHarness(const ServableId& id, std::unique_ptr loader, const Options& options = Options()); - // Constructor to create a harness with additional state, which a manager - // needs. + /// Constructor to create a harness with additional state, which a manager + /// needs. template LoaderHarness(const ServableId& id, std::unique_ptr loader, std::unique_ptr additional_state, @@ -112,93 +121,103 @@ class LoaderHarness final { : id_(id), loader_(std::move(loader)), additional_state_(std::move(additional_state)), - options_(options) {} + options_(options), + should_retry_([&](absl::Status status) { return true; }) {} - // Legal to destruct iff current state is one of kNew, kDisabled or kError. - // Check-fails if violated. + /// Legal to destruct iff current state is one of kNew, kDisabled or kError. + /// Check-fails if violated. ~LoaderHarness(); - // Returns the identifier of underlying Servable. + /// Returns the identifier of underlying Servable. ServableId id() const { return id_; } - // Returns the current state of underlying Servable. - State state() const LOCKS_EXCLUDED(mu_); + /// Returns the current state of underlying Servable. + State state() const TF_LOCKS_EXCLUDED(mu_); - // Returns a pointer to the wrapped loader. - // Ownership remains with this class. + /// Returns a pointer to the wrapped loader. + /// Ownership remains with this class. Loader* loader() const { return loader_.get(); } - // Returns the current overall state snapshot of the underlying Servable. + /// Returns the current overall state snapshot of the underlying Servable. template - ServableStateSnapshot loader_state_snapshot() const LOCKS_EXCLUDED(mu_); - - // Transitions the state of the harness to kLoadRequested. Returns ok if the - // state was transitioned successfully, else returns an error status. - // - // REQUIRES: That the harness is in state kNew when this method is called. - Status LoadRequested() LOCKS_EXCLUDED(mu_); - - // Transitions to kLoadApproved. - // - // Legal to call iff current state is kLoadRequested. Returns an error status - // if violated. - Status LoadApproved() LOCKS_EXCLUDED(mu_); - - // Transitions to kLoading, delegates to Servable::Load(), then transitions - // either to kReady if Load() succeeds, or to kError if Load() fails. This - // call may take a long time. - // - // We retry the Servable::Load() according to the options set during - // construction of this class. We stop retrying and give up if 1. we have - // reached max_num_load_retries or, 2. if cancel_load() is set to true. - // - // Legal to call iff current state is kApprovedForLoading. Returns an error - // status if violated. - Status Load(const ResourceAllocation& available_resources) - LOCKS_EXCLUDED(mu_); - - // Transitions the state of the harness to kUnloadRequested. Returns ok if the - // state was transitioned successfully, else returns an error status. - // - // REQUIRES: That the harness is in state kReady when this method is called. - Status UnloadRequested() LOCKS_EXCLUDED(mu_); - - // Cancels retrying the load of the servable. This is best-effort, and does - // not preempt a Load() which is already happening, only subsequent calls. - // - // If the retries are cancelled, the servable goes into a state dependent on - // the last Load() called on it. If the last Load() was successful, it will be - // in state kReady, else in kError. - void set_cancel_load_retry(bool value) LOCKS_EXCLUDED(mu_); - bool cancel_load_retry() LOCKS_EXCLUDED(mu_); - - // Transitions to kUnloading, delegates to Servable::Unload(), then - // transitions to kDisabled when Unload() is done. - void Unload() LOCKS_EXCLUDED(mu_); - - // Transitions the state to kQuiescing, which means that we would like to not - // give out any more handles to this servable. - // - // REQUIRES: State be kReady when called, else returns an error status. - Status StartQuiescing() LOCKS_EXCLUDED(mu_); - - // Transitions the state to kQuiesced, which means that there are no more live - // handles to this servable available in the frontend. At this point, we can - // unload this object. - void DoneQuiescing() LOCKS_EXCLUDED(mu_); - - // Transitions the state to kError. - void Error(Status status) LOCKS_EXCLUDED(mu_); - - // Whether anything has gone wrong with this servable. If state is kError, - // this will be non-OK. If not OK, the error could be something that occurred - // in a Source or SourceAdapter, in the Loader, in the Manager, or elsewhere. - // All errors pertaining to the servable are reported here, regardless of - // origin. - Status status() const LOCKS_EXCLUDED(mu_); - - // Gets the additional state. Returns nullptr if the type mismatches or if it - // wasn't set. + ServableStateSnapshot loader_state_snapshot() const TF_LOCKS_EXCLUDED(mu_); + + /// Transitions the state of the harness to kLoadRequested iff its current + /// state is kNew. The test-and-change is done transactionally, so this method + /// can be used to ensure that at most one Load() request can proceed. + Status LoadRequested() TF_LOCKS_EXCLUDED(mu_); + + /// Transitions to kLoadApproved. + /// + /// REQUIRES: State is kLoadRequested when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. + Status LoadApproved() TF_LOCKS_EXCLUDED(mu_); + + /// Transitions to kLoading, delegates to Servable::Load(), then transitions + /// either to kReady if Load() succeeds, or to kError (and invokes 'options_. + /// error_callback') if Load() fails. This call may take a long time. + /// + /// We retry the Servable::Load() according to the options set during + /// construction of this class. We stop retrying and give up if 1. we have + /// reached max_num_load_retries or, 2. if cancel_load() is set to true. + /// + /// REQUIRES: State is kLoadApproved when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. + Status Load() TF_LOCKS_EXCLUDED(mu_); + + /// Transitions the state of the harness to kUnloadRequested iff its current + /// state is kReady. The test-and-change is done transactionally, so this + /// method can be used to ensure that at most one Load() request can proceed. + Status UnloadRequested() TF_LOCKS_EXCLUDED(mu_); + + /// Sets the retry behavior for the servable using a function which accepts + /// the status of the last load attempt and returns a boolean. If the boolean + /// is false, we cancel the next retry. This is best-effort, and does not + /// preempt a Load() which is already happening, only subsequent calls. + /// + /// If the retries are cancelled, the servable goes into a state dependent on + /// the last Load() called on it. If the last Load() was successful, it will + /// be in state kReady, else in kError. + void set_should_retry(std::function should_retry) + TF_LOCKS_EXCLUDED(mu_); + + /// Returns true if the servable should be retried. + bool should_retry(absl::Status status) TF_LOCKS_EXCLUDED(mu_); + + /// Transitions to kUnloading, delegates to Servable::Unload(), then + /// transitions to kDisabled when Unload() is done. + /// + /// REQUIRES: State is kQuiesced when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. + Status Unload() TF_LOCKS_EXCLUDED(mu_); + + /// Transitions the state to kQuiescing, which means that we would like to not + /// give out any more handles to this servable. + /// + /// REQUIRES: State is kUnloadRequested when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. + Status StartQuiescing() TF_LOCKS_EXCLUDED(mu_); + + /// Transitions the state to kQuiesced, which means that there are no more + /// live handles to this servable available in the frontend. At this point, we + /// can unload this object. + /// + /// REQUIRES: State is kQuiescing when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. + Status DoneQuiescing() TF_LOCKS_EXCLUDED(mu_); + + /// Transitions the state to kError and invokes 'options_.error_callback'. + void Error(const Status& status) TF_LOCKS_EXCLUDED(mu_); + + /// Whether anything has gone wrong with this servable. If state is kError, + /// this will be non-OK. If not OK, the error could be something that occurred + /// in a Source or SourceAdapter, in the Loader, in the Manager, or elsewhere. + /// All errors pertaining to the servable are reported here, regardless of + /// origin. + Status status() const TF_LOCKS_EXCLUDED(mu_); + + /// Gets the additional state. Returns nullptr if the type mismatches or if it + /// wasn't set. template T* additional_state() { return additional_state_.get(); @@ -207,9 +226,18 @@ class LoaderHarness final { static string StateDebugString(State state); private: - // Transitions the state to kError. Private method to be used when we want to - // set an error from another method in this class, where mu_ is already held. - void ErrorInternal(Status status) EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Transitions the state to kError and invokes 'options_.error_callback'. + // Private method to be used when we want to set an error from another method + // in this class, where mu_ is already held. + void ErrorInternal(const Status& status) TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); + + // Expects 'state_' to equal 'from', and if so transitions it to 'to'. If not, + // DCHECK-fails, calls ErrorInternal() with a suitable error and returns the + // same error. + Status TransitionState(State from, State to) TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); + + Status UnloadInternal(State from_state) TF_LOCKS_EXCLUDED(mu_); + Status UnloadDueToCancelledLoad() TF_LOCKS_EXCLUDED(mu_); const ServableId id_; const std::unique_ptr loader_; @@ -217,23 +245,27 @@ class LoaderHarness final { const UniqueAnyPtr additional_state_; const Options options_; mutable mutex mu_; - State state_ GUARDED_BY(mu_) = State::kNew; + State state_ TF_GUARDED_BY(mu_) = State::kNew; // If state_ is kError, this will be non-OK. - Status status_ GUARDED_BY(mu_); - // If set to true, we don't try to retry the load of the servable, if not - // loaded by the first attempt. - bool cancel_load_retry_ GUARDED_BY(mu_) = false; + Status status_ TF_GUARDED_BY(mu_); + // The retry policy for the servable. If the function returns false, we cancel + // the next retry. This does not affect the current load action already + // running. + // There is no retry if the last action was successful. + std::function should_retry_ TF_GUARDED_BY(mu_); TF_DISALLOW_COPY_AND_ASSIGN(LoaderHarness); }; +// START_SKIP_DOXYGEN + // A snapshot of a servable's state and aspiredness, from the LoaderHarness's // perspective. template struct ServableStateSnapshot final { ServableId id; LoaderHarness::State state; - optional additional_state; + absl::optional additional_state; }; template @@ -278,6 +310,8 @@ LoaderHarness::loader_state_snapshot() const { return {id_, state_, {}}; } +// END_SKIP_DOXYGEN + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/loader_harness_test.cc b/tensorflow_serving/core/loader_harness_test.cc index e02274d314f..e4b0ce54020 100644 --- a/tensorflow_serving/core/loader_harness_test.cc +++ b/tensorflow_serving/core/loader_harness_test.cc @@ -16,14 +16,18 @@ limitations under the License. #include "tensorflow_serving/core/loader_harness.h" #include +#include #include #include -#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "absl/status/status.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/env.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/servable_id.h" #include "tensorflow_serving/core/test_util/mock_loader.h" #include "tensorflow_serving/test_util/test_util.h" #include "tensorflow_serving/util/any_ptr.h" @@ -32,22 +36,23 @@ namespace tensorflow { namespace serving { namespace { -using ::testing::_; using ::testing::HasSubstr; using ::testing::InvokeWithoutArgs; -using ::testing::InSequence; -using ::testing::IsNull; using ::testing::NiceMock; using ::testing::Return; -using ::testing::ReturnRef; using ::testing::StrictMock; -using test_util::EqualsProto; +// Walks 'harness' through a sequence of transitions from kReady to kDisabled. void QuiesceAndUnload(LoaderHarness* const harness) { - harness->UnloadRequested(); - harness->StartQuiescing(); - harness->DoneQuiescing(); - harness->Unload(); + TF_ASSERT_OK(harness->UnloadRequested()); + TF_ASSERT_OK(harness->StartQuiescing()); + TF_ASSERT_OK(harness->DoneQuiescing()); + TF_ASSERT_OK(harness->Unload()); +} + +// Makes it s.t. it's legal to destruct 'harness'. +void EnableDestruction(LoaderHarness* const harness) { + harness->Error(errors::Unknown("Erroring servable on purpose for shutdown")); } TEST(LoaderHarnessTest, Init) { @@ -72,75 +77,61 @@ TEST(LoaderHarnessTest, LoadRequested) { TEST(LoaderHarnessTest, Quiesce) { test_util::MockLoader* loader = new StrictMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader)); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(Return(absl::OkStatus())); EXPECT_CALL(*loader, Unload()).WillOnce(Return()); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); TF_ASSERT_OK(harness.UnloadRequested()); TF_ASSERT_OK(harness.StartQuiescing()); EXPECT_EQ(LoaderHarness::State::kQuiescing, harness.state()); - harness.DoneQuiescing(); + TF_ASSERT_OK(harness.DoneQuiescing()); EXPECT_EQ(LoaderHarness::State::kQuiesced, harness.state()); // Otherwise we break the dtor invariant and check-fail. - harness.Unload(); + TF_ASSERT_OK(harness.Unload()); } TEST(LoaderHarnessTest, Load) { test_util::MockLoader* loader = new StrictMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Unload()).WillOnce(Return()); + const ServableId servable_id = {"test", 0}; - const auto available_resources = test_util::CreateProto( - "resource_quantities { " - " resource { " - " device: 'main' " - " kind: 'processing' " - " } " - " quantity: 100 " - "} " - "resource_quantities { " - " resource { " - " device: 'gpu' " - " kind: 'ram' " - " } " - " quantity: 4 " - "} "); - - Notification load_called; - Notification load_should_return; - EXPECT_CALL(*loader, Load(EqualsProto(available_resources))) - .WillOnce(Return(Status::OK())); + LoaderHarness harness(servable_id, std::unique_ptr(loader)); + + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(Return(absl::OkStatus())); { - std::unique_ptr test_thread(Env::Default()->StartThread( - ThreadOptions(), "test", [&available_resources, &harness]() { + std::unique_ptr test_thread( + Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - EXPECT_TRUE(harness.Load(available_resources).ok()); + EXPECT_TRUE(harness.Load().ok()); })); // Deleting the thread here forces join and ensures that // LoaderHarness::Load() returns. } EXPECT_EQ(LoaderHarness::State::kReady, harness.state()); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, Unload) { test_util::MockLoader* loader = new StrictMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + const ServableId servable_id = {"test", 0}; + + LoaderHarness harness(servable_id, std::unique_ptr(loader)); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(Return(absl::OkStatus())); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); - Notification unload_called; - Notification unload_should_return; EXPECT_CALL(*loader, Unload()).WillOnce(Return()); { std::unique_ptr test_thread(Env::Default()->StartThread( @@ -153,16 +144,18 @@ TEST(LoaderHarnessTest, Unload) { TEST(LoaderHarnessTest, UnloadRequested) { test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader)); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(Return(absl::OkStatus())); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); TF_ASSERT_OK(harness.UnloadRequested()); EXPECT_EQ(LoaderHarness::State::kUnloadRequested, harness.state()); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, LoadApproved) { @@ -179,33 +172,52 @@ TEST(LoaderHarnessTest, LoadApproved) { TEST(LoaderHarnessTest, LoadError) { test_util::MockLoader* loader = new StrictMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader)); - Notification load_called; - Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) .WillOnce(Return(errors::Unknown("test load error"))); { std::unique_ptr test_thread( Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - Status status = harness.Load(ResourceAllocation()); - EXPECT_THAT(status.error_message(), HasSubstr("test load error")); + absl::Status status = harness.Load(); + EXPECT_THAT(status.message(), HasSubstr("test load error")); })); } EXPECT_EQ(LoaderHarness::State::kError, harness.state()); } TEST(LoaderHarnessTest, ExternallySignalledError) { - LoaderHarness harness(ServableId{"test", 0}, nullptr); + LoaderHarness harness(ServableId{"test", 0}, nullptr /* no loader */); EXPECT_EQ(LoaderHarness::State::kNew, harness.state()); - const Status status = Status(error::UNKNOWN, "Some unknown error"); + const absl::Status status = + absl::Status(static_cast(absl::StatusCode::kUnknown), + "Some unknown error"); harness.Error(status); EXPECT_EQ(LoaderHarness::State::kError, harness.state()); EXPECT_EQ(status, harness.status()); } +TEST(LoaderHarnessTest, ExternallySignalledErrorWithCallback) { + const ServableId id = {"test_servable", 42}; + const absl::Status error = + absl::Status(static_cast(absl::StatusCode::kUnknown), + "Some unknown error"); + Notification callback_called; + LoaderHarness::Options options; + options.error_callback = [&](const ServableId& callback_id, + const absl::Status& callback_error) { + EXPECT_EQ(id, callback_id); + EXPECT_EQ(callback_error, error); + callback_called.Notify(); + }; + LoaderHarness harness(id, nullptr /* no loader */, options); + harness.Error(error); + callback_called.WaitForNotification(); +} + TEST(LoaderHarnessTest, AdditionalState) { std::unique_ptr object(new int(10)); LoaderHarness harness({"test", 42}, nullptr, std::move(object)); @@ -224,85 +236,40 @@ TEST(LoaderHarnessTest, NoAdditionalState) { EXPECT_EQ(nullptr, harness.additional_state()); } -TEST(LoaderHarnessTest, NonApprovedLoadFails) { +TEST(LoaderHarnessTest, MultipleLoadRequestsOnlyFirstOneSucceeds) { test_util::MockLoader* loader = new NiceMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_FALSE(harness.Load(ResourceAllocation()).ok()); -} - -TEST(LoaderHarnessTest, MultipleLoadApprovedOnlyFirstOneSucceeds) { - test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); - TF_ASSERT_OK(harness.LoadRequested()); - TF_ASSERT_OK(harness.LoadApproved()); - const Status second_approve_for_loading_status = harness.LoadApproved(); - EXPECT_FALSE(second_approve_for_loading_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, - second_approve_for_loading_status.code()); - EXPECT_THAT(second_approve_for_loading_status.error_message(), - HasSubstr("cannot be approved for loading")); - - TF_ASSERT_OK(harness.Load(ResourceAllocation())); - QuiesceAndUnload(&harness); -} - -TEST(LoaderHarnessTest, MultipleLoadsOnlyFirstOneSucceeds) { - test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - - EXPECT_CALL(*loader, Load(_)).WillRepeatedly(Return(Status::OK())); - TF_ASSERT_OK(harness.LoadRequested()); - TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); - const Status second_load_status = harness.Load(ResourceAllocation()); - EXPECT_FALSE(second_load_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, second_load_status.code()); - EXPECT_THAT(second_load_status.error_message(), - HasSubstr("cannot be loaded")); - - QuiesceAndUnload(&harness); -} - -TEST(LoaderHarnessTest, MultipleUnloadRequestedOnlyFirstOneSucceeds) { - test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - - TF_ASSERT_OK(harness.LoadRequested()); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); - TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); - - TF_ASSERT_OK(harness.UnloadRequested()); - const Status second_status = harness.UnloadRequested(); - EXPECT_FALSE(second_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, second_status.code()); - EXPECT_THAT(second_status.error_message(), - HasSubstr("cannot be transitioned to unload-requested")); + const absl::Status second_request_status = harness.LoadRequested(); + EXPECT_FALSE(second_request_status.ok()); + EXPECT_EQ(error::FAILED_PRECONDITION, second_request_status.code()); + EXPECT_THAT(second_request_status.message(), + HasSubstr("Duplicate load request")); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } -TEST(LoaderHarnessTest, MultipleStartQuiescingOnlyFirstOneSucceeds) { +TEST(LoaderHarnessTest, MultipleUnloadRequestsOnlyFirstOneSucceeds) { test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader)); TF_ASSERT_OK(harness.LoadRequested()); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(Return(absl::OkStatus())); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); TF_ASSERT_OK(harness.UnloadRequested()); - TF_ASSERT_OK(harness.StartQuiescing()); - const Status second_status = harness.StartQuiescing(); + const absl::Status second_status = harness.UnloadRequested(); EXPECT_FALSE(second_status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, second_status.code()); - EXPECT_THAT(second_status.error_message(), HasSubstr("cannot be quiesced")); + EXPECT_THAT( + second_status.message(), + HasSubstr("Servable not loaded, or unload already requested/ongoing")); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, RetryOnLoadErrorFinallySucceeds) { @@ -310,18 +277,18 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorFinallySucceeds) { LoaderHarness::Options options; options.max_num_load_retries = 1; options.load_retry_interval_micros = 1; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader), - options); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader), options); - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) .WillOnce(InvokeWithoutArgs( []() { return errors::Unknown("test load error"); })) - .WillOnce(InvokeWithoutArgs([]() { return Status::OK(); })); - TF_EXPECT_OK(harness.LoadRequested()); - TF_EXPECT_OK(harness.LoadApproved()); - TF_EXPECT_OK(harness.Load(ResourceAllocation())); + .WillOnce(InvokeWithoutArgs([]() { return absl::OkStatus(); })); + TF_ASSERT_OK(harness.LoadRequested()); + TF_ASSERT_OK(harness.LoadApproved()); + TF_ASSERT_OK(harness.Load()); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, RetryOnLoadErrorFinallyFails) { @@ -329,17 +296,17 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorFinallyFails) { LoaderHarness::Options options; options.max_num_load_retries = 1; options.load_retry_interval_micros = 0; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader), - options); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader), options); - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) .Times(2) .WillRepeatedly(InvokeWithoutArgs( []() { return errors::Unknown("test load error"); })); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - const Status status = harness.Load(ResourceAllocation()); - EXPECT_THAT(status.error_message(), HasSubstr("test load error")); + const absl::Status status = harness.Load(); + EXPECT_THAT(status.message(), HasSubstr("test load error")); } // Tests cancelling load retries. @@ -348,60 +315,70 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorCancelledLoad) { LoaderHarness::Options options; options.max_num_load_retries = 10; options.load_retry_interval_micros = 0; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader), - options); - - Notification load_called; - Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) - .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { - return errors::Unknown("test load error"); - })) - // If the load is called again, we return Status::OK() to fail the test. - .WillRepeatedly(InvokeWithoutArgs([]() { return Status::OK(); })); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader), options); + + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(InvokeWithoutArgs( + []() { return errors::Unknown("test load error"); })) + // If the load is called again, we return OkStatus() to fail the test. + .WillRepeatedly(InvokeWithoutArgs([]() { return absl::OkStatus(); })); std::unique_ptr test_thread( Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - harness.set_cancel_load_retry(true); - const Status status = harness.Load(ResourceAllocation()); - EXPECT_THAT(status.error_message(), HasSubstr("test load error")); + harness.set_should_retry([](absl::Status status) { return false; }); + const absl::Status status = harness.Load(); + EXPECT_THAT(status.message(), HasSubstr("test load error")); })); } -TEST(LoaderHarnessTest, LoadAfterCancelledLoad) { +// Tests unload when ongoing load is cancelled. +TEST(LoaderHarnessTest, UnloadDueToCancelledLoad) { test_util::MockLoader* loader = new NiceMock; - LoaderHarness::Options options; - options.max_num_load_retries = 10; - options.load_retry_interval_micros = 0; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader), - options); - - Notification load_called; - Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) - .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { - load_called.Notify(); - load_should_return.WaitForNotification(); - return errors::Unknown("test load error"); - })) - .WillRepeatedly(InvokeWithoutArgs([]() { return Status::OK(); })); - { - std::unique_ptr test_thread( - Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { - TF_ASSERT_OK(harness.LoadRequested()); - TF_ASSERT_OK(harness.LoadApproved()); - const Status status = harness.Load(ResourceAllocation()); - EXPECT_THAT(status.error_message(), HasSubstr("test load error")); - })); - load_called.WaitForNotification(); - harness.set_cancel_load_retry(true); - load_should_return.Notify(); - } - const Status second_load_status = harness.Load(ResourceAllocation()); - ASSERT_FALSE(second_load_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, second_load_status.code()); + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader)); + + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(InvokeWithoutArgs([]() { + Env::Default()->SleepForMicroseconds(1000000); + return absl::OkStatus(); + })); + + std::unique_ptr test_thread( + Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { + TF_ASSERT_OK(harness.LoadRequested()); + TF_ASSERT_OK(harness.LoadApproved()); + harness.set_should_retry([](absl::Status status) { return false; }); + const absl::Status status = harness.Load(); + EXPECT_THAT(status.message(), HasSubstr("cancelled")); + })); +} + +TEST(LoaderHarnessTest, UnloadDueToNonRetriableError) { + test_util::MockLoader* loader = new NiceMock; + + const ServableId servable_id = {"test", 0}; + LoaderHarness harness(servable_id, std::unique_ptr(loader)); + + EXPECT_CALL(*loader, LoadWithMetadata(Loader::Metadata{servable_id})) + .WillOnce(Return(absl::InvalidArgumentError("Non-retriable error."))) + .WillRepeatedly(InvokeWithoutArgs([]() { + Env::Default()->SleepForMicroseconds(1000000); + return absl::OkStatus(); + })); + + std::unique_ptr test_thread( + Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { + TF_ASSERT_OK(harness.LoadRequested()); + TF_ASSERT_OK(harness.LoadApproved()); + harness.set_should_retry([](absl::Status status) { + return !absl::IsInvalidArgument(status); + }); + const absl::Status status = harness.Load(); + EXPECT_THAT(status.message(), HasSubstr("Non-retriable error.")); + })); } } // namespace diff --git a/tensorflow_serving/core/log_collector.cc b/tensorflow_serving/core/log_collector.cc new file mode 100644 index 00000000000..c8862f1f80a --- /dev/null +++ b/tensorflow_serving/core/log_collector.cc @@ -0,0 +1,85 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/log_collector.h" + +#include +#include + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/strings/scanner.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/thread_annotations.h" + +namespace tensorflow { +namespace serving { +namespace { + +// This class is thread-safe. +class Registry { + public: + absl::Status Register(const string& type, + const LogCollector::Factory& factory) + TF_LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + const auto found_it = factory_map_.find(type); + if (found_it != factory_map_.end()) { + return errors::AlreadyExists("Type ", type, " already registered."); + } + factory_map_.insert({type, factory}); + return absl::OkStatus(); + } + + const LogCollector::Factory* Lookup(const string& type) const + TF_LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + const auto found_it = factory_map_.find(type); + if (found_it == factory_map_.end()) { + return nullptr; + } + return &(found_it->second); + } + + private: + mutable mutex mu_; + std::unordered_map factory_map_ + TF_GUARDED_BY(mu_); +}; + +Registry* GetRegistry() { + static auto* registry = new Registry(); + return registry; +} + +} // namespace + +absl::Status LogCollector::RegisterFactory(const string& type, + const Factory& factory) { + return GetRegistry()->Register(type, factory); +} + +absl::Status LogCollector::Create( + const LogCollectorConfig& config, const uint32 id, + std::unique_ptr* const log_collector) { + auto* factory = GetRegistry()->Lookup(config.type()); + if (factory == nullptr) { + return errors::NotFound("Cannot find LogCollector::Factory for type: ", + config.type()); + } + return (*factory)(config, id, log_collector); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/log_collector.h b/tensorflow_serving/core/log_collector.h new file mode 100644 index 00000000000..8e842124f99 --- /dev/null +++ b/tensorflow_serving/core/log_collector.h @@ -0,0 +1,92 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ +#define TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ + +#include +#include +#include + +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/config/log_collector_config.pb.h" + +namespace tensorflow { +namespace serving { + +// LogCollector defines an abstract interface to use for collecting logs. +// +// Each LogCollector implementation is registered along with a 'type', and if a +// LogCollector is created using this API, we create the LogCollector +// corresponding to the 'type' specified. +class LogCollector { + public: + virtual ~LogCollector() = default; + + // Creates a log-collector for a given 'log_collector_config' and 'id'. The + // factory registered for the type, mentioned in the config, can then be used + // to create the log-collector. The 'id' argument helps in disambiguating logs + // from replicated servers (processes), so it could be a combination of + // task-id and replica-id or process-id and timestamp, etc. + static Status Create(const LogCollectorConfig& log_collector_config, + const uint32 id, + std::unique_ptr* log_collector); + + using Factory = std::function; + // Registers a factory for creating log-collectors for a particular 'type'. + // Returns an error status if a factory is already registered for the + // particular 'type'. + static Status RegisterFactory(const string& type, const Factory& factory); + + // Collects the log as a protocol buffer. + virtual Status CollectMessage(const google::protobuf::Message& message) = 0; + + // Flushes buffered data so that the data can survive an application crash + // (but not an OS crash). + virtual Status Flush() = 0; + + protected: + LogCollector() = default; +}; + +namespace register_log_collector { + +struct RegisterFactory { + RegisterFactory(const string& type, const LogCollector::Factory& factory) { + // This check happens during global object construction time, even before + // control reaches main(), so we are ok with the crash. + TF_CHECK_OK(LogCollector::RegisterFactory(type, factory)); // Crash ok. + } +}; + +} // namespace register_log_collector + +} // namespace serving +} // namespace tensorflow + +#define REGISTER_LOG_COLLECTOR_UNIQ_HELPER(ctr, type, factory) \ + REGISTER_LOG_COLLECTOR_UNIQ(ctr, type, factory) +#define REGISTER_LOG_COLLECTOR_UNIQ(ctr, type, factory) \ + static ::tensorflow::serving::register_log_collector::RegisterFactory \ + register_lgc##ctr TF_ATTRIBUTE_UNUSED = \ + ::tensorflow::serving::register_log_collector::RegisterFactory( \ + type, factory) + +// Registers a LogCollector factory implementation for a type. +#define REGISTER_LOG_COLLECTOR(type, factory) \ + REGISTER_LOG_COLLECTOR_UNIQ_HELPER(__COUNTER__, type, factory) + +#endif // TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ diff --git a/tensorflow_serving/core/log_collector_test.cc b/tensorflow_serving/core/log_collector_test.cc new file mode 100644 index 00000000000..effbc5bdf06 --- /dev/null +++ b/tensorflow_serving/core/log_collector_test.cc @@ -0,0 +1,98 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/log_collector.h" + +#include + +#include +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" +#include "tensorflow_serving/config/log_collector_config.pb.h" + +namespace tensorflow { +namespace serving { +namespace { + +class FakeLogCollector : public LogCollector { + public: + Status CollectMessage(const google::protobuf::Message& message) { return OkStatus(); } + Status Flush() override { return OkStatus(); } +}; + +LogCollectorConfig CreateConfig(const string& type, + const string& filename_prefix) { + LogCollectorConfig config; + config.set_type(type); + config.set_filename_prefix(filename_prefix); + return config; +} + +TEST(LogCollectorTest, NotRegistered) { + std::unique_ptr log_collector; + const auto status = LogCollector::Create( + CreateConfig("notregistered", "filename_prefix"), 0, &log_collector); + EXPECT_EQ(status.code(), error::NOT_FOUND); +} + +TEST(LogCollectorTest, Registration) { + TF_ASSERT_OK(LogCollector::RegisterFactory( + "registered", [](const LogCollectorConfig& config, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return OkStatus(); + })); + std::unique_ptr log_collector; + TF_ASSERT_OK(LogCollector::Create( + CreateConfig("registered", "filename_prefix"), 0, &log_collector)); +} + +auto duplicate_factory = [](const LogCollectorConfig& config, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return OkStatus(); +}; +REGISTER_LOG_COLLECTOR("duplicate", duplicate_factory); + +TEST(LogCollectorTest, DuplicateRegistration) { + const auto status = LogCollector::RegisterFactory( + "duplicate", [](const LogCollectorConfig& config, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return OkStatus(); + }); + EXPECT_EQ(status.code(), error::ALREADY_EXISTS); +} + +auto creation_factory = [](const LogCollectorConfig& config, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return OkStatus(); +}; +REGISTER_LOG_COLLECTOR("creation", creation_factory); + +TEST(LogCollectorTest, Creation) { + std::unique_ptr log_collector0; + TF_ASSERT_OK(LogCollector::Create(CreateConfig("creation", "filename_prefix"), + 0, &log_collector0)); + std::unique_ptr log_collector1; + TF_ASSERT_OK(LogCollector::Create(CreateConfig("creation", "filename_prefix"), + 0, &log_collector1)); + EXPECT_NE(log_collector0.get(), log_collector1.get()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/manager.h b/tensorflow_serving/core/manager.h index 6db9854d741..4eb92aa1541 100644 --- a/tensorflow_serving/core/manager.h +++ b/tensorflow_serving/core/manager.h @@ -22,59 +22,80 @@ limitations under the License. #include #include +#include "absl/types/optional.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { -// A query for a specific loaded servable object. The request can either specify -// a specific version number, or simply opt to use the latest loaded version. +/// A query for a specific loaded servable object. The request can either +/// specify a specific version number, or simply opt to use the earliest or +/// latest loaded version. struct ServableRequest { // Initialization factories, for convenience and readability. - static ServableRequest Specific(const string& name, const int64 version); + static ServableRequest Specific(const string& name, const int64_t version); + static ServableRequest Earliest(const string& name); static ServableRequest Latest(const string& name); static ServableRequest FromId(const ServableId& id); // The name of a servable stream. string name; - // The version number to use. If unset, the largest loaded version is used. - optional version; + // An optional specific version number to use. + absl::optional version; + + // How to choose a version number automatically, if 'version' is left unset. + enum class AutoVersionPolicy { + // Choose the loaded version with the smallest version number. + kEarliest, + + // Choose the loaded version with the largest version number. + kLatest, + }; + AutoVersionPolicy auto_version_policy = AutoVersionPolicy::kLatest; // Emits a string representation; for logging and debugging use only. string DebugString() const; + + //////// + // Legacy constructors. Do not use in new code. + ServableRequest() = default; + ServableRequest(const string& name_in, + const absl::optional& version_in) + : name(name_in), + version(version_in), + auto_version_policy(AutoVersionPolicy::kLatest) {} }; -// Manager is responsible for loading, unloading, lookup and lifetime -// management of all Servable objects via their Loaders. +/// Manager is responsible for loading, unloading, lookup and lifetime +/// management of all Servable objects via their Loaders. class Manager { public: virtual ~Manager() = default; - // Gets a list of all available servable ids, i.e. each of these can - // be retrieved using GetServableHandle. + /// Gets a list of all available servable ids, i.e. each of these can + /// be retrieved using GetServableHandle. virtual std::vector ListAvailableServableIds() const = 0; - // Returns a map of all the currently available servables of a particular type - // T. The map is from the servable's id to its corresponding handle. - // - // IMPORTANT: The caller should not hold onto the handles for a long time, - // because holding them will delay servable loading and unloading. + /// Returns a map of all the currently available servables of a particular + /// type T. The map is from the servable's id to its corresponding handle. + /// + /// IMPORTANT: The caller should not hold onto the handles for a long time, + /// because holding them will delay servable loading and unloading. template std::map> GetAvailableServableHandles() const; - // Returns a ServableHandle given a ServableRequest. Returns error if no such - // Servable is available -- e.g. not yet loaded, has been quiesced/unloaded, - // etc. Callers may assume that an OK status indicates a non-null handle. - // - // IMPORTANT: The caller should not hold onto the handles for a long time, - // because holding them will delay servable loading and unloading. + /// Returns a ServableHandle given a ServableRequest. Returns error if no such + /// Servable is available -- e.g. not yet loaded, has been quiesced/unloaded, + /// etc. Callers may assume that an OK status indicates a non-null handle. + /// + /// IMPORTANT: The caller should not hold onto the handles for a long time, + /// because holding them will delay servable loading and unloading. template Status GetServableHandle(const ServableRequest& request, ServableHandle* const handle); @@ -100,12 +121,25 @@ class Manager { // inline ServableRequest ServableRequest::Specific(const string& name, - const int64 version) { - return ServableRequest{name, version}; + const int64_t version) { + ServableRequest request; + request.name = name; + request.version = version; + return request; +} + +inline ServableRequest ServableRequest::Earliest(const string& name) { + ServableRequest request; + request.name = name; + request.auto_version_policy = AutoVersionPolicy::kEarliest; + return request; } inline ServableRequest ServableRequest::Latest(const string& name) { - return ServableRequest{name, nullopt}; + ServableRequest request; + request.name = name; + request.auto_version_policy = AutoVersionPolicy::kLatest; + return request; } inline ServableRequest ServableRequest::FromId(const ServableId& id) { @@ -117,7 +151,12 @@ inline string ServableRequest::DebugString() const { if (version) { return strings::StrCat("Specific(", name, ", ", version.value(), ")"); } else { - return strings::StrCat("Latest(", name, ")"); + switch (auto_version_policy) { + case AutoVersionPolicy::kEarliest: + return strings::StrCat("Earliest(", name, ")"); + case AutoVersionPolicy::kLatest: + return strings::StrCat("Latest(", name, ")"); + } } } @@ -134,7 +173,7 @@ Status Manager::GetServableHandle(const ServableRequest& request, return errors::InvalidArgument( "Servable type doesn't match the asked for type."); } - return Status::OK(); + return Status(); } template diff --git a/tensorflow_serving/core/manager_test.cc b/tensorflow_serving/core/manager_test.cc index b2747e121f9..b6bbba13bc7 100644 --- a/tensorflow_serving/core/manager_test.cc +++ b/tensorflow_serving/core/manager_test.cc @@ -15,8 +15,13 @@ limitations under the License. #include "tensorflow_serving/core/manager.h" +#include +#include +#include + #include #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow_serving/core/test_util/servable_handle_test_util.h" #include "tensorflow_serving/util/any_ptr.h" namespace tensorflow { @@ -50,7 +55,7 @@ class TestManager : public Manager { const ServableRequest& request, std::unique_ptr* result) override { result->reset(new TestHandle); - return Status::OK(); + return OkStatus(); } std::map> @@ -103,7 +108,7 @@ class ReturnNullManager : public TestManager { const ServableRequest& request, std::unique_ptr* result) override { *result = nullptr; - return Status::OK(); + return OkStatus(); } }; @@ -132,56 +137,13 @@ TEST(ManagerTest, ErrorReturnsNullHandle) { EXPECT_EQ(nullptr, handle.get()); } -// Wraps a pointer as a servable. Does a bit of type-gymnastics since servable -// handles can only be constructed by managers. -template -ServableHandle WrapAsHandle(const ServableId& id, T* t) { - // Perform some type gymnastics to create a handle that points to 't'. - class DummyHandle : public UntypedServableHandle { - public: - explicit DummyHandle(const ServableId& id, T* servable) - : id_(id), servable_(servable) {} - - AnyPtr servable() override { return servable_; } - - const ServableId& id() const override { return id_; } - - private: - const ServableId id_; - T* servable_; - }; - - // Always returns the same servable. - class DummyManager : public TestManager { - public: - explicit DummyManager(const ServableId& id, T* servable) - : id_(id), servable_(servable) {} - - Status GetUntypedServableHandle( - const ServableRequest& request, - std::unique_ptr* result) override { - result->reset(new DummyHandle(id_, servable_)); - return Status::OK(); - } - - private: - const ServableId id_; - T* servable_; - }; - - DummyManager manager{id, t}; - ServableHandle handle; - TF_CHECK_OK(manager.GetServableHandle({"Dummy", 0}, &handle)); - return handle; -} - TEST(ServableHandleTest, PointerOps) { TestServable servables[2]; ServableHandle handles[2]; const ServableId id = {"servable", 7}; - handles[0] = WrapAsHandle(id, &servables[0]); - handles[1] = WrapAsHandle(id, &servables[1]); + handles[0] = test_util::WrapAsHandle(id, &servables[0]); + handles[1] = test_util::WrapAsHandle(id, &servables[1]); // Equality. EXPECT_EQ(handles[0], handles[0]); @@ -203,8 +165,8 @@ TEST(ServableHandleTest, Id) { ServableHandle handles[2]; const ServableId id = {"servable", 7}; - handles[0] = WrapAsHandle(id, &servables[0]); - handles[1] = WrapAsHandle(id, &servables[1]); + handles[0] = test_util::WrapAsHandle(id, &servables[0]); + handles[1] = test_util::WrapAsHandle(id, &servables[1]); EXPECT_EQ(id, handles[0].id()); EXPECT_EQ(id, handles[1].id()); diff --git a/tensorflow_serving/core/manager_wrapper.cc b/tensorflow_serving/core/manager_wrapper.cc index b6bf42ab1ca..054a8f4aab6 100644 --- a/tensorflow_serving/core/manager_wrapper.cc +++ b/tensorflow_serving/core/manager_wrapper.cc @@ -15,6 +15,11 @@ limitations under the License. #include "tensorflow_serving/core/manager_wrapper.h" +#include +#include +#include +#include + namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/core/prefix_storage_path_source_adapter.cc b/tensorflow_serving/core/prefix_storage_path_source_adapter.cc new file mode 100644 index 00000000000..5981fe24af4 --- /dev/null +++ b/tensorflow_serving/core/prefix_storage_path_source_adapter.cc @@ -0,0 +1,38 @@ +/* Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/prefix_storage_path_source_adapter.h" + +#include + +#include "tensorflow/core/platform/path.h" + +namespace tensorflow { +namespace serving { + +PrefixStoragePathSourceAdapter::PrefixStoragePathSourceAdapter( + const std::string& prefix) + : prefix_(prefix) {} + +PrefixStoragePathSourceAdapter::~PrefixStoragePathSourceAdapter() { Detach(); } + +Status PrefixStoragePathSourceAdapter::Convert(const StoragePath& source, + StoragePath* destination) { + *destination = tensorflow::io::JoinPath(prefix_, source); + return OkStatus(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/prefix_storage_path_source_adapter.h b/tensorflow_serving/core/prefix_storage_path_source_adapter.h new file mode 100644 index 00000000000..1c01ebb73af --- /dev/null +++ b/tensorflow_serving/core/prefix_storage_path_source_adapter.h @@ -0,0 +1,49 @@ +/* Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_PREFIX_STORAGE_PATH_SOURCE_ADAPTER_H_ +#define TENSORFLOW_SERVING_CORE_PREFIX_STORAGE_PATH_SOURCE_ADAPTER_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow_serving/core/source_adapter.h" +#include "tensorflow_serving/core/storage_path.h" + +namespace tensorflow { +namespace serving { + +// A SourceAdapter that adds a prefix to StoragePath. +// +// This can be useful for filesystems which wrap other filesystems via namespace +// prefixes, adding additional functionality like buffering for example. +class PrefixStoragePathSourceAdapter final + : public UnarySourceAdapter { + public: + explicit PrefixStoragePathSourceAdapter(const std::string& prefix); + ~PrefixStoragePathSourceAdapter() override; + + protected: + Status Convert(const StoragePath& source, StoragePath* destination) final; + + private: + const std::string prefix_; + + TF_DISALLOW_COPY_AND_ASSIGN(PrefixStoragePathSourceAdapter); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_PREFIX_STORAGE_PATH_SOURCE_ADAPTER_H_ diff --git a/tensorflow_serving/core/prefix_storage_path_source_adapter_test.cc b/tensorflow_serving/core/prefix_storage_path_source_adapter_test.cc new file mode 100644 index 00000000000..9839dd88682 --- /dev/null +++ b/tensorflow_serving/core/prefix_storage_path_source_adapter_test.cc @@ -0,0 +1,58 @@ +/* Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/prefix_storage_path_source_adapter.h" + +#include +#include +#include "tensorflow_serving/core/servable_data.h" +#include "tensorflow_serving/core/storage_path.h" + +namespace tensorflow { +namespace serving { +namespace { + +TEST(PrefixStoragePathSourceAdapterTest, Basic) { + { + PrefixStoragePathSourceAdapter adapter(""); + ServableData output = + adapter.AdaptOneVersion(ServableData({"foo", 42}, "bar")); + EXPECT_EQ("foo", output.id().name); + EXPECT_EQ(42, output.id().version); + EXPECT_EQ("bar", output.DataOrDie()); + } + + { + PrefixStoragePathSourceAdapter adapter("/baz"); + ServableData output = + adapter.AdaptOneVersion(ServableData({"foo", 42}, "bar")); + EXPECT_EQ("foo", output.id().name); + EXPECT_EQ(42, output.id().version); + EXPECT_EQ("/baz/bar", output.DataOrDie()); + } + + { + PrefixStoragePathSourceAdapter adapter("/baz/"); + ServableData output = + adapter.AdaptOneVersion(ServableData({"foo", 42}, "bar")); + EXPECT_EQ("foo", output.id().name); + EXPECT_EQ(42, output.id().version); + EXPECT_EQ("/baz/bar", output.DataOrDie()); + } +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/request_logger.cc b/tensorflow_serving/core/request_logger.cc new file mode 100644 index 00000000000..386b46c6afb --- /dev/null +++ b/tensorflow_serving/core/request_logger.cc @@ -0,0 +1,82 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/request_logger.h" + +#include +#include +#include +#include + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/monitoring/counter.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" +#include "tensorflow_serving/apis/model.pb.h" + +namespace tensorflow { +namespace serving { +namespace { + +auto* request_log_count = monitoring::Counter<2>::New( + "/tensorflow/serving/request_log_count", + "The total number of requests logged from the model server sliced " + "down by model_name and status code.", + "model_name", "status_code"); +} + +RequestLogger::RequestLogger(const LoggingConfig& logging_config, + const std::vector& saved_model_tags, + std::unique_ptr log_collector) + : logging_config_(logging_config), + saved_model_tags_(saved_model_tags), + log_collector_(std::move(log_collector)), + uniform_sampler_() {} + +Status RequestLogger::Log(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata) { + const double sampling_rate = + logging_config_.sampling_config().sampling_rate(); + LogMetadata log_metadata_with_config = log_metadata; + *log_metadata_with_config.mutable_sampling_config() = + logging_config_.sampling_config(); + if (!saved_model_tags_.empty()) { + *log_metadata_with_config.mutable_saved_model_tags() = { + saved_model_tags_.begin(), saved_model_tags_.end()}; + } + if (uniform_sampler_.Sample(sampling_rate)) { + const auto status = [&]() { + std::unique_ptr log; + TF_RETURN_IF_ERROR( + CreateLogMessage(request, response, log_metadata_with_config, &log)); + return Log(*log); + }(); + request_log_count + ->GetCell(log_metadata.model_spec().name(), + error::Code_Name(static_cast(status.code()))) + ->IncrementBy(1); + return status; + } + return OkStatus(); +} + +Status RequestLogger::Log(const google::protobuf::Message& log) { + return log_collector_->CollectMessage(log); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/request_logger.h b/tensorflow_serving/core/request_logger.h new file mode 100644 index 00000000000..e2b1c2b410d --- /dev/null +++ b/tensorflow_serving/core/request_logger.h @@ -0,0 +1,130 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ + +#include +#include + +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/logging.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/log_collector.h" +#include "tensorflow_serving/core/stream_logger.h" + +namespace tensorflow { +namespace serving { + +// Abstraction to log requests and responses hitting a server. The log storage +// is handled by the log-collector. We sample requests based on the config. +// All subclasses must only implement a factory method that returns a +// shared_ptr. +class RequestLogger : public std::enable_shared_from_this { + public: + RequestLogger(const LoggingConfig& logging_config, + const std::vector& saved_model_tags, + std::unique_ptr log_collector); + + virtual ~RequestLogger() = default; + + // Writes the log for the particular request, response and metadata, if we + // decide to sample it. + Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, + const LogMetadata& log_metadata); + + // Starts logging a stream through returning a StreamLogger through + // `get_stream_logger_fn` and registers a log callback. Returns NULL if the + // stream should not be logged. + template + using GetStreamLoggerFn = std::function*()>; + template + void MaybeStartLoggingStream( + const LogMetadata& log_metadata, + GetStreamLoggerFn get_stream_logger_fn); + + const LoggingConfig& logging_config() const { return logging_config_; } + + private: + // Creates the log message given the request, response and metadata. + // Implementations override it to create the particular message that they want + // to be logged. + virtual Status CreateLogMessage(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata, + std::unique_ptr* log) = 0; + + // Implementations can fill up additional information to LogMetadata. + virtual LogMetadata FillLogMetadata(const LogMetadata& lm_in) = 0; + + // Writes the log. + Status Log(const google::protobuf::Message& log); + + // A sampler which samples uniformly at random. + class UniformSampler { + public: + UniformSampler() : rd_(), gen_(rd_()), dist_(0, 1) {} + + // Returns true if the sampler decides to sample it with a probability + // 'rate'. + bool Sample(const double rate) { return dist_(gen_) < rate; } + + private: + std::random_device rd_; + std::mt19937 gen_; + std::uniform_real_distribution dist_; + }; + + const LoggingConfig logging_config_; + const std::vector saved_model_tags_; + std::unique_ptr log_collector_; + UniformSampler uniform_sampler_; +}; + +/**************************Implementation Detail******************************/ +template +void RequestLogger::MaybeStartLoggingStream( + const LogMetadata& log_metadata, + GetStreamLoggerFn get_stream_logger_fn) { + // Sampling happens at the beginning of logging to avoid logging overhead. + // if request logger goes away during a stream which could happen due to + // loggin config update, the stream won't be logged. + if (!uniform_sampler_.Sample( + logging_config_.sampling_config().sampling_rate())) { + return; + } + + auto* stream_logger = get_stream_logger_fn(); + if (stream_logger == nullptr) return; + + LogMetadata lm_out = FillLogMetadata(log_metadata); + std::weak_ptr logger_ref(shared_from_this()); + stream_logger->AddLogCallback( + lm_out, [logger_ref = std::move(logger_ref)](const google::protobuf::Message& log) { + // The callback refers back to the request logger. If the + // request logger goes away after creation but before stream + // ends, we simply skip this sink. + if (auto logger = logger_ref.lock(); logger != nullptr) { + TF_RETURN_IF_ERROR(logger->Log(log)); + } + return OkStatus(); + }); +} + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ diff --git a/tensorflow_serving/core/request_logger_test.cc b/tensorflow_serving/core/request_logger_test.cc new file mode 100644 index 00000000000..ec262a0a907 --- /dev/null +++ b/tensorflow_serving/core/request_logger_test.cc @@ -0,0 +1,185 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/request_logger.h" + +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/wrappers.pb.h" +#include "google/protobuf/message.h" +#include +#include +#include "tensorflow/cc/saved_model/tag_constants.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/logging.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/log_collector.h" +#include "tensorflow_serving/core/test_util/mock_log_collector.h" +#include "tensorflow_serving/core/test_util/mock_prediction_stream_logger.h" +#include "tensorflow_serving/core/test_util/mock_request_logger.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using test_util::MockPredictionStreamLogger; +using ::testing::_; +using ::testing::DoAll; +using ::testing::HasSubstr; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Return; +using ::testing::WithArg; + +class RequestLoggerTest : public ::testing::Test { + protected: + RequestLoggerTest() { + LoggingConfig logging_config; + logging_config.mutable_sampling_config()->set_sampling_rate(1.0); + log_collector_ = new NiceMock(); + request_logger_ = std::shared_ptr>( + new NiceMock(logging_config, model_tags_, + log_collector_)); + } + + const std::vector model_tags_ = {kSavedModelTagServe, + kSavedModelTagTpu}; + NiceMock* log_collector_; + std::shared_ptr> request_logger_; +}; + +TEST_F(RequestLoggerTest, Simple) { + ModelSpec model_spec; + model_spec.set_name("model"); + model_spec.mutable_version()->set_value(10); + + PredictRequest request; + *request.mutable_model_spec() = model_spec; + + PredictResponse response; + response.mutable_outputs()->insert({"tensor", TensorProto()}); + LogMetadata log_metadata; + *log_metadata.mutable_model_spec() = model_spec; + + EXPECT_CALL(*request_logger_, CreateLogMessage(_, _, _, _)) + .WillOnce(Invoke([&](const google::protobuf::Message& actual_request, + const google::protobuf::Message& actual_response, + const LogMetadata& actual_log_metadata, + std::unique_ptr* log) { + EXPECT_THAT(static_cast(actual_request), + test_util::EqualsProto(request)); + EXPECT_THAT(static_cast(actual_response), + test_util::EqualsProto(PredictResponse())); + LogMetadata expected_log_metadata = log_metadata; + expected_log_metadata.mutable_sampling_config()->set_sampling_rate(1.0); + *expected_log_metadata.mutable_saved_model_tags() = { + model_tags_.begin(), model_tags_.end()}; + EXPECT_THAT(actual_log_metadata, + test_util::EqualsProto(expected_log_metadata)); + *log = + std::unique_ptr(new google::protobuf::Any()); + return OkStatus(); + })); + EXPECT_CALL(*log_collector_, CollectMessage(_)).WillOnce(Return(OkStatus())); + TF_ASSERT_OK(request_logger_->Log(request, PredictResponse(), log_metadata)); +} + +TEST_F(RequestLoggerTest, ErroringCreateLogMessage) { + EXPECT_CALL(*request_logger_, CreateLogMessage(_, _, _, _)) + .WillRepeatedly(Return(errors::Internal("Error"))); + EXPECT_CALL(*log_collector_, CollectMessage(_)).Times(0); + const auto error_status = + request_logger_->Log(PredictRequest(), PredictResponse(), LogMetadata()); + ASSERT_FALSE(error_status.ok()); + EXPECT_THAT(error_status.message(), HasSubstr("Error")); +} + +TEST_F(RequestLoggerTest, ErroringCollectMessage) { + EXPECT_CALL(*request_logger_, CreateLogMessage(_, _, _, _)) + .WillRepeatedly(Invoke([&](const google::protobuf::Message& actual_request, + const google::protobuf::Message& actual_response, + const LogMetadata& actual_log_metadata, + std::unique_ptr* log) { + *log = + std::unique_ptr(new google::protobuf::Any()); + return OkStatus(); + })); + EXPECT_CALL(*log_collector_, CollectMessage(_)) + .WillRepeatedly(Return(errors::Internal("Error"))); + const auto error_status = + request_logger_->Log(PredictRequest(), PredictResponse(), LogMetadata()); + ASSERT_FALSE(error_status.ok()); + EXPECT_THAT(error_status.message(), HasSubstr("Error")); +} + +TEST_F(RequestLoggerTest, LoggingStreamSucceeds) { + auto logger = std::make_unique(); + + LogMetadata expected_log_metadata; + expected_log_metadata.mutable_model_spec()->set_name("model"); + EXPECT_CALL(*request_logger_, + FillLogMetadata(test_util::EqualsProto(expected_log_metadata))) + .WillOnce(Return(expected_log_metadata)); + + request_logger_->MaybeStartLoggingStream( + expected_log_metadata, + [logger_ptr = logger.get()]() { return logger_ptr; }); + + EXPECT_CALL(*logger, CreateLogMessage( + test_util::EqualsProto(expected_log_metadata), _)) + .WillOnce(DoAll(WithArg<1>([](std::unique_ptr* log) { + *log = std::make_unique(); + }), + Return(OkStatus()))); + EXPECT_CALL(*log_collector_, CollectMessage(_)).WillOnce(Return(OkStatus())); + TF_ASSERT_OK(logger->LogMessage()); +} + +TEST_F(RequestLoggerTest, LoggingStreamRequestLoggerDiesBeforeStreamCloses) { + auto logger = std::make_unique(); + + LogMetadata expected_log_metadata; + expected_log_metadata.mutable_model_spec()->set_name("model"); + + EXPECT_CALL(*request_logger_, + FillLogMetadata(test_util::EqualsProto(expected_log_metadata))) + .WillOnce(Return(expected_log_metadata)); + request_logger_->MaybeStartLoggingStream( + expected_log_metadata, + [logger_ptr = logger.get()]() { return logger_ptr; }); + + EXPECT_CALL(*logger, CreateLogMessage( + test_util::EqualsProto(expected_log_metadata), _)) + .WillOnce(DoAll(WithArg<1>([](std::unique_ptr* log) { + *log = std::make_unique(); + }), + Return(OkStatus()))); + EXPECT_CALL(*log_collector_, CollectMessage(_)).Times(0); + + request_logger_.reset(); + TF_ASSERT_OK(logger->LogMessage()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/resource_preserving_policy.cc b/tensorflow_serving/core/resource_preserving_policy.cc new file mode 100644 index 00000000000..9cc06bd0454 --- /dev/null +++ b/tensorflow_serving/core/resource_preserving_policy.cc @@ -0,0 +1,64 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/resource_preserving_policy.h" + +#include +#include + +namespace tensorflow { +namespace serving { + +absl::optional +ResourcePreservingPolicy::GetNextAction( + const std::vector& all_versions) const { + // First iterate over all_versions and find any in kReady that are no longer + // aspired. Unload the first if any. + for (const auto& version : all_versions) { + if (version.state == LoaderHarness::State::kReady && !version.is_aspired) { + VLOG(1) << "ResourcePreservingPolicy requesting to unload servable " + << version.id; + return {{Action::kUnload, version.id}}; + } + } + + // Second, see if there are any not-aspired versions that aren't in an end + // state (kDisabled or kError). If so, do nothing for now. + const bool not_aspired_not_finished = + std::any_of(all_versions.begin(), all_versions.end(), + [](const AspiredServableStateSnapshot& version) { + return !version.is_aspired && + version.state != LoaderHarness::State::kDisabled && + version.state != LoaderHarness::State::kError; + }); + if (not_aspired_not_finished) { + return absl::nullopt; + } + + // Third and only if no action was found earlier, load the aspired new + // servable with the highest version if any. + absl::optional highest_aspired_new_version_id = + GetHighestAspiredNewServableId(all_versions); + if (highest_aspired_new_version_id) { + VLOG(1) << "ResourcePreservingPolicy requesting to load servable " + << highest_aspired_new_version_id.value(); + return {{Action::kLoad, highest_aspired_new_version_id.value()}}; + } + + return absl::nullopt; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/eager_unload_policy.h b/tensorflow_serving/core/resource_preserving_policy.h similarity index 80% rename from tensorflow_serving/core/eager_unload_policy.h rename to tensorflow_serving/core/resource_preserving_policy.h index 761cfd2bfb3..6c6a6c8c848 100644 --- a/tensorflow_serving/core/eager_unload_policy.h +++ b/tensorflow_serving/core/resource_preserving_policy.h @@ -13,20 +13,21 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ -#define TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ +#ifndef TENSORFLOW_SERVING_CORE_RESOURCE_PRESERVING_POLICY_H_ +#define TENSORFLOW_SERVING_CORE_RESOURCE_PRESERVING_POLICY_H_ #include +#include "absl/types/optional.h" #include "tensorflow_serving/core/aspired_version_policy.h" #include "tensorflow_serving/core/loader_harness.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { // ServablePolicy that eagerly unloads any no-longer-aspired versions of a -// servable stream and only after done unloading, loads newly aspired versions. +// servable stream and only after done unloading, loads newly aspired versions +// in the order of descending version number. // // This policy minimizes resource consumption with the trade-off of temporary // servable unavailability while all old versions unload followed by the new @@ -38,9 +39,9 @@ namespace serving { // interruptions to a single servable's availability on a replica. // // NB: This policy does not in any way solve cross-replica availability. -class EagerUnloadPolicy final : public AspiredVersionPolicy { +class ResourcePreservingPolicy final : public AspiredVersionPolicy { public: - optional GetNextAction( + absl::optional GetNextAction( const std::vector& all_versions) const override; }; @@ -48,4 +49,4 @@ class EagerUnloadPolicy final : public AspiredVersionPolicy { } // namespace serving } // namespace tensorflow -#endif // TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ +#endif // TENSORFLOW_SERVING_CORE_RESOURCE_PRESERVING_POLICY_H_ diff --git a/tensorflow_serving/core/eager_unload_policy_test.cc b/tensorflow_serving/core/resource_preserving_policy_test.cc similarity index 53% rename from tensorflow_serving/core/eager_unload_policy_test.cc rename to tensorflow_serving/core/resource_preserving_policy_test.cc index 7745d4386a8..cb6bd331aa0 100644 --- a/tensorflow_serving/core/eager_unload_policy_test.cc +++ b/tensorflow_serving/core/resource_preserving_policy_test.cc @@ -13,7 +13,9 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#include "tensorflow_serving/core/eager_unload_policy.h" +#include "tensorflow_serving/core/resource_preserving_policy.h" + +#include #include #include "tensorflow_serving/core/servable_id.h" @@ -23,7 +25,7 @@ namespace serving { namespace { // Test that the first ready and non-aspired version is unloaded first. -TEST(EagerUnloadPolicy, UnloadsFirstNonAspired) { +TEST(ResourcePreservingPolicyTest, UnloadsFirstNonAspired) { std::vector versions; versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); versions.push_back({{"test", 2}, LoaderHarness::State::kReady, true}); @@ -31,15 +33,16 @@ TEST(EagerUnloadPolicy, UnloadsFirstNonAspired) { versions.push_back({{"test", 4}, LoaderHarness::State::kNew, true}); versions.push_back({{"test", 5}, LoaderHarness::State::kReady, false}); - EagerUnloadPolicy policy; + ResourcePreservingPolicy policy; const auto action = policy.GetNextAction(versions); ASSERT_TRUE(action); EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); EXPECT_EQ(3, action->id.version); } -// Test that the first aspired version is loaded when there are none to unload. -TEST(EagerUnloadPolicy, LoadsFirstAspiredWhenNoneToUnload) { +// Test that the aspired new version with the highest version is loaded when +// there are none to unload. +TEST(ResourcePreservingPolicyTest, LoadsFirstAspiredWhenNoneToUnload) { std::vector versions; versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); @@ -47,16 +50,16 @@ TEST(EagerUnloadPolicy, LoadsFirstAspiredWhenNoneToUnload) { versions.push_back({{"test", 4}, LoaderHarness::State::kDisabled, false}); versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); - EagerUnloadPolicy policy; + ResourcePreservingPolicy policy; const auto action = policy.GetNextAction(versions); ASSERT_TRUE(action); EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); - EXPECT_EQ(3, action->id.version); + EXPECT_EQ(5, action->id.version); } // Test that no action is returned (empty optional) when there are no versions // needing loading or unloading. -TEST(EagerUnloadPolicy, ReturnsNoActionWhenNone) { +TEST(ResourcePreservingPolicyTest, ReturnsNoActionWhenNone) { std::vector versions; versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); @@ -64,11 +67,59 @@ TEST(EagerUnloadPolicy, ReturnsNoActionWhenNone) { versions.push_back({{"test", 4}, LoaderHarness::State::kUnloading, false}); versions.push_back({{"test", 5}, LoaderHarness::State::kDisabled, false}); - EagerUnloadPolicy policy; + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(ResourcePreservingPolicyTest, DoesNotLoadWhenOthersStillUnloading) { + std::vector versions; + versions.push_back( + {{"test", 1}, LoaderHarness::State::kUnloadRequested, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; const auto action = policy.GetNextAction(versions); EXPECT_FALSE(action); } +TEST(ResourcePreservingPolicyTest, LoadIfUnaspiredIsError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(2, action->id.version); +} + +TEST(ResourcePreservingPolicyTest, ErrorAndUnloadRequestedPreventLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back( + {{"test", 2}, LoaderHarness::State::kUnloadRequested, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(ResourcePreservingPolicyTest, ErrorAndDisabledAllowLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kDisabled, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(3, action->id.version); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/servable_data.h b/tensorflow_serving/core/servable_data.h index e628e40067d..5771368d08e 100644 --- a/tensorflow_serving/core/servable_data.h +++ b/tensorflow_serving/core/servable_data.h @@ -84,7 +84,7 @@ ServableData CreateServableData(const ServableId& id, T data); template ServableData::ServableData(const ServableId& id, T data) - : id_(id), status_(Status::OK()), data_(std::move(data)) {} + : id_(id), status_(Status()), data_(std::move(data)) {} template ServableData::ServableData(const ServableId& id, const Status& error) diff --git a/tensorflow_serving/core/servable_handle.h b/tensorflow_serving/core/servable_handle.h index e77de60c523..a1d66cf594f 100644 --- a/tensorflow_serving/core/servable_handle.h +++ b/tensorflow_serving/core/servable_handle.h @@ -22,17 +22,16 @@ limitations under the License. #include #include "tensorflow_serving/core/loader.h" -#include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/util/any_ptr.h" namespace tensorflow { namespace serving { -// A non-templatized handle to a servable, used internally in the -// Manager to retrieve a type-erased servable object from the Loader. -// The handle keeps the underlying object alive as long as the handle is alive. -// The frontend should not hold onto it for a long time, because holding it can -// delay servable reloading. +/// A non-templatized handle to a servable, used internally in the +/// Manager to retrieve a type-erased servable object from the Loader. +/// The handle keeps the underlying object alive as long as the handle is alive. +/// The frontend should not hold onto it for a long time, because holding it can +/// delay servable reloading. class UntypedServableHandle { public: virtual ~UntypedServableHandle() = default; @@ -42,34 +41,36 @@ class UntypedServableHandle { virtual AnyPtr servable() = 0; }; -// A smart pointer to the underlying servable object T retrieved from the -// Loader. Frontend code gets these handles from the ServableManager. The -// handle keeps the underlying object alive as long as the handle is alive. The -// frontend should not hold onto it for a long time, because holding it can -// delay servable reloading. -// -// The T returned from the handle is generally shared among multiple requests, -// which means any mutating changes made to T must preserve correctness -// vis-a-vis the application logic. Moreover, in the presence of multiple -// request threads, thread-safe usage of T must be ensured. -// -// T is expected to be a value type, and is internally stored as a pointer. -// Using a pointer type for T will fail to compile, since it would be a mistake -// to do so in most situations. -// -// Example Use: -// // Define or use an existing servable: -// class MyServable { -// public: -// void MyMethod(); -// }; -// -// // Get your handle from a manager. -// ServableHandle handle; -// TF_RETURN_IF_ERROR(manager->GetServableHandle(id, &handle)); -// -// // Use your handle as a smart-pointer: -// handle->MyMethod(); +/// A smart pointer to the underlying servable object T retrieved from the +/// Loader. Frontend code gets these handles from the ServableManager. The +/// handle keeps the underlying object alive as long as the handle is alive. The +/// frontend should not hold onto it for a long time, because holding it can +/// delay servable reloading. +/// +/// The T returned from the handle is generally shared among multiple requests, +/// which means any mutating changes made to T must preserve correctness +/// vis-a-vis the application logic. Moreover, in the presence of multiple +/// request threads, thread-safe usage of T must be ensured. +/// +/// T is expected to be a value type, and is internally stored as a pointer. +/// Using a pointer type for T will fail to compile, since it would be a mistake +/// to do so in most situations. +/// +/// Example use: +/// +/// // Define or use an existing servable: +/// class MyServable { +/// public: +/// void MyMethod(); +/// }; +/// +/// // Get your handle from a manager. +/// ServableHandle handle; +/// TF_RETURN_IF_ERROR(manager->GetServableHandle(id, &handle)); +/// +/// // Use your handle as a smart-pointer: +/// handle->MyMethod(); +/// template class ServableHandle { public: @@ -77,7 +78,7 @@ class ServableHandle { "Servables are implicitly passed as pointers, please use T " "instead of T*."); - // ServableHandle is null by default. + /// ServableHandle is null by default. ServableHandle() = default; const ServableId& id() const { return untyped_handle_->id(); } @@ -108,8 +109,8 @@ class ServableHandle { T* servable_ = nullptr; }; -// An implementation of UntypedServableHandle using shared_ptr to do -// ref-counting on the Loader that owns the Servable. +/// An implementation of UntypedServableHandle using shared_ptr to do +/// ref-counting on the Loader that owns the Servable. class SharedPtrHandle final : public UntypedServableHandle { public: ~SharedPtrHandle() override = default; diff --git a/tensorflow_serving/core/servable_id.h b/tensorflow_serving/core/servable_id.h index 1395bac6eb3..cffa2993eb1 100644 --- a/tensorflow_serving/core/servable_id.h +++ b/tensorflow_serving/core/servable_id.h @@ -41,7 +41,7 @@ struct ServableId { // active servable with the largest version number. // // Must be non-negative. - int64 version; + int64_t version; // Returns a string representation of this object. Useful in logging. string DebugString() const { @@ -50,7 +50,7 @@ struct ServableId { }; struct HashServableId { - uint64 operator()(const ServableId& id) const { + uint64_t operator()(const ServableId& id) const { // Hash codes for many common types are remarkably bad, often clustering // around the same values of the low and/or high bits for linear // sequences of inputs such as 1, 2, 3; or addresses of consecutively @@ -61,9 +61,9 @@ struct HashServableId { // make the high bits contain more entropy from the entire hash code. // It's based on Fibonacci hashing from Knuth's Art of Computer // Programming volume 3, section 6.4. - const uint64 version_hash = [&]() -> uint64 { + const uint64_t version_hash = [&]() -> uint64_t { if (id.version >= 0) { - return std::hash()(id.version) * + return std::hash()(id.version) * 0x9E3779B97F4A7C13; // (sqrt(5) - 1)/2 as a binary fraction. } else { return 0xDECAFCAFFE; diff --git a/tensorflow_serving/core/servable_state.h b/tensorflow_serving/core/servable_state.h index 3fef423b19d..978b71d26c4 100644 --- a/tensorflow_serving/core/servable_state.h +++ b/tensorflow_serving/core/servable_state.h @@ -57,9 +57,27 @@ struct ServableState { // unavailable. kUnloading, - // The manager is done tracking this servable, and is about to delete it. + // This servable has reached the end of its journey in the manager. Either + // it loaded and ultimately unloaded successfully, or it hit an error at + // some point in its lifecycle. kEnd, }; + + static string ManagerStateString(ManagerState state) { + switch (state) { + case ManagerState::kStart: + return "Start"; + case ManagerState::kLoading: + return "Loading"; + case ManagerState::kAvailable: + return "Available"; + case ManagerState::kUnloading: + return "Unloading"; + case ManagerState::kEnd: + return "End"; + } + } + ManagerState manager_state; // Whether anything has gone wrong with this servable. If not OK, the error @@ -71,8 +89,8 @@ struct ServableState { // Returns a string representation of this object. Useful in logging. string DebugString() const { return strings::StrCat("id: ", id.DebugString(), " manager_state: ", - static_cast(manager_state), " health: ", - health.ToString()); + ManagerStateString(manager_state), + " health: ", health.ToString()); } }; diff --git a/tensorflow_serving/core/servable_state_monitor.cc b/tensorflow_serving/core/servable_state_monitor.cc index 0a69e81bf51..46db7821b24 100644 --- a/tensorflow_serving/core/servable_state_monitor.cc +++ b/tensorflow_serving/core/servable_state_monitor.cc @@ -15,7 +15,14 @@ limitations under the License. #include "tensorflow_serving/core/servable_state_monitor.h" +#include +#include +#include + +#include "absl/time/time.h" #include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/gtl/cleanup.h" +#include "tensorflow_serving/core/servable_state.h" namespace tensorflow { namespace serving { @@ -25,20 +32,14 @@ void EraseLiveStatesEntry( const ServableStateMonitor::ServableStateAndTime& state_and_time, ServableStateMonitor::ServableMap* const live_states) { const string& servable_name = state_and_time.state.id.name; - const int64 version = state_and_time.state.id.version; + const int64_t version = state_and_time.state.id.version; auto servable_map_it = live_states->find(servable_name); if (servable_map_it == live_states->end()) { - DCHECK(!state_and_time.state.health.ok()) - << "Servable: " << state_and_time - << " is not in error and directly went to state kEnd."; return; } auto& version_map = servable_map_it->second; auto version_map_it = version_map.find(version); if (version_map_it == version_map.end()) { - DCHECK(!state_and_time.state.health.ok()) - << "Servable: " << state_and_time - << " is not in error and directly went to state kEnd."; return; } @@ -52,7 +53,7 @@ void UpdateLiveStates( const ServableStateMonitor::ServableStateAndTime& state_and_time, ServableStateMonitor::ServableMap* const live_states) { const string& servable_name = state_and_time.state.id.name; - const int64 version = state_and_time.state.id.version; + const int64_t version = state_and_time.state.id.version; if (state_and_time.state.manager_state != ServableState::ManagerState::kEnd) { (*live_states)[servable_name][version] = state_and_time; } else { @@ -62,9 +63,9 @@ void UpdateLiveStates( // Returns the state reached iff the servable has reached 'goal_state' or kEnd, // otherwise nullopt. -optional HasSpecificServableReachedState( +absl::optional HasSpecificServableReachedState( const ServableId& servable_id, const ServableState::ManagerState goal_state, - const optional + const absl::optional opt_servable_state_time) { if (!opt_servable_state_time) { return {}; @@ -79,10 +80,10 @@ optional HasSpecificServableReachedState( // Returns the id of the servable in the stream which has reached 'goal_state' // or kEnd. If no servable has done so, returns nullopt. -optional HasAnyServableInStreamReachedState( +absl::optional HasAnyServableInStreamReachedState( const string& stream_name, const ServableState::ManagerState goal_state, const ServableStateMonitor::ServableMap& states) { - optional opt_servable_id; + absl::optional opt_servable_id; const auto found_it = states.find(stream_name); if (found_it == states.end()) { return {}; @@ -109,39 +110,49 @@ string ServableStateMonitor::ServableStateAndTime::DebugString() const { ServableStateMonitor::ServableStateMonitor(EventBus* bus, const Options& options) - : options_(options), - bus_subscription_(bus->Subscribe( - [this](const EventBus::EventAndTime& state_and_time) { - this->HandleEvent(state_and_time); - })) {} + : options_(options) { + // Important: We must allow the state members ('states_', 'live_states_' and + // so on) to be initialized *before* we start the bus subscription, in case an + // event comes in while we are initializing. + bus_subscription_ = bus->Subscribe( + [this](const EventBus::EventAndTime& state_and_time) { + this->HandleEvent(state_and_time); + }); +} + +ServableStateMonitor::~ServableStateMonitor() { + // Halt event handling first, before tearing down state that event handling + // may access such as 'servable_state_notification_requests_'. + bus_subscription_ = nullptr; +} -optional +absl::optional ServableStateMonitor::GetStateAndTimeInternal( const ServableId& servable_id) const { auto it = states_.find(servable_id.name); if (it == states_.end()) { - return nullopt; + return absl::nullopt; } const VersionMap& versions = it->second; auto it2 = versions.find(servable_id.version); if (it2 == versions.end()) { - return nullopt; + return absl::nullopt; } return it2->second; } -optional +absl::optional ServableStateMonitor::GetStateAndTime(const ServableId& servable_id) const { mutex_lock l(mu_); return GetStateAndTimeInternal(servable_id); } -optional ServableStateMonitor::GetState( +absl::optional ServableStateMonitor::GetState( const ServableId& servable_id) const { - const optional& state_and_time = + const absl::optional& state_and_time = GetStateAndTime(servable_id); if (!state_and_time) { - return nullopt; + return absl::nullopt; } return state_and_time->state; } @@ -168,6 +179,42 @@ ServableStateMonitor::ServableMap ServableStateMonitor::GetLiveServableStates() return live_states_; } +void ServableStateMonitor::ForgetUnloadedServableStates() { + mutex_lock l(mu_); + + for (auto& state : states_) { + std::vector versions_to_forget; + auto& version_map = state.second; + for (const auto& version : version_map) { + if (version.second.state.manager_state == + ServableState::ManagerState::kEnd) { + versions_to_forget.emplace_back(version.first); + } + } + for (const auto& version : versions_to_forget) { + version_map.erase(version); + } + } +} + +ServableStateMonitor::ServableSet +ServableStateMonitor::GetAvailableServableStates() const { + ServableSet available_servable_set; + mutex_lock l(mu_); + for (const auto& live_state : live_states_) { + const string& servable_name = live_state.first; + const auto& version_map = live_state.second; + for (const auto& version : version_map) { + const ServableStateAndTime state_and_time = version.second; + if (state_and_time.state.manager_state == + ServableState::ManagerState::kAvailable) { + available_servable_set.insert(servable_name); + } + } + } + return available_servable_set; +} + ServableStateMonitor::BoundedLog ServableStateMonitor::GetBoundedLog() const { mutex_lock l(mu_); return log_; @@ -180,13 +227,19 @@ void ServableStateMonitor::NotifyWhenServablesReachState( mutex_lock l(mu_); servable_state_notification_requests_.push_back( {servables, goal_state, notifier_fn}); + MaybeSendStateReachedNotifications(); } -bool ServableStateMonitor::WaitUntilServablesReachState( +void ServableStateMonitor::Notify(const NotifyFn& notify_fn) { + mutex_lock l(notify_mu_); + notify_fns_.push_back(notify_fn); +} + +bool ServableStateMonitor::WaitUntilServablesReachStateWithTimeout( const std::vector& servables, - const ServableState::ManagerState goal_state, + const ServableState::ManagerState goal_state, absl::Duration timeout, std::map* const states_reached) { - bool reached_goal_state; + bool reached_goal_state = false; Notification notified; NotifyWhenServablesReachState( servables, goal_state, @@ -199,10 +252,19 @@ bool ServableStateMonitor::WaitUntilServablesReachState( reached_goal_state = incoming_reached_goal_state; notified.Notify(); }); - notified.WaitForNotification(); + notified.WaitForNotificationWithTimeout(timeout); return reached_goal_state; } +bool ServableStateMonitor::WaitUntilServablesReachState( + const std::vector& servables, + const ServableState::ManagerState goal_state, + std::map* const states_reached) { + return WaitUntilServablesReachStateWithTimeout( + servables, goal_state, + /*timeout=*/absl::InfiniteDuration(), states_reached); +} + void ServableStateMonitor::PreHandleEvent( const EventBus::EventAndTime& state_and_time) {} @@ -210,13 +272,16 @@ void ServableStateMonitor::HandleEvent( const EventBus::EventAndTime& event_and_time) { PreHandleEvent(event_and_time); + auto cleanup = + gtl::MakeCleanup([&]() { SendNotifications(event_and_time.event); }); + mutex_lock l(mu_); const ServableStateAndTime state_and_time = { event_and_time.event, event_and_time.event_time_micros}; states_[state_and_time.state.id.name][state_and_time.state.id.version] = state_and_time; UpdateLiveStates(state_and_time, &live_states_); - MaybeSendNotifications(); + MaybeSendStateReachedNotifications(); if (options_.max_count_log_events == 0) { return; @@ -227,8 +292,9 @@ void ServableStateMonitor::HandleEvent( log_.emplace_back(state_and_time.state, state_and_time.event_time_micros); } -optional>> -ServableStateMonitor::ShouldSendNotification( +absl::optional< + std::pair>> +ServableStateMonitor::ShouldSendStateReachedNotification( const ServableStateNotificationRequest& notification_request) { bool reached_goal_state = true; std::map states_reached; @@ -236,7 +302,7 @@ ServableStateMonitor::ShouldSendNotification( if (servable_request.version) { const ServableId servable_id = {servable_request.name, *servable_request.version}; - const optional opt_state = + const absl::optional opt_state = HasSpecificServableReachedState(servable_id, notification_request.goal_state, GetStateAndTimeInternal(servable_id)); @@ -248,7 +314,7 @@ ServableStateMonitor::ShouldSendNotification( reached_goal_state && *opt_state == notification_request.goal_state; states_reached[servable_id] = *opt_state; } else { - const optional opt_servable_id = + const absl::optional opt_servable_id = HasAnyServableInStreamReachedState( servable_request.name, notification_request.goal_state, states_); if (!opt_servable_id) { @@ -265,14 +331,14 @@ ServableStateMonitor::ShouldSendNotification( return {{reached_goal_state, states_reached}}; } -void ServableStateMonitor::MaybeSendNotifications() { +void ServableStateMonitor::MaybeSendStateReachedNotifications() { for (auto iter = servable_state_notification_requests_.begin(); iter != servable_state_notification_requests_.end();) { const ServableStateNotificationRequest& notification_request = *iter; - const optional< + const absl::optional< std::pair>> opt_state_and_states_reached = - ShouldSendNotification(notification_request); + ShouldSendStateReachedNotification(notification_request); if (opt_state_and_states_reached) { notification_request.notifier_fn(opt_state_and_states_reached->first, opt_state_and_states_reached->second); @@ -283,5 +349,13 @@ void ServableStateMonitor::MaybeSendNotifications() { } } +void ServableStateMonitor::SendNotifications( + const ServableState& servable_state) { + mutex_lock l(notify_mu_); + for (const auto& notify_fn : notify_fns_) { + notify_fn(servable_state); + } +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/servable_state_monitor.h b/tensorflow_serving/core/servable_state_monitor.h index e4c4f08c1ba..23f8528e289 100644 --- a/tensorflow_serving/core/servable_state_monitor.h +++ b/tensorflow_serving/core/servable_state_monitor.h @@ -20,6 +20,8 @@ limitations under the License. #include #include +#include "absl/time/time.h" +#include "absl/types/optional.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" @@ -28,35 +30,36 @@ limitations under the License. #include "tensorflow_serving/core/servable_id.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/util/event_bus.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { -// A utility that listens to an EventBus, and keeps track of the -// state of each servable mentioned on the bus. The intended use case is to -// track the states of servables in a Manager. -// -// Offers an interface for querying the servable states. It may be useful as the -// basis for dashboards, as well as for testing a manager. -// -// IMPORTANT: You must create this monitor before arranging for events to be -// published on the event bus, e.g. giving the event bus to a Manager. +/// A utility that listens to an EventBus&lt;ServableState>, and keeps track +/// of the state of each servable mentioned on the bus. The intended use case is +/// to track the states of servables in a Manager. +/// +/// Offers an interface for querying the servable states. It may be useful as +/// the basis for dashboards, as well as for testing a manager. +/// +/// IMPORTANT: You must create this monitor before arranging for events to be +/// published on the event bus, e.g. giving the event bus to a Manager. class ServableStateMonitor { public: + // START_SKIP_DOXYGEN struct ServableStateAndTime { ServableStateAndTime() = default; - ServableStateAndTime(ServableState servable_state, const uint64 event_time) + ServableStateAndTime(ServableState servable_state, + const uint64_t event_time) : state(std::move(servable_state)), event_time_micros(event_time) {} - // State of the servable. + /// State of the servable. ServableState state; - // Time at which servable state event was published. - uint64 event_time_micros; + /// Time at which servable state event was published. + uint64_t event_time_micros; - // Returns a string representation of this struct useful for debugging or - // logging. + /// Returns a string representation of this struct useful for debugging or + /// logging. string DebugString() const; }; @@ -65,90 +68,118 @@ class ServableStateMonitor { using VersionMap = std::map>; using ServableMap = std::map; + using ServableSet = std::set; struct Options { Options() {} - // Upper bound for the number of events captured in the bounded log. If set - // to 0, logging is disabled. - uint64 max_count_log_events = 0; + /// Upper bound for the number of events captured in the bounded log. If set + /// to 0, logging is disabled. + uint64_t max_count_log_events = 0; }; + // END_SKIP_DOXYGEN + using BoundedLog = std::deque; explicit ServableStateMonitor(EventBus* bus, const Options& options = Options()); - virtual ~ServableStateMonitor() = default; + virtual ~ServableStateMonitor(); - // Returns the current state of one servable, or nullopt if that servable is - // not being tracked. - optional GetState(const ServableId& servable_id) const - LOCKS_EXCLUDED(mu_); + /// Returns the current state of one servable, or nullopt if that servable is + /// not being tracked. + absl::optional GetState(const ServableId& servable_id) const + TF_LOCKS_EXCLUDED(mu_); - // Returns the current state and time of one servable, or nullopt if that - // servable is not being tracked. - optional GetStateAndTime( - const ServableId& servable_id) const LOCKS_EXCLUDED(mu_); + /// Returns the current state and time of one servable, or nullopt if that + /// servable is not being tracked. + absl::optional GetStateAndTime( + const ServableId& servable_id) const TF_LOCKS_EXCLUDED(mu_); - // Returns the current states of all tracked versions of the given servable, - // if any. + /// Returns the current states of all tracked versions of the given servable, + /// if any. VersionMap GetVersionStates(const string& servable_name) const - LOCKS_EXCLUDED(mu_); - - // Returns the current states of all tracked versions of all servables. - ServableMap GetAllServableStates() const LOCKS_EXCLUDED(mu_); - - // Returns the current states of all versions of all servables which have not - // transitioned to state ServableState::ManagerState::kEnd. - ServableMap GetLiveServableStates() const LOCKS_EXCLUDED(mu_); - - // Returns the current bounded log of handled servable state events. - BoundedLog GetBoundedLog() const LOCKS_EXCLUDED(mu_); - - // Notifies when all of the servables have reached the 'goal_state'. - // - // Servables can be specified in two ways: - // 1. As specific versions of a servable stream name. In this case, we check - // whether the specific version has reached the 'goal_state' or kEnd. - // 2. As latest versions, in which case any version for a servable stream - // name will be matched against the 'goal_state' or kEnd. - // - // We call the 'notifier_fn' when both conditions are true - - // 1. All of the specific servable requests have either reached the - // 'goal_state' or kEnd. - // 2. All of the latest servable requests have reached 'goal_state' or kEnd. - // The 'notifier_fn' will be called only once, and not repeatedly. - // - // The 'reached_goal_state' argument is set as true iff all of the specific - // servables have reached 'goal_state'. So callers should verify that - // 'reached_goal_state' is true in the 'notifier_fn'. - // - // The 'states_reached' argument is populated with the servable's id and the - // state it reached. The state would be 'goal_state' if 'reached_goal_state' - // is true, else it will contain one or more servables in kEnd state. For - // latest servable requests, the servable id will be the id of the servable in - // the stream which reached the state. + TF_LOCKS_EXCLUDED(mu_); + + /// Returns the current states of all tracked versions of all servables. + ServableMap GetAllServableStates() const TF_LOCKS_EXCLUDED(mu_); + + /// Returns the current states of all versions of all servables which have not + /// transitioned to state ServableState::ManagerState::kEnd. + ServableMap GetLiveServableStates() const TF_LOCKS_EXCLUDED(mu_); + + /// Removes all servable versions from the ServableMap whose + /// states have transitioned to kEnd. + void ForgetUnloadedServableStates() TF_LOCKS_EXCLUDED(mu_); + + // Returns all servables that are in state + // ServableState::ManagerState::kAvailable. + // Note that as opposed to GetAllServableStates() and GetLiveServableStates(), + // this method loops over all the tracked servables. + ServableSet GetAvailableServableStates() const TF_LOCKS_EXCLUDED(mu_); + + /// Returns the current bounded log of handled servable state events. + BoundedLog GetBoundedLog() const TF_LOCKS_EXCLUDED(mu_); + + /// Notifies when all of the servables have reached the 'goal_state'. + /// + /// Servables can be specified in two ways: + /// 1. As specific versions of a servable stream name. In this case, we + /// check whether the specific version has reached the 'goal_state' or kEnd. + /// 2. As latest versions, in which case any version for a servable stream + /// name will be matched against the 'goal_state' or kEnd. + /// + /// We call the 'notifier_fn' when both conditions are true - + /// 1. All of the specific servable requests have either reached the + /// 'goal_state' or kEnd. + /// 2. All of the latest servable requests have reached 'goal_state' or + /// kEnd. + /// The 'notifier_fn' will be called only once, and not repeatedly. + /// + /// The 'reached_goal_state' argument is set as true iff all of the specific + /// servables have reached 'goal_state'. So callers should verify that + /// 'reached_goal_state' is true in the 'notifier_fn'. + /// + /// The 'states_reached' argument is populated with the servable's id and the + /// state it reached. The state would be 'goal_state' if 'reached_goal_state' + /// is true, else it will contain one or more servables in kEnd state. For + /// latest servable requests, the servable id will be the id of the servable + /// in the stream which reached the state. using ServableStateNotifierFn = std::function& states_reached)>; void NotifyWhenServablesReachState( const std::vector& servables, ServableState::ManagerState goal_state, - const ServableStateNotifierFn& notifier_fn) LOCKS_EXCLUDED(mu_); - - // Similar to NotifyWhenServablesReachState(...), but instead of notifying, we - // wait until the 'goal_state' or kEnd is reached. - // - // To understand the return value and the return parameter 'states_reached', - // please read the documentation on NotifyWhenServablesReachState(...). + const ServableStateNotifierFn& notifier_fn) TF_LOCKS_EXCLUDED(mu_); + + /// Similar to NotifyWhenServablesReachState(...), but instead of notifying, + /// we wait until the 'goal_state' or kEnd is reached. + /// + /// To understand the return value and the return parameter 'states_reached', + /// please read the documentation on NotifyWhenServablesReachState(...). + /// WaitUntilServablesReachStateWithTimeout and WaitUntilServablesReachState + /// perform the same function, but the former has a timeout while the latter + /// waits indefinitely. + bool WaitUntilServablesReachStateWithTimeout( + const std::vector& servables, + ServableState::ManagerState goal_state, absl::Duration timeout, + std::map* states_reached = + nullptr) TF_LOCKS_EXCLUDED(mu_) TF_MUST_USE_RESULT; bool WaitUntilServablesReachState( const std::vector& servables, ServableState::ManagerState goal_state, std::map* states_reached = - nullptr) LOCKS_EXCLUDED(mu_) TF_MUST_USE_RESULT; + nullptr) TF_MUST_USE_RESULT; + + // Subscribes to all servable state changes hitting this monitor. This is + // called after the monitor updates its own state based on the event. + using NotifyFn = std::function; + void Notify(const NotifyFn& notify_fn) TF_LOCKS_EXCLUDED(notify_mu_); private: - optional GetStateAndTimeInternal( - const ServableId& servable_id) const EXCLUSIVE_LOCKS_REQUIRED(mu_); + absl::optional + GetStateAndTimeInternal(const ServableId& servable_id) const + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Request to send notification, setup using // NotifyWhenServablesReachState(...). @@ -161,14 +192,20 @@ class ServableStateMonitor { // Checks whether the notification request is satisfied and we cand send it. // If so, returns the 'reached_goal_state' bool and the 'states_reached' by // each servable. Oterwise returns nullopt. - optional>> - ShouldSendNotification( + absl::optional< + std::pair>> + ShouldSendStateReachedNotification( const ServableStateNotificationRequest& notification_request) - EXCLUSIVE_LOCKS_REQUIRED(mu_); + TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); // Goes through the notification requests and tries to see if any of them can // be sent. If a notification is sent, the corresponding request is removed. - void MaybeSendNotifications() EXCLUSIVE_LOCKS_REQUIRED(mu_); + void MaybeSendStateReachedNotifications() TF_EXCLUSIVE_LOCKS_REQUIRED(mu_); + + // Goes through the notify_fns list and calls each one with the currently + // received ServableState. + void SendNotifications(const ServableState& servable_state) + TF_LOCKS_EXCLUDED(notify_mu_); // This method is called when an event comes in, but before we update our // state with the contents of the event. Subclasses may override this method @@ -178,8 +215,8 @@ class ServableStateMonitor { const EventBus::EventAndTime& state_and_time); // Handles a bus event. - void HandleEvent(const EventBus::EventAndTime& state_and_time) - LOCKS_EXCLUDED(mu_); + void HandleEvent(const EventBus::EventAndTime& event_and_time) + TF_LOCKS_EXCLUDED(mu_, notify_mu_); const Options options_; @@ -189,19 +226,26 @@ class ServableStateMonitor { // The current state of each servable version that has appeared on the bus. // (Entries are never removed, even when they enter state kEnd.) - ServableMap states_ GUARDED_BY(mu_); + ServableMap states_ TF_GUARDED_BY(mu_); // The current state of each servable version that has not transitioned to // state ServableState::ManagerState::kEnd. - ServableMap live_states_ GUARDED_BY(mu_); + ServableMap live_states_ TF_GUARDED_BY(mu_); // Deque of pairs of timestamp and ServableState, corresponding to the most // recent servable state events handled by the monitor. The size of this deque // is upper bounded by max_count_log_events in Options. - BoundedLog log_ GUARDED_BY(mu_); + BoundedLog log_ TF_GUARDED_BY(mu_); std::vector - servable_state_notification_requests_ GUARDED_BY(mu_); + servable_state_notification_requests_ TF_GUARDED_BY(mu_); + + // Separate mutex to protect the notify_fns_ so that they can be updated + // independently. This also allows these notify_fns_ to call other methods + // in ServableStateMonitor which don't depend on this mutex without being + // deadlocked. + mutable mutex notify_mu_; + std::vector notify_fns_ TF_GUARDED_BY(notify_mu_); TF_DISALLOW_COPY_AND_ASSIGN(ServableStateMonitor); }; diff --git a/tensorflow_serving/core/servable_state_monitor_test.cc b/tensorflow_serving/core/servable_state_monitor_test.cc index 6a9ace4ed70..8f2b5665d7d 100644 --- a/tensorflow_serving/core/servable_state_monitor_test.cc +++ b/tensorflow_serving/core/servable_state_monitor_test.cc @@ -15,163 +15,184 @@ limitations under the License. #include "tensorflow_serving/core/servable_state_monitor.h" +#include +#include +#include + #include #include +#include "tensorflow/core/kernels/batching_util/fake_clock_env.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" +#include "tensorflow/core/platform/notification.h" +#include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/servable_id.h" +#include "tensorflow_serving/core/servable_state.h" +#include "tensorflow_serving/util/event_bus.h" namespace tensorflow { namespace serving { namespace { using ::testing::ElementsAre; +using ::testing::IsEmpty; using ::testing::Pair; using ::testing::UnorderedElementsAre; using ServableStateAndTime = ServableStateMonitor::ServableStateAndTime; -TEST(ServableStateMonitorTest, AddingStates) { - test_util::FakeClockEnv env(Env::Default()); - EventBus::Options bus_options; - bus_options.env = &env; - auto bus = EventBus::CreateEventBus(bus_options); - - ServableStateMonitor::Options monitor_options; - monitor_options.max_count_log_events = 4; - - ServableStateMonitor monitor(bus.get(), monitor_options); - EXPECT_FALSE(monitor.GetState(ServableId{"foo", 42})); - EXPECT_TRUE(monitor.GetVersionStates("foo").empty()); - EXPECT_TRUE(monitor.GetAllServableStates().empty()); - EXPECT_TRUE(monitor.GetBoundedLog().empty()); +class ServableStateMonitorTest : public ::testing::Test { + protected: + ServableStateMonitorTest() { + env_ = std::make_unique(Env::Default()); + EventBus::Options bus_options; + bus_options.env = env_.get(); + bus_ = EventBus::CreateEventBus(bus_options); + } + void CreateMonitor(int max_count_log_events = 0) { + ServableStateMonitor::Options monitor_options; + monitor_options.max_count_log_events = max_count_log_events; + monitor_ = + std::make_unique(bus_.get(), monitor_options); + } + std::unique_ptr env_; + std::shared_ptr> bus_; + std::unique_ptr monitor_; +}; + +TEST_F(ServableStateMonitorTest, AddingStates) { + CreateMonitor(/*max_count_log_events=*/4); + ServableState notified_state; + monitor_->Notify([&](const ServableState& servable_state) { + notified_state = servable_state; + }); + EXPECT_FALSE(monitor_->GetState(ServableId{"foo", 42})); + EXPECT_TRUE(monitor_->GetVersionStates("foo").empty()); + EXPECT_TRUE(monitor_->GetAllServableStates().empty()); + EXPECT_TRUE(monitor_->GetBoundedLog().empty()); // Initial servable. const ServableState state_0 = { - ServableId{"foo", 42}, ServableState::ManagerState::kStart, Status::OK()}; - env.AdvanceByMicroseconds(1); + ServableId{"foo", 42}, ServableState::ManagerState::kStart, OkStatus()}; + env_->AdvanceByMicroseconds(1); const ServableStateAndTime state_0_and_time = {state_0, 1}; - bus->Publish(state_0); - ASSERT_TRUE(monitor.GetState(ServableId{"foo", 42})); - EXPECT_EQ(state_0, *monitor.GetState(ServableId{"foo", 42})); - EXPECT_FALSE(monitor.GetState(ServableId{"foo", 99})); - EXPECT_FALSE(monitor.GetState(ServableId{"bar", 42})); - EXPECT_THAT(monitor.GetVersionStates("foo"), + bus_->Publish(state_0); + ASSERT_TRUE(monitor_->GetState(ServableId{"foo", 42})); + EXPECT_EQ(state_0, *monitor_->GetState(ServableId{"foo", 42})); + EXPECT_EQ(state_0, notified_state); + EXPECT_FALSE(monitor_->GetState(ServableId{"foo", 99})); + EXPECT_FALSE(monitor_->GetState(ServableId{"bar", 42})); + EXPECT_THAT(monitor_->GetVersionStates("foo"), ElementsAre(Pair(42, state_0_and_time))); - EXPECT_TRUE(monitor.GetVersionStates("bar").empty()); - EXPECT_THAT(monitor.GetAllServableStates(), + EXPECT_TRUE(monitor_->GetVersionStates("bar").empty()); + EXPECT_THAT(monitor_->GetAllServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(42, state_0_and_time))))); - EXPECT_THAT(monitor.GetBoundedLog(), ElementsAre(state_0_and_time)); + EXPECT_THAT(monitor_->GetBoundedLog(), ElementsAre(state_0_and_time)); // New version of existing servable. const ServableState state_1 = {ServableId{"foo", 43}, ServableState::ManagerState::kAvailable, errors::Unknown("error")}; - env.AdvanceByMicroseconds(2); + env_->AdvanceByMicroseconds(2); const ServableStateAndTime state_1_and_time = {state_1, 3}; - bus->Publish(state_1); - ASSERT_TRUE(monitor.GetState(ServableId{"foo", 42})); - EXPECT_EQ(state_0, *monitor.GetState(ServableId{"foo", 42})); - ASSERT_TRUE(monitor.GetState(ServableId{"foo", 43})); - EXPECT_EQ(state_1, *monitor.GetState(ServableId{"foo", 43})); - EXPECT_FALSE(monitor.GetState(ServableId{"foo", 99})); - EXPECT_FALSE(monitor.GetState(ServableId{"bar", 42})); + bus_->Publish(state_1); + ASSERT_TRUE(monitor_->GetState(ServableId{"foo", 42})); + EXPECT_EQ(state_0, *monitor_->GetState(ServableId{"foo", 42})); + ASSERT_TRUE(monitor_->GetState(ServableId{"foo", 43})); + EXPECT_EQ(state_1, *monitor_->GetState(ServableId{"foo", 43})); + EXPECT_EQ(state_1, notified_state); + EXPECT_FALSE(monitor_->GetState(ServableId{"foo", 99})); + EXPECT_FALSE(monitor_->GetState(ServableId{"bar", 42})); EXPECT_THAT( - monitor.GetVersionStates("foo"), + monitor_->GetVersionStates("foo"), ElementsAre(Pair(43, state_1_and_time), Pair(42, state_0_and_time))); - EXPECT_TRUE(monitor.GetVersionStates("bar").empty()); - EXPECT_THAT(monitor.GetAllServableStates(), + EXPECT_TRUE(monitor_->GetVersionStates("bar").empty()); + EXPECT_THAT(monitor_->GetAllServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(43, state_1_and_time), Pair(42, state_0_and_time))))); - EXPECT_THAT(monitor.GetBoundedLog(), + EXPECT_THAT(monitor_->GetBoundedLog(), ElementsAre(state_0_and_time, state_1_and_time)); // New servable name. const ServableState state_2 = {ServableId{"bar", 7}, ServableState::ManagerState::kUnloading, - Status::OK()}; - env.AdvanceByMicroseconds(4); + OkStatus()}; + env_->AdvanceByMicroseconds(4); const ServableStateAndTime state_2_and_time = {state_2, 7}; - bus->Publish(state_2); - ASSERT_TRUE(monitor.GetState(ServableId{"foo", 42})); - EXPECT_EQ(state_0, *monitor.GetState(ServableId{"foo", 42})); - ASSERT_TRUE(monitor.GetState(ServableId{"foo", 43})); - EXPECT_EQ(state_1, *monitor.GetState(ServableId{"foo", 43})); - ASSERT_TRUE(monitor.GetState(ServableId{"bar", 7})); - EXPECT_EQ(state_2, *monitor.GetState(ServableId{"bar", 7})); - EXPECT_FALSE(monitor.GetState(ServableId{"bar", 42})); + bus_->Publish(state_2); + ASSERT_TRUE(monitor_->GetState(ServableId{"foo", 42})); + EXPECT_EQ(state_0, *monitor_->GetState(ServableId{"foo", 42})); + ASSERT_TRUE(monitor_->GetState(ServableId{"foo", 43})); + EXPECT_EQ(state_1, *monitor_->GetState(ServableId{"foo", 43})); + ASSERT_TRUE(monitor_->GetState(ServableId{"bar", 7})); + EXPECT_EQ(state_2, *monitor_->GetState(ServableId{"bar", 7})); + EXPECT_EQ(state_2, notified_state); + EXPECT_FALSE(monitor_->GetState(ServableId{"bar", 42})); EXPECT_THAT( - monitor.GetVersionStates("foo"), + monitor_->GetVersionStates("foo"), ElementsAre(Pair(43, state_1_and_time), Pair(42, state_0_and_time))); - EXPECT_THAT(monitor.GetVersionStates("bar"), + EXPECT_THAT(monitor_->GetVersionStates("bar"), ElementsAre(Pair(7, state_2_and_time))); - EXPECT_TRUE(monitor.GetVersionStates("baz").empty()); - EXPECT_THAT(monitor.GetAllServableStates(), + EXPECT_TRUE(monitor_->GetVersionStates("baz").empty()); + EXPECT_THAT(monitor_->GetAllServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(43, state_1_and_time), Pair(42, state_0_and_time))), Pair("bar", ElementsAre(Pair(7, state_2_and_time))))); EXPECT_THAT( - monitor.GetBoundedLog(), + monitor_->GetBoundedLog(), ElementsAre(state_0_and_time, state_1_and_time, state_2_and_time)); } -TEST(ServableStateMonitorTest, UpdatingStates) { - test_util::FakeClockEnv env(Env::Default()); - EventBus::Options bus_options; - bus_options.env = &env; - auto bus = EventBus::CreateEventBus(bus_options); - - ServableStateMonitor::Options monitor_options; - monitor_options.max_count_log_events = 3; - ServableStateMonitor monitor(bus.get(), monitor_options); +TEST_F(ServableStateMonitorTest, UpdatingStates) { + CreateMonitor(/*max_count_log_events=*/3); // Initial servables. const ServableState state_0 = { - ServableId{"foo", 42}, ServableState::ManagerState::kStart, Status::OK()}; - env.AdvanceByMicroseconds(4); + ServableId{"foo", 42}, ServableState::ManagerState::kStart, OkStatus()}; + env_->AdvanceByMicroseconds(4); const ServableStateAndTime state_0_and_time = {state_0, 4}; - bus->Publish(state_0); + bus_->Publish(state_0); const ServableState state_1 = {ServableId{"foo", 43}, ServableState::ManagerState::kAvailable, errors::Unknown("error")}; const ServableStateAndTime state_1_and_time = {state_1, 4}; - bus->Publish(state_1); + bus_->Publish(state_1); const ServableState state_2 = {ServableId{"bar", 7}, ServableState::ManagerState::kUnloading, - Status::OK()}; + OkStatus()}; const ServableStateAndTime state_2_and_time = {state_2, 4}; - bus->Publish(state_2); - EXPECT_THAT(monitor.GetAllServableStates(), + bus_->Publish(state_2); + EXPECT_THAT(monitor_->GetAllServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(43, state_1_and_time), Pair(42, state_0_and_time))), Pair("bar", ElementsAre(Pair(7, state_2_and_time))))); EXPECT_THAT( - monitor.GetBoundedLog(), + monitor_->GetBoundedLog(), ElementsAre(state_0_and_time, state_1_and_time, state_2_and_time)); // Update one of them. - const ServableState state_1_updated = {ServableId{"foo", 43}, - ServableState::ManagerState::kLoading, - Status::OK()}; - env.AdvanceByMicroseconds(4); + const ServableState state_1_updated = { + ServableId{"foo", 43}, ServableState::ManagerState::kLoading, OkStatus()}; + env_->AdvanceByMicroseconds(4); const ServableStateAndTime state_1_updated_and_time = {state_1_updated, 8}; - bus->Publish(state_1_updated); - ASSERT_TRUE(monitor.GetState(ServableId{"foo", 42})); - EXPECT_EQ(state_0, *monitor.GetState(ServableId{"foo", 42})); - ASSERT_TRUE(monitor.GetState(ServableId{"foo", 43})); - EXPECT_EQ(state_1_updated, *monitor.GetState(ServableId{"foo", 43})); - ASSERT_TRUE(monitor.GetState(ServableId{"bar", 7})); - EXPECT_EQ(state_2, *monitor.GetState(ServableId{"bar", 7})); - EXPECT_THAT(monitor.GetVersionStates("foo"), + bus_->Publish(state_1_updated); + ASSERT_TRUE(monitor_->GetState(ServableId{"foo", 42})); + EXPECT_EQ(state_0, *monitor_->GetState(ServableId{"foo", 42})); + ASSERT_TRUE(monitor_->GetState(ServableId{"foo", 43})); + EXPECT_EQ(state_1_updated, *monitor_->GetState(ServableId{"foo", 43})); + ASSERT_TRUE(monitor_->GetState(ServableId{"bar", 7})); + EXPECT_EQ(state_2, *monitor_->GetState(ServableId{"bar", 7})); + EXPECT_THAT(monitor_->GetVersionStates("foo"), ElementsAre(Pair(43, state_1_updated_and_time), Pair(42, state_0_and_time))); - EXPECT_THAT(monitor.GetVersionStates("bar"), + EXPECT_THAT(monitor_->GetVersionStates("bar"), ElementsAre(Pair(7, state_2_and_time))); - EXPECT_THAT(monitor.GetAllServableStates(), + EXPECT_THAT(monitor_->GetAllServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(43, state_1_updated_and_time), Pair(42, state_0_and_time))), @@ -180,54 +201,45 @@ TEST(ServableStateMonitorTest, UpdatingStates) { // The max count for events logged in the bounded log is 3, so the first entry // corresponding to state_0 is removed and an entry is added for // state_1_updated. - EXPECT_THAT(monitor.GetBoundedLog(), + EXPECT_THAT(monitor_->GetBoundedLog(), ElementsAre(state_1_and_time, state_2_and_time, state_1_updated_and_time)); } -TEST(ServableStateMonitorTest, DisableBoundedLogging) { - test_util::FakeClockEnv env(Env::Default()); - EventBus::Options bus_options; - bus_options.env = &env; - auto bus = EventBus::CreateEventBus(bus_options); - +TEST_F(ServableStateMonitorTest, DisableBoundedLogging) { // The default value for max_count_log_events in options is 0, which disables // logging. - ServableStateMonitor monitor(bus.get()); + CreateMonitor(); const ServableState state_0 = { - ServableId{"foo", 42}, ServableState::ManagerState::kStart, Status::OK()}; - env.AdvanceByMicroseconds(1); + ServableId{"foo", 42}, ServableState::ManagerState::kStart, OkStatus()}; + env_->AdvanceByMicroseconds(1); const ServableStateAndTime state_0_and_time = {state_0, 1}; - bus->Publish(state_0); - EXPECT_THAT(monitor.GetAllServableStates(), + bus_->Publish(state_0); + EXPECT_THAT(monitor_->GetAllServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(42, state_0_and_time))))); - EXPECT_TRUE(monitor.GetBoundedLog().empty()); + EXPECT_TRUE(monitor_->GetBoundedLog().empty()); } -TEST(ServableStateMonitorTest, GetLiveServableStates) { - test_util::FakeClockEnv env(Env::Default()); - EventBus::Options bus_options; - bus_options.env = &env; - auto bus = EventBus::CreateEventBus(bus_options); - ServableStateMonitor monitor(bus.get()); +TEST_F(ServableStateMonitorTest, GetLiveServableStates) { + CreateMonitor(); const ServableState state_0 = { - ServableId{"foo", 42}, ServableState::ManagerState::kStart, Status::OK()}; - env.AdvanceByMicroseconds(1); + ServableId{"foo", 42}, ServableState::ManagerState::kStart, OkStatus()}; + env_->AdvanceByMicroseconds(1); const ServableStateAndTime state_0_and_time = {state_0, 1}; - bus->Publish(state_0); - EXPECT_THAT(monitor.GetLiveServableStates(), + bus_->Publish(state_0); + EXPECT_THAT(monitor_->GetLiveServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(42, state_0_and_time))))); const ServableState state_1 = {ServableId{"bar", 7}, ServableState::ManagerState::kAvailable, - Status::OK()}; - env.AdvanceByMicroseconds(1); + OkStatus()}; + env_->AdvanceByMicroseconds(1); const ServableStateAndTime state_1_and_time = {state_1, 2}; - bus->Publish(state_1); - EXPECT_THAT(monitor.GetLiveServableStates(), + bus_->Publish(state_1); + EXPECT_THAT(monitor_->GetLiveServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(42, state_0_and_time))), Pair("bar", ElementsAre(Pair(7, state_1_and_time))))); @@ -235,54 +247,163 @@ TEST(ServableStateMonitorTest, GetLiveServableStates) { // Servable {foo, 42} moves to state kEnd and is removed from the live states // servables. const ServableState state_0_update = { - ServableId{"foo", 42}, ServableState::ManagerState::kEnd, Status::OK()}; - env.AdvanceByMicroseconds(1); - bus->Publish(state_0_update); - EXPECT_THAT(monitor.GetLiveServableStates(), + ServableId{"foo", 42}, ServableState::ManagerState::kEnd, OkStatus()}; + env_->AdvanceByMicroseconds(1); + bus_->Publish(state_0_update); + EXPECT_THAT(monitor_->GetLiveServableStates(), UnorderedElementsAre( Pair("bar", ElementsAre(Pair(7, state_1_and_time))))); } -TEST(ServableStateMonitorTest, VersionMapDescendingOrder) { - test_util::FakeClockEnv env(Env::Default()); - EventBus::Options bus_options; - bus_options.env = &env; - auto bus = EventBus::CreateEventBus(bus_options); - ServableStateMonitor monitor(bus.get()); +TEST_F(ServableStateMonitorTest, GetAvailableServableStates) { + CreateMonitor(); + + const ServableState state_0 = { + ServableId{"foo", 42}, ServableState::ManagerState::kStart, OkStatus()}; + env_->AdvanceByMicroseconds(1); + const ServableStateAndTime state_0_and_time = {state_0, 1}; + bus_->Publish(state_0); + EXPECT_THAT(monitor_->GetAvailableServableStates(), testing::IsEmpty()); + + env_->AdvanceByMicroseconds(1); + std::vector servable_state_and_time; + for (const auto& servable_id : {ServableId{"bar", 6}, ServableId{"bar", 7}}) { + const ServableState state = { + servable_id, ServableState::ManagerState::kAvailable, OkStatus()}; + const ServableStateAndTime state_and_time = {state, 2}; + servable_state_and_time.push_back({state, 2}); + bus_->Publish(state); + } + + EXPECT_THAT(monitor_->GetAvailableServableStates(), + UnorderedElementsAre("bar")); + + // Servable {bar, 6} moves to state kUnloading and is removed from available + // servable states. + const ServableState state_0_update = {ServableId{"bar", 6}, + ServableState::ManagerState::kUnloading, + OkStatus()}; + env_->AdvanceByMicroseconds(1); + bus_->Publish(state_0_update); + EXPECT_THAT(monitor_->GetAvailableServableStates(), + UnorderedElementsAre("bar")); + // Servable {bar, 7} moves to state kEnd and is removed from available + // servable states. + const ServableState state_1_update = { + ServableId{"bar", 7}, ServableState::ManagerState::kEnd, OkStatus()}; + env_->AdvanceByMicroseconds(1); + bus_->Publish(state_1_update); + // No available state now. + EXPECT_THAT(monitor_->GetAvailableServableStates(), ::testing::IsEmpty()); +} + +TEST_F(ServableStateMonitorTest, VersionMapDescendingOrder) { + CreateMonitor(); const ServableState state_0 = { - ServableId{"foo", 42}, ServableState::ManagerState::kStart, Status::OK()}; - env.AdvanceByMicroseconds(1); + ServableId{"foo", 42}, ServableState::ManagerState::kStart, OkStatus()}; + env_->AdvanceByMicroseconds(1); const ServableStateAndTime state_0_and_time = {state_0, 1}; - bus->Publish(state_0); - EXPECT_THAT(monitor.GetLiveServableStates(), + bus_->Publish(state_0); + EXPECT_THAT(monitor_->GetLiveServableStates(), UnorderedElementsAre( Pair("foo", ElementsAre(Pair(42, state_0_and_time))))); const ServableState state_1 = {ServableId{"foo", 7}, ServableState::ManagerState::kAvailable, - Status::OK()}; - env.AdvanceByMicroseconds(1); + OkStatus()}; + env_->AdvanceByMicroseconds(1); const ServableStateAndTime state_1_and_time = {state_1, 2}; - bus->Publish(state_1); - EXPECT_THAT(monitor.GetLiveServableStates(), + bus_->Publish(state_1); + EXPECT_THAT(monitor_->GetLiveServableStates(), ElementsAre(Pair("foo", ElementsAre(Pair(42, state_0_and_time), Pair(7, state_1_and_time))))); } -TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateSpecificAvailable) { - auto bus = EventBus::CreateEventBus({}); - ServableStateMonitor monitor(bus.get()); +TEST_F(ServableStateMonitorTest, ForgetUnloadedServableStates) { + CreateMonitor(); + + const ServableState state_0 = {ServableId{"foo", 42}, + ServableState::ManagerState::kAvailable, + OkStatus()}; + env_->AdvanceByMicroseconds(1); + const ServableStateAndTime state_0_and_time = {state_0, 1}; + bus_->Publish(state_0); + EXPECT_THAT(monitor_->GetLiveServableStates(), + UnorderedElementsAre( + Pair("foo", ElementsAre(Pair(42, state_0_and_time))))); + + const ServableState state_1 = {ServableId{"bar", 1}, + ServableState::ManagerState::kAvailable, + OkStatus()}; + env_->AdvanceByMicroseconds(1); + const ServableStateAndTime state_1_and_time = {state_1, 2}; + bus_->Publish(state_1); + EXPECT_THAT(monitor_->GetLiveServableStates(), + UnorderedElementsAre( + Pair("foo", ElementsAre(Pair(42, state_0_and_time))), + Pair("bar", ElementsAre(Pair(1, state_1_and_time))))); + + const ServableState state_2 = {ServableId{"foo", 42}, + ServableState::ManagerState::kUnloading, + OkStatus()}; + env_->AdvanceByMicroseconds(1); + const ServableStateAndTime state_2_and_time = {state_2, 3}; + bus_->Publish(state_2); + monitor_->ForgetUnloadedServableStates(); + // "foo" state should still be recorded since it hasn't reached kEnd. + EXPECT_THAT(monitor_->GetAllServableStates(), + UnorderedElementsAre( + Pair("foo", ElementsAre(Pair(42, state_2_and_time))), + Pair("bar", ElementsAre(Pair(1, state_1_and_time))))); + + const ServableState state_3 = {ServableId{"foo", 42}, + ServableState::ManagerState::kEnd, OkStatus()}; + env_->AdvanceByMicroseconds(1); + const ServableStateAndTime state_3_and_time = {state_3, 4}; + bus_->Publish(state_3); + EXPECT_THAT(monitor_->GetAllServableStates(), + UnorderedElementsAre( + Pair("foo", ElementsAre(Pair(42, state_3_and_time))), + Pair("bar", ElementsAre(Pair(1, state_1_and_time))))); + monitor_->ForgetUnloadedServableStates(); + EXPECT_THAT(monitor_->GetAllServableStates(), + UnorderedElementsAre( + Pair("foo", IsEmpty()), + Pair("bar", ElementsAre(Pair(1, state_1_and_time))))); +} + +TEST_F(ServableStateMonitorTest, NotifyWhenServablesReachStateZeroServables) { + CreateMonitor(); + const std::vector servables = {}; + + using ManagerState = ServableState::ManagerState; + + Notification notified; + monitor_->NotifyWhenServablesReachState( + servables, ManagerState::kAvailable, + [&](const bool reached, + std::map states_reached) { + EXPECT_TRUE(reached); + EXPECT_THAT(states_reached, IsEmpty()); + notified.Notify(); + }); + notified.WaitForNotification(); +} + +TEST_F(ServableStateMonitorTest, + NotifyWhenServablesReachStateSpecificAvailable) { + CreateMonitor(); std::vector servables; const ServableId specific_goal_state_id = {"specific_goal_state", 42}; servables.push_back(ServableRequest::FromId(specific_goal_state_id)); using ManagerState = ServableState::ManagerState; const ServableState specific_goal_state = { - specific_goal_state_id, ManagerState::kAvailable, Status::OK()}; + specific_goal_state_id, ManagerState::kAvailable, OkStatus()}; Notification notified; - monitor.NotifyWhenServablesReachState( + monitor_->NotifyWhenServablesReachState( servables, ManagerState::kAvailable, [&](const bool reached, std::map states_reached) { @@ -292,13 +413,12 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateSpecificAvailable) { ManagerState::kAvailable))); notified.Notify(); }); - bus->Publish(specific_goal_state); + bus_->Publish(specific_goal_state); notified.WaitForNotification(); } -TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateSpecificError) { - auto bus = EventBus::CreateEventBus({}); - ServableStateMonitor monitor(bus.get()); +TEST_F(ServableStateMonitorTest, NotifyWhenServablesReachStateSpecificError) { + CreateMonitor(); std::vector servables; const ServableId specific_error_state_id = {"specific_error_state", 42}; servables.push_back(ServableRequest::FromId(specific_error_state_id)); @@ -308,7 +428,7 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateSpecificError) { specific_error_state_id, ManagerState::kEnd, errors::Internal("error")}; Notification notified; - monitor.NotifyWhenServablesReachState( + monitor_->NotifyWhenServablesReachState( servables, ManagerState::kAvailable, [&](const bool reached, std::map states_reached) { @@ -318,25 +438,23 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateSpecificError) { Pair(specific_error_state_id, ManagerState::kEnd))); notified.Notify(); }); - bus->Publish(specific_error_state); + bus_->Publish(specific_error_state); notified.WaitForNotification(); } -TEST(ServableStateMonitorTest, - NotifyWhenServablesReachStateServableLatestAvailable) { - auto bus = EventBus::CreateEventBus({}); - ServableStateMonitor monitor(bus.get()); +TEST_F(ServableStateMonitorTest, + NotifyWhenServablesReachStateServableLatestAvailable) { + CreateMonitor(); std::vector servables; servables.push_back(ServableRequest::Latest("servable_stream")); const ServableId servable_stream_available_state_id = {"servable_stream", 42}; using ManagerState = ServableState::ManagerState; const ServableState servable_stream_available_state = { - servable_stream_available_state_id, ManagerState::kAvailable, - Status::OK()}; + servable_stream_available_state_id, ManagerState::kAvailable, OkStatus()}; Notification notified; - monitor.NotifyWhenServablesReachState( + monitor_->NotifyWhenServablesReachState( servables, ManagerState::kAvailable, [&](const bool reached, std::map states_reached) { @@ -346,13 +464,12 @@ TEST(ServableStateMonitorTest, ManagerState::kAvailable))); notified.Notify(); }); - bus->Publish(servable_stream_available_state); + bus_->Publish(servable_stream_available_state); notified.WaitForNotification(); } -TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateLatestError) { - auto bus = EventBus::CreateEventBus({}); - ServableStateMonitor monitor(bus.get()); +TEST_F(ServableStateMonitorTest, NotifyWhenServablesReachStateLatestError) { + CreateMonitor(); std::vector servables; servables.push_back(ServableRequest::Latest("servable_stream")); const ServableId servable_stream_error_state_id = {"servable_stream", 7}; @@ -363,7 +480,7 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateLatestError) { errors::Internal("error")}; Notification notified; - monitor.NotifyWhenServablesReachState( + monitor_->NotifyWhenServablesReachState( servables, ManagerState::kAvailable, [&](const bool reached, std::map states_reached) { @@ -373,15 +490,15 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateLatestError) { ManagerState::kEnd))); notified.Notify(); }); - bus->Publish(servable_stream_error_state); + bus_->Publish(servable_stream_error_state); notified.WaitForNotification(); } -TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateFullFunctionality) { +TEST_F(ServableStateMonitorTest, + NotifyWhenServablesReachStateFullFunctionality) { using ManagerState = ServableState::ManagerState; - auto bus = EventBus::CreateEventBus({}); - ServableStateMonitor monitor(bus.get()); + CreateMonitor(); std::vector servables; const ServableId specific_goal_state_id = {"specific_goal_state", 42}; servables.push_back(ServableRequest::FromId(specific_goal_state_id)); @@ -391,7 +508,7 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateFullFunctionality) { const ServableId servable_stream_id = {"servable_stream", 7}; Notification notified; - monitor.NotifyWhenServablesReachState( + monitor_->NotifyWhenServablesReachState( servables, ManagerState::kAvailable, [&](const bool reached, std::map states_reached) { @@ -405,33 +522,33 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateFullFunctionality) { }); const ServableState specific_goal_state = { - specific_goal_state_id, ManagerState::kAvailable, Status::OK()}; + specific_goal_state_id, ManagerState::kAvailable, OkStatus()}; const ServableState specific_error_state = { specific_error_state_id, ManagerState::kEnd, errors::Internal("error")}; const ServableState servable_stream_state = { - servable_stream_id, ManagerState::kAvailable, Status::OK()}; + servable_stream_id, ManagerState::kAvailable, OkStatus()}; - bus->Publish(specific_goal_state); + bus_->Publish(specific_goal_state); ASSERT_FALSE(notified.HasBeenNotified()); - bus->Publish(specific_error_state); + bus_->Publish(specific_error_state); ASSERT_FALSE(notified.HasBeenNotified()); - bus->Publish(servable_stream_state); + bus_->Publish(servable_stream_state); notified.WaitForNotification(); } -TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateOnlyNotifiedOnce) { - auto bus = EventBus::CreateEventBus({}); - ServableStateMonitor monitor(bus.get()); +TEST_F(ServableStateMonitorTest, + NotifyWhenServablesReachStateOnlyNotifiedOnce) { + CreateMonitor(); std::vector servables; const ServableId specific_goal_state_id = {"specific_goal_state", 42}; servables.push_back(ServableRequest::FromId(specific_goal_state_id)); using ManagerState = ServableState::ManagerState; const ServableState specific_goal_state = { - specific_goal_state_id, ManagerState::kAvailable, Status::OK()}; + specific_goal_state_id, ManagerState::kAvailable, OkStatus()}; Notification notified; - monitor.NotifyWhenServablesReachState( + monitor_->NotifyWhenServablesReachState( servables, ManagerState::kAvailable, [&](const bool reached, std::map states_reached) { @@ -443,16 +560,16 @@ TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateOnlyNotifiedOnce) { ManagerState::kAvailable))); notified.Notify(); }); - bus->Publish(specific_goal_state); + bus_->Publish(specific_goal_state); notified.WaitForNotification(); - bus->Publish(specific_goal_state); + bus_->Publish(specific_goal_state); } -TEST(ServableStateMonitorTest, WaitUntilServablesReachStateFullFunctionality) { +TEST_F(ServableStateMonitorTest, + WaitUntilServablesReachStateFullFunctionality) { using ManagerState = ServableState::ManagerState; - auto bus = EventBus::CreateEventBus({}); - ServableStateMonitor monitor(bus.get()); + CreateMonitor(); std::vector servables; const ServableId specific_goal_state_id = {"specific_goal_state", 42}; servables.push_back(ServableRequest::FromId(specific_goal_state_id)); @@ -462,20 +579,20 @@ TEST(ServableStateMonitorTest, WaitUntilServablesReachStateFullFunctionality) { const ServableId servable_stream_id = {"servable_stream", 7}; const ServableState specific_goal_state = { - specific_goal_state_id, ManagerState::kAvailable, Status::OK()}; + specific_goal_state_id, ManagerState::kAvailable, OkStatus()}; const ServableState specific_error_state = { specific_error_state_id, ManagerState::kEnd, errors::Internal("error")}; const ServableState servable_stream_state = { - servable_stream_id, ManagerState::kAvailable, Status::OK()}; + servable_stream_id, ManagerState::kAvailable, OkStatus()}; - bus->Publish(specific_goal_state); - bus->Publish(specific_error_state); + bus_->Publish(specific_goal_state); + bus_->Publish(specific_error_state); std::map states_reached; Notification waiting_done; std::unique_ptr wait_till_servable_state_reached( Env::Default()->StartThread({}, "WaitUntilServablesReachState", [&]() { - EXPECT_FALSE(monitor.WaitUntilServablesReachState( + EXPECT_FALSE(monitor_->WaitUntilServablesReachState( servables, ManagerState::kAvailable, &states_reached)); EXPECT_THAT(states_reached, UnorderedElementsAre( @@ -487,7 +604,7 @@ TEST(ServableStateMonitorTest, WaitUntilServablesReachStateFullFunctionality) { // We publish till waiting is finished, otherwise we could publish before we // could start waiting. while (!waiting_done.HasBeenNotified()) { - bus->Publish(servable_stream_state); + bus_->Publish(servable_stream_state); } } diff --git a/tensorflow_serving/core/server_request_logger.cc b/tensorflow_serving/core/server_request_logger.cc new file mode 100644 index 00000000000..7784f70a39d --- /dev/null +++ b/tensorflow_serving/core/server_request_logger.cc @@ -0,0 +1,177 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/server_request_logger.h" + +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/synchronization/mutex.h" +#include "xla/tsl/platform/errors.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/strings/proto_serialization.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/apis/logging.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/core/request_logger.h" + +namespace tensorflow { +namespace serving { + +using UpdateRequest = ServerRequestLogger::UpdateRequest; + +// static +absl::Status ServerRequestLogger::Create( + LoggerCreator request_logger_creator, + std::unique_ptr* server_request_logger) { + server_request_logger->reset( + new ServerRequestLogger(std::move(request_logger_creator))); + return absl::OkStatus(); +} + +ServerRequestLogger::ServerRequestLogger(LoggerCreator request_logger_creator) + : request_logger_creator_(std::move(request_logger_creator)) {} + +absl::Status ServerRequestLogger::FindOrCreateLogger( + const LoggingConfig& config, + StringToUniqueRequestLoggerMap* new_config_to_logger_map, + std::shared_ptr* result) const { + string serialized_config; + if (!SerializeToStringDeterministic(config, &serialized_config)) { + return errors::InvalidArgument("Cannot serialize config."); + } + + auto find_new_it = new_config_to_logger_map->find(serialized_config); + if (find_new_it != new_config_to_logger_map->end()) { + // The logger is already in new_config_to_logger_map, simply return it. + *result = find_new_it->second; + return absl::OkStatus(); + } + + const auto find_old_it = config_to_logger_map_.find(serialized_config); + if (find_old_it != config_to_logger_map_.end()) { + // The logger is in old_config_to_logger_map. Create a copy to + // new_config_to_logger_map. Note we cannot move, as entries in + // config_to_logger_map_ should not be updated here. + *result = find_old_it->second; + new_config_to_logger_map->emplace( + std::make_pair(serialized_config, find_old_it->second)); + return absl::OkStatus(); + } + + // The logger does not exist. Create a new logger, insert it into + // new_config_to_logger_map and return it. + TF_RETURN_IF_ERROR(request_logger_creator_(config, result)); + new_config_to_logger_map->emplace(std::make_pair(serialized_config, *result)); + return absl::OkStatus(); +} + +absl::StatusOr ServerRequestLogger::CreateUpdateRequestLocked( + const std::map>& logging_config_map) + const { + if (!logging_config_map.empty() && !request_logger_creator_) { + return absl::InvalidArgumentError("No request-logger-creator provided."); + } + + auto model_to_loggers_map = std::make_unique(); + auto config_to_logger_map = + std::make_unique(); + + for (const auto& [model_name, logging_configs] : logging_config_map) { + for (const auto& logging_config : logging_configs) { + std::shared_ptr logger; + absl::Status creation_status = FindOrCreateLogger( + logging_config, config_to_logger_map.get(), &logger); + if (!creation_status.ok()) { + return creation_status; + } + (*model_to_loggers_map)[model_name].push_back(logger); + } + } + + return UpdateRequest{.model_to_loggers_map = std::move(model_to_loggers_map), + .config_to_logger_map = std::move(config_to_logger_map)}; +} + +absl::StatusOr ServerRequestLogger::CreateUpdateRequest( + const std::map>& logging_config_map) + const { + absl::MutexLock l(&update_mu_); + return CreateUpdateRequestLocked(logging_config_map); +} + +void ServerRequestLogger::ApplyUpdateRequestLocked(UpdateRequest& req) { + model_to_loggers_map_.Update(std::move(req.model_to_loggers_map)); + config_to_logger_map_ = std::move(*req.config_to_logger_map); +} + +void ServerRequestLogger::ApplyUpdateRequest(UpdateRequest& req) { + absl::MutexLock l(&update_mu_); + ApplyUpdateRequestLocked(req); +} + +absl::Status ServerRequestLogger::Update( + const std::map>& logging_config_map) { + absl::MutexLock l(&update_mu_); + + absl::StatusOr req = + CreateUpdateRequestLocked(logging_config_map); + if (!req.ok()) { + return req.status(); + } + + ApplyUpdateRequestLocked(*req); + return absl::OkStatus(); +} + +absl::Status ServerRequestLogger::Log(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata) { + absl::Status status; + InvokeLoggerForModel( + log_metadata, [&status, &request, &response, &log_metadata]( + const std::shared_ptr& logger) { + // Note: Only first error will be tracked/returned. + status.Update(logger->Log(request, response, log_metadata)); + }); + return status; +} + +void ServerRequestLogger::InvokeLoggerForModel( + const LogMetadata& log_metadata, + std::function&)> fn) { + const string& model_name = log_metadata.model_spec().name(); + auto model_to_loggers_map = model_to_loggers_map_.get(); + if (!model_to_loggers_map || model_to_loggers_map->empty()) { + VLOG(2) << "Request loggers map is empty."; + return; + } + auto found_it = model_to_loggers_map->find(model_name); + if (found_it == model_to_loggers_map->end()) { + VLOG(2) << "Cannot find request-loggers for model: " << model_name; + return; + } + for (const auto& logger : found_it->second) { + fn(logger); + } +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/server_request_logger.h b/tensorflow_serving/core/server_request_logger.h new file mode 100644 index 00000000000..6eb23fdae6b --- /dev/null +++ b/tensorflow_serving/core/server_request_logger.h @@ -0,0 +1,193 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_SERVER_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_SERVER_REQUEST_LOGGER_H_ + +#include +#include +#include +#include +#include + +#include "absl/base/nullability.h" +#include "absl/status/status.h" +#include "absl/status/statusor.h" +#include "absl/synchronization/mutex.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/apis/logging.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/request_logger.h" +#include "tensorflow_serving/core/stream_logger.h" +#include "tensorflow_serving/util/fast_read_dynamic_ptr.h" + +namespace tensorflow { +namespace serving { + +// Logs a sample of requests hitting all the models in the server. +// +// Constructed based on the logging config for the server, which contains the +// sampling config. +class ServerRequestLogger { + public: + using LoggerCreator = std::function*)>; + + // Creates the ServerRequestLogger based on a custom request_logger_creator + // method. + // + // You can create an empty ServerRequestLogger with an empty + // request_logger_creator. + static absl::Status Create( + LoggerCreator request_logger_creator, + std::unique_ptr* server_request_logger); + + virtual ~ServerRequestLogger() = default; + + using StringToRequestLoggersMap = + std::unordered_map>>; + using StringToUniqueRequestLoggerMap = + std::unordered_map>; + struct UpdateRequest { + /*absl_nonnull - not yet supported*/ std::unique_ptr + model_to_loggers_map; + /*absl_nonnull - not yet supported*/ std::unique_ptr + config_to_logger_map; + }; + + // `CreateUpdateRequest()` allows you to validate the given logging config + // map without actually applying the update. A `UpdateRequest` struct is + // returned if the given `logging_config_map` passed all the validation + // checks, and the returned object can be passed to `ApplyUpdateRequest()` + // function which allows an infallible update later. This allows the caller to + // perform the validation step and the apply step separately. + // + // Note, if you plan to do validation and apply as separate steps, you must + // lock this object to ensure no `Update()` or `ApplyUpdateRequest()` is + // called concurrently. + virtual absl::StatusOr CreateUpdateRequest( + const std::map>& logging_config_map) + const; + + // Infallible update function. The `req` provided in this function is not + // validated, for this reason the caller is advised to validate the request + // before calling this function. Most optimally the caller should use the + // output generated from the `CreateUpdateRequest()` function. + virtual void ApplyUpdateRequest(UpdateRequest& req); + + // Fallibly updates the logger with the new 'logging_config_map'. + // + // If the ServerRequestLogger was created using an empty + // request_logger_creator, this will return an error if a non-empty + // logging_config_map is passed in. + virtual absl::Status Update( + const std::map>& logging_config_map); + + // Similar to RequestLogger::Log(). + // + // If request is logged/written to multiple sinks, we return error from + // the first failed write (and continue attempting to write to all). + virtual absl::Status Log(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata); + + // Starts logging a stream. Returns a StreamLogger created through + // `create_stream_logger_fn`. Returns NULL if the stream should not be logged. + // + // Even if the stream is logged/written to multiple sinks, we return a single + // stream logger but register multiple callbacks, one for each sink. + template + using CreateStreamLoggerFn = + std::function>()>; + template + std::unique_ptr> StartLoggingStream( + const LogMetadata& log_metadata, + CreateStreamLoggerFn create_stream_logger_fn); + + protected: + explicit ServerRequestLogger(LoggerCreator request_logger_creator); + + private: + // Locked version of `CreateUpdateRequest()`. `update_mu_` must be held when + // calling this function. + absl::StatusOr CreateUpdateRequestLocked( + const std::map>& logging_config_map) + const; + + // Locked version of `ApplyUpdateRequest()`. `update_mu_` must be held when + // calling this function. + void ApplyUpdateRequestLocked(UpdateRequest& req); + + // Find a logger for config in either config_to_logger_map_ or + // new_config_to_logger_map. If the logger was found in one of existing + // maps, a shared_ptr that points to the existing logging config will be + // added to the new_config_to_logger_map, this avoids unnecessary creation + // of duplicate configs. If such a logger does not exist, create a new + // logger and insert it into new_config_to_logger_map. Return the logger + // in `result`. + absl::Status FindOrCreateLogger( + const LoggingConfig& config, + StringToUniqueRequestLoggerMap* new_config_to_logger_map, + std::shared_ptr* result) const; + + // Invokes `fn` with all loggers for the corresponding model. + void InvokeLoggerForModel( + const LogMetadata& log_metadata, + std::function&)> fn); + + // Mutex to ensure concurrent calls to Update() are serialized. + mutable absl::Mutex update_mu_; + // A map from serialized model logging config to its corresponding + // RequestLogger. If two models have the same logging config, they + // will share the RequestLogger. + // This is only used during calls to Update(). + StringToUniqueRequestLoggerMap config_to_logger_map_; + + // A map from model_name to its corresponding RequestLoggers. + // The RequestLoggers are owned by config_to_logger_map_. + FastReadDynamicPtr model_to_loggers_map_; + + LoggerCreator request_logger_creator_; +}; + +/***************************Implementation Details***************************/ +template +std::unique_ptr> +ServerRequestLogger::StartLoggingStream( + const LogMetadata& log_metadata, + CreateStreamLoggerFn create_stream_logger_fn) { + StreamLogger* stream_logger = nullptr; + InvokeLoggerForModel( + log_metadata, [create_stream_logger_fn, &stream_logger, &log_metadata]( + const std::shared_ptr& request_logger) { + RequestLogger::GetStreamLoggerFn create_logger = + [&create_stream_logger_fn, &stream_logger]() { + stream_logger = create_stream_logger_fn().release(); + return stream_logger; + }; + RequestLogger::GetStreamLoggerFn get_logger = + [stream_logger]() { return stream_logger; }; + request_logger->MaybeStartLoggingStream( + log_metadata, stream_logger == nullptr ? std::move(create_logger) + : std::move(get_logger)); + // If the stream logger has been created, reuse it. + }); + return absl::WrapUnique(stream_logger); +} + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_SERVER_REQUEST_LOGGER_H_ diff --git a/tensorflow_serving/core/server_request_logger_test.cc b/tensorflow_serving/core/server_request_logger_test.cc new file mode 100644 index 00000000000..ee695b16863 --- /dev/null +++ b/tensorflow_serving/core/server_request_logger_test.cc @@ -0,0 +1,494 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/server_request_logger.h" + +#include +#include +#include +#include +#include +#include + +#include "google/protobuf/any.pb.h" +#include +#include +#include "absl/memory/memory.h" +#include "absl/status/status.h" +#include "absl/strings/str_cat.h" +#include "absl/synchronization/mutex.h" +#include "tensorflow/cc/saved_model/tag_constants.h" +#include "xla/tsl/lib/core/status_test_util.h" +#include "xla/tsl/platform/status.h" +#include "xla/tsl/platform/status_matchers.h" +#include "xla/tsl/platform/statusor.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/apis/logging.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" +#include "tensorflow_serving/config/log_collector_config.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/request_logger.h" +#include "tensorflow_serving/core/stream_logger.h" +#include "tensorflow_serving/core/test_util/fake_log_collector.h" +#include "tensorflow_serving/core/test_util/mock_prediction_stream_logger.h" +#include "tensorflow_serving/core/test_util/mock_request_logger.h" + +namespace tensorflow { +namespace serving { +namespace { + +using test_util::MockPredictionStreamLogger; +using ::testing::_; +using ::testing::Eq; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Pair; +using ::testing::SizeIs; +using ::testing::UnorderedElementsAre; +using ::tsl::testing::StatusIs; + +using UpdateRequest = ServerRequestLogger::UpdateRequest; + +LogCollectorConfig CreateLogCollectorConfig(const string& type, + const string& filename_prefix) { + LogCollectorConfig config; + config.set_type(type); + config.set_filename_prefix(filename_prefix); + return config; +} + +std::pair CreateLoggingConfigForModel( + const string& model_name, const string& log_filename_suffix = "") { + string filename_prefix = absl::StrCat("/file/", model_name); + if (!log_filename_suffix.empty()) { + absl::StrAppend(&filename_prefix, "-", log_filename_suffix); + } + LoggingConfig logging_config; + *logging_config.mutable_log_collector_config() = + CreateLogCollectorConfig("", filename_prefix); + logging_config.mutable_sampling_config()->set_sampling_rate(1.0); + return {model_name, logging_config}; +} + +std::map> CreateLoggingConfigMap( + const std::vector>& model_configs) { + std::map> config_map; + for (const auto& model_config : model_configs) { + config_map[model_config.first].push_back(model_config.second); + } + return config_map; +} + +class ServerRequestLoggerTest : public ::testing::Test { + protected: + ServerRequestLoggerTest() { + TF_CHECK_OK(ServerRequestLogger::Create( + [&](const LoggingConfig& logging_config, + std::shared_ptr* const request_logger) { + if (logging_config.has_sampling_config() && + logging_config.sampling_config().sampling_rate() < 0) { + return absl::InvalidArgumentError( + "Negative log sampling rate provided."); + } + + const string& filename_prefix = + logging_config.log_collector_config().filename_prefix(); + log_collector_map_[filename_prefix] = new FakeLogCollector(); + increment_created_logger_counter(); + auto logger_destruction_notifier = [this]() { + increment_deleted_logger_counter(); + }; + const std::vector& tags = {kSavedModelTagServe}; + auto mock_request_logger = + std::shared_ptr>( + new NiceMock( + logging_config, tags, log_collector_map_[filename_prefix], + logger_destruction_notifier)); + ON_CALL(*mock_request_logger, CreateLogMessage(_, _, _, _)) + .WillByDefault(Invoke([&](const google::protobuf::Message& actual_request, + const google::protobuf::Message& actual_response, + const LogMetadata& actual_log_metadata, + std::unique_ptr* log) { + *log = std::unique_ptr( + new google::protobuf::Any()); + return request_logger_status_cb_(); + })); + ON_CALL(*mock_request_logger, FillLogMetadata(_)) + .WillByDefault(Invoke([&](const LogMetadata& log_metadata) { + return log_metadata; + })); + *request_logger = std::move(mock_request_logger); + return absl::OkStatus(); + }, + &server_request_logger_)); + } + + void increment_created_logger_counter() { + absl::MutexLock l(&m_); + created_logger_counter_++; + } + + int created_logger_counter() const { + absl::MutexLock l(&m_); + return created_logger_counter_; + } + + void increment_deleted_logger_counter() { + absl::MutexLock l(&m_); + deleted_logger_counter_++; + } + + int deleted_logger_counter() const { + absl::MutexLock l(&m_); + return deleted_logger_counter_; + } + + mutable absl::Mutex m_; + int created_logger_counter_ = 0; + int deleted_logger_counter_ = 0; + std::function request_logger_status_cb_ = []() { + return absl::OkStatus(); + }; + std::unordered_map log_collector_map_; + std::unique_ptr server_request_logger_; +}; + +TEST_F(ServerRequestLoggerTest, Empty) { + TF_ASSERT_OK(server_request_logger_->Update({})); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + LogMetadata())); + // No log-collectors should have been made. + EXPECT_TRUE(log_collector_map_.empty()); +} + +TEST_F(ServerRequestLoggerTest, AbsentModel) { + TF_ASSERT_OK(server_request_logger_->Update( + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0")}))); + LogMetadata log_metadata; + auto* const model_spec = log_metadata.mutable_model_spec(); + model_spec->set_name("absent_model"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata)); + ASSERT_EQ(1, log_collector_map_.size()); + EXPECT_EQ(0, log_collector_map_["/file/model0"]->collect_count()); +} + +TEST_F(ServerRequestLoggerTest, MultipleModels) { + TF_ASSERT_OK(server_request_logger_->Update( + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0"), + CreateLoggingConfigForModel("model1")}))); + + LogMetadata log_metadata0; + auto* const model_spec0 = log_metadata0.mutable_model_spec(); + model_spec0->set_name("model0"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata0)); + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0"]->collect_count()); + EXPECT_EQ(0, log_collector_map_["/file/model1"]->collect_count()); + + LogMetadata log_metadata1; + auto* const model_spec = log_metadata1.mutable_model_spec(); + model_spec->set_name("model1"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata1)); + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0"]->collect_count()); + EXPECT_EQ(1, log_collector_map_["/file/model1"]->collect_count()); +} + +TEST_F(ServerRequestLoggerTest, CreateAndDeleteLogger) { + auto model_logging_configs = + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0")}); + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + EXPECT_EQ(1, created_logger_counter()); + EXPECT_EQ(0, deleted_logger_counter()); + + model_logging_configs.clear(); + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + EXPECT_EQ(1, created_logger_counter()); + EXPECT_EQ(1, deleted_logger_counter()); +} + +TEST_F(ServerRequestLoggerTest, CreateAndModifyLogger) { + auto model_logging_configs = + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0")}); + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + EXPECT_EQ(1, created_logger_counter()); + EXPECT_EQ(0, deleted_logger_counter()); + + model_logging_configs["model0"][0] + .mutable_sampling_config() + ->set_sampling_rate(0.17); + + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + EXPECT_EQ(2, created_logger_counter()); + EXPECT_EQ(1, deleted_logger_counter()); +} + +TEST_F(ServerRequestLoggerTest, SameConfigForTwoModelsCreatesOneLogger) { + std::pair model_and_config1 = + CreateLoggingConfigForModel("model"); + std::pair model_and_config2 = { + "model2", model_and_config1.second}; + TF_ASSERT_OK(server_request_logger_->Update( + CreateLoggingConfigMap({model_and_config1, model_and_config2}))); + + EXPECT_EQ(1, created_logger_counter()); + EXPECT_EQ(0, deleted_logger_counter()); +} + +TEST_F(ServerRequestLoggerTest, MultipleConfigForOneModel) { + TF_ASSERT_OK(server_request_logger_->Update(CreateLoggingConfigMap( + {CreateLoggingConfigForModel("model0"), + CreateLoggingConfigForModel("model0", "infra")}))); + EXPECT_EQ(2, created_logger_counter()); + EXPECT_EQ(0, deleted_logger_counter()); +} + +TEST_F(ServerRequestLoggerTest, PartiallyBadUpdate) { + // Initially create a logger with 2 OK configs. + std::pair model0_ok_config = + CreateLoggingConfigForModel(/*model_name=*/"model0", + /*log_filename_suffix=*/"path0"); + std::pair model1_ok_config = + CreateLoggingConfigForModel(/*model_name=*/"model1", + /*log_filename_suffix=*/"path1"); + TF_ASSERT_OK(server_request_logger_->Update( + CreateLoggingConfigMap({model0_ok_config, model1_ok_config}))); + EXPECT_EQ(created_logger_counter(), 2); + EXPECT_EQ(deleted_logger_counter(), 0); + + // Now, attempt to update model1's config with a bad config, expect an update + // failure, and existing ok configs should not modified. + std::pair model1_bad_config = + CreateLoggingConfigForModel(/*model_name=*/"model1", + /*log_filename_suffix=*/"path2"); + model1_bad_config.second.mutable_sampling_config()->set_sampling_rate(-1); + EXPECT_THAT(server_request_logger_->Update(CreateLoggingConfigMap( + {model0_ok_config, model1_bad_config})), + StatusIs(absl::StatusCode::kInvalidArgument)); + EXPECT_EQ(created_logger_counter(), 2); + EXPECT_EQ(deleted_logger_counter(), 0); + + // Model0 is still using the old config. + LogMetadata log_metadata0; + log_metadata0.mutable_model_spec()->set_name("model0"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata0)); + ASSERT_EQ(log_collector_map_.size(), 2); + EXPECT_EQ(log_collector_map_.count("/file/model0-path0"), 1); + EXPECT_EQ(log_collector_map_["/file/model0-path0"]->collect_count(), 1); + + // Model1 is still using the old config. + LogMetadata log_metadata1; + log_metadata1.mutable_model_spec()->set_name("model1"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata1)); + EXPECT_EQ(log_collector_map_.count("/file/model1-path1"), 1); + EXPECT_EQ(log_collector_map_["/file/model1-path1"]->collect_count(), 1); + EXPECT_EQ(log_collector_map_.count("/file/model1-path2"), 0); +} + +TEST_F(ServerRequestLoggerTest, CreateUpdateRequestErrors) { + // Null. + TF_ASSERT_OK_AND_ASSIGN(UpdateRequest req, + server_request_logger_->CreateUpdateRequest({})); + EXPECT_TRUE(req.config_to_logger_map->empty()); + EXPECT_TRUE(req.model_to_loggers_map->empty()); + + // Bad config. + std::pair model0_bad_config = + CreateLoggingConfigForModel(/*model_name=*/"model0", + /*log_filename_suffix=*/"bad_config_path"); + model0_bad_config.second.mutable_sampling_config()->set_sampling_rate(-1); + EXPECT_THAT(server_request_logger_->CreateUpdateRequest( + CreateLoggingConfigMap({model0_bad_config})), + StatusIs(absl::StatusCode::kInvalidArgument)); + + // Ok+bad config. + std::pair model0_ok_config = + CreateLoggingConfigForModel(/*model_name=*/"model0", + /*log_filename_suffix=*/"ok_conig_path"); + EXPECT_THAT( + server_request_logger_->CreateUpdateRequest( + CreateLoggingConfigMap({model0_ok_config, model0_bad_config})), + StatusIs(absl::StatusCode::kInvalidArgument)); +} + +TEST_F(ServerRequestLoggerTest, CreateUpdateRequestSingleConfigOk) { + std::pair model0_ok_config = + CreateLoggingConfigForModel(/*model_name=*/"model0", + /*log_filename_suffix=*/"ok_conig_path"); + TF_ASSERT_OK_AND_ASSIGN(UpdateRequest req, + server_request_logger_->CreateUpdateRequest( + CreateLoggingConfigMap({model0_ok_config}))); + EXPECT_THAT(*req.model_to_loggers_map, + UnorderedElementsAre(Pair(Eq("model0"), SizeIs(1)))); + EXPECT_THAT(*req.config_to_logger_map, SizeIs(1)); +} + +TEST_F(ServerRequestLoggerTest, CreateUpdateRequestMultipleConfigsOk) { + // Both model0 and model1 are using the same config, expect only 1 logger + // created. + std::pair model0_v1_config = + CreateLoggingConfigForModel(/*model_name=*/"model0", + /*log_filename_suffix=*/"same_conig_path"); + std::pair model0_v2_config = model0_v1_config; + model0_v2_config.first = "model0_v2"; + + TF_ASSERT_OK_AND_ASSIGN( + UpdateRequest req, + server_request_logger_->CreateUpdateRequest( + CreateLoggingConfigMap({model0_v1_config, model0_v2_config}))); + EXPECT_THAT(*req.model_to_loggers_map, + UnorderedElementsAre(Pair(Eq("model0"), SizeIs(1)), + Pair(Eq("model0_v2"), SizeIs(1)))); + + // Only 1 creation and 1 entry in the config map. As the same config is + // reused. + EXPECT_THAT(*req.config_to_logger_map, SizeIs(1)); + EXPECT_EQ(created_logger_counter(), 1); +} + +TEST_F(ServerRequestLoggerTest, MultipleLoggersForOneModel) { + TF_ASSERT_OK(server_request_logger_->Update(CreateLoggingConfigMap( + {CreateLoggingConfigForModel("model0"), + CreateLoggingConfigForModel("model0", "infra")}))); + + LogMetadata log_metadata0; + auto* const model_spec0 = log_metadata0.mutable_model_spec(); + model_spec0->set_name("model0"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata0)); + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0"]->collect_count()); + EXPECT_EQ(1, log_collector_map_["/file/model0-infra"]->collect_count()); + + LogMetadata log_metadata1; + auto* const model_spec = log_metadata1.mutable_model_spec(); + model_spec->set_name("model1"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata1)); + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0"]->collect_count()); + EXPECT_EQ(1, log_collector_map_["/file/model0-infra"]->collect_count()); + EXPECT_EQ(log_collector_map_.end(), log_collector_map_.find("/file/model1")); +} + +TEST_F(ServerRequestLoggerTest, MultipleLoggersOneModelErrors) { + TF_ASSERT_OK(server_request_logger_->Update(CreateLoggingConfigMap( + {CreateLoggingConfigForModel("model0"), + CreateLoggingConfigForModel("model0", "infra")}))); + + // Inject errors for all Log() calls. + int req_count = 0; + request_logger_status_cb_ = [&]() { + return errors::InvalidArgument(absl::StrCat(req_count++)); + }; + + LogMetadata log_metadata0; + auto* const model_spec0 = log_metadata0.mutable_model_spec(); + model_spec0->set_name("model0"); + EXPECT_EQ(errors::InvalidArgument("0"), + server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata0)); + + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(0, log_collector_map_["/file/model0"]->collect_count()); + EXPECT_EQ(0, log_collector_map_["/file/model0-infra"]->collect_count()); +} + +TEST_F(ServerRequestLoggerTest, MultipleUpdatesSingleCreation) { + const auto& model_logging_configs = + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0"), + CreateLoggingConfigForModel("model1")}); + for (int i = 0; i < 100; i++) { + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + } + + EXPECT_EQ(2, created_logger_counter()); + EXPECT_EQ(0, deleted_logger_counter()); +} + +TEST_F(ServerRequestLoggerTest, StreamLoggingBasic) { + auto model_logging_configs = + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0", "file1"), + CreateLoggingConfigForModel("model0", "file2")}); + + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + LogMetadata log_metadata; + log_metadata.mutable_model_spec()->set_name("model0"); + auto* logger_ptr = new MockPredictionStreamLogger(); + auto logger = server_request_logger_ + ->StartLoggingStream( + log_metadata, [logger_ptr]() { + return absl::WrapUnique(logger_ptr); + }); + EXPECT_CALL(*logger_ptr, CreateLogMessage(_, _)) + .Times(2) + .WillRepeatedly(Invoke([](const LogMetadata& log_metadata, + std::unique_ptr* log) { + *log = std::make_unique(); + return absl::OkStatus(); + })); + TF_ASSERT_OK(logger->LogMessage()); + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0-file1"]->collect_count()); + EXPECT_EQ(1, log_collector_map_["/file/model0-file2"]->collect_count()); +} + +TEST_F(ServerRequestLoggerTest, StreamLoggingUpdateLoggingConfig) { + auto model_logging_configs = + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0", "file1"), + CreateLoggingConfigForModel("model0", "file2")}); + + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + LogMetadata log_metadata; + log_metadata.mutable_model_spec()->set_name("model0"); + auto* logger_ptr = new MockPredictionStreamLogger(); + auto logger = server_request_logger_->StartLoggingStream( + log_metadata, [logger_ptr]() { + return std::unique_ptr>( + logger_ptr); + }); + + model_logging_configs = + CreateLoggingConfigMap({CreateLoggingConfigForModel("model0", "file2"), + CreateLoggingConfigForModel("model0", "file3")}); + + // Updates to a new logging config. Since the stream hasn't finished, the + // stream logger will not use the new config. + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + EXPECT_CALL(*logger_ptr, CreateLogMessage(_, _)) + .Times(2) + .WillRepeatedly(Invoke([](const LogMetadata& log_metadata, + std::unique_ptr* log) { + *log = + std::unique_ptr(new google::protobuf::Any()); + return absl::OkStatus(); + })); + TF_ASSERT_OK(logger->LogMessage()); + ASSERT_EQ(3, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0-file2"]->collect_count()); + EXPECT_EQ(0, log_collector_map_["/file/model0-file3"]->collect_count()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/simple_loader.h b/tensorflow_serving/core/simple_loader.h index acce1ae1903..971d83f3d48 100644 --- a/tensorflow_serving/core/simple_loader.h +++ b/tensorflow_serving/core/simple_loader.h @@ -19,16 +19,19 @@ limitations under the License. #include #include +#include "absl/types/optional.h" +#include "absl/types/variant.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mem.h" +#include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/source_adapter.h" +#include "tensorflow_serving/resources/resource_util.h" #include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/util/any_ptr.h" -#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -54,19 +57,25 @@ namespace serving { // auto servable_creator = [](std::unique_ptr* servable) { // servable->reset(new time_t); // *servable = time(nullptr); -// return Status::OK(); +// return Status(); // }; // auto resource_estimator = [](ResourceAllocation* estimate) { // estimate->mutable_...(...)->set_...(...); -// return Status::OK(); +// return Status(); // }; // std::unique_ptr loader(new SimpleLoader( // servable_creator, resource_estimator)); +// +// This class is not thread-safe. Synchronization is assumed to be done by the +// caller. template class SimpleLoader : public Loader { public: // Creator is called in Load and used to create the servable. using Creator = std::function*)>; + using CreatorWithMetadata = + std::function*)>; + using CreatorVariant = absl::variant; // A callback for estimating a servable's resource usage. using ResourceEstimator = std::function; @@ -80,24 +89,69 @@ class SimpleLoader : public Loader { // and hence the serving system cannot enforce resource safety. static ResourceEstimator EstimateNoResources(); + // Constructor that takes a single resource estimator, to use for estimating + // the resources needed during load as well as post-load. SimpleLoader(Creator creator, ResourceEstimator resource_estimator); + + // Similar to the above constructor, but accepts a CreatorWithMetadata + // function. + SimpleLoader(CreatorWithMetadata creator_with_metadata, + ResourceEstimator resource_estimator); + + // Constructor that takes two resource estimators: one to use for estimating + // the resources needed during load, as well as a second one that gives a + // different estimate after loading has finished. See the documentation on + // Loader::EstimateResources() for (a) potential reasons the estimate might + // decrease, and (b) correctness constraints on how the estimate is allowed to + // change over time. + SimpleLoader(Creator creator, ResourceEstimator resource_estimator, + ResourceEstimator post_load_resource_estimator); + + // Similar to the above constructor, but accepts a CreatorWithMetadata + // function. + SimpleLoader(CreatorWithMetadata creator_with_metadata, + ResourceEstimator resource_estimator, + ResourceEstimator post_load_resource_estimator); + + // Constructor which accepts all variations of the params. + SimpleLoader(CreatorVariant creator_variant, + ResourceEstimator resource_estimator, + absl::optional post_load_resource_estimator); + ~SimpleLoader() override = default; Status EstimateResources(ResourceAllocation* estimate) const override; - Status Load(const ResourceAllocation& available_resources) override; + // REQUIRES: That the ctor with Creator be used, otherwise returns an error + // status. + Status Load() override; + + Status LoadWithMetadata(const Metadata& metadata) override; void Unload() override; AnyPtr servable() override { return AnyPtr{servable_.get()}; } private: - Creator creator_; + Status EstimateResourcesPostLoad(); + CreatorVariant creator_variant_; + + // A function that estimates the resources needed to load the servable. ResourceEstimator resource_estimator_; - // The memoized estimated resource requirement of the session bundle servable. - mutable optional memoized_resource_estimate_; + // An optional function that estimates the resources needed for the servable + // after it has been loaded. (If omitted, 'resource_estimator_' should be used + // for all estimates, i.e. before, during and after load.) + absl::optional post_load_resource_estimator_; + + // The memoized estimated resource requirement of the servable. + mutable absl::optional memoized_resource_estimate_ + TF_GUARDED_BY(memoized_resource_estimate_mu_); + mutable mutex memoized_resource_estimate_mu_; + + std::unique_ptr resource_util_; + Resource ram_resource_; std::unique_ptr servable_; @@ -173,34 +227,125 @@ typename SimpleLoader::ResourceEstimator SimpleLoader::EstimateNoResources() { return [](ResourceAllocation* estimate) { estimate->Clear(); - return Status::OK(); + return Status(); }; } template SimpleLoader::SimpleLoader(Creator creator, ResourceEstimator resource_estimator) - : creator_(creator), resource_estimator_(resource_estimator) {} + : SimpleLoader(CreatorVariant(creator), resource_estimator, absl::nullopt) { +} + +template +SimpleLoader::SimpleLoader( + CreatorWithMetadata creator_with_metadata, + ResourceEstimator resource_estimator) + : SimpleLoader(CreatorVariant(creator_with_metadata), resource_estimator, + absl::nullopt) {} + +template +SimpleLoader::SimpleLoader( + Creator creator, ResourceEstimator resource_estimator, + ResourceEstimator post_load_resource_estimator) + : SimpleLoader(CreatorVariant(creator), resource_estimator, + {post_load_resource_estimator}) {} + +template +SimpleLoader::SimpleLoader( + CreatorWithMetadata creator_with_metadata, + ResourceEstimator resource_estimator, + ResourceEstimator post_load_resource_estimator) + : SimpleLoader(CreatorVariant(creator_with_metadata), resource_estimator, + {post_load_resource_estimator}) {} + +template +SimpleLoader::SimpleLoader( + CreatorVariant creator_variant, ResourceEstimator resource_estimator, + absl::optional post_load_resource_estimator) + : creator_variant_(creator_variant), + resource_estimator_(resource_estimator), + post_load_resource_estimator_(post_load_resource_estimator) { + ResourceUtil::Options resource_util_options; + resource_util_options.devices = {{device_types::kMain, 1}}; + resource_util_ = + std::unique_ptr(new ResourceUtil(resource_util_options)); + + ram_resource_ = resource_util_->CreateBoundResource( + device_types::kMain, resource_kinds::kRamBytes); +} template Status SimpleLoader::EstimateResources( ResourceAllocation* estimate) const { + mutex_lock l(memoized_resource_estimate_mu_); if (memoized_resource_estimate_) { *estimate = *memoized_resource_estimate_; - return Status::OK(); + return Status(); } // Compute and memoize the resource estimate. TF_RETURN_IF_ERROR(resource_estimator_(estimate)); memoized_resource_estimate_ = *estimate; - return Status::OK(); + return Status(); +} + +template +Status SimpleLoader::Load() { + if (absl::holds_alternative(creator_variant_)) { + return errors::FailedPrecondition( + "SimpleLoader::Load() called even though " + "SimpleLoader::CreatorWithMetadata was setup. Please use " + "SimpleLoader::LoadWithMetadata() instead."); + } + TF_RETURN_IF_ERROR(absl::get(creator_variant_)(&servable_)); + return EstimateResourcesPostLoad(); +} + +template +Status SimpleLoader::LoadWithMetadata(const Metadata& metadata) { + if (absl::holds_alternative(creator_variant_)) { + TF_RETURN_IF_ERROR( + absl::get(creator_variant_)(metadata, &servable_)); + } else { + TF_RETURN_IF_ERROR(absl::get(creator_variant_)(&servable_)); + } + return EstimateResourcesPostLoad(); } template -Status SimpleLoader::Load( - const ResourceAllocation& available_resources) { - const Status status = creator_(&servable_); - return status; +Status SimpleLoader::EstimateResourcesPostLoad() { + if (post_load_resource_estimator_) { + // Save the during-load estimate (may be able to use the memoized value). + ResourceAllocation during_load_resource_estimate; + TF_RETURN_IF_ERROR(EstimateResources(&during_load_resource_estimate)); + + // Obtain the post-load estimate, and store it as the memoized value. + ResourceAllocation post_load_resource_estimate; + TF_RETURN_IF_ERROR( + (*post_load_resource_estimator_)(&post_load_resource_estimate)); + { + mutex_lock l(memoized_resource_estimate_mu_); + memoized_resource_estimate_ = post_load_resource_estimate; + } + + // Release any transient memory used only during load to the OS. + const uint64_t during_load_ram_estimate = resource_util_->GetQuantity( + ram_resource_, during_load_resource_estimate); + const uint64_t post_load_ram_estimate = + resource_util_->GetQuantity(ram_resource_, post_load_resource_estimate); + if (post_load_ram_estimate < during_load_ram_estimate) { + const uint64_t transient_ram_estimate = + during_load_ram_estimate - post_load_ram_estimate; + LOG(INFO) << "Calling MallocExtension_ReleaseToSystem() after servable " + "load with " + << transient_ram_estimate; + ::tensorflow::port::MallocExtension_ReleaseToSystem( + transient_ram_estimate); + } + } + + return Status(); } template @@ -220,12 +365,13 @@ void SimpleLoader::Unload() { // If we have a main-memory footprint estimate, release that amount of memory // to the OS. - for (const ResourceAllocation::Entry& entry : - resource_estimate.resource_quantities()) { - if (entry.resource().device() == device_types::kMain && - entry.resource().kind() == resource_kinds::kRamBytes) { - ::tensorflow::port::MallocExtension_ReleaseToSystem(entry.quantity()); - } + const uint64_t memory_estimate = + resource_util_->GetQuantity(ram_resource_, resource_estimate); + if (memory_estimate > 0) { + LOG(INFO) << "Calling MallocExtension_ReleaseToSystem() after servable " + "unload with " + << memory_estimate; + ::tensorflow::port::MallocExtension_ReleaseToSystem(memory_estimate); } } @@ -238,7 +384,7 @@ typename SimpleLoaderSourceAdapter::ResourceEstimator SimpleLoaderSourceAdapter::EstimateNoResources() { return [](const DataType& data, ResourceAllocation* estimate) { estimate->Clear(); - return Status::OK(); + return Status(); }; } @@ -262,7 +408,7 @@ Status SimpleLoaderSourceAdapter::Convert( [resource_estimator, data](ResourceAllocation* estimate) { return resource_estimator(data, estimate); })); - return Status::OK(); + return Status(); } } // namespace serving diff --git a/tensorflow_serving/core/simple_loader_test.cc b/tensorflow_serving/core/simple_loader_test.cc index b9fbbb7827e..7cd99db197f 100644 --- a/tensorflow_serving/core/simple_loader_test.cc +++ b/tensorflow_serving/core/simple_loader_test.cc @@ -15,16 +15,19 @@ limitations under the License. #include "tensorflow_serving/core/simple_loader.h" +#include #include #include #include -#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "absl/memory/memory.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/protobuf/error_codes.pb.h" #include "tensorflow_serving/core/servable_data.h" +#include "tensorflow_serving/core/servable_id.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { @@ -53,18 +56,57 @@ class Caller { State* state_; }; +Loader::Metadata CreateMetadata() { return {ServableId{"name", 42}}; } + +class LoaderCreatorWithoutMetadata { + public: + template + static std::unique_ptr CreateSimpleLoader( + typename SimpleLoader::Creator creator, Args... args) { + return absl::make_unique>(creator, args...); + } + + static absl::Status Load(Loader* loader) { return loader->Load(); } +}; + +class LoaderCreatorWithMetadata { + public: + template + static std::unique_ptr CreateSimpleLoader( + typename SimpleLoader::Creator creator, Args... args) { + return absl::make_unique>( + [creator](const Loader::Metadata& metadata, + std::unique_ptr* servable) { + const auto& expected_metadata = CreateMetadata(); + EXPECT_EQ(expected_metadata.servable_id, metadata.servable_id); + return creator(servable); + }, + args...); + } + + static absl::Status Load(Loader* loader) { + return loader->LoadWithMetadata(CreateMetadata()); + } +}; + +template +class SimpleLoaderTest : public ::testing::Test {}; +using LoaderCreatorTypes = + ::testing::Types; +TYPED_TEST_SUITE(SimpleLoaderTest, LoaderCreatorTypes); + // Move a Loader through its lifetime and ensure the servable is in the state // we expect. -TEST(SimpleLoaderTest, VerifyServableStates) { +TYPED_TEST(SimpleLoaderTest, VerifyServableStates) { State state = State::kNone; - std::unique_ptr loader(new SimpleLoader( + auto loader = TypeParam::template CreateSimpleLoader( [&state](std::unique_ptr* caller) { caller->reset(new Caller(&state)); - return Status::OK(); + return absl::OkStatus(); }, - SimpleLoader::EstimateNoResources())); + SimpleLoader::EstimateNoResources()); EXPECT_EQ(State::kNone, state); - const Status status = loader->Load(ResourceAllocation()); + const absl::Status status = TypeParam::Load(loader.get()); TF_EXPECT_OK(status); EXPECT_EQ(State::kCtor, state); AnyPtr servable = loader->servable(); @@ -78,7 +120,7 @@ TEST(SimpleLoaderTest, VerifyServableStates) { EXPECT_EQ(State::kNone, state); } -TEST(SimpleLoaderTest, ResourceEstimation) { +TYPED_TEST(SimpleLoaderTest, ResourceEstimation) { const auto want = CreateProto( "resource_quantities { " " resource { " @@ -87,33 +129,118 @@ TEST(SimpleLoaderTest, ResourceEstimation) { " } " " quantity: 42 " "} "); - std::unique_ptr loader(new SimpleLoader( + auto loader = TypeParam::template CreateSimpleLoader( [](std::unique_ptr* servable) { servable->reset(new int); - return Status::OK(); + return absl::OkStatus(); }, [&want](ResourceAllocation* estimate) { *estimate = want; - return Status::OK(); + return absl::OkStatus(); + }); + + { + ResourceAllocation got; + TF_ASSERT_OK(loader->EstimateResources(&got)); + EXPECT_THAT(got, EqualsProto(want)); + } + + // The estimate should remain the same after load. + TF_ASSERT_OK(TypeParam::Load(loader.get())); + { + ResourceAllocation got; + TF_ASSERT_OK(loader->EstimateResources(&got)); + EXPECT_THAT(got, EqualsProto(want)); + } +} + +TYPED_TEST(SimpleLoaderTest, ResourceEstimationWithPostLoadRelease) { + const auto pre_load_resources = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'processing' " + " } " + " quantity: 42 " + "} "); + const auto post_load_resources = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'processing' " + " } " + " quantity: 17 " + "} "); + auto loader = TypeParam::template CreateSimpleLoader( + [](std::unique_ptr* servable) { + servable->reset(new int); + return absl::OkStatus(); + }, + [&pre_load_resources](ResourceAllocation* estimate) { + *estimate = pre_load_resources; + return absl::OkStatus(); + }, + absl::make_optional([&post_load_resources](ResourceAllocation* estimate) { + *estimate = post_load_resources; + return absl::OkStatus(); })); + + // Run it twice, to exercise memoization. for (int i = 0; i < 2; ++i) { ResourceAllocation got; TF_ASSERT_OK(loader->EstimateResources(&got)); - EXPECT_THAT(got, EqualsProto(want)); + EXPECT_THAT(got, EqualsProto(pre_load_resources)); + } + + // The estimate should switch to the post-load one after load. + TF_ASSERT_OK(TypeParam::Load(loader.get())); + { + ResourceAllocation got; + TF_ASSERT_OK(loader->EstimateResources(&got)); + EXPECT_THAT(got, EqualsProto(post_load_resources)); } } // Verify that the error returned by the Creator is propagates back through // Load. -TEST(SimpleLoaderTest, LoadError) { - std::unique_ptr loader(new SimpleLoader( +TYPED_TEST(SimpleLoaderTest, LoadError) { + auto loader = TypeParam::template CreateSimpleLoader( [](std::unique_ptr* caller) { return errors::InvalidArgument("No way!"); }, - SimpleLoader::EstimateNoResources())); - const Status status = loader->Load(ResourceAllocation()); + SimpleLoader::EstimateNoResources()); + const absl::Status status = TypeParam::Load(loader.get()); EXPECT_EQ(error::INVALID_ARGUMENT, status.code()); - EXPECT_EQ("No way!", status.error_message()); + EXPECT_EQ("No way!", status.message()); +} + +TEST(SimpleLoaderCompatibilityTest, WithoutMetadata) { + auto loader_without_metadata = absl::make_unique>( + [](std::unique_ptr* servable) { + servable->reset(new int); + return absl::OkStatus(); + }, + SimpleLoader::EstimateNoResources()); + // If the creator without metadata is used, both Load() and LoadWithMetadata() + // are fine, for compatibility. + TF_EXPECT_OK(loader_without_metadata->Load()); + TF_EXPECT_OK(loader_without_metadata->LoadWithMetadata(CreateMetadata())); +} + +TEST(SimpleLoaderCompatibilityTest, WithMetadata) { + auto loader_with_metadata = absl::make_unique>( + [](const Loader::Metadata& metadata, std::unique_ptr* servable) { + const auto& expected_metadata = CreateMetadata(); + EXPECT_EQ(expected_metadata.servable_id, metadata.servable_id); + servable->reset(new int); + return absl::OkStatus(); + }, + SimpleLoader::EstimateNoResources()); + // If the creator with metadata is used, we allow only LoadWithMetadata() + // to be invoked. + const absl::Status error_status = loader_with_metadata->Load(); + EXPECT_EQ(error::FAILED_PRECONDITION, error_status.code()); + TF_EXPECT_OK(loader_with_metadata->LoadWithMetadata(CreateMetadata())); } // A pass-through implementation of SimpleLoaderSourceAdapter, which can be @@ -137,19 +264,19 @@ TEST(SimpleLoaderSourceAdapterTest, Basic) { [](const string& data, std::unique_ptr* servable) { servable->reset(new string); **servable = strings::StrCat(data, "_was_here"); - return Status::OK(); + return absl::OkStatus(); }, [](const string& data, ResourceAllocation* output) { ResourceAllocation::Entry* entry = output->add_resource_quantities(); entry->mutable_resource()->set_device(data); entry->set_quantity(42); - return Status::OK(); + return absl::OkStatus(); }); const string kServableName = "test_servable_name"; bool callback_called; adapter.SetAspiredVersionsCallback( - [&](const StringPiece servable_name, + [&](const absl::string_view servable_name, std::vector>> versions) { callback_called = true; EXPECT_EQ(kServableName, servable_name); @@ -165,7 +292,7 @@ TEST(SimpleLoaderSourceAdapterTest, Basic) { " } " " quantity: 42 " "} "))); - TF_ASSERT_OK(loader->Load(ResourceAllocation())); + TF_ASSERT_OK(loader->Load()); AnyPtr servable = loader->servable(); ASSERT_TRUE(servable.get() != nullptr); EXPECT_EQ("test_data_was_here", *servable.get()); @@ -186,13 +313,13 @@ TEST(SimpleLoaderSourceAdapterTest, OkayToDeleteAdapter) { [](const string& data, std::unique_ptr* servable) { servable->reset(new string); **servable = strings::StrCat(data, "_was_here"); - return Status::OK(); + return absl::OkStatus(); }, SimpleLoaderSourceAdapter::EstimateNoResources())); const string kServableName = "test_servable_name"; adapter->SetAspiredVersionsCallback( - [&](const StringPiece servable_name, + [&](const absl::string_view servable_name, std::vector>> versions) { ASSERT_EQ(1, versions.size()); TF_ASSERT_OK(versions[0].status()); @@ -208,7 +335,7 @@ TEST(SimpleLoaderSourceAdapterTest, OkayToDeleteAdapter) { // callbacks, despite the fact that 'adapter' has been deleted. ResourceAllocation estimate_given; TF_ASSERT_OK(loader->EstimateResources(&estimate_given)); - TF_ASSERT_OK(loader->Load(ResourceAllocation())); + TF_ASSERT_OK(loader->Load()); } } // namespace diff --git a/tensorflow_serving/core/source.h b/tensorflow_serving/core/source.h index 083f8c66d8a..e9347968123 100644 --- a/tensorflow_serving/core/source.h +++ b/tensorflow_serving/core/source.h @@ -28,64 +28,66 @@ limitations under the License. namespace tensorflow { namespace serving { -// An abstraction for a module that sources servables to load, or, more -// precisely, handles to data that can be used to load those servables. -// Examples of such data handles are: -// - a file-system path to a serialized vocabulary map -// - a handle to an incoming RPC that specifies a machine-learned model to load -// - a Loader (see loader.h) -// The data handles are generally assumed to be small. -// -// A Source monitors some external resource (e.g. file system, RPC calls) to -// find out about new servables and/or new versions of servables and/or the -// need to unload servable versions. It uses the provided callback to instruct -// a Target module (e.g. AspiredVersionsManager) which version(s) of a given -// servable to load. Furthermore, depending on the semantics of the Target -// module, the Source implicitly instructs it which ones to unload by omitting -// those servables. -// -// A common case is that a Source emits versions for exactly one servable. An -// even simpler case is that a servable has a single, static version for the -// lifetime of the server. -// -// Sources can house state that is shared among multiple emitted servables, e.g. -// 1. A shared thread pool or other resource that multiple servables use. -// 2. A shared read-only data structure that multiple servables use, to avoid -// the time and space overhead of replicating the data structure in each -// servable instance. -// Shared state whose initialization time and size is negligible (e.g. thread -// pools) can be created eagerly by the source, which then embeds a pointer to -// it in each emitted ServableData item. Creation of expensive or large shared -// state should be deferred to the first applicable Loader::Load() call, i.e. -// governed by the manager. Symmetrically, the Loader::Unload() call to the -// final servable using the expensive/large shared state should tear it down. +/// An abstraction for a module that sources servables to load, or, more +/// precisely, handles to data that can be used to load those servables. +/// Examples of such data handles are: +/// - a file-system path to a serialized vocabulary map +/// - a handle to an incoming RPC that specifies a machine-learned model to +/// load +/// - a Loader (see loader.h) +/// The data handles are generally assumed to be small. +/// +/// A Source monitors some external resource (e.g. file system, RPC calls) to +/// find out about new servables and/or new versions of servables and/or the +/// need to unload servable versions. It uses the provided callback to instruct +/// a Target module (e.g. AspiredVersionsManager) which version(s) of a given +/// servable to load. Furthermore, depending on the semantics of the Target +/// module, the Source implicitly instructs it which ones to unload by omitting +/// those servables. +/// +/// A common case is that a Source emits versions for exactly one servable. An +/// even simpler case is that a servable has a single, static version for the +/// lifetime of the server. +/// +/// Sources can house state that is shared among multiple emitted servables, +/// e.g. +/// 1. A shared thread pool or other resource that multiple servables use. +/// 2. A shared read-only data structure that multiple servables use, to avoid +/// the time and space overhead of replicating the data structure in each +/// servable instance. +/// Shared state whose initialization time and size is negligible (e.g. thread +/// pools) can be created eagerly by the source, which then embeds a pointer to +/// it in each emitted ServableData item. Creation of expensive or large shared +/// state should be deferred to the first applicable Loader::Load() call, i.e. +/// governed by the manager. Symmetrically, the Loader::Unload() call to the +/// final servable using the expensive/large shared state should tear it down. template class Source { public: virtual ~Source() = default; - // A callback for a Source to supply version(s) of a servable to a Target, to - // be loaded. - // - // A single invocation of the callback pertains to a single servable stream - // (given by 'servable_name'). All versions supplied in a call must be for the - // servable identified in 'servable_name'. Invocations on different servable - // streams are orthogonal to one another. - // - // Multiple invocations may supply servable-data objects with identical - // ids (i.e. same servable name and version). Such servable-data objects are - // treated as semantically equivalent. The recipient will ultimately retain - // one and discard the rest. - // - // If a servable version V is supplied in a first invocation, and subsequently - // omitted from a second invocation, the implication of omitting V depends on - // the semantics of the Target of the callback. Certain Targets will interpret - // V's omission as an implicit instruction to unload V. Each Target must - // document its semantics in this regard. + /// A callback for a Source to supply version(s) of a servable to a Target, to + /// be loaded. + /// + /// A single invocation of the callback pertains to a single servable stream + /// (given by 'servable_name'). All versions supplied in a call must be for + /// the servable identified in 'servable_name'. Invocations on different + /// servable streams are orthogonal to one another. + /// + /// Multiple invocations may supply servable-data objects with identical + /// ids (i.e. same servable name and version). Such servable-data objects are + /// treated as semantically equivalent. The recipient will ultimately retain + /// one and discard the rest. + /// + /// If a servable version V is supplied in a first invocation, and + /// subsequently omitted from a second invocation, the implication of omitting + /// V depends on the semantics of the Target of the callback. Certain Targets + /// will interpret V's omission as an implicit instruction to unload V. Each + /// Target must document its semantics in this regard. using AspiredVersionsCallback = std::function> versions)>; - // Supplies an AspiredVersionsCallback to use. Can be called at most once. + /// Supplies an AspiredVersionsCallback to use. Can be called at most once. virtual void SetAspiredVersionsCallback(AspiredVersionsCallback callback) = 0; }; diff --git a/tensorflow_serving/core/source_adapter.h b/tensorflow_serving/core/source_adapter.h index a6909e7096d..d4e92507adb 100644 --- a/tensorflow_serving/core/source_adapter.h +++ b/tensorflow_serving/core/source_adapter.h @@ -25,55 +25,63 @@ limitations under the License. #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/source.h" +#include "tensorflow_serving/core/storage_path.h" #include "tensorflow_serving/core/target.h" +#include "tensorflow_serving/util/class_registration.h" namespace tensorflow { namespace serving { -// An abstraction for a module that receives aspired-version callbacks with data -// of type InputType and converts them into calls with data of type OutputType. -// -// A common example uses InputType=StoragePath, OutputType=unique_ptr, -// in which case the module "converts" each incoming storage path into a loader -// capable of loading a (particular type of) servable based on the path. -// -// SourceAdapters are typically stateless. However, as with all Sources they can -// house state that is shared among multiple emitted servables. See the -// discussion in source.h. -// -// Implementing subclasses supply an implementation of the Adapt() virtual -// method, which converts a servable version list from InputType to OutputType. -// -// IMPORTANT: Every leaf derived class must call Detach() at the top of its -// destructor. (See documentation on TargetBase::Detach() in target.h.) Doing so -// ensures that no Adapt() calls are in flight during destruction of member -// variables. +/// An abstraction for a module that receives aspired-version callbacks with +/// data of type InputType and converts them into calls with data of type +/// OutputType. +/// +/// A common example uses InputType=StoragePath, +/// OutputType=unique_ptr&lt;Loader>, in which case the module "converts" +/// each incoming storage path into a loader capable of loading a (particular +/// type of) servable based on the path. +/// +/// SourceAdapters are typically stateless. However, as with all Sources they +/// can house state that is shared among multiple emitted servables. See the +/// discussion in source.h. +/// +/// Implementing subclasses supply an implementation of the Adapt() virtual +/// method, which converts a servable version list from InputType to OutputType. +/// +/// IMPORTANT: Every leaf derived class must call Detach() at the top of its +/// destructor. (See documentation on TargetBase::Detach() in target.h.) Doing +/// so ensures that no Adapt() calls are in flight during destruction of member +/// variables. template class SourceAdapter : public TargetBase, public Source { public: ~SourceAdapter() override = 0; - // This method is implemented in terms of Adapt(), which the implementing - // subclass must supply. + /// This method is implemented in terms of Adapt(), which the implementing + /// subclass must supply. void SetAspiredVersions(const StringPiece servable_name, std::vector> versions) final; void SetAspiredVersionsCallback( typename Source::AspiredVersionsCallback callback) final; + /// Given an InputType-based aspired-versions request, produces a + /// corresponding OutputType-based request. + virtual std::vector> Adapt( + const StringPiece servable_name, + std::vector> versions) = 0; + + /// Adapts a single servable data item. (Implemented on top of Adapt().) + ServableData AdaptOneVersion(ServableData input); + protected: // This is an abstract class. SourceAdapter() = default; private: - // Given an InputType-based aspired-versions request, produces a corresponding - // OutputType-based request. - virtual std::vector> Adapt( - const StringPiece servable_name, - std::vector> versions) = 0; - // The callback for emitting OutputType-based aspired-version lists. typename Source::AspiredVersionsCallback outgoing_callback_; @@ -82,6 +90,18 @@ class SourceAdapter : public TargetBase, public Source { Notification outgoing_callback_set_; }; +// START_SKIP_DOXYGEN + +// Define a SourceAdapter registry for the common case of adapting from a +// storage path to a loader. +using StoragePathSourceAdapter = + SourceAdapter>; +DEFINE_CLASS_REGISTRY(StoragePathSourceAdapterRegistry, + StoragePathSourceAdapter); +#define REGISTER_STORAGE_PATH_SOURCE_ADAPTER(ClassCreator, ConfigProto) \ + REGISTER_CLASS(StoragePathSourceAdapterRegistry, StoragePathSourceAdapter, \ + ClassCreator, ConfigProto); + // A source adapter that converts InputType instances to OutputType instances // one at a time (i.e. there is no interaction among members of a given aspired- // version list). Most source adapters can subclass UnarySourceAdapter, and do @@ -167,6 +187,17 @@ void SourceAdapter::SetAspiredVersionsCallback( outgoing_callback_set_.Notify(); } +template +ServableData SourceAdapter::AdaptOneVersion( + ServableData input) { + const StringPiece servable_name(input.id().name); + std::vector> input_versions = {input}; + std::vector> output_versions = + Adapt(servable_name, input_versions); + DCHECK_EQ(1, output_versions.size()); + return std::move(output_versions[0]); +} + template UnarySourceAdapter::~UnarySourceAdapter() {} @@ -217,7 +248,7 @@ ErrorInjectingSourceAdapter::Adapt( for (const ServableData& version : versions) { if (version.status().ok()) { LOG(INFO) << "Injecting error for servable " << version.id() << ": " - << error_.error_message(); + << error_.message(); adapted_versions.emplace_back( ServableData{version.id(), error_}); } else { @@ -228,6 +259,8 @@ ErrorInjectingSourceAdapter::Adapt( return adapted_versions; } +// END_SKIP_DOXYGEN + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/source_adapter_test.cc b/tensorflow_serving/core/source_adapter_test.cc index 8a9253fae54..5a6d522efa6 100644 --- a/tensorflow_serving/core/source_adapter_test.cc +++ b/tensorflow_serving/core/source_adapter_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include #include #include @@ -33,7 +34,6 @@ limitations under the License. using ::testing::ElementsAre; using ::testing::Eq; using ::testing::IsEmpty; -using ::testing::Return; using ::testing::StrictMock; namespace tensorflow { @@ -48,7 +48,7 @@ class LimitedAdapter final : public SourceAdapter { protected: std::vector> Adapt( - const StringPiece servable_name, + const absl::string_view servable_name, std::vector> versions) override { CHECK(versions.empty()); return {}; @@ -58,6 +58,15 @@ class LimitedAdapter final : public SourceAdapter { TF_DISALLOW_COPY_AND_ASSIGN(LimitedAdapter); }; +TEST(SourceAdapterTest, AdaptOneVersion) { + test_util::FakeStoragePathSourceAdapter adapter("baz"); + ServableData output = + adapter.AdaptOneVersion(ServableData({"foo", 42}, "bar")); + EXPECT_EQ("foo", output.id().name); + EXPECT_EQ(42, output.id().version); + EXPECT_EQ("bar/baz", output.DataOrDie()); +} + TEST(SourceAdapterTest, SetAspiredVersionsBlocksUntilTargetConnected) { LimitedAdapter adapter; std::unique_ptr target( diff --git a/tensorflow_serving/core/source_router.h b/tensorflow_serving/core/source_router.h index 51bf8f0ab17..93c39e2d9fc 100644 --- a/tensorflow_serving/core/source_router.h +++ b/tensorflow_serving/core/source_router.h @@ -51,6 +51,8 @@ namespace serving { template class SourceRouter : public TargetBase { public: + static constexpr int kNoRoute = -1; + ~SourceRouter() override = 0; // Returns a vector of N source pointers, corresponding to the N output ports @@ -72,8 +74,10 @@ class SourceRouter : public TargetBase { // of the router. To be written by the implementing subclass. virtual int num_output_ports() const = 0; - // Returns the output port # to which to route a given aspired-versions - // request, in [0, num_output_ports() - 1]. To be written by the implementing + // Returns `kNoRoute` or a valid output port # in [0, num_output_ports() - 1]. + // Aspired-versions requests will be routed to the output port corresponding + // to the returned port number. If `kNoRoute` is returned, the aspired-version + // request will be discarded silently. To be written by the implementing // subclass. virtual int Route(const StringPiece servable_name, const std::vector>& versions) = 0; @@ -96,21 +100,25 @@ namespace internal { // A SourceAdapter that passes through data unchanged. Used to implement the // output ports. template -class IdentitySourceAdapter final : public UnarySourceAdapter { +class IdentitySourceAdapter final : public SourceAdapter { public: IdentitySourceAdapter() = default; ~IdentitySourceAdapter() override { TargetBase::Detach(); } - protected: - Status Convert(const T& data, T* converted_data) override { - *converted_data = data; - return Status::OK(); - } - private: + std::vector> Adapt( + const StringPiece servable_name, + std::vector> versions) final; + TF_DISALLOW_COPY_AND_ASSIGN(IdentitySourceAdapter); }; +template +std::vector> IdentitySourceAdapter::Adapt( + const StringPiece servable_name, std::vector> versions) { + return versions; +} + } // namespace internal template @@ -144,10 +152,13 @@ void SourceRouter::SetAspiredVersions( const StringPiece servable_name, std::vector> versions) { output_ports_created_.WaitForNotification(); int output_port = Route(servable_name, versions); + if (output_port == kNoRoute) { + return; + } if (output_port < 0 || output_port > output_ports_.size() - 1) { LOG(ERROR) - << "SourceRouter abstraction used improperly; Route() must return a " - "value in [0, num_output_ports()-1]; suppressing the " + << "SourceRouter abstraction used improperly; Route() must return " + "kNoRoute or a value in [0, num_output_ports()-1]; suppressing the " "aspired-versions request"; DCHECK(false); return; diff --git a/tensorflow_serving/core/source_router_test.cc b/tensorflow_serving/core/source_router_test.cc index 053e23d4809..eafd84497f8 100644 --- a/tensorflow_serving/core/source_router_test.cc +++ b/tensorflow_serving/core/source_router_test.cc @@ -15,7 +15,10 @@ limitations under the License. #include "tensorflow_serving/core/source_router.h" +#include #include +#include +#include #include #include @@ -25,13 +28,13 @@ limitations under the License. #include "tensorflow/core/platform/macros.h" #include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/core/target.h" #include "tensorflow_serving/core/test_util/mock_storage_path_target.h" using ::testing::_; using ::testing::ElementsAre; using ::testing::Eq; using ::testing::IsEmpty; -using ::testing::Return; using ::testing::StrictMock; namespace tensorflow { @@ -40,18 +43,21 @@ namespace { class TestSourceRouter final : public SourceRouter { public: - TestSourceRouter() = default; + TestSourceRouter(int num_ports = 2) : num_output_ports_(num_ports) {} ~TestSourceRouter() override { Detach(); } protected: - int num_output_ports() const override { return 2; } + const int num_output_ports_; + int num_output_ports() const override { return num_output_ports_; } - int Route(const StringPiece servable_name, + int Route(const absl::string_view servable_name, const std::vector>& versions) override { if (servable_name == "zero") { return 0; } else if (servable_name == "one") { return 1; + } else if (servable_name == "no_route") { + return kNoRoute; } else { LOG(FATAL) << "Unexpected test data"; } @@ -87,6 +93,21 @@ TEST(SourceRouterTest, Basic) { {ServableData({"one", 1}, "floo")}); } +TEST(SourceRouterTest, NumPorts) { + TestSourceRouter router(1); + std::vector*> output_ports = router.GetOutputPorts(); + ASSERT_EQ(1, output_ports.size()); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(output_ports[0], target.get()); + + EXPECT_CALL(*target, SetAspiredVersions(Eq("zero"), + ElementsAre(ServableData( + {"zero", 0}, "mrop")))); + router.SetAspiredVersions("zero", + {ServableData({"zero", 0}, "mrop")}); +} + TEST(SourceRouterTest, SetAspiredVersionsBlocksUntilAllTargetsConnected_1) { // Scenario 1: When SetAspiredVersions() is invoked, GetOutputPorts() has not // yet been called. The SetAspiredVersions() call should block until the ports @@ -157,6 +178,22 @@ TEST(SourceRouterTest, SetAspiredVersionsBlocksUntilAllTargetsConnected_2) { router.SetAspiredVersions("one", {}); } +TEST(SourceRouterTest, DiscardRequest) { + // Testing return kNoRoute to discard a request + + TestSourceRouter router; + std::vector*> output_ports = router.GetOutputPorts(); + std::vector> targets; + for (int i = 0; i < output_ports.size(); ++i) { + targets.emplace_back(new StrictMock); + ConnectSourceToTarget(output_ports[i], targets.back().get()); + } + + router.SetAspiredVersions("no_route", {}); + + // Expect no `SetAspiredVersions` call on `output_ports[0]`. +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/static_manager.cc b/tensorflow_serving/core/static_manager.cc index 7b838fa3d64..fd437228f07 100644 --- a/tensorflow_serving/core/static_manager.cc +++ b/tensorflow_serving/core/static_manager.cc @@ -15,13 +15,17 @@ limitations under the License. #include "tensorflow_serving/core/static_manager.h" +#include +#include + namespace tensorflow { namespace serving { StaticManagerBuilder::StaticManagerBuilder() { BasicManager::Options basic_manager_options; // We don't want multithreading. - basic_manager_options.num_load_unload_threads = 0; + basic_manager_options.num_load_threads = 0; + basic_manager_options.num_unload_threads = 0; const Status basic_manager_status = BasicManager::Create(std::move(basic_manager_options), &basic_manager_); if (!basic_manager_status.ok()) { diff --git a/tensorflow_serving/core/static_manager.h b/tensorflow_serving/core/static_manager.h index 481bff1c767..691316989c2 100644 --- a/tensorflow_serving/core/static_manager.h +++ b/tensorflow_serving/core/static_manager.h @@ -66,13 +66,13 @@ Status StaticManagerBuilder::AddServable(const ServableId& id, TF_RETURN_IF_ERROR(health_); DCHECK(basic_manager_ != nullptr); - basic_manager_->ManageServable(CreateServableData( + TF_RETURN_IF_ERROR(basic_manager_->ManageServable(CreateServableData( id, std::unique_ptr(new SimpleLoader( [&servable](std::unique_ptr* const returned_servable) { *returned_servable = std::move(servable); - return Status::OK(); + return Status(); }, - SimpleLoader::EstimateNoResources())))); + SimpleLoader::EstimateNoResources()))))); Status load_status; Notification load_done; basic_manager_->LoadServable(id, [&](const Status& status) { diff --git a/tensorflow_serving/core/static_source_router.h b/tensorflow_serving/core/static_source_router.h index 6dfb1003358..4ef0e76c0f9 100644 --- a/tensorflow_serving/core/static_source_router.h +++ b/tensorflow_serving/core/static_source_router.h @@ -20,26 +20,26 @@ limitations under the License. #include #include +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/regexp.h" #include "tensorflow_serving/core/source_router.h" namespace tensorflow { namespace serving { // A SourceRouter with N statically-configured output ports. Items are routed to -// output ports based on regular-expression matching against the servable name. -// The router is configured with N-1 regular expressions (regexps), with "fall- -// through" semantics. In particular: The regexps are numbered 0, 1, ..., N-2. -// Items whose servable name matches regexp 0 are sent to port 0; items that -// fail to match regexp 0 but do match regexp 1 are sent to port 1; and so on. -// Items that match none of the regexps are sent to port N-1. +// output ports based on substring matching against the servable name. The +// router is configured with N-1 substrings, with "fall-through" semantics. In +// particular: The substrings are numbered 0, 1, ..., N-2. Items whose servable +// name matches substring 0 are sent to port 0; items that fail to match +// substring 0 but do match substring 1 are sent to port 1; and so on. Items +// that match none of the substrings are sent to port N-1. template class StaticSourceRouter final : public SourceRouter { public: - // Creates a StaticSourceRouter with 'route_regexps.size() + 1' output ports, - // based on cascading regular expression matching as described above. - static Status Create(const std::vector& route_regexps, + // Creates a StaticSourceRouter with 'route_substrings.size() + 1' output + // ports, based on cascading substring matching as described above. + static Status Create(const std::vector& route_substrings, std::unique_ptr>* result); ~StaticSourceRouter() override; @@ -52,10 +52,11 @@ class StaticSourceRouter final : public SourceRouter { const std::vector>& versions) override; private: - explicit StaticSourceRouter(const std::vector& route_regexps); + explicit StaticSourceRouter(const std::vector& route_substrings); - // The regexps of the first N-1 routes (the Nth route is the default route). - std::vector> routes_except_default_; + // The substrings of the first N-1 routes (the Nth route is the default + // route). + std::vector routes_except_default_; TF_DISALLOW_COPY_AND_ASSIGN(StaticSourceRouter); }; @@ -65,10 +66,10 @@ class StaticSourceRouter final : public SourceRouter { template Status StaticSourceRouter::Create( - const std::vector& route_regexps, + const std::vector& route_substrings, std::unique_ptr>* result) { - result->reset(new StaticSourceRouter(route_regexps)); - return Status::OK(); + result->reset(new StaticSourceRouter(route_substrings)); + return Status(); } template @@ -80,25 +81,22 @@ template int StaticSourceRouter::Route(const StringPiece servable_name, const std::vector>& versions) { for (int i = 0; i < routes_except_default_.size(); ++i) { - if (RE2::FullMatch(servable_name.ToString(), *routes_except_default_[i])) { - LOG(INFO) << "Routing servable(s) from stream " << servable_name - << " to route " << i; + if (str_util::StrContains(servable_name, routes_except_default_[i])) { + VLOG(2) << "Routing servable(s) from stream " << servable_name + << " to route " << i; return i; } } - // None of the regexps matched, so return the "default" Nth route. - LOG(INFO) << "Routing servable(s) from stream " << servable_name - << " to default route " << routes_except_default_.size(); + // None of the substrings matched, so return the "default" Nth route. + VLOG(2) << "Routing servable(s) from stream " << servable_name + << " to default route " << routes_except_default_.size(); return routes_except_default_.size(); } template StaticSourceRouter::StaticSourceRouter( - const std::vector& route_regexps) { - for (const string& route_regexp : route_regexps) { - routes_except_default_.emplace_back(new RE2(route_regexp)); - } -} + const std::vector& route_substrings) + : routes_except_default_(route_substrings) {} } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/static_source_router_test.cc b/tensorflow_serving/core/static_source_router_test.cc index 91dd3127988..c9b87bce0d9 100644 --- a/tensorflow_serving/core/static_source_router_test.cc +++ b/tensorflow_serving/core/static_source_router_test.cc @@ -16,6 +16,9 @@ limitations under the License. #include "tensorflow_serving/core/static_source_router.h" #include +#include +#include +#include #include #include @@ -31,7 +34,6 @@ limitations under the License. using ::testing::ElementsAre; using ::testing::Eq; -using ::testing::Return; using ::testing::StrictMock; namespace tensorflow { @@ -39,7 +41,7 @@ namespace serving { namespace { TEST(StaticSourceRouterTest, Basic) { - const std::vector regexps = {"0th.*", ".*1st"}; + const std::vector regexps = {"0th", "1st"}; std::unique_ptr> router; TF_ASSERT_OK(StaticSourceRouter::Create(regexps, &router)); std::vector*> output_ports = router->GetOutputPorts(); diff --git a/tensorflow_serving/core/stream_logger.h b/tensorflow_serving/core/stream_logger.h new file mode 100644 index 00000000000..31ed9f92ced --- /dev/null +++ b/tensorflow_serving/core/stream_logger.h @@ -0,0 +1,111 @@ +/* Copyright 2023 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_SERVING_CORE_STREAM_LOGGER_H_ +#define THIRD_PARTY_TENSORFLOW_SERVING_CORE_STREAM_LOGGER_H_ + +#include +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow_serving/apis/logging.pb.h" + +namespace tensorflow { +namespace serving { + +// Simple logger for a stream of requests and responses. In practice, the +// lifetime of this class should be attached to the lifetime of a stream. +// +// The class being templated on requests and responses is to avoid RTTI in the +// subclasses. +// Not thread-safe. +template +class StreamLogger { + public: + StreamLogger() { + static_assert((std::is_base_of::value), + "Request must be a proto type."); + static_assert((std::is_base_of::value), + "Response must be a proto type."); + } + + virtual ~StreamLogger() = default; + + virtual void LogStreamRequest(Request request) = 0; + virtual void LogStreamResponse(Response response) = 0; + + using LogMessageFn = std::function; + // Registers a log callback to be invoked when calling LogMessage(); + void AddLogCallback(const LogMetadata& log_metadata, + LogMessageFn log_message_fn); + + // Logs the message with all requests and responses accumulated so far, and + // invokes all log callbacks sequentially. Upon return, any subsequent calls + // to any other methods of this class will result in undefined behavior. On + // multiple callbacks, we return error from the first failed one (and continue + // attempting the rest). + absl::Status LogMessage(); + + private: + virtual absl::Status CreateLogMessage( + const LogMetadata& log_metadata, + std::unique_ptr* log) = 0; + + struct StreamLogCallback { + LogMetadata log_metadata; + LogMessageFn log_message_fn; + }; + + std::vector callbacks_; +}; + +/*************************Implementation Details******************************/ + +template +absl::Status StreamLogger::LogMessage() { + absl::Status status; + for (const auto& callback : callbacks_) { + std::unique_ptr log; + absl::Status create_status = CreateLogMessage(callback.log_metadata, &log); + if (create_status.ok()) { + status.Update(callback.log_message_fn(*log)); + } else { + LOG_EVERY_N_SEC(ERROR, 30) + << "Failed creating log message for streaming request. Log metadata: " + << callback.log_metadata.DebugString() + << ", error: " << create_status; + status.Update(create_status); + } + } + return status; +} + +template +void StreamLogger::AddLogCallback( + const LogMetadata& log_metadata, LogMessageFn log_message_fn) { + StreamLogCallback callback; + callback.log_metadata = log_metadata; + callback.log_message_fn = std::move(log_message_fn); + callbacks_.push_back(std::move(callback)); +} + +} // namespace serving +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_SERVING_CORE_STREAM_LOGGER_H_ diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 3d501805ed3..4278437b018 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -1,18 +1,15 @@ # Description: Tensorflow Serving core test utils. +load("//tensorflow_serving:serving.bzl", "serving_proto_library") + package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) +licenses(["notice"]) filegroup( name = "all_files", @@ -31,9 +28,16 @@ cc_library( testonly = 1, srcs = ["test_main.cc"], linkopts = ["-lm"], + tags = ["keep_dep"], # Tell build_cleaner to keep dependencies on this. deps = [ - "//external:gtest", + "@com_google_absl//absl/strings", + "@com_google_googletest//:gtest", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", "@org_tensorflow//tensorflow/core:testlib", + "@org_tensorflow//tensorflow/core/platform", + "@org_tensorflow//tensorflow/core/platform:test_benchmark", + "@org_tensorflow//tensorflow/core/platform:types", ], ) @@ -42,9 +46,9 @@ cc_library( testonly = 1, hdrs = ["mock_loader.h"], deps = [ - "//external:gtest", "//tensorflow_serving/core:loader", "//tensorflow_serving/util:any_ptr", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -68,9 +72,21 @@ cc_library( hdrs = ["fake_loader_source_adapter.h"], visibility = ["//visibility:public"], deps = [ + ":fake_loader_source_adapter_cc_proto", + "//tensorflow_serving/core:loader", "//tensorflow_serving/core:simple_loader", + "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", + "@org_tensorflow//tensorflow/core:lib", ], + alwayslink = 1, +) + +serving_proto_library( + name = "fake_loader_source_adapter_proto", + testonly = 1, + srcs = ["fake_loader_source_adapter.proto"], + visibility = ["//visibility:public"], ) cc_library( @@ -90,23 +106,9 @@ cc_library( testonly = 1, hdrs = ["mock_storage_path_target.h"], deps = [ - "//external:gtest", "//tensorflow_serving/core:storage_path", "//tensorflow_serving/core:target", - "@org_tensorflow//tensorflow/core:lib", - ], -) - -cc_library( - name = "source_adapter_test_util", - testonly = 1, - hdrs = ["source_adapter_test_util.h"], - visibility = ["//visibility:public"], - deps = [ - "//external:gtest", - "//tensorflow_serving/core:loader", - "//tensorflow_serving/core:servable_data", - "//tensorflow_serving/core:servable_id", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -118,8 +120,9 @@ cc_library( hdrs = ["availability_test_util.h"], visibility = ["//visibility:public"], deps = [ - "//external:gtest", "//tensorflow_serving/core:servable_state_monitor", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -134,3 +137,102 @@ cc_library( "//tensorflow_serving/core:caching_manager", ], ) + +cc_library( + name = "servable_handle_test_util", + testonly = 1, + hdrs = ["servable_handle_test_util.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow_serving/core:manager", + "//tensorflow_serving/core:servable_handle", + "//tensorflow_serving/core:servable_id", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "mock_log_collector", + testonly = 1, + hdrs = ["mock_log_collector.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow_serving/core:log_collector", + "@com_google_googletest//:gtest", + "@com_google_protobuf//:protobuf", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "fake_log_collector", + testonly = 1, + hdrs = ["fake_log_collector.h"], + deps = [ + "//tensorflow_serving/core:log_collector", + "@com_google_protobuf//:protobuf", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "mock_request_logger", + testonly = 1, + hdrs = ["mock_request_logger.h"], + deps = [ + "//tensorflow_serving/apis:logging_cc_proto", + "//tensorflow_serving/config:logging_config_cc_proto", + "//tensorflow_serving/core:log_collector", + "//tensorflow_serving/core:request_logger", + "@com_google_googletest//:gtest", + "@com_google_protobuf//:protobuf", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "mock_server_request_logger", + testonly = 1, + hdrs = ["mock_server_request_logger.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow_serving/core:server_request_logger", + "@com_google_googletest//:gtest", + ], +) + +cc_library( + name = "mock_session", + testonly = 1, + hdrs = ["mock_session.h"], + deps = [ + "@com_google_googletest//:gtest", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "session_test_util", + testonly = 1, + srcs = ["session_test_util.cc"], + hdrs = ["session_test_util.h"], + deps = [ + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + ], + alwayslink = 1, +) + +cc_library( + name = "mock_prediction_stream_logger", + testonly = 1, + hdrs = ["mock_prediction_stream_logger.h"], + deps = [ + "//tensorflow_serving/apis:logging_cc_proto", + "//tensorflow_serving/apis:predict_cc_proto", + "//tensorflow_serving/core:stream_logger", + "@com_google_googletest//:gtest", + ], +) diff --git a/tensorflow_serving/core/test_util/availability_test_util.cc b/tensorflow_serving/core/test_util/availability_test_util.cc index 5606809df44..05cea96a01e 100644 --- a/tensorflow_serving/core/test_util/availability_test_util.cc +++ b/tensorflow_serving/core/test_util/availability_test_util.cc @@ -14,6 +14,12 @@ limitations under the License. ==============================================================================*/ #include "tensorflow_serving/core/test_util/availability_test_util.h" + +#include +#include + +#include "absl/types/optional.h" +#include "absl/types/span.h" #include "tensorflow/core/platform/env.h" namespace tensorflow { @@ -27,7 +33,7 @@ namespace { bool ServableManagerStateIsOneOf( const ServableStateMonitor& monitor, const ServableId& servable, const std::vector& states) { - optional maybe_state = monitor.GetState(servable); + absl::optional maybe_state = monitor.GetState(servable); if (!maybe_state) { return false; } @@ -51,6 +57,16 @@ void WaitUntilServableManagerStateIsOneOf( } } +void WaitUntilVersionsAvailable( + const ServableStateMonitor& monitor, const string& servable_id_name, + absl::Span servable_id_versions) { + for (int64_t version : servable_id_versions) { + const ServableId servable_id = {servable_id_name, version}; + WaitUntilServableManagerStateIsOneOf( + monitor, servable_id, {ServableState::ManagerState::kAvailable}); + } +} + } // namespace test_util } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/test_util/availability_test_util.h b/tensorflow_serving/core/test_util/availability_test_util.h index baa724e13ec..edf84efd39c 100644 --- a/tensorflow_serving/core/test_util/availability_test_util.h +++ b/tensorflow_serving/core/test_util/availability_test_util.h @@ -31,6 +31,12 @@ void WaitUntilServableManagerStateIsOneOf( const ServableStateMonitor& monitor, const ServableId& servable, const std::vector& states); +// Waits until 'monitor' shows that the manager state servable ids is +// kAvailable. +void WaitUntilVersionsAvailable(const ServableStateMonitor& monitor, + const string& servable_id_name, + absl::Span servable_id_versions); + } // namespace test_util } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/test_util/fake_loader.cc b/tensorflow_serving/core/test_util/fake_loader.cc index 03b0c93c31e..c5692285f51 100644 --- a/tensorflow_serving/core/test_util/fake_loader.cc +++ b/tensorflow_serving/core/test_util/fake_loader.cc @@ -25,7 +25,7 @@ thread_local bool FakeLoader::was_deleted_in_this_thread_; int FakeLoader::num_fake_loaders_ = 0; mutex FakeLoader::num_fake_loaders_mu_(LINKER_INITIALIZED); -FakeLoader::FakeLoader(int64 servable, const Status load_status) +FakeLoader::FakeLoader(int64_t servable, const absl::Status load_status) : servable_(servable), load_status_(load_status) { was_deleted_in_this_thread_ = false; { @@ -42,9 +42,9 @@ FakeLoader::~FakeLoader() { was_deleted_in_this_thread_ = true; } -Status FakeLoader::load_status() { return load_status_; } +absl::Status FakeLoader::load_status() { return load_status_; } -Status FakeLoader::Load() { return load_status_; } +absl::Status FakeLoader::Load() { return load_status_; } void FakeLoader::Unload() {} diff --git a/tensorflow_serving/core/test_util/fake_loader.h b/tensorflow_serving/core/test_util/fake_loader.h index 41579c2f09d..30dc605a3fd 100644 --- a/tensorflow_serving/core/test_util/fake_loader.h +++ b/tensorflow_serving/core/test_util/fake_loader.h @@ -36,7 +36,7 @@ namespace test_util { // This class is thread-safe. class FakeLoader : public ResourceUnsafeLoader { public: - explicit FakeLoader(int64 servable, const Status load_status = Status::OK()); + explicit FakeLoader(int64_t servable, const Status load_status = Status()); ~FakeLoader() override; @@ -59,13 +59,13 @@ class FakeLoader : public ResourceUnsafeLoader { static thread_local bool was_deleted_in_this_thread_; // Counts the number of FakeLoader objects alive. - static int num_fake_loaders_ GUARDED_BY(num_fake_loaders_mu_); + static int num_fake_loaders_ TF_GUARDED_BY(num_fake_loaders_mu_); static mutex num_fake_loaders_mu_; // The servable returned from this loader. // // Don't make const or you'll have to change the handle type to 'const int64'. - int64 servable_; + int64_t servable_; // The status returned during load. const Status load_status_; }; diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc index 53fadba8aa8..dada97df8b0 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc @@ -15,6 +15,14 @@ limitations under the License. #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" +#include +#include + +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/source_adapter.h" + namespace tensorflow { namespace serving { namespace test_util { @@ -28,7 +36,7 @@ FakeLoaderSourceAdapter::FakeLoaderSourceAdapter( ? strings::StrCat(path, "/", suffix_) : path; servable_ptr->reset(new string(servable)); - return Status::OK(); + return absl::Status(); }, SimpleLoaderSourceAdapter::EstimateNoResources()), @@ -42,15 +50,20 @@ FakeLoaderSourceAdapter::~FakeLoaderSourceAdapter() { } } -std::function>>*)> -FakeLoaderSourceAdapter::GetCreator() { - return [](std::unique_ptr>>* source) { - source->reset(new FakeLoaderSourceAdapter); - return Status::OK(); - }; -} +// Register the source adapter. +class FakeLoaderSourceAdapterCreator { + public: + static absl::Status Create( + const FakeLoaderSourceAdapterConfig& config, + std::unique_ptr>>* + adapter) { + adapter->reset(new FakeLoaderSourceAdapter(config.suffix())); + return absl::Status(); + } +}; +REGISTER_STORAGE_PATH_SOURCE_ADAPTER(FakeLoaderSourceAdapterCreator, + FakeLoaderSourceAdapterConfig); + } // namespace test_util } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h index 6c460c153d7..c024d9f1c45 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h @@ -16,9 +16,14 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_SOURCE_ADAPTER_H_ #define TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_SOURCE_ADAPTER_H_ +#include + #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/simple_loader.h" #include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.pb.h" namespace tensorflow { namespace serving { @@ -43,11 +48,6 @@ class FakeLoaderSourceAdapter final ~FakeLoaderSourceAdapter() override; - // Returns a function to create a fake source adapter. - static std::function>>*)> - GetCreator(); - private: const string suffix_; std::function call_on_destruct_; diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto b/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto new file mode 100644 index 00000000000..d3d4adb1e90 --- /dev/null +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package tensorflow.serving.test_util; + +// Config proto for FakeLoaderSourceAdapter. +message FakeLoaderSourceAdapterConfig { + // FakeLoaderSourceAdapter's 'suffix' ctor parameter. + string suffix = 1; +} diff --git a/tensorflow_serving/core/test_util/fake_log_collector.h b/tensorflow_serving/core/test_util/fake_log_collector.h new file mode 100644 index 00000000000..be4ca0d7a03 --- /dev/null +++ b/tensorflow_serving/core/test_util/fake_log_collector.h @@ -0,0 +1,46 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOG_COLLECTOR_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOG_COLLECTOR_H_ + +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/log_collector.h" + +namespace tensorflow { +namespace serving { + +// FakeLogCollector which does nothing except count the number of times +// CollectMessage has been called on it. +class FakeLogCollector : public LogCollector { + public: + Status CollectMessage(const google::protobuf::Message& message) override { + ++collect_count_; + return Status(); + } + + Status Flush() override { return Status(); } + + int collect_count() const { return collect_count_; } + + private: + int collect_count_ = 0; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOG_COLLECTOR_H_ diff --git a/tensorflow_serving/core/test_util/fake_storage_path_source_adapter.cc b/tensorflow_serving/core/test_util/fake_storage_path_source_adapter.cc index 83e3c19a30a..55bc3e2b53a 100644 --- a/tensorflow_serving/core/test_util/fake_storage_path_source_adapter.cc +++ b/tensorflow_serving/core/test_util/fake_storage_path_source_adapter.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow_serving/core/test_util/fake_storage_path_source_adapter.h" +#include + #include "tensorflow/core/lib/core/errors.h" namespace tensorflow { @@ -32,7 +34,7 @@ FakeStoragePathSourceAdapter::~FakeStoragePathSourceAdapter() { } } -Status FakeStoragePathSourceAdapter::Convert( +absl::Status FakeStoragePathSourceAdapter::Convert( const StoragePath& data, StoragePath* const converted_data) { if (data == "invalid") { return errors::InvalidArgument( @@ -42,7 +44,7 @@ Status FakeStoragePathSourceAdapter::Convert( } *converted_data = suffix_.empty() ? data : strings::StrCat(data, "/", suffix_); - return Status::OK(); + return absl::Status(); } } // namespace test_util diff --git a/tensorflow_serving/core/test_util/manager_test_util.cc b/tensorflow_serving/core/test_util/manager_test_util.cc index 086f750bce6..cd5394ae74f 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.cc +++ b/tensorflow_serving/core/test_util/manager_test_util.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow_serving/core/test_util/manager_test_util.h" +#include + namespace tensorflow { namespace serving { namespace test_util { @@ -23,6 +25,10 @@ AspiredVersionsManagerTestAccess::AspiredVersionsManagerTestAccess( AspiredVersionsManager* manager) : manager_(manager) {} +void AspiredVersionsManagerTestAccess::FlushServables() { + manager_->FlushServables(); +} + void AspiredVersionsManagerTestAccess::HandlePendingAspiredVersionsRequests() { manager_->HandlePendingAspiredVersionsRequests(); } @@ -31,10 +37,35 @@ void AspiredVersionsManagerTestAccess::InvokePolicyAndExecuteAction() { manager_->InvokePolicyAndExecuteAction(); } +void AspiredVersionsManagerTestAccess::SetNumLoadThreads( + const uint32 num_load_threads) { + manager_->SetNumLoadThreads(num_load_threads); +} + +uint32 AspiredVersionsManagerTestAccess::num_load_threads() const { + return manager_->num_load_threads(); +} + +void AspiredVersionsManagerTestAccess::SetCustomSortActions( + AspiredVersionsManager::CustomSortActionsFn custom_sort_actions) { + manager_->custom_sort_actions_ = std::move(custom_sort_actions); +} + +BasicManagerTestAccess::BasicManagerTestAccess(BasicManager* manager) + : manager_(manager) {} + +void BasicManagerTestAccess::SetNumLoadThreads(const uint32 num_load_threads) { + manager_->SetNumLoadThreads(num_load_threads); +} + +uint32 BasicManagerTestAccess::num_load_threads() const { + return manager_->num_load_threads(); +} + CachingManagerTestAccess::CachingManagerTestAccess(CachingManager* manager) : manager_(manager) {} -int64 CachingManagerTestAccess::GetLoadMutexMapSize() const { +int64_t CachingManagerTestAccess::GetLoadMutexMapSize() const { return manager_->GetLoadMutexMapSize(); } diff --git a/tensorflow_serving/core/test_util/manager_test_util.h b/tensorflow_serving/core/test_util/manager_test_util.h index d84a312e2e4..1b5da2626fa 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.h +++ b/tensorflow_serving/core/test_util/manager_test_util.h @@ -29,18 +29,43 @@ class AspiredVersionsManagerTestAccess { public: explicit AspiredVersionsManagerTestAccess(AspiredVersionsManager* manager); + // Invokes FlushServables() on the manager. + void FlushServables(); + // Invokes HandlePendingAspiredVersionsRequests() on the manager. void HandlePendingAspiredVersionsRequests(); // Invokes InvokePolicyAndExecuteAction() on the manager. void InvokePolicyAndExecuteAction(); + void SetNumLoadThreads(uint32 num_load_threads); + + uint32 num_load_threads() const; + + void SetCustomSortActions( + AspiredVersionsManager::CustomSortActionsFn custom_sort_actions); + private: AspiredVersionsManager* const manager_; TF_DISALLOW_COPY_AND_ASSIGN(AspiredVersionsManagerTestAccess); }; +// A test utility that provides access to private BasicManager members. +class BasicManagerTestAccess { + public: + explicit BasicManagerTestAccess(BasicManager* manager); + + void SetNumLoadThreads(uint32 num_load_threads); + + uint32 num_load_threads() const; + + private: + BasicManager* const manager_; + + TF_DISALLOW_COPY_AND_ASSIGN(BasicManagerTestAccess); +}; + // A test utility that provides access to private CachingManager members. class CachingManagerTestAccess { public: @@ -48,7 +73,7 @@ class CachingManagerTestAccess { // Returns the size of the load-mutex map that stores the mutex reference per // servable-id requested for load. - int64 GetLoadMutexMapSize() const; + int64_t GetLoadMutexMapSize() const; private: CachingManager* const manager_; diff --git a/tensorflow_serving/core/test_util/mock_loader.h b/tensorflow_serving/core/test_util/mock_loader.h index c82f5fe86f6..8e14c4fd0a7 100644 --- a/tensorflow_serving/core/test_util/mock_loader.h +++ b/tensorflow_serving/core/test_util/mock_loader.h @@ -27,10 +27,12 @@ namespace test_util { class MockLoader : public Loader { public: - MOCK_CONST_METHOD1(EstimateResources, Status(ResourceAllocation* estimate)); - MOCK_METHOD1(Load, Status(const ResourceAllocation& available_resources)); - MOCK_METHOD0(Unload, void()); - MOCK_METHOD0(servable, AnyPtr()); + MOCK_METHOD(Status, EstimateResources, (ResourceAllocation * estimate), + (const, override)); + MOCK_METHOD(Status, Load, (), (override)); + MOCK_METHOD(Status, LoadWithMetadata, (const Metadata&), (override)); + MOCK_METHOD(void, Unload, (), (override)); + MOCK_METHOD(AnyPtr, servable, (), (override)); }; } // namespace test_util diff --git a/tensorflow_serving/core/test_util/mock_log_collector.h b/tensorflow_serving/core/test_util/mock_log_collector.h new file mode 100644 index 00000000000..3b1d4607567 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_log_collector.h @@ -0,0 +1,38 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOG_COLLECTOR_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOG_COLLECTOR_H_ + +#include "google/protobuf/message.h" +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/log_collector.h" + +namespace tensorflow { +namespace serving { + +class MockLogCollector : public LogCollector { + public: + MockLogCollector() = default; + MOCK_METHOD(Status, CollectMessage, (const google::protobuf::Message& message), + (override)); + MOCK_METHOD(Status, Flush, (), (override)); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOG_COLLECTOR_H_ diff --git a/tensorflow_serving/core/test_util/mock_prediction_stream_logger.h b/tensorflow_serving/core/test_util/mock_prediction_stream_logger.h new file mode 100644 index 00000000000..ce7cb8491a8 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_prediction_stream_logger.h @@ -0,0 +1,46 @@ +/* Copyright 2023 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef THIRD_PARTY_TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_PREDICTION_STREAM_LOGGER_H_ +#define THIRD_PARTY_TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_PREDICTION_STREAM_LOGGER_H_ + +#include + +#include +#include "tensorflow_serving/apis/logging.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" +#include "tensorflow_serving/core/stream_logger.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +class MockPredictionStreamLogger + : public StreamLogger { + public: + ~MockPredictionStreamLogger() override = default; + + MOCK_METHOD(void, LogStreamRequest, (PredictRequest), (override)); + MOCK_METHOD(void, LogStreamResponse, (PredictResponse), (override)); + MOCK_METHOD(absl::Status, CreateLogMessage, + (const LogMetadata&, std::unique_ptr*), + (override)); +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_PREDICTION_STREAM_LOGGER_H_ diff --git a/tensorflow_serving/core/test_util/mock_request_logger.h b/tensorflow_serving/core/test_util/mock_request_logger.h new file mode 100644 index 00000000000..20eae1fe381 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_request_logger.h @@ -0,0 +1,66 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_REQUEST_LOGGER_H_ + +#include + +#include "google/protobuf/message.h" +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/logging.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/log_collector.h" +#include "tensorflow_serving/core/request_logger.h" + +namespace tensorflow { +namespace serving { + +class MockRequestLogger : public RequestLogger { + public: + // Unfortunately NiceMock doesn't support ctors with move-only types, so we + // have to do this workaround. + MockRequestLogger(const LoggingConfig& logging_config, + const std::vector& saved_model_tags, + LogCollector* log_collector, + std::function notify_destruction = + std::function()) + : RequestLogger(logging_config, saved_model_tags, + std::unique_ptr(log_collector)), + notify_destruction_(std::move(notify_destruction)) {} + + virtual ~MockRequestLogger() { + if (notify_destruction_) { + notify_destruction_(); + } + } + + MOCK_METHOD(Status, CreateLogMessage, + (const google::protobuf::Message& request, const google::protobuf::Message& response, + const LogMetadata& log_metadata, + std::unique_ptr* log), + (override)); + + MOCK_METHOD(LogMetadata, FillLogMetadata, (const LogMetadata&), (override)); + + private: + std::function notify_destruction_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_REQUEST_LOGGER_H_ diff --git a/tensorflow_serving/core/test_util/mock_server_request_logger.h b/tensorflow_serving/core/test_util/mock_server_request_logger.h new file mode 100644 index 00000000000..fdfe0d78f63 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_server_request_logger.h @@ -0,0 +1,48 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SERVER_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SERVER_REQUEST_LOGGER_H_ + +#include +#include + +#include +#include "tensorflow_serving/core/server_request_logger.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +class MockServerRequestLogger : public ServerRequestLogger { + public: + MockServerRequestLogger() : ServerRequestLogger({}) {} + + MOCK_METHOD(Status, Update, + ((const std::map>& + logging_config_map)), + (override)); + + MOCK_METHOD(Status, Log, + (const google::protobuf::Message& request, const google::protobuf::Message& response, + const LogMetadata& log_metadata), + (override)); +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SERVER_REQUEST_LOGGER_H_ diff --git a/tensorflow_serving/core/test_util/mock_session.h b/tensorflow_serving/core/test_util/mock_session.h new file mode 100644 index 00000000000..4e66287c284 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_session.h @@ -0,0 +1,82 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SESSION_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SESSION_H_ + +#include +#include "tensorflow/core/platform/threadpool_options.h" +#include "tensorflow/core/public/session.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// A mock of tensorflow::Session. +class MockSession : public tensorflow::Session { + public: + MockSession() : Session() { + ON_CALL(*this, Close()).WillByDefault(::testing::Return(Status())); + } + MOCK_METHOD(::tensorflow::Status, Create, (const GraphDef& graph), + (override)); + MOCK_METHOD(::tensorflow::Status, Extend, (const GraphDef& graph), + (override)); + MOCK_METHOD(::tensorflow::Status, Run, + ((const std::vector>& inputs), + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs), + (override)); + MOCK_METHOD(::tensorflow::Status, Run, + (const RunOptions& run_options, + (const std::vector>& inputs), + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs, RunMetadata* run_metadata), + (override)); + MOCK_METHOD( + ::tensorflow::Status, Run, + (const RunOptions& run_options, + (const std::vector>& inputs), + const std::vector& output_names, + const std::vector& target_nodes, std::vector* outputs, + RunMetadata* run_metadata, + const tensorflow::thread::ThreadPoolOptions& thread_pool_options), + (override)); + MOCK_METHOD(::tensorflow::Status, PRunSetup, + (const std::vector& input_names, + const std::vector& output_names, + const std::vector& target_nodes, string* handle), + (override)); + MOCK_METHOD(::tensorflow::Status, PRun, + (const string& handle, + (const std::vector>& inputs), + const std::vector& output_names, + std::vector* outputs), + (override)); + + MOCK_METHOD(::tensorflow::Status, ListDevices, + (std::vector<::tensorflow::DeviceAttributes> * response), + (override)); + + MOCK_METHOD(::tensorflow::Status, Close, (), (override)); +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SESSION_H_ diff --git a/tensorflow_serving/core/test_util/mock_storage_path_target.h b/tensorflow_serving/core/test_util/mock_storage_path_target.h index fc9af700a8a..449ba21e4c0 100644 --- a/tensorflow_serving/core/test_util/mock_storage_path_target.h +++ b/tensorflow_serving/core/test_util/mock_storage_path_target.h @@ -30,8 +30,9 @@ namespace test_util { class MockStoragePathTarget : public TargetBase { public: ~MockStoragePathTarget() override { Detach(); } - MOCK_METHOD2(SetAspiredVersions, - void(const StringPiece, std::vector>)); + MOCK_METHOD(void, SetAspiredVersions, + (const StringPiece, std::vector>), + (override)); }; } // namespace test_util diff --git a/tensorflow_serving/core/test_util/servable_handle_test_util.h b/tensorflow_serving/core/test_util/servable_handle_test_util.h new file mode 100644 index 00000000000..6495f0c1942 --- /dev/null +++ b/tensorflow_serving/core/test_util/servable_handle_test_util.h @@ -0,0 +1,99 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_SERVABLE_HANDLE_TEST_UTIL_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_SERVABLE_HANDLE_TEST_UTIL_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/core/servable_id.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// Wraps a pointer as a servable. +// +// The ownership of @p t is *not* transferred to this function; t must outlive +// the returned ServableHandle. +// +// Does a bit of type-gymnastics since ServableHandles can only be constructed +// by Managers. +template +static ServableHandle WrapAsHandle(const ServableId& id, T* t) { + // A basic Handle that wraps a ServableId and a pointer to a servable. + // + // Exactly the same objects that are used to construct the DummyHandle are + // returned when the appropriate getter functions are invoked. + class DummyHandle : public UntypedServableHandle { + public: + explicit DummyHandle(const ServableId& id, T* servable) + : id_(id), servable_(servable) {} + + AnyPtr servable() override { return servable_; } + + const ServableId& id() const override { return id_; } + + private: + const ServableId id_; + T* servable_; + }; + + // A Manager that always returns the same servable when + // GetUntypedServableHandle is invoked. + class DummyManager : public Manager { + public: + explicit DummyManager(const ServableId& id, T* servable) + : id_(id), servable_(servable) {} + + // Resets the UntypedServableHandle to a new DummyHandle that wraps the + // Servable and ServableId which this DummyManager was created with. + // + // Always returns OK status. + Status GetUntypedServableHandle( + const ServableRequest& request, + std::unique_ptr* result) override { + result->reset(new DummyHandle(id_, servable_)); + return Status(); + } + + // Unimplemented: always returns an empty map. + std::map> + GetAvailableUntypedServableHandles() const override { + return {}; + } + + // Unimplemented: always returns an empty vector. + std::vector ListAvailableServableIds() const override { + return {}; + } + + private: + const ServableId id_; + T* servable_; + }; + + DummyManager manager{id, t}; + ServableHandle handle; + TF_CHECK_OK(manager.GetServableHandle({"Dummy", 0}, &handle)); + return handle; +} + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_SERVABLE_HANDLE_TEST_UTIL_H_ diff --git a/tensorflow_serving/core/test_util/session_test_util.cc b/tensorflow_serving/core/test_util/session_test_util.cc new file mode 100644 index 00000000000..d9f4b91a690 --- /dev/null +++ b/tensorflow_serving/core/test_util/session_test_util.cc @@ -0,0 +1,87 @@ +/* Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/test_util/session_test_util.h" + +#include +#include +#include + +#include "absl/memory/memory.h" +#include "absl/strings/strip.h" +#include "tensorflow/core/common_runtime/session_factory.h" +#include "tensorflow/core/public/session.h" + +namespace tensorflow { +namespace serving { +namespace test_util { +namespace { + +using NewSessionHook = std::function; +NewSessionHook new_session_hook_; + +NewSessionHook GetNewSessionHook() { return new_session_hook_; } + +// A DelegatingSessionFactory is used to setup the new-session-hook. +// +// This SessionFactory accepts "new_session_hook/" as the +// session target. While returning the created session, it calls the +// new-session-hook and returns the session created when target is +// "". +class DelegatingSessionFactory : public SessionFactory { + public: + DelegatingSessionFactory() {} + + bool AcceptsOptions(const SessionOptions& options) override { + return absl::StartsWith(options.target, "new_session_hook/"); + } + + absl::Status NewSession(const SessionOptions& options, + Session** out_session) override { + auto actual_session_options = options; + actual_session_options.target = std::string( + absl::StripPrefix(options.target, kNewSessionHookSessionTargetPrefix)); + auto new_session_hook = GetNewSessionHook(); + if (new_session_hook) { + TF_RETURN_IF_ERROR(new_session_hook(actual_session_options)); + } + Session* actual_session; + TF_RETURN_IF_ERROR( + tensorflow::NewSession(actual_session_options, &actual_session)); + *out_session = actual_session; + return absl::Status(); + } +}; + +class DelegatingSessionRegistrar { + public: + DelegatingSessionRegistrar() { + SessionFactory::Register("DELEGATING_SESSION", + new DelegatingSessionFactory()); + } +}; +static DelegatingSessionRegistrar registrar; + +} // namespace + +const char kNewSessionHookSessionTargetPrefix[] = "new_session_hook/"; + +void SetNewSessionHook(NewSessionHook hook) { + new_session_hook_ = std::move(hook); +} + +} // namespace test_util +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/test_util/session_test_util.h b/tensorflow_serving/core/test_util/session_test_util.h new file mode 100644 index 00000000000..13d795c40fe --- /dev/null +++ b/tensorflow_serving/core/test_util/session_test_util.h @@ -0,0 +1,48 @@ +/* Copyright 2019 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_SESSION_TEST_UTIL_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_SESSION_TEST_UTIL_H_ + +#include + +#include "absl/base/attributes.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/public/session_options.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// Sets a 'hook' function, which will be called when a new session is created +// via the tensorflow::NewSession() API. If the hook returns an error status, +// the session creation fails. +// +// For this hook to be enabled, create a session by setting +// SessionOptions::target as "new_session_hook/". This +// will call the hook as well as return the session created when target is +// "". +// +// Calling this method again replaces the previous hook. +// +// This method is NOT thread-safe. +ABSL_CONST_INIT extern const char kNewSessionHookSessionTargetPrefix[]; +void SetNewSessionHook(std::function hook); + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_SESSION_TEST_UTIL_H_ diff --git a/tensorflow_serving/core/test_util/source_adapter_test_util.h b/tensorflow_serving/core/test_util/source_adapter_test_util.h deleted file mode 100644 index 25f98347162..00000000000 --- a/tensorflow_serving/core/test_util/source_adapter_test_util.h +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_SOURCE_ADAPTER_TEST_UTIL_H_ -#define TENSORFLOW_SERVING_CORE_TEST_UTIL_SOURCE_ADAPTER_TEST_UTIL_H_ - -#include -#include -#include - -#include -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/core/loader.h" -#include "tensorflow_serving/core/servable_data.h" -#include "tensorflow_serving/core/servable_id.h" - -namespace tensorflow { -namespace serving { -template -class SourceAdapter; -} // namespace serving -} // namespace tensorflow - -namespace tensorflow { -namespace serving { -namespace test_util { - -// Takes a SourceAdapter, and arranges to push a single data item of type -// InputType through its aspired-versions API. Returns the resulting -// ServableData object (which has a trivial servable id of {"", 0}). -// -// Assumes 'adapter->SetAspiredVersionsCallback()' has not yet been called. -// Assumes 'adapter->SetAspiredVersions()' is synchronous and internally calls -// the outgoing aspired-versions callback. -// -// Mutates 'adapter' and leaves it in an unspecified state. -template -ServableData RunSourceAdapter( - const InputType& in, SourceAdapter* adapter); - -////////// -// Implementation details follow. API users need not read. - -template -ServableData RunSourceAdapter( - const InputType& in, SourceAdapter* adapter) { - ServableId servable_id = {"", 0}; - std::vector> servable_data; - bool outgoing_callback_called = false; - adapter->SetAspiredVersionsCallback( - [&servable_id, &servable_data, &outgoing_callback_called]( - const StringPiece servable_name, - std::vector> versions) { - outgoing_callback_called = true; - CHECK_EQ(servable_id.name, servable_name); - servable_data = std::move(versions); - }); - adapter->SetAspiredVersions(servable_id.name, - {CreateServableData(servable_id, in)}); - CHECK(outgoing_callback_called) - << "Supplied adapter appears to have asynchronous behavior"; - CHECK_EQ(1, servable_data.size()); - CHECK_EQ(servable_id, servable_data[0].id()); - return std::move(servable_data[0]); -} - -} // namespace test_util -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_SOURCE_ADAPTER_TEST_UTIL_H_ diff --git a/tensorflow_serving/core/test_util/test_main.cc b/tensorflow_serving/core/test_util/test_main.cc index b2b05cc74c9..3b718a68423 100644 --- a/tensorflow_serving/core/test_util/test_main.cc +++ b/tensorflow_serving/core/test_util/test_main.cc @@ -24,21 +24,39 @@ limitations under the License. #if defined(PLATFORM_GOOGLE) || defined(__ANDROID__) // main() is supplied by gunit_main #else +#include +#include + #include "gtest/gtest.h" -#include "tensorflow/core/lib/core/stringpiece.h" +#include "absl/strings/match.h" +#include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/stacktrace_handler.h" +#include "tensorflow/core/platform/test.h" #include "tensorflow/core/platform/test_benchmark.h" GTEST_API_ int main(int argc, char** argv) { std::cout << "Running main() from test_main.cc\n"; - testing::InitGoogleTest(&argc, argv); + tensorflow::testing::InstallStacktraceHandler(); + for (int i = 1; i < argc; i++) { - if (tensorflow::StringPiece(argv[i]).starts_with("--benchmarks=")) { - const char* pattern = argv[i] + strlen("--benchmarks="); - tensorflow::testing::Benchmark::Run(pattern); + if (absl::StartsWith(argv[i], "--benchmark_filter=")) { + tensorflow::testing::InitializeBenchmarks(&argc, argv); + + // XXX: Must be called after benchmark's init because + // InitGoogleTest eventually calls absl::ParseCommandLine() which would + // complain that benchmark_filter flag is not known because that flag is + // defined by the benchmark library via its own command-line flag + // facility, which is not known to absl flags. + // FIXME(b/195442215): Fix this mess once we make benchmark use absl flags + testing::InitGoogleTest(&argc, argv); + tensorflow::testing::RunBenchmarks(); return 0; } } + + testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } + #endif diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 9ea4a93c5c1..3ea144f96fe 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -2,17 +2,10 @@ package( default_visibility = ["//tensorflow_serving:internal"], - features = [ - "-parse_headers", - "no_layering_check", - ], + features = ["no_layering_check"], ) -licenses(["notice"]) # Apache 2.0 - -exports_files(["LICENSE"]) - -load("//tensorflow_serving:serving.bzl", "serving_proto_library") +licenses(["notice"]) filegroup( name = "all_files", @@ -25,140 +18,17 @@ filegroup( ), ) -serving_proto_library( - name = "mnist_inference_proto", - srcs = ["mnist_inference.proto"], - has_services = 1, - cc_api_version = 2, - cc_grpc_version = 1, -) - -py_library( - name = "mnist_input_data", - srcs = ["mnist_input_data.py"], -) - -py_binary( - name = "mnist_export", - srcs = [ - "mnist_export.py", - ], - deps = [ - ":mnist_input_data", - "@org_tensorflow//tensorflow:tensorflow_py", - "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", - ], -) - cc_binary( - name = "mnist_inference", + name = "resnet_client_cc", srcs = [ - "mnist_inference.cc", + "resnet_client.cc", ], - linkopts = ["-lm"], deps = [ - ":mnist_inference_proto", - "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", - "//tensorflow_serving/servables/tensorflow:session_bundle_factory", - "@grpc//:grpc++", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:core_cpu", + "//tensorflow_serving/apis:prediction_service_cc_proto", + "@com_github_grpc_grpc//:grpc++", + "@com_google_protobuf//:protobuf_lite", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - -cc_binary( - name = "mnist_inference_2", - srcs = [ - "mnist_inference_2.cc", - ], - linkopts = ["-lm"], - deps = [ - ":mnist_inference_proto", - "//tensorflow_serving/batching:basic_batch_scheduler", - "//tensorflow_serving/batching:batch_scheduler", - "//tensorflow_serving/core:manager", - "//tensorflow_serving/core:servable_handle", - "//tensorflow_serving/core:servable_id", - "//tensorflow_serving/servables/tensorflow:simple_servers", - "@grpc//:grpc++", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - -py_binary( - name = "mnist_client", - srcs = [ - "mnist_client.py", - "mnist_inference_pb2.py", - ], - deps = [ - ":mnist_input_data", - "@org_tensorflow//tensorflow:tensorflow_py", - ], -) - -serving_proto_library( - name = "inception_inference_proto", - srcs = ["inception_inference.proto"], - has_services = 1, - cc_api_version = 2, - cc_grpc_version = 1, -) - -py_binary( - name = "inception_export", - srcs = [ - "inception_export.py", - ], - deps = [ - "@inception_model//inception", - "@org_tensorflow//tensorflow:tensorflow_py", - "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", - ], -) - -cc_binary( - name = "inception_inference", - srcs = [ - "inception_inference.cc", - ], - linkopts = ["-lm"], - deps = [ - ":inception_inference_proto", - "//tensorflow_serving/batching:basic_batch_scheduler", - "//tensorflow_serving/batching:batch_scheduler", - "//tensorflow_serving/core:manager", - "//tensorflow_serving/core:servable_handle", - "//tensorflow_serving/core:servable_id", - "//tensorflow_serving/servables/tensorflow:simple_servers", - "@grpc//:grpc++", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - -py_binary( - name = "inception_client", - srcs = [ - "inception_client.py", - "inception_inference_pb2.py", + "@org_tensorflow//tensorflow/core/platform:jpeg", ], - deps = ["@org_tensorflow//tensorflow:tensorflow_py"], ) diff --git a/tensorflow_serving/example/imagenet_lsvrc_2015_synsets.txt b/tensorflow_serving/example/imagenet_lsvrc_2015_synsets.txt deleted file mode 100644 index 88aa58f966b..00000000000 --- a/tensorflow_serving/example/imagenet_lsvrc_2015_synsets.txt +++ /dev/null @@ -1,1000 +0,0 @@ -n01440764 -n01443537 -n01484850 -n01491361 -n01494475 -n01496331 -n01498041 -n01514668 -n01514859 -n01518878 -n01530575 -n01531178 -n01532829 -n01534433 -n01537544 -n01558993 -n01560419 -n01580077 -n01582220 -n01592084 -n01601694 -n01608432 -n01614925 -n01616318 -n01622779 -n01629819 -n01630670 -n01631663 -n01632458 -n01632777 -n01641577 -n01644373 -n01644900 -n01664065 -n01665541 -n01667114 -n01667778 -n01669191 -n01675722 -n01677366 -n01682714 -n01685808 -n01687978 -n01688243 -n01689811 -n01692333 -n01693334 -n01694178 -n01695060 -n01697457 -n01698640 -n01704323 -n01728572 -n01728920 -n01729322 -n01729977 -n01734418 -n01735189 -n01737021 -n01739381 -n01740131 -n01742172 -n01744401 -n01748264 -n01749939 -n01751748 -n01753488 -n01755581 -n01756291 -n01768244 -n01770081 -n01770393 -n01773157 -n01773549 -n01773797 -n01774384 -n01774750 -n01775062 -n01776313 -n01784675 -n01795545 -n01796340 -n01797886 -n01798484 -n01806143 -n01806567 -n01807496 -n01817953 -n01818515 -n01819313 -n01820546 -n01824575 -n01828970 -n01829413 -n01833805 -n01843065 -n01843383 -n01847000 -n01855032 -n01855672 -n01860187 -n01871265 -n01872401 -n01873310 -n01877812 -n01882714 -n01883070 -n01910747 -n01914609 -n01917289 -n01924916 -n01930112 -n01943899 -n01944390 -n01945685 -n01950731 -n01955084 -n01968897 -n01978287 -n01978455 -n01980166 -n01981276 -n01983481 -n01984695 -n01985128 -n01986214 -n01990800 -n02002556 -n02002724 -n02006656 -n02007558 -n02009229 -n02009912 -n02011460 -n02012849 -n02013706 -n02017213 -n02018207 -n02018795 -n02025239 -n02027492 -n02028035 -n02033041 -n02037110 -n02051845 -n02056570 -n02058221 -n02066245 -n02071294 -n02074367 -n02077923 -n02085620 -n02085782 -n02085936 -n02086079 -n02086240 -n02086646 -n02086910 -n02087046 -n02087394 -n02088094 -n02088238 -n02088364 -n02088466 -n02088632 -n02089078 -n02089867 -n02089973 -n02090379 -n02090622 -n02090721 -n02091032 -n02091134 -n02091244 -n02091467 -n02091635 -n02091831 -n02092002 -n02092339 -n02093256 -n02093428 -n02093647 -n02093754 -n02093859 -n02093991 -n02094114 -n02094258 -n02094433 -n02095314 -n02095570 -n02095889 -n02096051 -n02096177 -n02096294 -n02096437 -n02096585 -n02097047 -n02097130 -n02097209 -n02097298 -n02097474 -n02097658 -n02098105 -n02098286 -n02098413 -n02099267 -n02099429 -n02099601 -n02099712 -n02099849 -n02100236 -n02100583 -n02100735 -n02100877 -n02101006 -n02101388 -n02101556 -n02102040 -n02102177 -n02102318 -n02102480 -n02102973 -n02104029 -n02104365 -n02105056 -n02105162 -n02105251 -n02105412 -n02105505 -n02105641 -n02105855 -n02106030 -n02106166 -n02106382 -n02106550 -n02106662 -n02107142 -n02107312 -n02107574 -n02107683 -n02107908 -n02108000 -n02108089 -n02108422 -n02108551 -n02108915 -n02109047 -n02109525 -n02109961 -n02110063 -n02110185 -n02110341 -n02110627 -n02110806 -n02110958 -n02111129 -n02111277 -n02111500 -n02111889 -n02112018 -n02112137 -n02112350 -n02112706 -n02113023 -n02113186 -n02113624 -n02113712 -n02113799 -n02113978 -n02114367 -n02114548 -n02114712 -n02114855 -n02115641 -n02115913 -n02116738 -n02117135 -n02119022 -n02119789 -n02120079 -n02120505 -n02123045 -n02123159 -n02123394 -n02123597 -n02124075 -n02125311 -n02127052 -n02128385 -n02128757 -n02128925 -n02129165 -n02129604 -n02130308 -n02132136 -n02133161 -n02134084 -n02134418 -n02137549 -n02138441 -n02165105 -n02165456 -n02167151 -n02168699 -n02169497 -n02172182 -n02174001 -n02177972 -n02190166 -n02206856 -n02219486 -n02226429 -n02229544 -n02231487 -n02233338 -n02236044 -n02256656 -n02259212 -n02264363 -n02268443 -n02268853 -n02276258 -n02277742 -n02279972 -n02280649 -n02281406 -n02281787 -n02317335 -n02319095 -n02321529 -n02325366 -n02326432 -n02328150 -n02342885 -n02346627 -n02356798 -n02361337 -n02363005 -n02364673 -n02389026 -n02391049 -n02395406 -n02396427 -n02397096 -n02398521 -n02403003 -n02408429 -n02410509 -n02412080 -n02415577 -n02417914 -n02422106 -n02422699 -n02423022 -n02437312 -n02437616 -n02441942 -n02442845 -n02443114 -n02443484 -n02444819 -n02445715 -n02447366 -n02454379 -n02457408 -n02480495 -n02480855 -n02481823 -n02483362 -n02483708 -n02484975 -n02486261 -n02486410 -n02487347 -n02488291 -n02488702 -n02489166 -n02490219 -n02492035 -n02492660 -n02493509 -n02493793 -n02494079 -n02497673 -n02500267 -n02504013 -n02504458 -n02509815 -n02510455 -n02514041 -n02526121 -n02536864 -n02606052 -n02607072 -n02640242 -n02641379 -n02643566 -n02655020 -n02666196 -n02667093 -n02669723 -n02672831 -n02676566 -n02687172 -n02690373 -n02692877 -n02699494 -n02701002 -n02704792 -n02708093 -n02727426 -n02730930 -n02747177 -n02749479 -n02769748 -n02776631 -n02777292 -n02782093 -n02783161 -n02786058 -n02787622 -n02788148 -n02790996 -n02791124 -n02791270 -n02793495 -n02794156 -n02795169 -n02797295 -n02799071 -n02802426 -n02804414 -n02804610 -n02807133 -n02808304 -n02808440 -n02814533 -n02814860 -n02815834 -n02817516 -n02823428 -n02823750 -n02825657 -n02834397 -n02835271 -n02837789 -n02840245 -n02841315 -n02843684 -n02859443 -n02860847 -n02865351 -n02869837 -n02870880 -n02871525 -n02877765 -n02879718 -n02883205 -n02892201 -n02892767 -n02894605 -n02895154 -n02906734 -n02909870 -n02910353 -n02916936 -n02917067 -n02927161 -n02930766 -n02939185 -n02948072 -n02950826 -n02951358 -n02951585 -n02963159 -n02965783 -n02966193 -n02966687 -n02971356 -n02974003 -n02977058 -n02978881 -n02979186 -n02980441 -n02981792 -n02988304 -n02992211 -n02992529 -n02999410 -n03000134 -n03000247 -n03000684 -n03014705 -n03016953 -n03017168 -n03018349 -n03026506 -n03028079 -n03032252 -n03041632 -n03042490 -n03045698 -n03047690 -n03062245 -n03063599 -n03063689 -n03065424 -n03075370 -n03085013 -n03089624 -n03095699 -n03100240 -n03109150 -n03110669 -n03124043 -n03124170 -n03125729 -n03126707 -n03127747 -n03127925 -n03131574 -n03133878 -n03134739 -n03141823 -n03146219 -n03160309 -n03179701 -n03180011 -n03187595 -n03188531 -n03196217 -n03197337 -n03201208 -n03207743 -n03207941 -n03208938 -n03216828 -n03218198 -n03220513 -n03223299 -n03240683 -n03249569 -n03250847 -n03255030 -n03259280 -n03271574 -n03272010 -n03272562 -n03290653 -n03291819 -n03297495 -n03314780 -n03325584 -n03337140 -n03344393 -n03345487 -n03347037 -n03355925 -n03372029 -n03376595 -n03379051 -n03384352 -n03388043 -n03388183 -n03388549 -n03393912 -n03394916 -n03400231 -n03404251 -n03417042 -n03424325 -n03425413 -n03443371 -n03444034 -n03445777 -n03445924 -n03447447 -n03447721 -n03450230 -n03452741 -n03457902 -n03459775 -n03461385 -n03467068 -n03476684 -n03476991 -n03478589 -n03481172 -n03482405 -n03483316 -n03485407 -n03485794 -n03492542 -n03494278 -n03495258 -n03496892 -n03498962 -n03527444 -n03529860 -n03530642 -n03532672 -n03534580 -n03535780 -n03538406 -n03544143 -n03584254 -n03584829 -n03590841 -n03594734 -n03594945 -n03595614 -n03598930 -n03599486 -n03602883 -n03617480 -n03623198 -n03627232 -n03630383 -n03633091 -n03637318 -n03642806 -n03649909 -n03657121 -n03658185 -n03661043 -n03662601 -n03666591 -n03670208 -n03673027 -n03676483 -n03680355 -n03690938 -n03691459 -n03692522 -n03697007 -n03706229 -n03709823 -n03710193 -n03710637 -n03710721 -n03717622 -n03720891 -n03721384 -n03724870 -n03729826 -n03733131 -n03733281 -n03733805 -n03742115 -n03743016 -n03759954 -n03761084 -n03763968 -n03764736 -n03769881 -n03770439 -n03770679 -n03773504 -n03775071 -n03775546 -n03776460 -n03777568 -n03777754 -n03781244 -n03782006 -n03785016 -n03786901 -n03787032 -n03788195 -n03788365 -n03791053 -n03792782 -n03792972 -n03793489 -n03794056 -n03796401 -n03803284 -n03804744 -n03814639 -n03814906 -n03825788 -n03832673 -n03837869 -n03838899 -n03840681 -n03841143 -n03843555 -n03854065 -n03857828 -n03866082 -n03868242 -n03868863 -n03871628 -n03873416 -n03874293 -n03874599 -n03876231 -n03877472 -n03877845 -n03884397 -n03887697 -n03888257 -n03888605 -n03891251 -n03891332 -n03895866 -n03899768 -n03902125 -n03903868 -n03908618 -n03908714 -n03916031 -n03920288 -n03924679 -n03929660 -n03929855 -n03930313 -n03930630 -n03933933 -n03935335 -n03937543 -n03938244 -n03942813 -n03944341 -n03947888 -n03950228 -n03954731 -n03956157 -n03958227 -n03961711 -n03967562 -n03970156 -n03976467 -n03976657 -n03977966 -n03980874 -n03982430 -n03983396 -n03991062 -n03992509 -n03995372 -n03998194 -n04004767 -n04005630 -n04008634 -n04009552 -n04019541 -n04023962 -n04026417 -n04033901 -n04033995 -n04037443 -n04039381 -n04040759 -n04041544 -n04044716 -n04049303 -n04065272 -n04067472 -n04069434 -n04070727 -n04074963 -n04081281 -n04086273 -n04090263 -n04099969 -n04111531 -n04116512 -n04118538 -n04118776 -n04120489 -n04125021 -n04127249 -n04131690 -n04133789 -n04136333 -n04141076 -n04141327 -n04141975 -n04146614 -n04147183 -n04149813 -n04152593 -n04153751 -n04154565 -n04162706 -n04179913 -n04192698 -n04200800 -n04201297 -n04204238 -n04204347 -n04208210 -n04209133 -n04209239 -n04228054 -n04229816 -n04235860 -n04238763 -n04239074 -n04243546 -n04251144 -n04252077 -n04252225 -n04254120 -n04254680 -n04254777 -n04258138 -n04259630 -n04263257 -n04264628 -n04265275 -n04266014 -n04270147 -n04273569 -n04275548 -n04277352 -n04285008 -n04286575 -n04296562 -n04310018 -n04311004 -n04311174 -n04317175 -n04325704 -n04326547 -n04328186 -n04330267 -n04332243 -n04335435 -n04336792 -n04344873 -n04346328 -n04347754 -n04350905 -n04355338 -n04355933 -n04356056 -n04357314 -n04366367 -n04367480 -n04370456 -n04371430 -n04371774 -n04372370 -n04376876 -n04380533 -n04389033 -n04392985 -n04398044 -n04399382 -n04404412 -n04409515 -n04417672 -n04418357 -n04423845 -n04428191 -n04429376 -n04435653 -n04442312 -n04443257 -n04447861 -n04456115 -n04458633 -n04461696 -n04462240 -n04465501 -n04467665 -n04476259 -n04479046 -n04482393 -n04483307 -n04485082 -n04486054 -n04487081 -n04487394 -n04493381 -n04501370 -n04505470 -n04507155 -n04509417 -n04515003 -n04517823 -n04522168 -n04523525 -n04525038 -n04525305 -n04532106 -n04532670 -n04536866 -n04540053 -n04542943 -n04548280 -n04548362 -n04550184 -n04552348 -n04553703 -n04554684 -n04557648 -n04560804 -n04562935 -n04579145 -n04579432 -n04584207 -n04589890 -n04590129 -n04591157 -n04591713 -n04592741 -n04596742 -n04597913 -n04599235 -n04604644 -n04606251 -n04612504 -n04613696 -n06359193 -n06596364 -n06785654 -n06794110 -n06874185 -n07248320 -n07565083 -n07579787 -n07583066 -n07584110 -n07590611 -n07613480 -n07614500 -n07615774 -n07684084 -n07693725 -n07695742 -n07697313 -n07697537 -n07711569 -n07714571 -n07714990 -n07715103 -n07716358 -n07716906 -n07717410 -n07717556 -n07718472 -n07718747 -n07720875 -n07730033 -n07734744 -n07742313 -n07745940 -n07747607 -n07749582 -n07753113 -n07753275 -n07753592 -n07754684 -n07760859 -n07768694 -n07802026 -n07831146 -n07836838 -n07860988 -n07871810 -n07873807 -n07875152 -n07880968 -n07892512 -n07920052 -n07930864 -n07932039 -n09193705 -n09229709 -n09246464 -n09256479 -n09288635 -n09332890 -n09399592 -n09421951 -n09428293 -n09468604 -n09472597 -n09835506 -n10148035 -n10565667 -n11879895 -n11939491 -n12057211 -n12144580 -n12267677 -n12620546 -n12768682 -n12985857 -n12998815 -n13037406 -n13040303 -n13044778 -n13052670 -n13054560 -n13133613 -n15075141 diff --git a/tensorflow_serving/example/imagenet_metadata.txt b/tensorflow_serving/example/imagenet_metadata.txt deleted file mode 100644 index 913a237e95d..00000000000 --- a/tensorflow_serving/example/imagenet_metadata.txt +++ /dev/null @@ -1,21842 +0,0 @@ -n00004475 organism, being -n00005787 benthos -n00006024 heterotroph -n00006484 cell -n00007846 person, individual, someone, somebody, mortal, soul -n00015388 animal, animate being, beast, brute, creature, fauna -n00017222 plant, flora, plant life -n00021265 food, nutrient -n00021939 artifact, artefact -n00120010 hop -n00141669 check-in -n00288000 dressage -n00288190 curvet, vaulting -n00288384 piaffe -n00324978 funambulism, tightrope walking -n00326094 rock climbing -n00433458 contact sport -n00433661 outdoor sport, field sport -n00433802 gymnastics, gymnastic exercise -n00434075 acrobatics, tumbling -n00439826 track and field -n00440039 track, running -n00440218 jumping -n00440382 broad jump, long jump -n00440509 high jump -n00440643 Fosbury flop -n00440747 skiing -n00440941 cross-country skiing -n00441073 ski jumping -n00441824 water sport, aquatics -n00442115 swimming, swim -n00442437 bathe -n00442847 dip, plunge -n00442981 dive, diving -n00443231 floating, natation -n00443375 dead-man's float, prone float -n00443517 belly flop, belly flopper, belly whop, belly whopper -n00443692 cliff diving -n00443803 flip -n00443917 gainer, full gainer -n00444142 half gainer -n00444340 jackknife -n00444490 swan dive, swallow dive -n00444651 skin diving, skin-dive -n00444846 scuba diving -n00444937 snorkeling, snorkel diving -n00445055 surfing, surfboarding, surfriding -n00445226 water-skiing -n00445351 rowing, row -n00445685 sculling -n00445802 boxing, pugilism, fisticuffs -n00446311 professional boxing -n00446411 in-fighting -n00446493 fight -n00446632 rope-a-dope -n00446804 spar, sparring -n00446980 archery -n00447073 sledding -n00447221 tobogganing -n00447361 luging -n00447463 bobsledding -n00447540 wrestling, rassling, grappling -n00447957 Greco-Roman wrestling -n00448126 professional wrestling -n00448232 sumo -n00448466 skating -n00448640 ice skating -n00448748 figure skating -n00448872 rollerblading -n00448958 roller skating -n00449054 skateboarding -n00449168 speed skating -n00449295 racing -n00449517 auto racing, car racing -n00449695 boat racing -n00449796 hydroplane racing -n00449892 camel racing -n00449977 greyhound racing -n00450070 horse racing -n00450335 riding, horseback riding, equitation -n00450700 equestrian sport -n00450866 pony-trekking -n00450998 showjumping, stadium jumping -n00451186 cross-country riding, cross-country jumping -n00451370 cycling -n00451563 bicycling -n00451635 motorcycling -n00451768 dune cycling -n00451866 blood sport -n00452034 bullfighting, tauromachy -n00452152 cockfighting -n00452293 hunt, hunting -n00452734 battue -n00452864 beagling -n00453126 coursing -n00453313 deer hunting, deer hunt -n00453396 ducking, duck hunting -n00453478 fox hunting, foxhunt -n00453631 pigsticking -n00453935 fishing, sportfishing -n00454237 angling -n00454395 fly-fishing -n00454493 troll, trolling -n00454624 casting, cast -n00454855 bait casting -n00454983 fly casting -n00455076 overcast -n00455173 surf casting, surf fishing -n00456465 day game -n00463246 athletic game -n00463543 ice hockey, hockey, hockey game -n00464277 tetherball -n00464478 water polo -n00464651 outdoor game -n00464894 golf, golf game -n00466273 professional golf -n00466377 round of golf, round -n00466524 medal play, stroke play -n00466630 match play -n00466712 miniature golf -n00466880 croquet -n00467320 quoits, horseshoes -n00467536 shuffleboard, shovelboard -n00467719 field game -n00467995 field hockey, hockey -n00468299 shinny, shinney -n00468480 football, football game -n00469651 American football, American football game -n00470554 professional football -n00470682 touch football -n00470830 hurling -n00470966 rugby, rugby football, rugger -n00471437 ball game, ballgame -n00471613 baseball, baseball game -n00474568 ball -n00474657 professional baseball -n00474769 hardball -n00474881 perfect game -n00475014 no-hit game, no-hitter -n00475142 one-hitter, 1-hitter -n00475273 two-hitter, 2-hitter -n00475403 three-hitter, 3-hitter -n00475535 four-hitter, 4-hitter -n00475661 five-hitter, 5-hitter -n00475787 softball, softball game -n00476140 rounders -n00476235 stickball, stickball game -n00476389 cricket -n00477392 lacrosse -n00477639 polo -n00477827 pushball -n00478262 soccer, association football -n00479076 court game -n00479440 handball -n00479616 racquetball -n00479734 fives -n00479887 squash, squash racquets, squash rackets -n00480211 volleyball, volleyball game -n00480366 jai alai, pelota -n00480508 badminton -n00480885 battledore, battledore and shuttlecock -n00480993 basketball, basketball game, hoops -n00481803 professional basketball -n00481938 deck tennis -n00482122 netball -n00482298 tennis, lawn tennis -n00483205 professional tennis -n00483313 singles -n00483409 singles -n00483508 doubles -n00483605 doubles -n00483705 royal tennis, real tennis, court tennis -n00483848 pallone -n00523513 sport, athletics -n00812526 clasp, clench, clutch, clutches, grasp, grip, hold -n00825773 judo -n00887544 team sport -n01035504 Last Supper, Lord's Supper -n01035667 Seder, Passover supper -n01055165 camping, encampment, bivouacking, tenting -n01314388 pest -n01314663 critter -n01314781 creepy-crawly -n01314910 darter -n01315213 peeper -n01315330 homeotherm, homoiotherm, homotherm -n01315581 poikilotherm, ectotherm -n01315805 range animal -n01316422 scavenger -n01316579 bottom-feeder, bottom-dweller -n01316734 bottom-feeder -n01316949 work animal -n01317089 beast of burden, jument -n01317294 draft animal -n01317391 pack animal, sumpter -n01317541 domestic animal, domesticated animal -n01317813 feeder -n01317916 feeder -n01318053 stocker -n01318279 hatchling -n01318381 head -n01318478 migrator -n01318660 molter, moulter -n01318894 pet -n01319001 stayer -n01319187 stunt -n01319467 marine animal, marine creature, sea animal, sea creature -n01319685 by-catch, bycatch -n01320872 female -n01321123 hen -n01321230 male -n01321456 adult -n01321579 young, offspring -n01321770 orphan -n01321854 young mammal -n01322221 baby -n01322343 pup, whelp -n01322508 wolf pup, wolf cub -n01322604 puppy -n01322685 cub, young carnivore -n01322898 lion cub -n01322983 bear cub -n01323068 tiger cub -n01323155 kit -n01323261 suckling -n01323355 sire -n01323493 dam -n01323599 thoroughbred, purebred, pureblood -n01323781 giant -n01324305 mutant -n01324431 carnivore -n01324610 herbivore -n01324799 insectivore -n01324916 acrodont -n01325060 pleurodont -n01326291 microorganism, micro-organism -n01327909 monohybrid -n01329186 arbovirus, arborvirus -n01330126 adenovirus -n01330497 arenavirus -n01332181 Marburg virus -n01333082 Arenaviridae -n01333483 vesiculovirus -n01333610 Reoviridae -n01334217 variola major, variola major virus -n01334690 viroid, virusoid -n01335218 coliphage -n01337191 paramyxovirus -n01337734 poliovirus -n01338685 herpes, herpes virus -n01339083 herpes simplex 1, HS1, HSV-1, HSV-I -n01339336 herpes zoster, herpes zoster virus -n01339471 herpes varicella zoster, herpes varicella zoster virus -n01339801 cytomegalovirus, CMV -n01340014 varicella zoster virus -n01340522 polyoma, polyoma virus -n01340785 lyssavirus -n01340935 reovirus -n01341090 rotavirus -n01342269 moneran, moneron -n01347583 archaebacteria, archaebacterium, archaeobacteria, archeobacteria -n01349735 bacteroid -n01350226 Bacillus anthracis, anthrax bacillus -n01350701 Yersinia pestis -n01351170 Brucella -n01351315 spirillum, spirilla -n01357328 botulinus, botulinum, Clostridium botulinum -n01357507 clostridium perfringens -n01358572 cyanobacteria, blue-green algae -n01359762 trichodesmium -n01362336 nitric bacteria, nitrobacteria -n01363719 spirillum -n01365474 Francisella, genus Francisella -n01365885 gonococcus, Neisseria gonorrhoeae -n01366700 Corynebacterium diphtheriae, C. diphtheriae, Klebs-Loeffler bacillus -n01367772 enteric bacteria, enterobacteria, enterics, entric -n01368672 klebsiella -n01369358 Salmonella typhimurium -n01369484 typhoid bacillus, Salmonella typhosa, Salmonella typhi -n01374703 nitrate bacterium, nitric bacterium -n01374846 nitrite bacterium, nitrous bacterium -n01375204 actinomycete -n01376237 streptomyces -n01376437 Streptomyces erythreus -n01376543 Streptomyces griseus -n01377278 tubercle bacillus, Mycobacterium tuberculosis -n01377510 pus-forming bacteria -n01377694 streptobacillus -n01378545 myxobacteria, myxobacterium, myxobacter, gliding bacteria, slime bacteria -n01379389 staphylococcus, staphylococci, staph -n01380610 diplococcus -n01380754 pneumococcus, Diplococcus pneumoniae -n01381044 streptococcus, streptococci, strep -n01382033 spirochete, spirochaete -n01384084 planktonic algae -n01384164 zooplankton -n01384687 parasite -n01385017 endoparasite, entoparasite, entozoan, entozoon, endozoan -n01385330 ectoparasite, ectozoan, ectozoon, epizoan, epizoon -n01386007 pathogen -n01386182 commensal -n01386354 myrmecophile -n01387065 protoctist -n01389507 protozoan, protozoon -n01390123 sarcodinian, sarcodine -n01390763 heliozoan -n01392275 endameba -n01392380 ameba, amoeba -n01393486 globigerina -n01394040 testacean -n01394492 arcella -n01394771 difflugia -n01395254 ciliate, ciliated protozoan, ciliophoran -n01396048 paramecium, paramecia -n01396617 stentor -n01397114 alga, algae -n01397690 arame -n01397871 seagrass -n01400247 golden algae -n01400391 yellow-green algae -n01402600 brown algae -n01403457 kelp -n01404365 fucoid, fucoid algae -n01404495 fucoid -n01405007 fucus -n01405616 bladderwrack, Ascophyllum nodosum -n01407798 green algae, chlorophyte -n01410457 pond scum -n01411450 chlorella -n01412694 stonewort -n01413457 desmid -n01414216 sea moss -n01415626 eukaryote, eucaryote -n01415920 prokaryote, procaryote -n01416213 zooid -n01418498 Leishmania, genus Leishmania -n01418620 zoomastigote, zooflagellate -n01419332 polymastigote -n01419573 costia, Costia necatrix -n01419888 giardia -n01421333 cryptomonad, cryptophyte -n01421807 sporozoan -n01422185 sporozoite -n01422335 trophozoite -n01422450 merozoite -n01423302 coccidium, eimeria -n01423617 gregarine -n01424420 plasmodium, Plasmodium vivax, malaria parasite -n01425223 leucocytozoan, leucocytozoon -n01427399 microsporidian -n01429172 Ostariophysi, order Ostariophysi -n01438208 cypriniform fish -n01438581 loach -n01439121 cyprinid, cyprinid fish -n01439514 carp -n01439808 domestic carp, Cyprinus carpio -n01440160 leather carp -n01440242 mirror carp -n01440467 European bream, Abramis brama -n01440764 tench, Tinca tinca -n01441117 dace, Leuciscus leuciscus -n01441272 chub, Leuciscus cephalus -n01441425 shiner -n01441910 common shiner, silversides, Notropis cornutus -n01442450 roach, Rutilus rutilus -n01442710 rudd, Scardinius erythrophthalmus -n01442972 minnow, Phoxinus phoxinus -n01443243 gudgeon, Gobio gobio -n01443537 goldfish, Carassius auratus -n01443831 crucian carp, Carassius carassius, Carassius vulgaris -n01444339 electric eel, Electrophorus electric -n01444783 catostomid -n01445429 buffalo fish, buffalofish -n01445593 black buffalo, Ictiobus niger -n01445857 hog sucker, hog molly, Hypentelium nigricans -n01446152 redhorse, redhorse sucker -n01446589 cyprinodont -n01446760 killifish -n01447139 mummichog, Fundulus heteroclitus -n01447331 striped killifish, mayfish, may fish, Fundulus majalis -n01447658 rivulus -n01447946 flagfish, American flagfish, Jordanella floridae -n01448291 swordtail, helleri, topminnow, Xyphophorus helleri -n01448594 guppy, rainbow fish, Lebistes reticulatus -n01448951 topminnow, poeciliid fish, poeciliid, live-bearer -n01449374 mosquitofish, Gambusia affinis -n01449712 platy, Platypoecilus maculatus -n01449980 mollie, molly -n01450661 squirrelfish -n01450950 reef squirrelfish, Holocentrus coruscus -n01451115 deepwater squirrelfish, Holocentrus bullisi -n01451295 Holocentrus ascensionis -n01451426 soldierfish, soldier-fish -n01451863 anomalops, flashlight fish -n01452345 flashlight fish, Photoblepharon palpebratus -n01453087 John Dory, Zeus faber -n01453475 boarfish, Capros aper -n01453742 boarfish -n01454545 cornetfish -n01454856 stickleback, prickleback -n01455317 three-spined stickleback, Gasterosteus aculeatus -n01455461 ten-spined stickleback, Gasterosteus pungitius -n01455778 pipefish, needlefish -n01456137 dwarf pipefish, Syngnathus hildebrandi -n01456454 deepwater pipefish, Cosmocampus profundus -n01456756 seahorse, sea horse -n01457082 snipefish, bellows fish -n01457407 shrimpfish, shrimp-fish -n01457852 trumpetfish, Aulostomus maculatus -n01458746 pellicle -n01458842 embryo, conceptus, fertilized egg -n01459791 fetus, foetus -n01460303 abortus -n01461315 spawn -n01461646 blastula, blastosphere -n01462042 blastocyst, blastodermic vessicle -n01462544 gastrula -n01462803 morula -n01464844 yolk, vitellus -n01466257 chordate -n01467336 cephalochordate -n01467804 lancelet, amphioxus -n01468238 tunicate, urochordate, urochord -n01468712 ascidian -n01469103 sea squirt -n01469723 salp, salpa -n01470145 doliolum -n01470479 larvacean -n01470733 appendicularia -n01470895 ascidian tadpole -n01471682 vertebrate, craniate -n01472303 Amniota -n01472502 amniote -n01473806 aquatic vertebrate -n01474283 jawless vertebrate, jawless fish, agnathan -n01474864 ostracoderm -n01475232 heterostracan -n01475940 anaspid -n01476418 conodont -n01477080 cyclostome -n01477525 lamprey, lamprey eel, lamper eel -n01477875 sea lamprey, Petromyzon marinus -n01478511 hagfish, hag, slime eels -n01478969 Myxine glutinosa -n01479213 eptatretus -n01479820 gnathostome -n01480106 placoderm -n01480516 cartilaginous fish, chondrichthian -n01480880 holocephalan, holocephalian -n01481331 chimaera -n01481498 rabbitfish, Chimaera monstrosa -n01482071 elasmobranch, selachian -n01482330 shark -n01483021 cow shark, six-gilled shark, Hexanchus griseus -n01483522 mackerel shark -n01483830 porbeagle, Lamna nasus -n01484097 mako, mako shark -n01484285 shortfin mako, Isurus oxyrhincus -n01484447 longfin mako, Isurus paucus -n01484562 bonito shark, blue pointed, Isurus glaucus -n01484850 great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias -n01485479 basking shark, Cetorhinus maximus -n01486010 thresher, thrasher, thresher shark, fox shark, Alopius vulpinus -n01486540 carpet shark, Orectolobus barbatus -n01486838 nurse shark, Ginglymostoma cirratum -n01487506 sand tiger, sand shark, Carcharias taurus, Odontaspis taurus -n01488038 whale shark, Rhincodon typus -n01488918 requiem shark -n01489501 bull shark, cub shark, Carcharhinus leucas -n01489709 sandbar shark, Carcharhinus plumbeus -n01489920 blacktip shark, sandbar shark, Carcharhinus limbatus -n01490112 whitetip shark, oceanic whitetip shark, white-tipped shark, Carcharinus longimanus -n01490360 dusky shark, Carcharhinus obscurus -n01490670 lemon shark, Negaprion brevirostris -n01491006 blue shark, great blue shark, Prionace glauca -n01491361 tiger shark, Galeocerdo cuvieri -n01491661 soupfin shark, soupfin, soup-fin, Galeorhinus zyopterus -n01491874 dogfish -n01492357 smooth dogfish -n01492569 smoothhound, smoothhound shark, Mustelus mustelus -n01492708 American smooth dogfish, Mustelus canis -n01492860 Florida smoothhound, Mustelus norrisi -n01493146 whitetip shark, reef whitetip shark, Triaenodon obseus -n01493541 spiny dogfish -n01493829 Atlantic spiny dogfish, Squalus acanthias -n01494041 Pacific spiny dogfish, Squalus suckleyi -n01494475 hammerhead, hammerhead shark -n01494757 smooth hammerhead, Sphyrna zygaena -n01494882 smalleye hammerhead, Sphyrna tudes -n01495006 shovelhead, bonnethead, bonnet shark, Sphyrna tiburo -n01495493 angel shark, angelfish, Squatina squatina, monkfish -n01495701 ray -n01496331 electric ray, crampfish, numbfish, torpedo -n01497118 sawfish -n01497413 smalltooth sawfish, Pristis pectinatus -n01497738 guitarfish -n01498041 stingray -n01498406 roughtail stingray, Dasyatis centroura -n01498699 butterfly ray -n01498989 eagle ray -n01499396 spotted eagle ray, spotted ray, Aetobatus narinari -n01499732 cownose ray, cow-nosed ray, Rhinoptera bonasus -n01500091 manta, manta ray, devilfish -n01500476 Atlantic manta, Manta birostris -n01500854 devil ray, Mobula hypostoma -n01501160 skate -n01501641 grey skate, gray skate, Raja batis -n01501777 little skate, Raja erinacea -n01501948 thorny skate, Raja radiata -n01502101 barndoor skate, Raja laevis -n01503061 bird -n01503976 dickeybird, dickey-bird, dickybird, dicky-bird -n01504179 fledgling, fledgeling -n01504344 nestling, baby bird -n01514668 cock -n01514752 gamecock, fighting cock -n01514859 hen -n01514926 nester -n01515078 night bird -n01515217 night raven -n01515303 bird of passage -n01516212 archaeopteryx, archeopteryx, Archaeopteryx lithographica -n01517389 archaeornis -n01517565 ratite, ratite bird, flightless bird -n01517966 carinate, carinate bird, flying bird -n01518878 ostrich, Struthio camelus -n01519563 cassowary -n01519873 emu, Dromaius novaehollandiae, Emu novaehollandiae -n01520576 kiwi, apteryx -n01521399 rhea, Rhea americana -n01521756 rhea, nandu, Pterocnemia pennata -n01522450 elephant bird, aepyornis -n01523105 moa -n01524359 passerine, passeriform bird -n01524761 nonpasserine bird -n01525720 oscine, oscine bird -n01526521 songbird, songster -n01526766 honey eater, honeysucker -n01527194 accentor -n01527347 hedge sparrow, sparrow, dunnock, Prunella modularis -n01527617 lark -n01527917 skylark, Alauda arvensis -n01528396 wagtail -n01528654 pipit, titlark, lark -n01528845 meadow pipit, Anthus pratensis -n01529672 finch -n01530439 chaffinch, Fringilla coelebs -n01530575 brambling, Fringilla montifringilla -n01531178 goldfinch, Carduelis carduelis -n01531344 linnet, lintwhite, Carduelis cannabina -n01531512 siskin, Carduelis spinus -n01531639 red siskin, Carduelis cucullata -n01531811 redpoll, Carduelis flammea -n01531971 redpoll, Carduelis hornemanni -n01532325 New World goldfinch, goldfinch, yellowbird, Spinus tristis -n01532511 pine siskin, pine finch, Spinus pinus -n01532829 house finch, linnet, Carpodacus mexicanus -n01533000 purple finch, Carpodacus purpureus -n01533339 canary, canary bird -n01533481 common canary, Serinus canaria -n01533651 serin -n01533893 crossbill, Loxia curvirostra -n01534155 bullfinch, Pyrrhula pyrrhula -n01534433 junco, snowbird -n01534582 dark-eyed junco, slate-colored junco, Junco hyemalis -n01534762 New World sparrow -n01535140 vesper sparrow, grass finch, Pooecetes gramineus -n01535469 white-throated sparrow, whitethroat, Zonotrichia albicollis -n01535690 white-crowned sparrow, Zonotrichia leucophrys -n01536035 chipping sparrow, Spizella passerina -n01536186 field sparrow, Spizella pusilla -n01536334 tree sparrow, Spizella arborea -n01536644 song sparrow, Melospiza melodia -n01536780 swamp sparrow, Melospiza georgiana -n01537134 bunting -n01537544 indigo bunting, indigo finch, indigo bird, Passerina cyanea -n01537895 ortolan, ortolan bunting, Emberiza hortulana -n01538059 reed bunting, Emberiza schoeniclus -n01538200 yellowhammer, yellow bunting, Emberiza citrinella -n01538362 yellow-breasted bunting, Emberiza aureola -n01538630 snow bunting, snowbird, snowflake, Plectrophenax nivalis -n01538955 honeycreeper -n01539272 banana quit -n01539573 sparrow, true sparrow -n01539925 English sparrow, house sparrow, Passer domesticus -n01540090 tree sparrow, Passer montanus -n01540233 grosbeak, grossbeak -n01540566 evening grosbeak, Hesperiphona vespertina -n01540832 hawfinch, Coccothraustes coccothraustes -n01541102 pine grosbeak, Pinicola enucleator -n01541386 cardinal, cardinal grosbeak, Richmondena Cardinalis, Cardinalis cardinalis, redbird -n01541760 pyrrhuloxia, Pyrrhuloxia sinuata -n01541922 towhee -n01542168 chewink, cheewink, Pipilo erythrophthalmus -n01542433 green-tailed towhee, Chlorura chlorura -n01542786 weaver, weaverbird, weaver finch -n01543175 baya, Ploceus philippinus -n01543383 whydah, whidah, widow bird -n01543632 Java sparrow, Java finch, ricebird, Padda oryzivora -n01543936 avadavat, amadavat -n01544208 grassfinch, grass finch -n01544389 zebra finch, Poephila castanotis -n01544704 honeycreeper, Hawaiian honeycreeper -n01545574 lyrebird -n01546039 scrubbird, scrub-bird, scrub bird -n01546506 broadbill -n01546921 tyrannid -n01547832 New World flycatcher, flycatcher, tyrant flycatcher, tyrant bird -n01548301 kingbird, Tyrannus tyrannus -n01548492 Arkansas kingbird, western kingbird -n01548694 Cassin's kingbird, Tyrannus vociferans -n01548865 eastern kingbird -n01549053 grey kingbird, gray kingbird, petchary, Tyrannus domenicensis domenicensis -n01549430 pewee, peewee, peewit, pewit, wood pewee, Contopus virens -n01549641 western wood pewee, Contopus sordidulus -n01549886 phoebe, phoebe bird, Sayornis phoebe -n01550172 vermillion flycatcher, firebird, Pyrocephalus rubinus mexicanus -n01550761 cotinga, chatterer -n01551080 cock of the rock, Rupicola rupicola -n01551300 cock of the rock, Rupicola peruviana -n01551711 manakin -n01552034 bellbird -n01552333 umbrella bird, Cephalopterus ornatus -n01552813 ovenbird -n01553142 antbird, ant bird -n01553527 ant thrush -n01553762 ant shrike -n01554017 spotted antbird, Hylophylax naevioides -n01554448 woodhewer, woodcreeper, wood-creeper, tree creeper -n01555004 pitta -n01555305 scissortail, scissortailed flycatcher, Muscivora-forficata -n01555809 Old World flycatcher, true flycatcher, flycatcher -n01556182 spotted flycatcher, Muscicapa striata, Muscicapa grisola -n01556514 thickhead, whistler -n01557185 thrush -n01557962 missel thrush, mistle thrush, mistletoe thrush, Turdus viscivorus -n01558149 song thrush, mavis, throstle, Turdus philomelos -n01558307 fieldfare, snowbird, Turdus pilaris -n01558461 redwing, Turdus iliacus -n01558594 blackbird, merl, merle, ouzel, ousel, European blackbird, Turdus merula -n01558765 ring ouzel, ring blackbird, ring thrush, Turdus torquatus -n01558993 robin, American robin, Turdus migratorius -n01559160 clay-colored robin, Turdus greyi -n01559477 hermit thrush, Hylocichla guttata -n01559639 veery, Wilson's thrush, Hylocichla fuscescens -n01559804 wood thrush, Hylocichla mustelina -n01560105 nightingale, Luscinia megarhynchos -n01560280 thrush nightingale, Luscinia luscinia -n01560419 bulbul -n01560636 Old World chat, chat -n01560793 stonechat, Saxicola torquata -n01560935 whinchat, Saxicola rubetra -n01561181 solitaire -n01561452 redstart, redtail -n01561732 wheatear -n01562014 bluebird -n01562265 robin, redbreast, robin redbreast, Old World robin, Erithacus rubecola -n01562451 bluethroat, Erithacus svecicus -n01563128 warbler -n01563449 gnatcatcher -n01563746 kinglet -n01563945 goldcrest, golden-crested kinglet, Regulus regulus -n01564101 gold-crowned kinglet, Regulus satrata -n01564217 ruby-crowned kinglet, ruby-crowned wren, Regulus calendula -n01564394 Old World warbler, true warbler -n01564773 blackcap, Silvia atricapilla -n01564914 greater whitethroat, whitethroat, Sylvia communis -n01565078 lesser whitethroat, whitethroat, Sylvia curruca -n01565345 wood warbler, Phylloscopus sibilatrix -n01565599 sedge warbler, sedge bird, sedge wren, reedbird, Acrocephalus schoenobaenus -n01565930 wren warbler -n01566207 tailorbird, Orthotomus sutorius -n01566645 babbler, cackler -n01567133 New World warbler, wood warbler -n01567678 parula warbler, northern parula, Parula americana -n01567879 Wilson's warbler, Wilson's blackcap, Wilsonia pusilla -n01568132 flycatching warbler -n01568294 American redstart, redstart, Setophaga ruticilla -n01568720 Cape May warbler, Dendroica tigrina -n01568892 yellow warbler, golden warbler, yellowbird, Dendroica petechia -n01569060 Blackburn, Blackburnian warbler, Dendroica fusca -n01569262 Audubon's warbler, Audubon warbler, Dendroica auduboni -n01569423 myrtle warbler, myrtle bird, Dendroica coronata -n01569566 blackpoll, Dendroica striate -n01569836 New World chat, chat -n01569971 yellow-breasted chat, Icteria virens -n01570267 ovenbird, Seiurus aurocapillus -n01570421 water thrush -n01570676 yellowthroat -n01570839 common yellowthroat, Maryland yellowthroat, Geothlypis trichas -n01571410 riflebird, Ptloris paradisea -n01571904 New World oriole, American oriole, oriole -n01572328 northern oriole, Icterus galbula -n01572489 Baltimore oriole, Baltimore bird, hangbird, firebird, Icterus galbula galbula -n01572654 Bullock's oriole, Icterus galbula bullockii -n01572782 orchard oriole, Icterus spurius -n01573074 meadowlark, lark -n01573240 eastern meadowlark, Sturnella magna -n01573360 western meadowlark, Sturnella neglecta -n01573627 cacique, cazique -n01573898 bobolink, ricebird, reedbird, Dolichonyx oryzivorus -n01574045 New World blackbird, blackbird -n01574390 grackle, crow blackbird -n01574560 purple grackle, Quiscalus quiscula -n01574801 rusty blackbird, rusty grackle, Euphagus carilonus -n01575117 cowbird -n01575401 red-winged blackbird, redwing, Agelaius phoeniceus -n01575745 Old World oriole, oriole -n01576076 golden oriole, Oriolus oriolus -n01576358 fig-bird -n01576695 starling -n01577035 common starling, Sturnus vulgaris -n01577458 rose-colored starling, rose-colored pastor, Pastor sturnus, Pastor roseus -n01577659 myna, mynah, mina, minah, myna bird, mynah bird -n01577941 crested myna, Acridotheres tristis -n01578180 hill myna, Indian grackle, grackle, Gracula religiosa -n01578575 corvine bird -n01579028 crow -n01579149 American crow, Corvus brachyrhyncos -n01579260 raven, Corvus corax -n01579410 rook, Corvus frugilegus -n01579578 jackdaw, daw, Corvus monedula -n01579729 chough -n01580077 jay -n01580379 Old World jay -n01580490 common European jay, Garullus garullus -n01580772 New World jay -n01580870 blue jay, jaybird, Cyanocitta cristata -n01581166 Canada jay, grey jay, gray jay, camp robber, whisker jack, Perisoreus canadensis -n01581434 Rocky Mountain jay, Perisoreus canadensis capitalis -n01581730 nutcracker -n01581874 common nutcracker, Nucifraga caryocatactes -n01581984 Clark's nutcracker, Nucifraga columbiana -n01582220 magpie -n01582398 European magpie, Pica pica -n01582498 American magpie, Pica pica hudsonia -n01582856 Australian magpie -n01583209 butcherbird -n01583495 currawong, bell magpie -n01583828 piping crow, piping crow-shrike, Gymnorhina tibicen -n01584225 wren, jenny wren -n01584695 winter wren, Troglodytes troglodytes -n01584853 house wren, Troglodytes aedon -n01585121 marsh wren -n01585287 long-billed marsh wren, Cistothorus palustris -n01585422 sedge wren, short-billed marsh wren, Cistothorus platensis -n01585715 rock wren, Salpinctes obsoletus -n01586020 Carolina wren, Thryothorus ludovicianus -n01586374 cactus wren -n01586941 mockingbird, mocker, Mimus polyglotktos -n01587278 blue mockingbird, Melanotis caerulescens -n01587526 catbird, grey catbird, gray catbird, Dumetella carolinensis -n01587834 thrasher, mocking thrush -n01588002 brown thrasher, brown thrush, Toxostoma rufums -n01588431 New Zealand wren -n01588725 rock wren, Xenicus gilviventris -n01588996 rifleman bird, Acanthisitta chloris -n01589286 creeper, tree creeper -n01589718 brown creeper, American creeper, Certhia americana -n01589893 European creeper, Certhia familiaris -n01590220 wall creeper, tichodrome, Tichodroma muriaria -n01591005 European nuthatch, Sitta europaea -n01591123 red-breasted nuthatch, Sitta canadensis -n01591301 white-breasted nuthatch, Sitta carolinensis -n01591697 titmouse, tit -n01592084 chickadee -n01592257 black-capped chickadee, blackcap, Parus atricapillus -n01592387 tufted titmouse, Parus bicolor -n01592540 Carolina chickadee, Parus carolinensis -n01592694 blue tit, tomtit, Parus caeruleus -n01593028 bushtit, bush tit -n01593282 wren-tit, Chamaea fasciata -n01593553 verdin, Auriparus flaviceps -n01594004 fairy bluebird, bluebird -n01594372 swallow -n01594787 barn swallow, chimney swallow, Hirundo rustica -n01594968 cliff swallow, Hirundo pyrrhonota -n01595168 tree swallow, tree martin, Hirundo nigricans -n01595450 white-bellied swallow, tree swallow, Iridoprocne bicolor -n01595624 martin -n01595974 house martin, Delichon urbica -n01596273 bank martin, bank swallow, sand martin, Riparia riparia -n01596608 purple martin, Progne subis -n01597022 wood swallow, swallow shrike -n01597336 tanager -n01597737 scarlet tanager, Piranga olivacea, redbird, firebird -n01597906 western tanager, Piranga ludoviciana -n01598074 summer tanager, summer redbird, Piranga rubra -n01598271 hepatic tanager, Piranga flava hepatica -n01598588 shrike -n01598988 butcherbird -n01599159 European shrike, Lanius excubitor -n01599269 northern shrike, Lanius borealis -n01599388 white-rumped shrike, Lanius ludovicianus excubitorides -n01599556 loggerhead shrike, Lanius lucovicianus -n01599741 migrant shrike, Lanius ludovicianus migrans -n01600085 bush shrike -n01600341 black-fronted bush shrike, Chlorophoneus nigrifrons -n01600657 bowerbird, catbird -n01601068 satin bowerbird, satin bird, Ptilonorhynchus violaceus -n01601410 great bowerbird, Chlamydera nuchalis -n01601694 water ouzel, dipper -n01602080 European water ouzel, Cinclus aquaticus -n01602209 American water ouzel, Cinclus mexicanus -n01602630 vireo -n01602832 red-eyed vireo, Vireo olivaceous -n01603000 solitary vireo, Vireo solitarius -n01603152 blue-headed vireo, Vireo solitarius solitarius -n01603600 waxwing -n01603812 cedar waxwing, cedarbird, Bombycilla cedrorun -n01603953 Bohemian waxwing, Bombycilla garrulus -n01604330 bird of prey, raptor, raptorial bird -n01604968 Accipitriformes, order Accipitriformes -n01605630 hawk -n01606097 eyas -n01606177 tiercel, tercel, tercelet -n01606522 goshawk, Accipiter gentilis -n01606672 sparrow hawk, Accipiter nisus -n01606809 Cooper's hawk, blue darter, Accipiter cooperii -n01606978 chicken hawk, hen hawk -n01607309 buteonine -n01607429 redtail, red-tailed hawk, Buteo jamaicensis -n01607600 rough-legged hawk, roughleg, Buteo lagopus -n01607812 red-shouldered hawk, Buteo lineatus -n01607962 buzzard, Buteo buteo -n01608265 honey buzzard, Pernis apivorus -n01608432 kite -n01608814 black kite, Milvus migrans -n01609062 swallow-tailed kite, swallow-tailed hawk, Elanoides forficatus -n01609391 white-tailed kite, Elanus leucurus -n01609751 harrier -n01609956 marsh harrier, Circus Aeruginosus -n01610100 Montagu's harrier, Circus pygargus -n01610226 marsh hawk, northern harrier, hen harrier, Circus cyaneus -n01610552 harrier eagle, short-toed eagle -n01610955 falcon -n01611472 peregrine, peregrine falcon, Falco peregrinus -n01611674 falcon-gentle, falcon-gentil -n01611800 gyrfalcon, gerfalcon, Falco rusticolus -n01611969 kestrel, Falco tinnunculus -n01612122 sparrow hawk, American kestrel, kestrel, Falco sparverius -n01612275 pigeon hawk, merlin, Falco columbarius -n01612476 hobby, Falco subbuteo -n01612628 caracara -n01612955 Audubon's caracara, Polyborus cheriway audubonii -n01613177 carancha, Polyborus plancus -n01613294 eagle, bird of Jove -n01613615 young bird -n01613807 eaglet -n01614038 harpy, harpy eagle, Harpia harpyja -n01614343 golden eagle, Aquila chrysaetos -n01614556 tawny eagle, Aquila rapax -n01614925 bald eagle, American eagle, Haliaeetus leucocephalus -n01615121 sea eagle -n01615303 Kamchatkan sea eagle, Stellar's sea eagle, Haliaeetus pelagicus -n01615458 ern, erne, grey sea eagle, gray sea eagle, European sea eagle, white-tailed sea eagle, Haliatus albicilla -n01615703 fishing eagle, Haliaeetus leucorhyphus -n01616086 osprey, fish hawk, fish eagle, sea eagle, Pandion haliaetus -n01616318 vulture -n01616551 Aegypiidae, family Aegypiidae -n01616764 Old World vulture -n01617095 griffon vulture, griffon, Gyps fulvus -n01617443 bearded vulture, lammergeier, lammergeyer, Gypaetus barbatus -n01617766 Egyptian vulture, Pharaoh's chicken, Neophron percnopterus -n01618082 black vulture, Aegypius monachus -n01618503 secretary bird, Sagittarius serpentarius -n01618922 New World vulture, cathartid -n01619310 buzzard, turkey buzzard, turkey vulture, Cathartes aura -n01619536 condor -n01619835 Andean condor, Vultur gryphus -n01620135 California condor, Gymnogyps californianus -n01620414 black vulture, carrion crow, Coragyps atratus -n01620735 king vulture, Sarcorhamphus papa -n01621127 owl, bird of Minerva, bird of night, hooter -n01621635 owlet -n01622120 little owl, Athene noctua -n01622352 horned owl -n01622483 great horned owl, Bubo virginianus -n01622779 great grey owl, great gray owl, Strix nebulosa -n01622959 tawny owl, Strix aluco -n01623110 barred owl, Strix varia -n01623425 screech owl, Otus asio -n01623615 screech owl -n01623706 scops owl -n01623880 spotted owl, Strix occidentalis -n01624115 Old World scops owl, Otus scops -n01624212 Oriental scops owl, Otus sunia -n01624305 hoot owl -n01624537 hawk owl, Surnia ulula -n01624833 long-eared owl, Asio otus -n01625121 laughing owl, laughing jackass, Sceloglaux albifacies -n01625562 barn owl, Tyto alba -n01627424 amphibian -n01628331 Ichyostega -n01628770 urodele, caudate -n01629276 salamander -n01629819 European fire salamander, Salamandra salamandra -n01629962 spotted salamander, fire salamander, Salamandra maculosa -n01630148 alpine salamander, Salamandra atra -n01630284 newt, triton -n01630670 common newt, Triturus vulgaris -n01630901 red eft, Notophthalmus viridescens -n01631175 Pacific newt -n01631354 rough-skinned newt, Taricha granulosa -n01631512 California newt, Taricha torosa -n01631663 eft -n01632047 ambystomid, ambystomid salamander -n01632308 mole salamander, Ambystoma talpoideum -n01632458 spotted salamander, Ambystoma maculatum -n01632601 tiger salamander, Ambystoma tigrinum -n01632777 axolotl, mud puppy, Ambystoma mexicanum -n01632952 waterdog -n01633406 hellbender, mud puppy, Cryptobranchus alleganiensis -n01633781 giant salamander, Megalobatrachus maximus -n01634227 olm, Proteus anguinus -n01634522 mud puppy, Necturus maculosus -n01635027 dicamptodon, dicamptodontid -n01635176 Pacific giant salamander, Dicamptodon ensatus -n01635480 olympic salamander, Rhyacotriton olympicus -n01636127 lungless salamander, plethodont -n01636352 eastern red-backed salamander, Plethodon cinereus -n01636510 western red-backed salamander, Plethodon vehiculum -n01636829 dusky salamander -n01637112 climbing salamander -n01637338 arboreal salamander, Aneides lugubris -n01637615 slender salamander, worm salamander -n01637932 web-toed salamander -n01638194 Shasta salamander, Hydromantes shastae -n01638329 limestone salamander, Hydromantes brunus -n01638722 amphiuma, congo snake, congo eel, blind eel -n01639187 siren -n01639765 frog, toad, toad frog, anuran, batrachian, salientian -n01640846 true frog, ranid -n01641206 wood-frog, wood frog, Rana sylvatica -n01641391 leopard frog, spring frog, Rana pipiens -n01641577 bullfrog, Rana catesbeiana -n01641739 green frog, spring frog, Rana clamitans -n01641930 cascades frog, Rana cascadae -n01642097 goliath frog, Rana goliath -n01642257 pickerel frog, Rana palustris -n01642391 tarahumara frog, Rana tarahumarae -n01642539 grass frog, Rana temporaria -n01642943 leptodactylid frog, leptodactylid -n01643255 robber frog -n01643507 barking frog, robber frog, Hylactophryne augusti -n01643896 crapaud, South American bullfrog, Leptodactylus pentadactylus -n01644373 tree frog, tree-frog -n01644900 tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui -n01645466 Liopelma hamiltoni -n01645776 true toad -n01646292 bufo -n01646388 agua, agua toad, Bufo marinus -n01646555 European toad, Bufo bufo -n01646648 natterjack, Bufo calamita -n01646802 American toad, Bufo americanus -n01646902 Eurasian green toad, Bufo viridis -n01647033 American green toad, Bufo debilis -n01647180 Yosemite toad, Bufo canorus -n01647303 Texas toad, Bufo speciosus -n01647466 southwestern toad, Bufo microscaphus -n01647640 western toad, Bufo boreas -n01648139 obstetrical toad, midwife toad, Alytes obstetricans -n01648356 midwife toad, Alytes cisternasi -n01648620 fire-bellied toad, Bombina bombina -n01649170 spadefoot, spadefoot toad -n01649412 western spadefoot, Scaphiopus hammondii -n01649556 southern spadefoot, Scaphiopus multiplicatus -n01649726 plains spadefoot, Scaphiopus bombifrons -n01650167 tree toad, tree frog, tree-frog -n01650690 spring peeper, Hyla crucifer -n01650901 Pacific tree toad, Hyla regilla -n01651059 canyon treefrog, Hyla arenicolor -n01651285 chameleon tree frog -n01651487 cricket frog -n01651641 northern cricket frog, Acris crepitans -n01651778 eastern cricket frog, Acris gryllus -n01652026 chorus frog -n01652297 lowland burrowing treefrog, northern casque-headed frog, Pternohyla fodiens -n01653026 western narrow-mouthed toad, Gastrophryne olivacea -n01653223 eastern narrow-mouthed toad, Gastrophryne carolinensis -n01653509 sheep frog -n01653773 tongueless frog -n01654083 Surinam toad, Pipa pipa, Pipa americana -n01654637 African clawed frog, Xenopus laevis -n01654863 South American poison toad -n01655344 caecilian, blindworm -n01661091 reptile, reptilian -n01661592 anapsid, anapsid reptile -n01661818 diapsid, diapsid reptile -n01662060 Diapsida, subclass Diapsida -n01662622 chelonian, chelonian reptile -n01662784 turtle -n01663401 sea turtle, marine turtle -n01663782 green turtle, Chelonia mydas -n01664065 loggerhead, loggerhead turtle, Caretta caretta -n01664369 ridley -n01664492 Atlantic ridley, bastard ridley, bastard turtle, Lepidochelys kempii -n01664674 Pacific ridley, olive ridley, Lepidochelys olivacea -n01664990 hawksbill turtle, hawksbill, hawkbill, tortoiseshell turtle, Eretmochelys imbricata -n01665541 leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea -n01665932 snapping turtle -n01666228 common snapping turtle, snapper, Chelydra serpentina -n01666585 alligator snapping turtle, alligator snapper, Macroclemys temmincki -n01667114 mud turtle -n01667432 musk turtle, stinkpot -n01667778 terrapin -n01668091 diamondback terrapin, Malaclemys centrata -n01668436 red-bellied terrapin, red-bellied turtle, redbelly, Pseudemys rubriventris -n01668665 slider, yellow-bellied terrapin, Pseudemys scripta -n01668892 cooter, river cooter, Pseudemys concinna -n01669191 box turtle, box tortoise -n01669372 Western box turtle, Terrapene ornata -n01669654 painted turtle, painted terrapin, painted tortoise, Chrysemys picta -n01670092 tortoise -n01670535 European tortoise, Testudo graeca -n01670802 giant tortoise -n01671125 gopher tortoise, gopher turtle, gopher, Gopherus polypemus -n01671479 desert tortoise, Gopherus agassizii -n01671705 Texas tortoise -n01672032 soft-shelled turtle, pancake turtle -n01672432 spiny softshell, Trionyx spiniferus -n01672611 smooth softshell, Trionyx muticus -n01673282 tuatara, Sphenodon punctatum -n01674216 saurian -n01674464 lizard -n01674990 gecko -n01675352 flying gecko, fringed gecko, Ptychozoon homalocephalum -n01675722 banded gecko -n01676755 iguanid, iguanid lizard -n01677366 common iguana, iguana, Iguana iguana -n01677747 marine iguana, Amblyrhynchus cristatus -n01678043 desert iguana, Dipsosaurus dorsalis -n01678343 chuckwalla, Sauromalus obesus -n01678657 zebra-tailed lizard, gridiron-tailed lizard, Callisaurus draconoides -n01679005 fringe-toed lizard, Uma notata -n01679307 earless lizard -n01679626 collared lizard -n01679962 leopard lizard -n01680264 spiny lizard -n01680478 fence lizard -n01680655 western fence lizard, swift, blue-belly, Sceloporus occidentalis -n01680813 eastern fence lizard, pine lizard, Sceloporus undulatus -n01680983 sagebrush lizard, Sceloporus graciosus -n01681328 side-blotched lizard, sand lizard, Uta stansburiana -n01681653 tree lizard, Urosaurus ornatus -n01681940 horned lizard, horned toad, horny frog -n01682172 Texas horned lizard, Phrynosoma cornutum -n01682435 basilisk -n01682714 American chameleon, anole, Anolis carolinensis -n01683201 worm lizard -n01683558 night lizard -n01684133 skink, scincid, scincid lizard -n01684578 western skink, Eumeces skiltonianus -n01684741 mountain skink, Eumeces callicephalus -n01685439 teiid lizard, teiid -n01685808 whiptail, whiptail lizard -n01686044 racerunner, race runner, six-lined racerunner, Cnemidophorus sexlineatus -n01686220 plateau striped whiptail, Cnemidophorus velox -n01686403 Chihuahuan spotted whiptail, Cnemidophorus exsanguis -n01686609 western whiptail, Cnemidophorus tigris -n01686808 checkered whiptail, Cnemidophorus tesselatus -n01687128 teju -n01687290 caiman lizard -n01687665 agamid, agamid lizard -n01687978 agama -n01688243 frilled lizard, Chlamydosaurus kingi -n01688961 moloch -n01689081 mountain devil, spiny lizard, Moloch horridus -n01689411 anguid lizard -n01689811 alligator lizard -n01690149 blindworm, slowworm, Anguis fragilis -n01690466 glass lizard, glass snake, joint snake -n01691217 legless lizard -n01691652 Lanthanotus borneensis -n01691951 venomous lizard -n01692333 Gila monster, Heloderma suspectum -n01692523 beaded lizard, Mexican beaded lizard, Heloderma horridum -n01692864 lacertid lizard, lacertid -n01693175 sand lizard, Lacerta agilis -n01693334 green lizard, Lacerta viridis -n01693783 chameleon, chamaeleon -n01694178 African chameleon, Chamaeleo chamaeleon -n01694311 horned chameleon, Chamaeleo oweni -n01694709 monitor, monitor lizard, varan -n01694955 African monitor, Varanus niloticus -n01695060 Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis -n01696633 crocodilian reptile, crocodilian -n01697178 crocodile -n01697457 African crocodile, Nile crocodile, Crocodylus niloticus -n01697611 Asian crocodile, Crocodylus porosus -n01697749 Morlett's crocodile -n01697978 false gavial, Tomistoma schlegeli -n01698434 alligator, gator -n01698640 American alligator, Alligator mississipiensis -n01698782 Chinese alligator, Alligator sinensis -n01699040 caiman, cayman -n01699254 spectacled caiman, Caiman sclerops -n01699675 gavial, Gavialis gangeticus -n01701551 armored dinosaur -n01701859 stegosaur, stegosaurus, Stegosaur stenops -n01702256 ankylosaur, ankylosaurus -n01702479 Edmontonia -n01703011 bone-headed dinosaur -n01703161 pachycephalosaur, pachycephalosaurus -n01703569 ceratopsian, horned dinosaur -n01704103 protoceratops -n01704323 triceratops -n01704626 styracosaur, styracosaurus -n01705010 psittacosaur, psittacosaurus -n01705591 ornithopod, ornithopod dinosaur -n01705934 hadrosaur, hadrosaurus, duck-billed dinosaur -n01707294 trachodon, trachodont -n01708106 saurischian, saurischian dinosaur -n01708998 sauropod, sauropod dinosaur -n01709484 apatosaur, apatosaurus, brontosaur, brontosaurus, thunder lizard, Apatosaurus excelsus -n01709876 barosaur, barosaurus -n01710177 diplodocus -n01711160 argentinosaur -n01712008 theropod, theropod dinosaur, bird-footed dinosaur -n01712752 ceratosaur, ceratosaurus -n01713170 coelophysis -n01713764 tyrannosaur, tyrannosaurus, Tyrannosaurus rex -n01714231 allosaur, allosaurus -n01715888 ornithomimid -n01717016 maniraptor -n01717229 oviraptorid -n01717467 velociraptor -n01718096 deinonychus -n01718414 utahraptor, superslasher -n01719403 synapsid, synapsid reptile -n01721174 dicynodont -n01721898 pelycosaur -n01722670 dimetrodon -n01722998 pterosaur, flying reptile -n01723579 pterodactyl -n01724231 ichthyosaur -n01724840 ichthyosaurus -n01725086 stenopterygius, Stenopterygius quadrisicissus -n01725713 plesiosaur, plesiosaurus -n01726203 nothosaur -n01726692 snake, serpent, ophidian -n01727646 colubrid snake, colubrid -n01728266 hoop snake -n01728572 thunder snake, worm snake, Carphophis amoenus -n01728920 ringneck snake, ring-necked snake, ring snake -n01729322 hognose snake, puff adder, sand viper -n01729672 leaf-nosed snake -n01729977 green snake, grass snake -n01730185 smooth green snake, Opheodrys vernalis -n01730307 rough green snake, Opheodrys aestivus -n01730563 green snake -n01730812 racer -n01730960 blacksnake, black racer, Coluber constrictor -n01731137 blue racer, Coluber constrictor flaviventris -n01731277 horseshoe whipsnake, Coluber hippocrepis -n01731545 whip-snake, whip snake, whipsnake -n01731764 coachwhip, coachwhip snake, Masticophis flagellum -n01731941 California whipsnake, striped racer, Masticophis lateralis -n01732093 Sonoran whipsnake, Masticophis bilineatus -n01732244 rat snake -n01732614 corn snake, red rat snake, Elaphe guttata -n01732789 black rat snake, blacksnake, pilot blacksnake, mountain blacksnake, Elaphe obsoleta -n01732989 chicken snake -n01733214 Indian rat snake, Ptyas mucosus -n01733466 glossy snake, Arizona elegans -n01733757 bull snake, bull-snake -n01733957 gopher snake, Pituophis melanoleucus -n01734104 pine snake -n01734418 king snake, kingsnake -n01734637 common kingsnake, Lampropeltis getulus -n01734808 milk snake, house snake, milk adder, checkered adder, Lampropeltis triangulum -n01735189 garter snake, grass snake -n01735439 common garter snake, Thamnophis sirtalis -n01735577 ribbon snake, Thamnophis sauritus -n01735728 Western ribbon snake, Thamnophis proximus -n01736032 lined snake, Tropidoclonion lineatum -n01736375 ground snake, Sonora semiannulata -n01736796 eastern ground snake, Potamophis striatula, Haldea striatula -n01737021 water snake -n01737472 common water snake, banded water snake, Natrix sipedon, Nerodia sipedon -n01737728 water moccasin -n01737875 grass snake, ring snake, ringed snake, Natrix natrix -n01738065 viperine grass snake, Natrix maura -n01738306 red-bellied snake, Storeria occipitamaculata -n01738601 sand snake -n01738731 banded sand snake, Chilomeniscus cinctus -n01739094 black-headed snake -n01739381 vine snake -n01739647 lyre snake -n01739871 Sonoran lyre snake, Trimorphodon lambda -n01740131 night snake, Hypsiglena torquata -n01740551 blind snake, worm snake -n01740885 western blind snake, Leptotyphlops humilis -n01741232 indigo snake, gopher snake, Drymarchon corais -n01741442 eastern indigo snake, Drymarchon corais couperi -n01741562 constrictor -n01741943 boa -n01742172 boa constrictor, Constrictor constrictor -n01742447 rubber boa, tow-headed snake, Charina bottae -n01742821 rosy boa, Lichanura trivirgata -n01743086 anaconda, Eunectes murinus -n01743605 python -n01743936 carpet snake, Python variegatus, Morelia spilotes variegatus -n01744100 reticulated python, Python reticulatus -n01744270 Indian python, Python molurus -n01744401 rock python, rock snake, Python sebae -n01744555 amethystine python -n01745125 elapid, elapid snake -n01745484 coral snake, harlequin-snake, New World coral snake -n01745902 eastern coral snake, Micrurus fulvius -n01746191 western coral snake, Micruroides euryxanthus -n01746359 coral snake, Old World coral snake -n01746952 African coral snake, Aspidelaps lubricus -n01747285 Australian coral snake, Rhynchoelaps australis -n01747589 copperhead, Denisonia superba -n01747885 cobra -n01748264 Indian cobra, Naja naja -n01748389 asp, Egyptian cobra, Naja haje -n01748686 black-necked cobra, spitting cobra, Naja nigricollis -n01748906 hamadryad, king cobra, Ophiophagus hannah, Naja hannah -n01749244 ringhals, rinkhals, spitting snake, Hemachatus haemachatus -n01749582 mamba -n01749742 black mamba, Dendroaspis augusticeps -n01749939 green mamba -n01750167 death adder, Acanthophis antarcticus -n01750437 tiger snake, Notechis scutatus -n01750743 Australian blacksnake, Pseudechis porphyriacus -n01751036 krait -n01751215 banded krait, banded adder, Bungarus fasciatus -n01751472 taipan, Oxyuranus scutellatus -n01751748 sea snake -n01752165 viper -n01752585 adder, common viper, Vipera berus -n01752736 asp, asp viper, Vipera aspis -n01753032 puff adder, Bitis arietans -n01753180 gaboon viper, Bitis gabonica -n01753488 horned viper, cerastes, sand viper, horned asp, Cerastes cornutus -n01753959 pit viper -n01754370 copperhead, Agkistrodon contortrix -n01754533 water moccasin, cottonmouth, cottonmouth moccasin, Agkistrodon piscivorus -n01754876 rattlesnake, rattler -n01755581 diamondback, diamondback rattlesnake, Crotalus adamanteus -n01755740 timber rattlesnake, banded rattlesnake, Crotalus horridus horridus -n01755952 canebrake rattlesnake, canebrake rattler, Crotalus horridus atricaudatus -n01756089 prairie rattlesnake, prairie rattler, Western rattlesnake, Crotalus viridis -n01756291 sidewinder, horned rattlesnake, Crotalus cerastes -n01756508 Western diamondback, Western diamondback rattlesnake, Crotalus atrox -n01756733 rock rattlesnake, Crotalus lepidus -n01756916 tiger rattlesnake, Crotalus tigris -n01757115 Mojave rattlesnake, Crotalus scutulatus -n01757343 speckled rattlesnake, Crotalus mitchellii -n01757677 massasauga, massasauga rattler, Sistrurus catenatus -n01757901 ground rattler, massasauga, Sistrurus miliaris -n01758141 fer-de-lance, Bothrops atrops -n01758757 carcase, carcass -n01758895 carrion -n01767661 arthropod -n01768244 trilobite -n01769347 arachnid, arachnoid -n01770081 harvestman, daddy longlegs, Phalangium opilio -n01770393 scorpion -n01770795 false scorpion, pseudoscorpion -n01771100 book scorpion, Chelifer cancroides -n01771417 whip-scorpion, whip scorpion -n01771766 vinegarroon, Mastigoproctus giganteus -n01772222 spider -n01772664 orb-weaving spider -n01773157 black and gold garden spider, Argiope aurantia -n01773549 barn spider, Araneus cavaticus -n01773797 garden spider, Aranea diademata -n01774097 comb-footed spider, theridiid -n01774384 black widow, Latrodectus mactans -n01774750 tarantula -n01775062 wolf spider, hunting spider -n01775370 European wolf spider, tarantula, Lycosa tarentula -n01775730 trap-door spider -n01776192 acarine -n01776313 tick -n01776705 hard tick, ixodid -n01777304 Ixodes dammini, deer tick -n01777467 Ixodes neotomae -n01777649 Ixodes pacificus, western black-legged tick -n01777909 Ixodes scapularis, black-legged tick -n01778217 sheep-tick, sheep tick, Ixodes ricinus -n01778487 Ixodes persulcatus -n01778621 Ixodes dentatus -n01778801 Ixodes spinipalpis -n01779148 wood tick, American dog tick, Dermacentor variabilis -n01779463 soft tick, argasid -n01779629 mite -n01779939 web-spinning mite -n01780142 acarid -n01780426 trombidiid -n01780696 trombiculid -n01781071 harvest mite, chigger, jigger, redbug -n01781570 acarus, genus Acarus -n01781698 itch mite, sarcoptid -n01781875 rust mite -n01782209 spider mite, tetranychid -n01782516 red spider, red spider mite, Panonychus ulmi -n01783017 myriapod -n01783706 garden centipede, garden symphilid, symphilid, Scutigerella immaculata -n01784293 tardigrade -n01784675 centipede -n01785667 house centipede, Scutigera coleoptrata -n01786646 millipede, millepede, milliped -n01787006 sea spider, pycnogonid -n01787191 Merostomata, class Merostomata -n01787835 horseshoe crab, king crab, Limulus polyphemus, Xiphosurus polyphemus -n01788291 Asian horseshoe crab -n01788579 eurypterid -n01788864 tongue worm, pentastomid -n01789386 gallinaceous bird, gallinacean -n01789740 domestic fowl, fowl, poultry -n01790171 Dorking -n01790304 Plymouth Rock -n01790398 Cornish, Cornish fowl -n01790557 Rock Cornish -n01790711 game fowl -n01790812 cochin, cochin china -n01791107 jungle fowl, gallina -n01791314 jungle cock -n01791388 jungle hen -n01791463 red jungle fowl, Gallus gallus -n01791625 chicken, Gallus gallus -n01791954 bantam -n01792042 chick, biddy -n01792158 cock, rooster -n01792429 cockerel -n01792530 capon -n01792640 hen, biddy -n01792808 cackler -n01792955 brood hen, broody, broody hen, setting hen, sitter -n01793085 mother hen -n01793159 layer -n01793249 pullet -n01793340 spring chicken -n01793435 Rhode Island red -n01793565 Dominique, Dominick -n01793715 Orpington -n01794158 turkey, Meleagris gallopavo -n01794344 turkey cock, gobbler, tom, tom turkey -n01794651 ocellated turkey, Agriocharis ocellata -n01795088 grouse -n01795545 black grouse -n01795735 European black grouse, heathfowl, Lyrurus tetrix -n01795900 Asian black grouse, Lyrurus mlokosiewiczi -n01796019 blackcock, black cock -n01796105 greyhen, grayhen, grey hen, gray hen, heath hen -n01796340 ptarmigan -n01796519 red grouse, moorfowl, moorbird, moor-bird, moorgame, Lagopus scoticus -n01796729 moorhen -n01797020 capercaillie, capercailzie, horse of the wood, Tetrao urogallus -n01797307 spruce grouse, Canachites canadensis -n01797601 sage grouse, sage hen, Centrocercus urophasianus -n01797886 ruffed grouse, partridge, Bonasa umbellus -n01798168 sharp-tailed grouse, sprigtail, sprig tail, Pedioecetes phasianellus -n01798484 prairie chicken, prairie grouse, prairie fowl -n01798706 greater prairie chicken, Tympanuchus cupido -n01798839 lesser prairie chicken, Tympanuchus pallidicinctus -n01798979 heath hen, Tympanuchus cupido cupido -n01799302 guan -n01799679 curassow -n01800195 piping guan -n01800424 chachalaca -n01800633 Texas chachalaca, Ortilis vetula macalli -n01801088 megapode, mound bird, mound-bird, mound builder, scrub fowl -n01801479 mallee fowl, leipoa, lowan, Leipoa ocellata -n01801672 mallee hen -n01801876 brush turkey, Alectura lathami -n01802159 maleo, Macrocephalon maleo -n01802721 phasianid -n01803078 pheasant -n01803362 ring-necked pheasant, Phasianus colchicus -n01803641 afropavo, Congo peafowl, Afropavo congensis -n01803893 argus, argus pheasant -n01804163 golden pheasant, Chrysolophus pictus -n01804478 bobwhite, bobwhite quail, partridge -n01804653 northern bobwhite, Colinus virginianus -n01804921 Old World quail -n01805070 migratory quail, Coturnix coturnix, Coturnix communis -n01805321 monal, monaul -n01805801 peafowl, bird of Juno -n01806061 peachick, pea-chick -n01806143 peacock -n01806297 peahen -n01806364 blue peafowl, Pavo cristatus -n01806467 green peafowl, Pavo muticus -n01806567 quail -n01806847 California quail, Lofortyx californicus -n01807105 tragopan -n01807496 partridge -n01807828 Hungarian partridge, grey partridge, gray partridge, Perdix perdix -n01808140 red-legged partridge, Alectoris ruffa -n01808291 Greek partridge, rock partridge, Alectoris graeca -n01808596 mountain quail, mountain partridge, Oreortyx picta palmeri -n01809106 guinea fowl, guinea, Numida meleagris -n01809371 guinea hen -n01809752 hoatzin, hoactzin, stinkbird, Opisthocomus hoazin -n01810268 tinamou, partridge -n01810700 columbiform bird -n01811243 dodo, Raphus cucullatus -n01811909 pigeon -n01812187 pouter pigeon, pouter -n01812337 dove -n01812662 rock dove, rock pigeon, Columba livia -n01812866 band-tailed pigeon, band-tail pigeon, bandtail, Columba fasciata -n01813088 wood pigeon, ringdove, cushat, Columba palumbus -n01813385 turtledove -n01813532 Streptopelia turtur -n01813658 ringdove, Streptopelia risoria -n01813948 Australian turtledove, turtledove, Stictopelia cuneata -n01814217 mourning dove, Zenaidura macroura -n01814370 domestic pigeon -n01814549 squab -n01814620 fairy swallow -n01814755 roller, tumbler, tumbler pigeon -n01814921 homing pigeon, homer -n01815036 carrier pigeon -n01815270 passenger pigeon, Ectopistes migratorius -n01815601 sandgrouse, sand grouse -n01816017 painted sandgrouse, Pterocles indicus -n01816140 pin-tailed sandgrouse, pin-tailed grouse, Pterocles alchata -n01816474 pallas's sandgrouse, Syrrhaptes paradoxus -n01816887 parrot -n01817263 popinjay -n01817346 poll, poll parrot -n01817953 African grey, African gray, Psittacus erithacus -n01818299 amazon -n01818515 macaw -n01818832 kea, Nestor notabilis -n01819115 cockatoo -n01819313 sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita -n01819465 pink cockatoo, Kakatoe leadbeateri -n01819734 cockateel, cockatiel, cockatoo parrot, Nymphicus hollandicus -n01820052 lovebird -n01820348 lory -n01820546 lorikeet -n01820801 varied Lorikeet, Glossopsitta versicolor -n01821076 rainbow lorikeet, Trichoglossus moluccanus -n01821203 parakeet, parrakeet, parroket, paraquet, paroquet, parroquet -n01821554 Carolina parakeet, Conuropsis carolinensis -n01821869 budgerigar, budgereegah, budgerygah, budgie, grass parakeet, lovebird, shell parakeet, Melopsittacus undulatus -n01822300 ring-necked parakeet, Psittacula krameri -n01822602 cuculiform bird -n01823013 cuckoo -n01823414 European cuckoo, Cuculus canorus -n01823740 black-billed cuckoo, Coccyzus erythropthalmus -n01824035 roadrunner, chaparral cock, Geococcyx californianus -n01824344 ani -n01824575 coucal -n01824749 crow pheasant, Centropus sinensis -n01825278 touraco, turaco, turacou, turakoo -n01825930 coraciiform bird -n01826364 roller -n01826680 European roller, Coracias garrulus -n01826844 ground roller -n01827403 kingfisher -n01827793 Eurasian kingfisher, Alcedo atthis -n01828096 belted kingfisher, Ceryle alcyon -n01828556 kookaburra, laughing jackass, Dacelo gigas -n01828970 bee eater -n01829413 hornbill -n01829869 hoopoe, hoopoo -n01830042 Euopean hoopoe, Upupa epops -n01830479 wood hoopoe -n01830915 motmot, momot -n01831360 tody -n01831712 apodiform bird -n01832167 swift -n01832493 European swift, Apus apus -n01832813 chimney swift, chimney swallow, Chateura pelagica -n01833112 swiftlet, Collocalia inexpectata -n01833415 tree swift, crested swift -n01833805 hummingbird -n01834177 Archilochus colubris -n01834540 thornbill -n01835276 goatsucker, nightjar, caprimulgid -n01835769 European goatsucker, European nightjar, Caprimulgus europaeus -n01835918 chuck-will's-widow, Caprimulgus carolinensis -n01836087 whippoorwill, Caprimulgus vociferus -n01836673 poorwill, Phalaenoptilus nuttallii -n01837072 frogmouth -n01837526 oilbird, guacharo, Steatornis caripensis -n01838038 piciform bird -n01838598 woodpecker, peckerwood, pecker -n01839086 green woodpecker, Picus viridis -n01839330 downy woodpecker -n01839598 flicker -n01839750 yellow-shafted flicker, Colaptes auratus, yellowhammer -n01839949 gilded flicker, Colaptes chrysoides -n01840120 red-shafted flicker, Colaptes caper collaris -n01840412 ivorybill, ivory-billed woodpecker, Campephilus principalis -n01840775 redheaded woodpecker, redhead, Melanerpes erythrocephalus -n01841102 sapsucker -n01841288 yellow-bellied sapsucker, Sphyrapicus varius -n01841441 red-breasted sapsucker, Sphyrapicus varius ruber -n01841679 wryneck -n01841943 piculet -n01842235 barbet -n01842504 puffbird -n01842788 honey guide -n01843065 jacamar -n01843383 toucan -n01843719 toucanet -n01844231 trogon -n01844551 quetzal, quetzal bird -n01844746 resplendent quetzel, resplendent trogon, Pharomacrus mocino -n01844917 aquatic bird -n01845132 waterfowl, water bird, waterbird -n01845477 anseriform bird -n01846331 duck -n01847000 drake -n01847089 quack-quack -n01847170 duckling -n01847253 diving duck -n01847407 dabbling duck, dabbler -n01847806 mallard, Anas platyrhynchos -n01847978 black duck, Anas rubripes -n01848123 teal -n01848323 greenwing, green-winged teal, Anas crecca -n01848453 bluewing, blue-winged teal, Anas discors -n01848555 garganey, Anas querquedula -n01848648 widgeon, wigeon, Anas penelope -n01848840 American widgeon, baldpate, Anas americana -n01848976 shoveler, shoveller, broadbill, Anas clypeata -n01849157 pintail, pin-tailed duck, Anas acuta -n01849466 sheldrake -n01849676 shelduck -n01849863 ruddy duck, Oxyura jamaicensis -n01850192 bufflehead, butterball, dipper, Bucephela albeola -n01850373 goldeneye, whistler, Bucephela clangula -n01850553 Barrow's goldeneye, Bucephala islandica -n01850873 canvasback, canvasback duck, Aythya valisineria -n01851038 pochard, Aythya ferina -n01851207 redhead, Aythya americana -n01851375 scaup, scaup duck, bluebill, broadbill -n01851573 greater scaup, Aythya marila -n01851731 lesser scaup, lesser scaup duck, lake duck, Aythya affinis -n01851895 wild duck -n01852142 wood duck, summer duck, wood widgeon, Aix sponsa -n01852329 wood drake -n01852400 mandarin duck, Aix galericulata -n01852671 muscovy duck, musk duck, Cairina moschata -n01852861 sea duck -n01853195 eider, eider duck -n01853498 scoter, scooter -n01853666 common scoter, Melanitta nigra -n01853870 old squaw, oldwife, Clangula hyemalis -n01854415 merganser, fish duck, sawbill, sheldrake -n01854700 goosander, Mergus merganser -n01854838 American merganser, Mergus merganser americanus -n01855032 red-breasted merganser, Mergus serrator -n01855188 smew, Mergus albellus -n01855476 hooded merganser, hooded sheldrake, Lophodytes cucullatus -n01855672 goose -n01856072 gosling -n01856155 gander -n01856380 Chinese goose, Anser cygnoides -n01856553 greylag, graylag, greylag goose, graylag goose, Anser anser -n01856890 blue goose, Chen caerulescens -n01857079 snow goose -n01857325 brant, brant goose, brent, brent goose -n01857512 common brant goose, Branta bernicla -n01857632 honker, Canada goose, Canadian goose, Branta canadensis -n01857851 barnacle goose, barnacle, Branta leucopsis -n01858281 coscoroba -n01858441 swan -n01858780 cob -n01858845 pen -n01858906 cygnet -n01859190 mute swan, Cygnus olor -n01859325 whooper, whooper swan, Cygnus cygnus -n01859496 tundra swan, Cygnus columbianus -n01859689 whistling swan, Cygnus columbianus columbianus -n01859852 Bewick's swan, Cygnus columbianus bewickii -n01860002 trumpeter, trumpeter swan, Cygnus buccinator -n01860187 black swan, Cygnus atratus -n01860497 screamer -n01860864 horned screamer, Anhima cornuta -n01861148 crested screamer -n01861330 chaja, Chauna torquata -n01861778 mammal, mammalian -n01862399 female mammal -n01871265 tusker -n01871543 prototherian -n01871875 monotreme, egg-laying mammal -n01872401 echidna, spiny anteater, anteater -n01872772 echidna, spiny anteater, anteater -n01873310 platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus -n01874434 marsupial, pouched mammal -n01874928 opossum, possum -n01875313 common opossum, Didelphis virginiana, Didelphis marsupialis -n01875610 crab-eating opossum -n01876034 opossum rat -n01876326 bandicoot -n01876667 rabbit-eared bandicoot, rabbit bandicoot, bilby, Macrotis lagotis -n01877134 kangaroo -n01877606 giant kangaroo, great grey kangaroo, Macropus giganteus -n01877812 wallaby, brush kangaroo -n01878061 common wallaby, Macropus agiles -n01878335 hare wallaby, kangaroo hare -n01878639 nail-tailed wallaby, nail-tailed kangaroo -n01878929 rock wallaby, rock kangaroo -n01879217 pademelon, paddymelon -n01879509 tree wallaby, tree kangaroo -n01879837 musk kangaroo, Hypsiprymnodon moschatus -n01880152 rat kangaroo, kangaroo rat -n01880473 potoroo -n01880716 bettong -n01880813 jerboa kangaroo, kangaroo jerboa -n01881171 phalanger, opossum, possum -n01881564 cuscus -n01881857 brush-tailed phalanger, Trichosurus vulpecula -n01882125 flying phalanger, flying opossum, flying squirrel -n01882714 koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus -n01883070 wombat -n01883513 dasyurid marsupial, dasyurid -n01883920 dasyure -n01884104 eastern dasyure, Dasyurus quoll -n01884203 native cat, Dasyurus viverrinus -n01884476 thylacine, Tasmanian wolf, Tasmanian tiger, Thylacinus cynocephalus -n01884834 Tasmanian devil, ursine dasyure, Sarcophilus hariisi -n01885158 pouched mouse, marsupial mouse, marsupial rat -n01885498 numbat, banded anteater, anteater, Myrmecobius fasciatus -n01886045 pouched mole, marsupial mole, Notoryctus typhlops -n01886756 placental, placental mammal, eutherian, eutherian mammal -n01887474 livestock, stock, farm animal -n01887623 bull -n01887787 cow -n01887896 calf -n01888045 calf -n01888181 yearling -n01888264 buck -n01888411 doe -n01889074 insectivore -n01889520 mole -n01889849 starnose mole, star-nosed mole, Condylura cristata -n01890144 brewer's mole, hair-tailed mole, Parascalops breweri -n01890564 golden mole -n01890860 shrew mole -n01891013 Asiatic shrew mole, Uropsilus soricipes -n01891274 American shrew mole, Neurotrichus gibbsii -n01891633 shrew, shrewmouse -n01892030 common shrew, Sorex araneus -n01892145 masked shrew, Sorex cinereus -n01892385 short-tailed shrew, Blarina brevicauda -n01892551 water shrew -n01892744 American water shrew, Sorex palustris -n01893021 European water shrew, Neomys fodiens -n01893164 Mediterranean water shrew, Neomys anomalus -n01893399 least shrew, Cryptotis parva -n01893825 hedgehog, Erinaceus europaeus, Erinaceus europeaeus -n01894207 tenrec, tendrac -n01894522 tailless tenrec, Tenrec ecaudatus -n01894956 otter shrew, potamogale, Potamogale velox -n01896844 eiderdown -n01897257 aftershaft -n01897426 sickle feather -n01897536 contour feather -n01897667 bastard wing, alula, spurious wing -n01898593 saddle hackle, saddle feather -n01899894 encolure -n01900150 hair -n01903234 squama -n01903346 scute -n01903498 sclerite -n01904029 plastron -n01904806 scallop shell -n01904886 oyster shell -n01905321 theca -n01905661 invertebrate -n01906749 sponge, poriferan, parazoan -n01907287 choanocyte, collar cell -n01907738 glass sponge -n01908042 Venus's flower basket -n01908958 metazoan -n01909422 coelenterate, cnidarian -n01909788 planula -n01909906 polyp -n01910252 medusa, medusoid, medusan -n01910747 jellyfish -n01911063 scyphozoan -n01911403 Chrysaora quinquecirrha -n01911839 hydrozoan, hydroid -n01912152 hydra -n01912454 siphonophore -n01912809 nanomia -n01913166 Portuguese man-of-war, man-of-war, jellyfish -n01913346 praya -n01913440 apolemia -n01914163 anthozoan, actinozoan -n01914609 sea anemone, anemone -n01914830 actinia, actinian, actiniarian -n01915700 sea pen -n01915811 coral -n01916187 gorgonian, gorgonian coral -n01916388 sea feather -n01916481 sea fan -n01916588 red coral -n01916925 stony coral, madrepore, madriporian coral -n01917289 brain coral -n01917611 staghorn coral, stag's-horn coral -n01917882 mushroom coral -n01918744 ctenophore, comb jelly -n01919385 beroe -n01920051 platyctenean -n01920438 sea gooseberry -n01921059 Venus's girdle, Cestum veneris -n01922303 worm -n01922717 helminth, parasitic worm -n01922948 woodworm -n01923025 woodborer, borer -n01923404 acanthocephalan, spiny-headed worm -n01923890 arrowworm, chaetognath -n01924800 bladder worm -n01924916 flatworm, platyhelminth -n01925270 planarian, planaria -n01925695 fluke, trematode, trematode worm -n01925916 cercaria -n01926379 liver fluke, Fasciola hepatica -n01926689 Fasciolopsis buski -n01927159 schistosome, blood fluke -n01927456 tapeworm, cestode -n01927928 echinococcus -n01928215 taenia -n01928517 ribbon worm, nemertean, nemertine, proboscis worm -n01928865 beard worm, pogonophoran -n01929186 rotifer -n01930112 nematode, nematode worm, roundworm -n01930852 common roundworm, Ascaris lumbricoides -n01931140 chicken roundworm, Ascaridia galli -n01931520 pinworm, threadworm, Enterobius vermicularis -n01931714 eelworm -n01932151 vinegar eel, vinegar worm, Anguillula aceti, Turbatrix aceti -n01932936 trichina, Trichinella spiralis -n01933151 hookworm -n01933478 filaria -n01933988 Guinea worm, Dracunculus medinensis -n01934440 annelid, annelid worm, segmented worm -n01934844 archiannelid -n01935176 oligochaete, oligochaete worm -n01935395 earthworm, angleworm, fishworm, fishing worm, wiggler, nightwalker, nightcrawler, crawler, dew worm, red worm -n01936391 polychaete, polychete, polychaete worm, polychete worm -n01936671 lugworm, lug, lobworm -n01936858 sea mouse -n01937579 bloodworm -n01937909 leech, bloodsucker, hirudinean -n01938454 medicinal leech, Hirudo medicinalis -n01938735 horseleech -n01940736 mollusk, mollusc, shellfish -n01941223 scaphopod -n01941340 tooth shell, tusk shell -n01942177 gastropod, univalve -n01942869 abalone, ear-shell -n01943087 ormer, sea-ear, Haliotis tuberculata -n01943541 scorpion shell -n01943899 conch -n01944118 giant conch, Strombus gigas -n01944390 snail -n01944812 edible snail, Helix pomatia -n01944955 garden snail -n01945143 brown snail, Helix aspersa -n01945340 Helix hortensis -n01945685 slug -n01945845 seasnail -n01946277 neritid, neritid gastropod -n01946630 nerita -n01946827 bleeding tooth, Nerita peloronta -n01947139 neritina -n01947396 whelk -n01947997 moon shell, moonshell -n01948446 periwinkle, winkle -n01948573 limpet -n01949085 common limpet, Patella vulgata -n01949499 keyhole limpet, Fissurella apertura, Diodora apertura -n01949973 river limpet, freshwater limpet, Ancylus fluviatilis -n01950731 sea slug, nudibranch -n01951274 sea hare, Aplysia punctata -n01951613 Hermissenda crassicornis -n01952029 bubble shell -n01952712 physa -n01953361 cowrie, cowry -n01953594 money cowrie, Cypraea moneta -n01953762 tiger cowrie, Cypraea tigris -n01954516 solenogaster, aplacophoran -n01955084 chiton, coat-of-mail shell, sea cradle, polyplacophore -n01955933 bivalve, pelecypod, lamellibranch -n01956344 spat -n01956481 clam -n01956764 seashell -n01957335 soft-shell clam, steamer, steamer clam, long-neck clam, Mya arenaria -n01958038 quahog, quahaug, hard-shell clam, hard clam, round clam, Venus mercenaria, Mercenaria mercenaria -n01958346 littleneck, littleneck clam -n01958435 cherrystone, cherrystone clam -n01958531 geoduck -n01959029 razor clam, jackknife clam, knife-handle -n01959492 giant clam, Tridacna gigas -n01959985 cockle -n01960177 edible cockle, Cardium edule -n01960459 oyster -n01961234 Japanese oyster, Ostrea gigas -n01961600 Virginia oyster -n01961985 pearl oyster, Pinctada margaritifera -n01962506 saddle oyster, Anomia ephippium -n01962788 window oyster, windowpane oyster, capiz, Placuna placenta -n01963317 ark shell -n01963479 blood clam -n01963571 mussel -n01964049 marine mussel, mytilid -n01964271 edible mussel, Mytilus edulis -n01964441 freshwater mussel, freshwater clam -n01964957 pearly-shelled mussel -n01965252 thin-shelled mussel -n01965529 zebra mussel, Dreissena polymorpha -n01965889 scallop, scollop, escallop -n01966377 bay scallop, Pecten irradians -n01966586 sea scallop, giant scallop, Pecten magellanicus -n01967094 shipworm, teredinid -n01967308 teredo -n01967963 piddock -n01968315 cephalopod, cephalopod mollusk -n01968897 chambered nautilus, pearly nautilus, nautilus -n01969726 octopod -n01970164 octopus, devilfish -n01970667 paper nautilus, nautilus, Argonaut, Argonauta argo -n01971094 decapod -n01971280 squid -n01971620 loligo -n01971850 ommastrephes -n01972131 architeuthis, giant squid -n01972541 cuttlefish, cuttle -n01973148 spirula, Spirula peronii -n01974773 crustacean -n01975687 malacostracan crustacean -n01976146 decapod crustacean, decapod -n01976868 brachyuran -n01976957 crab -n01977485 stone crab, Menippe mercenaria -n01978010 hard-shell crab -n01978136 soft-shell crab, soft-shelled crab -n01978287 Dungeness crab, Cancer magister -n01978455 rock crab, Cancer irroratus -n01978587 Jonah crab, Cancer borealis -n01978930 swimming crab -n01979269 English lady crab, Portunus puber -n01979526 American lady crab, lady crab, calico crab, Ovalipes ocellatus -n01979874 blue crab, Callinectes sapidus -n01980166 fiddler crab -n01980655 pea crab -n01981276 king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica -n01981702 spider crab -n01982068 European spider crab, king crab, Maja squinado -n01982347 giant crab, Macrocheira kaempferi -n01982650 lobster -n01983048 true lobster -n01983481 American lobster, Northern lobster, Maine lobster, Homarus americanus -n01983674 European lobster, Homarus vulgaris -n01983829 Cape lobster, Homarus capensis -n01984245 Norway lobster, Nephrops norvegicus -n01984695 spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish -n01985128 crayfish, crawfish, crawdad, crawdaddy -n01985493 Old World crayfish, ecrevisse -n01985797 American crayfish -n01986214 hermit crab -n01986806 shrimp -n01987076 snapping shrimp, pistol shrimp -n01987545 prawn -n01987727 long-clawed prawn, river prawn, Palaemon australis -n01988203 tropical prawn -n01988701 krill -n01988869 Euphausia pacifica -n01989516 opossum shrimp -n01989869 stomatopod, stomatopod crustacean -n01990007 mantis shrimp, mantis crab -n01990516 squilla, mantis prawn -n01990800 isopod -n01991028 woodlouse, slater -n01991520 pill bug -n01992262 sow bug -n01992423 sea louse, sea slater -n01992773 amphipod -n01993525 skeleton shrimp -n01993830 whale louse -n01994910 daphnia, water flea -n01995514 fairy shrimp -n01995686 brine shrimp, Artemia salina -n01996280 tadpole shrimp -n01996585 copepod, copepod crustacean -n01997119 cyclops, water flea -n01997825 seed shrimp, mussel shrimp, ostracod -n01998183 barnacle, cirriped, cirripede -n01998741 acorn barnacle, rock barnacle, Balanus balanoides -n01999186 goose barnacle, gooseneck barnacle, Lepas fascicularis -n01999767 onychophoran, velvet worm, peripatus -n02000954 wading bird, wader -n02002075 stork -n02002556 white stork, Ciconia ciconia -n02002724 black stork, Ciconia nigra -n02003037 adjutant bird, adjutant, adjutant stork, Leptoptilus dubius -n02003204 marabou, marabout, marabou stork, Leptoptilus crumeniferus -n02003577 openbill -n02003839 jabiru, Jabiru mycteria -n02004131 saddlebill, jabiru, Ephippiorhynchus senegalensis -n02004492 policeman bird, black-necked stork, jabiru, Xenorhyncus asiaticus -n02004855 wood ibis, wood stork, flinthead, Mycteria americana -n02005399 shoebill, shoebird, Balaeniceps rex -n02005790 ibis -n02006063 wood ibis, wood stork, Ibis ibis -n02006364 sacred ibis, Threskiornis aethiopica -n02006656 spoonbill -n02006985 common spoonbill, Platalea leucorodia -n02007284 roseate spoonbill, Ajaia ajaja -n02007558 flamingo -n02008041 heron -n02008497 great blue heron, Ardea herodius -n02008643 great white heron, Ardea occidentalis -n02008796 egret -n02009229 little blue heron, Egretta caerulea -n02009380 snowy egret, snowy heron, Egretta thula -n02009508 little egret, Egretta garzetta -n02009750 great white heron, Casmerodius albus -n02009912 American egret, great white heron, Egretta albus -n02010272 cattle egret, Bubulcus ibis -n02010453 night heron, night raven -n02010728 black-crowned night heron, Nycticorax nycticorax -n02011016 yellow-crowned night heron, Nyctanassa violacea -n02011281 boatbill, boat-billed heron, broadbill, Cochlearius cochlearius -n02011460 bittern -n02011805 American bittern, stake driver, Botaurus lentiginosus -n02011943 European bittern, Botaurus stellaris -n02012185 least bittern, Ixobrychus exilis -n02012849 crane -n02013177 whooping crane, whooper, Grus americana -n02013567 courlan, Aramus guarauna -n02013706 limpkin, Aramus pictus -n02014237 crested cariama, seriema, Cariama cristata -n02014524 chunga, seriema, Chunga burmeisteri -n02014941 rail -n02015357 weka, maori hen, wood hen -n02015554 crake -n02015797 corncrake, land rail, Crex crex -n02016066 spotted crake, Porzana porzana -n02016358 gallinule, marsh hen, water hen, swamphen -n02016659 Florida gallinule, Gallinula chloropus cachinnans -n02016816 moorhen, Gallinula chloropus -n02016956 purple gallinule -n02017213 European gallinule, Porphyrio porphyrio -n02017475 American gallinule, Porphyrula martinica -n02017725 notornis, takahe, Notornis mantelli -n02018027 coot -n02018207 American coot, marsh hen, mud hen, water hen, Fulica americana -n02018368 Old World coot, Fulica atra -n02018795 bustard -n02019190 great bustard, Otis tarda -n02019438 plain turkey, Choriotis australis -n02019929 button quail, button-quail, bustard quail, hemipode -n02020219 striped button quail, Turnix sylvatica -n02020578 plain wanderer, Pedionomus torquatus -n02021050 trumpeter -n02021281 Brazilian trumpeter, Psophia crepitans -n02021795 seabird, sea bird, seafowl -n02022684 shorebird, shore bird, limicoline bird -n02023341 plover -n02023855 piping plover, Charadrius melodus -n02023992 killdeer, kildeer, killdeer plover, Charadrius vociferus -n02024185 dotterel, dotrel, Charadrius morinellus, Eudromias morinellus -n02024479 golden plover -n02024763 lapwing, green plover, peewit, pewit -n02025043 turnstone -n02025239 ruddy turnstone, Arenaria interpres -n02025389 black turnstone, Arenaria-Melanocephala -n02026059 sandpiper -n02026629 surfbird, Aphriza virgata -n02026948 European sandpiper, Actitis hypoleucos -n02027075 spotted sandpiper, Actitis macularia -n02027357 least sandpiper, stint, Erolia minutilla -n02027492 red-backed sandpiper, dunlin, Erolia alpina -n02027897 greenshank, Tringa nebularia -n02028035 redshank, Tringa totanus -n02028175 yellowlegs -n02028342 greater yellowlegs, Tringa melanoleuca -n02028451 lesser yellowlegs, Tringa flavipes -n02028727 pectoral sandpiper, jacksnipe, Calidris melanotos -n02028900 knot, greyback, grayback, Calidris canutus -n02029087 curlew sandpiper, Calidris Ferruginea -n02029378 sanderling, Crocethia alba -n02029706 upland sandpiper, upland plover, Bartramian sandpiper, Bartramia longicauda -n02030035 ruff, Philomachus pugnax -n02030224 reeve -n02030287 tattler -n02030568 Polynesian tattler, Heteroscelus incanus -n02030837 willet, Catoptrophorus semipalmatus -n02030996 woodcock -n02031298 Eurasian woodcock, Scolopax rusticola -n02031585 American woodcock, woodcock snipe, Philohela minor -n02031934 snipe -n02032222 whole snipe, Gallinago gallinago -n02032355 Wilson's snipe, Gallinago gallinago delicata -n02032480 great snipe, woodcock snipe, Gallinago media -n02032769 jacksnipe, half snipe, Limnocryptes minima -n02033041 dowitcher -n02033208 greyback, grayback, Limnodromus griseus -n02033324 red-breasted snipe, Limnodromus scolopaceus -n02033561 curlew -n02033779 European curlew, Numenius arquata -n02033882 Eskimo curlew, Numenius borealis -n02034129 godwit -n02034295 Hudsonian godwit, Limosa haemastica -n02034661 stilt, stiltbird, longlegs, long-legs, stilt plover, Himantopus stilt -n02034971 black-necked stilt, Himantopus mexicanus -n02035210 black-winged stilt, Himantopus himantopus -n02035402 white-headed stilt, Himantopus himantopus leucocephalus -n02035656 kaki, Himantopus novae-zelandiae -n02036053 stilt, Australian stilt -n02036228 banded stilt, Cladorhyncus leucocephalum -n02036711 avocet -n02037110 oystercatcher, oyster catcher -n02037464 phalarope -n02037869 red phalarope, Phalaropus fulicarius -n02038141 northern phalarope, Lobipes lobatus -n02038466 Wilson's phalarope, Steganopus tricolor -n02038993 pratincole, glareole -n02039171 courser -n02039497 cream-colored courser, Cursorius cursor -n02039780 crocodile bird, Pluvianus aegyptius -n02040266 stone curlew, thick-knee, Burhinus oedicnemus -n02040505 coastal diving bird -n02041085 larid -n02041246 gull, seagull, sea gull -n02041678 mew, mew gull, sea mew, Larus canus -n02041875 black-backed gull, great black-backed gull, cob, Larus marinus -n02042046 herring gull, Larus argentatus -n02042180 laughing gull, blackcap, pewit, pewit gull, Larus ridibundus -n02042472 ivory gull, Pagophila eburnea -n02042759 kittiwake -n02043063 tern -n02043333 sea swallow, Sterna hirundo -n02043808 skimmer -n02044178 jaeger -n02044517 parasitic jaeger, arctic skua, Stercorarius parasiticus -n02044778 skua, bonxie -n02044908 great skua, Catharacta skua -n02045369 auk -n02045596 auklet -n02045864 razorbill, razor-billed auk, Alca torda -n02046171 little auk, dovekie, Plautus alle -n02046759 guillemot -n02046939 black guillemot, Cepphus grylle -n02047045 pigeon guillemot, Cepphus columba -n02047260 murre -n02047411 common murre, Uria aalge -n02047517 thick-billed murre, Uria lomvia -n02047614 puffin -n02047975 Atlantic puffin, Fratercula arctica -n02048115 horned puffin, Fratercula corniculata -n02048353 tufted puffin, Lunda cirrhata -n02048698 gaviiform seabird -n02049088 loon, diver -n02049532 podicipitiform seabird -n02050004 grebe -n02050313 great crested grebe, Podiceps cristatus -n02050442 red-necked grebe, Podiceps grisegena -n02050586 black-necked grebe, eared grebe, Podiceps nigricollis -n02050809 dabchick, little grebe, Podiceps ruficollis -n02051059 pied-billed grebe, Podilymbus podiceps -n02051474 pelecaniform seabird -n02051845 pelican -n02052204 white pelican, Pelecanus erythrorhynchos -n02052365 Old world white pelican, Pelecanus onocrotalus -n02052775 frigate bird, man-of-war bird -n02053083 gannet -n02053425 solan, solan goose, solant goose, Sula bassana -n02053584 booby -n02054036 cormorant, Phalacrocorax carbo -n02054502 snakebird, anhinga, darter -n02054711 water turkey, Anhinga anhinga -n02055107 tropic bird, tropicbird, boatswain bird -n02055658 sphenisciform seabird -n02055803 penguin -n02056228 Adelie, Adelie penguin, Pygoscelis adeliae -n02056570 king penguin, Aptenodytes patagonica -n02056728 emperor penguin, Aptenodytes forsteri -n02057035 jackass penguin, Spheniscus demersus -n02057330 rock hopper, crested penguin -n02057731 pelagic bird, oceanic bird -n02057898 procellariiform seabird -n02058221 albatross, mollymawk -n02058594 wandering albatross, Diomedea exulans -n02058747 black-footed albatross, gooney, gooney bird, goonie, goony, Diomedea nigripes -n02059162 petrel -n02059541 white-chinned petrel, Procellaria aequinoctialis -n02059852 giant petrel, giant fulmar, Macronectes giganteus -n02060133 fulmar, fulmar petrel, Fulmarus glacialis -n02060411 shearwater -n02060569 Manx shearwater, Puffinus puffinus -n02060889 storm petrel -n02061217 stormy petrel, northern storm petrel, Hydrobates pelagicus -n02061560 Mother Carey's chicken, Mother Carey's hen, Oceanites oceanicus -n02061853 diving petrel -n02062017 aquatic mammal -n02062430 cetacean, cetacean mammal, blower -n02062744 whale -n02063224 baleen whale, whalebone whale -n02063662 right whale -n02064000 bowhead, bowhead whale, Greenland whale, Balaena mysticetus -n02064338 rorqual, razorback -n02064816 blue whale, sulfur bottom, Balaenoptera musculus -n02065026 finback, finback whale, fin whale, common rorqual, Balaenoptera physalus -n02065263 sei whale, Balaenoptera borealis -n02065407 lesser rorqual, piked whale, minke whale, Balaenoptera acutorostrata -n02065726 humpback, humpback whale, Megaptera novaeangliae -n02066245 grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus -n02066707 toothed whale -n02067240 sperm whale, cachalot, black whale, Physeter catodon -n02067603 pygmy sperm whale, Kogia breviceps -n02067768 dwarf sperm whale, Kogia simus -n02068206 beaked whale -n02068541 bottle-nosed whale, bottlenose whale, bottlenose, Hyperoodon ampullatus -n02068974 dolphin -n02069412 common dolphin, Delphinus delphis -n02069701 bottlenose dolphin, bottle-nosed dolphin, bottlenose -n02069974 Atlantic bottlenose dolphin, Tursiops truncatus -n02070174 Pacific bottlenose dolphin, Tursiops gilli -n02070430 porpoise -n02070624 harbor porpoise, herring hog, Phocoena phocoena -n02070776 vaquita, Phocoena sinus -n02071028 grampus, Grampus griseus -n02071294 killer whale, killer, orca, grampus, sea wolf, Orcinus orca -n02071636 pilot whale, black whale, common blackfish, blackfish, Globicephala melaena -n02072040 river dolphin -n02072493 narwhal, narwal, narwhale, Monodon monoceros -n02072798 white whale, beluga, Delphinapterus leucas -n02073250 sea cow, sirenian mammal, sirenian -n02073831 manatee, Trichechus manatus -n02074367 dugong, Dugong dugon -n02074726 Steller's sea cow, Hydrodamalis gigas -n02075296 carnivore -n02075612 omnivore -n02075927 pinniped mammal, pinniped, pinnatiped -n02076196 seal -n02076402 crabeater seal, crab-eating seal -n02076779 eared seal -n02077152 fur seal -n02077384 guadalupe fur seal, Arctocephalus philippi -n02077658 fur seal -n02077787 Alaska fur seal, Callorhinus ursinus -n02077923 sea lion -n02078292 South American sea lion, Otaria Byronia -n02078574 California sea lion, Zalophus californianus, Zalophus californicus -n02078738 Australian sea lion, Zalophus lobatus -n02079005 Steller sea lion, Steller's sea lion, Eumetopias jubatus -n02079389 earless seal, true seal, hair seal -n02079851 harbor seal, common seal, Phoca vitulina -n02080146 harp seal, Pagophilus groenlandicus -n02080415 elephant seal, sea elephant -n02080713 bearded seal, squareflipper square flipper, Erignathus barbatus -n02081060 hooded seal, bladdernose, Cystophora cristata -n02081571 walrus, seahorse, sea horse -n02081798 Atlantic walrus, Odobenus rosmarus -n02081927 Pacific walrus, Odobenus divergens -n02082056 Fissipedia -n02082190 fissiped mammal, fissiped -n02082791 aardvark, ant bear, anteater, Orycteropus afer -n02083346 canine, canid -n02083672 bitch -n02083780 brood bitch -n02084071 dog, domestic dog, Canis familiaris -n02084732 pooch, doggie, doggy, barker, bow-wow -n02084861 cur, mongrel, mutt -n02085019 feist, fice -n02085118 pariah dog, pye-dog, pie-dog -n02085272 lapdog -n02085374 toy dog, toy -n02085620 Chihuahua -n02085782 Japanese spaniel -n02085936 Maltese dog, Maltese terrier, Maltese -n02086079 Pekinese, Pekingese, Peke -n02086240 Shih-Tzu -n02086346 toy spaniel -n02086478 English toy spaniel -n02086646 Blenheim spaniel -n02086753 King Charles spaniel -n02086910 papillon -n02087046 toy terrier -n02087122 hunting dog -n02087314 courser -n02087394 Rhodesian ridgeback -n02087551 hound, hound dog -n02088094 Afghan hound, Afghan -n02088238 basset, basset hound -n02088364 beagle -n02088466 bloodhound, sleuthhound -n02088632 bluetick -n02088745 boarhound -n02088839 coonhound -n02088992 coondog -n02089078 black-and-tan coonhound -n02089232 dachshund, dachsie, badger dog -n02089468 sausage dog, sausage hound -n02089555 foxhound -n02089725 American foxhound -n02089867 Walker hound, Walker foxhound -n02089973 English foxhound -n02090129 harrier -n02090253 Plott hound -n02090379 redbone -n02090475 wolfhound -n02090622 borzoi, Russian wolfhound -n02090721 Irish wolfhound -n02090827 greyhound -n02091032 Italian greyhound -n02091134 whippet -n02091244 Ibizan hound, Ibizan Podenco -n02091467 Norwegian elkhound, elkhound -n02091635 otterhound, otter hound -n02091831 Saluki, gazelle hound -n02092002 Scottish deerhound, deerhound -n02092173 staghound -n02092339 Weimaraner -n02092468 terrier -n02093056 bullterrier, bull terrier -n02093256 Staffordshire bullterrier, Staffordshire bull terrier -n02093428 American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier -n02093647 Bedlington terrier -n02093754 Border terrier -n02093859 Kerry blue terrier -n02093991 Irish terrier -n02094114 Norfolk terrier -n02094258 Norwich terrier -n02094433 Yorkshire terrier -n02094562 rat terrier, ratter -n02094721 Manchester terrier, black-and-tan terrier -n02094931 toy Manchester, toy Manchester terrier -n02095050 fox terrier -n02095212 smooth-haired fox terrier -n02095314 wire-haired fox terrier -n02095412 wirehair, wirehaired terrier, wire-haired terrier -n02095570 Lakeland terrier -n02095727 Welsh terrier -n02095889 Sealyham terrier, Sealyham -n02096051 Airedale, Airedale terrier -n02096177 cairn, cairn terrier -n02096294 Australian terrier -n02096437 Dandie Dinmont, Dandie Dinmont terrier -n02096585 Boston bull, Boston terrier -n02096756 schnauzer -n02097047 miniature schnauzer -n02097130 giant schnauzer -n02097209 standard schnauzer -n02097298 Scotch terrier, Scottish terrier, Scottie -n02097474 Tibetan terrier, chrysanthemum dog -n02097658 silky terrier, Sydney silky -n02097786 Skye terrier -n02097967 Clydesdale terrier -n02098105 soft-coated wheaten terrier -n02098286 West Highland white terrier -n02098413 Lhasa, Lhasa apso -n02098550 sporting dog, gun dog -n02098806 bird dog -n02098906 water dog -n02099029 retriever -n02099267 flat-coated retriever -n02099429 curly-coated retriever -n02099601 golden retriever -n02099712 Labrador retriever -n02099849 Chesapeake Bay retriever -n02099997 pointer, Spanish pointer -n02100236 German short-haired pointer -n02100399 setter -n02100583 vizsla, Hungarian pointer -n02100735 English setter -n02100877 Irish setter, red setter -n02101006 Gordon setter -n02101108 spaniel -n02101388 Brittany spaniel -n02101556 clumber, clumber spaniel -n02101670 field spaniel -n02101861 springer spaniel, springer -n02102040 English springer, English springer spaniel -n02102177 Welsh springer spaniel -n02102318 cocker spaniel, English cocker spaniel, cocker -n02102480 Sussex spaniel -n02102605 water spaniel -n02102806 American water spaniel -n02102973 Irish water spaniel -n02103181 griffon, wire-haired pointing griffon -n02103406 working dog -n02103841 watchdog, guard dog -n02104029 kuvasz -n02104184 attack dog -n02104280 housedog -n02104365 schipperke -n02104523 shepherd dog, sheepdog, sheep dog -n02104882 Belgian sheepdog, Belgian shepherd -n02105056 groenendael -n02105162 malinois -n02105251 briard -n02105412 kelpie -n02105505 komondor -n02105641 Old English sheepdog, bobtail -n02105855 Shetland sheepdog, Shetland sheep dog, Shetland -n02106030 collie -n02106166 Border collie -n02106382 Bouvier des Flandres, Bouviers des Flandres -n02106550 Rottweiler -n02106662 German shepherd, German shepherd dog, German police dog, alsatian -n02106854 police dog -n02106966 pinscher -n02107142 Doberman, Doberman pinscher -n02107312 miniature pinscher -n02107420 Sennenhunde -n02107574 Greater Swiss Mountain dog -n02107683 Bernese mountain dog -n02107908 Appenzeller -n02108000 EntleBucher -n02108089 boxer -n02108254 mastiff -n02108422 bull mastiff -n02108551 Tibetan mastiff -n02108672 bulldog, English bulldog -n02108915 French bulldog -n02109047 Great Dane -n02109150 guide dog -n02109256 Seeing Eye dog -n02109391 hearing dog -n02109525 Saint Bernard, St Bernard -n02109687 seizure-alert dog -n02109811 sled dog, sledge dog -n02109961 Eskimo dog, husky -n02110063 malamute, malemute, Alaskan malamute -n02110185 Siberian husky -n02110341 dalmatian, coach dog, carriage dog -n02110532 liver-spotted dalmatian -n02110627 affenpinscher, monkey pinscher, monkey dog -n02110806 basenji -n02110958 pug, pug-dog -n02111129 Leonberg -n02111277 Newfoundland, Newfoundland dog -n02111500 Great Pyrenees -n02111626 spitz -n02111889 Samoyed, Samoyede -n02112018 Pomeranian -n02112137 chow, chow chow -n02112350 keeshond -n02112497 griffon, Brussels griffon, Belgian griffon -n02112706 Brabancon griffon -n02112826 corgi, Welsh corgi -n02113023 Pembroke, Pembroke Welsh corgi -n02113186 Cardigan, Cardigan Welsh corgi -n02113335 poodle, poodle dog -n02113624 toy poodle -n02113712 miniature poodle -n02113799 standard poodle -n02113892 large poodle -n02113978 Mexican hairless -n02114100 wolf -n02114367 timber wolf, grey wolf, gray wolf, Canis lupus -n02114548 white wolf, Arctic wolf, Canis lupus tundrarum -n02114712 red wolf, maned wolf, Canis rufus, Canis niger -n02114855 coyote, prairie wolf, brush wolf, Canis latrans -n02115012 coydog -n02115096 jackal, Canis aureus -n02115335 wild dog -n02115641 dingo, warrigal, warragal, Canis dingo -n02115913 dhole, Cuon alpinus -n02116185 crab-eating dog, crab-eating fox, Dusicyon cancrivorus -n02116450 raccoon dog, Nyctereutes procyonides -n02116738 African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus -n02117135 hyena, hyaena -n02117512 striped hyena, Hyaena hyaena -n02117646 brown hyena, strand wolf, Hyaena brunnea -n02117900 spotted hyena, laughing hyena, Crocuta crocuta -n02118176 aardwolf, Proteles cristata -n02118333 fox -n02118643 vixen -n02118707 Reynard -n02119022 red fox, Vulpes vulpes -n02119247 black fox -n02119359 silver fox -n02119477 red fox, Vulpes fulva -n02119634 kit fox, prairie fox, Vulpes velox -n02119789 kit fox, Vulpes macrotis -n02120079 Arctic fox, white fox, Alopex lagopus -n02120278 blue fox -n02120505 grey fox, gray fox, Urocyon cinereoargenteus -n02120997 feline, felid -n02121620 cat, true cat -n02121808 domestic cat, house cat, Felis domesticus, Felis catus -n02122298 kitty, kitty-cat, puss, pussy, pussycat -n02122430 mouser -n02122510 alley cat -n02122580 stray -n02122725 tom, tomcat -n02122810 gib -n02122878 tabby, queen -n02122948 kitten, kitty -n02123045 tabby, tabby cat -n02123159 tiger cat -n02123242 tortoiseshell, tortoiseshell-cat, calico cat -n02123394 Persian cat -n02123478 Angora, Angora cat -n02123597 Siamese cat, Siamese -n02123785 blue point Siamese -n02123917 Burmese cat -n02124075 Egyptian cat -n02124157 Maltese, Maltese cat -n02124313 Abyssinian, Abyssinian cat -n02124484 Manx, Manx cat -n02124623 wildcat -n02125010 sand cat -n02125081 European wildcat, catamountain, Felis silvestris -n02125311 cougar, puma, catamount, mountain lion, painter, panther, Felis concolor -n02125494 ocelot, panther cat, Felis pardalis -n02125689 jaguarundi, jaguarundi cat, jaguarondi, eyra, Felis yagouaroundi -n02125872 kaffir cat, caffer cat, Felis ocreata -n02126028 jungle cat, Felis chaus -n02126139 serval, Felis serval -n02126317 leopard cat, Felis bengalensis -n02126640 margay, margay cat, Felis wiedi -n02126787 manul, Pallas's cat, Felis manul -n02127052 lynx, catamount -n02127292 common lynx, Lynx lynx -n02127381 Canada lynx, Lynx canadensis -n02127482 bobcat, bay lynx, Lynx rufus -n02127586 spotted lynx, Lynx pardina -n02127678 caracal, desert lynx, Lynx caracal -n02127808 big cat, cat -n02128385 leopard, Panthera pardus -n02128598 leopardess -n02128669 panther -n02128757 snow leopard, ounce, Panthera uncia -n02128925 jaguar, panther, Panthera onca, Felis onca -n02129165 lion, king of beasts, Panthera leo -n02129463 lioness -n02129530 lionet -n02129604 tiger, Panthera tigris -n02129837 Bengal tiger -n02129923 tigress -n02129991 liger -n02130086 tiglon, tigon -n02130308 cheetah, chetah, Acinonyx jubatus -n02130545 saber-toothed tiger, sabertooth -n02130925 Smiledon californicus -n02131653 bear -n02132136 brown bear, bruin, Ursus arctos -n02132320 bruin -n02132466 Syrian bear, Ursus arctos syriacus -n02132580 grizzly, grizzly bear, silvertip, silver-tip, Ursus horribilis, Ursus arctos horribilis -n02132788 Alaskan brown bear, Kodiak bear, Kodiak, Ursus middendorffi, Ursus arctos middendorffi -n02133161 American black bear, black bear, Ursus americanus, Euarctos americanus -n02133400 cinnamon bear -n02133704 Asiatic black bear, black bear, Ursus thibetanus, Selenarctos thibetanus -n02134084 ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus -n02134418 sloth bear, Melursus ursinus, Ursus ursinus -n02134971 viverrine, viverrine mammal -n02135220 civet, civet cat -n02135610 large civet, Viverra zibetha -n02135844 small civet, Viverricula indica, Viverricula malaccensis -n02136103 binturong, bearcat, Arctictis bintourong -n02136285 Cryptoprocta, genus Cryptoprocta -n02136452 fossa, fossa cat, Cryptoprocta ferox -n02136794 fanaloka, Fossa fossa -n02137015 genet, Genetta genetta -n02137302 banded palm civet, Hemigalus hardwickii -n02137549 mongoose -n02137722 Indian mongoose, Herpestes nyula -n02137888 ichneumon, Herpestes ichneumon -n02138169 palm cat, palm civet -n02138441 meerkat, mierkat -n02138647 slender-tailed meerkat, Suricata suricatta -n02138777 suricate, Suricata tetradactyla -n02139199 bat, chiropteran -n02139671 fruit bat, megabat -n02140049 flying fox -n02140179 Pteropus capestratus -n02140268 Pteropus hypomelanus -n02140491 harpy, harpy bat, tube-nosed bat, tube-nosed fruit bat -n02140858 Cynopterus sphinx -n02141306 carnivorous bat, microbat -n02141611 mouse-eared bat -n02141713 leafnose bat, leaf-nosed bat -n02142407 macrotus, Macrotus californicus -n02142734 spearnose bat -n02142898 Phyllostomus hastatus -n02143142 hognose bat, Choeronycteris mexicana -n02143439 horseshoe bat -n02143891 horseshoe bat -n02144251 orange bat, orange horseshoe bat, Rhinonicteris aurantius -n02144593 false vampire, false vampire bat -n02144936 big-eared bat, Megaderma lyra -n02145424 vespertilian bat, vespertilionid -n02145910 frosted bat, Vespertilio murinus -n02146201 red bat, Lasiurus borealis -n02146371 brown bat -n02146700 little brown bat, little brown myotis, Myotis leucifugus -n02146879 cave myotis, Myotis velifer -n02147173 big brown bat, Eptesicus fuscus -n02147328 serotine, European brown bat, Eptesicus serotinus -n02147591 pallid bat, cave bat, Antrozous pallidus -n02147947 pipistrelle, pipistrel, Pipistrellus pipistrellus -n02148088 eastern pipistrel, Pipistrellus subflavus -n02148512 jackass bat, spotted bat, Euderma maculata -n02148835 long-eared bat -n02148991 western big-eared bat, Plecotus townsendi -n02149420 freetail, free-tailed bat, freetailed bat -n02149653 guano bat, Mexican freetail bat, Tadarida brasiliensis -n02149861 pocketed bat, pocketed freetail bat, Tadirida femorosacca -n02150134 mastiff bat -n02150482 vampire bat, true vampire bat -n02150885 Desmodus rotundus -n02151230 hairy-legged vampire bat, Diphylla ecaudata -n02152740 predator, predatory animal -n02152881 prey, quarry -n02152991 game -n02153109 big game -n02153203 game bird -n02153809 fossorial mammal -n02156732 tetrapod -n02156871 quadruped -n02157206 hexapod -n02157285 biped -n02159955 insect -n02160947 social insect -n02161225 holometabola, metabola -n02161338 defoliator -n02161457 pollinator -n02161588 gallfly -n02162561 scorpion fly -n02163008 hanging fly -n02163297 collembolan, springtail -n02164464 beetle -n02165105 tiger beetle -n02165456 ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle -n02165877 two-spotted ladybug, Adalia bipunctata -n02166229 Mexican bean beetle, bean beetle, Epilachna varivestis -n02166567 Hippodamia convergens -n02166826 vedalia, Rodolia cardinalis -n02167151 ground beetle, carabid beetle -n02167505 bombardier beetle -n02167820 calosoma -n02167944 searcher, searcher beetle, Calosoma scrutator -n02168245 firefly, lightning bug -n02168427 glowworm -n02168699 long-horned beetle, longicorn, longicorn beetle -n02169023 sawyer, sawyer beetle -n02169218 pine sawyer -n02169497 leaf beetle, chrysomelid -n02169705 flea beetle -n02169974 Colorado potato beetle, Colorado beetle, potato bug, potato beetle, Leptinotarsa decemlineata -n02170400 carpet beetle, carpet bug -n02170599 buffalo carpet beetle, Anthrenus scrophulariae -n02170738 black carpet beetle -n02170993 clerid beetle, clerid -n02171164 bee beetle -n02171453 lamellicorn beetle -n02171869 scarabaeid beetle, scarabaeid, scarabaean -n02172182 dung beetle -n02172518 scarab, scarabaeus, Scarabaeus sacer -n02172678 tumblebug -n02172761 dorbeetle -n02172870 June beetle, June bug, May bug, May beetle -n02173113 green June beetle, figeater -n02173373 Japanese beetle, Popillia japonica -n02173784 Oriental beetle, Asiatic beetle, Anomala orientalis -n02174001 rhinoceros beetle -n02174355 melolonthid beetle -n02174659 cockchafer, May bug, May beetle, Melolontha melolontha -n02175014 rose chafer, rose bug, Macrodactylus subspinosus -n02175569 rose chafer, rose beetle, Cetonia aurata -n02175916 stag beetle -n02176261 elaterid beetle, elater, elaterid -n02176439 click beetle, skipjack, snapping beetle -n02176747 firefly, fire beetle, Pyrophorus noctiluca -n02176916 wireworm -n02177196 water beetle -n02177506 whirligig beetle -n02177775 deathwatch beetle, deathwatch, Xestobium rufovillosum -n02177972 weevil -n02178411 snout beetle -n02178717 boll weevil, Anthonomus grandis -n02179012 blister beetle, meloid -n02179192 oil beetle -n02179340 Spanish fly -n02179891 Dutch-elm beetle, Scolytus multistriatus -n02180233 bark beetle -n02180427 spruce bark beetle, Dendroctonus rufipennis -n02180875 rove beetle -n02181235 darkling beetle, darkling groung beetle, tenebrionid -n02181477 mealworm -n02181724 flour beetle, flour weevil -n02182045 seed beetle, seed weevil -n02182355 pea weevil, Bruchus pisorum -n02182642 bean weevil, Acanthoscelides obtectus -n02182930 rice weevil, black weevil, Sitophylus oryzae -n02183096 Asian longhorned beetle, Anoplophora glabripennis -n02183507 web spinner -n02183857 louse, sucking louse -n02184473 common louse, Pediculus humanus -n02184589 head louse, Pediculus capitis -n02184720 body louse, cootie, Pediculus corporis -n02185167 crab louse, pubic louse, crab, Phthirius pubis -n02185481 bird louse, biting louse, louse -n02186153 flea -n02186717 Pulex irritans -n02187150 dog flea, Ctenocephalides canis -n02187279 cat flea, Ctenocephalides felis -n02187554 chigoe, chigger, chigoe flea, Tunga penetrans -n02187900 sticktight, sticktight flea, Echidnophaga gallinacea -n02188699 dipterous insect, two-winged insects, dipteran, dipteron -n02189363 gall midge, gallfly, gall gnat -n02189670 Hessian fly, Mayetiola destructor -n02190166 fly -n02190790 housefly, house fly, Musca domestica -n02191273 tsetse fly, tsetse, tzetze fly, tzetze, glossina -n02191773 blowfly, blow fly -n02191979 bluebottle, Calliphora vicina -n02192252 greenbottle, greenbottle fly -n02192513 flesh fly, Sarcophaga carnaria -n02192814 tachina fly -n02193009 gadfly -n02193163 botfly -n02194249 human botfly, Dermatobia hominis -n02194750 sheep botfly, sheep gadfly, Oestrus ovis -n02195091 warble fly -n02195526 horsefly, cleg, clegg, horse fly -n02195819 bee fly -n02196119 robber fly, bee killer -n02196344 fruit fly, pomace fly -n02196896 apple maggot, railroad worm, Rhagoletis pomonella -n02197185 Mediterranean fruit fly, medfly, Ceratitis capitata -n02197689 drosophila, Drosophila melanogaster -n02197877 vinegar fly -n02198129 leaf miner, leaf-miner -n02198532 louse fly, hippoboscid -n02198859 horse tick, horsefly, Hippobosca equina -n02199170 sheep ked, sheep-tick, sheep tick, Melophagus Ovinus -n02199502 horn fly, Haematobia irritans -n02200198 mosquito -n02200509 wiggler, wriggler -n02200630 gnat -n02200850 yellow-fever mosquito, Aedes aegypti -n02201000 Asian tiger mosquito, Aedes albopictus -n02201497 anopheline -n02201626 malarial mosquito, malaria mosquito -n02202006 common mosquito, Culex pipiens -n02202124 Culex quinquefasciatus, Culex fatigans -n02202287 gnat -n02202678 punkie, punky, punkey, no-see-um, biting midge -n02203152 midge -n02203592 fungus gnat -n02203978 psychodid -n02204249 sand fly, sandfly, Phlebotomus papatasii -n02204722 fungus gnat, sciara, sciarid -n02204907 armyworm -n02205219 crane fly, daddy longlegs -n02205673 blackfly, black fly, buffalo gnat -n02206270 hymenopterous insect, hymenopteran, hymenopteron, hymenopter -n02206856 bee -n02207179 drone -n02207345 queen bee -n02207449 worker -n02207647 soldier -n02207805 worker bee -n02208280 honeybee, Apis mellifera -n02208498 Africanized bee, Africanized honey bee, killer bee, Apis mellifera scutellata, Apis mellifera adansonii -n02208848 black bee, German bee -n02208979 Carniolan bee -n02209111 Italian bee -n02209354 carpenter bee -n02209624 bumblebee, humblebee -n02209964 cuckoo-bumblebee -n02210427 andrena, andrenid, mining bee -n02210921 Nomia melanderi, alkali bee -n02211444 leaf-cutting bee, leaf-cutter, leaf-cutter bee -n02211627 mason bee -n02211896 potter bee -n02212062 wasp -n02212602 vespid, vespid wasp -n02212958 paper wasp -n02213107 hornet -n02213239 giant hornet, Vespa crabro -n02213543 common wasp, Vespula vulgaris -n02213663 bald-faced hornet, white-faced hornet, Vespula maculata -n02213788 yellow jacket, yellow hornet, Vespula maculifrons -n02214096 Polistes annularis -n02214341 mason wasp -n02214499 potter wasp -n02214660 Mutillidae, family Mutillidae -n02214773 velvet ant -n02215161 sphecoid wasp, sphecoid -n02215621 mason wasp -n02215770 digger wasp -n02216211 cicada killer, Sphecius speciosis -n02216365 mud dauber -n02216740 gall wasp, gallfly, cynipid wasp, cynipid gall wasp -n02217563 chalcid fly, chalcidfly, chalcid, chalcid wasp -n02217839 strawworm, jointworm -n02218134 chalcis fly -n02218371 ichneumon fly -n02218713 sawfly -n02219015 birch leaf miner, Fenusa pusilla -n02219486 ant, emmet, pismire -n02220055 pharaoh ant, pharaoh's ant, Monomorium pharaonis -n02220225 little black ant, Monomorium minimum -n02220518 army ant, driver ant, legionary ant -n02220804 carpenter ant -n02221083 fire ant -n02221414 wood ant, Formica rufa -n02221571 slave ant -n02221715 Formica fusca -n02221820 slave-making ant, slave-maker -n02222035 sanguinary ant, Formica sanguinea -n02222321 bulldog ant -n02222582 Amazon ant, Polyergus rufescens -n02223266 termite, white ant -n02223520 dry-wood termite -n02224023 Reticulitermes lucifugus -n02224713 Mastotermes darwiniensis -n02225081 Mastotermes electrodominicus -n02225798 powder-post termite, Cryptotermes brevis -n02226183 orthopterous insect, orthopteron, orthopteran -n02226429 grasshopper, hopper -n02226821 short-horned grasshopper, acridid -n02226970 locust -n02227247 migratory locust, Locusta migratoria -n02227604 migratory grasshopper -n02227966 long-horned grasshopper, tettigoniid -n02228341 katydid -n02228697 mormon cricket, Anabrus simplex -n02229156 sand cricket, Jerusalem cricket, Stenopelmatus fuscus -n02229544 cricket -n02229765 mole cricket -n02230023 European house cricket, Acheta domestica -n02230187 field cricket, Acheta assimilis -n02230480 tree cricket -n02230634 snowy tree cricket, Oecanthus fultoni -n02231052 phasmid, phasmid insect -n02231487 walking stick, walkingstick, stick insect -n02231803 diapheromera, Diapheromera femorata -n02232223 walking leaf, leaf insect -n02233338 cockroach, roach -n02233943 oriental cockroach, oriental roach, Asiatic cockroach, blackbeetle, Blatta orientalis -n02234355 American cockroach, Periplaneta americana -n02234570 Australian cockroach, Periplaneta australasiae -n02234848 German cockroach, Croton bug, crotonbug, water bug, Blattella germanica -n02235205 giant cockroach -n02236044 mantis, mantid -n02236241 praying mantis, praying mantid, Mantis religioso -n02236355 bug -n02236896 hemipterous insect, bug, hemipteran, hemipteron -n02237424 leaf bug, plant bug -n02237581 mirid bug, mirid, capsid -n02237868 four-lined plant bug, four-lined leaf bug, Poecilocapsus lineatus -n02238235 lygus bug -n02238358 tarnished plant bug, Lygus lineolaris -n02238594 lace bug -n02238887 lygaeid, lygaeid bug -n02239192 chinch bug, Blissus leucopterus -n02239528 coreid bug, coreid -n02239774 squash bug, Anasa tristis -n02240068 leaf-footed bug, leaf-foot bug -n02240517 bedbug, bed bug, chinch, Cimex lectularius -n02241008 backswimmer, Notonecta undulata -n02241426 true bug -n02241569 heteropterous insect -n02241799 water bug -n02242137 giant water bug -n02242455 water scorpion -n02243209 water boatman, boat bug -n02243562 water strider, pond-skater, water skater -n02243878 common pond-skater, Gerris lacustris -n02244173 assassin bug, reduviid -n02244515 conenose, cone-nosed bug, conenose bug, big bedbug, kissing bug -n02244797 wheel bug, Arilus cristatus -n02245111 firebug -n02245443 cotton stainer -n02246011 homopterous insect, homopteran -n02246628 whitefly -n02246941 citrus whitefly, Dialeurodes citri -n02247216 greenhouse whitefly, Trialeurodes vaporariorum -n02247511 sweet-potato whitefly -n02247655 superbug, Bemisia tabaci, poinsettia strain -n02248062 cotton strain -n02248368 coccid insect -n02248510 scale insect -n02248887 soft scale -n02249134 brown soft scale, Coccus hesperidum -n02249515 armored scale -n02249809 San Jose scale, Aspidiotus perniciosus -n02250280 cochineal insect, cochineal, Dactylopius coccus -n02250822 mealybug, mealy bug -n02251067 citrophilous mealybug, citrophilus mealybug, Pseudococcus fragilis -n02251233 Comstock mealybug, Comstock's mealybug, Pseudococcus comstocki -n02251593 citrus mealybug, Planococcus citri -n02251775 plant louse, louse -n02252226 aphid -n02252799 apple aphid, green apple aphid, Aphis pomi -n02252972 blackfly, bean aphid, Aphis fabae -n02253127 greenfly -n02253264 green peach aphid -n02253494 ant cow -n02253715 woolly aphid, woolly plant louse -n02253913 woolly apple aphid, American blight, Eriosoma lanigerum -n02254246 woolly alder aphid, Prociphilus tessellatus -n02254697 adelgid -n02254901 balsam woolly aphid, Adelges piceae -n02255023 spruce gall aphid, Adelges abietis -n02255391 woolly adelgid -n02256172 jumping plant louse, psylla, psyllid -n02256656 cicada, cicala -n02257003 dog-day cicada, harvest fly -n02257284 seventeen-year locust, periodical cicada, Magicicada septendecim -n02257715 spittle insect, spittlebug -n02257985 froghopper -n02258198 meadow spittlebug, Philaenus spumarius -n02258508 pine spittlebug -n02258629 Saratoga spittlebug, Aphrophora saratogensis -n02259212 leafhopper -n02259377 plant hopper, planthopper -n02259708 treehopper -n02259987 lantern fly, lantern-fly -n02260421 psocopterous insect -n02260863 psocid -n02261063 bark-louse, bark louse -n02261419 booklouse, book louse, deathwatch, Liposcelis divinatorius -n02261757 common booklouse, Trogium pulsatorium -n02262178 ephemerid, ephemeropteran -n02262449 mayfly, dayfly, shadfly -n02262803 stonefly, stone fly, plecopteran -n02263378 neuropteron, neuropteran, neuropterous insect -n02264021 ant lion, antlion, antlion fly -n02264232 doodlebug, ant lion, antlion -n02264363 lacewing, lacewing fly -n02264591 aphid lion, aphis lion -n02264885 green lacewing, chrysopid, stink fly -n02265330 brown lacewing, hemerobiid, hemerobiid fly -n02266050 dobson, dobsonfly, dobson fly, Corydalus cornutus -n02266269 hellgrammiate, dobson -n02266421 fish fly, fish-fly -n02266864 alderfly, alder fly, Sialis lutaria -n02267208 snakefly -n02267483 mantispid -n02268148 odonate -n02268443 dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk -n02268853 damselfly -n02269196 trichopterous insect, trichopteran, trichopteron -n02269340 caddis fly, caddis-fly, caddice fly, caddice-fly -n02269522 caseworm -n02269657 caddisworm, strawworm -n02270011 thysanuran insect, thysanuron -n02270200 bristletail -n02270623 silverfish, Lepisma saccharina -n02270945 firebrat, Thermobia domestica -n02271222 jumping bristletail, machilid -n02271570 thysanopter, thysanopteron, thysanopterous insect -n02271897 thrips, thrip, thripid -n02272286 tobacco thrips, Frankliniella fusca -n02272552 onion thrips, onion louse, Thrips tobaci -n02272871 earwig -n02273392 common European earwig, Forficula auricularia -n02274024 lepidopterous insect, lepidopteron, lepidopteran -n02274259 butterfly -n02274822 nymphalid, nymphalid butterfly, brush-footed butterfly, four-footed butterfly -n02275560 mourning cloak, mourning cloak butterfly, Camberwell beauty, Nymphalis antiopa -n02275773 tortoiseshell, tortoiseshell butterfly -n02276078 painted beauty, Vanessa virginiensis -n02276258 admiral -n02276355 red admiral, Vanessa atalanta -n02276749 white admiral, Limenitis camilla -n02276902 banded purple, white admiral, Limenitis arthemis -n02277094 red-spotted purple, Limenitis astyanax -n02277268 viceroy, Limenitis archippus -n02277422 anglewing -n02277742 ringlet, ringlet butterfly -n02278024 comma, comma butterfly, Polygonia comma -n02278210 fritillary -n02278463 silverspot -n02278839 emperor butterfly, emperor -n02278980 purple emperor, Apatura iris -n02279257 peacock, peacock butterfly, Inachis io -n02279637 danaid, danaid butterfly -n02279972 monarch, monarch butterfly, milkweed butterfly, Danaus plexippus -n02280458 pierid, pierid butterfly -n02280649 cabbage butterfly -n02281015 small white, Pieris rapae -n02281136 large white, Pieris brassicae -n02281267 southern cabbage butterfly, Pieris protodice -n02281406 sulphur butterfly, sulfur butterfly -n02281787 lycaenid, lycaenid butterfly -n02282257 blue -n02282385 copper -n02282553 American copper, Lycaena hypophlaeas -n02282903 hairstreak, hairstreak butterfly -n02283077 Strymon melinus -n02283201 moth -n02283617 moth miller, miller -n02283951 tortricid, tortricid moth -n02284224 leaf roller, leaf-roller -n02284611 tea tortrix, tortrix, Homona coffearia -n02284884 orange tortrix, tortrix, Argyrotaenia citrana -n02285179 codling moth, codlin moth, Carpocapsa pomonella -n02285548 lymantriid, tussock moth -n02285801 tussock caterpillar -n02286089 gypsy moth, gipsy moth, Lymantria dispar -n02286425 browntail, brown-tail moth, Euproctis phaeorrhoea -n02286654 gold-tail moth, Euproctis chrysorrhoea -n02287004 geometrid, geometrid moth -n02287352 Paleacrita vernata -n02287622 Alsophila pometaria -n02287799 cankerworm -n02287987 spring cankerworm -n02288122 fall cankerworm -n02288268 measuring worm, inchworm, looper -n02288789 pyralid, pyralid moth -n02289307 bee moth, wax moth, Galleria mellonella -n02289610 corn borer, European corn borer moth, corn borer moth, Pyrausta nubilalis -n02289988 Mediterranean flour moth, Anagasta kuehniella -n02290340 tobacco moth, cacao moth, Ephestia elutella -n02290664 almond moth, fig moth, Cadra cautella -n02290870 raisin moth, Cadra figulilella -n02291220 tineoid, tineoid moth -n02291572 tineid, tineid moth -n02291748 clothes moth -n02292085 casemaking clothes moth, Tinea pellionella -n02292401 webbing clothes moth, webbing moth, Tineola bisselliella -n02292692 carpet moth, tapestry moth, Trichophaga tapetzella -n02293352 gelechiid, gelechiid moth -n02293868 grain moth -n02294097 angoumois moth, angoumois grain moth, Sitotroga cerealella -n02294407 potato moth, potato tuber moth, splitworm, Phthorimaea operculella -n02294577 potato tuberworm, Phthorimaea operculella -n02295064 noctuid moth, noctuid, owlet moth -n02295390 cutworm -n02295870 underwing -n02296021 red underwing, Catocala nupta -n02296276 antler moth, Cerapteryx graminis -n02296612 heliothis moth, Heliothis zia -n02296912 army cutworm, Chorizagrotis auxiliaris -n02297294 armyworm, Pseudaletia unipuncta -n02297442 armyworm, army worm, Pseudaletia unipuncta -n02297819 Spodoptera exigua -n02297938 beet armyworm, Spodoptera exigua -n02298095 Spodoptera frugiperda -n02298218 fall armyworm, Spodoptera frugiperda -n02298541 hawkmoth, hawk moth, sphingid, sphinx moth, hummingbird moth -n02299039 Manduca sexta -n02299157 tobacco hornworm, tomato worm, Manduca sexta -n02299378 Manduca quinquemaculata -n02299505 tomato hornworm, potato worm, Manduca quinquemaculata -n02299846 death's-head moth, Acherontia atropos -n02300173 bombycid, bombycid moth, silkworm moth -n02300554 domestic silkworm moth, domesticated silkworm moth, Bombyx mori -n02300797 silkworm -n02301452 saturniid, saturniid moth -n02301935 emperor, emperor moth, Saturnia pavonia -n02302244 imperial moth, Eacles imperialis -n02302459 giant silkworm moth, silkworm moth -n02302620 silkworm, giant silkworm, wild wilkworm -n02302969 luna moth, Actias luna -n02303284 cecropia, cecropia moth, Hyalophora cecropia -n02303585 cynthia moth, Samia cynthia, Samia walkeri -n02303777 ailanthus silkworm, Samia cynthia -n02304036 io moth, Automeris io -n02304432 polyphemus moth, Antheraea polyphemus -n02304657 pernyi moth, Antheraea pernyi -n02304797 tussah, tusseh, tussur, tussore, tusser, Antheraea mylitta -n02305085 atlas moth, Atticus atlas -n02305407 arctiid, arctiid moth -n02305636 tiger moth -n02305929 cinnabar, cinnabar moth, Callimorpha jacobeae -n02306433 lasiocampid, lasiocampid moth -n02306825 eggar, egger -n02307176 tent-caterpillar moth, Malacosoma americana -n02307325 tent caterpillar -n02307515 tent-caterpillar moth, Malacosoma disstria -n02307681 forest tent caterpillar, Malacosoma disstria -n02307910 lappet, lappet moth -n02308033 lappet caterpillar -n02308139 webworm -n02308471 webworm moth -n02308618 Hyphantria cunea -n02308735 fall webworm, Hyphantria cunea -n02309120 garden webworm, Loxostege similalis -n02309242 instar -n02309337 caterpillar -n02309841 corn borer, Pyrausta nubilalis -n02310000 bollworm -n02310149 pink bollworm, Gelechia gossypiella -n02310334 corn earworm, cotton bollworm, tomato fruitworm, tobacco budworm, vetchworm, Heliothis zia -n02310585 cabbageworm, Pieris rapae -n02310717 woolly bear, woolly bear caterpillar -n02310941 woolly bear moth -n02311060 larva -n02311617 nymph -n02311748 leptocephalus -n02312006 grub -n02312175 maggot -n02312325 leatherjacket -n02312427 pupa -n02312640 chrysalis -n02312912 imago -n02313008 queen -n02313360 phoronid -n02313709 bryozoan, polyzoan, sea mat, sea moss, moss animal -n02315487 brachiopod, lamp shell, lampshell -n02315821 peanut worm, sipunculid -n02316707 echinoderm -n02317335 starfish, sea star -n02317781 brittle star, brittle-star, serpent star -n02318167 basket star, basket fish -n02318687 Astrophyton muricatum -n02319095 sea urchin -n02319308 edible sea urchin, Echinus esculentus -n02319555 sand dollar -n02319829 heart urchin -n02320127 crinoid -n02320465 sea lily -n02321170 feather star, comatulid -n02321529 sea cucumber, holothurian -n02322047 trepang, Holothuria edulis -n02322992 Duplicidentata -n02323449 lagomorph, gnawing mammal -n02323902 leporid, leporid mammal -n02324045 rabbit, coney, cony -n02324431 rabbit ears -n02324514 lapin -n02324587 bunny, bunny rabbit -n02324850 European rabbit, Old World rabbit, Oryctolagus cuniculus -n02325366 wood rabbit, cottontail, cottontail rabbit -n02325722 eastern cottontail, Sylvilagus floridanus -n02325884 swamp rabbit, canecutter, swamp hare, Sylvilagus aquaticus -n02326074 marsh hare, swamp rabbit, Sylvilagus palustris -n02326432 hare -n02326763 leveret -n02326862 European hare, Lepus europaeus -n02327028 jackrabbit -n02327175 white-tailed jackrabbit, whitetail jackrabbit, Lepus townsendi -n02327435 blacktail jackrabbit, Lepus californicus -n02327656 polar hare, Arctic hare, Lepus arcticus -n02327842 snowshoe hare, snowshoe rabbit, varying hare, Lepus americanus -n02328009 Belgian hare, leporide -n02328150 Angora, Angora rabbit -n02328429 pika, mouse hare, rock rabbit, coney, cony -n02328820 little chief hare, Ochotona princeps -n02328942 collared pika, Ochotona collaris -n02329401 rodent, gnawer -n02330245 mouse -n02331046 rat -n02331309 pocket rat -n02331842 murine -n02332156 house mouse, Mus musculus -n02332447 harvest mouse, Micromyx minutus -n02332755 field mouse, fieldmouse -n02332954 nude mouse -n02333190 European wood mouse, Apodemus sylvaticus -n02333546 brown rat, Norway rat, Rattus norvegicus -n02333733 wharf rat -n02333819 sewer rat -n02333909 black rat, roof rat, Rattus rattus -n02334201 bandicoot rat, mole rat -n02334460 jerboa rat -n02334728 kangaroo mouse -n02335127 water rat -n02335231 beaver rat -n02336011 New World mouse -n02336275 American harvest mouse, harvest mouse -n02336641 wood mouse -n02336826 white-footed mouse, vesper mouse, Peromyscus leucopus -n02337001 deer mouse, Peromyscus maniculatus -n02337171 cactus mouse, Peromyscus eremicus -n02337332 cotton mouse, Peromyscus gossypinus -n02337598 pygmy mouse, Baiomys taylori -n02337902 grasshopper mouse -n02338145 muskrat, musquash, Ondatra zibethica -n02338449 round-tailed muskrat, Florida water rat, Neofiber alleni -n02338722 cotton rat, Sigmodon hispidus -n02338901 wood rat, wood-rat -n02339282 dusky-footed wood rat -n02339376 vole, field mouse -n02339922 packrat, pack rat, trade rat, bushytail woodrat, Neotoma cinerea -n02340186 dusky-footed woodrat, Neotoma fuscipes -n02340358 eastern woodrat, Neotoma floridana -n02340640 rice rat, Oryzomys palustris -n02340930 pine vole, pine mouse, Pitymys pinetorum -n02341288 meadow vole, meadow mouse, Microtus pennsylvaticus -n02341475 water vole, Richardson vole, Microtus richardsoni -n02341616 prairie vole, Microtus ochrogaster -n02341974 water vole, water rat, Arvicola amphibius -n02342250 red-backed mouse, redback vole -n02342534 phenacomys -n02342885 hamster -n02343058 Eurasian hamster, Cricetus cricetus -n02343320 golden hamster, Syrian hamster, Mesocricetus auratus -n02343772 gerbil, gerbille -n02344175 jird -n02344270 tamarisk gerbil, Meriones unguiculatus -n02344408 sand rat, Meriones longifrons -n02344528 lemming -n02344918 European lemming, Lemmus lemmus -n02345078 brown lemming, Lemmus trimucronatus -n02345340 grey lemming, gray lemming, red-backed lemming -n02345600 pied lemming -n02345774 Hudson bay collared lemming, Dicrostonyx hudsonius -n02345997 southern bog lemming, Synaptomys cooperi -n02346170 northern bog lemming, Synaptomys borealis -n02346627 porcupine, hedgehog -n02346998 Old World porcupine -n02347274 brush-tailed porcupine, brush-tail porcupine -n02347573 long-tailed porcupine, Trichys lipura -n02347744 New World porcupine -n02348173 Canada porcupine, Erethizon dorsatum -n02348788 pocket mouse -n02349205 silky pocket mouse, Perognathus flavus -n02349390 plains pocket mouse, Perognathus flavescens -n02349557 hispid pocket mouse, Perognathus hispidus -n02349847 Mexican pocket mouse, Liomys irroratus -n02350105 kangaroo rat, desert rat, Dipodomys phillipsii -n02350357 Ord kangaroo rat, Dipodomys ordi -n02350670 kangaroo mouse, dwarf pocket rat -n02350989 jumping mouse -n02351343 meadow jumping mouse, Zapus hudsonius -n02351870 jerboa -n02352002 typical jerboa -n02352290 Jaculus jaculus -n02352591 dormouse -n02352932 loir, Glis glis -n02353172 hazel mouse, Muscardinus avellanarius -n02353411 lerot -n02353861 gopher, pocket gopher, pouched rat -n02354162 plains pocket gopher, Geomys bursarius -n02354320 southeastern pocket gopher, Geomys pinetis -n02354621 valley pocket gopher, Thomomys bottae -n02354781 northern pocket gopher, Thomomys talpoides -n02355227 squirrel -n02355477 tree squirrel -n02356381 eastern grey squirrel, eastern gray squirrel, cat squirrel, Sciurus carolinensis -n02356612 western grey squirrel, western gray squirrel, Sciurus griseus -n02356798 fox squirrel, eastern fox squirrel, Sciurus niger -n02356977 black squirrel -n02357111 red squirrel, cat squirrel, Sciurus vulgaris -n02357401 American red squirrel, spruce squirrel, red squirrel, Sciurus hudsonicus, Tamiasciurus hudsonicus -n02357585 chickeree, Douglas squirrel, Tamiasciurus douglasi -n02357911 antelope squirrel, whitetail antelope squirrel, antelope chipmunk, Citellus leucurus -n02358091 ground squirrel, gopher, spermophile -n02358390 mantled ground squirrel, Citellus lateralis -n02358584 suslik, souslik, Citellus citellus -n02358712 flickertail, Richardson ground squirrel, Citellus richardsoni -n02358890 rock squirrel, Citellus variegatus -n02359047 Arctic ground squirrel, parka squirrel, Citellus parryi -n02359324 prairie dog, prairie marmot -n02359556 blacktail prairie dog, Cynomys ludovicianus -n02359667 whitetail prairie dog, Cynomys gunnisoni -n02359915 eastern chipmunk, hackee, striped squirrel, ground squirrel, Tamias striatus -n02360282 chipmunk -n02360480 baronduki, baranduki, barunduki, burunduki, Eutamius asiaticus, Eutamius sibiricus -n02360781 American flying squirrel -n02360933 southern flying squirrel, Glaucomys volans -n02361090 northern flying squirrel, Glaucomys sabrinus -n02361337 marmot -n02361587 groundhog, woodchuck, Marmota monax -n02361706 hoary marmot, whistler, whistling marmot, Marmota caligata -n02361850 yellowbelly marmot, rockchuck, Marmota flaviventris -n02362194 Asiatic flying squirrel -n02363005 beaver -n02363245 Old World beaver, Castor fiber -n02363351 New World beaver, Castor canadensis -n02363996 mountain beaver, sewellel, Aplodontia rufa -n02364520 cavy -n02364673 guinea pig, Cavia cobaya -n02364840 aperea, wild cavy, Cavia porcellus -n02365108 mara, Dolichotis patagonum -n02365480 capybara, capibara, Hydrochoerus hydrochaeris -n02366002 agouti, Dasyprocta aguti -n02366301 paca, Cuniculus paca -n02366579 mountain paca -n02366959 coypu, nutria, Myocastor coypus -n02367492 chinchilla, Chinchilla laniger -n02367812 mountain chinchilla, mountain viscacha -n02368116 viscacha, chinchillon, Lagostomus maximus -n02368399 abrocome, chinchilla rat, rat chinchilla -n02368821 mole rat -n02369293 mole rat -n02369555 sand rat -n02369680 naked mole rat -n02369935 queen, queen mole rat -n02370137 Damaraland mole rat -n02370525 Ungulata -n02370806 ungulate, hoofed mammal -n02371344 unguiculate, unguiculate mammal -n02372140 dinoceras, uintathere -n02372584 hyrax, coney, cony, dassie, das -n02372952 rock hyrax, rock rabbit, Procavia capensis -n02373336 odd-toed ungulate, perissodactyl, perissodactyl mammal -n02374149 equine, equid -n02374451 horse, Equus caballus -n02375302 roan -n02375438 stablemate, stable companion -n02375757 gee-gee -n02375862 eohippus, dawn horse -n02376542 foal -n02376679 filly -n02376791 colt -n02376918 male horse -n02377063 ridgeling, ridgling, ridgel, ridgil -n02377181 stallion, entire -n02377291 stud, studhorse -n02377388 gelding -n02377480 mare, female horse -n02377603 broodmare, stud mare -n02377703 saddle horse, riding horse, mount -n02378149 remount -n02378299 palfrey -n02378415 warhorse -n02378541 cavalry horse -n02378625 charger, courser -n02378755 steed -n02378870 prancer -n02378969 hack -n02379081 cow pony -n02379183 quarter horse -n02379329 Morgan -n02379430 Tennessee walker, Tennessee walking horse, Walking horse, Plantation walking horse -n02379630 American saddle horse -n02379743 Appaloosa -n02379908 Arabian, Arab -n02380052 Lippizan, Lipizzan, Lippizaner -n02380335 pony -n02380464 polo pony -n02380583 mustang -n02380745 bronco, bronc, broncho -n02380875 bucking bronco -n02381004 buckskin -n02381119 crowbait, crow-bait -n02381261 dun -n02381364 grey, gray -n02381460 wild horse -n02381609 tarpan, Equus caballus gomelini -n02381831 Przewalski's horse, Przevalski's horse, Equus caballus przewalskii, Equus caballus przevalskii -n02382039 cayuse, Indian pony -n02382132 hack -n02382204 hack, jade, nag, plug -n02382338 plow horse, plough horse -n02382437 pony -n02382635 Shetland pony -n02382750 Welsh pony -n02382850 Exmoor -n02382948 racehorse, race horse, bangtail -n02383231 thoroughbred -n02384741 steeplechaser -n02384858 racer -n02385002 finisher -n02385098 pony -n02385214 yearling -n02385580 dark horse -n02385676 mudder -n02385776 nonstarter -n02385898 stalking-horse -n02386014 harness horse -n02386141 cob -n02386224 hackney -n02386310 workhorse -n02386496 draft horse, draught horse, dray horse -n02386746 packhorse -n02386853 carthorse, cart horse, drayhorse -n02386968 Clydesdale -n02387093 Percheron -n02387254 farm horse, dobbin -n02387346 shire, shire horse -n02387452 pole horse, poler -n02387722 post horse, post-horse, poster -n02387887 coach horse -n02387983 pacer -n02388143 pacer, pacemaker, pacesetter -n02388276 trotting horse, trotter -n02388453 pole horse -n02388588 stepper, high stepper -n02388735 chestnut -n02388832 liver chestnut -n02388917 bay -n02389026 sorrel -n02389128 palomino -n02389261 pinto -n02389346 ass -n02389559 domestic ass, donkey, Equus asinus -n02389779 burro -n02389865 moke -n02389943 jack, jackass -n02390015 jennet, jenny, jenny ass -n02390101 mule -n02390258 hinny -n02390454 wild ass -n02390640 African wild ass, Equus asinus -n02390738 kiang, Equus kiang -n02390834 onager, Equus hemionus -n02390938 chigetai, dziggetai, Equus hemionus hemionus -n02391049 zebra -n02391234 common zebra, Burchell's zebra, Equus Burchelli -n02391373 mountain zebra, Equus zebra zebra -n02391508 grevy's zebra, Equus grevyi -n02391617 quagga, Equus quagga -n02391994 rhinoceros, rhino -n02392434 Indian rhinoceros, Rhinoceros unicornis -n02392555 woolly rhinoceros, Rhinoceros antiquitatis -n02392824 white rhinoceros, Ceratotherium simum, Diceros simus -n02393161 black rhinoceros, Diceros bicornis -n02393580 tapir -n02393807 New World tapir, Tapirus terrestris -n02393940 Malayan tapir, Indian tapir, Tapirus indicus -n02394477 even-toed ungulate, artiodactyl, artiodactyl mammal -n02395003 swine -n02395406 hog, pig, grunter, squealer, Sus scrofa -n02395694 piglet, piggy, shoat, shote -n02395855 sucking pig -n02395931 porker -n02396014 boar -n02396088 sow -n02396157 razorback, razorback hog, razorbacked hog -n02396427 wild boar, boar, Sus scrofa -n02396796 babirusa, babiroussa, babirussa, Babyrousa Babyrussa -n02397096 warthog -n02397529 peccary, musk hog -n02397744 collared peccary, javelina, Tayassu angulatus, Tayassu tajacu, Peccari angulatus -n02397987 white-lipped peccary, Tayassu pecari -n02398521 hippopotamus, hippo, river horse, Hippopotamus amphibius -n02399000 ruminant -n02401031 bovid -n02402010 bovine -n02402175 ox, wild ox -n02402425 cattle, cows, kine, oxen, Bos taurus -n02403003 ox -n02403153 stirk -n02403231 bullock, steer -n02403325 bull -n02403454 cow, moo-cow -n02403740 heifer -n02403820 bullock -n02403920 dogie, dogy, leppy -n02404028 maverick -n02404186 beef, beef cattle -n02404432 longhorn, Texas longhorn -n02404573 Brahman, Brahma, Brahmin, Bos indicus -n02404906 zebu -n02405101 aurochs, urus, Bos primigenius -n02405302 yak, Bos grunniens -n02405440 banteng, banting, tsine, Bos banteng -n02405577 Welsh, Welsh Black -n02405692 red poll -n02405799 Santa Gertrudis -n02405929 Aberdeen Angus, Angus, black Angus -n02406046 Africander -n02406174 dairy cattle, dairy cow, milch cow, milk cow, milcher, milker -n02406432 Ayrshire -n02406533 Brown Swiss -n02406647 Charolais -n02406749 Jersey -n02406859 Devon -n02406952 grade -n02407071 Durham, shorthorn -n02407172 milking shorthorn -n02407276 Galloway -n02407390 Friesian, Holstein, Holstein-Friesian -n02407521 Guernsey -n02407625 Hereford, whiteface -n02407763 cattalo, beefalo -n02407959 Old World buffalo, buffalo -n02408429 water buffalo, water ox, Asiatic buffalo, Bubalus bubalis -n02408660 Indian buffalo -n02408817 carabao -n02409038 anoa, dwarf buffalo, Anoa depressicornis -n02409202 tamarau, tamarao, Bubalus mindorensis, Anoa mindorensis -n02409508 Cape buffalo, Synercus caffer -n02409870 Asian wild ox -n02410011 gaur, Bibos gaurus -n02410141 gayal, mithan, Bibos frontalis -n02410509 bison -n02410702 American bison, American buffalo, buffalo, Bison bison -n02410900 wisent, aurochs, Bison bonasus -n02411206 musk ox, musk sheep, Ovibos moschatus -n02411705 sheep -n02411999 ewe -n02412080 ram, tup -n02412210 wether -n02412440 lamb -n02412629 lambkin -n02412700 baa-lamb -n02412787 hog, hogget, hogg -n02412909 teg -n02412977 Persian lamb -n02413050 black sheep -n02413131 domestic sheep, Ovis aries -n02413484 Cotswold -n02413593 Hampshire, Hampshire down -n02413717 Lincoln -n02413824 Exmoor -n02413917 Cheviot -n02414043 broadtail, caracul, karakul -n02414209 longwool -n02414290 merino, merino sheep -n02414442 Rambouillet -n02414578 wild sheep -n02414763 argali, argal, Ovis ammon -n02414904 Marco Polo sheep, Marco Polo's sheep, Ovis poli -n02415130 urial, Ovis vignei -n02415253 Dall sheep, Dall's sheep, white sheep, Ovis montana dalli -n02415435 mountain sheep -n02415577 bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis -n02415829 mouflon, moufflon, Ovis musimon -n02416104 aoudad, arui, audad, Barbary sheep, maned sheep, Ammotragus lervia -n02416519 goat, caprine animal -n02416820 kid -n02416880 billy, billy goat, he-goat -n02416964 nanny, nanny-goat, she-goat -n02417070 domestic goat, Capra hircus -n02417242 Cashmere goat, Kashmir goat -n02417387 Angora, Angora goat -n02417534 wild goat -n02417663 bezoar goat, pasang, Capra aegagrus -n02417785 markhor, markhoor, Capra falconeri -n02417914 ibex, Capra ibex -n02418064 goat antelope -n02418465 mountain goat, Rocky Mountain goat, Oreamnos americanus -n02418770 goral, Naemorhedus goral -n02419056 serow -n02419336 chamois, Rupicapra rupicapra -n02419634 takin, gnu goat, Budorcas taxicolor -n02419796 antelope -n02420509 blackbuck, black buck, Antilope cervicapra -n02420828 gerenuk, Litocranius walleri -n02421136 addax, Addax nasomaculatus -n02421449 gnu, wildebeest -n02421792 dik-dik -n02422106 hartebeest -n02422391 sassaby, topi, Damaliscus lunatus -n02422699 impala, Aepyceros melampus -n02423022 gazelle -n02423218 Thomson's gazelle, Gazella thomsoni -n02423362 Gazella subgutturosa -n02423589 springbok, springbuck, Antidorcas marsupialis, Antidorcas euchore -n02424085 bongo, Tragelaphus eurycerus, Boocercus eurycerus -n02424305 kudu, koodoo, koudou -n02424486 greater kudu, Tragelaphus strepsiceros -n02424589 lesser kudu, Tragelaphus imberbis -n02424695 harnessed antelope -n02424909 nyala, Tragelaphus angasi -n02425086 mountain nyala, Tragelaphus buxtoni -n02425228 bushbuck, guib, Tragelaphus scriptus -n02425532 nilgai, nylghai, nylghau, blue bull, Boselaphus tragocamelus -n02425887 sable antelope, Hippotragus niger -n02426176 saiga, Saiga tatarica -n02426481 steenbok, steinbok, Raphicerus campestris -n02426813 eland -n02427032 common eland, Taurotragus oryx -n02427183 giant eland, Taurotragus derbianus -n02427470 kob, Kobus kob -n02427576 lechwe, Kobus leche -n02427724 waterbuck -n02428089 puku, Adenota vardoni -n02428349 oryx, pasang -n02428508 gemsbok, gemsbuck, Oryx gazella -n02428842 forest goat, spindle horn, Pseudoryx nghetinhensis -n02429456 pronghorn, prongbuck, pronghorn antelope, American antelope, Antilocapra americana -n02430045 deer, cervid -n02430559 stag -n02430643 royal, royal stag -n02430748 pricket -n02430830 fawn -n02431122 red deer, elk, American elk, wapiti, Cervus elaphus -n02431337 hart, stag -n02431441 hind -n02431542 brocket -n02431628 sambar, sambur, Cervus unicolor -n02431785 wapiti, elk, American elk, Cervus elaphus canadensis -n02431976 Japanese deer, sika, Cervus nipon, Cervus sika -n02432291 Virginia deer, white tail, whitetail, white-tailed deer, whitetail deer, Odocoileus Virginianus -n02432511 mule deer, burro deer, Odocoileus hemionus -n02432704 black-tailed deer, blacktail deer, blacktail, Odocoileus hemionus columbianus -n02432983 elk, European elk, moose, Alces alces -n02433318 fallow deer, Dama dama -n02433546 roe deer, Capreolus capreolus -n02433729 roebuck -n02433925 caribou, reindeer, Greenland caribou, Rangifer tarandus -n02434190 woodland caribou, Rangifer caribou -n02434415 barren ground caribou, Rangifer arcticus -n02434712 brocket -n02434954 muntjac, barking deer -n02435216 musk deer, Moschus moschiferus -n02435517 pere david's deer, elaphure, Elaphurus davidianus -n02435853 chevrotain, mouse deer -n02436224 kanchil, Tragulus kanchil -n02436353 napu, Tragulus Javanicus -n02436645 water chevrotain, water deer, Hyemoschus aquaticus -n02437136 camel -n02437312 Arabian camel, dromedary, Camelus dromedarius -n02437482 Bactrian camel, Camelus bactrianus -n02437616 llama -n02437971 domestic llama, Lama peruana -n02438173 guanaco, Lama guanicoe -n02438272 alpaca, Lama pacos -n02438580 vicuna, Vicugna vicugna -n02439033 giraffe, camelopard, Giraffa camelopardalis -n02439398 okapi, Okapia johnstoni -n02441326 musteline mammal, mustelid, musteline -n02441942 weasel -n02442172 ermine, shorttail weasel, Mustela erminea -n02442336 stoat -n02442446 New World least weasel, Mustela rixosa -n02442572 Old World least weasel, Mustela nivalis -n02442668 longtail weasel, long-tailed weasel, Mustela frenata -n02442845 mink -n02443015 American mink, Mustela vison -n02443114 polecat, fitch, foulmart, foumart, Mustela putorius -n02443346 ferret -n02443484 black-footed ferret, ferret, Mustela nigripes -n02443808 muishond -n02443959 snake muishond, Poecilogale albinucha -n02444251 striped muishond, Ictonyx striata -n02444819 otter -n02445004 river otter, Lutra canadensis -n02445171 Eurasian otter, Lutra lutra -n02445394 sea otter, Enhydra lutris -n02445715 skunk, polecat, wood pussy -n02446206 striped skunk, Mephitis mephitis -n02446352 hooded skunk, Mephitis macroura -n02446645 hog-nosed skunk, hognosed skunk, badger skunk, rooter skunk, Conepatus leuconotus -n02447021 spotted skunk, little spotted skunk, Spilogale putorius -n02447366 badger -n02447762 American badger, Taxidea taxus -n02448060 Eurasian badger, Meles meles -n02448318 ratel, honey badger, Mellivora capensis -n02448633 ferret badger -n02448885 hog badger, hog-nosed badger, sand badger, Arctonyx collaris -n02449183 wolverine, carcajou, skunk bear, Gulo luscus -n02449350 glutton, Gulo gulo, wolverine -n02449699 grison, Grison vittatus, Galictis vittatus -n02450034 marten, marten cat -n02450295 pine marten, Martes martes -n02450426 sable, Martes zibellina -n02450561 American marten, American sable, Martes americana -n02450677 stone marten, beech marten, Martes foina -n02450829 fisher, pekan, fisher cat, black cat, Martes pennanti -n02451125 yellow-throated marten, Charronia flavigula -n02451415 tayra, taira, Eira barbara -n02451575 fictional animal -n02453108 pachyderm -n02453611 edentate -n02454379 armadillo -n02454794 peba, nine-banded armadillo, Texas armadillo, Dasypus novemcinctus -n02455135 apar, three-banded armadillo, Tolypeutes tricinctus -n02455428 tatouay, cabassous, Cabassous unicinctus -n02455720 peludo, poyou, Euphractus sexcinctus -n02456008 giant armadillo, tatou, tatu, Priodontes giganteus -n02456275 pichiciago, pichiciego, fairy armadillo, chlamyphore, Chlamyphorus truncatus -n02456962 sloth, tree sloth -n02457408 three-toed sloth, ai, Bradypus tridactylus -n02457945 two-toed sloth, unau, unai, Choloepus didactylus -n02458135 two-toed sloth, unau, unai, Choloepus hoffmanni -n02458517 megatherian, megatheriid, megatherian mammal -n02459190 mylodontid -n02460009 anteater, New World anteater -n02460451 ant bear, giant anteater, great anteater, tamanoir, Myrmecophaga jubata -n02460817 silky anteater, two-toed anteater, Cyclopes didactylus -n02461128 tamandua, tamandu, lesser anteater, Tamandua tetradactyla -n02461830 pangolin, scaly anteater, anteater -n02462213 coronet -n02469248 scapular -n02469472 tadpole, polliwog, pollywog -n02469914 primate -n02470238 simian -n02470325 ape -n02470709 anthropoid -n02470899 anthropoid ape -n02471300 hominoid -n02471762 hominid -n02472293 homo, man, human being, human -n02472987 world, human race, humanity, humankind, human beings, humans, mankind, man -n02473307 Homo erectus -n02473554 Pithecanthropus, Pithecanthropus erectus, genus Pithecanthropus -n02473720 Java man, Trinil man -n02473857 Peking man -n02473983 Sinanthropus, genus Sinanthropus -n02474110 Homo soloensis -n02474282 Javanthropus, genus Javanthropus -n02474605 Homo habilis -n02474777 Homo sapiens -n02475078 Neandertal man, Neanderthal man, Neandertal, Neanderthal, Homo sapiens neanderthalensis -n02475358 Cro-magnon -n02475669 Homo sapiens sapiens, modern man -n02476219 australopithecine -n02476567 Australopithecus afarensis -n02476870 Australopithecus africanus -n02477028 Australopithecus boisei -n02477187 Zinjanthropus, genus Zinjanthropus -n02477329 Australopithecus robustus -n02477516 Paranthropus, genus Paranthropus -n02477782 Sivapithecus -n02478239 rudapithecus, Dryopithecus Rudapithecus hungaricus -n02478875 proconsul -n02479332 Aegyptopithecus -n02480153 great ape, pongid -n02480495 orangutan, orang, orangutang, Pongo pygmaeus -n02480855 gorilla, Gorilla gorilla -n02481103 western lowland gorilla, Gorilla gorilla gorilla -n02481235 eastern lowland gorilla, Gorilla gorilla grauri -n02481366 mountain gorilla, Gorilla gorilla beringei -n02481500 silverback -n02481823 chimpanzee, chimp, Pan troglodytes -n02482060 western chimpanzee, Pan troglodytes verus -n02482286 eastern chimpanzee, Pan troglodytes schweinfurthii -n02482474 central chimpanzee, Pan troglodytes troglodytes -n02482650 pygmy chimpanzee, bonobo, Pan paniscus -n02483092 lesser ape -n02483362 gibbon, Hylobates lar -n02483708 siamang, Hylobates syndactylus, Symphalangus syndactylus -n02484322 monkey -n02484473 Old World monkey, catarrhine -n02484975 guenon, guenon monkey -n02485225 talapoin, Cercopithecus talapoin -n02485371 grivet, Cercopithecus aethiops -n02485536 vervet, vervet monkey, Cercopithecus aethiops pygerythrus -n02485688 green monkey, African green monkey, Cercopithecus aethiops sabaeus -n02485988 mangabey -n02486261 patas, hussar monkey, Erythrocebus patas -n02486410 baboon -n02486657 chacma, chacma baboon, Papio ursinus -n02486908 mandrill, Mandrillus sphinx -n02487079 drill, Mandrillus leucophaeus -n02487347 macaque -n02487547 rhesus, rhesus monkey, Macaca mulatta -n02487675 bonnet macaque, bonnet monkey, capped macaque, crown monkey, Macaca radiata -n02487847 Barbary ape, Macaca sylvana -n02488003 crab-eating macaque, croo monkey, Macaca irus -n02488291 langur -n02488415 entellus, hanuman, Presbytes entellus, Semnopithecus entellus -n02488702 colobus, colobus monkey -n02488894 guereza, Colobus guereza -n02489166 proboscis monkey, Nasalis larvatus -n02489589 New World monkey, platyrrhine, platyrrhinian -n02490219 marmoset -n02490597 true marmoset -n02490811 pygmy marmoset, Cebuella pygmaea -n02491107 tamarin, lion monkey, lion marmoset, leoncita -n02491329 silky tamarin, Leontocebus rosalia -n02491474 pinche, Leontocebus oedipus -n02492035 capuchin, ringtail, Cebus capucinus -n02492356 douroucouli, Aotus trivirgatus -n02492660 howler monkey, howler -n02492948 saki -n02493224 uakari -n02493509 titi, titi monkey -n02493793 spider monkey, Ateles geoffroyi -n02494079 squirrel monkey, Saimiri sciureus -n02494383 woolly monkey -n02495242 tree shrew -n02496052 prosimian -n02496913 lemur -n02497673 Madagascar cat, ring-tailed lemur, Lemur catta -n02498153 aye-aye, Daubentonia madagascariensis -n02498743 slender loris, Loris gracilis -n02499022 slow loris, Nycticebus tardigradua, Nycticebus pygmaeus -n02499316 potto, kinkajou, Perodicticus potto -n02499568 angwantibo, golden potto, Arctocebus calabarensis -n02499808 galago, bushbaby, bush baby -n02500267 indri, indris, Indri indri, Indri brevicaudatus -n02500596 woolly indris, Avahi laniger -n02501583 tarsier -n02501923 Tarsius syrichta -n02502006 Tarsius glis -n02502514 flying lemur, flying cat, colugo -n02502807 Cynocephalus variegatus -n02503127 proboscidean, proboscidian -n02503517 elephant -n02503756 rogue elephant -n02504013 Indian elephant, Elephas maximus -n02504458 African elephant, Loxodonta africana -n02504770 mammoth -n02505063 woolly mammoth, northern mammoth, Mammuthus primigenius -n02505238 columbian mammoth, Mammuthus columbi -n02505485 imperial mammoth, imperial elephant, Archidiskidon imperator -n02505998 mastodon, mastodont -n02506947 plantigrade mammal, plantigrade -n02507148 digitigrade mammal, digitigrade -n02507649 procyonid -n02508021 raccoon, racoon -n02508213 common raccoon, common racoon, coon, ringtail, Procyon lotor -n02508346 crab-eating raccoon, Procyon cancrivorus -n02508742 bassarisk, cacomistle, cacomixle, coon cat, raccoon fox, ringtail, ring-tailed cat, civet cat, miner's cat, Bassariscus astutus -n02509197 kinkajou, honey bear, potto, Potos flavus, Potos caudivolvulus -n02509515 coati, coati-mondi, coati-mundi, coon cat, Nasua narica -n02509815 lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens -n02510455 giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca -n02511730 twitterer -n02512053 fish -n02512752 fingerling -n02512830 game fish, sport fish -n02512938 food fish -n02513248 rough fish -n02513355 groundfish, bottom fish -n02513560 young fish -n02513727 parr -n02513805 mouthbreeder -n02513939 spawner -n02514041 barracouta, snoek -n02515214 crossopterygian, lobefin, lobe-finned fish -n02515713 coelacanth, Latimeria chalumnae -n02516188 lungfish -n02516776 ceratodus -n02517442 catfish, siluriform fish -n02517938 silurid, silurid fish -n02518324 European catfish, sheatfish, Silurus glanis -n02518622 electric catfish, Malopterurus electricus -n02519148 bullhead, bullhead catfish -n02519340 horned pout, hornpout, pout, Ameiurus Melas -n02519472 brown bullhead -n02519686 channel catfish, channel cat, Ictalurus punctatus -n02519862 blue catfish, blue cat, blue channel catfish, blue channel cat -n02520147 flathead catfish, mudcat, goujon, shovelnose catfish, spoonbill catfish, Pylodictus olivaris -n02520525 armored catfish -n02520810 sea catfish -n02521646 gadoid, gadoid fish -n02522399 cod, codfish -n02522637 codling -n02522722 Atlantic cod, Gadus morhua -n02522866 Pacific cod, Alaska cod, Gadus macrocephalus -n02523110 whiting, Merlangus merlangus, Gadus merlangus -n02523427 burbot, eelpout, ling, cusk, Lota lota -n02523877 haddock, Melanogrammus aeglefinus -n02524202 pollack, pollock, Pollachius pollachius -n02524524 hake -n02524659 silver hake, Merluccius bilinearis, whiting -n02524928 ling -n02525382 cusk, torsk, Brosme brosme -n02525703 grenadier, rattail, rattail fish -n02526121 eel -n02526425 elver -n02526818 common eel, freshwater eel -n02527057 tuna, Anguilla sucklandii -n02527271 moray, moray eel -n02527622 conger, conger eel -n02528163 teleost fish, teleost, teleostan -n02529293 beaked salmon, sandfish, Gonorhynchus gonorhynchus -n02529772 clupeid fish, clupeid -n02530052 whitebait -n02530188 brit, britt -n02530421 shad -n02530637 common American shad, Alosa sapidissima -n02530831 river shad, Alosa chrysocloris -n02530999 allice shad, allis shad, allice, allis, Alosa alosa -n02531114 alewife, Alosa pseudoharengus, Pomolobus pseudoharengus -n02531625 menhaden, Brevoortia tyrannis -n02532028 herring, Clupea harangus -n02532272 Atlantic herring, Clupea harengus harengus -n02532451 Pacific herring, Clupea harengus pallasii -n02532602 sardine -n02532786 sild -n02532918 brisling, sprat, Clupea sprattus -n02533209 pilchard, sardine, Sardina pilchardus -n02533545 Pacific sardine, Sardinops caerulea -n02533834 anchovy -n02534165 mediterranean anchovy, Engraulis encrasicholus -n02534559 salmonid -n02534734 salmon -n02535080 parr -n02535163 blackfish -n02535258 redfish -n02535537 Atlantic salmon, Salmo salar -n02535759 landlocked salmon, lake salmon -n02536165 sockeye, sockeye salmon, red salmon, blueback salmon, Oncorhynchus nerka -n02536456 chinook, chinook salmon, king salmon, quinnat salmon, Oncorhynchus tshawytscha -n02536864 coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch -n02537085 trout -n02537319 brown trout, salmon trout, Salmo trutta -n02537525 rainbow trout, Salmo gairdneri -n02537716 sea trout -n02538010 lake trout, salmon trout, Salvelinus namaycush -n02538216 brook trout, speckled trout, Salvelinus fontinalis -n02538406 char, charr -n02538562 Arctic char, Salvelinus alpinus -n02538985 whitefish -n02539424 lake whitefish, Coregonus clupeaformis -n02539573 cisco, lake herring, Coregonus artedi -n02539894 round whitefish, Menominee whitefish, Prosopium cylindraceum -n02540412 smelt -n02540983 sparling, European smelt, Osmerus eperlanus -n02541257 capelin, capelan, caplin -n02541687 tarpon, Tarpon atlanticus -n02542017 ladyfish, tenpounder, Elops saurus -n02542432 bonefish, Albula vulpes -n02542958 argentine -n02543255 lanternfish -n02543565 lizardfish, snakefish, snake-fish -n02544274 lancetfish, lancet fish, wolffish -n02545841 opah, moonfish, Lampris regius -n02546028 New World opah, Lampris guttatus -n02546331 ribbonfish -n02546627 dealfish, Trachipterus arcticus -n02547014 oarfish, king of the herring, ribbonfish, Regalecus glesne -n02547733 batfish -n02548247 goosefish, angler, anglerfish, angler fish, monkfish, lotte, allmouth, Lophius Americanus -n02548689 toadfish, Opsanus tau -n02548884 oyster fish, oyster-fish, oysterfish -n02549248 frogfish -n02549376 sargassum fish -n02549989 needlefish, gar, billfish -n02550203 timucu -n02550460 flying fish -n02550655 monoplane flying fish, two-wing flying fish -n02551134 halfbeak -n02551668 saury, billfish, Scomberesox saurus -n02552171 spiny-finned fish, acanthopterygian -n02553028 lingcod, Ophiodon elongatus -n02554730 percoid fish, percoid, percoidean -n02555863 perch -n02556373 climbing perch, Anabas testudineus, A. testudineus -n02556846 perch -n02557182 yellow perch, Perca flavescens -n02557318 European perch, Perca fluviatilis -n02557591 pike-perch, pike perch -n02557749 walleye, walleyed pike, jack salmon, dory, Stizostedion vitreum -n02557909 blue pike, blue pickerel, blue pikeperch, blue walleye, Strizostedion vitreum glaucum -n02558206 snail darter, Percina tanasi -n02558860 cusk-eel -n02559144 brotula -n02559383 pearlfish, pearl-fish -n02559862 robalo -n02560110 snook -n02561108 pike -n02561381 northern pike, Esox lucius -n02561514 muskellunge, Esox masquinongy -n02561661 pickerel -n02561803 chain pickerel, chain pike, Esox niger -n02561937 redfin pickerel, barred pickerel, Esox americanus -n02562315 sunfish, centrarchid -n02562796 crappie -n02562971 black crappie, Pomoxis nigromaculatus -n02563079 white crappie, Pomoxis annularis -n02563182 freshwater bream, bream -n02563648 pumpkinseed, Lepomis gibbosus -n02563792 bluegill, Lepomis macrochirus -n02563949 spotted sunfish, stumpknocker, Lepomis punctatus -n02564270 freshwater bass -n02564403 rock bass, rock sunfish, Ambloplites rupestris -n02564720 black bass -n02564935 Kentucky black bass, spotted black bass, Micropterus pseudoplites -n02565072 smallmouth, smallmouth bass, smallmouthed bass, smallmouth black bass, smallmouthed black bass, Micropterus dolomieu -n02565324 largemouth, largemouth bass, largemouthed bass, largemouth black bass, largemouthed black bass, Micropterus salmoides -n02565573 bass -n02566109 serranid fish, serranid -n02566489 white perch, silver perch, Morone americana -n02566665 yellow bass, Morone interrupta -n02567334 blackmouth bass, Synagrops bellus -n02567633 rock sea bass, rock bass, Centropristis philadelphica -n02568087 striped bass, striper, Roccus saxatilis, rockfish -n02568447 stone bass, wreckfish, Polyprion americanus -n02568959 grouper -n02569484 hind -n02569631 rock hind, Epinephelus adscensionis -n02569905 creole-fish, Paranthias furcifer -n02570164 jewfish, Mycteroperca bonaci -n02570484 soapfish -n02570838 surfperch, surffish, surf fish -n02571167 rainbow seaperch, rainbow perch, Hipsurus caryi -n02571652 bigeye -n02571810 catalufa, Priacanthus arenatus -n02572196 cardinalfish -n02572484 flame fish, flamefish, Apogon maculatus -n02573249 tilefish, Lopholatilus chamaeleonticeps -n02573704 bluefish, Pomatomus saltatrix -n02574271 cobia, Rachycentron canadum, sergeant fish -n02574910 remora, suckerfish, sucking fish -n02575325 sharksucker, Echeneis naucrates -n02575590 whale sucker, whalesucker, Remilegia australis -n02576223 carangid fish, carangid -n02576575 jack -n02576906 crevalle jack, jack crevalle, Caranx hippos -n02577041 yellow jack, Caranx bartholomaei -n02577164 runner, blue runner, Caranx crysos -n02577403 rainbow runner, Elagatis bipinnulata -n02577662 leatherjacket, leatherjack -n02577952 threadfish, thread-fish, Alectis ciliaris -n02578233 moonfish, Atlantic moonfish, horsefish, horsehead, horse-head, dollarfish, Selene setapinnis -n02578454 lookdown, lookdown fish, Selene vomer -n02578771 amberjack, amberfish -n02578928 yellowtail, Seriola dorsalis -n02579303 kingfish, Seriola grandis -n02579557 pompano -n02579762 Florida pompano, Trachinotus carolinus -n02579928 permit, Trachinotus falcatus -n02580336 scad -n02580679 horse mackerel, jack mackerel, Spanish mackerel, saurel, Trachurus symmetricus -n02580830 horse mackerel, saurel, Trachurus trachurus -n02581108 bigeye scad, big-eyed scad, goggle-eye, Selar crumenophthalmus -n02581482 mackerel scad, mackerel shad, Decapterus macarellus -n02581642 round scad, cigarfish, quiaquia, Decapterus punctatus -n02581957 dolphinfish, dolphin, mahimahi -n02582220 Coryphaena hippurus -n02582349 Coryphaena equisetis -n02582721 pomfret, Brama raii -n02583567 characin, characin fish, characid -n02583890 tetra -n02584145 cardinal tetra, Paracheirodon axelrodi -n02584449 piranha, pirana, caribe -n02585872 cichlid, cichlid fish -n02586238 bolti, Tilapia nilotica -n02586543 snapper -n02587051 red snapper, Lutjanus blackfordi -n02587300 grey snapper, gray snapper, mangrove snapper, Lutjanus griseus -n02587479 mutton snapper, muttonfish, Lutjanus analis -n02587618 schoolmaster, Lutjanus apodus -n02587877 yellowtail, yellowtail snapper, Ocyurus chrysurus -n02588286 grunt -n02588794 margate, Haemulon album -n02588945 Spanish grunt, Haemulon macrostomum -n02589062 tomtate, Haemulon aurolineatum -n02589196 cottonwick, Haemulon malanurum -n02589316 sailor's-choice, sailors choice, Haemulon parra -n02589623 porkfish, pork-fish, Anisotremus virginicus -n02589796 pompon, black margate, Anisotremus surinamensis -n02590094 pigfish, hogfish, Orthopristis chrysopterus -n02590495 sparid, sparid fish -n02590702 sea bream, bream -n02590987 porgy -n02591330 red porgy, Pagrus pagrus -n02591613 European sea bream, Pagellus centrodontus -n02591911 Atlantic sea bream, Archosargus rhomboidalis -n02592055 sheepshead, Archosargus probatocephalus -n02592371 pinfish, sailor's-choice, squirrelfish, Lagodon rhomboides -n02592734 sheepshead porgy, Calamus penna -n02593019 snapper, Chrysophrys auratus -n02593191 black bream, Chrysophrys australis -n02593453 scup, northern porgy, northern scup, Stenotomus chrysops -n02593679 scup, southern porgy, southern scup, Stenotomus aculeatus -n02594250 sciaenid fish, sciaenid -n02594942 striped drum, Equetus pulcher -n02595056 jackknife-fish, Equetus lanceolatus -n02595339 silver perch, mademoiselle, Bairdiella chrysoura -n02595702 red drum, channel bass, redfish, Sciaenops ocellatus -n02596067 mulloway, jewfish, Sciaena antarctica -n02596252 maigre, maiger, Sciaena aquila -n02596381 croaker -n02596720 Atlantic croaker, Micropogonias undulatus -n02597004 yellowfin croaker, surffish, surf fish, Umbrina roncador -n02597367 whiting -n02597608 kingfish -n02597818 king whiting, Menticirrhus americanus -n02597972 northern whiting, Menticirrhus saxatilis -n02598134 corbina, Menticirrhus undulatus -n02598573 white croaker, chenfish, kingfish, Genyonemus lineatus -n02598878 white croaker, queenfish, Seriphus politus -n02599052 sea trout -n02599347 weakfish, Cynoscion regalis -n02599557 spotted weakfish, spotted sea trout, spotted squeateague, Cynoscion nebulosus -n02599958 mullet -n02600298 goatfish, red mullet, surmullet, Mullus surmuletus -n02600503 red goatfish, Mullus auratus -n02600798 yellow goatfish, Mulloidichthys martinicus -n02601344 mullet, grey mullet, gray mullet -n02601767 striped mullet, Mugil cephalus -n02601921 white mullet, Mugil curema -n02602059 liza, Mugil liza -n02602405 silversides, silverside -n02602760 jacksmelt, Atherinopsis californiensis -n02603317 barracuda -n02603540 great barracuda, Sphyraena barracuda -n02603862 sweeper -n02604157 sea chub -n02604480 Bermuda chub, rudderfish, Kyphosus sectatrix -n02604954 spadefish, angelfish, Chaetodipterus faber -n02605316 butterfly fish -n02605703 chaetodon -n02605936 angelfish -n02606052 rock beauty, Holocanthus tricolor -n02606384 damselfish, demoiselle -n02606751 beaugregory, Pomacentrus leucostictus -n02607072 anemone fish -n02607201 clown anemone fish, Amphiprion percula -n02607470 sergeant major, Abudefduf saxatilis -n02607862 wrasse -n02608284 pigfish, giant pigfish, Achoerodus gouldii -n02608547 hogfish, hog snapper, Lachnolaimus maximus -n02608860 slippery dick, Halicoeres bivittatus -n02608996 puddingwife, pudding-wife, Halicoeres radiatus -n02609302 bluehead, Thalassoma bifasciatum -n02609823 pearly razorfish, Hemipteronatus novacula -n02610066 tautog, blackfish, Tautoga onitis -n02610373 cunner, bergall, Tautogolabrus adspersus -n02610664 parrotfish, polly fish, pollyfish -n02610980 threadfin -n02611561 jawfish -n02611898 stargazer -n02612167 sand stargazer -n02613181 blenny, combtooth blenny -n02613572 shanny, Blennius pholis -n02613820 Molly Miller, Scartella cristata -n02614140 clinid, clinid fish -n02614482 pikeblenny -n02614653 bluethroat pikeblenny, Chaenopsis ocellata -n02614978 gunnel, bracketed blenny -n02615298 rock gunnel, butterfish, Pholis gunnellus -n02616128 eelblenny -n02616397 wrymouth, ghostfish, Cryptacanthodes maculatus -n02616851 wolffish, wolf fish, catfish -n02617537 viviparous eelpout, Zoarces viviparus -n02618094 ocean pout, Macrozoarces americanus -n02618513 sand lance, sand launce, sand eel, launce -n02618827 dragonet -n02619165 goby, gudgeon -n02619550 mudskipper, mudspringer -n02619861 sleeper, sleeper goby -n02620167 flathead -n02620578 archerfish, Toxotes jaculatrix -n02621258 surgeonfish -n02621908 gempylid -n02622249 snake mackerel, Gempylus serpens -n02622547 escolar, Lepidocybium flavobrunneum -n02622712 oilfish, Ruvettus pretiosus -n02622955 cutlassfish, frost fish, hairtail -n02623445 scombroid, scombroid fish -n02624167 mackerel -n02624551 common mackerel, shiner, Scomber scombrus -n02624807 Spanish mackerel, Scomber colias -n02624987 chub mackerel, tinker, Scomber japonicus -n02625258 wahoo, Acanthocybium solandri -n02625612 Spanish mackerel -n02625851 king mackerel, cavalla, cero, Scomberomorus cavalla -n02626089 Scomberomorus maculatus -n02626265 cero, pintado, kingfish, Scomberomorus regalis -n02626471 sierra, Scomberomorus sierra -n02626762 tuna, tunny -n02627037 albacore, long-fin tunny, Thunnus alalunga -n02627292 bluefin, bluefin tuna, horse mackerel, Thunnus thynnus -n02627532 yellowfin, yellowfin tuna, Thunnus albacares -n02627835 bonito -n02628062 skipjack, Atlantic bonito, Sarda sarda -n02628259 Chile bonito, Chilean bonito, Pacific bonito, Sarda chiliensis -n02628600 skipjack, skipjack tuna, Euthynnus pelamis -n02629230 bonito, oceanic bonito, Katsuwonus pelamis -n02629716 swordfish, Xiphias gladius -n02630281 sailfish -n02630615 Atlantic sailfish, Istiophorus albicans -n02630739 billfish -n02631041 marlin -n02631330 blue marlin, Makaira nigricans -n02631475 black marlin, Makaira mazara, Makaira marlina -n02631628 striped marlin, Makaira mitsukurii -n02631775 white marlin, Makaira albida -n02632039 spearfish -n02632494 louvar, Luvarus imperialis -n02633422 dollarfish, Poronotus triacanthus -n02633677 palometa, California pompano, Palometa simillima -n02633977 harvestfish, Paprilus alepidotus -n02634545 driftfish -n02635154 barrelfish, black rudderfish, Hyperglyphe perciformis -n02635580 clingfish -n02636170 tripletail -n02636405 Atlantic tripletail, Lobotes surinamensis -n02636550 Pacific tripletail, Lobotes pacificus -n02636854 mojarra -n02637179 yellowfin mojarra, Gerres cinereus -n02637475 silver jenny, Eucinostomus gula -n02637977 whiting -n02638596 ganoid, ganoid fish -n02639087 bowfin, grindle, dogfish, Amia calva -n02639605 paddlefish, duckbill, Polyodon spathula -n02639922 Chinese paddlefish, Psephurus gladis -n02640242 sturgeon -n02640626 Pacific sturgeon, white sturgeon, Sacramento sturgeon, Acipenser transmontanus -n02640857 beluga, hausen, white sturgeon, Acipenser huso -n02641379 gar, garfish, garpike, billfish, Lepisosteus osseus -n02642107 scorpaenoid, scorpaenoid fish -n02642644 scorpaenid, scorpaenid fish -n02643112 scorpionfish, scorpion fish, sea scorpion -n02643316 plumed scorpionfish, Scorpaena grandicornis -n02643566 lionfish -n02643836 stonefish, Synanceja verrucosa -n02644113 rockfish -n02644360 copper rockfish, Sebastodes caurinus -n02644501 vermillion rockfish, rasher, Sebastodes miniatus -n02644665 red rockfish, Sebastodes ruberrimus -n02644817 rosefish, ocean perch, Sebastodes marinus -n02645538 bullhead -n02645691 miller's-thumb -n02645953 sea raven, Hemitripterus americanus -n02646667 lumpfish, Cyclopterus lumpus -n02646892 lumpsucker -n02648035 pogge, armed bullhead, Agonus cataphractus -n02648625 greenling -n02648916 kelp greenling, Hexagrammos decagrammus -n02649218 painted greenling, convict fish, convictfish, Oxylebius pictus -n02649546 flathead -n02650050 gurnard -n02650413 tub gurnard, yellow gurnard, Trigla lucerna -n02650541 sea robin, searobin -n02651060 northern sea robin, Prionotus carolinus -n02652132 flying gurnard, flying robin, butterflyfish -n02652668 plectognath, plectognath fish -n02653145 triggerfish -n02653497 queen triggerfish, Bessy cerca, oldwench, oldwife, Balistes vetula -n02653786 filefish -n02654112 leatherjacket, leatherfish -n02654425 boxfish, trunkfish -n02654745 cowfish, Lactophrys quadricornis -n02655020 puffer, pufferfish, blowfish, globefish -n02655523 spiny puffer -n02655848 porcupinefish, porcupine fish, Diodon hystrix -n02656032 balloonfish, Diodon holocanthus -n02656301 burrfish -n02656670 ocean sunfish, sunfish, mola, headfish -n02656969 sharptail mola, Mola lanceolata -n02657368 flatfish -n02657694 flounder -n02658079 righteye flounder, righteyed flounder -n02658531 plaice, Pleuronectes platessa -n02658811 European flatfish, Platichthys flesus -n02659176 yellowtail flounder, Limanda ferruginea -n02659478 winter flounder, blackback flounder, lemon sole, Pseudopleuronectes americanus -n02659808 lemon sole, Microstomus kitt -n02660091 American plaice, Hippoglossoides platessoides -n02660208 halibut, holibut -n02660519 Atlantic halibut, Hippoglossus hippoglossus -n02660640 Pacific halibut, Hippoglossus stenolepsis -n02661017 lefteye flounder, lefteyed flounder -n02661473 southern flounder, Paralichthys lethostigmus -n02661618 summer flounder, Paralichthys dentatus -n02662239 whiff -n02662397 horned whiff, Citharichthys cornutus -n02662559 sand dab -n02662825 windowpane, Scophthalmus aquosus -n02662993 brill, Scophthalmus rhombus -n02663211 turbot, Psetta maxima -n02663485 tonguefish, tongue-fish -n02663849 sole -n02664285 European sole, Solea solea -n02664642 English sole, lemon sole, Parophrys vitulus -n02665250 hogchoker, Trinectes maculatus -n02665985 aba -n02666196 abacus -n02666501 abandoned ship, derelict -n02666624 A battery -n02666943 abattoir, butchery, shambles, slaughterhouse -n02667093 abaya -n02667244 Abbe condenser -n02667379 abbey -n02667478 abbey -n02667576 abbey -n02667693 Abney level -n02668393 abrader, abradant -n02668613 abrading stone -n02669295 abutment -n02669442 abutment arch -n02669534 academic costume -n02669723 academic gown, academic robe, judge's robe -n02670186 accelerator, throttle, throttle valve -n02670382 accelerator, particle accelerator, atom smasher -n02670683 accelerator, accelerator pedal, gas pedal, gas, throttle, gun -n02670935 accelerometer -n02671780 accessory, accoutrement, accouterment -n02672152 accommodating lens implant, accommodating IOL -n02672371 accommodation -n02672831 accordion, piano accordion, squeeze box -n02675077 acetate disk, phonograph recording disk -n02675219 acetate rayon, acetate -n02675522 achromatic lens -n02676097 acoustic delay line, sonic delay line -n02676261 acoustic device -n02676566 acoustic guitar -n02676670 acoustic modem -n02676938 acropolis -n02677028 acrylic -n02677136 acrylic, acrylic paint -n02677436 actinometer -n02677718 action, action mechanism -n02678010 active matrix screen -n02678384 actuator -n02678897 adapter, adaptor -n02679142 adder -n02679257 adding machine, totalizer, totaliser -n02679961 addressing machine, Addressograph -n02680110 adhesive bandage -n02680512 adit -n02680638 adjoining room -n02680754 adjustable wrench, adjustable spanner -n02681392 adobe, adobe brick -n02682311 adz, adze -n02682407 aeolian harp, aeolian lyre, wind harp -n02682569 aerator -n02682811 aerial torpedo -n02682922 aerosol, aerosol container, aerosol can, aerosol bomb, spray can -n02683183 Aertex -n02683323 afghan -n02683454 Afro-wig -n02683558 afterburner -n02683791 after-shave, after-shave lotion -n02684248 agateware -n02684356 agglomerator -n02684515 aglet, aiglet, aiguilette -n02684649 aglet, aiglet -n02684962 agora, public square -n02685082 aigrette, aigret -n02685253 aileron -n02685365 air bag -n02685701 airbrake -n02685995 airbrush -n02686121 airbus -n02686227 air compressor -n02686379 air conditioner, air conditioning -n02686568 aircraft -n02687172 aircraft carrier, carrier, flattop, attack aircraft carrier -n02687423 aircraft engine -n02687682 air cushion, air spring -n02687821 airdock, hangar, repair shed -n02687992 airfield, landing field, flying field, field -n02688273 air filter, air cleaner -n02688443 airfoil, aerofoil, control surface, surface -n02689144 airframe -n02689274 air gun, airgun, air rifle -n02689434 air hammer, jackhammer, pneumatic hammer -n02689748 air horn -n02689819 airing cupboard -n02690373 airliner -n02690715 airmailer -n02691156 airplane, aeroplane, plane -n02692086 airplane propeller, airscrew, prop -n02692232 airport, airdrome, aerodrome, drome -n02692513 air pump, vacuum pump -n02692680 air search radar -n02692877 airship, dirigible -n02693246 air terminal, airport terminal -n02693413 air-to-air missile -n02693540 air-to-ground missile, air-to-surface missile -n02694045 aisle -n02694279 Aladdin's lamp -n02694426 alarm, warning device, alarm system -n02694662 alarm clock, alarm -n02694966 alb -n02695627 alcazar -n02695762 alcohol thermometer, alcohol-in-glass thermometer -n02696165 alehouse -n02696246 alembic -n02696569 algometer -n02696843 alidade, alidad -n02697022 alidade, alidad -n02697221 A-line -n02697576 Allen screw -n02697675 Allen wrench -n02697876 alligator wrench -n02698244 alms dish, alms tray -n02698473 alpaca -n02698634 alpenstock -n02699494 altar -n02699629 altar, communion table, Lord's table -n02699770 altarpiece, reredos -n02699915 altazimuth -n02700064 alternator -n02700258 altimeter -n02700895 Amati -n02701002 ambulance -n02701260 amen corner -n02701730 American organ -n02702989 ammeter -n02703124 ammonia clock -n02703275 ammunition, ammo -n02704645 amphibian, amphibious aircraft -n02704792 amphibian, amphibious vehicle -n02704949 amphitheater, amphitheatre, coliseum -n02705201 amphitheater, amphitheatre -n02705429 amphora -n02705944 amplifier -n02706221 ampulla -n02706806 amusement arcade -n02708093 analog clock -n02708224 analog computer, analogue computer -n02708433 analog watch -n02708555 analytical balance, chemical balance -n02708711 analyzer, analyser -n02708885 anamorphosis, anamorphism -n02709101 anastigmat -n02709367 anchor, ground tackle -n02709637 anchor chain, anchor rope -n02709763 anchor light, riding light, riding lamp -n02709908 AND circuit, AND gate -n02710044 andiron, firedog, dog, dog-iron -n02710201 android, humanoid, mechanical man -n02710324 anechoic chamber -n02710429 anemometer, wind gauge, wind gage -n02710600 aneroid barometer, aneroid -n02711237 angiocardiogram -n02711780 angioscope -n02712545 angle bracket, angle iron -n02712643 angledozer -n02713003 ankle brace -n02713218 anklet, anklets, bobbysock, bobbysocks -n02713364 anklet -n02713496 ankus -n02714315 anode -n02714535 anode -n02714751 answering machine -n02715229 antenna, aerial, transmitting aerial -n02715513 anteroom, antechamber, entrance hall, hall, foyer, lobby, vestibule -n02715712 antiaircraft, antiaircraft gun, flak, flack, pom-pom, ack-ack, ack-ack gun -n02716626 antiballistic missile, ABM -n02720048 antifouling paint -n02720576 anti-G suit, G suit -n02721813 antimacassar -n02723165 antiperspirant -n02724722 anti-submarine rocket -n02725872 anvil -n02726017 ao dai -n02726210 apadana -n02726305 apartment, flat -n02726681 apartment building, apartment house -n02727016 aperture -n02727141 aperture -n02727426 apiary, bee house -n02727825 apparatus, setup -n02728440 apparel, wearing apparel, dress, clothes -n02729222 applecart -n02729837 appliance -n02729965 appliance, contraption, contrivance, convenience, gadget, gizmo, gismo, widget -n02730265 applicator, applier -n02730568 appointment, fitting -n02730930 apron -n02731251 apron string -n02731398 apse, apsis -n02731629 aqualung, Aqua-Lung, scuba -n02731900 aquaplane -n02732072 aquarium, fish tank, marine museum -n02732572 arabesque -n02732827 arbor, arbour, bower, pergola -n02733213 arcade, colonnade -n02733524 arch -n02734725 architecture -n02734835 architrave -n02735268 arch support -n02735361 arc lamp, arc light -n02735538 arctic, galosh, golosh, rubber, gumshoe -n02735688 area -n02736396 areaway -n02736798 argyle, argyll -n02737351 ark -n02737660 arm -n02738031 armament -n02738271 armature -n02738449 armband -n02738535 armchair -n02738741 armet -n02738859 arm guard, arm pad -n02738978 armhole -n02739123 armilla -n02739427 armlet, arm band -n02739550 armoire -n02739668 armor, armour -n02739889 armored car, armoured car -n02740061 armored car, armoured car -n02740300 armored personnel carrier, armoured personnel carrier, APC -n02740533 armored vehicle, armoured vehicle -n02740764 armor plate, armour plate, armor plating, plate armor, plate armour -n02741367 armory, armoury, arsenal -n02741475 armrest -n02742070 arquebus, harquebus, hackbut, hagbut -n02742194 array -n02742322 array, raiment, regalia -n02742468 arrester, arrester hook -n02742753 arrow -n02743426 arsenal, armory, armoury -n02744323 arterial road -n02744844 arthrogram -n02744961 arthroscope -n02745492 artificial heart -n02745611 artificial horizon, gyro horizon, flight indicator -n02745816 artificial joint -n02746008 artificial kidney, hemodialyzer -n02746225 artificial skin -n02746365 artillery, heavy weapon, gun, ordnance -n02746595 artillery shell -n02746683 artist's loft -n02746978 art school -n02747063 ascot -n02747177 ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin -n02747672 ash-pan -n02747802 ashtray -n02748183 aspergill, aspersorium -n02748359 aspersorium -n02748491 aspirator -n02749169 aspirin powder, headache powder -n02749292 assault gun -n02749479 assault rifle, assault gun -n02749670 assegai, assagai -n02749790 assembly -n02749953 assembly -n02750070 assembly hall -n02750169 assembly plant -n02750320 astatic coils -n02750652 astatic galvanometer -n02751067 astrodome -n02751215 astrolabe -n02751295 astronomical telescope -n02751490 astronomy satellite -n02752199 athenaeum, atheneum -n02752496 athletic sock, sweat sock, varsity sock -n02752615 athletic supporter, supporter, suspensor, jockstrap, jock -n02752810 atlas, telamon -n02752917 atmometer, evaporometer -n02753044 atom bomb, atomic bomb, A-bomb, fission bomb, plutonium bomb -n02753394 atomic clock -n02753710 atomic pile, atomic reactor, pile, chain reactor -n02754103 atomizer, atomiser, spray, sprayer, nebulizer, nebuliser -n02754656 atrium -n02755140 attache case, attache -n02755352 attachment, bond -n02755529 attack submarine -n02755675 attenuator -n02755823 attic -n02755984 attic fan -n02756098 attire, garb, dress -n02756854 audio amplifier -n02756977 audiocassette -n02757061 audio CD, audio compact disc -n02757337 audiometer, sonometer -n02757462 audio system, sound system -n02757714 audiotape -n02757810 audiotape -n02757927 audiovisual, audiovisual aid -n02758134 auditorium -n02758490 auger, gimlet, screw auger, wimble -n02758863 autobahn -n02758960 autoclave, sterilizer, steriliser -n02759257 autofocus -n02759387 autogiro, autogyro, gyroplane -n02759700 autoinjector -n02759963 autoloader, self-loader -n02760099 automat -n02760199 automat -n02760298 automatic choke -n02760429 automatic firearm, automatic gun, automatic weapon -n02760658 automatic pistol, automatic -n02760855 automatic rifle, automatic, machine rifle -n02761034 automatic transmission, automatic drive -n02761206 automation -n02761392 automaton, robot, golem -n02761557 automobile engine -n02761696 automobile factory, auto factory, car factory -n02761834 automobile horn, car horn, motor horn, horn, hooter -n02762169 autopilot, automatic pilot, robot pilot -n02762371 autoradiograph -n02762508 autostrada -n02762725 auxiliary boiler, donkey boiler -n02762909 auxiliary engine, donkey engine -n02763083 auxiliary pump, donkey pump -n02763198 auxiliary research submarine -n02763306 auxiliary storage, external storage, secondary storage -n02763604 aviary, bird sanctuary, volary -n02763714 awl -n02763901 awning, sunshade, sunblind -n02764044 ax, axe -n02764398 ax handle, axe handle -n02764505 ax head, axe head -n02764614 axis, axis of rotation -n02764779 axle -n02764935 axle bar -n02765028 axletree -n02766168 babushka -n02766320 baby bed, baby's bed -n02766534 baby buggy, baby carriage, carriage, perambulator, pram, stroller, go-cart, pushchair, pusher -n02766792 baby grand, baby grand piano, parlor grand, parlor grand piano, parlour grand, parlour grand piano -n02767038 baby powder -n02767147 baby shoe -n02767433 back, backrest -n02767665 back -n02767956 backbench -n02768114 backboard -n02768226 backboard, basketball backboard -n02768433 backbone -n02768655 back brace -n02768973 backgammon board -n02769075 background, desktop, screen background -n02769290 backhoe -n02769669 backlighting -n02769748 backpack, back pack, knapsack, packsack, rucksack, haversack -n02769963 backpacking tent, pack tent -n02770078 backplate -n02770211 back porch -n02770585 backsaw, back saw -n02770721 backscratcher -n02770830 backseat -n02771004 backspace key, backspace, backspacer -n02771166 backstairs -n02771286 backstay -n02771547 backstop -n02771750 backsword -n02772101 backup system -n02772435 badminton court -n02772554 badminton equipment -n02772700 badminton racket, badminton racquet, battledore -n02773037 bag -n02773838 bag, traveling bag, travelling bag, grip, suitcase -n02774152 bag, handbag, pocketbook, purse -n02774630 baggage, luggage -n02774921 baggage -n02775039 baggage car, luggage van -n02775178 baggage claim -n02775483 bagpipe -n02775689 bailey -n02775813 bailey -n02775897 Bailey bridge -n02776007 bain-marie -n02776205 bait, decoy, lure -n02776505 baize -n02776631 bakery, bakeshop, bakehouse -n02776825 balaclava, balaclava helmet -n02776978 balalaika -n02777100 balance -n02777292 balance beam, beam -n02777402 balance wheel, balance -n02777638 balbriggan -n02777734 balcony -n02777927 balcony -n02778131 baldachin -n02778294 baldric, baldrick -n02778456 bale -n02778588 baling wire -n02778669 ball -n02779435 ball -n02779609 ball and chain -n02779719 ball-and-socket joint -n02779971 ballast, light ballast -n02780315 ball bearing, needle bearing, roller bearing -n02780445 ball cartridge -n02780588 ballcock, ball cock -n02780704 balldress -n02780815 ballet skirt, tutu -n02781121 ball gown -n02781213 ballistic galvanometer -n02781338 ballistic missile -n02781517 ballistic pendulum -n02781764 ballistocardiograph, cardiograph -n02782093 balloon -n02782432 balloon bomb, Fugo -n02782602 balloon sail -n02782681 ballot box -n02782778 ballpark, park -n02783035 ball-peen hammer -n02783161 ballpoint, ballpoint pen, ballpen, Biro -n02783324 ballroom, dance hall, dance palace -n02783459 ball valve -n02783900 balsa raft, Kon Tiki -n02783994 baluster -n02784124 banana boat -n02784998 band -n02785648 bandage, patch -n02786058 Band Aid -n02786198 bandanna, bandana -n02786331 bandbox -n02786463 banderilla -n02786611 bandoleer, bandolier -n02786736 bandoneon -n02786837 bandsaw, band saw -n02787120 bandwagon -n02787269 bangalore torpedo -n02787435 bangle, bauble, gaud, gewgaw, novelty, fallal, trinket -n02787622 banjo -n02788021 banner, streamer -n02788148 bannister, banister, balustrade, balusters, handrail -n02788386 banquette -n02788462 banyan, banian -n02788572 baptismal font, baptistry, baptistery, font -n02788689 bar -n02789487 bar -n02790669 barbecue, barbeque -n02790823 barbed wire, barbwire -n02790996 barbell -n02791124 barber chair -n02791270 barbershop -n02791532 barbette carriage -n02791665 barbican, barbacan -n02791795 bar bit -n02792409 bareboat -n02792552 barge, flatboat, hoy, lighter -n02792948 barge pole -n02793089 baritone, baritone horn -n02793199 bark, barque -n02793296 bar magnet -n02793414 bar mask -n02793495 barn -n02793684 barndoor -n02793842 barn door -n02793930 barnyard -n02794008 barograph -n02794156 barometer -n02794368 barong -n02794474 barouche -n02794664 bar printer -n02794779 barrack -n02794972 barrage balloon -n02795169 barrel, cask -n02795528 barrel, gun barrel -n02795670 barrelhouse, honky-tonk -n02795783 barrel knot, blood knot -n02795978 barrel organ, grind organ, hand organ, hurdy gurdy, hurdy-gurdy, street organ -n02796207 barrel vault -n02796318 barrette -n02796412 barricade -n02796623 barrier -n02796995 barroom, bar, saloon, ginmill, taproom -n02797295 barrow, garden cart, lawn cart, wheelbarrow -n02797535 bascule -n02797692 base, pedestal, stand -n02797881 base, bag -n02799071 baseball -n02799175 baseball bat, lumber -n02799323 baseball cap, jockey cap, golf cap -n02799897 baseball equipment -n02800213 baseball glove, glove, baseball mitt, mitt -n02800497 basement, cellar -n02800675 basement -n02800940 basic point defense missile system -n02801047 basilica, Roman basilica -n02801184 basilica -n02801450 basilisk -n02801525 basin -n02801823 basinet -n02801938 basket, handbasket -n02802215 basket, basketball hoop, hoop -n02802426 basketball -n02802544 basketball court -n02802721 basketball equipment -n02802990 basket weave -n02803349 bass -n02803539 bass clarinet -n02803666 bass drum, gran casa -n02803809 basset horn -n02803934 bass fiddle, bass viol, bull fiddle, double bass, contrabass, string bass -n02804123 bass guitar -n02804252 bass horn, sousaphone, tuba -n02804414 bassinet -n02804515 bassinet -n02804610 bassoon -n02805283 baster -n02805845 bastinado -n02805983 bastion -n02806088 bastion, citadel -n02806379 bat -n02806530 bath -n02806762 bath chair -n02806875 bathhouse, bagnio -n02806992 bathhouse, bathing machine -n02807133 bathing cap, swimming cap -n02807523 bath oil -n02807616 bathrobe -n02807731 bathroom, bath -n02808185 bath salts -n02808304 bath towel -n02808440 bathtub, bathing tub, bath, tub -n02808829 bathyscaphe, bathyscaph, bathyscape -n02808968 bathysphere -n02809105 batik -n02809241 batiste -n02809364 baton, wand -n02809491 baton -n02809605 baton -n02809736 baton -n02810139 battering ram -n02810270 batter's box -n02810471 battery, electric battery -n02810782 battery, stamp battery -n02811059 batting cage, cage -n02811204 batting glove -n02811350 batting helmet -n02811468 battle-ax, battle-axe -n02811618 battle cruiser -n02811719 battle dress -n02811936 battlement, crenelation, crenellation -n02812201 battleship, battlewagon -n02812342 battle sight, battlesight -n02812631 bay -n02812785 bay -n02812949 bayonet -n02813252 bay rum -n02813399 bay window, bow window -n02813544 bazaar, bazar -n02813645 bazaar, bazar -n02813752 bazooka -n02813981 B battery -n02814116 BB gun -n02814338 beach house -n02814428 beach towel -n02814533 beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon -n02814774 beachwear -n02814860 beacon, lighthouse, beacon light, pharos -n02815478 beading plane -n02815749 beaker -n02815834 beaker -n02815950 beam -n02816494 beam balance -n02816656 beanbag -n02816768 beanie, beany -n02817031 bearing -n02817251 bearing rein, checkrein -n02817386 bearing wall -n02817516 bearskin, busby, shako -n02817650 beater -n02817799 beating-reed instrument, reed instrument, reed -n02818135 beaver, castor -n02818254 beaver -n02818687 Beckman thermometer -n02818832 bed -n02819697 bed -n02820085 bed and breakfast, bed-and-breakfast -n02820210 bedclothes, bed clothing, bedding -n02820556 Bedford cord -n02820675 bed jacket -n02821202 bedpan -n02821415 bedpost -n02821543 bedroll -n02821627 bedroom, sleeping room, sleeping accommodation, chamber, bedchamber -n02821943 bedroom furniture -n02822064 bedsitting room, bedsitter, bedsit -n02822220 bedspread, bedcover, bed cover, bed covering, counterpane, spread -n02822399 bedspring -n02822579 bedstead, bedframe -n02822762 beefcake -n02822865 beehive, hive -n02823124 beeper, pager -n02823335 beer barrel, beer keg -n02823428 beer bottle -n02823510 beer can -n02823586 beer garden -n02823750 beer glass -n02823848 beer hall -n02823964 beer mat -n02824058 beer mug, stein -n02824152 belaying pin -n02824319 belfry -n02824448 bell -n02825153 bell arch -n02825240 bellarmine, longbeard, long-beard, greybeard -n02825442 bellbottom trousers, bell-bottoms, bellbottom pants -n02825657 bell cote, bell cot -n02825872 bell foundry -n02825961 bell gable -n02826068 bell jar, bell glass -n02826259 bellows -n02826459 bellpull -n02826589 bell push -n02826683 bell seat, balloon seat -n02826812 bell tent -n02826886 bell tower -n02827148 bellyband -n02827606 belt -n02828115 belt, belt ammunition, belted ammunition -n02828299 belt buckle -n02828427 belting -n02828884 bench -n02829246 bench clamp -n02829353 bench hook -n02829510 bench lathe -n02829596 bench press -n02830157 bender -n02831237 beret -n02831335 berlin -n02831595 Bermuda shorts, Jamaica shorts -n02831724 berth, bunk, built in bed -n02831894 besom -n02831998 Bessemer converter -n02833040 bethel -n02833140 betting shop -n02833275 bevatron -n02833403 bevel, bevel square -n02833793 bevel gear, pinion and crown wheel, pinion and ring gear -n02834027 B-flat clarinet, licorice stick -n02834397 bib -n02834506 bib-and-tucker -n02834642 bicorn, bicorne -n02834778 bicycle, bike, wheel, cycle -n02835271 bicycle-built-for-two, tandem bicycle, tandem -n02835412 bicycle chain -n02835551 bicycle clip, trouser clip -n02835724 bicycle pump -n02835829 bicycle rack -n02835915 bicycle seat, saddle -n02836035 bicycle wheel -n02836174 bidet -n02836268 bier -n02836392 bier -n02836513 bi-fold door -n02836607 bifocals -n02836900 Big Blue, BLU-82 -n02837134 big board -n02837567 bight -n02837789 bikini, two-piece -n02837887 bikini pants -n02838014 bilge -n02838178 bilge keel -n02838345 bilge pump -n02838577 bilge well -n02838728 bill, peak, eyeshade, visor, vizor -n02838958 bill, billhook -n02839110 billboard, hoarding -n02839351 billiard ball -n02839592 billiard room, billiard saloon, billiard parlor, billiard parlour, billiard hall -n02839910 bin -n02840134 binder, ligature -n02840245 binder, ring-binder -n02840515 bindery -n02840619 binding, book binding, cover, back -n02841063 bin liner -n02841187 binnacle -n02841315 binoculars, field glasses, opera glasses -n02841506 binocular microscope -n02841641 biochip -n02841847 biohazard suit -n02842133 bioscope -n02842573 biplane -n02842809 birch, birch rod -n02843029 birchbark canoe, birchbark, birch bark -n02843158 birdbath -n02843276 birdcage -n02843465 birdcall -n02843553 bird feeder, birdfeeder, feeder -n02843684 birdhouse -n02843777 bird shot, buckshot, duck shot -n02843909 biretta, berretta, birretta -n02844056 bishop -n02844214 bistro -n02844307 bit -n02844714 bit -n02845130 bite plate, biteplate -n02845293 bitewing -n02845985 bitumastic -n02846141 black -n02846260 black -n02846511 blackboard, chalkboard -n02846619 blackboard eraser -n02846733 black box -n02846874 blackface -n02847461 blackjack, cosh, sap -n02847631 black tie -n02847852 blackwash -n02848118 bladder -n02848216 blade -n02848523 blade, vane -n02848806 blade -n02848921 blank, dummy, blank shell -n02849154 blanket, cover -n02849885 blast furnace -n02850060 blasting cap -n02850358 blazer, sport jacket, sport coat, sports jacket, sports coat -n02850732 blender, liquidizer, liquidiser -n02850950 blimp, sausage balloon, sausage -n02851099 blind, screen -n02851795 blind curve, blind bend -n02851939 blindfold -n02852043 bling, bling bling -n02852173 blinker, flasher -n02852360 blister pack, bubble pack -n02853016 block -n02853218 blockade -n02853336 blockade-runner -n02853745 block and tackle -n02853870 blockbuster -n02854378 blockhouse -n02854532 block plane -n02854630 bloodmobile -n02854739 bloomers, pants, drawers, knickers -n02854926 blouse -n02855089 blower -n02855390 blowtorch, torch, blowlamp -n02855701 blucher -n02855793 bludgeon -n02855925 blue -n02856013 blue chip -n02856237 blunderbuss -n02856362 blunt file -n02857365 boarding -n02857477 boarding house, boardinghouse -n02857644 boardroom, council chamber -n02857907 boards -n02858304 boat -n02859184 boater, leghorn, Panama, Panama hat, sailor, skimmer, straw hat -n02859343 boat hook -n02859443 boathouse -n02859557 boatswain's chair, bosun's chair -n02859729 boat train -n02859955 boatyard -n02860415 bobbin, spool, reel -n02860640 bobby pin, hairgrip, grip -n02860847 bobsled, bobsleigh, bob -n02861022 bobsled, bobsleigh -n02861147 bocce ball, bocci ball, boccie ball -n02861286 bodega -n02861387 bodice -n02861509 bodkin, threader -n02861658 bodkin -n02861777 bodkin -n02861886 body -n02862048 body armor, body armour, suit of armor, suit of armour, coat of mail, cataphract -n02862916 body lotion -n02863014 body stocking -n02863176 body plethysmograph -n02863340 body pad -n02863426 bodywork -n02863536 Bofors gun -n02863638 bogy, bogie, bogey -n02863750 boiler, steam boiler -n02864122 boiling water reactor, BWR -n02864504 bolero -n02864593 bollard, bitt -n02864987 bolo, bolo knife -n02865351 bolo tie, bolo, bola tie, bola -n02865665 bolt -n02865931 bolt, deadbolt -n02866106 bolt -n02866386 bolt cutter -n02866578 bomb -n02867401 bombazine -n02867592 bomb calorimeter, bomb -n02867715 bomber -n02867966 bomber jacket -n02868240 bomblet, cluster bomblet -n02868429 bomb rack -n02868546 bombshell -n02868638 bomb shelter, air-raid shelter, bombproof -n02868975 bone-ash cup, cupel, refractory pot -n02869155 bone china -n02869249 bones, castanets, clappers, finger cymbals -n02869563 boneshaker -n02869737 bongo, bongo drum -n02869837 bonnet, poke bonnet -n02870526 book -n02870676 book bag -n02870772 bookbindery -n02870880 bookcase -n02871005 bookend -n02871147 bookmark, bookmarker -n02871314 bookmobile -n02871439 bookshelf -n02871525 bookshop, bookstore, bookstall -n02871631 boom -n02871824 boom, microphone boom -n02871963 boomerang, throwing stick, throw stick -n02872333 booster, booster rocket, booster unit, takeoff booster, takeoff rocket -n02872529 booster, booster amplifier, booster station, relay link, relay station, relay transmitter -n02872752 boot -n02873520 boot -n02873623 boot camp -n02873733 bootee, bootie -n02873839 booth, cubicle, stall, kiosk -n02874086 booth -n02874214 booth -n02874336 boothose -n02874442 bootjack -n02874537 bootlace -n02874642 bootleg -n02874750 bootstrap -n02875436 bore bit, borer, rock drill, stone drill -n02875626 boron chamber -n02875948 borstal -n02876084 bosom -n02876326 Boston rocker -n02876457 bota -n02876657 bottle -n02877266 bottle, feeding bottle, nursing bottle -n02877513 bottle bank -n02877642 bottlebrush -n02877765 bottlecap -n02877962 bottle opener -n02878107 bottling plant -n02878222 bottom, freighter, merchantman, merchant ship -n02878425 boucle -n02878534 boudoir -n02878628 boulle, boule, buhl -n02878796 bouncing betty -n02879087 bouquet, corsage, posy, nosegay -n02879309 boutique, dress shop -n02879422 boutonniere -n02879517 bow -n02879718 bow -n02880189 bow, bowknot -n02880393 bow and arrow -n02880546 bowed stringed instrument, string -n02880842 Bowie knife -n02880940 bowl -n02881193 bowl -n02881546 bowl -n02881757 bowler hat, bowler, derby hat, derby, plug hat -n02881906 bowline, bowline knot -n02882190 bowling alley -n02882301 bowling ball, bowl -n02882483 bowling equipment -n02882647 bowling pin, pin -n02882894 bowling shoe -n02883004 bowsprit -n02883101 bowstring -n02883205 bow tie, bow-tie, bowtie -n02883344 box -n02884225 box, loge -n02884450 box, box seat -n02884859 box beam, box girder -n02884994 box camera, box Kodak -n02885108 boxcar -n02885233 box coat -n02885338 boxing equipment -n02885462 boxing glove, glove -n02885882 box office, ticket office, ticket booth -n02886321 box spring -n02886434 box wrench, box end wrench -n02886599 brace, bracing -n02887079 brace, braces, orthodontic braces -n02887209 brace -n02887489 brace, suspender, gallus -n02887832 brace and bit -n02887970 bracelet, bangle -n02888270 bracer, armguard -n02888429 brace wrench -n02888569 bracket, wall bracket -n02888898 bradawl, pricker -n02889425 brake -n02889646 brake -n02889856 brake band -n02889996 brake cylinder, hydraulic brake cylinder, master cylinder -n02890188 brake disk -n02890351 brake drum, drum -n02890513 brake lining -n02890662 brake pad -n02890804 brake pedal -n02890940 brake shoe, shoe, skid -n02891188 brake system, brakes -n02891788 brass, brass instrument -n02892201 brass, memorial tablet, plaque -n02892304 brass -n02892392 brassard -n02892499 brasserie -n02892626 brassie -n02892767 brassiere, bra, bandeau -n02892948 brass knucks, knucks, brass knuckles, knuckles, knuckle duster -n02893269 brattice -n02893418 brazier, brasier -n02893608 breadbasket -n02893692 bread-bin, breadbox -n02893941 bread knife -n02894024 breakable -n02894158 breakfast area, breakfast nook -n02894337 breakfast table -n02894605 breakwater, groin, groyne, mole, bulwark, seawall, jetty -n02894847 breast drill -n02895008 breast implant -n02895154 breastplate, aegis, egis -n02895328 breast pocket -n02895438 breathalyzer, breathalyser -n02896074 breechblock, breech closer -n02896294 breechcloth, breechclout, loincloth -n02896442 breeches, knee breeches, knee pants, knickerbockers, knickers -n02896694 breeches buoy -n02896856 breechloader -n02896949 breeder reactor -n02897097 Bren, Bren gun -n02897389 brewpub -n02897820 brick -n02898093 brickkiln -n02898173 bricklayer's hammer -n02898269 brick trowel, mason's trowel -n02898369 brickwork -n02898585 bridal gown, wedding gown, wedding dress -n02898711 bridge, span -n02899439 bridge, nosepiece -n02900160 bridle -n02900459 bridle path, bridle road -n02900594 bridoon -n02900705 briefcase -n02900857 briefcase bomb -n02900987 briefcase computer -n02901114 briefs, Jockey shorts -n02901259 brig -n02901377 brig -n02901481 brigandine -n02901620 brigantine, hermaphrodite brig -n02901793 brilliantine -n02901901 brilliant pebble -n02902079 brim -n02902687 bristle brush -n02902816 britches -n02902916 broad arrow -n02903006 broadax, broadaxe -n02903126 brochette -n02903204 broadcaster, spreader -n02903727 broadcloth -n02903852 broadcloth -n02904109 broad hatchet -n02904233 broadloom -n02904505 broadside -n02904640 broadsword -n02904803 brocade -n02904927 brogan, brogue, clodhopper, work shoe -n02905036 broiler -n02905152 broken arch -n02905886 bronchoscope -n02906734 broom -n02906963 broom closet -n02907082 broomstick, broom handle -n02907296 brougham -n02907391 Browning automatic rifle, BAR -n02907656 Browning machine gun, Peacemaker -n02907873 brownstone -n02908123 brunch coat -n02908217 brush -n02908773 Brussels carpet -n02908951 Brussels lace -n02909053 bubble -n02909165 bubble chamber -n02909285 bubble jet printer, bubble-jet printer, bubblejet -n02909706 buckboard -n02909870 bucket, pail -n02910145 bucket seat -n02910241 bucket shop -n02910353 buckle -n02910542 buckram -n02910701 bucksaw -n02910864 buckskins -n02910964 buff, buffer -n02911332 buffer, polisher -n02911485 buffer, buffer storage, buffer store -n02912065 buffet, counter, sideboard -n02912319 buffing wheel -n02912557 buggy, roadster -n02912894 bugle -n02913152 building, edifice -n02914991 building complex, complex -n02915904 bulldog clip, alligator clip -n02916065 bulldog wrench -n02916179 bulldozer, dozer -n02916350 bullet, slug -n02916936 bulletproof vest -n02917067 bullet train, bullet -n02917377 bullhorn, loud hailer, loud-hailer -n02917521 bullion -n02917607 bullnose, bullnosed plane -n02917742 bullpen, detention cell, detention centre -n02917964 bullpen -n02918112 bullring -n02918330 bulwark -n02918455 bumboat -n02918595 bumper -n02918831 bumper -n02918964 bumper car, Dodgem -n02919148 bumper guard -n02919308 bumper jack -n02919414 bundle, sheaf -n02919648 bung, spile -n02919792 bungalow, cottage -n02919890 bungee, bungee cord -n02919976 bunghole -n02920083 bunk -n02920164 bunk, feed bunk -n02920259 bunk bed, bunk -n02920369 bunker, sand trap, trap -n02920503 bunker, dugout -n02920658 bunker -n02921029 bunsen burner, bunsen, etna -n02921195 bunting -n02921292 bur, burr -n02921406 Burberry -n02921592 burette, buret -n02921756 burglar alarm -n02921884 burial chamber, sepulcher, sepulchre, sepulture -n02922159 burial garment -n02922292 burial mound, grave mound, barrow, tumulus -n02922461 burin -n02922578 burqa, burka -n02922798 burlap, gunny -n02922877 burn bag -n02923129 burner -n02923535 burnous, burnoose, burnouse -n02923682 burp gun, machine pistol -n02923915 burr -n02924116 bus, autobus, coach, charabanc, double-decker, jitney, motorbus, motorcoach, omnibus, passenger vehicle -n02925009 bushel basket -n02925107 bushing, cylindrical lining -n02925385 bush jacket -n02925519 business suit -n02925666 buskin, combat boot, desert boot, half boot, top boot -n02926426 bustier -n02926591 bustle -n02927053 butcher knife -n02927161 butcher shop, meat market -n02927764 butter dish -n02927887 butterfly valve -n02928049 butter knife -n02928299 butt hinge -n02928413 butt joint, butt -n02928608 button -n02929184 buttonhook -n02929289 buttress, buttressing -n02929462 butt shaft -n02929582 butt weld, butt-weld -n02929923 buzz bomb, robot bomb, flying bomb, doodlebug, V-1 -n02930080 buzzer -n02930214 BVD, BVD's -n02930339 bypass condenser, bypass capacitor -n02930645 byway, bypath, byroad -n02930766 cab, hack, taxi, taxicab -n02931013 cab, cabriolet -n02931148 cab -n02931294 cabana -n02931417 cabaret, nightclub, night club, club, nightspot -n02931836 caber -n02932019 cabin -n02932400 cabin -n02932523 cabin car, caboose -n02932693 cabin class, second class, economy class -n02932891 cabin cruiser, cruiser, pleasure boat, pleasure craft -n02933112 cabinet -n02933340 cabinet, console -n02933462 cabinet, locker, storage locker -n02933649 cabinetwork -n02933750 cabin liner -n02933990 cable, cable television, cable system, cable television service -n02934168 cable, line, transmission line -n02934451 cable car, car -n02935017 cache, memory cache -n02935387 caddy, tea caddy -n02935490 caesium clock -n02935658 cafe, coffeehouse, coffee shop, coffee bar -n02935891 cafeteria -n02936176 cafeteria tray -n02936281 caff -n02936402 caftan, kaftan -n02936570 caftan, kaftan -n02936714 cage, coop -n02936921 cage -n02937010 cagoule -n02937336 caisson -n02937958 calash, caleche, calash top -n02938218 calceus -n02938321 calcimine -n02938886 calculator, calculating machine -n02939185 caldron, cauldron -n02939763 calico -n02939866 caliper, calliper -n02940289 call-board -n02940385 call center, call centre -n02940570 caller ID -n02940706 calliope, steam organ -n02941095 calorimeter -n02941228 calpac, calpack, kalpac -n02941845 camail, aventail, ventail -n02942015 camber arch -n02942147 cambric -n02942349 camcorder -n02942460 camel's hair, camelhair -n02942699 camera, photographic camera -n02943241 camera lens, optical lens -n02943465 camera lucida -n02943686 camera obscura -n02943871 camera tripod -n02943964 camise -n02944075 camisole -n02944146 camisole, underbodice -n02944256 camlet -n02944459 camouflage -n02944579 camouflage, camo -n02944826 camp, encampment, cantonment, bivouac -n02945161 camp -n02945813 camp, refugee camp -n02945964 campaign hat -n02946127 campanile, belfry -n02946270 camp chair -n02946348 camper, camping bus, motor home -n02946509 camper trailer -n02946753 campstool -n02946824 camshaft -n02946921 can, tin, tin can -n02947212 canal -n02947660 canal boat, narrow boat, narrowboat -n02947818 candelabrum, candelabra -n02947977 candid camera -n02948072 candle, taper, wax light -n02948293 candlepin -n02948403 candlesnuffer -n02948557 candlestick, candle holder -n02948834 candlewick -n02948942 candy thermometer -n02949084 cane -n02949202 cane -n02949356 cangue -n02949542 canister, cannister, tin -n02950018 cannery -n02950120 cannikin -n02950186 cannikin -n02950256 cannon -n02950482 cannon -n02950632 cannon -n02950826 cannon -n02950943 cannonball, cannon ball, round shot -n02951358 canoe -n02951585 can opener, tin opener -n02951703 canopic jar, canopic vase -n02951843 canopy -n02952109 canopy -n02952237 canopy -n02952374 canteen -n02952485 canteen -n02952585 canteen -n02952674 canteen, mobile canteen -n02952798 canteen -n02952935 cant hook -n02953056 cantilever -n02953197 cantilever bridge -n02953455 cantle -n02953552 Canton crepe -n02953673 canvas, canvass -n02953850 canvas, canvass -n02954163 canvas tent, canvas, canvass -n02954340 cap -n02954938 cap -n02955065 cap -n02955247 capacitor, capacitance, condenser, electrical condenser -n02955540 caparison, trapping, housing -n02955767 cape, mantle -n02956393 capital ship -n02956699 capitol -n02956795 cap opener -n02956883 capote, hooded cloak -n02957008 capote, hooded coat -n02957135 cap screw -n02957252 capstan -n02957427 capstone, copestone, coping stone, stretcher -n02957755 capsule -n02957862 captain's chair -n02958343 car, auto, automobile, machine, motorcar -n02959942 car, railcar, railway car, railroad car -n02960352 car, elevator car -n02960690 carabiner, karabiner, snap ring -n02960903 carafe, decanter -n02961035 caravansary, caravanserai, khan, caravan inn -n02961225 car battery, automobile battery -n02961451 carbine -n02961544 car bomb -n02961947 carbon arc lamp, carbon arc -n02962061 carboy -n02962200 carburetor, carburettor -n02962414 car carrier -n02962843 cardcase -n02962938 cardiac monitor, heart monitor -n02963159 cardigan -n02963302 card index, card catalog, card catalogue -n02963503 cardiograph, electrocardiograph -n02963692 cardioid microphone -n02963821 car door -n02963987 cardroom -n02964075 card table -n02964196 card table -n02964295 car-ferry -n02964634 cargo area, cargo deck, cargo hold, hold, storage area -n02964843 cargo container -n02964934 cargo door -n02965024 cargo hatch -n02965122 cargo helicopter -n02965216 cargo liner -n02965300 cargo ship, cargo vessel -n02965529 carillon -n02965783 car mirror -n02966068 caroche -n02966193 carousel, carrousel, merry-go-round, roundabout, whirligig -n02966545 carpenter's hammer, claw hammer, clawhammer -n02966687 carpenter's kit, tool kit -n02966786 carpenter's level -n02966942 carpenter's mallet -n02967081 carpenter's rule -n02967170 carpenter's square -n02967294 carpetbag -n02967407 carpet beater, rug beater -n02967540 carpet loom -n02967626 carpet pad, rug pad, underlay, underlayment -n02967782 carpet sweeper, sweeper -n02967991 carpet tack -n02968074 carport, car port -n02968210 carrack, carack -n02968333 carrel, carrell, cubicle, stall -n02968473 carriage, equipage, rig -n02969010 carriage -n02969163 carriage bolt -n02969323 carriageway -n02969527 carriage wrench -n02969634 carrick bend -n02969886 carrier -n02970408 carryall, holdall, tote, tote bag -n02970534 carrycot -n02970685 car seat -n02970849 cart -n02971167 car tire, automobile tire, auto tire, rubber tire -n02971356 carton -n02971473 cartouche, cartouch -n02971579 car train -n02971691 cartridge -n02971940 cartridge, pickup -n02972397 cartridge belt -n02972714 cartridge extractor, cartridge remover, extractor -n02972934 cartridge fuse -n02973017 cartridge holder, cartridge clip, clip, magazine -n02973236 cartwheel -n02973805 carving fork -n02973904 carving knife -n02974003 car wheel -n02974348 caryatid -n02974454 cascade liquefier -n02974565 cascade transformer -n02974697 case -n02975212 case, display case, showcase, vitrine -n02975589 case, compositor's case, typesetter's case -n02975994 casein paint, casein -n02976123 case knife, sheath knife -n02976249 case knife -n02976350 casement -n02976455 casement window -n02976552 casern -n02976641 case shot, canister, canister shot -n02976815 cash bar -n02976939 cashbox, money box, till -n02977058 cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM -n02977330 cashmere -n02977438 cash register, register -n02977619 casing, case -n02977936 casino, gambling casino -n02978055 casket, jewel casket -n02978205 casque -n02978367 casquet, casquetel -n02978478 Cassegrainian telescope, Gregorian telescope -n02978753 casserole -n02978881 cassette -n02979074 cassette deck -n02979186 cassette player -n02979290 cassette recorder -n02979399 cassette tape -n02979516 cassock -n02979836 cast, plaster cast, plaster bandage -n02980036 caster, castor -n02980203 caster, castor -n02980441 castle -n02980625 castle, rook -n02981024 catacomb -n02981198 catafalque -n02981321 catalytic converter -n02981565 catalytic cracker, cat cracker -n02981792 catamaran -n02981911 catapult, arbalest, arbalist, ballista, bricole, mangonel, onager, trebuchet, trebucket -n02982232 catapult, launcher -n02982416 catboat -n02982515 cat box -n02982599 catch -n02983072 catchall -n02983189 catcher's mask -n02983357 catchment -n02983507 Caterpillar, cat -n02983904 cathedra, bishop's throne -n02984061 cathedral -n02984203 cathedral, duomo -n02984469 catheter -n02984699 cathode -n02985137 cathode-ray tube, CRT -n02985606 cat-o'-nine-tails, cat -n02985828 cat's-paw -n02985963 catsup bottle, ketchup bottle -n02986066 cattle car -n02986160 cattle guard, cattle grid -n02986348 cattleship, cattle boat -n02987047 cautery, cauterant -n02987379 cavalier hat, slouch hat -n02987492 cavalry sword, saber, sabre -n02987706 cavetto -n02987823 cavity wall -n02987950 C battery -n02988066 C-clamp -n02988156 CD drive -n02988304 CD player -n02988486 CD-R, compact disc recordable, CD-WO, compact disc write-once -n02988679 CD-ROM, compact disc read-only memory -n02988963 CD-ROM drive -n02989099 cedar chest -n02990373 ceiling -n02990758 celesta -n02991048 cell, electric cell -n02991302 cell, jail cell, prison cell -n02991847 cellar, wine cellar -n02992032 cellblock, ward -n02992211 cello, violoncello -n02992368 cellophane -n02992529 cellular telephone, cellular phone, cellphone, cell, mobile phone -n02992795 cellulose tape, Scotch tape, Sellotape -n02993194 cenotaph, empty tomb -n02993368 censer, thurible -n02993546 center, centre -n02994573 center punch -n02994743 Centigrade thermometer -n02995345 central processing unit, CPU, C.P.U., central processor, processor, mainframe -n02995871 centrifugal pump -n02995998 centrifuge, extractor, separator -n02997391 ceramic -n02997607 ceramic ware -n02997910 cereal bowl -n02998003 cereal box -n02998107 cerecloth -n02998563 cesspool, cesspit, sink, sump -n02998696 chachka, tsatske, tshatshke, tchotchke -n02998841 chador, chadar, chaddar, chuddar -n02999138 chafing dish -n02999410 chain -n02999936 chain -n03000134 chainlink fence -n03000247 chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour -n03000530 chain printer -n03000684 chain saw, chainsaw -n03001115 chain store -n03001282 chain tongs -n03001540 chain wrench -n03001627 chair -n03002096 chair -n03002210 chair of state -n03002341 chairlift, chair lift -n03002555 chaise, shay -n03002711 chaise longue, chaise, daybed -n03002816 chalet -n03002948 chalice, goblet -n03003091 chalk -n03003633 challis -n03004275 chamberpot, potty, thunder mug -n03004409 chambray -n03004531 chamfer bit -n03004620 chamfer plane -n03004713 chamois cloth -n03004824 chancel, sanctuary, bema -n03005033 chancellery -n03005147 chancery -n03005285 chandelier, pendant, pendent -n03005515 chandlery -n03005619 chanfron, chamfron, testiere, frontstall, front-stall -n03006626 chanter, melody pipe -n03006788 chantry -n03006903 chap -n03007130 chapel -n03007297 chapterhouse, fraternity house, frat house -n03007444 chapterhouse -n03007591 character printer, character-at-a-time printer, serial printer -n03008177 charcuterie -n03008817 charge-exchange accelerator -n03008976 charger, battery charger -n03009111 chariot -n03009269 chariot -n03009794 charnel house, charnel -n03010473 chassis -n03010656 chassis -n03010795 chasuble -n03010915 chateau -n03011018 chatelaine -n03011355 checker, chequer -n03011741 checkout, checkout counter -n03012013 cheekpiece -n03012159 cheeseboard, cheese tray -n03012373 cheesecloth -n03012499 cheese cutter -n03012644 cheese press -n03012734 chemical bomb, gas bomb -n03012897 chemical plant -n03013006 chemical reactor -n03013438 chemise, sack, shift -n03013580 chemise, shimmy, shift, slip, teddy -n03013850 chenille -n03014440 chessman, chess piece -n03014705 chest -n03015149 chesterfield -n03015254 chest of drawers, chest, bureau, dresser -n03015478 chest protector -n03015631 cheval-de-frise, chevaux-de-frise -n03015851 cheval glass -n03016209 chicane -n03016389 chicken coop, coop, hencoop, henhouse -n03016609 chicken wire -n03016737 chicken yard, hen yard, chicken run, fowl run -n03016868 chiffon -n03016953 chiffonier, commode -n03017070 child's room -n03017168 chime, bell, gong -n03017698 chimney breast -n03017835 chimney corner, inglenook -n03018209 china -n03018349 china cabinet, china closet -n03018614 chinchilla -n03018712 Chinese lantern -n03018848 Chinese puzzle -n03019198 chinning bar -n03019304 chino -n03019434 chino -n03019685 chin rest -n03019806 chin strap -n03019938 chintz -n03020034 chip, microchip, micro chip, silicon chip, microprocessor chip -n03020416 chip, poker chip -n03020692 chisel -n03021228 chlamys -n03024064 choir -n03024233 choir loft -n03024333 choke -n03024518 choke, choke coil, choking coil -n03025070 chokey, choky -n03025165 choo-choo -n03025250 chopine, platform -n03025886 chordophone -n03026506 Christmas stocking -n03026907 chronograph -n03027001 chronometer -n03027108 chronoscope -n03027250 chuck -n03027505 chuck wagon -n03027625 chukka, chukka boot -n03028079 church, church building -n03028596 church bell -n03028785 church hat -n03029066 church key -n03029197 church tower -n03029296 churidars -n03029445 churn, butter churn -n03029925 ciderpress -n03030262 cigar band -n03030353 cigar box -n03030557 cigar cutter -n03030880 cigarette butt -n03031012 cigarette case -n03031152 cigarette holder -n03031422 cigar lighter, cigarette lighter, pocket lighter -n03031756 cinch, girth -n03032252 cinema, movie theater, movie theatre, movie house, picture palace -n03032453 cinquefoil -n03032811 circle, round -n03033267 circlet -n03033362 circuit, electrical circuit, electric circuit -n03033986 circuit board, circuit card, board, card, plug-in, add-in -n03034244 circuit breaker, breaker -n03034405 circuitry -n03034516 circular plane, compass plane -n03034663 circular saw, buzz saw -n03035252 circus tent, big top, round top, top -n03035510 cistern -n03035715 cistern, water tank -n03035832 cittern, cithern, cither, citole, gittern -n03036022 city hall -n03036149 cityscape -n03036244 city university -n03036341 civies, civvies -n03036469 civilian clothing, civilian dress, civilian garb, plain clothes -n03036701 clack valve, clack, clapper valve -n03036866 clamp, clinch -n03037108 clamshell, grapple -n03037228 clapper, tongue -n03037404 clapperboard -n03037590 clarence -n03037709 clarinet -n03038041 Clark cell, Clark standard cell -n03038281 clasp -n03038480 clasp knife, jackknife -n03038685 classroom, schoolroom -n03038870 clavichord -n03039015 clavier, Klavier -n03039259 clay pigeon -n03039353 claymore mine, claymore -n03039493 claymore -n03039827 cleaners, dry cleaners -n03039947 cleaning implement, cleaning device, cleaning equipment -n03040229 cleaning pad -n03040376 clean room, white room -n03040836 clearway -n03041114 cleat -n03041265 cleat -n03041449 cleats -n03041632 cleaver, meat cleaver, chopper -n03041810 clerestory, clearstory -n03042139 clevis -n03042384 clews -n03042490 cliff dwelling -n03042697 climbing frame -n03042829 clinch -n03042984 clinch, clench -n03043173 clincher -n03043274 clinic -n03043423 clinical thermometer, mercury-in-glass clinical thermometer -n03043693 clinker, clinker brick -n03043798 clinometer, inclinometer -n03043958 clip -n03044671 clip lead -n03044801 clip-on -n03044934 clipper -n03045074 clipper -n03045228 clipper, clipper ship -n03045337 cloak -n03045698 cloak -n03045800 cloakroom, coatroom -n03046029 cloche -n03046133 cloche -n03046257 clock -n03046802 clock pendulum -n03046921 clock radio -n03047052 clock tower -n03047171 clockwork -n03047690 clog, geta, patten, sabot -n03047799 cloisonne -n03047941 cloister -n03048883 closed circuit, loop -n03049066 closed-circuit television -n03049326 closed loop, closed-loop system -n03049457 closet -n03049782 closeup lens -n03049924 cloth cap, flat cap -n03050026 cloth covering -n03050453 clothesbrush -n03050546 clothes closet, clothespress -n03050655 clothes dryer, clothes drier -n03050864 clothes hamper, laundry basket, clothes basket, voider -n03051041 clotheshorse -n03051249 clothespin, clothes pin, clothes peg -n03051396 clothes tree, coat tree, coat stand -n03051540 clothing, article of clothing, vesture, wear, wearable, habiliment -n03052464 clothing store, haberdashery, haberdashery store, mens store -n03052917 clout nail, clout -n03053047 clove hitch -n03053976 club car, lounge car -n03054491 clubroom -n03054605 cluster bomb -n03054901 clutch -n03055159 clutch, clutch pedal -n03055418 clutch bag, clutch -n03055670 coach, four-in-hand, coach-and-four -n03055857 coach house, carriage house, remise -n03056097 coal car -n03056215 coal chute -n03056288 coal house -n03056493 coal shovel -n03056583 coaming -n03056873 coaster brake -n03057021 coat -n03057541 coat button -n03057636 coat closet -n03057724 coatdress -n03057841 coatee -n03057920 coat hanger, clothes hanger, dress hanger -n03058107 coating, coat -n03058603 coating -n03058949 coat of paint -n03059103 coatrack, coat rack, hatrack -n03059236 coattail -n03059366 coaxial cable, coax, coax cable -n03059685 cobweb -n03059934 cobweb -n03060728 Cockcroft and Walton accelerator, Cockcroft-Walton accelerator, Cockcroft and Walton voltage multiplier, Cockcroft-Walton voltage multiplier -n03061050 cocked hat -n03061211 cockhorse -n03061345 cockleshell -n03061505 cockpit -n03061674 cockpit -n03061819 cockpit -n03061893 cockscomb, coxcomb -n03062015 cocktail dress, sheath -n03062122 cocktail lounge -n03062245 cocktail shaker -n03062336 cocotte -n03062651 codpiece -n03062798 coelostat -n03062985 coffee can -n03063073 coffee cup -n03063199 coffee filter -n03063338 coffee maker -n03063485 coffee mill, coffee grinder -n03063599 coffee mug -n03063689 coffeepot -n03063834 coffee stall -n03063968 coffee table, cocktail table -n03064250 coffee urn -n03064350 coffer -n03064562 Coffey still -n03064758 coffin, casket -n03064935 cog, sprocket -n03065243 coif -n03065424 coil, spiral, volute, whorl, helix -n03065708 coil -n03066232 coil -n03066359 coil spring, volute spring -n03066464 coin box -n03066849 colander, cullender -n03067093 cold cathode -n03067212 cold chisel, set chisel -n03067339 cold cream, coldcream, face cream, vanishing cream -n03067518 cold frame -n03068181 collar, neckband -n03068998 collar -n03069752 college -n03070059 collet, collet chuck -n03070193 collider -n03070396 colliery, pit -n03070587 collimator -n03070854 collimator -n03071021 cologne, cologne water, eau de cologne -n03071160 colonnade -n03071288 colonoscope -n03071552 colorimeter, tintometer -n03072056 colors, colours -n03072201 color television, colour television, color television system, colour television system, color TV, colour TV -n03072440 color tube, colour tube, color television tube, colour television tube, color TV tube, colour TV tube -n03072682 color wash, colour wash -n03073296 Colt -n03073384 colter, coulter -n03073545 columbarium -n03073694 columbarium, cinerarium -n03073977 column, pillar -n03074380 column, pillar -n03074855 comb -n03075097 comb -n03075248 comber -n03075370 combination lock -n03075500 combination plane -n03075634 combine -n03075768 comforter, pacifier, baby's dummy, teething ring -n03075946 command module -n03076411 commissary -n03076623 commissary -n03076708 commodity, trade good, good -n03077442 common ax, common axe, Dayton ax, Dayton axe -n03077616 common room -n03077741 communications satellite -n03078287 communication system -n03078506 community center, civic center -n03078670 commutator -n03078802 commuter, commuter train -n03078995 compact, powder compact -n03079136 compact, compact car -n03079230 compact disk, compact disc, CD -n03079494 compact-disk burner, CD burner -n03079616 companionway -n03079741 compartment -n03080309 compartment -n03080497 compass -n03080633 compass -n03080731 compass card, mariner's compass -n03080904 compass saw -n03081859 compound -n03081986 compound lens -n03082127 compound lever -n03082280 compound microscope -n03082450 compress -n03082656 compression bandage, tourniquet -n03082807 compressor -n03082979 computer, computing machine, computing device, data processor, electronic computer, information processing system -n03084420 computer circuit -n03084834 computerized axial tomography scanner, CAT scanner -n03085013 computer keyboard, keypad -n03085219 computer monitor -n03085333 computer network -n03085602 computer screen, computer display -n03085781 computer store -n03085915 computer system, computing system, automatic data processing system, ADP system, ADPS -n03086183 concentration camp, stockade -n03086457 concert grand, concert piano -n03086580 concert hall -n03086670 concertina -n03086868 concertina -n03087069 concrete mixer, cement mixer -n03087245 condensation pump, diffusion pump -n03087366 condenser, optical condenser -n03087521 condenser -n03087643 condenser -n03087816 condenser microphone, capacitor microphone -n03088389 condominium -n03088580 condominium, condo -n03088707 conductor -n03089477 cone clutch, cone friction clutch -n03089624 confectionery, confectionary, candy store -n03089753 conference center, conference house -n03089879 conference room -n03090000 conference table, council table, council board -n03090172 confessional -n03090437 conformal projection, orthomorphic projection -n03090710 congress boot, congress shoe, congress gaiter -n03090856 conic projection, conical projection -n03091044 connecting rod -n03091223 connecting room -n03091374 connection, connexion, connector, connecter, connective -n03091907 conning tower -n03092053 conning tower -n03092166 conservatory, hothouse, indoor garden -n03092314 conservatory, conservatoire -n03092476 console -n03092656 console -n03092883 console table, console -n03093427 consulate -n03093792 contact, tangency -n03094159 contact, contact lens -n03094503 container -n03095699 container ship, containership, container vessel -n03095965 containment -n03096439 contrabassoon, contrafagotto, double bassoon -n03096960 control, controller -n03097362 control center -n03097535 control circuit, negative feedback circuit -n03097673 control key, command key -n03098140 control panel, instrument panel, control board, board, panel -n03098515 control rod -n03098688 control room -n03098806 control system -n03098959 control tower -n03099147 convector -n03099274 convenience store -n03099454 convent -n03099622 conventicle, meetinghouse -n03099771 converging lens, convex lens -n03099945 converter, convertor -n03100240 convertible -n03100346 convertible, sofa bed -n03100490 conveyance, transport -n03100897 conveyer belt, conveyor belt, conveyer, conveyor, transporter -n03101156 cooker -n03101302 cookfire -n03101375 cookhouse -n03101517 cookie cutter -n03101664 cookie jar, cooky jar -n03101796 cookie sheet, baking tray -n03101986 cooking utensil, cookware -n03102371 cookstove -n03102516 coolant system -n03102654 cooler, ice chest -n03102859 cooling system, cooling -n03103128 cooling system, engine cooling system -n03103396 cooling tower -n03103563 coonskin cap, coonskin -n03103904 cope -n03104019 coping saw -n03104512 copperware -n03105088 copyholder -n03105214 coquille -n03105306 coracle -n03105467 corbel, truss -n03105645 corbel arch -n03105810 corbel step, corbie-step, corbiestep, crow step -n03105974 corbie gable -n03106722 cord, corduroy -n03106898 cord, electric cord -n03107046 cordage -n03107488 cords, corduroys -n03107716 core -n03108455 core bit -n03108624 core drill -n03108759 corer -n03108853 cork, bottle cork -n03109033 corker -n03109150 corkscrew, bottle screw -n03109253 corncrib -n03109693 corner, quoin -n03109881 corner, nook -n03110202 corner post -n03110669 cornet, horn, trumpet, trump -n03111041 cornice -n03111177 cornice -n03111296 cornice, valance, valance board, pelmet -n03111690 correctional institution -n03112240 corrugated fastener, wiggle nail -n03112719 corselet, corslet -n03112869 corset, girdle, stays -n03113152 cosmetic -n03113505 cosmotron -n03113657 costume -n03113835 costume -n03114041 costume -n03114236 costume -n03114379 cosy, tea cosy, cozy, tea cozy -n03114504 cot, camp bed -n03114743 cottage tent -n03114839 cotter, cottar -n03115014 cotter pin -n03115180 cotton -n03115400 cotton flannel, Canton flannel -n03115663 cotton mill -n03115762 couch -n03115897 couch -n03116008 couchette -n03116163 coude telescope, coude system -n03116530 counter -n03116767 counter, tabulator -n03117199 counter -n03117642 counterbore, countersink, countersink bit -n03118346 counter tube -n03118969 country house -n03119203 country store, general store, trading post -n03119396 coupe -n03119510 coupling, coupler -n03120198 court, courtyard -n03120491 court -n03120778 court, courtroom -n03121040 court -n03121190 Courtelle -n03121298 courthouse -n03121431 courthouse -n03121897 coverall -n03122073 covered bridge -n03122202 covered couch -n03122295 covered wagon, Conestoga wagon, Conestoga, prairie wagon, prairie schooner -n03122748 covering -n03123553 coverlet -n03123666 cover plate -n03123809 cowbarn, cowshed, cow barn, cowhouse, byre -n03123917 cowbell -n03124043 cowboy boot -n03124170 cowboy hat, ten-gallon hat -n03124313 cowhide -n03124474 cowl -n03124590 cow pen, cattle pen, corral -n03125057 CPU board, mother board -n03125588 crackle, crackleware, crackle china -n03125729 cradle -n03125870 craft -n03126090 cramp, cramp iron -n03126385 crampon, crampoon, climbing iron, climber -n03126580 crampon, crampoon -n03126707 crane -n03126927 craniometer -n03127024 crank, starter -n03127203 crankcase -n03127408 crankshaft -n03127531 crash barrier -n03127747 crash helmet -n03127925 crate -n03128085 cravat -n03128248 crayon, wax crayon -n03128427 crazy quilt -n03128519 cream, ointment, emollient -n03129001 cream pitcher, creamer -n03129471 creche, foundling hospital -n03129636 creche -n03129753 credenza, credence -n03129848 creel -n03130066 crematory, crematorium, cremation chamber -n03130233 crematory, crematorium -n03130563 crepe, crape -n03130761 crepe de Chine -n03130866 crescent wrench -n03131193 cretonne -n03131574 crib, cot -n03131669 crib -n03131967 cricket ball -n03132076 cricket bat, bat -n03132261 cricket equipment -n03132438 cringle, eyelet, loop, grommet, grummet -n03132666 crinoline -n03132776 crinoline -n03133050 crochet needle, crochet hook -n03133415 crock, earthenware jar -n03133878 Crock Pot -n03134118 crook, shepherd's crook -n03134232 Crookes radiometer -n03134394 Crookes tube -n03134739 croquet ball -n03134853 croquet equipment -n03135030 croquet mallet -n03135532 cross -n03135656 crossbar -n03135788 crossbar -n03135917 crossbar -n03136051 crossbench -n03136254 cross bit -n03136369 crossbow -n03136504 crosscut saw, crosscut handsaw, cutoff saw -n03137473 crossjack, mizzen course -n03137579 crosspiece -n03138128 crotchet -n03138217 croupier's rake -n03138344 crowbar, wrecking bar, pry, pry bar -n03138669 crown, diadem -n03139089 crown, crownwork, jacket, jacket crown, cap -n03139464 crown jewels -n03139640 crown lens -n03139998 crow's nest -n03140126 crucible, melting pot -n03140292 crucifix, rood, rood-tree -n03140431 cruet, crewet -n03140546 cruet-stand -n03140652 cruise control -n03140771 cruise missile -n03140900 cruiser -n03141065 cruiser, police cruiser, patrol car, police car, prowl car, squad car -n03141327 cruise ship, cruise liner -n03141455 crupper -n03141612 cruse -n03141702 crusher -n03141823 crutch -n03142099 cryometer -n03142205 cryoscope -n03142325 cryostat -n03142431 crypt -n03142679 crystal, watch crystal, watch glass -n03143400 crystal detector -n03143572 crystal microphone -n03143754 crystal oscillator, quartz oscillator -n03144156 crystal set -n03144873 cubitiere -n03144982 cucking stool, ducking stool -n03145147 cuckoo clock -n03145277 cuddy -n03145384 cudgel -n03145522 cue, cue stick, pool cue, pool stick -n03145719 cue ball -n03145843 cuff, turnup -n03146219 cuirass -n03146342 cuisse -n03146449 cul, cul de sac, dead end -n03146560 culdoscope -n03146687 cullis -n03146777 culotte -n03146846 cultivator, tiller -n03147084 culverin -n03147156 culverin -n03147280 culvert -n03147509 cup -n03148324 cupboard, closet -n03148518 cup hook -n03148727 cupola -n03148808 cupola -n03149135 curb, curb bit -n03149401 curb roof -n03149686 curbstone, kerbstone -n03149810 curette, curet -n03150232 curler, hair curler, roller, crimper -n03150511 curling iron -n03150661 currycomb -n03150795 cursor, pointer -n03151077 curtain, drape, drapery, mantle, pall -n03152303 customhouse, customshouse -n03152951 cutaway, cutaway drawing, cutaway model -n03153246 cutlas, cutlass -n03153585 cutoff -n03153948 cutout -n03154073 cutter, cutlery, cutting tool -n03154316 cutter -n03154446 cutting implement -n03154616 cutting room -n03154745 cutty stool -n03154895 cutwork -n03155178 cybercafe -n03155502 cyclopean masonry -n03155915 cyclostyle -n03156071 cyclotron -n03156279 cylinder -n03156405 cylinder, piston chamber -n03156767 cylinder lock -n03157348 cymbal -n03158186 dacha -n03158414 Dacron, Terylene -n03158668 dado -n03158796 dado plane -n03158885 dagger, sticker -n03159535 dairy, dairy farm -n03159640 dais, podium, pulpit, rostrum, ambo, stump, soapbox -n03160001 daisy print wheel, daisy wheel -n03160186 daisywheel printer -n03160309 dam, dike, dyke -n03160740 damask -n03161016 dampener, moistener -n03161450 damper, muffler -n03161893 damper block, piano damper -n03162297 dark lantern, bull's-eye -n03162460 darkroom -n03162556 darning needle, embroidery needle -n03162714 dart -n03162818 dart -n03163222 dashboard, fascia -n03163381 dashiki, daishiki -n03163488 dash-pot -n03163798 data converter -n03163973 data input device, input device -n03164192 data multiplexer -n03164344 data system, information system -n03164605 davenport -n03164722 davenport -n03164929 davit -n03165096 daybed, divan bed -n03165211 daybook, ledger -n03165466 day nursery, day care center -n03165616 day school -n03165823 dead axle -n03165955 deadeye -n03166120 deadhead -n03166514 deanery -n03166600 deathbed -n03166685 death camp -n03166809 death house, death row -n03166951 death knell, death bell -n03167153 death seat -n03167978 deck -n03168107 deck -n03168217 deck chair, beach chair -n03168543 deck-house -n03168663 deckle -n03168774 deckle edge, deckle -n03168933 declinometer, transit declinometer -n03169063 decoder -n03169176 decolletage -n03170292 decoupage -n03170459 dedicated file server -n03170635 deep-freeze, Deepfreeze, deep freezer, freezer -n03170872 deerstalker -n03171228 defense system, defence system -n03171356 defensive structure, defense, defence -n03171635 defibrillator -n03171910 defilade -n03172038 deflector -n03172738 delayed action -n03172965 delay line -n03173270 delft -n03173387 delicatessen, deli, food shop -n03173929 delivery truck, delivery van, panel truck -n03174079 delta wing -n03174450 demijohn -n03174731 demitasse -n03175081 den -n03175189 denim, dungaree, jean -n03175301 densimeter, densitometer -n03175457 densitometer -n03175604 dental appliance -n03175843 dental floss, floss -n03175983 dental implant -n03176238 dentist's drill, burr drill -n03176386 denture, dental plate, plate -n03176594 deodorant, deodourant -n03176763 department store, emporium -n03177059 departure lounge -n03177165 depilatory, depilator, epilator -n03177708 depressor -n03178000 depth finder -n03178173 depth gauge, depth gage -n03178430 derrick -n03178538 derrick -n03178674 derringer -n03179701 desk -n03179910 desk phone -n03180011 desktop computer -n03180384 dessert spoon -n03180504 destroyer, guided missile destroyer -n03180732 destroyer escort -n03180865 detached house, single dwelling -n03180969 detector, sensor, sensing element -n03181293 detector -n03181667 detention home, detention house, house of detention, detention camp -n03182140 detonating fuse -n03182232 detonator, detonating device, cap -n03182912 developer -n03183080 device -n03185868 Dewar flask, Dewar -n03186199 dhoti -n03186285 dhow -n03186818 dial, telephone dial -n03187037 dial -n03187153 dial -n03187268 dialog box, panel -n03187595 dial telephone, dial phone -n03187751 dialyzer, dialysis machine -n03188290 diamante -n03188531 diaper, nappy, napkin -n03188725 diaper -n03188871 diaphone -n03189083 diaphragm, stop -n03189311 diaphragm -n03189818 diathermy machine -n03190458 dibble, dibber -n03191286 dice cup, dice box -n03191451 dicer -n03191561 dickey, dickie, dicky, shirtfront -n03191776 dickey, dickie, dicky, dickey-seat, dickie-seat, dicky-seat -n03192543 Dictaphone -n03192907 die -n03193107 diesel, diesel engine, diesel motor -n03193260 diesel-electric locomotive, diesel-electric -n03193423 diesel-hydraulic locomotive, diesel-hydraulic -n03193597 diesel locomotive -n03193754 diestock -n03194170 differential analyzer -n03194297 differential gear, differential -n03194812 diffuser, diffusor -n03194992 diffuser, diffusor -n03195332 digester -n03195485 diggings, digs, domiciliation, lodgings, pad -n03195799 digital-analog converter, digital-to-analog converter -n03195959 digital audiotape, DAT -n03196062 digital camera -n03196217 digital clock -n03196324 digital computer -n03196598 digital display, alphanumeric display -n03196990 digital subscriber line, DSL -n03197201 digital voltmeter -n03197337 digital watch -n03197446 digitizer, digitiser, analog-digital converter, analog-to-digital converter -n03198223 dilator, dilater -n03198500 dildo -n03199358 dimity -n03199488 dimmer -n03199647 diner -n03199775 dinette -n03199901 dinghy, dory, rowboat -n03200231 dining area -n03200357 dining car, diner, dining compartment, buffet car -n03200539 dining-hall -n03200701 dining room, dining-room -n03200906 dining-room furniture -n03201035 dining-room table -n03201208 dining table, board -n03201529 dinner bell -n03201638 dinner dress, dinner gown, formal, evening gown -n03201776 dinner jacket, tux, tuxedo, black tie -n03201895 dinner napkin -n03201996 dinner pail, dinner bucket -n03202354 dinner table -n03202481 dinner theater, dinner theatre -n03202760 diode, semiconductor diode, junction rectifier, crystal rectifier -n03202940 diode, rectifying tube, rectifying valve -n03203089 dip -n03203806 diplomatic building -n03204134 dipole, dipole antenna -n03204306 dipper -n03204436 dipstick -n03204558 DIP switch, dual inline package switch -n03204955 directional antenna -n03205143 directional microphone -n03205304 direction finder -n03205458 dirk -n03205574 dirndl -n03205669 dirndl -n03205903 dirty bomb -n03206023 discharge lamp -n03206158 discharge pipe -n03206282 disco, discotheque -n03206405 discount house, discount store, discounter, wholesale house -n03206602 discus, saucer -n03206718 disguise -n03206908 dish -n03207305 dish, dish aerial, dish antenna, saucer -n03207548 dishpan -n03207630 dish rack -n03207743 dishrag, dishcloth -n03207835 dishtowel, dish towel, tea towel -n03207941 dishwasher, dish washer, dishwashing machine -n03208556 disk, disc -n03208938 disk brake, disc brake -n03209359 disk clutch -n03209477 disk controller -n03209666 disk drive, disc drive, hard drive, Winchester drive -n03209910 diskette, floppy, floppy disk -n03210245 disk harrow, disc harrow -n03210372 dispatch case, dispatch box -n03210552 dispensary -n03210683 dispenser -n03211117 display, video display -n03211413 display adapter, display adaptor -n03211616 display panel, display board, board -n03211789 display window, shop window, shopwindow, show window -n03212114 disposal, electric pig, garbage disposal -n03212247 disrupting explosive, bursting explosive -n03212406 distaff -n03212811 distillery, still -n03213014 distributor, distributer, electrical distributor -n03213361 distributor cam -n03213538 distributor cap -n03213715 distributor housing -n03213826 distributor point, breaker point, point -n03214253 ditch -n03214450 ditch spade, long-handled spade -n03214582 ditty bag -n03214966 divan -n03215076 divan, diwan -n03215191 dive bomber -n03215337 diverging lens, concave lens -n03215508 divided highway, dual carriageway -n03215749 divider -n03215930 diving bell -n03216199 divining rod, dowser, dowsing rod, waterfinder, water finder -n03216402 diving suit, diving dress -n03216562 dixie -n03216710 Dixie cup, paper cup -n03216828 dock, dockage, docking facility -n03217653 doeskin -n03217739 dogcart -n03217889 doggie bag, doggy bag -n03218198 dogsled, dog sled, dog sleigh -n03218446 dog wrench -n03219010 doily, doyley, doyly -n03219135 doll, dolly -n03219483 dollhouse, doll's house -n03219612 dolly -n03219859 dolman -n03219966 dolman, dolman jacket -n03220095 dolman sleeve -n03220237 dolmen, cromlech, portal tomb -n03220513 dome -n03220692 dome, domed stadium, covered stadium -n03221059 domino, half mask, eye mask -n03221351 dongle -n03221540 donkey jacket -n03221720 door -n03222176 door -n03222318 door -n03222516 doorbell, bell, buzzer -n03222722 doorframe, doorcase -n03222857 doorjamb, doorpost -n03223162 doorlock -n03223299 doormat, welcome mat -n03223441 doornail -n03223553 doorplate -n03223686 doorsill, doorstep, threshold -n03223923 doorstop, doorstopper -n03224490 Doppler radar -n03224603 dormer, dormer window -n03224753 dormer window -n03224893 dormitory, dorm, residence hall, hall, student residence -n03225108 dormitory, dormitory room, dorm room -n03225458 dosemeter, dosimeter -n03225616 dossal, dossel -n03225777 dot matrix printer, matrix printer, dot printer -n03225988 double bed -n03226090 double-bitted ax, double-bitted axe, Western ax, Western axe -n03226254 double boiler, double saucepan -n03226375 double-breasted jacket -n03226538 double-breasted suit -n03226880 double door -n03227010 double glazing -n03227184 double-hung window -n03227317 double knit -n03227721 doubler -n03227856 double reed -n03228016 double-reed instrument, double reed -n03228254 doublet -n03228365 doubletree -n03228533 douche, douche bag -n03228692 dovecote, columbarium, columbary -n03228796 Dover's powder -n03228967 dovetail, dovetail joint -n03229115 dovetail plane -n03229244 dowel, dowel pin, joggle -n03229526 downstage -n03231160 drafting instrument -n03231368 drafting table, drawing table -n03231819 Dragunov -n03232309 drainage ditch -n03232417 drainage system -n03232543 drain basket -n03232815 drainplug -n03232923 drape -n03233123 drapery -n03233624 drawbar -n03233744 drawbridge, lift bridge -n03233905 drawer -n03234164 drawers, underdrawers, shorts, boxers, boxershorts -n03234952 drawing chalk -n03235042 drawing room, withdrawing room -n03235180 drawing room -n03235327 drawknife, drawshave -n03235796 drawstring bag -n03235979 dray, camion -n03236093 dreadnought, dreadnaught -n03236217 dredge -n03236423 dredger -n03236580 dredging bucket -n03236735 dress, frock -n03237212 dress blues, dress whites -n03237340 dresser -n03237416 dress hat, high hat, opera hat, silk hat, stovepipe, top hat, topper, beaver -n03237639 dressing, medical dressing -n03237839 dressing case -n03237992 dressing gown, robe-de-chambre, lounging robe -n03238131 dressing room -n03238286 dressing sack, dressing sacque -n03238586 dressing table, dresser, vanity, toilet table -n03238762 dress rack -n03238879 dress shirt, evening shirt -n03239054 dress suit, full dress, tailcoat, tail coat, tails, white tie, white tie and tails -n03239259 dress uniform -n03239607 drift net -n03239726 drill -n03240140 electric drill -n03240683 drilling platform, offshore rig -n03240892 drill press -n03241093 drill rig, drilling rig, oilrig, oil rig -n03241335 drinking fountain, water fountain, bubbler -n03241496 drinking vessel -n03241903 drip loop -n03242120 drip mat -n03242264 drip pan -n03242390 dripping pan, drip pan -n03242506 drip pot -n03242995 drive -n03243218 drive -n03243625 drive line, drive line system -n03244047 driver, number one wood -n03244231 driveshaft -n03244388 driveway, drive, private road -n03244775 driving iron, one iron -n03244919 driving wheel -n03245271 drogue, drogue chute, drogue parachute -n03245421 drogue parachute -n03245724 drone, drone pipe, bourdon -n03245889 drone, pilotless aircraft, radio-controlled aircraft -n03246197 drop arch -n03246312 drop cloth -n03246454 drop curtain, drop cloth, drop -n03246653 drop forge, drop hammer, drop press -n03246933 drop-leaf table -n03247083 dropper, eye dropper -n03247351 droshky, drosky -n03247495 drove, drove chisel -n03248835 drugget -n03249342 drugstore, apothecary's shop, chemist's, chemist's shop, pharmacy -n03249569 drum, membranophone, tympan -n03249956 drum, metal drum -n03250089 drum brake -n03250279 drumhead, head -n03250405 drum printer -n03250588 drum sander, electric sander, sander, smoother -n03250847 drumstick -n03250952 dry battery -n03251100 dry-bulb thermometer -n03251280 dry cell -n03251533 dry dock, drydock, graving dock -n03251766 dryer, drier -n03251932 dry fly -n03252231 dry kiln -n03252324 dry masonry -n03252422 dry point -n03252637 dry wall, dry-stone wall -n03252787 dual scan display -n03253071 duck -n03253187 duckboard -n03253279 duckpin -n03253714 dudeen -n03253796 duffel, duffle -n03253886 duffel bag, duffle bag, duffel, duffle -n03254046 duffel coat, duffle coat -n03254189 dugout -n03254374 dugout canoe, dugout, pirogue -n03254625 dulciana -n03254737 dulcimer -n03254862 dulcimer -n03255030 dumbbell -n03255167 dumb bomb, gravity bomb -n03255322 dumbwaiter, food elevator -n03255488 dumdum, dumdum bullet -n03255899 dumpcart -n03256032 Dumpster -n03256166 dump truck, dumper, tipper truck, tipper lorry, tip truck, tipper -n03256472 Dumpy level -n03256631 dunce cap, dunce's cap, fool's cap -n03256788 dune buggy, beach buggy -n03256928 dungeon -n03257065 duplex apartment, duplex -n03257210 duplex house, duplex, semidetached house -n03257586 duplicator, copier -n03258192 dust bag, vacuum bag -n03258330 dustcloth, dustrag, duster -n03258456 dust cover -n03258577 dust cover, dust sheet -n03258905 dustmop, dust mop, dry mop -n03259009 dustpan -n03259280 Dutch oven -n03259401 Dutch oven -n03259505 dwelling, home, domicile, abode, habitation, dwelling house -n03260206 dye-works -n03260504 dynamo -n03260733 dynamometer, ergometer -n03260849 Eames chair -n03261019 earflap, earlap -n03261263 early warning radar -n03261395 early warning system -n03261603 earmuff -n03261776 earphone, earpiece, headphone, phone -n03262072 earplug -n03262248 earplug -n03262519 earthenware -n03262717 earthwork -n03262809 easel -n03262932 easy chair, lounge chair, overstuffed chair -n03263076 eaves -n03263338 ecclesiastical attire, ecclesiastical robe -n03263640 echinus -n03263758 echocardiograph -n03264906 edger -n03265032 edge tool -n03265754 efficiency apartment -n03266195 egg-and-dart, egg-and-anchor, egg-and-tongue -n03266371 eggbeater, eggwhisk -n03266620 egg timer -n03266749 eiderdown, duvet, continental quilt -n03267113 eight ball -n03267468 ejection seat, ejector seat, capsule -n03267696 elastic -n03267821 elastic bandage -n03268142 Elastoplast -n03268311 elbow -n03268645 elbow pad -n03268790 electric, electric automobile, electric car -n03268918 electrical cable -n03269073 electrical contact -n03269203 electrical converter -n03269401 electrical device -n03270165 electrical system -n03270695 electric bell -n03270854 electric blanket -n03271030 electric chair, chair, death chair, hot seat -n03271260 electric clock -n03271376 electric-discharge lamp, gas-discharge lamp -n03271574 electric fan, blower -n03271765 electric frying pan -n03271865 electric furnace -n03272010 electric guitar -n03272125 electric hammer -n03272239 electric heater, electric fire -n03272383 electric lamp -n03272562 electric locomotive -n03272810 electric meter, power meter -n03272940 electric mixer -n03273061 electric motor -n03273551 electric organ, electronic organ, Hammond organ, organ -n03273740 electric range -n03273913 electric refrigerator, fridge -n03274265 electric toothbrush -n03274435 electric typewriter -n03274561 electro-acoustic transducer -n03274796 electrode -n03275125 electrodynamometer -n03275311 electroencephalograph -n03275566 electrograph -n03275681 electrolytic, electrolytic capacitor, electrolytic condenser -n03275864 electrolytic cell -n03276179 electromagnet -n03276696 electrometer -n03276839 electromyograph -n03277004 electron accelerator -n03277149 electron gun -n03277459 electronic balance -n03277602 electronic converter -n03277771 electronic device -n03278248 electronic equipment -n03278914 electronic fetal monitor, electronic foetal monitor, fetal monitor, foetal monitor -n03279153 electronic instrument, electronic musical instrument -n03279364 electronic voltmeter -n03279508 electron microscope -n03279804 electron multiplier -n03279918 electrophorus -n03280216 electroscope -n03280394 electrostatic generator, electrostatic machine, Wimshurst machine, Van de Graaff generator -n03280644 electrostatic printer -n03281145 elevator, lift -n03281524 elevator -n03281673 elevator shaft -n03282060 embankment -n03282295 embassy -n03282401 embellishment -n03283221 emergency room, ER -n03283413 emesis basin -n03283827 emitter -n03284308 empty -n03284482 emulsion, photographic emulsion -n03284743 enamel -n03284886 enamel -n03284981 enamelware -n03285578 encaustic -n03285730 encephalogram, pneumoencephalogram -n03285912 enclosure -n03286572 endoscope -n03287351 energizer, energiser -n03287733 engine -n03288003 engine -n03288500 engineering, engine room -n03288643 enginery -n03288742 English horn, cor anglais -n03288886 English saddle, English cavalry saddle -n03289660 enlarger -n03289985 ensemble -n03290096 ensign -n03290195 entablature -n03290653 entertainment center -n03291413 entrenching tool, trenching spade -n03291551 entrenchment, intrenchment -n03291741 envelope -n03291819 envelope -n03291963 envelope, gasbag -n03292085 eolith -n03292362 epauliere -n03292475 epee -n03292603 epergne -n03292736 epicyclic train, epicyclic gear train -n03292960 epidiascope -n03293095 epilating wax -n03293741 equalizer, equaliser -n03293863 equatorial -n03294048 equipment -n03294604 erasable programmable read-only memory, EPROM -n03294833 eraser -n03295012 erecting prism -n03295140 erection -n03295246 Erlenmeyer flask -n03295928 escape hatch -n03296081 escapement -n03296217 escape wheel -n03296328 escarpment, escarp, scarp, protective embankment -n03296478 escutcheon, scutcheon -n03296963 esophagoscope, oesophagoscope -n03297103 espadrille -n03297226 espalier -n03297495 espresso maker -n03297644 espresso shop -n03297735 establishment -n03298089 estaminet -n03298352 estradiol patch -n03298716 etagere -n03298858 etamine, etamin -n03299406 etching -n03300216 ethernet -n03300443 ethernet cable -n03301175 Eton jacket -n03301291 etui -n03301389 eudiometer -n03301568 euphonium -n03301833 evaporative cooler -n03301940 evening bag -n03302671 exercise bike, exercycle -n03302790 exercise device -n03302938 exhaust, exhaust system -n03303217 exhaust fan -n03303669 exhaust valve -n03303831 exhibition hall, exhibition area -n03304197 Exocet -n03304323 expansion bit, expansive bit -n03304465 expansion bolt -n03305300 explosive detection system, EDS -n03305522 explosive device -n03305953 explosive trace detection, ETD -n03306385 express, limited -n03306869 extension, telephone extension, extension phone -n03307037 extension cord -n03307573 external-combustion engine -n03307792 external drive -n03308152 extractor -n03308481 eyebrow pencil -n03308614 eyecup, eyebath, eye cup -n03309110 eyeliner -n03309356 eyepatch, patch -n03309465 eyepiece, ocular -n03309687 eyeshadow -n03309808 fabric, cloth, material, textile -n03313333 facade, frontage, frontal -n03314227 face guard -n03314378 face mask -n03314608 faceplate -n03314780 face powder -n03314884 face veil -n03315644 facing, cladding -n03315805 facing -n03315990 facing, veneer -n03316105 facsimile, facsimile machine, fax -n03316406 factory, mill, manufacturing plant, manufactory -n03316873 factory ship -n03317233 fagot, faggot -n03317510 fagot stitch, faggot stitch -n03317673 Fahrenheit thermometer -n03317788 faience -n03317889 faille -n03318136 fairlead -n03318294 fairy light -n03318865 falchion -n03318983 fallboard, fall-board -n03319167 fallout shelter -n03319457 false face -n03319576 false teeth -n03319745 family room -n03320046 fan -n03320262 fan belt -n03320421 fan blade -n03320519 fancy dress, masquerade, masquerade costume -n03320845 fanion -n03320959 fanlight -n03321103 fanjet, fan-jet, fanjet engine, turbojet, turbojet engine, turbofan, turbofan engine -n03321419 fanjet, fan-jet, turbofan, turbojet -n03321563 fanny pack, butt pack -n03321843 fan tracery -n03321954 fan vaulting -n03322570 farm building -n03322704 farmer's market, green market, greenmarket -n03322836 farmhouse -n03322940 farm machine -n03323096 farmplace, farm-place, farmstead -n03323211 farmyard -n03323319 farthingale -n03323703 fastener, fastening, holdfast, fixing -n03324629 fast reactor -n03324814 fat farm -n03324928 fatigues -n03325088 faucet, spigot -n03325288 fauld -n03325403 fauteuil -n03325584 feather boa, boa -n03325691 featheredge -n03325941 fedora, felt hat, homburg, Stetson, trilby -n03326073 feedback circuit, feedback loop -n03326371 feedlot -n03326475 fell, felled seam -n03326660 felloe, felly -n03326795 felt -n03326948 felt-tip pen, felt-tipped pen, felt tip, Magic Marker -n03327133 felucca -n03327234 fence, fencing -n03327553 fencing mask, fencer's mask -n03327691 fencing sword -n03327841 fender, wing -n03328201 fender, buffer, cowcatcher, pilot -n03329302 Ferris wheel -n03329536 ferrule, collet -n03329663 ferry, ferryboat -n03330002 ferule -n03330665 festoon -n03330792 fetoscope, foetoscope -n03330947 fetter, hobble -n03331077 fez, tarboosh -n03331244 fiber, fibre, vulcanized fiber -n03331599 fiber optic cable, fibre optic cable -n03332005 fiberscope -n03332173 fichu -n03332271 fiddlestick, violin bow -n03332393 field artillery, field gun -n03332591 field coil, field winding -n03332784 field-effect transistor, FET -n03332989 field-emission microscope -n03333129 field glass, glass, spyglass -n03333252 field hockey ball -n03333349 field hospital -n03333610 field house, sports arena -n03333711 field lens -n03333851 field magnet -n03334017 field-sequential color television, field-sequential color TV, field-sequential color television system, field-sequential color TV system -n03334291 field tent -n03334382 fieldwork -n03334492 fife -n03334912 fifth wheel, spare -n03335030 fighter, fighter aircraft, attack aircraft -n03335333 fighting chair -n03335461 fig leaf -n03335846 figure eight, figure of eight -n03336168 figure loom, figured-fabric loom -n03336282 figure skate -n03336575 filament -n03336742 filature -n03336839 file -n03337140 file, file cabinet, filing cabinet -n03337383 file folder -n03337494 file server -n03337822 filigree, filagree, fillagree -n03338287 filling -n03338821 film, photographic film -n03339296 film, plastic film -n03339529 film advance -n03339643 filter -n03340009 filter -n03340723 finder, viewfinder, view finder -n03340923 finery -n03341035 fine-tooth comb, fine-toothed comb -n03341153 finger -n03341297 fingerboard -n03341606 finger bowl -n03342015 finger paint, fingerpaint -n03342127 finger-painting -n03342262 finger plate, escutcheon, scutcheon -n03342432 fingerstall, cot -n03342657 finish coat, finishing coat -n03342863 finish coat, finishing coat -n03342961 finisher -n03343047 fin keel -n03343234 fipple -n03343354 fipple flute, fipple pipe, recorder, vertical flute -n03343560 fire -n03343737 fire alarm, smoke alarm -n03343853 firearm, piece, small-arm -n03344305 fire bell -n03344393 fireboat -n03344509 firebox -n03344642 firebrick -n03344784 fire control radar -n03344935 fire control system -n03345487 fire engine, fire truck -n03345837 fire extinguisher, extinguisher, asphyxiator -n03346135 fire iron -n03346289 fireman's ax, fireman's axe -n03346455 fireplace, hearth, open fireplace -n03347037 fire screen, fireguard -n03347472 fire tongs, coal tongs -n03347617 fire tower -n03348142 firewall -n03348868 firing chamber, gun chamber -n03349020 firing pin -n03349296 firkin -n03349367 firmer chisel -n03349469 first-aid kit -n03349599 first-aid station -n03349771 first base -n03349892 first class -n03350204 fishbowl, fish bowl, goldfish bowl -n03350352 fisherman's bend -n03350456 fisherman's knot, true lover's knot, truelove knot -n03350602 fisherman's lure, fish lure -n03351151 fishhook -n03351262 fishing boat, fishing smack, fishing vessel -n03351434 fishing gear, tackle, fishing tackle, fishing rig, rig -n03351979 fishing rod, fishing pole -n03352232 fish joint -n03352366 fish knife -n03352628 fishnet, fishing net -n03352961 fish slice -n03353281 fitment -n03353951 fixative -n03354207 fixer-upper -n03354903 flag -n03355468 flageolet, treble recorder, shepherd's pipe -n03355768 flagon -n03355925 flagpole, flagstaff -n03356038 flagship -n03356279 flail -n03356446 flambeau -n03356559 flamethrower -n03356858 flange, rim -n03356982 flannel -n03357081 flannel, gabardine, tweed, white -n03357267 flannelette -n03357716 flap, flaps -n03358172 flash, photoflash, flash lamp, flashgun, flashbulb, flash bulb -n03358380 flash -n03358726 flash camera -n03358841 flasher -n03359137 flashlight, torch -n03359285 flashlight battery -n03359436 flash memory -n03359566 flask -n03360133 flat arch, straight arch -n03360300 flatbed -n03360431 flatbed press, cylinder press -n03360622 flat bench -n03360731 flatcar, flatbed, flat -n03361109 flat file -n03361297 flatlet -n03361380 flat panel display, FPD -n03361550 flats -n03361683 flat tip screwdriver -n03362639 fleece -n03362771 fleet ballistic missile submarine -n03362890 fleur-de-lis, fleur-de-lys -n03363363 flight simulator, trainer -n03363549 flintlock -n03363749 flintlock, firelock -n03364008 flip-flop, thong -n03364156 flipper, fin -n03364599 float, plasterer's float -n03364937 floating dock, floating dry dock -n03365231 floatplane, pontoon plane -n03365374 flood, floodlight, flood lamp, photoflood -n03365592 floor, flooring -n03365991 floor, level, storey, story -n03366464 floor -n03366721 floorboard -n03366823 floor cover, floor covering -n03366974 floor joist -n03367059 floor lamp -n03367321 flophouse, dosshouse -n03367410 florist, florist shop, flower store -n03367545 floss -n03367875 flotsam, jetsam -n03367969 flour bin -n03368048 flour mill -n03368352 flowerbed, flower bed, bed of flowers -n03369276 flugelhorn, fluegelhorn -n03369407 fluid drive -n03369512 fluid flywheel -n03369866 flume -n03370387 fluorescent lamp -n03370646 fluoroscope, roentgenoscope -n03371875 flush toilet, lavatory -n03372029 flute, transverse flute -n03372549 flute, flute glass, champagne flute -n03372822 flux applicator -n03372933 fluxmeter -n03373237 fly -n03373611 flying boat -n03373943 flying buttress, arc-boutant -n03374102 flying carpet -n03374282 flying jib -n03374372 fly rod -n03374473 fly tent -n03374570 flytrap -n03374649 flywheel -n03374838 fob, watch chain, watch guard -n03375171 foghorn -n03375329 foglamp -n03375575 foil -n03376159 fold, sheepfold, sheep pen, sheepcote -n03376279 folder -n03376595 folding chair -n03376771 folding door, accordion door -n03376938 folding saw -n03378005 food court -n03378174 food processor -n03378342 food hamper -n03378442 foot -n03378593 footage -n03378765 football -n03379051 football helmet -n03379204 football stadium -n03379343 footbath -n03379719 foot brake -n03379828 footbridge, overcrossing, pedestrian bridge -n03379989 foothold, footing -n03380301 footlocker, locker -n03380647 foot rule -n03380724 footstool, footrest, ottoman, tuffet -n03380867 footwear, footgear -n03381126 footwear -n03381231 forceps -n03381450 force pump -n03381565 fore-and-after -n03381776 fore-and-aft sail -n03382104 forecastle, fo'c'sle -n03382292 forecourt -n03382413 foredeck -n03382533 fore edge, foredge -n03382708 foreground -n03382856 foremast -n03382969 fore plane -n03383099 foresail -n03383211 forestay -n03383378 foretop -n03383468 fore-topmast -n03383562 fore-topsail -n03383821 forge -n03384167 fork -n03384352 forklift -n03384891 formalwear, eveningwear, evening dress, evening clothes -n03385295 Formica -n03385557 fortification, munition -n03386011 fortress, fort -n03386343 forty-five -n03386544 Foucault pendulum -n03386726 foulard -n03386870 foul-weather gear -n03387323 foundation garment, foundation -n03387653 foundry, metalworks -n03388043 fountain -n03388183 fountain pen -n03388323 four-in-hand -n03388549 four-poster -n03388711 four-pounder -n03388990 four-stroke engine, four-stroke internal-combustion engine -n03389611 four-wheel drive, 4WD -n03389761 four-wheel drive, 4WD -n03389889 four-wheeler -n03389983 fowling piece -n03390075 foxhole, fox hole -n03390327 fragmentation bomb, antipersonnel bomb, anti-personnel bomb, daisy cutter -n03390673 frail -n03390786 fraise -n03390983 frame, framing -n03391301 frame -n03391613 frame buffer -n03391770 framework -n03392648 Francis turbine -n03392741 franking machine -n03393017 free house -n03393199 free-reed -n03393324 free-reed instrument -n03393761 freewheel -n03393912 freight car -n03394149 freight elevator, service elevator -n03394272 freight liner, liner train -n03394480 freight train, rattler -n03394649 French door -n03394916 French horn, horn -n03395256 French polish, French polish shellac -n03395401 French roof -n03395514 French window -n03395859 Fresnel lens -n03396074 fret -n03396580 friary -n03396654 friction clutch -n03396997 frieze -n03397087 frieze -n03397266 frigate -n03397412 frigate -n03397532 frill, flounce, ruffle, furbelow -n03397947 Frisbee -n03398153 frock -n03398228 frock coat -n03399579 frontlet, frontal -n03399677 front porch -n03399761 front projector -n03399971 fruit machine -n03400231 frying pan, frypan, skillet -n03400972 fuel filter -n03401129 fuel gauge, fuel indicator -n03401279 fuel injection, fuel injection system -n03401721 fuel system -n03402188 full-dress uniform -n03402369 full metal jacket -n03402511 full skirt -n03402785 fumigator -n03402941 funeral home, funeral parlor, funeral parlour, funeral chapel, funeral church, funeral-residence -n03403643 funnel -n03404012 funny wagon -n03404149 fur -n03404251 fur coat -n03404360 fur hat -n03404449 furnace -n03404900 furnace lining, refractory -n03405111 furnace room -n03405265 furnishing -n03405595 furnishing, trappings -n03405725 furniture, piece of furniture, article of furniture -n03406759 fur-piece -n03406966 furrow -n03407369 fuse, electrical fuse, safety fuse -n03407865 fusee drive, fusee -n03408054 fuselage -n03408264 fusil -n03408340 fustian -n03408444 futon -n03409297 gabardine -n03409393 gable, gable end, gable wall -n03409591 gable roof, saddle roof, saddleback, saddleback roof -n03409920 gadgetry -n03410022 gaff -n03410147 gaff -n03410303 gaff -n03410423 gaffsail, gaff-headed sail -n03410571 gaff topsail, fore-and-aft topsail -n03410740 gag, muzzle -n03410938 gaiter -n03411079 gaiter -n03411208 Galilean telescope -n03411339 galleon -n03411927 gallery -n03412058 gallery, art gallery, picture gallery -n03412220 galley, ship's galley, caboose, cookhouse -n03412387 galley -n03412511 galley -n03412906 gallows -n03413124 gallows tree, gallows-tree, gibbet, gallous -n03413264 galvanometer -n03413428 gambling house, gambling den, gambling hell, gaming house -n03413684 gambrel, gambrel roof -n03413828 game -n03414029 gamebag -n03414162 game equipment -n03414676 gaming table -n03415252 gamp, brolly -n03415486 gangplank, gangboard, gangway -n03415626 gangsaw -n03415749 gangway -n03415868 gantlet -n03416094 gantry, gauntry -n03416489 garage -n03416640 garage, service department -n03416775 Garand rifle, Garand, M-1, M-1 rifle -n03416900 garbage -n03417042 garbage truck, dustcart -n03417202 garboard, garboard plank, garboard strake -n03417345 garden -n03417749 garden -n03417970 garden rake -n03418158 garden spade -n03418242 garden tool, lawn tool -n03418402 garden trowel -n03418618 gargoyle -n03418749 garibaldi -n03418915 garlic press -n03419014 garment -n03420345 garment bag -n03420801 garrison cap, overseas cap -n03420935 garrote, garotte, garrotte, iron collar -n03421117 garter, supporter -n03421324 garter belt, suspender belt -n03421485 garter stitch -n03421669 gas guzzler -n03421768 gas shell -n03421960 gas bracket -n03422072 gas burner, gas jet -n03422484 gas-cooled reactor -n03422589 gas-discharge tube -n03422771 gas engine -n03423099 gas fixture -n03423224 gas furnace -n03423306 gas gun -n03423479 gas heater -n03423568 gas holder, gasometer -n03423719 gasket -n03423877 gas lamp -n03424204 gas maser -n03424325 gasmask, respirator, gas helmet -n03424489 gas meter, gasometer -n03424630 gasoline engine, petrol engine -n03424862 gasoline gauge, gasoline gage, gas gauge, gas gage, petrol gauge, petrol gage -n03425241 gas oven -n03425325 gas oven -n03425413 gas pump, gasoline pump, petrol pump, island dispenser -n03425595 gas range, gas stove, gas cooker -n03425769 gas ring -n03426134 gas tank, gasoline tank, petrol tank -n03426285 gas thermometer, air thermometer -n03426462 gastroscope -n03426574 gas turbine -n03426871 gas-turbine ship -n03427202 gat, rod -n03427296 gate -n03428090 gatehouse -n03428226 gateleg table -n03428349 gatepost -n03429003 gathered skirt -n03429137 Gatling gun -n03429288 gauge, gage -n03429682 gauntlet, gantlet -n03429771 gauntlet, gantlet, metal glove -n03429914 gauze, netting, veiling -n03430091 gauze, gauze bandage -n03430313 gavel -n03430418 gazebo, summerhouse -n03430551 gear, gear wheel, geared wheel, cogwheel -n03430959 gear, paraphernalia, appurtenance -n03431243 gear, gear mechanism -n03431570 gearbox, gear box, gear case -n03431745 gearing, gear, geartrain, power train, train -n03432061 gearset -n03432129 gearshift, gearstick, shifter, gear lever -n03432360 Geiger counter, Geiger-Muller counter -n03432509 Geiger tube, Geiger-Muller tube -n03433247 gene chip, DNA chip -n03433637 general-purpose bomb, GP bomb -n03433877 generator -n03434188 generator -n03434285 generator -n03434830 Geneva gown -n03435593 geodesic dome -n03435743 georgette -n03435991 gharry -n03436075 ghat -n03436182 ghetto blaster, boom box -n03436417 gift shop, novelty shop -n03436549 gift wrapping -n03436656 gig -n03436772 gig -n03436891 gig -n03436990 gig -n03437184 gildhall -n03437295 gill net -n03437430 gilt, gilding -n03437581 gimbal -n03437741 gingham -n03437829 girandole, girandola -n03437941 girder -n03438071 girdle, cincture, sash, waistband, waistcloth -n03438257 glass, drinking glass -n03438661 glass -n03438780 glass cutter -n03438863 glasses case -n03439348 glebe house -n03439631 Glengarry -n03439814 glider, sailplane -n03440216 Global Positioning System, GPS -n03440682 glockenspiel, orchestral bells -n03440876 glory hole, lazaretto -n03441112 glove -n03441345 glove compartment -n03441465 glow lamp -n03441582 glow tube -n03442288 glyptic art, glyptography -n03442487 glyptics, lithoglyptics -n03442597 gnomon -n03442756 goal -n03443005 goalmouth -n03443149 goalpost -n03443371 goblet -n03443543 godown -n03443912 goggles -n03444034 go-kart -n03445326 gold plate -n03445617 golf bag -n03445777 golf ball -n03445924 golfcart, golf cart -n03446070 golf club, golf-club, club -n03446268 golf-club head, club head, club-head, clubhead -n03446832 golf equipment -n03447075 golf glove -n03447358 golliwog, golliwogg -n03447447 gondola -n03447721 gong, tam-tam -n03447894 goniometer -n03448031 Gordian knot -n03448590 gorget -n03448696 gossamer -n03448956 Gothic arch -n03449217 gouache -n03449309 gouge -n03449451 gourd, calabash -n03449564 government building -n03449858 government office -n03450230 gown -n03450516 gown, robe -n03450734 gown, surgical gown, scrubs -n03450881 grab -n03450974 grab bag -n03451120 grab bar -n03451253 grace cup -n03451365 grade separation -n03451711 graduated cylinder -n03451798 graffito, graffiti -n03452267 gramophone, acoustic gramophone -n03452449 granary, garner -n03452594 grandfather clock, longcase clock -n03452741 grand piano, grand -n03453231 graniteware -n03453320 granny knot, granny -n03453443 grape arbor, grape arbour -n03454110 grapnel, grapnel anchor -n03454211 grapnel, grapple, grappler, grappling hook, grappling iron -n03454442 grass skirt -n03454536 grate, grating -n03454707 grate, grating -n03454885 grater -n03455355 graver, graving tool, pointel, pointrel -n03455488 gravestone, headstone, tombstone -n03455642 gravimeter, gravity meter -n03455802 gravure, photogravure, heliogravure -n03456024 gravy boat, gravy holder, sauceboat, boat -n03456186 grey, gray -n03456299 grease-gun, gun -n03456447 greasepaint -n03456548 greasy spoon -n03456665 greatcoat, overcoat, topcoat -n03457008 great hall -n03457451 greave, jambeau -n03457686 greengrocery -n03457902 greenhouse, nursery, glasshouse -n03458271 grenade -n03458422 grid, gridiron -n03459328 griddle -n03459591 grill, grille, grillwork -n03459775 grille, radiator grille -n03459914 grillroom, grill -n03460040 grinder -n03460147 grinding wheel, emery wheel -n03460297 grindstone -n03460455 gripsack -n03460899 gristmill -n03461288 grocery bag -n03461385 grocery store, grocery, food market, market -n03461651 grogram -n03461882 groined vault -n03461988 groover -n03462110 grosgrain -n03462315 gros point -n03462747 ground, earth -n03462972 ground bait -n03463185 ground control -n03463381 ground floor, first floor, ground level -n03463666 groundsheet, ground cloth -n03464053 G-string, thong -n03464467 guard, safety, safety device -n03464628 guard boat -n03464952 guardroom -n03465040 guardroom -n03465151 guard ship -n03465320 guard's van -n03465426 gueridon -n03465500 Guarnerius -n03465605 guesthouse -n03465718 guestroom -n03465818 guidance system, guidance device -n03466162 guided missile -n03466493 guided missile cruiser -n03466600 guided missile frigate -n03466839 guildhall -n03466947 guilloche -n03467068 guillotine -n03467254 guimpe -n03467380 guimpe -n03467517 guitar -n03467796 guitar pick -n03467887 gulag -n03467984 gun -n03468570 gunboat -n03468696 gun carriage -n03468821 gun case -n03469031 gun emplacement, weapons emplacement -n03469175 gun enclosure, gun turret, turret -n03469493 gunlock, firing mechanism -n03469832 gunnery -n03469903 gunnysack, gunny sack, burlap bag -n03470005 gun pendulum -n03470222 gun room -n03470387 gunsight, gun-sight -n03470629 gun trigger, trigger -n03470948 gurney -n03471030 gusher -n03471190 gusset, inset -n03471347 gusset, gusset plate -n03471779 guy, guy cable, guy wire, guy rope -n03472232 gymnastic apparatus, exerciser -n03472535 gym shoe, sneaker, tennis shoe -n03472672 gym suit -n03472796 gymslip -n03472937 gypsy cab -n03473078 gyrocompass -n03473227 gyroscope, gyro -n03473465 gyrostabilizer, gyrostabiliser -n03473817 habergeon -n03473966 habit -n03474167 habit, riding habit -n03474352 hacienda -n03474779 hacksaw, hack saw, metal saw -n03474896 haft, helve -n03475581 hairbrush -n03475674 haircloth, hair -n03475823 hairdressing, hair tonic, hair oil, hair grease -n03475961 hairnet -n03476083 hairpiece, false hair, postiche -n03476313 hairpin -n03476542 hair shirt -n03476684 hair slide -n03476991 hair spray -n03477143 hairspring -n03477303 hair trigger -n03477410 halberd -n03477512 half binding -n03477773 half hatchet -n03477902 half hitch -n03478589 half track -n03478756 hall -n03478907 hall -n03479121 hall -n03479266 Hall of Fame -n03479397 hall of residence -n03479502 hallstand -n03480579 halter -n03480719 halter, hackamore -n03480973 hame -n03481172 hammer -n03481521 hammer, power hammer -n03482001 hammer -n03482128 hammerhead -n03482252 hammock, sack -n03482405 hamper -n03482523 hand -n03482877 handball -n03483086 handbarrow -n03483230 handbell -n03483316 hand blower, blow dryer, blow drier, hair dryer, hair drier -n03483531 handbow -n03483637 hand brake, emergency, emergency brake, parking brake -n03483823 hand calculator, pocket calculator -n03483971 handcar -n03484083 handcart, pushcart, cart, go-cart -n03484487 hand cream -n03484576 handcuff, cuff, handlock, manacle -n03484809 hand drill, handheld drill -n03484931 hand glass, simple microscope, magnifying glass -n03485198 hand glass, hand mirror -n03485309 hand grenade -n03485407 hand-held computer, hand-held microcomputer -n03485575 handhold -n03485794 handkerchief, hankie, hanky, hankey -n03487090 handlebar -n03487331 handloom -n03487444 hand lotion -n03487533 hand luggage -n03487642 hand-me-down -n03487774 hand mower -n03487886 hand pump -n03488111 handrest -n03488188 handsaw, hand saw, carpenter's saw -n03488438 handset, French telephone -n03488603 hand shovel -n03488784 handspike -n03488887 handstamp, rubber stamp -n03489048 hand throttle -n03489162 hand tool -n03490006 hand towel, face towel -n03490119 hand truck, truck -n03490324 handwear, hand wear -n03490449 handwheel -n03490649 handwheel -n03490784 hangar queen -n03490884 hanger -n03491032 hang glider -n03491724 hangman's rope, hangman's halter, halter, hemp, hempen necktie -n03491988 hank -n03492087 hansom, hansom cab -n03492250 harbor, harbour -n03492542 hard disc, hard disk, fixed disk -n03492922 hard hat, tin hat, safety hat -n03493219 hardtop -n03493792 hardware, ironware -n03493911 hardware store, ironmonger, ironmonger's shop -n03494278 harmonica, mouth organ, harp, mouth harp -n03494537 harmonium, organ, reed organ -n03494706 harness -n03495039 harness -n03495258 harp -n03495570 harp -n03495671 harpoon -n03495941 harpoon gun -n03496183 harpoon log -n03496296 harpsichord, cembalo -n03496486 Harris Tweed -n03496612 harrow -n03496892 harvester, reaper -n03497100 hash house -n03497352 hasp -n03497657 hat, chapeau, lid -n03498441 hatbox -n03498536 hatch -n03498662 hatchback, hatchback door -n03498781 hatchback -n03498866 hatchel, heckle -n03498962 hatchet -n03499354 hatpin -n03499468 hauberk, byrnie -n03499907 Hawaiian guitar, steel guitar -n03500090 hawse, hawsehole, hawsepipe -n03500209 hawser -n03500295 hawser bend -n03500389 hay bale -n03500457 hayfork -n03500557 hayloft, haymow, mow -n03500699 haymaker, hay conditioner -n03500838 hayrack, hayrig -n03500971 hayrack -n03501152 hazard -n03501288 head -n03501520 head -n03501614 head -n03502200 headboard -n03502331 head covering, veil -n03502509 headdress, headgear -n03502777 header -n03502897 header -n03503097 header, coping, cope -n03503233 header, lintel -n03503358 headfast -n03503477 head gasket -n03503567 head gate -n03503718 headgear -n03503997 headlight, headlamp -n03504205 headpiece -n03504293 headpin, kingpin -n03504723 headquarters, central office, main office, home office, home base -n03505015 headrace -n03505133 headrest -n03505383 headsail -n03505504 headscarf -n03505667 headset -n03505764 head shop -n03506028 headstall, headpiece -n03506184 headstock -n03506370 health spa, spa, health club -n03506560 hearing aid, ear trumpet -n03506727 hearing aid, deaf-aid -n03506880 hearse -n03507241 hearth, fireside -n03507458 hearthrug -n03507658 heart-lung machine -n03507963 heat engine -n03508101 heater, warmer -n03508485 heat exchanger -n03508881 heating pad, hot pad -n03509394 heat lamp, infrared lamp -n03509608 heat pump -n03509843 heat-seeking missile -n03510072 heat shield -n03510244 heat sink -n03510384 heaume -n03510487 heaver -n03510583 heavier-than-air craft -n03510866 heckelphone, basset oboe -n03510987 hectograph, heliotype -n03511175 hedge, hedgerow -n03511333 hedge trimmer -n03512030 helicon, bombardon -n03512147 helicopter, chopper, whirlybird, eggbeater -n03512452 heliograph -n03512624 heliometer -n03512911 helm -n03513137 helmet -n03513376 helmet -n03514129 hematocrit, haematocrit -n03514340 hemming-stitch -n03514451 hemostat, haemostat -n03514693 hemstitch, hemstitching -n03514894 henroost -n03515338 heraldry -n03515934 hermitage -n03516266 herringbone -n03516367 herringbone, herringbone pattern -n03516647 Herschelian telescope, off-axis reflector -n03516844 Hessian boot, hessian, jackboot, Wellington, Wellington boot -n03516996 heterodyne receiver, superheterodyne receiver, superhet -n03517509 hibachi -n03517647 hideaway, retreat -n03517760 hi-fi, high fidelity sound system -n03517899 high altar -n03517982 high-angle gun -n03518135 highball glass -n03518230 highboard -n03518305 highboy, tallboy -n03518445 highchair, feeding chair -n03518631 high gear, high -n03518829 high-hat cymbal, high hat -n03518943 highlighter -n03519081 highlighter -n03519226 high-pass filter -n03519387 high-rise, tower block -n03519674 high table -n03519848 high-warp loom -n03520493 hijab -n03521076 hinge, flexible joint -n03521431 hinging post, swinging post -n03521544 hip boot, thigh boot -n03521675 hipflask, pocket flask -n03521771 hip pad -n03521899 hip pocket -n03522003 hippodrome -n03522100 hip roof, hipped roof -n03522634 hitch -n03522863 hitch -n03522990 hitching post -n03523134 hitchrack, hitching bar -n03523398 hob -n03523506 hobble skirt -n03523987 hockey skate -n03524150 hockey stick -n03524287 hod -n03524425 hodoscope -n03524574 hoe -n03524745 hoe handle -n03524976 hogshead -n03525074 hoist -n03525252 hold, keep -n03525454 holder -n03525693 holding cell -n03525827 holding device -n03526062 holding pen, holding paddock, holding yard -n03527149 hollowware, holloware -n03527444 holster -n03527565 holster -n03527675 holy of holies, sanctum sanctorum -n03528100 home, nursing home, rest home -n03528263 home appliance, household appliance -n03528523 home computer -n03528901 home plate, home base, home, plate -n03529175 home room, homeroom -n03529444 homespun -n03529629 homestead -n03529860 home theater, home theatre -n03530189 homing torpedo -n03530511 hone -n03530642 honeycomb -n03530910 hood, bonnet, cowl, cowling -n03531281 hood -n03531447 hood -n03531546 hood, exhaust hood -n03531691 hood -n03531982 hood latch -n03532342 hook -n03532672 hook, claw -n03532919 hook -n03533014 hookah, narghile, nargileh, sheesha, shisha, chicha, calean, kalian, water pipe, hubble-bubble, hubbly-bubbly -n03533392 hook and eye -n03533486 hookup, assemblage -n03533654 hookup -n03533845 hook wrench, hook spanner -n03534580 hoopskirt, crinoline -n03534695 hoosegow, hoosgow -n03534776 Hoover -n03535024 hope chest, wedding chest -n03535284 hopper -n03535647 hopsacking, hopsack -n03535780 horizontal bar, high bar -n03536122 horizontal stabilizer, horizontal stabiliser, tailplane -n03536568 horizontal tail -n03536761 horn -n03537085 horn -n03537241 horn -n03537412 horn button -n03537550 hornpipe, pibgorn, stockhorn -n03538037 horse, gymnastic horse -n03538179 horsebox -n03538300 horsecar -n03538406 horse cart, horse-cart -n03538542 horsecloth -n03538634 horse-drawn vehicle -n03538817 horsehair -n03538957 horsehair wig -n03539103 horseless carriage -n03539293 horse pistol, horse-pistol -n03539433 horseshoe, shoe -n03539546 horseshoe -n03539678 horse-trail -n03539754 horsewhip -n03540090 hose -n03540267 hosiery, hose -n03540476 hospice -n03540595 hospital, infirmary -n03540914 hospital bed -n03541091 hospital room -n03541269 hospital ship -n03541393 hospital train -n03541537 hostel, youth hostel, student lodging -n03541696 hostel, hostelry, inn, lodge, auberge -n03541923 hot-air balloon -n03542333 hotel -n03542605 hotel-casino, casino-hotel -n03542727 hotel-casino, casino-hotel -n03542860 hotel room -n03543012 hot line -n03543112 hot pants -n03543254 hot plate, hotplate -n03543394 hot rod, hot-rod -n03543511 hot spot, hotspot -n03543603 hot tub -n03543735 hot-water bottle, hot-water bag -n03543945 houndstooth check, hound's-tooth check, dogstooth check, dogs-tooth check, dog's-tooth check -n03544143 hourglass -n03544238 hour hand, little hand -n03544360 house -n03545150 house -n03545470 houseboat -n03545585 houselights -n03545756 house of cards, cardhouse, card-house, cardcastle -n03545961 house of correction -n03546112 house paint, housepaint -n03546235 housetop -n03546340 housing, lodging, living accommodations -n03547054 hovel, hut, hutch, shack, shanty -n03547229 hovercraft, ground-effect machine -n03547397 howdah, houdah -n03547530 huarache, huaraches -n03547861 hub-and-spoke, hub-and-spoke system -n03548086 hubcap -n03548195 huck, huckaback -n03548320 hug-me-tight -n03548402 hula-hoop -n03548533 hulk -n03548626 hull -n03548930 humeral veil, veil -n03549199 Humvee, Hum-Vee -n03549350 hunter, hunting watch -n03549473 hunting knife -n03549589 hurdle -n03549732 hurricane deck, hurricane roof, promenade deck, awning deck -n03549897 hurricane lamp, hurricane lantern, tornado lantern, storm lantern, storm lamp -n03550153 hut, army hut, field hut -n03550289 hutch -n03550420 hutment -n03551084 hydraulic brake, hydraulic brakes -n03551395 hydraulic press -n03551582 hydraulic pump, hydraulic ram -n03551790 hydraulic system -n03552001 hydraulic transmission, hydraulic transmission system -n03552449 hydroelectric turbine -n03552749 hydrofoil, hydroplane -n03553019 hydrofoil, foil -n03553248 hydrogen bomb, H-bomb, fusion bomb, thermonuclear bomb -n03553486 hydrometer, gravimeter -n03554375 hygrodeik -n03554460 hygrometer -n03554645 hygroscope -n03555006 hyperbaric chamber -n03555217 hypercoaster -n03555426 hypermarket -n03555564 hypodermic needle -n03555662 hypodermic syringe, hypodermic, hypo -n03555862 hypsometer -n03555996 hysterosalpingogram -n03556173 I-beam -n03556679 ice ax, ice axe, piolet -n03556811 iceboat, ice yacht, scooter -n03556992 icebreaker, iceboat -n03557270 iced-tea spoon -n03557360 ice hockey rink, ice-hockey rink -n03557590 ice machine -n03557692 ice maker -n03557840 ice pack, ice bag -n03558007 icepick, ice pick -n03558176 ice rink, ice-skating rink, ice -n03558404 ice skate -n03558633 ice tongs -n03558739 icetray -n03559373 iconoscope -n03559531 Identikit, Identikit picture -n03559999 idle pulley, idler pulley, idle wheel -n03560430 igloo, iglu -n03560860 ignition coil -n03561047 ignition key -n03561169 ignition switch -n03561573 imaret -n03562565 immovable bandage -n03563200 impact printer -n03563460 impeller -n03563710 implant -n03563967 implement -n03564849 impression -n03565288 imprint -n03565565 improvised explosive device, I.E.D., IED -n03565710 impulse turbine -n03565830 in-basket, in-tray -n03565991 incendiary bomb, incendiary, firebomb -n03566193 incinerator -n03566329 inclined plane -n03566555 inclinometer, dip circle -n03566730 inclinometer -n03566860 incrustation, encrustation -n03567066 incubator, brooder -n03567635 index register -n03567788 Indiaman -n03567912 Indian club -n03568117 indicator -n03568818 induction coil -n03569014 inductor, inductance -n03569174 industrial watercourse -n03569293 inertial guidance system, inertial navigation system -n03569494 inflater, inflator -n03571280 inhaler, inhalator -n03571439 injector -n03571625 ink bottle, inkpot -n03571853 ink eraser -n03571942 ink-jet printer -n03572107 inkle -n03572205 inkstand -n03572321 inkwell, inkstand -n03572631 inlay -n03573574 inside caliper -n03573848 insole, innersole -n03574243 instep -n03574416 instillator -n03574555 institution -n03574816 instrument -n03575958 instrument of punishment -n03576215 instrument of torture -n03576443 intaglio, diaglyph -n03576955 intake valve -n03577090 integrated circuit, microcircuit -n03577312 integrator, planimeter -n03577474 Intelnet -n03577672 interceptor -n03577818 interchange -n03578055 intercommunication system, intercom -n03578251 intercontinental ballistic missile, ICBM -n03578656 interface, port -n03578981 interferometer -n03579538 interior door -n03579982 internal-combustion engine, ICE -n03580518 internal drive -n03580615 internet, net, cyberspace -n03580845 interphone -n03580990 interrupter -n03581125 intersection, crossroad, crossway, crossing, carrefour -n03581531 interstice -n03581897 intraocular lens -n03582508 intravenous pyelogram, IVP -n03582959 inverter -n03583419 ion engine -n03583621 ionization chamber, ionization tube -n03584254 iPod -n03584400 video iPod -n03584829 iron, smoothing iron -n03585073 iron -n03585337 iron, branding iron -n03585438 irons, chains -n03585551 ironclad -n03585682 iron foundry -n03585778 iron horse -n03585875 ironing -n03586219 iron lung -n03586631 ironmongery -n03586911 ironworks -n03587205 irrigation ditch -n03588216 izar -n03588841 jabot -n03588951 jack -n03589313 jack, jackstones -n03589513 jack -n03589672 jack -n03589791 jacket -n03590306 jacket -n03590475 jacket -n03590588 jack-in-the-box -n03590841 jack-o'-lantern -n03590932 jack plane -n03591116 Jacob's ladder, jack ladder, pilot ladder -n03591313 jaconet -n03591592 Jacquard loom, Jacquard -n03591798 jacquard -n03591901 jag, dag -n03592245 jail, jailhouse, gaol, clink, slammer, poky, pokey -n03592669 jalousie -n03592773 jamb -n03592931 jammer -n03593122 jampot, jamjar -n03593222 japan -n03593526 jar -n03593862 Jarvik heart, Jarvik artificial heart -n03594010 jaunting car, jaunty car -n03594148 javelin -n03594277 jaw -n03594523 Jaws of Life -n03594734 jean, blue jean, denim -n03594945 jeep, landrover -n03595055 jellaba -n03595264 jerkin -n03595409 jeroboam, double-magnum -n03595523 jersey -n03595614 jersey, T-shirt, tee shirt -n03595860 jet, jet plane, jet-propelled plane -n03596099 jet bridge -n03596285 jet engine -n03596543 jetliner -n03597147 jeweler's glass -n03597317 jewelled headdress, jeweled headdress -n03597916 jew's harp, jews' harp, mouth bow -n03598151 jib -n03598299 jibboom -n03598385 jig -n03598515 jig -n03598646 jiggermast, jigger -n03598783 jigsaw, scroll saw, fretsaw -n03598930 jigsaw puzzle -n03599486 jinrikisha, ricksha, rickshaw -n03599964 jobcentre -n03600285 jodhpurs, jodhpur breeches, riding breeches -n03600475 jodhpur, jodhpur boot, jodhpur shoe -n03600722 joinery -n03600977 joint -n03601442 Joint Direct Attack Munition, JDAM -n03601638 jointer, jointer plane, jointing plane, long plane -n03601840 joist -n03602081 jolly boat, jolly -n03602194 jorum -n03602365 joss house -n03602686 journal bearing -n03602790 journal box -n03602883 joystick -n03603442 jungle gym -n03603594 junk -n03603722 jug -n03604156 jukebox, nickelodeon -n03604311 jumbojet, jumbo jet -n03604400 jumper, pinafore, pinny -n03604536 jumper -n03604629 jumper -n03604763 jumper -n03604843 jumper cable, jumper lead, lead, booster cable -n03605417 jump seat -n03605504 jump suit -n03605598 jump suit, jumpsuit -n03605722 junction -n03605915 junction, conjunction -n03606106 junction barrier, barrier strip -n03606251 junk shop -n03606347 jury box -n03606465 jury mast -n03607029 kachina -n03607186 kaffiyeh -n03607527 kalansuwa -n03607659 Kalashnikov -n03607923 kameez -n03608504 kanzu -n03609147 katharometer -n03609235 kayak -n03609397 kazoo -n03609542 keel -n03609786 keelboat -n03609959 keelson -n03610098 keep, donjon, dungeon -n03610418 keg -n03610524 kennel, doghouse, dog house -n03610682 kepi, peaked cap, service cap, yachting cap -n03610836 keratoscope -n03610992 kerchief -n03612010 ketch -n03612814 kettle, boiler -n03612965 kettle, kettledrum, tympanum, tympani, timpani -n03613294 key -n03613592 key -n03614007 keyboard -n03614383 keyboard buffer -n03614532 keyboard instrument -n03614782 keyhole -n03614887 keyhole saw -n03615300 khadi, khaddar -n03615406 khaki -n03615563 khakis -n03615655 khimar -n03615790 khukuri -n03616091 kick pleat -n03616225 kicksorter, pulse height analyzer -n03616428 kickstand -n03616763 kick starter, kick start -n03616979 kid glove, suede glove -n03617095 kiln -n03617312 kilt -n03617480 kimono -n03617594 kinescope, picture tube, television tube -n03617834 Kinetoscope -n03618101 king -n03618339 king -n03618546 kingbolt, kingpin, swivel pin -n03618678 king post -n03618797 Kipp's apparatus -n03618982 kirk -n03619050 kirpan -n03619196 kirtle -n03619275 kirtle -n03619396 kit, outfit -n03619650 kit -n03619793 kitbag, kit bag -n03619890 kitchen -n03620052 kitchen appliance -n03620353 kitchenette -n03620967 kitchen table -n03621049 kitchen utensil -n03621377 kitchenware -n03621694 kite balloon -n03622058 klaxon, claxon -n03622401 klieg light -n03622526 klystron -n03622839 knee brace -n03622931 knee-high, knee-hi -n03623198 knee pad -n03623338 knee piece -n03623556 knife -n03624134 knife -n03624400 knife blade -n03624767 knight, horse -n03625355 knit -n03625539 knitting machine -n03625646 knitting needle -n03625943 knitwear -n03626115 knob, boss -n03626272 knob, pommel -n03626418 knobble -n03626502 knobkerrie, knobkerry -n03626760 knocker, doorknocker, rapper -n03627232 knot -n03627954 knuckle joint, hinge joint -n03628071 kohl -n03628215 koto -n03628421 kraal -n03628511 kremlin -n03628728 kris, creese, crease -n03628831 krummhorn, crumhorn, cromorne -n03628984 Kundt's tube -n03629100 Kurdistan -n03629231 kurta -n03629520 kylix, cylix -n03629643 kymograph, cymograph -n03630262 lab bench, laboratory bench -n03630383 lab coat, laboratory coat -n03631177 lace -n03631811 lacquer -n03631922 lacquerware -n03632100 lacrosse ball -n03632577 ladder-back -n03632729 ladder-back, ladder-back chair -n03632852 ladder truck, aerial ladder truck -n03632963 ladies' room, powder room -n03633091 ladle -n03633341 lady chapel -n03633632 lagerphone -n03633886 lag screw, lag bolt -n03634034 lake dwelling, pile dwelling -n03634899 lally, lally column -n03635032 lamasery -n03635108 lambrequin -n03635330 lame -n03635516 laminar flow clean room -n03635668 laminate -n03635932 lamination -n03636248 lamp -n03636649 lamp -n03637027 lamp house, lamphouse, lamp housing -n03637181 lamppost -n03637318 lampshade, lamp shade -n03637480 lanai -n03637787 lancet arch, lancet -n03637898 lancet window -n03638014 landau -n03638180 lander -n03638623 landing craft -n03638743 landing flap -n03638883 landing gear -n03639077 landing net -n03639230 landing skid -n03639497 land line, landline -n03639675 land mine, ground-emplaced mine, booby trap -n03639880 land office -n03640850 lanolin -n03640988 lantern -n03641569 lanyard, laniard -n03641947 lap, lap covering -n03642144 laparoscope -n03642341 lapboard -n03642444 lapel -n03642573 lap joint, splice -n03642806 laptop, laptop computer -n03643149 laryngoscope -n03643253 laser, optical maser -n03643491 laser-guided bomb, LGB -n03643737 laser printer -n03643907 lash, thong -n03644073 lashing -n03644378 lasso, lariat, riata, reata -n03644858 latch -n03645011 latch, door latch -n03645168 latchet -n03645290 latchkey -n03645577 lateen, lateen sail -n03646020 latex paint, latex, rubber-base paint -n03646148 lath -n03646296 lathe -n03646809 latrine -n03646916 lattice, latticework, fretwork -n03647423 launch -n03647520 launcher, rocket launcher -n03648219 laundry, wash, washing, washables -n03648431 laundry cart -n03648667 laundry truck -n03649003 lavalava -n03649161 lavaliere, lavalier, lavalliere -n03649288 laver -n03649674 lawn chair, garden chair -n03649797 lawn furniture -n03649909 lawn mower, mower -n03650551 layette -n03651388 lead-acid battery, lead-acid accumulator -n03651605 lead-in -n03651843 leading rein -n03652100 lead pencil -n03652389 leaf spring -n03652729 lean-to -n03652826 lean-to tent -n03652932 leash, tether, lead -n03653110 leatherette, imitation leather -n03653220 leather strip -n03653454 Leclanche cell -n03653583 lectern, reading desk -n03653740 lecture room -n03653833 lederhosen -n03653975 ledger board -n03654576 leg -n03654826 leg -n03655072 legging, leging, leg covering -n03655470 Leiden jar, Leyden jar -n03655720 leisure wear -n03656484 lens, lense, lens system -n03656957 lens, electron lens -n03657121 lens cap, lens cover -n03657239 lens implant, interocular lens implant, IOL -n03657511 leotard, unitard, body suit, cat suit -n03658102 letter case -n03658185 letter opener, paper knife, paperknife -n03658635 levee -n03658858 level, spirit level -n03659292 lever -n03659686 lever, lever tumbler -n03659809 lever -n03659950 lever lock -n03660124 Levi's, levis -n03660562 Liberty ship -n03660909 library -n03661043 library -n03661340 lid -n03662301 Liebig condenser -n03662452 lie detector -n03662601 lifeboat -n03662719 life buoy, lifesaver, life belt, life ring -n03662887 life jacket, life vest, cork jacket -n03663433 life office -n03663531 life preserver, preserver, flotation device -n03663910 life-support system, life support -n03664159 life-support system, life support -n03664675 lifting device -n03664840 lift pump -n03664943 ligament -n03665232 ligature -n03665366 light, light source -n03665851 light arm -n03665924 light bulb, lightbulb, bulb, incandescent lamp, electric light, electric-light bulb -n03666238 light circuit, lighting circuit -n03666362 light-emitting diode, LED -n03666591 lighter, light, igniter, ignitor -n03666917 lighter-than-air craft -n03667060 light filter, diffusing screen -n03667235 lighting -n03667552 light machine gun -n03667664 light meter, exposure meter, photometer -n03667829 light microscope -n03668067 lightning rod, lightning conductor -n03668279 light pen, electronic stylus -n03668488 lightship -n03668803 Lilo -n03669245 limber -n03669534 limekiln -n03669886 limiter, clipper -n03670208 limousine, limo -n03671914 linear accelerator, linac -n03672521 linen -n03672827 line printer, line-at-a-time printer -n03673027 liner, ocean liner -n03673270 liner, lining -n03673450 lingerie, intimate apparel -n03673767 lining, liner -n03674270 link, data link -n03674440 linkage -n03674731 Link trainer -n03674842 linocut -n03675076 linoleum knife, linoleum cutter -n03675235 Linotype, Linotype machine -n03675445 linsey-woolsey -n03675558 linstock -n03675907 lion-jaw forceps -n03676087 lip-gloss -n03676483 lipstick, lip rouge -n03676623 liqueur glass -n03676759 liquid crystal display, LCD -n03677115 liquid metal reactor -n03677682 lisle -n03677766 lister, lister plow, lister plough, middlebreaker, middle buster -n03678558 litterbin, litter basket, litter-basket -n03678729 little theater, little theatre -n03678879 live axle, driving axle -n03679384 living quarters, quarters -n03679712 living room, living-room, sitting room, front room, parlor, parlour -n03680248 load -n03680355 Loafer -n03680512 loaner -n03680734 lobe -n03680858 lobster pot -n03680942 local -n03681477 local area network, LAN -n03681813 local oscillator, heterodyne oscillator -n03682380 Lochaber ax -n03682487 lock -n03682877 lock, ignition lock -n03683079 lock, lock chamber -n03683341 lock -n03683457 lockage -n03683606 locker -n03683708 locker room -n03683995 locket -n03684143 lock-gate -n03684224 locking pliers -n03684489 lockring, lock ring, lock washer -n03684611 lockstitch -n03684740 lockup -n03684823 locomotive, engine, locomotive engine, railway locomotive -n03685307 lodge, indian lodge -n03685486 lodge, hunting lodge -n03685640 lodge -n03685820 lodging house, rooming house -n03686130 loft, attic, garret -n03686363 loft, pigeon loft -n03686470 loft -n03686924 log cabin -n03687137 loggia -n03687928 longbow -n03688066 long iron -n03688192 long johns -n03688405 long sleeve -n03688504 long tom -n03688605 long trousers, long pants -n03688707 long underwear, union suit -n03688832 looking glass, glass -n03688943 lookout, observation tower, lookout station, observatory -n03689157 loom -n03689570 loop knot -n03690168 lorgnette -n03690279 Lorraine cross, cross of Lorraine -n03690473 lorry, camion -n03690851 lota -n03690938 lotion -n03691459 loudspeaker, speaker, speaker unit, loudspeaker system, speaker system -n03691817 lounge, waiting room, waiting area -n03692004 lounger -n03692136 lounging jacket, smoking jacket -n03692272 lounging pajama, lounging pyjama -n03692379 loungewear -n03692522 loupe, jeweler's loupe -n03692842 louvered window, jalousie -n03693293 love knot, lovers' knot, lover's knot, true lovers' knot, true lover's knot -n03693474 love seat, loveseat, tete-a-tete, vis-a-vis -n03693707 loving cup -n03693860 lowboy -n03694196 low-pass filter -n03694356 low-warp-loom -n03694639 LP, L-P -n03694761 L-plate -n03694949 lubber's hole -n03695122 lubricating system, force-feed lubricating system, force feed, pressure-feed lubricating system, pressure feed -n03695452 luff -n03695616 lug -n03695753 luge -n03695857 Luger -n03695957 luggage carrier -n03696065 luggage compartment, automobile trunk, trunk -n03696301 luggage rack, roof rack -n03696445 lugger -n03696568 lugsail, lug -n03696746 lug wrench -n03696909 lumberjack, lumber jacket -n03697007 lumbermill, sawmill -n03697366 lunar excursion module, lunar module, LEM -n03697552 lunchroom -n03697812 lunette -n03697913 lungi, lungyi, longyi -n03698123 lunula -n03698226 lusterware -n03698360 lute -n03698604 luxury liner, express luxury liner -n03698723 lyceum -n03698815 lychgate, lichgate -n03699280 lyre -n03699591 machete, matchet, panga -n03699754 machicolation -n03699975 machine -n03700963 machine, simple machine -n03701191 machine bolt -n03701391 machine gun -n03701640 machinery -n03701790 machine screw -n03702248 machine tool -n03702440 machinist's vise, metalworking vise -n03702582 machmeter -n03703075 mackinaw -n03703203 mackinaw, Mackinaw boat -n03703463 mackinaw, Mackinaw coat -n03703590 mackintosh, macintosh -n03703730 macrame -n03703862 madras -n03703945 Mae West, air jacket -n03704549 magazine rack -n03704834 magic lantern -n03705379 magnet -n03705808 magnetic bottle -n03706229 magnetic compass -n03706415 magnetic core memory, core memory -n03706653 magnetic disk, magnetic disc, disk, disc -n03706939 magnetic head -n03707171 magnetic mine -n03707372 magnetic needle -n03707597 magnetic recorder -n03707766 magnetic stripe -n03708036 magnetic tape, mag tape, tape -n03708425 magneto, magnetoelectric machine -n03708843 magnetometer, gaussmeter -n03708962 magnetron -n03709206 magnifier -n03709363 magnum -n03709545 magnus hitch -n03709644 mail -n03709823 mailbag, postbag -n03709960 mailbag, mail pouch -n03710079 mailboat, mail boat, packet, packet boat -n03710193 mailbox, letter box -n03710294 mail car -n03710421 maildrop -n03710528 mailer -n03710637 maillot -n03710721 maillot, tank suit -n03710937 mailsorter -n03711044 mail train -n03711711 mainframe, mainframe computer -n03711999 mainmast -n03712111 main rotor -n03712337 mainsail -n03712444 mainspring -n03712887 main-topmast -n03712981 main-topsail -n03713069 main yard -n03713151 maisonette, maisonnette -n03713436 majolica, maiolica -n03714235 makeup, make-up, war paint -n03715114 Maksutov telescope -n03715275 malacca, malacca cane -n03715386 mallet, beetle -n03715669 mallet, hammer -n03715892 mallet -n03716228 mammogram -n03716887 mandola -n03716966 mandolin -n03717131 manger, trough -n03717285 mangle -n03717447 manhole -n03717622 manhole cover -n03718212 man-of-war, ship of the line -n03718335 manometer -n03718458 manor, manor house -n03718581 manor hall, hall -n03718699 MANPAD -n03718789 mansard, mansard roof -n03718935 manse -n03719053 mansion, mansion house, manse, hall, residence -n03719343 mantel, mantelpiece, mantle, mantlepiece, chimneypiece -n03719560 mantelet, mantilla -n03719743 mantilla -n03720005 Mao jacket -n03720163 map -n03720665 maquiladora -n03720891 maraca -n03721047 marble -n03721252 marching order -n03721384 marimba, xylophone -n03721590 marina -n03722007 marker -n03722288 marketplace, market place, mart, market -n03722646 marlinespike, marlinspike, marlingspike -n03722944 marocain, crepe marocain -n03723153 marquee, marquise -n03723267 marquetry, marqueterie -n03723439 marriage bed -n03723781 martello tower -n03723885 martingale -n03724066 mascara -n03724176 maser -n03724417 masher -n03724538 mashie, five iron -n03724623 mashie niblick, seven iron -n03724756 masjid, musjid -n03724870 mask -n03725035 mask -n03725506 Masonite -n03725600 Mason jar -n03725717 masonry -n03725869 mason's level -n03726116 massage parlor -n03726233 massage parlor -n03726371 mass spectrograph -n03726516 mass spectrometer, spectrometer -n03726760 mast -n03726993 mast -n03727067 mastaba, mastabah -n03727465 master bedroom -n03727605 masterpiece, chef-d'oeuvre -n03727837 mat -n03727946 mat, gym mat -n03728437 match, lucifer, friction match -n03728982 match -n03729131 matchboard -n03729308 matchbook -n03729402 matchbox -n03729482 matchlock -n03729647 match plane, tonguing and grooving plane -n03729826 matchstick -n03729951 material -n03730153 materiel, equipage -n03730334 maternity hospital -n03730494 maternity ward -n03730655 matrix -n03730788 Matthew Walker, Matthew Walker knot -n03730893 matting -n03731019 mattock -n03731483 mattress cover -n03731695 maul, sledge, sledgehammer -n03731882 maulstick, mahlstick -n03732020 Mauser -n03732114 mausoleum -n03732458 maxi -n03732543 Maxim gun -n03732658 maximum and minimum thermometer -n03733131 maypole -n03733281 maze, labyrinth -n03733465 mazer -n03733547 means -n03733644 measure -n03733805 measuring cup -n03733925 measuring instrument, measuring system, measuring device -n03735637 measuring stick, measure, measuring rod -n03735963 meat counter -n03736064 meat grinder -n03736147 meat hook -n03736269 meat house -n03736372 meat safe -n03736470 meat thermometer -n03736970 mechanical device -n03738066 mechanical piano, Pianola, player piano -n03738241 mechanical system -n03738472 mechanism -n03739518 medical building, health facility, healthcare facility -n03739693 medical instrument -n03742019 medicine ball -n03742115 medicine chest, medicine cabinet -n03742238 MEDLINE -n03743016 megalith, megalithic structure -n03743279 megaphone -n03743902 memorial, monument -n03744276 memory, computer memory, storage, computer storage, store, memory board -n03744684 memory chip -n03744840 memory device, storage device -n03745146 menagerie, zoo, zoological garden -n03745487 mending -n03745571 menhir, standing stone -n03746005 menorah -n03746155 Menorah -n03746330 man's clothing -n03746486 men's room, men's -n03748162 mercantile establishment, retail store, sales outlet, outlet -n03749504 mercury barometer -n03749634 mercury cell -n03749807 mercury thermometer, mercury-in-glass thermometer -n03750206 mercury-vapor lamp -n03750437 mercy seat -n03750614 merlon -n03751065 mess, mess hall -n03751269 mess jacket, monkey jacket, shell jacket -n03751458 mess kit -n03751590 messuage -n03751757 metal detector -n03752071 metallic -n03752185 metal screw -n03752398 metal wood -n03752922 meteorological balloon -n03753077 meter -n03753514 meterstick, metrestick -n03757604 metronome -n03758089 mezzanine, mezzanine floor, entresol -n03758220 mezzanine, first balcony -n03758894 microbalance -n03758992 microbrewery -n03759243 microfiche -n03759432 microfilm -n03759661 micrometer, micrometer gauge, micrometer caliper -n03759954 microphone, mike -n03760310 microprocessor -n03760671 microscope -n03760944 microtome -n03761084 microwave, microwave oven -n03761588 microwave diathermy machine -n03761731 microwave linear accelerator -n03762238 middy, middy blouse -n03762332 midiron, two iron -n03762434 mihrab -n03762602 mihrab -n03762982 military hospital -n03763727 military quarters -n03763968 military uniform -n03764276 military vehicle -n03764606 milk bar -n03764736 milk can -n03764822 milk float -n03764995 milking machine -n03765128 milking stool -n03765467 milk wagon, milkwagon -n03765561 mill, grinder, milling machinery -n03765934 milldam -n03766044 miller, milling machine -n03766218 milliammeter -n03766322 millinery, woman's hat -n03766508 millinery, hat shop -n03766600 milling -n03766697 millivoltmeter -n03766935 millstone -n03767112 millstone -n03767203 millwheel, mill wheel -n03767459 mimeograph, mimeo, mimeograph machine, Roneo, Roneograph -n03767745 minaret -n03767966 mincer, mincing machine -n03768132 mine -n03768683 mine detector -n03768823 minelayer -n03768916 mineshaft -n03769610 minibar, cellaret -n03769722 minibike, motorbike -n03769881 minibus -n03770085 minicar -n03770224 minicomputer -n03770316 ministry -n03770439 miniskirt, mini -n03770520 minisub, minisubmarine -n03770679 minivan -n03770834 miniver -n03770954 mink, mink coat -n03772077 minster -n03772269 mint -n03772584 minute hand, big hand -n03772674 Minuteman -n03773035 mirror -n03773504 missile -n03773835 missile defense system, missile defence system -n03774327 miter box, mitre box -n03774461 miter joint, mitre joint, miter, mitre -n03775071 mitten -n03775199 mixer -n03775388 mixer -n03775546 mixing bowl -n03775636 mixing faucet -n03775747 mizzen, mizen -n03775847 mizzenmast, mizenmast, mizzen, mizen -n03776167 mobcap -n03776460 mobile home, manufactured home -n03776877 moccasin, mocassin -n03776997 mock-up -n03777126 mod con -n03777568 Model T -n03777754 modem -n03778459 modillion -n03778817 module -n03779000 module -n03779128 mohair -n03779246 moire, watered-silk -n03779370 mold, mould, cast -n03779884 moldboard, mouldboard -n03780047 moldboard plow, mouldboard plough -n03780799 moleskin -n03781055 Molotov cocktail, petrol bomb, gasoline bomb -n03781244 monastery -n03781467 monastic habit -n03781594 moneybag -n03781683 money belt -n03781787 monitor -n03782006 monitor -n03782190 monitor, monitoring device -n03782794 monkey-wrench, monkey wrench -n03782929 monk's cloth -n03783304 monochrome -n03783430 monocle, eyeglass -n03783575 monofocal lens implant, monofocal IOL -n03783873 monoplane -n03784139 monotype -n03784270 monstrance, ostensorium -n03784793 mooring tower, mooring mast -n03784896 Moorish arch, horseshoe arch -n03785016 moped -n03785142 mop handle -n03785237 moquette -n03785499 morgue, mortuary, dead room -n03785721 morion, cabasset -n03786096 morning dress -n03786194 morning dress -n03786313 morning room -n03786621 Morris chair -n03786715 mortar, howitzer, trench mortar -n03786901 mortar -n03787032 mortarboard -n03787523 mortise joint, mortise-and-tenon joint -n03788047 mosaic -n03788195 mosque -n03788365 mosquito net -n03788498 motel -n03788601 motel room -n03788914 Mother Hubbard, muumuu -n03789171 motion-picture camera, movie camera, cine-camera -n03789400 motion-picture film, movie film, cine-film -n03789603 motley -n03789794 motley -n03789946 motor -n03790230 motorboat, powerboat -n03790512 motorcycle, bike -n03790755 motor hotel, motor inn, motor lodge, tourist court, court -n03790953 motorized wheelchair -n03791053 motor scooter, scooter -n03791235 motor vehicle, automotive vehicle -n03792048 mound, hill -n03792334 mound, hill, pitcher's mound -n03792526 mount, setting -n03792782 mountain bike, all-terrain bike, off-roader -n03792972 mountain tent -n03793489 mouse, computer mouse -n03793850 mouse button -n03794056 mousetrap -n03794136 mousse, hair mousse, hair gel -n03794798 mouthpiece, embouchure -n03795123 mouthpiece -n03795269 mouthpiece, gumshield -n03795758 movement -n03795976 movie projector, cine projector, film projector -n03796181 moving-coil galvanometer -n03796401 moving van -n03796522 mud brick -n03796605 mudguard, splash guard, splash-guard -n03796848 mudhif -n03796974 muff -n03797062 muffle -n03797182 muffler -n03797264 mufti -n03797390 mug -n03797896 mulch -n03798061 mule, scuff -n03798442 multichannel recorder -n03798610 multiengine airplane, multiengine plane -n03798982 multiplex -n03799113 multiplexer -n03799240 multiprocessor -n03799375 multistage rocket, step rocket -n03799610 munition, ordnance, ordnance store -n03799876 Murphy bed -n03800371 musette, shepherd's pipe -n03800485 musette pipe -n03800563 museum -n03800772 mushroom anchor -n03800933 musical instrument, instrument -n03801353 music box, musical box -n03801533 music hall, vaudeville theater, vaudeville theatre -n03801671 music school -n03801760 music stand, music rack -n03801880 music stool, piano stool -n03802007 musket -n03802228 musket ball, ball -n03802393 muslin -n03802643 mustache cup, moustache cup -n03802800 mustard plaster, sinapism -n03802973 mute -n03803116 muzzle loader -n03803284 muzzle -n03803780 myelogram -n03804211 nacelle -n03804744 nail -n03805180 nailbrush -n03805280 nailfile -n03805374 nailhead -n03805503 nailhead -n03805725 nail polish, nail enamel, nail varnish -n03805933 nainsook -n03807334 Napier's bones, Napier's rods -n03809211 nard, spikenard -n03809312 narrowbody aircraft, narrow-body aircraft, narrow-body -n03809603 narrow wale -n03809686 narthex -n03809802 narthex -n03810412 nasotracheal tube -n03810952 national monument -n03811295 nautilus, nuclear submarine, nuclear-powered submarine -n03811444 navigational system -n03811847 naval equipment -n03811965 naval gun -n03812263 naval missile -n03812382 naval radar -n03812789 naval tactical data system -n03812924 naval weaponry -n03813078 nave -n03813176 navigational instrument -n03813946 nebuchadnezzar -n03814528 neckband -n03814639 neck brace -n03814727 neckcloth, stock -n03814817 neckerchief -n03814906 necklace -n03815149 necklet -n03815278 neckline -n03815482 neckpiece -n03815615 necktie, tie -n03816005 neckwear -n03816136 needle -n03816394 needle -n03816530 needlenose pliers -n03816849 needlework, needlecraft -n03817191 negative -n03817331 negative magnetic pole, negative pole, south-seeking pole -n03817522 negative pole -n03817647 negligee, neglige, peignoir, wrapper, housecoat -n03818001 neolith -n03818343 neon lamp, neon induction lamp, neon tube -n03819047 nephoscope -n03819336 nest -n03819448 nest egg -n03819595 net, network, mesh, meshing, meshwork -n03819994 net -n03820154 net -n03820318 net -n03820728 network, electronic network -n03820950 network -n03821145 neutron bomb -n03821424 newel -n03821518 newel post, newel -n03822171 newspaper, paper -n03822361 newsroom -n03822504 newsroom -n03822656 newsstand -n03822767 Newtonian telescope, Newtonian reflector -n03823111 nib, pen nib -n03823216 niblick, nine iron -n03823312 nicad, nickel-cadmium accumulator -n03823673 nickel-iron battery, nickel-iron accumulator -n03823906 Nicol prism -n03824197 night bell -n03824284 nightcap -n03824381 nightgown, gown, nightie, night-robe, nightdress -n03824589 night latch -n03824713 night-light -n03824999 nightshirt -n03825080 nightwear, sleepwear, nightclothes -n03825271 ninepin, skittle, skittle pin -n03825442 ninepin ball, skittle ball -n03825673 ninon -n03825788 nipple -n03825913 nipple shield -n03826039 niqab -n03826186 Nissen hut, Quonset hut -n03827420 nogging -n03827536 noisemaker -n03828020 nonsmoker, nonsmoking car -n03829340 non-volatile storage, nonvolatile storage -n03829857 Norfolk jacket -n03829954 noria -n03831203 nosebag, feedbag -n03831382 noseband, nosepiece -n03831757 nose flute -n03832144 nosewheel -n03832673 notebook, notebook computer -n03833907 nuclear-powered ship -n03834040 nuclear reactor, reactor -n03834472 nuclear rocket -n03834604 nuclear weapon, atomic weapon -n03835197 nude, nude painting -n03835729 numdah, numdah rug, nammad -n03835941 nun's habit -n03836062 nursery, baby's room -n03836451 nut and bolt -n03836602 nutcracker -n03836906 nylon -n03836976 nylons, nylon stocking, rayons, rayon stocking, silk stocking -n03837422 oar -n03837606 oast -n03837698 oast house -n03837869 obelisk -n03838024 object ball -n03838298 objective, objective lens, object lens, object glass -n03838748 oblique bandage -n03838899 oboe, hautboy, hautbois -n03839172 oboe da caccia -n03839276 oboe d'amore -n03839424 observation dome -n03839671 observatory -n03839795 obstacle -n03840327 obturator -n03840681 ocarina, sweet potato -n03840823 octant -n03841011 odd-leg caliper -n03841143 odometer, hodometer, mileometer, milometer -n03841290 oeil de boeuf -n03841666 office, business office -n03842012 office building, office block -n03842156 office furniture -n03842276 officer's mess -n03842377 off-line equipment, auxiliary equipment -n03842585 ogee, cyma reversa -n03842754 ogee arch, keel arch -n03842986 ohmmeter -n03843092 oil, oil color, oil colour -n03843316 oilcan -n03843438 oilcloth -n03843555 oil filter -n03843883 oil heater, oilstove, kerosene heater, kerosine heater -n03844045 oil lamp, kerosene lamp, kerosine lamp -n03844233 oil paint -n03844550 oil pump -n03844673 oil refinery, petroleum refinery -n03844815 oilskin, slicker -n03844965 oil slick -n03845107 oilstone -n03845190 oil tanker, oiler, tanker, tank ship -n03845990 old school tie -n03846100 olive drab -n03846234 olive drab, olive-drab uniform -n03846431 Olympian Zeus -n03846677 omelet pan, omelette pan -n03846772 omnidirectional antenna, nondirectional antenna -n03846970 omnirange, omnidirectional range, omnidirectional radio range -n03847471 onion dome -n03847823 open-air market, open-air marketplace, market square -n03848033 open circuit -n03848168 open-end wrench, tappet wrench -n03848348 opener -n03848537 open-hearth furnace -n03849275 openside plane, rabbet plane -n03849412 open sight -n03849679 openwork -n03849814 opera, opera house -n03849943 opera cloak, opera hood -n03850053 operating microscope -n03850245 operating room, OR, operating theater, operating theatre, surgery -n03850492 operating table -n03850613 ophthalmoscope -n03851341 optical device -n03851787 optical disk, optical disc -n03852280 optical instrument -n03852544 optical pyrometer, pyroscope -n03852688 optical telescope -n03853291 orchestra pit, pit -n03853924 ordinary, ordinary bicycle -n03854065 organ, pipe organ -n03854421 organdy, organdie -n03854506 organic light-emitting diode, OLED -n03854722 organ loft -n03854815 organ pipe, pipe, pipework -n03855214 organza -n03855333 oriel, oriel window -n03855464 oriflamme -n03855604 O ring -n03855756 Orlon -n03855908 orlop deck, orlop, fourth deck -n03856012 orphanage, orphans' asylum -n03856335 orphrey -n03856465 orrery -n03856728 orthicon, image orthicon -n03857026 orthochromatic film -n03857156 orthopter, ornithopter -n03857291 orthoscope -n03857687 oscillograph -n03857828 oscilloscope, scope, cathode-ray oscilloscope, CRO -n03858085 ossuary -n03858183 otoscope, auriscope, auroscope -n03858418 ottoman, pouf, pouffe, puff, hassock -n03858533 oubliette -n03858837 out-basket, out-tray -n03859000 outboard motor, outboard -n03859170 outboard motorboat, outboard -n03859280 outbuilding -n03859495 outerwear, overclothes -n03859608 outfall -n03859958 outfit, getup, rig, turnout -n03860234 outfitter -n03860404 outhouse, privy, earth-closet, jakes -n03861048 output device -n03861271 outrigger -n03861430 outrigger canoe -n03861596 outside caliper -n03861842 outside mirror -n03862379 outwork -n03862676 oven -n03862862 oven thermometer -n03863108 overall -n03863262 overall, boilersuit, boilers suit -n03863657 overcoat, overcoating -n03863783 overdrive -n03863923 overgarment, outer garment -n03864139 overhand knot -n03864356 overhang -n03864692 overhead projector -n03865288 overmantel -n03865371 overnighter, overnight bag, overnight case -n03865557 overpass, flyover -n03865820 override -n03865949 overshoe -n03866082 overskirt -n03867854 oxbow -n03868044 Oxbridge -n03868242 oxcart -n03868324 oxeye -n03868406 oxford -n03868643 oximeter -n03868763 oxyacetylene torch -n03868863 oxygen mask -n03869838 oyster bar -n03869976 oyster bed, oyster bank, oyster park -n03870105 pace car -n03870290 pacemaker, artificial pacemaker -n03870546 pack -n03870672 pack -n03870980 pack, face pack -n03871083 package, parcel -n03871371 package store, liquor store, off-licence -n03871524 packaging -n03871628 packet -n03871724 packing box, packing case -n03871860 packinghouse, packing plant -n03872016 packinghouse -n03872167 packing needle -n03872273 packsaddle -n03873416 paddle, boat paddle -n03873699 paddle -n03873848 paddle -n03873996 paddle box, paddle-box -n03874138 paddle steamer, paddle-wheeler -n03874293 paddlewheel, paddle wheel -n03874487 paddock -n03874599 padlock -n03874823 page printer, page-at-a-time printer -n03875218 paint, pigment -n03875806 paintball -n03875955 paintball gun -n03876111 paintbox -n03876231 paintbrush -n03877351 paisley -n03877472 pajama, pyjama, pj's, jammies -n03877674 pajama, pyjama -n03877845 palace -n03878066 palace, castle -n03878211 palace -n03878294 palanquin, palankeen -n03878418 paleolith -n03878511 palestra, palaestra -n03878674 palette, pallet -n03878828 palette knife -n03878963 palisade -n03879456 pallet -n03879705 pallette, palette -n03880032 pallium -n03880129 pallium -n03880323 pan -n03880531 pan, cooking pan -n03881305 pancake turner -n03881404 panchromatic film -n03881534 panda car -n03882611 paneling, panelling, pane -n03882960 panhandle -n03883054 panic button -n03883385 pannier -n03883524 pannier -n03883664 pannikin -n03883773 panopticon -n03883944 panopticon -n03884397 panpipe, pandean pipe, syrinx -n03884554 pantaloon -n03884639 pantechnicon -n03884778 pantheon -n03884926 pantheon -n03885028 pantie, panty, scanty, step-in -n03885194 panting, trousering -n03885293 pant leg, trouser leg -n03885410 pantograph -n03885535 pantry, larder, buttery -n03885669 pants suit, pantsuit -n03885788 panty girdle -n03885904 pantyhose -n03886053 panzer -n03886641 paper chain -n03886762 paper clip, paperclip, gem clip -n03886940 paper cutter -n03887185 paper fastener -n03887330 paper feed -n03887512 paper mill -n03887697 paper towel -n03887899 parabolic mirror -n03888022 parabolic reflector, paraboloid reflector -n03888257 parachute, chute -n03888605 parallel bars, bars -n03888808 parallel circuit, shunt circuit -n03888998 parallel interface, parallel port -n03889397 parang -n03889503 parapet, breastwork -n03889626 parapet -n03889726 parasail -n03889871 parasol, sunshade -n03890093 parer, paring knife -n03890233 parfait glass -n03890358 pargeting, pargetting, pargetry -n03890514 pari-mutuel machine, totalizer, totaliser, totalizator, totalisator -n03891051 parka, windbreaker, windcheater, anorak -n03891251 park bench -n03891332 parking meter -n03891538 parlor, parlour -n03892178 parquet, parquet floor -n03892425 parquetry, parqueterie -n03892557 parsonage, vicarage, rectory -n03892728 Parsons table -n03893935 partial denture -n03894051 particle detector -n03894379 partition, divider -n03894677 parts bin -n03894933 party line -n03895038 party wall -n03895170 parvis -n03895866 passenger car, coach, carriage -n03896103 passenger ship -n03896233 passenger train -n03896419 passenger van -n03896526 passe-partout -n03896628 passive matrix display -n03896984 passkey, passe-partout, master key, master -n03897130 pass-through -n03897634 pastry cart -n03897943 patch -n03898129 patchcord -n03898271 patchouli, patchouly, pachouli -n03898395 patch pocket -n03898633 patchwork, patchwork quilt -n03898787 patent log, screw log, taffrail log -n03899100 paternoster -n03899612 patina -n03899768 patio, terrace -n03899933 patisserie -n03900028 patka -n03900194 patrol boat, patrol ship -n03900301 patty-pan -n03900393 pave -n03900979 pavilion, marquee -n03901229 pavior, paviour, paving machine -n03901338 pavis, pavise -n03901750 pawn -n03901974 pawnbroker's shop, pawnshop, loan office -n03902125 pay-phone, pay-station -n03902220 PC board -n03902482 peach orchard -n03902756 pea jacket, peacoat -n03903133 peavey, peavy, cant dog, dog hook -n03903290 pectoral, pectoral medallion -n03903424 pedal, treadle, foot pedal, foot lever -n03903733 pedal pusher, toreador pants -n03903868 pedestal, plinth, footstall -n03904060 pedestal table -n03904183 pedestrian crossing, zebra crossing -n03904433 pedicab, cycle rickshaw -n03904657 pediment -n03904782 pedometer -n03904909 peeler -n03905361 peep sight -n03905540 peg, nog -n03905730 peg, pin, thole, tholepin, rowlock, oarlock -n03905947 peg -n03906106 peg, wooden leg, leg, pegleg -n03906224 pegboard -n03906463 Pelham -n03906590 pelican crossing -n03906789 pelisse -n03906894 pelvimeter -n03906997 pen -n03907475 penal colony -n03907654 penal institution, penal facility -n03907908 penalty box -n03908111 pen-and-ink -n03908204 pencil -n03908456 pencil -n03908618 pencil box, pencil case -n03908714 pencil sharpener -n03909020 pendant earring, drop earring, eardrop -n03909160 pendulum -n03909406 pendulum clock -n03909516 pendulum watch -n03909658 penetration bomb -n03911406 penile implant -n03911513 penitentiary, pen -n03911658 penknife -n03911767 penlight -n03911866 pennant, pennon, streamer, waft -n03912218 pennywhistle, tin whistle, whistle -n03912821 penthouse -n03913343 pentode -n03913930 peplos, peplus, peplum -n03914106 peplum -n03914337 pepper mill, pepper grinder -n03914438 pepper shaker, pepper box, pepper pot -n03914583 pepper spray -n03914831 percale -n03915118 percolator -n03915320 percussion cap -n03915437 percussion instrument, percussive instrument -n03915900 perforation -n03916031 perfume, essence -n03916289 perfumery -n03916385 perfumery -n03916470 perfumery -n03916720 peripheral, computer peripheral, peripheral device -n03917048 periscope -n03917198 peristyle -n03917327 periwig, peruke -n03917814 permanent press, durable press -n03918074 perpetual motion machine -n03918480 personal computer, PC, microcomputer -n03918737 personal digital assistant, PDA, personal organizer, personal organiser, organizer, organiser -n03919096 personnel carrier -n03919289 pestle -n03919430 pestle, muller, pounder -n03919808 petcock -n03920288 Petri dish -n03920384 petrolatum gauze -n03920641 pet shop -n03920737 petticoat, half-slip, underskirt -n03920867 pew, church bench -n03923379 phial, vial, ampule, ampul, ampoule -n03923564 Phillips screw -n03923692 Phillips screwdriver -n03923918 phonograph needle, needle -n03924069 phonograph record, phonograph recording, record, disk, disc, platter -n03924407 photocathode -n03924532 photocoagulator -n03924679 photocopier -n03926148 photographic equipment -n03926412 photographic paper, photographic material -n03926876 photometer -n03927091 photomicrograph -n03927299 Photostat, Photostat machine -n03927539 photostat -n03927792 physical pendulum, compound pendulum -n03928116 piano, pianoforte, forte-piano -n03928589 piano action -n03928814 piano keyboard, fingerboard, clavier -n03928994 piano wire -n03929091 piccolo -n03929202 pick, pickax, pickaxe -n03929443 pick -n03929660 pick, plectrum, plectron -n03929855 pickelhaube -n03930229 picket boat -n03930313 picket fence, paling -n03930431 picket ship -n03930515 pickle barrel -n03930630 pickup, pickup truck -n03931765 picture frame -n03931885 picture hat -n03931980 picture rail -n03932080 picture window -n03932670 piece of cloth, piece of material -n03933391 pied-a-terre -n03933933 pier -n03934042 pier -n03934229 pier arch -n03934311 pier glass, pier mirror -n03934565 pier table -n03934656 pieta -n03934890 piezometer -n03935116 pig bed, pig -n03935234 piggery, pig farm -n03935335 piggy bank, penny bank -n03935883 pilaster -n03936269 pile, spile, piling, stilt -n03936466 pile driver -n03937543 pill bottle -n03937835 pillbox, toque, turban -n03937931 pillion -n03938037 pillory -n03938244 pillow -n03938401 pillow block -n03938522 pillow lace, bobbin lace -n03938725 pillow sham -n03939062 pilot bit -n03939178 pilot boat -n03939281 pilot burner, pilot light, pilot -n03939440 pilot cloth -n03939565 pilot engine -n03939677 pilothouse, wheelhouse -n03939844 pilot light, pilot lamp, indicator lamp -n03940256 pin -n03940894 pin, flag -n03941013 pin, pin tumbler -n03941231 pinata -n03941417 pinball machine, pin table -n03941586 pince-nez -n03941684 pincer, pair of pincers, tweezer, pair of tweezers -n03941887 pinch bar -n03942028 pincurl clip -n03942600 pinfold -n03942813 ping-pong ball -n03942920 pinhead -n03943115 pinion -n03943266 pinnacle -n03943623 pinprick -n03943714 pinstripe -n03943833 pinstripe -n03943920 pinstripe -n03944024 pintle -n03944138 pinwheel, pinwheel wind collector -n03944341 pinwheel -n03945459 tabor pipe -n03945615 pipe -n03945817 pipe bomb -n03945928 pipe cleaner -n03946076 pipe cutter -n03946162 pipefitting, pipe fitting -n03947111 pipet, pipette -n03947343 pipe vise, pipe clamp -n03947466 pipe wrench, tube wrench -n03947798 pique -n03947888 pirate, pirate ship -n03948242 piste -n03948459 pistol, handgun, side arm, shooting iron -n03948830 pistol grip -n03948950 piston, plunger -n03949145 piston ring -n03949317 piston rod -n03949761 pit -n03950228 pitcher, ewer -n03950359 pitchfork -n03950537 pitching wedge -n03950647 pitch pipe -n03950899 pith hat, pith helmet, sun helmet, topee, topi -n03951068 piton -n03951213 Pitot-static tube, Pitot head, Pitot tube -n03951453 Pitot tube, Pitot -n03951800 pitsaw -n03951971 pivot, pin -n03952150 pivoting window -n03952576 pizzeria, pizza shop, pizza parlor -n03953020 place of business, business establishment -n03953416 place of worship, house of prayer, house of God, house of worship -n03953901 placket -n03954393 planchet, coin blank -n03954731 plane, carpenter's plane, woodworking plane -n03955296 plane, planer, planing machine -n03955489 plane seat -n03955809 planetarium -n03955941 planetarium -n03956157 planetarium -n03956331 planetary gear, epicyclic gear, planet wheel, planet gear -n03956531 plank-bed -n03956623 planking -n03956785 planner -n03956922 plant, works, industrial plant -n03957315 planter -n03957420 plaster, adhesive plaster, sticking plaster -n03957762 plasterboard, gypsum board -n03957991 plastering trowel -n03958227 plastic bag -n03958338 plastic bomb -n03958630 plastic laminate -n03958752 plastic wrap -n03959014 plastron -n03959123 plastron -n03959227 plastron -n03959701 plate, scale, shell -n03960374 plate, collection plate -n03960490 plate -n03961394 platen -n03961630 platen -n03961711 plate rack -n03961828 plate rail -n03961939 platform -n03962525 platform, weapons platform -n03962685 platform -n03962852 platform bed -n03962932 platform rocker -n03963028 plating, metal plating -n03963198 platter -n03963294 playback -n03963483 playbox, play-box -n03963645 playground -n03964495 playpen, pen -n03964611 playsuit -n03965456 plaza, mall, center, shopping mall, shopping center, shopping centre -n03965907 pleat, plait -n03966206 plenum -n03966325 plethysmograph -n03966582 pleximeter, plessimeter -n03966751 plexor, plessor, percussor -n03966976 pliers, pair of pliers, plyers -n03967270 plimsoll -n03967396 plotter -n03967562 plow, plough -n03967942 plug, stopper, stopple -n03968293 plug, male plug -n03968479 plug fuse -n03968581 plughole -n03968728 plumb bob, plumb, plummet -n03969510 plumb level -n03970156 plunger, plumber's helper -n03970363 plus fours -n03970546 plush -n03971218 plywood, plyboard -n03971321 pneumatic drill -n03971960 p-n junction -n03972146 p-n-p transistor -n03972372 poacher -n03972524 pocket -n03973003 pocket battleship -n03973285 pocketcomb, pocket comb -n03973402 pocket flap -n03973520 pocket-handkerchief -n03973628 pocketknife, pocket knife -n03973839 pocket watch -n03973945 pod, fuel pod -n03974070 pogo stick -n03974915 point-and-shoot camera -n03975035 pointed arch -n03975657 pointing trowel -n03975788 point lace, needlepoint -n03975926 poker, stove poker, fire hook, salamander -n03976105 polarimeter, polariscope -n03976268 Polaroid -n03976467 Polaroid camera, Polaroid Land camera -n03976657 pole -n03977158 pole -n03977266 poleax, poleaxe -n03977430 poleax, poleaxe -n03977592 police boat -n03977966 police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria -n03978421 polling booth -n03978575 polo ball -n03978686 polo mallet, polo stick -n03978815 polonaise -n03978966 polo shirt, sport shirt -n03979377 polyester -n03979492 polygraph -n03980026 pomade, pomatum -n03980478 pommel horse, side horse -n03980874 poncho -n03980986 pongee -n03981094 poniard, bodkin -n03981340 pontifical -n03981566 pontoon -n03981760 pontoon bridge, bateau bridge, floating bridge -n03981924 pony cart, ponycart, donkey cart, tub-cart -n03982232 pool ball -n03982331 poolroom -n03982430 pool table, billiard table, snooker table -n03982642 poop deck -n03982767 poor box, alms box, mite box -n03982895 poorhouse -n03983396 pop bottle, soda bottle -n03983499 popgun -n03983612 poplin -n03983712 popper -n03983928 poppet, poppet valve -n03984125 pop tent -n03984234 porcelain -n03984381 porch -n03984643 porkpie, porkpie hat -n03984759 porringer -n03985069 portable -n03985232 portable computer -n03985441 portable circular saw, portable saw -n03985881 portcullis -n03986071 porte-cochere -n03986224 porte-cochere -n03986355 portfolio -n03986562 porthole -n03986704 portico -n03986857 portiere -n03986949 portmanteau, Gladstone, Gladstone bag -n03987266 portrait camera -n03987376 portrait lens -n03987674 positive pole, positive magnetic pole, north-seeking pole -n03987865 positive pole -n03987990 positron emission tomography scanner, PET scanner -n03988170 post -n03988758 postage meter -n03988926 post and lintel -n03989199 post chaise -n03989349 postern -n03989447 post exchange, PX -n03989665 posthole digger, post-hole digger -n03989777 post horn -n03989898 posthouse, post house -n03990474 pot -n03991062 pot, flowerpot -n03991202 potbelly, potbelly stove -n03991321 Potemkin village -n03991443 potential divider, voltage divider -n03991646 potentiometer, pot -n03991837 potentiometer -n03992325 potpourri -n03992436 potsherd -n03992509 potter's wheel -n03992703 pottery, clayware -n03992975 pottle -n03993053 potty seat, potty chair -n03993180 pouch -n03993403 poultice, cataplasm, plaster -n03993703 pound, dog pound -n03993878 pound net -n03994008 powder -n03994297 powder and shot -n03994417 powdered mustard, dry mustard -n03994614 powder horn, powder flask -n03994757 powder keg -n03995018 power brake -n03995265 power cord -n03995372 power drill -n03995535 power line, power cable -n03995661 power loom -n03995856 power mower, motor mower -n03996004 power pack -n03996145 power saw, saw, sawing machine -n03996416 power shovel, excavator, digger, shovel -n03996849 power steering, power-assisted steering -n03997274 power takeoff, PTO -n03997484 power tool -n03997875 praetorium, pretorium -n03998194 prayer rug, prayer mat -n03998333 prayer shawl, tallith, tallis -n03998673 precipitator, electrostatic precipitator, Cottrell precipitator -n03999064 prefab -n03999160 presbytery -n03999621 presence chamber -n03999992 press, mechanical press -n04000311 press, printing press -n04000480 press -n04000592 press box -n04000716 press gallery -n04000998 press of sail, press of canvas -n04001132 pressure cabin -n04001265 pressure cooker -n04001397 pressure dome -n04001499 pressure gauge, pressure gage -n04001661 pressurized water reactor, PWR -n04001845 pressure suit -n04002262 pricket -n04002371 prie-dieu -n04002629 primary coil, primary winding, primary -n04003241 Primus stove, Primus -n04003359 Prince Albert -n04003856 print -n04004099 print buffer -n04004210 printed circuit -n04004475 printer, printing machine -n04004767 printer -n04004990 printer cable -n04005197 priory -n04005630 prison, prison house -n04005912 prison camp, internment camp, prisoner of war camp, POW camp -n04006067 privateer -n04006227 private line -n04006330 privet hedge -n04006411 probe -n04007415 proctoscope -n04007664 prod, goad -n04008385 production line, assembly line, line -n04008634 projectile, missile -n04009552 projector -n04009801 projector -n04009923 prolonge -n04010057 prolonge knot, sailor's breastplate -n04010779 prompter, autocue -n04010927 prong -n04011827 propeller, propellor -n04012084 propeller plane -n04012482 propjet, turboprop, turbo-propeller plane -n04012665 proportional counter tube, proportional counter -n04013060 propulsion system -n04013176 proscenium, proscenium wall -n04013600 proscenium arch -n04013729 prosthesis, prosthetic device -n04014297 protective covering, protective cover, protection -n04015204 protective garment -n04015786 proton accelerator -n04015908 protractor -n04016240 pruner, pruning hook, lopper -n04016479 pruning knife -n04016576 pruning saw -n04016684 pruning shears -n04016846 psaltery -n04017571 psychrometer -n04017807 PT boat, mosquito boat, mosquito craft, motor torpedo boat -n04018155 public address system, P.A. system, PA system, P.A., PA -n04018399 public house, pub, saloon, pothouse, gin mill, taphouse -n04018667 public toilet, comfort station, public convenience, convenience, public lavatory, restroom, toilet facility, wash room -n04019101 public transport -n04019335 public works -n04019541 puck, hockey puck -n04019696 pull -n04019881 pullback, tieback -n04020087 pull chain -n04020298 pulley, pulley-block, pulley block, block -n04020744 pull-off, rest area, rest stop, layby, lay-by -n04020912 Pullman, Pullman car -n04021028 pullover, slipover -n04021164 pull-through -n04021362 pulse counter -n04021503 pulse generator -n04021704 pulse timing circuit -n04021798 pump -n04022332 pump -n04022434 pump action, slide action -n04022708 pump house, pumping station -n04022866 pump room -n04023021 pump-type pliers -n04023119 pump well -n04023249 punch, puncher -n04023422 punchboard -n04023695 punch bowl -n04023962 punching bag, punch bag, punching ball, punchball -n04024137 punch pliers -n04024274 punch press -n04024862 punnet -n04024983 punt -n04025508 pup tent, shelter tent -n04025633 purdah -n04026053 purifier -n04026180 purl, purl stitch -n04026417 purse -n04026813 push-bike -n04026918 push broom -n04027023 push button, push, button -n04027367 push-button radio -n04027706 pusher, zori -n04027820 put-put -n04027935 puttee -n04028074 putter, putting iron -n04028221 putty knife -n04028315 puzzle -n04028581 pylon, power pylon -n04028764 pylon -n04029416 pyramidal tent -n04029647 pyrograph -n04029734 pyrometer -n04029913 pyrometric cone -n04030054 pyrostat -n04030161 pyx, pix -n04030274 pyx, pix, pyx chest, pix chest -n04030414 pyxis -n04030518 quad, quadrangle -n04030846 quadrant -n04030965 quadraphony, quadraphonic system, quadriphonic system -n04031884 quartering -n04032509 quarterstaff -n04032603 quartz battery, quartz mill -n04032936 quartz lamp -n04033287 queen -n04033425 queen -n04033557 queen post -n04033801 quern -n04033901 quill, quill pen -n04033995 quilt, comforter, comfort, puff -n04034262 quilted bedspread -n04034367 quilting -n04035231 quipu -n04035634 quirk molding, quirk moulding -n04035748 quirt -n04035836 quiver -n04035912 quoin, coign, coigne -n04036155 quoit -n04036303 QWERTY keyboard -n04036776 rabbet, rebate -n04036963 rabbet joint -n04037076 rabbit ears -n04037220 rabbit hutch -n04037298 raceabout -n04037443 racer, race car, racing car -n04037873 raceway, race -n04037964 racing boat -n04038231 racing gig -n04038338 racing skiff, single shell -n04038440 rack, stand -n04038727 rack -n04039041 rack, wheel -n04039209 rack and pinion -n04039381 racket, racquet -n04039742 racquetball -n04039848 radar, microwave radar, radio detection and ranging, radiolocation -n04040247 radial, radial tire, radial-ply tire -n04040373 radial engine, rotary engine -n04040540 radiation pyrometer -n04040759 radiator -n04041069 radiator -n04041243 radiator cap -n04041408 radiator hose -n04041544 radio, wireless -n04041747 radio antenna, radio aerial -n04042076 radio chassis -n04042204 radio compass -n04042358 radiogram, radiograph, shadowgraph, skiagraph, skiagram -n04042632 radio interferometer -n04042795 radio link, link -n04042985 radiometer -n04043168 radiomicrometer -n04043411 radio-phonograph, radio-gramophone -n04043733 radio receiver, receiving set, radio set, radio, tuner, wireless -n04044307 radiotelegraph, radiotelegraphy, wireless telegraph, wireless telegraphy -n04044498 radiotelephone, radiophone, wireless telephone -n04044716 radio telescope, radio reflector -n04044955 radiotherapy equipment -n04045085 radio transmitter -n04045255 radome, radar dome -n04045397 raft -n04045644 rafter, balk, baulk -n04045787 raft foundation -n04045941 rag, shred, tag, tag end, tatter -n04046091 ragbag -n04046277 raglan -n04046400 raglan sleeve -n04046590 rail -n04046974 rail fence -n04047139 railhead -n04047401 railing, rail -n04047733 railing -n04047834 railroad bed -n04048441 railroad tunnel -n04049303 rain barrel -n04049405 raincoat, waterproof -n04049585 rain gauge, rain gage, pluviometer, udometer -n04049753 rain stick -n04050066 rake -n04050313 rake handle -n04050600 RAM disk -n04050933 ramekin, ramequin -n04051269 ramjet, ramjet engine, atherodyde, athodyd, flying drainpipe -n04051439 rammer -n04051549 ramp, incline -n04051705 rampant arch -n04051825 rampart, bulwark, wall -n04052235 ramrod -n04052346 ramrod -n04052442 ranch, spread, cattle ranch, cattle farm -n04052658 ranch house -n04052757 random-access memory, random access memory, random memory, RAM, read/write memory -n04053508 rangefinder, range finder -n04053677 range hood -n04053767 range pole, ranging pole, flagpole -n04054361 rapier, tuck -n04054566 rariora -n04054670 rasp, wood file -n04055180 ratchet, rachet, ratch -n04055447 ratchet wheel -n04055700 rathskeller -n04055861 ratline, ratlin -n04056073 rat-tail file -n04056180 rattan, ratan -n04056413 rattrap -n04056932 rayon -n04057047 razor -n04057215 razorblade -n04057435 reaction-propulsion engine, reaction engine -n04057673 reaction turbine -n04057846 reactor -n04057981 reading lamp -n04058096 reading room -n04058239 read-only memory, ROM, read-only storage, fixed storage -n04058486 read-only memory chip -n04058594 readout, read-out -n04058721 read/write head, head -n04059157 ready-to-wear -n04059298 real storage -n04059399 reamer -n04059516 reamer, juicer, juice reamer -n04059947 rearview mirror -n04060198 Reaumur thermometer -n04060448 rebozo -n04060647 receiver, receiving system -n04060904 receptacle -n04061681 reception desk -n04061793 reception room -n04061969 recess, niche -n04062179 reciprocating engine -n04062428 recliner, reclining chair, lounger -n04062644 reconnaissance plane -n04062807 reconnaissance vehicle, scout car -n04063154 record changer, auto-changer, changer -n04063373 recorder, recording equipment, recording machine -n04063868 recording -n04064213 recording system -n04064401 record player, phonograph -n04064747 record sleeve, record cover -n04064862 recovery room -n04065272 recreational vehicle, RV, R.V. -n04065464 recreation room, rec room -n04065789 recycling bin -n04065909 recycling plant -n04066023 redbrick university -n04066270 red carpet -n04066388 redoubt -n04066476 redoubt -n04066767 reduction gear -n04067143 reed pipe -n04067231 reed stop -n04067353 reef knot, flat knot -n04067472 reel -n04067658 reel -n04067818 refectory -n04067921 refectory table -n04068441 refinery -n04068601 reflecting telescope, reflector -n04069166 reflectometer -n04069276 reflector -n04069434 reflex camera -n04069582 reflux condenser -n04069777 reformatory, reform school, training school -n04070003 reformer -n04070207 refracting telescope -n04070415 refractometer -n04070545 refrigeration system -n04070727 refrigerator, icebox -n04070964 refrigerator car -n04071102 refuge, sanctuary, asylum -n04071263 regalia -n04071393 regimentals -n04072193 regulator -n04072551 rein -n04072960 relay, electrical relay -n04073425 release, button -n04073948 religious residence, cloister -n04074185 reliquary -n04074963 remote control, remote -n04075291 remote terminal, link-attached terminal, remote station, link-attached station -n04075468 removable disk -n04075715 rendering -n04075813 rep, repp -n04075916 repair shop, fix-it shop -n04076052 repeater -n04076284 repeating firearm, repeater -n04076713 repository, monument -n04077430 reproducer -n04077594 rerebrace, upper cannon -n04077734 rescue equipment -n04077889 research center, research facility -n04078002 reseau -n04078574 reservoir -n04078955 reset -n04079106 reset button -n04079244 residence -n04079603 resistance pyrometer -n04079933 resistor, resistance -n04080138 resonator -n04080454 resonator, cavity resonator, resonating chamber -n04080705 resort hotel, spa -n04080833 respirator, inhalator -n04081281 restaurant, eating house, eating place, eatery -n04081699 rest house -n04081844 restraint, constraint -n04082344 resuscitator -n04082562 retainer -n04082710 retaining wall -n04082886 reticle, reticule, graticule -n04083113 reticulation -n04083309 reticule -n04083649 retort -n04083800 retractor -n04084517 return key, return -n04084682 reverberatory furnace -n04084889 revers, revere -n04085017 reverse, reverse gear -n04085574 reversible -n04085873 revetment, revetement, stone facing -n04086066 revetment -n04086273 revolver, six-gun, six-shooter -n04086446 revolving door, revolver -n04086663 rheometer -n04086794 rheostat, variable resistor -n04086937 rhinoscope -n04087126 rib -n04087432 riband, ribband -n04087709 ribbed vault -n04087826 ribbing -n04088229 ribbon development -n04088343 rib joint pliers -n04088441 ricer -n04088696 riddle -n04088797 ride -n04089152 ridge, ridgepole, rooftree -n04089376 ridge rope -n04089666 riding boot -n04089836 riding crop, hunting crop -n04089976 riding mower -n04090263 rifle -n04090548 rifle ball -n04090781 rifle grenade -n04091097 rig -n04091466 rigger, rigger brush -n04091584 rigger -n04091693 rigging, tackle -n04092168 rigout -n04093157 ringlet -n04093223 rings -n04093625 rink, skating rink -n04093775 riot gun -n04093915 ripcord -n04094060 ripcord -n04094250 ripping bar -n04094438 ripping chisel -n04094608 ripsaw, splitsaw -n04094720 riser -n04094859 riser, riser pipe, riser pipeline, riser main -n04095109 Ritz -n04095210 river boat -n04095342 rivet -n04095577 riveting machine, riveter, rivetter -n04095938 roach clip, roach holder -n04096066 road, route -n04096733 roadbed -n04096848 roadblock, barricade -n04097085 roadhouse -n04097373 roadster, runabout, two-seater -n04097622 roadway -n04097760 roaster -n04097866 robe -n04098169 robotics equipment -n04098260 Rochon prism, Wollaston prism -n04098399 rock bit, roller bit -n04098513 rocker -n04098795 rocker, cradle -n04099003 rocker arm, valve rocker -n04099175 rocket, rocket engine -n04099429 rocket, projectile -n04099969 rocking chair, rocker -n04100174 rod -n04100519 rodeo -n04101375 roll -n04101497 roller -n04101701 roller -n04101860 roller bandage -n04102037 in-line skate -n04102162 Rollerblade -n04102285 roller blind -n04102406 roller coaster, big dipper, chute-the-chute -n04102618 roller skate -n04102760 roller towel -n04102872 roll film -n04102962 rolling hitch -n04103094 rolling mill -n04103206 rolling pin -n04103364 rolling stock -n04103665 roll-on -n04103769 roll-on -n04103918 roll-on roll-off -n04104147 Rolodex -n04104384 Roman arch, semicircular arch -n04104500 Roman building -n04104770 romper, romper suit -n04104925 rood screen -n04105068 roof -n04105438 roof -n04105704 roofing -n04105893 room -n04107598 roomette -n04107743 room light -n04107984 roost -n04108268 rope -n04108822 rope bridge -n04108999 rope tow -n04110068 rose water -n04110178 rose window, rosette -n04110281 rosin bag -n04110439 rotary actuator, positioner -n04110654 rotary engine -n04110841 rotary press -n04110955 rotating mechanism -n04111190 rotating shaft, shaft -n04111414 rotisserie -n04111531 rotisserie -n04111668 rotor -n04111962 rotor, rotor coil -n04112147 rotor -n04112252 rotor blade, rotary wing -n04112430 rotor head, rotor shaft -n04112579 rotunda -n04112654 rotunda -n04112752 rouge, paint, blusher -n04112921 roughcast -n04113038 rouleau -n04113194 roulette, toothed wheel -n04113316 roulette ball -n04113406 roulette wheel, wheel -n04113641 round, unit of ammunition, one shot -n04113765 round arch -n04113968 round-bottom flask -n04114069 roundel -n04114301 round file -n04114428 roundhouse -n04114719 router -n04114844 router -n04114996 router plane -n04115144 rowel -n04115256 row house, town house -n04115456 rowing boat -n04115542 rowlock arch -n04115802 royal -n04115996 royal mast -n04116098 rubber band, elastic band, elastic -n04116294 rubber boot, gum boot -n04116389 rubber bullet -n04116512 rubber eraser, rubber, pencil eraser -n04117216 rudder -n04117464 rudder -n04117639 rudder blade -n04118021 rug, carpet, carpeting -n04118538 rugby ball -n04118635 ruin -n04118776 rule, ruler -n04119091 rumble -n04119230 rumble seat -n04119360 rummer -n04119478 rumpus room, playroom, game room -n04119630 runcible spoon -n04119751 rundle, spoke, rung -n04120489 running shoe -n04120695 running suit -n04120842 runway -n04121228 rushlight, rush candle -n04121342 russet -n04121426 rya, rya rug -n04121511 saber, sabre -n04121728 saber saw, jigsaw, reciprocating saw -n04122262 sable -n04122349 sable, sable brush, sable's hair pencil -n04122492 sable coat -n04122578 sabot, wooden shoe -n04122685 sachet -n04122825 sack, poke, paper bag, carrier bag -n04123026 sack, sacque -n04123123 sackbut -n04123228 sackcloth -n04123317 sackcloth -n04123448 sack coat -n04123567 sacking, bagging -n04123740 saddle -n04124098 saddlebag -n04124202 saddle blanket, saddlecloth, horse blanket -n04124370 saddle oxford, saddle shoe -n04124488 saddlery -n04124573 saddle seat -n04124887 saddle stitch -n04125021 safe -n04125116 safe -n04125257 safe-deposit, safe-deposit box, safety-deposit, safety deposit box, deposit box, lockbox -n04125541 safe house -n04125692 safety arch -n04125853 safety belt, life belt, safety harness -n04126066 safety bicycle, safety bike -n04126244 safety bolt, safety lock -n04126541 safety curtain -n04126659 safety fuse -n04126852 safety lamp, Davy lamp -n04126980 safety match, book matches -n04127117 safety net -n04127249 safety pin -n04127395 safety rail, guardrail -n04127521 safety razor -n04127633 safety valve, relief valve, escape valve, escape cock, escape -n04127904 sail, canvas, canvass, sheet -n04128413 sail -n04128499 sailboat, sailing boat -n04128710 sailcloth -n04128837 sailing vessel, sailing ship -n04129490 sailing warship -n04129688 sailor cap -n04129766 sailor suit -n04130143 salad bar -n04130257 salad bowl -n04130566 salinometer -n04130907 sallet, salade -n04131015 salon -n04131113 salon -n04131208 salon, beauty salon, beauty parlor, beauty parlour, beauty shop -n04131368 saltbox -n04131499 saltcellar -n04131690 saltshaker, salt shaker -n04131811 saltworks -n04131929 salver -n04132158 salwar, shalwar -n04132465 Sam Browne belt -n04132603 samisen, shamisen -n04132829 samite -n04132985 samovar -n04133114 sampan -n04133789 sandal -n04134008 sandbag -n04134170 sandblaster -n04134523 sandbox -n04134632 sandglass -n04135024 sand wedge -n04135118 sandwich board -n04135315 sanitary napkin, sanitary towel, Kotex -n04135710 cling film, clingfilm, Saran Wrap -n04135933 sarcenet, sarsenet -n04136045 sarcophagus -n04136161 sari, saree -n04136333 sarong -n04136510 sash, window sash -n04136800 sash fastener, sash lock, window lock -n04137089 sash window -n04137217 satchel -n04137355 sateen -n04137444 satellite, artificial satellite, orbiter -n04137773 satellite receiver -n04137897 satellite television, satellite TV -n04138131 satellite transmitter -n04138261 satin -n04138869 Saturday night special -n04138977 saucepan -n04139140 saucepot -n04139395 sauna, sweat room -n04139859 savings bank, coin bank, money box, bank -n04140064 saw -n04140539 sawed-off shotgun -n04140631 sawhorse, horse, sawbuck, buck -n04140777 sawmill -n04140853 saw set -n04141076 sax, saxophone -n04141198 saxhorn -n04141327 scabbard -n04141712 scaffolding, staging -n04141838 scale -n04141975 scale, weighing machine -n04142175 scaler -n04142327 scaling ladder -n04142434 scalpel -n04142731 scanner, electronic scanner -n04142999 scanner -n04143140 scanner, digital scanner, image scanner -n04143365 scantling, stud -n04143897 scarf -n04144241 scarf joint, scarf -n04144539 scatter rug, throw rug -n04144651 scauper, scorper -n04145863 Schmidt telescope, Schmidt camera -n04146050 school, schoolhouse -n04146343 schoolbag -n04146504 school bell -n04146614 school bus -n04146862 school ship, training ship -n04146976 school system -n04147183 schooner -n04147291 schooner -n04147495 scientific instrument -n04147793 scimitar -n04147916 scintillation counter -n04148054 scissors, pair of scissors -n04148285 sclerometer -n04148464 scoinson arch, sconcheon arch -n04148579 sconce -n04148703 sconce -n04149083 scoop -n04149374 scooter -n04149813 scoreboard -n04150153 scouring pad -n04150273 scow -n04150371 scow -n04150980 scraper -n04151108 scratcher -n04151581 screen -n04151940 screen, cover, covert, concealment -n04152387 screen -n04152593 screen, CRT screen -n04153025 screen door, screen -n04153330 screening -n04153751 screw -n04154152 screw, screw propeller -n04154340 screw -n04154565 screwdriver -n04154753 screw eye -n04154854 screw key -n04154938 screw thread, thread -n04155068 screwtop -n04155177 screw wrench -n04155457 scriber, scribe, scratch awl -n04155625 scrim -n04155735 scrimshaw -n04155889 scriptorium -n04156040 scrubber -n04156140 scrub brush, scrubbing brush, scrubber -n04156297 scrub plane -n04156411 scuffer -n04156591 scuffle, scuffle hoe, Dutch hoe -n04156814 scull -n04156946 scull -n04157099 scullery -n04157320 sculpture -n04158002 scuttle, coal scuttle -n04158138 scyphus -n04158250 scythe -n04158672 seabag -n04158807 sea boat -n04158956 sea chest -n04160036 sealing wax, seal -n04160261 sealskin -n04160372 seam -n04160586 seaplane, hydroplane -n04160847 searchlight -n04161010 searing iron -n04161358 seat -n04161981 seat -n04162433 seat -n04162706 seat belt, seatbelt -n04163530 secateurs -n04164002 secondary coil, secondary winding, secondary -n04164199 second balcony, family circle, upper balcony, peanut gallery -n04164406 second base -n04164757 second hand -n04164868 secretary, writing table, escritoire, secretaire -n04165409 sectional -n04165675 security blanket -n04165945 security system, security measure, security -n04166111 security system -n04166281 sedan, saloon -n04166436 sedan, sedan chair -n04167346 seeder -n04167489 seeker -n04167661 seersucker -n04168084 segmental arch -n04168199 Segway, Segway Human Transporter, Segway HT -n04168472 seidel -n04168541 seine -n04168840 seismograph -n04169437 selector, selector switch -n04169597 selenium cell -n04170037 self-propelled vehicle -n04170384 self-registering thermometer -n04170515 self-starter -n04170694 selsyn, synchro -n04170933 selvage, selvedge -n04171208 semaphore -n04171459 semiautomatic firearm -n04171629 semiautomatic pistol, semiautomatic -n04171831 semiconductor device, semiconductor unit, semiconductor -n04172107 semi-detached house -n04172230 semigloss -n04172342 semitrailer, semi -n04172512 sennit -n04172607 sensitometer -n04172776 sentry box -n04172904 separate -n04173046 septic tank -n04173172 sequence, episode -n04173511 sequencer, sequenator -n04173907 serape, sarape -n04174026 serge -n04174101 serger -n04174234 serial port -n04174500 serpent -n04174705 serration -n04175039 server -n04175147 server, host -n04175574 service club -n04176068 serving cart -n04176190 serving dish -n04176295 servo, servomechanism, servosystem -n04176528 set -n04177041 set gun, spring gun -n04177329 setscrew -n04177545 setscrew -n04177654 set square -n04177755 settee -n04177820 settle, settee -n04177931 settlement house -n04178190 seventy-eight, 78 -n04178329 Seven Wonders of the Ancient World, Seven Wonders of the World -n04178668 sewage disposal plant, disposal plant -n04179126 sewer, sewerage, cloaca -n04179712 sewing basket -n04179824 sewing kit -n04179913 sewing machine -n04180063 sewing needle -n04180229 sewing room -n04180888 sextant -n04181083 sgraffito -n04181228 shackle, bond, hamper, trammel -n04181561 shackle -n04181718 shade -n04182152 shadow box -n04182322 shaft -n04183217 shag rug -n04183329 shaker -n04183957 shank -n04184095 shank, stem -n04184316 shantung -n04184435 shaper, shaping machine -n04184600 shaping tool -n04184880 sharkskin -n04185071 sharpener -n04185529 Sharpie -n04185804 shaver, electric shaver, electric razor -n04185946 shaving brush -n04186051 shaving cream, shaving soap -n04186268 shaving foam -n04186455 shawl -n04186624 shawm -n04186848 shears -n04187061 sheath -n04187233 sheathing, overlay, overlayer -n04187547 shed -n04187751 sheep bell -n04187885 sheepshank -n04187970 sheepskin coat, afghan -n04188064 sheepwalk, sheeprun -n04188179 sheet, bed sheet -n04189092 sheet bend, becket bend, weaver's knot, weaver's hitch -n04189282 sheeting -n04189651 sheet pile, sheath pile, sheet piling -n04189816 Sheetrock -n04190052 shelf -n04190376 shelf bracket -n04190464 shell -n04190747 shell, case, casing -n04190997 shell, racing shell -n04191150 shellac, shellac varnish -n04191595 shelter -n04191943 shelter -n04192238 shelter -n04192361 sheltered workshop -n04192521 Sheraton -n04192698 shield, buckler -n04192858 shield -n04193179 shielding -n04193377 shift key, shift -n04193742 shillelagh, shillalah -n04193883 shim -n04194009 shingle -n04194127 shin guard, shinpad -n04194289 ship -n04196080 shipboard system -n04196502 shipping, cargo ships, merchant marine, merchant vessels -n04196803 shipping room -n04196925 ship-towed long-range acoustic detection system -n04197110 shipwreck -n04197391 shirt -n04197781 shirt button -n04197878 shirtdress -n04198015 shirtfront -n04198233 shirting -n04198355 shirtsleeve -n04198453 shirttail -n04198562 shirtwaist, shirtwaister -n04198722 shiv -n04198797 shock absorber, shock, cushion -n04199027 shoe -n04200000 shoe -n04200258 shoebox -n04200537 shoehorn -n04200800 shoe shop, shoe-shop, shoe store -n04200908 shoetree -n04201064 shofar, shophar -n04201297 shoji -n04201733 shooting brake -n04202142 shooting lodge, shooting box -n04202282 shooting stick -n04202417 shop, store -n04203356 shop bell -n04204081 shopping bag -n04204238 shopping basket -n04204347 shopping cart -n04204755 short circuit, short -n04205062 short iron -n04205318 short pants, shorts, trunks -n04205505 short sleeve -n04205613 shortwave diathermy machine -n04206070 shot -n04206225 shot glass, jigger, pony -n04206356 shotgun, scattergun -n04206570 shotgun shell -n04206790 shot tower -n04207151 shoulder -n04207343 shoulder bag -n04207596 shouldered arch -n04207763 shoulder holster -n04207903 shoulder pad -n04208065 shoulder patch -n04208210 shovel -n04208427 shovel -n04208582 shovel hat -n04208760 showboat -n04208936 shower -n04209133 shower cap -n04209239 shower curtain -n04209509 shower room -n04209613 shower stall, shower bath -n04209811 showroom, salesroom, saleroom -n04210012 shrapnel -n04210120 shredder -n04210288 shrimper -n04210390 shrine -n04210591 shrink-wrap -n04210858 shunt -n04211001 shunt, electrical shunt, bypass -n04211219 shunter -n04211356 shutter -n04211528 shutter -n04211857 shuttle -n04211970 shuttle -n04212165 shuttle bus -n04212282 shuttlecock, bird, birdie, shuttle -n04212467 shuttle helicopter -n04212810 Sibley tent -n04213105 sickbay, sick berth -n04213264 sickbed -n04213353 sickle, reaping hook, reap hook -n04213530 sickroom -n04214046 sideboard -n04214282 sidecar -n04214413 side chapel -n04214649 sidelight, running light -n04215153 sidesaddle -n04215402 sidewalk, pavement -n04215588 sidewall -n04215800 side-wheeler -n04215910 sidewinder -n04216634 sieve, screen -n04216860 sifter -n04216963 sights -n04217387 sigmoidoscope, flexible sigmoidoscope -n04217546 signal box, signal tower -n04217718 signaling device -n04217882 signboard, sign -n04218564 silencer, muffler -n04218921 silent butler -n04219185 Silex -n04219424 silk -n04219580 silks -n04220250 silo -n04220805 silver plate -n04221076 silverpoint -n04221673 simple pendulum -n04221823 simulator -n04222210 single bed -n04222307 single-breasted jacket -n04222470 single-breasted suit -n04222723 single prop, single-propeller plane -n04222847 single-reed instrument, single-reed woodwind -n04223066 single-rotor helicopter -n04223170 singlestick, fencing stick, backsword -n04223299 singlet, vest, undershirt -n04224395 siren -n04224543 sister ship -n04224842 sitar -n04225031 sitz bath, hip bath -n04225222 six-pack, six pack, sixpack -n04225729 skate -n04225987 skateboard -n04226322 skeg -n04226464 skein -n04226537 skeleton, skeletal frame, frame, underframe -n04226826 skeleton key -n04226962 skep -n04227050 skep -n04227144 sketch, study -n04227519 sketcher -n04227787 skew arch -n04227900 skewer -n04228054 ski -n04228215 ski binding, binding -n04228422 skibob -n04228581 ski boot -n04228693 ski cap, stocking cap, toboggan cap -n04229007 skidder -n04229107 skid lid -n04229480 skiff -n04229620 ski jump -n04229737 ski lodge -n04229816 ski mask -n04229959 skimmer -n04230387 ski parka, ski jacket -n04230487 ski-plane -n04230603 ski pole -n04230707 ski rack -n04230808 skirt -n04231272 skirt -n04231693 ski tow, ski lift, lift -n04231905 Skivvies -n04232153 skullcap -n04232312 skybox -n04232437 skyhook -n04232800 skylight, fanlight -n04233027 skysail -n04233124 skyscraper -n04233295 skywalk -n04233715 slacks -n04233832 slack suit -n04234160 slasher -n04234260 slash pocket -n04234455 slat, spline -n04234670 slate -n04234763 slate pencil -n04234887 slate roof -n04235291 sled, sledge, sleigh -n04235646 sleeper -n04235771 sleeper -n04235860 sleeping bag -n04236001 sleeping car, sleeper, wagon-lit -n04236377 sleeve, arm -n04236702 sleeve -n04236809 sleigh bed -n04236935 sleigh bell, cascabel -n04237174 slice bar -n04237287 slicer -n04237423 slicer -n04238128 slide, playground slide, sliding board -n04238321 slide fastener, zip, zipper, zip fastener -n04238617 slide projector -n04238763 slide rule, slipstick -n04238953 slide valve -n04239074 sliding door -n04239218 sliding seat -n04239333 sliding window -n04239436 sling, scarf bandage, triangular bandage -n04239639 sling -n04239786 slingback, sling -n04239900 slinger ring -n04240434 slip clutch, slip friction clutch -n04240752 slipcover -n04240867 slip-joint pliers -n04241042 slipknot -n04241249 slip-on -n04241394 slipper, carpet slipper -n04241573 slip ring -n04242084 slit lamp -n04242315 slit trench -n04242408 sloop -n04242587 sloop of war -n04242704 slop basin, slop bowl -n04243003 slop pail, slop jar -n04243142 slops -n04243251 slopshop, slopseller's shop -n04243546 slot, one-armed bandit -n04243941 slot machine, coin machine -n04244379 sluice, sluiceway, penstock -n04244847 smack -n04244997 small boat -n04245218 small computer system interface, SCSI -n04245412 small ship -n04245508 small stores -n04245847 smart bomb -n04246060 smelling bottle -n04246271 smocking -n04246459 smoke bomb, smoke grenade -n04246731 smokehouse, meat house -n04246855 smoker, smoking car, smoking carriage, smoking compartment -n04247011 smoke screen, smokescreen -n04247440 smoking room -n04247544 smoothbore -n04247630 smooth plane, smoothing plane -n04247736 snack bar, snack counter, buffet -n04247876 snaffle, snaffle bit -n04248209 snap, snap fastener, press stud -n04248396 snap brim -n04248507 snap-brim hat -n04248851 snare, gin, noose -n04249415 snare drum, snare, side drum -n04249582 snatch block -n04249882 snifter, brandy snifter, brandy glass -n04250224 sniper rifle, precision rifle -n04250473 snips, tinsnips -n04250599 Sno-cat -n04250692 snood -n04250850 snorkel, schnorkel, schnorchel, snorkel breather, breather -n04251144 snorkel -n04251701 snowbank, snow bank -n04251791 snowboard -n04252077 snowmobile -n04252225 snowplow, snowplough -n04252331 snowshoe -n04252560 snowsuit -n04252653 snow thrower, snow blower -n04253057 snuffbox -n04253168 snuffer -n04253304 snuffers -n04253931 soapbox -n04254009 soap dish -n04254120 soap dispenser -n04254450 soap pad -n04254680 soccer ball -n04254777 sock -n04255163 socket -n04255346 socket wrench -n04255499 socle -n04255586 soda can -n04255670 soda fountain -n04255768 soda fountain -n04255899 sod house, soddy, adobe house -n04256318 sodium-vapor lamp, sodium-vapour lamp -n04256520 sofa, couch, lounge -n04256758 soffit -n04256891 softball, playground ball -n04257223 soft pedal -n04257684 soil pipe -n04257790 solar array, solar battery, solar panel -n04257986 solar cell, photovoltaic cell -n04258138 solar dish, solar collector, solar furnace -n04258333 solar heater -n04258438 solar house -n04258618 solar telescope -n04258732 solar thermal system -n04258859 soldering iron -n04259202 solenoid -n04259468 solleret, sabaton -n04259630 sombrero -n04260192 sonic depth finder, fathometer -n04260364 sonogram, echogram -n04260589 sonograph -n04261116 sorter -n04261281 souk -n04261369 sound bow -n04261506 soundbox, body -n04261638 sound camera -n04261767 sounder -n04261868 sound film -n04262161 sounding board, soundboard -n04262530 sounding rocket -n04262678 sound recording, audio recording, audio -n04262869 sound spectrograph -n04263257 soup bowl -n04263336 soup ladle -n04263502 soupspoon, soup spoon -n04263760 source of illumination -n04263950 sourdine -n04264134 soutache -n04264233 soutane -n04264361 sou'wester -n04264485 soybean future -n04264628 space bar -n04264765 space capsule, capsule -n04264914 spacecraft, ballistic capsule, space vehicle -n04265275 space heater -n04265428 space helmet -n04265904 space rocket -n04266014 space shuttle -n04266162 space station, space platform, space laboratory -n04266375 spacesuit -n04266486 spade -n04266849 spade bit -n04266968 spaghetti junction -n04267091 Spandau -n04267165 spandex -n04267246 spandrel, spandril -n04267435 spanker -n04267577 spar -n04267985 sparge pipe -n04268142 spark arrester, sparker -n04268275 spark arrester -n04268418 spark chamber, spark counter -n04268565 spark coil -n04268799 spark gap -n04269086 spark lever -n04269270 spark plug, sparking plug, plug -n04269502 sparkplug wrench -n04269668 spark transmitter -n04269822 spat, gaiter -n04269944 spatula -n04270147 spatula -n04270371 speakerphone -n04270576 speaking trumpet -n04270891 spear, lance, shaft -n04271148 spear, gig, fizgig, fishgig, lance -n04271531 specialty store -n04271793 specimen bottle -n04271891 spectacle -n04272054 spectacles, specs, eyeglasses, glasses -n04272389 spectator pump, spectator -n04272782 spectrograph -n04272928 spectrophotometer -n04273064 spectroscope, prism spectroscope -n04273285 speculum -n04273569 speedboat -n04273659 speed bump -n04273796 speedometer, speed indicator -n04273972 speed skate, racing skate -n04274686 spherometer -n04274985 sphygmomanometer -n04275093 spicemill -n04275175 spice rack -n04275283 spider -n04275548 spider web, spider's web -n04275661 spike -n04275904 spike -n04277352 spindle -n04277493 spindle, mandrel, mandril, arbor -n04277669 spindle -n04277826 spin dryer, spin drier -n04278247 spinet -n04278353 spinet -n04278447 spinnaker -n04278605 spinner -n04278932 spinning frame -n04279063 spinning jenny -n04279172 spinning machine -n04279353 spinning rod -n04279462 spinning wheel -n04279858 spiral bandage -n04279987 spiral ratchet screwdriver, ratchet screwdriver -n04280259 spiral spring -n04280373 spirit lamp -n04280487 spirit stove -n04280845 spirometer -n04280970 spit -n04281260 spittoon, cuspidor -n04281375 splashboard, splasher, dashboard -n04281571 splasher -n04281998 splice, splicing -n04282231 splicer -n04282494 splint -n04282872 split rail, fence rail -n04282992 Spode -n04283096 spoiler -n04283255 spoiler -n04283378 spoke, wheel spoke, radius -n04283585 spokeshave -n04283784 sponge cloth -n04283905 sponge mop -n04284002 spoon -n04284341 spoon -n04284438 Spork -n04284572 sporran -n04284869 sport kite, stunt kite -n04285008 sports car, sport car -n04285146 sports equipment -n04285622 sports implement -n04285803 sportswear, athletic wear, activewear -n04285965 sport utility, sport utility vehicle, S.U.V., SUV -n04286128 spot -n04286575 spotlight, spot -n04286960 spot weld, spot-weld -n04287351 spouter -n04287451 sprag -n04287747 spray gun -n04287898 spray paint -n04287986 spreader -n04288165 sprig -n04288272 spring -n04288533 spring balance, spring scale -n04288673 springboard -n04289027 sprinkler -n04289195 sprinkler system -n04289449 sprit -n04289576 spritsail -n04289690 sprocket, sprocket wheel -n04289827 sprocket -n04290079 spun yarn -n04290259 spur, gad -n04290507 spur gear, spur wheel -n04290615 sputnik -n04290762 spy satellite -n04291069 squad room -n04291242 square -n04291759 square knot -n04291992 square-rigger -n04292080 square sail -n04292221 squash ball -n04292414 squash racket, squash racquet, bat -n04292572 squawk box, squawker, intercom speaker -n04292921 squeegee -n04293119 squeezer -n04293258 squelch circuit, squelch, squelcher -n04293744 squinch -n04294212 stabilizer, stabiliser -n04294426 stabilizer -n04294614 stabilizer bar, anti-sway bar -n04294879 stable, stalls, horse barn -n04295081 stable gear, saddlery, tack -n04295353 stabling -n04295571 stacks -n04295777 staddle -n04295881 stadium, bowl, arena, sports stadium -n04296562 stage -n04297098 stagecoach, stage -n04297750 stained-glass window -n04297847 stair-carpet -n04298053 stair-rod -n04298661 stairwell -n04298765 stake -n04299215 stall, stand, sales booth -n04299370 stall -n04299963 stamp -n04300358 stamp mill, stamping mill -n04300509 stamping machine, stamper -n04300643 stanchion -n04301000 stand -n04301242 standard -n04301474 standard cell -n04301760 standard transmission, stick shift -n04302200 standing press -n04302863 stanhope -n04302988 Stanley Steamer -n04303095 staple -n04303258 staple -n04303357 staple gun, staplegun, tacker -n04303497 stapler, stapling machine -n04304215 starship, spaceship -n04304375 starter, starter motor, starting motor -n04304680 starting gate, starting stall -n04305016 Stassano furnace, electric-arc furnace -n04305210 Statehouse -n04305323 stately home -n04305471 state prison -n04305572 stateroom -n04305947 static tube -n04306080 station -n04306592 stator, stator coil -n04306847 statue -n04307419 stay -n04307767 staysail -n04307878 steakhouse, chophouse -n04307986 steak knife -n04308084 stealth aircraft -n04308273 stealth bomber -n04308397 stealth fighter -n04308583 steam bath, steam room, vapor bath, vapour bath -n04308807 steamboat -n04308915 steam chest -n04309049 steam engine -n04309348 steamer, steamship -n04309548 steamer -n04309833 steam iron -n04310018 steam locomotive -n04310157 steamroller, road roller -n04310507 steam shovel -n04310604 steam turbine -n04310721 steam whistle -n04310904 steel -n04311004 steel arch bridge -n04311174 steel drum -n04311595 steel mill, steelworks, steel plant, steel factory -n04312020 steel-wool pad -n04312154 steelyard, lever scale, beam scale -n04312432 steeple, spire -n04312654 steerage -n04312756 steering gear -n04312916 steering linkage -n04313220 steering system, steering mechanism -n04313503 steering wheel, wheel -n04313628 stele, stela -n04314107 stem-winder -n04314216 stencil -n04314522 Sten gun -n04314632 stenograph -n04314914 step, stair -n04315342 step-down transformer -n04315713 step stool -n04315828 step-up transformer -n04315948 stereo, stereophony, stereo system, stereophonic system -n04316498 stereoscope -n04316815 stern chaser -n04316924 sternpost -n04317063 sternwheeler -n04317175 stethoscope -n04317325 stewing pan, stewpan -n04317420 stick -n04317833 stick -n04317976 stick, control stick, joystick -n04318131 stick -n04318787 stile -n04318892 stiletto -n04318982 still -n04319545 stillroom, still room -n04319774 Stillson wrench -n04319937 stilt -n04320405 Stinger -n04320598 stink bomb, stench bomb -n04320871 stirrer -n04320973 stirrup, stirrup iron -n04321121 stirrup pump -n04321453 stob -n04322026 stock, gunstock -n04322531 stockade -n04322692 stockcar -n04322801 stock car -n04323519 stockinet, stockinette -n04323819 stocking -n04324120 stock-in-trade -n04324297 stockpot -n04324387 stockroom, stock room -n04324515 stocks -n04325041 stock saddle, Western saddle -n04325208 stockyard -n04325704 stole -n04325804 stomacher -n04325968 stomach pump -n04326547 stone wall -n04326676 stoneware -n04326799 stonework -n04326896 stool -n04327204 stoop, stoep -n04327544 stop bath, short-stop, short-stop bath -n04327682 stopcock, cock, turncock -n04328054 stopper knot -n04328186 stopwatch, stop watch -n04328329 storage battery, accumulator -n04328580 storage cell, secondary cell -n04328703 storage ring -n04328946 storage space -n04329477 storeroom, storage room, stowage -n04329681 storm cellar, cyclone cellar, tornado cellar -n04329834 storm door -n04329958 storm window, storm sash -n04330109 stoup, stoop -n04330189 stoup -n04330267 stove -n04330340 stove, kitchen stove, range, kitchen range, cooking stove -n04330669 stove bolt -n04330746 stovepipe -n04330896 stovepipe iron -n04330998 Stradavarius, Strad -n04331277 straight chair, side chair -n04331443 straightedge -n04331639 straightener -n04331765 straight flute, straight-fluted drill -n04331892 straight pin -n04332074 straight razor -n04332243 strainer -n04332580 straitjacket, straightjacket -n04332987 strap -n04333129 strap -n04333869 strap hinge, joint hinge -n04334105 strapless -n04334365 streamer fly -n04334504 streamliner -n04334599 street -n04335209 street -n04335435 streetcar, tram, tramcar, trolley, trolley car -n04335693 street clothes -n04335886 streetlight, street lamp -n04336792 stretcher -n04337157 stretcher -n04337287 stretch pants -n04337503 strickle -n04337650 strickle -n04338517 stringed instrument -n04338963 stringer -n04339062 stringer -n04339191 string tie -n04339638 strip -n04339879 strip lighting -n04340019 strip mall -n04340521 stroboscope, strobe, strobe light -n04340750 strongbox, deedbox -n04340935 stronghold, fastness -n04341133 strongroom -n04341288 strop -n04341414 structural member -n04341686 structure, construction -n04343511 student center -n04343630 student lamp -n04343740 student union -n04344003 stud finder -n04344734 studio apartment, studio -n04344873 studio couch, day bed -n04345028 study -n04345201 study hall -n04345787 stuffing nut, packing nut -n04346003 stump -n04346157 stun gun, stun baton -n04346328 stupa, tope -n04346428 sty, pigsty, pigpen -n04346511 stylus, style -n04346679 stylus -n04346855 sub-assembly -n04347119 subcompact, subcompact car -n04347519 submachine gun -n04347754 submarine, pigboat, sub, U-boat -n04348070 submarine torpedo -n04348184 submersible, submersible warship -n04348359 submersible -n04348988 subtracter -n04349189 subway token -n04349306 subway train -n04349401 subwoofer -n04349913 suction cup -n04350104 suction pump -n04350235 sudatorium, sudatory -n04350458 suede cloth, suede -n04350581 sugar bowl -n04350688 sugar refinery -n04350769 sugar spoon, sugar shell -n04350905 suit, suit of clothes -n04351550 suite, rooms -n04351699 suiting -n04353573 sulky -n04354026 summer house -n04354182 sumo ring -n04354387 sump -n04354487 sump pump -n04354589 sunbonnet -n04355115 Sunday best, Sunday clothes -n04355267 sun deck -n04355338 sundial -n04355511 sundress -n04355684 sundries -n04355821 sun gear -n04355933 sunglass -n04356056 sunglasses, dark glasses, shades -n04356595 sunhat, sun hat -n04356772 sunlamp, sun lamp, sunray lamp, sun-ray lamp -n04356925 sun parlor, sun parlour, sun porch, sunporch, sunroom, sun lounge, solarium -n04357121 sunroof, sunshine-roof -n04357314 sunscreen, sunblock, sun blocker -n04357531 sunsuit -n04357930 supercharger -n04358117 supercomputer -n04358256 superconducting supercollider -n04358491 superhighway, information superhighway -n04358707 supermarket -n04358874 superstructure -n04359034 supertanker -n04359124 supper club -n04359217 supplejack -n04359335 supply chamber -n04359500 supply closet -n04359589 support -n04360501 support -n04360798 support column -n04360914 support hose, support stocking -n04361095 supporting structure -n04361260 supporting tower -n04361937 surcoat -n04362624 surface gauge, surface gage, scribing block -n04362821 surface lift -n04362972 surface search radar -n04363082 surface ship -n04363210 surface-to-air missile, SAM -n04363412 surface-to-air missile system -n04363671 surfboat -n04363777 surcoat -n04363874 surgeon's knot -n04363991 surgery -n04364160 surge suppressor, surge protector, spike suppressor, spike arrester, lightning arrester -n04364397 surgical dressing -n04364545 surgical instrument -n04364827 surgical knife -n04364994 surplice -n04365112 surrey -n04365229 surtout -n04365328 surveillance system -n04365484 surveying instrument, surveyor's instrument -n04365751 surveyor's level -n04366033 sushi bar -n04366116 suspension, suspension system -n04366367 suspension bridge -n04366832 suspensory, suspensory bandage -n04367011 sustaining pedal, loud pedal -n04367371 suture, surgical seam -n04367480 swab, swob, mop -n04367746 swab -n04367950 swaddling clothes, swaddling bands -n04368109 swag -n04368235 swage block -n04368365 swagger stick -n04368496 swallow-tailed coat, swallowtail, morning coat -n04368695 swamp buggy, marsh buggy -n04368840 swan's down -n04369025 swathe, wrapping -n04369282 swatter, flyswatter, flyswat -n04369485 sweat bag -n04369618 sweatband -n04370048 sweater, jumper -n04370288 sweat pants, sweatpants -n04370456 sweatshirt -n04370600 sweatshop -n04370774 sweat suit, sweatsuit, sweats, workout suit -n04370955 sweep, sweep oar -n04371050 sweep hand, sweep-second -n04371430 swimming trunks, bathing trunks -n04371563 swimsuit, swimwear, bathing suit, swimming costume, bathing costume -n04371774 swing -n04371979 swing door, swinging door -n04372370 switch, electric switch, electrical switch -n04373089 switchblade, switchblade knife, flick-knife, flick knife -n04373428 switch engine, donkey engine -n04373563 swivel -n04373704 swivel chair -n04373795 swizzle stick -n04373894 sword, blade, brand, steel -n04374315 sword cane, sword stick -n04374521 S wrench -n04374735 synagogue, temple, tabernacle -n04374907 synchrocyclotron -n04375080 synchroflash -n04375241 synchromesh -n04375405 synchronous converter, rotary, rotary converter -n04375615 synchronous motor -n04375775 synchrotron -n04375926 synchroscope, synchronoscope, synchronizer, synchroniser -n04376400 synthesizer, synthesiser -n04376876 syringe -n04377057 system -n04378489 tabard -n04378651 Tabernacle -n04378956 tabi, tabis -n04379096 tab key, tab -n04379243 table -n04379964 table -n04380255 tablefork -n04380346 table knife -n04380533 table lamp -n04380916 table saw -n04381073 tablespoon -n04381450 tablet-armed chair -n04381587 table-tennis table, ping-pong table, pingpong table -n04381724 table-tennis racquet, table-tennis bat, pingpong paddle -n04381860 tabletop -n04381994 tableware -n04382334 tabor, tabour -n04382438 taboret, tabouret -n04382537 tachistoscope, t-scope -n04382695 tachograph -n04382880 tachometer, tach -n04383015 tachymeter, tacheometer -n04383130 tack -n04383301 tack hammer -n04383839 taffeta -n04383923 taffrail -n04384593 tailgate, tailboard -n04384910 taillight, tail lamp, rear light, rear lamp -n04385079 tailor-made -n04385157 tailor's chalk -n04385536 tailpipe -n04385799 tail rotor, anti-torque rotor -n04386051 tailstock -n04386456 take-up -n04386664 talaria -n04386792 talcum, talcum powder -n04387095 tam, tam-o'-shanter, tammy -n04387201 tambour -n04387261 tambour, embroidery frame, embroidery hoop -n04387400 tambourine -n04387531 tammy -n04387706 tamp, tamper, tamping bar -n04387932 Tampax -n04388040 tampion, tompion -n04388162 tampon -n04388473 tandoor -n04388574 tangram -n04388743 tank, storage tank -n04389033 tank, army tank, armored combat vehicle, armoured combat vehicle -n04389430 tankard -n04389521 tank car, tank -n04389718 tank destroyer -n04389854 tank engine, tank locomotive -n04389999 tanker plane -n04390483 tank shell -n04390577 tank top -n04390873 tannoy -n04390977 tap, spigot -n04391445 tapa, tappa -n04391838 tape, tape recording, taping -n04392113 tape, tapeline, tape measure -n04392526 tape deck -n04392764 tape drive, tape transport, transport -n04392985 tape player -n04393095 tape recorder, tape machine -n04393301 taper file -n04393549 tapestry, tapis -n04393808 tappet -n04393913 tap wrench -n04394031 tare -n04394261 target, butt -n04394421 target acquisition system -n04394630 tarmacadam, tarmac, macadam -n04395024 tarpaulin, tarp -n04395106 tartan, plaid -n04395332 tasset, tasse -n04395651 tattoo -n04395875 tavern, tap house -n04396226 tawse -n04396335 taximeter -n04396650 T-bar lift, T-bar, Alpine lift -n04396808 tea bag -n04396902 tea ball -n04397027 tea cart, teacart, tea trolley, tea wagon -n04397168 tea chest -n04397261 teaching aid -n04397452 teacup -n04397645 tea gown -n04397768 teakettle -n04397860 tea maker -n04398044 teapot -n04398497 teashop, teahouse, tearoom, tea parlor, tea parlour -n04398688 teaspoon -n04398834 tea-strainer -n04398951 tea table -n04399046 tea tray -n04399158 tea urn -n04399382 teddy, teddy bear -n04399537 tee, golf tee -n04399846 tee hinge, T hinge -n04400109 telecom hotel, telco building -n04400289 telecommunication system, telecom system, telecommunication equipment, telecom equipment -n04400499 telegraph, telegraphy -n04400737 telegraph key -n04400899 telemeter -n04401088 telephone, phone, telephone set -n04401578 telephone bell -n04401680 telephone booth, phone booth, call box, telephone box, telephone kiosk -n04401828 telephone cord, phone cord -n04401949 telephone jack, phone jack -n04402057 telephone line, phone line, telephone circuit, subscriber line, line -n04402342 telephone plug, phone plug -n04402449 telephone pole, telegraph pole, telegraph post -n04402580 telephone receiver, receiver -n04402746 telephone system, phone system -n04402984 telephone wire, telephone line, telegraph wire, telegraph line -n04403413 telephoto lens, zoom lens -n04403524 Teleprompter -n04403638 telescope, scope -n04403925 telescopic sight, telescope sight -n04404072 telethermometer -n04404200 teletypewriter, teleprinter, teletype machine, telex, telex machine -n04404412 television, television system -n04404817 television antenna, tv-antenna -n04404997 television camera, tv camera, camera -n04405540 television equipment, video equipment -n04405762 television monitor, tv monitor -n04405907 television receiver, television, television set, tv, tv set, idiot box, boob tube, telly, goggle box -n04406239 television room, tv room -n04406552 television transmitter -n04406687 telpher, telfer -n04406817 telpherage, telferage -n04407257 tempera, poster paint, poster color, poster colour -n04407435 temple -n04407686 temple -n04408871 temporary hookup, patch -n04409011 tender, supply ship -n04409128 tender, ship's boat, pinnace, cutter -n04409279 tender -n04409384 tenement, tenement house -n04409515 tennis ball -n04409625 tennis camp -n04409806 tennis racket, tennis racquet -n04409911 tenon -n04410086 tenor drum, tom-tom -n04410365 tenoroon -n04410485 tenpenny nail -n04410565 tenpin -n04410663 tensimeter -n04410760 tensiometer -n04410886 tensiometer -n04411019 tensiometer -n04411264 tent, collapsible shelter -n04411835 tenter -n04411966 tenterhook -n04412097 tent-fly, rainfly, fly sheet, fly, tent flap -n04412300 tent peg -n04412416 tepee, tipi, teepee -n04413151 terminal, pole -n04413419 terminal -n04413969 terraced house -n04414101 terra cotta -n04414199 terrarium -n04414319 terra sigillata, Samian ware -n04414476 terry, terry cloth, terrycloth -n04414675 Tesla coil -n04414909 tessera -n04415257 test equipment -n04415663 test rocket, research rocket, test instrument vehicle -n04415815 test room, testing room -n04416005 testudo -n04416901 tetraskelion, tetraskele -n04417086 tetrode -n04417180 textile machine -n04417361 textile mill -n04417672 thatch, thatched roof -n04417809 theater, theatre, house -n04418357 theater curtain, theatre curtain -n04418644 theater light -n04419073 theodolite, transit -n04419642 theremin -n04419868 thermal printer -n04420024 thermal reactor -n04420720 thermocouple, thermocouple junction -n04421083 thermoelectric thermometer, thermel, electric thermometer -n04421258 thermograph, thermometrograph -n04421417 thermograph -n04421582 thermohydrometer, thermogravimeter -n04421740 thermojunction -n04421872 thermometer -n04422409 thermonuclear reactor, fusion reactor -n04422566 thermopile -n04422727 thermos, thermos bottle, thermos flask -n04422875 thermostat, thermoregulator -n04423552 thigh pad -n04423687 thill -n04423845 thimble -n04424692 thinning shears -n04425804 third base, third -n04425977 third gear, third -n04426184 third rail -n04426316 thong -n04426427 thong -n04427216 three-centered arch, basket-handle arch -n04427473 three-decker -n04427559 three-dimensional radar, 3d radar -n04427715 three-piece suit -n04427857 three-quarter binding -n04428008 three-way switch, three-point switch -n04428191 thresher, thrasher, threshing machine -n04428382 threshing floor -n04428634 thriftshop, second-hand store -n04429038 throat protector -n04429376 throne -n04430475 thrust bearing -n04430605 thruster -n04430896 thumb -n04431025 thumbhole -n04431436 thumbscrew -n04431648 thumbstall -n04431745 thumbtack, drawing pin, pushpin -n04431925 thunderer -n04432043 thwart, cross thwart -n04432203 tiara -n04432662 ticking -n04432785 tickler coil -n04433377 tie, tie beam -n04433585 tie, railroad tie, crosstie, sleeper -n04434207 tie rack -n04434531 tie rod -n04434932 tights, leotards -n04435180 tile -n04435552 tile cutter -n04435653 tile roof -n04435759 tiller -n04435870 tilter -n04436012 tilt-top table, tip-top table, tip table -n04436185 timber -n04436329 timber -n04436401 timber hitch -n04436542 timbrel -n04436832 time bomb, infernal machine -n04436992 time capsule -n04437276 time clock -n04437380 time-delay measuring instrument, time-delay measuring system -n04437670 time-fuse -n04437953 timepiece, timekeeper, horologe -n04438304 timer -n04438507 timer -n04438643 time-switch -n04438897 tin -n04439505 tinderbox -n04439585 tine -n04439712 tinfoil, tin foil -n04440597 tippet -n04440963 tire chain, snow chain -n04441093 tire iron, tire tool -n04441528 titfer -n04441662 tithe barn -n04441790 titrator -n04442312 toaster -n04442441 toaster oven -n04442582 toasting fork -n04442741 toastrack -n04443164 tobacco pouch -n04443257 tobacco shop, tobacconist shop, tobacconist -n04443433 toboggan -n04443766 toby, toby jug, toby fillpot jug -n04444121 tocsin, warning bell -n04444218 toe -n04444749 toecap -n04444953 toehold -n04445040 toga -n04445154 toga virilis -n04445327 toggle -n04445610 toggle bolt -n04445782 toggle joint -n04445952 toggle switch, toggle, on-off switch, on/off switch -n04446162 togs, threads, duds -n04446276 toilet, lavatory, lav, can, john, privy, bathroom -n04446844 toilet bag, sponge bag -n04447028 toilet bowl -n04447156 toilet kit, travel kit -n04447276 toilet powder, bath powder, dusting powder -n04447443 toiletry, toilet articles -n04447861 toilet seat -n04448070 toilet water, eau de toilette -n04448185 tokamak -n04448361 token -n04449290 tollbooth, tolbooth, tollhouse -n04449449 toll bridge -n04449550 tollgate, tollbar -n04449700 toll line -n04449966 tomahawk, hatchet -n04450133 Tommy gun, Thompson submachine gun -n04450243 tomograph -n04450465 tone arm, pickup, pickup arm -n04450640 toner -n04450749 tongs, pair of tongs -n04450994 tongue -n04451139 tongue and groove joint -n04451318 tongue depressor -n04451636 tonometer -n04451818 tool -n04452528 tool bag -n04452615 toolbox, tool chest, tool cabinet, tool case -n04452757 toolshed, toolhouse -n04452848 tooth -n04453037 tooth -n04453156 toothbrush -n04453390 toothpick -n04453666 top -n04453910 top, cover -n04454654 topgallant, topgallant mast -n04454792 topgallant, topgallant sail -n04454908 topiary -n04455048 topknot -n04455250 topmast -n04455579 topper -n04455652 topsail -n04456011 toque -n04456115 torch -n04456472 torpedo -n04456734 torpedo -n04457157 torpedo -n04457326 torpedo boat -n04457474 torpedo-boat destroyer -n04457638 torpedo tube -n04457767 torque converter -n04457910 torque wrench -n04458201 torture chamber -n04458633 totem pole -n04458843 touch screen, touchscreen -n04459018 toupee, toupe -n04459122 touring car, phaeton, tourer -n04459243 tourist class, third class -n04459362 towel -n04459610 toweling, towelling -n04459773 towel rack, towel horse -n04459909 towel rail, towel bar -n04460130 tower -n04461437 town hall -n04461570 towpath, towing path -n04461696 tow truck, tow car, wrecker -n04461879 toy -n04462011 toy box, toy chest -n04462240 toyshop -n04462576 trace detector -n04463679 track, rail, rails, runway -n04464125 track -n04464615 trackball -n04464852 tracked vehicle -n04465050 tract house -n04465203 tract housing -n04465358 traction engine -n04465501 tractor -n04465666 tractor -n04466871 trail bike, dirt bike, scrambler -n04467099 trailer, house trailer -n04467307 trailer -n04467506 trailer camp, trailer park -n04467665 trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi -n04467899 trailing edge -n04468005 train, railroad train -n04469003 tramline, tramway, streetcar track -n04469251 trammel -n04469514 trampoline -n04469684 tramp steamer, tramp -n04469813 tramway, tram, aerial tramway, cable tramway, ropeway -n04470741 transdermal patch, skin patch -n04471148 transept -n04471315 transformer -n04471632 transistor, junction transistor, electronic transistor -n04471912 transit instrument -n04472243 transmission, transmission system -n04472563 transmission shaft -n04472726 transmitter, sender -n04472961 transom, traverse -n04473108 transom, transom window, fanlight -n04473275 transponder -n04473884 transporter -n04474035 transporter, car transporter -n04474187 transport ship -n04474466 trap -n04475309 trap door -n04475411 trapeze -n04475496 trave, traverse, crossbeam, crosspiece -n04475631 travel iron -n04475749 trawl, dragnet, trawl net -n04475900 trawl, trawl line, spiller, setline, trotline -n04476116 trawler, dragger -n04476259 tray -n04476526 tray cloth -n04476831 tread -n04476972 tread -n04477219 treadmill, treadwheel, tread-wheel -n04477387 treadmill -n04477548 treasure chest -n04477725 treasure ship -n04478066 treenail, trenail, trunnel -n04478383 trefoil arch -n04478512 trellis, treillage -n04478657 trench -n04479046 trench coat -n04479287 trench knife -n04479405 trepan -n04479526 trepan, trephine -n04479694 trestle -n04479823 trestle -n04479939 trestle bridge -n04480033 trestle table -n04480141 trestlework -n04480303 trews -n04480527 trial balloon -n04480853 triangle -n04480995 triangle -n04481524 triclinium -n04481642 triclinium -n04482177 tricorn, tricorne -n04482297 tricot -n04482393 tricycle, trike, velocipede -n04482975 trident -n04483073 trigger -n04483307 trimaran -n04483925 trimmer -n04484024 trimmer arch -n04484432 triode -n04485082 tripod -n04485423 triptych -n04485586 trip wire -n04485750 trireme -n04485884 triskelion, triskele -n04486054 triumphal arch -n04486213 trivet -n04486322 trivet -n04486616 troika -n04486934 troll -n04487081 trolleybus, trolley coach, trackless trolley -n04487394 trombone -n04487724 troop carrier, troop transport -n04487894 troopship -n04488202 trophy case -n04488427 trough -n04488530 trouser -n04488742 trouser cuff -n04488857 trouser press, pants presser -n04489008 trouser, pant -n04489695 trousseau -n04489817 trowel -n04490091 truck, motortruck -n04491312 trumpet arch -n04491388 truncheon, nightstick, baton, billy, billystick, billy club -n04491638 trundle bed, trundle, truckle bed, truckle -n04491769 trunk -n04491934 trunk hose -n04492060 trunk lid -n04492157 trunk line -n04492375 truss -n04492749 truss bridge -n04493109 try square -n04493259 T-square -n04493381 tub, vat -n04494204 tube, vacuum tube, thermionic vacuum tube, thermionic tube, electron tube, thermionic valve -n04495051 tuck box -n04495183 tucker -n04495310 tucker-bag -n04495450 tuck shop -n04495555 Tudor arch, four-centered arch -n04495698 tudung -n04495843 tugboat, tug, towboat, tower -n04496614 tulle -n04496726 tumble-dryer, tumble drier -n04496872 tumbler -n04497249 tumbrel, tumbril -n04497442 tun -n04497570 tunic -n04497801 tuning fork -n04498275 tupik, tupek, sealskin tent -n04498389 turban -n04498523 turbine -n04498873 turbogenerator -n04499062 tureen -n04499300 Turkish bath -n04499446 Turkish towel, terry towel -n04499554 Turk's head -n04499810 turnbuckle -n04500060 turner, food turner -n04500390 turnery -n04501127 turnpike -n04501281 turnspit -n04501370 turnstile -n04501550 turntable -n04501837 turntable, lazy Susan -n04501947 turret -n04502059 turret clock -n04502197 turtleneck, turtle, polo-neck -n04502502 tweed -n04502670 tweeter -n04502851 twenty-two, .22 -n04502989 twenty-two pistol -n04503073 twenty-two rifle -n04503155 twill -n04503269 twill, twill weave -n04503413 twin bed -n04503499 twinjet -n04503593 twist bit, twist drill -n04503705 two-by-four -n04504038 two-man tent -n04504141 two-piece, two-piece suit, lounge suit -n04504770 typesetting machine -n04505036 typewriter -n04505345 typewriter carriage -n04505470 typewriter keyboard -n04505888 tyrolean, tirolean -n04506289 uke, ukulele -n04506402 ulster -n04506506 ultracentrifuge -n04506688 ultramicroscope, dark-field microscope -n04506895 Ultrasuede -n04506994 ultraviolet lamp, ultraviolet source -n04507155 umbrella -n04507326 umbrella tent -n04507453 undercarriage -n04507689 undercoat, underseal -n04508163 undergarment, unmentionable -n04508489 underpants -n04508949 underwear, underclothes, underclothing -n04509171 undies -n04509260 uneven parallel bars, uneven bars -n04509417 unicycle, monocycle -n04509592 uniform -n04510706 universal joint, universal -n04511002 university -n04513827 upholstery -n04513998 upholstery material -n04514095 upholstery needle -n04514241 uplift -n04514648 upper berth, upper -n04515003 upright, upright piano -n04515444 upset, swage -n04515729 upstairs -n04515890 urceole -n04516116 urn -n04516214 urn -n04516354 used-car, secondhand car -n04516672 utensil -n04517211 Uzi -n04517408 vacation home -n04517823 vacuum, vacuum cleaner -n04517999 vacuum chamber -n04518132 vacuum flask, vacuum bottle -n04518343 vacuum gauge, vacuum gage -n04518643 Valenciennes, Valenciennes lace -n04518764 valise -n04519153 valve -n04519536 valve -n04519728 valve-in-head engine -n04519887 vambrace, lower cannon -n04520170 van -n04520382 van, caravan -n04520784 vane -n04520962 vaporizer, vaporiser -n04521571 variable-pitch propeller -n04521863 variometer -n04521987 varnish -n04522168 vase -n04523525 vault -n04523831 vault, bank vault -n04524142 vaulting horse, long horse, buck -n04524313 vehicle -n04524594 Velcro -n04524716 velocipede -n04524941 velour, velours -n04525038 velvet -n04525191 velveteen -n04525305 vending machine -n04525417 veneer, veneering -n04525584 Venetian blind -n04525821 Venn diagram, Venn's diagram -n04526520 ventilation, ventilation system, ventilating system -n04526800 ventilation shaft -n04526964 ventilator -n04527648 veranda, verandah, gallery -n04528079 verdigris -n04528968 vernier caliper, vernier micrometer -n04529108 vernier scale, vernier -n04529681 vertical file -n04529962 vertical stabilizer, vertical stabiliser, vertical fin, tail fin, tailfin -n04530283 vertical tail -n04530456 Very pistol, Verey pistol -n04530566 vessel, watercraft -n04531098 vessel -n04531873 vest, waistcoat -n04532022 vestiture -n04532106 vestment -n04532398 vest pocket -n04532504 vestry, sacristy -n04532670 viaduct -n04532831 vibraphone, vibraharp, vibes -n04533042 vibrator -n04533199 vibrator -n04533499 Victrola -n04533594 vicuna -n04533700 videocassette -n04533802 videocassette recorder, VCR -n04533946 videodisk, videodisc, DVD -n04534127 video recording, video -n04534359 videotape -n04534520 videotape -n04534895 vigil light, vigil candle -n04535252 villa -n04535370 villa -n04535524 villa -n04536153 viol -n04536335 viola -n04536465 viola da braccio -n04536595 viola da gamba, gamba, bass viol -n04536765 viola d'amore -n04536866 violin, fiddle -n04537436 virginal, pair of virginals -n04538249 viscometer, viscosimeter -n04538403 viscose rayon, viscose -n04538552 vise, bench vise -n04538878 visor, vizor -n04539053 visual display unit, VDU -n04539203 vivarium -n04539407 Viyella -n04539794 voile -n04540053 volleyball -n04540255 volleyball net -n04540397 voltage regulator -n04540761 voltaic cell, galvanic cell, primary cell -n04541136 voltaic pile, pile, galvanic pile -n04541320 voltmeter -n04541662 vomitory -n04541777 von Neumann machine -n04541987 voting booth -n04542095 voting machine -n04542329 voussoir -n04542474 vox angelica, voix celeste -n04542595 vox humana -n04542715 waders -n04542858 wading pool -n04542943 waffle iron -n04543158 wagon, waggon -n04543509 wagon, coaster wagon -n04543636 wagon tire -n04543772 wagon wheel -n04543924 wain -n04543996 wainscot, wainscoting, wainscotting -n04544325 wainscoting, wainscotting -n04544450 waist pack, belt bag -n04545305 walker, baby-walker, go-cart -n04545471 walker, Zimmer, Zimmer frame -n04545748 walker -n04545858 walkie-talkie, walky-talky -n04545984 walk-in -n04546081 walking shoe -n04546194 walking stick -n04546340 Walkman -n04546595 walk-up apartment, walk-up -n04546855 wall -n04547592 wall -n04548280 wall clock -n04548362 wallet, billfold, notecase, pocketbook -n04549028 wall tent -n04549122 wall unit -n04549629 wand -n04549721 Wankel engine, Wankel rotary engine, epitrochoidal engine -n04549919 ward, hospital ward -n04550184 wardrobe, closet, press -n04550676 wardroom -n04551055 warehouse, storage warehouse -n04551833 warming pan -n04552097 war paint -n04552348 warplane, military plane -n04552551 war room -n04552696 warship, war vessel, combat ship -n04553389 wash -n04553561 wash-and-wear -n04553703 washbasin, handbasin, washbowl, lavabo, wash-hand basin -n04554211 washboard, splashboard -n04554406 washboard -n04554684 washer, automatic washer, washing machine -n04554871 washer -n04554998 washhouse -n04555291 washroom -n04555400 washstand, wash-hand stand -n04555600 washtub -n04555700 wastepaper basket, waste-paper basket, wastebasket, waste basket, circular file -n04555897 watch, ticker -n04556408 watch cap -n04556533 watch case -n04556664 watch glass -n04556948 watchtower -n04557308 water-base paint -n04557522 water bed -n04557648 water bottle -n04557751 water butt -n04558059 water cart -n04558199 water chute -n04558478 water closet, closet, W.C., loo -n04558804 watercolor, water-color, watercolour, water-colour -n04559023 water-cooled reactor -n04559166 water cooler -n04559451 water faucet, water tap, tap, hydrant -n04559620 water filter -n04559730 water gauge, water gage, water glass -n04559910 water glass -n04559994 water hazard -n04560113 water heater, hot-water heater, hot-water tank -n04560292 watering can, watering pot -n04560502 watering cart -n04560619 water jacket -n04560804 water jug -n04560882 water jump -n04561010 water level -n04561287 water meter -n04561422 water mill -n04561734 waterproof -n04561857 waterproofing -n04561965 water pump -n04562122 water scooter, sea scooter, scooter -n04562262 water ski -n04562496 waterspout -n04562935 water tower -n04563020 water wagon, water waggon -n04563204 waterwheel, water wheel -n04563413 waterwheel, water wheel -n04563560 water wings -n04563790 waterworks -n04564278 wattmeter -n04564581 waxwork, wax figure -n04565039 ways, shipway, slipway -n04565375 weapon, arm, weapon system -n04566257 weaponry, arms, implements of war, weapons system, munition -n04566561 weapons carrier -n04566756 weathercock -n04567098 weatherglass -n04567593 weather satellite, meteorological satellite -n04567746 weather ship -n04568069 weathervane, weather vane, vane, wind vane -n04568557 web, entanglement -n04568713 web -n04568841 webbing -n04569063 webcam -n04569520 wedge -n04569822 wedge -n04570118 wedgie -n04570214 Wedgwood -n04570416 weeder, weed-whacker -n04570532 weeds, widow's weeds -n04570815 weekender -n04570958 weighbridge -n04571292 weight, free weight, exercising weight -n04571566 weir -n04571686 weir -n04571800 welcome wagon -n04571958 weld -n04572121 welder's mask -n04572235 weldment -n04572935 well -n04573045 wellhead -n04573281 welt -n04573379 Weston cell, cadmium cell -n04573513 wet bar -n04573625 wet-bulb thermometer -n04573832 wet cell -n04573937 wet fly -n04574067 wet suit -n04574348 whaleboat -n04574471 whaler, whaling ship -n04574606 whaling gun -n04574999 wheel -n04575723 wheel -n04575824 wheel and axle -n04576002 wheelchair -n04576211 wheeled vehicle -n04576971 wheelwork -n04577139 wherry -n04577293 wherry, Norfolk wherry -n04577426 whetstone -n04577567 whiffletree, whippletree, swingletree -n04577769 whip -n04578112 whipcord -n04578329 whipping post -n04578559 whipstitch, whipping, whipstitching -n04578708 whirler -n04578801 whisk, whisk broom -n04578934 whisk -n04579056 whiskey bottle -n04579145 whiskey jug -n04579230 whispering gallery, whispering dome -n04579432 whistle -n04579667 whistle -n04579986 white -n04580493 white goods -n04581102 whitewash -n04581595 whorehouse, brothel, bordello, bagnio, house of prostitution, house of ill repute, bawdyhouse, cathouse, sporting house -n04581829 wick, taper -n04582205 wicker, wickerwork, caning -n04582349 wicker basket -n04582771 wicket, hoop -n04582869 wicket -n04583022 wickiup, wikiup -n04583212 wide-angle lens, fisheye lens -n04583620 widebody aircraft, wide-body aircraft, wide-body, twin-aisle airplane -n04583888 wide wale -n04583967 widow's walk -n04584056 Wiffle, Wiffle Ball -n04584207 wig -n04584373 wigwam -n04585128 Wilton, Wilton carpet -n04585318 wimple -n04585456 wincey -n04585626 winceyette -n04585745 winch, windlass -n04585980 Winchester -n04586072 windbreak, shelterbelt -n04586581 winder, key -n04586932 wind instrument, wind -n04587327 windjammer -n04587404 windmill, aerogenerator, wind generator -n04587559 windmill -n04587648 window -n04588739 window -n04589190 window blind -n04589325 window box -n04589434 window envelope -n04589593 window frame -n04589890 window screen -n04590021 window seat -n04590129 window shade -n04590263 windowsill -n04590553 windshield, windscreen -n04590746 windshield wiper, windscreen wiper, wiper, wiper blade -n04590933 Windsor chair -n04591056 Windsor knot -n04591157 Windsor tie -n04591249 wind tee -n04591359 wind tunnel -n04591517 wind turbine -n04591631 wine bar -n04591713 wine bottle -n04591887 wine bucket, wine cooler -n04592005 wine cask, wine barrel -n04592099 wineglass -n04592356 winepress -n04592465 winery, wine maker -n04592596 wineskin -n04592741 wing -n04593077 wing chair -n04593185 wing nut, wing-nut, wing screw, butterfly nut, thumbnut -n04593376 wing tip -n04593524 wing tip -n04593629 winker, blinker, blinder -n04593866 wiper, wiper arm, contact arm -n04594114 wiper motor -n04594218 wire -n04594489 wire, conducting wire -n04594742 wire cloth -n04594828 wire cutter -n04594919 wire gauge, wire gage -n04595028 wireless local area network, WLAN, wireless fidelity, WiFi -n04595285 wire matrix printer, wire printer, stylus printer -n04595501 wire recorder -n04595611 wire stripper -n04595762 wirework, grillwork -n04595855 wiring -n04596116 wishing cap -n04596492 witness box, witness stand -n04596742 wok -n04596852 woman's clothing -n04597066 wood -n04597309 woodcarving -n04597400 wood chisel -n04597804 woodenware -n04597913 wooden spoon -n04598136 woodscrew -n04598318 woodshed -n04598416 wood vise, woodworking vise, shoulder vise -n04598582 woodwind, woodwind instrument, wood -n04598965 woof, weft, filling, pick -n04599124 woofer -n04599235 wool, woolen, woollen -n04600312 workbasket, workbox, workbag -n04600486 workbench, work bench, bench -n04600912 work-clothing, work-clothes -n04601041 workhouse -n04601159 workhouse -n04601938 workpiece -n04602762 workroom -n04602840 works, workings -n04602956 work-shirt -n04603399 workstation -n04603729 worktable, work table -n04603872 workwear -n04604276 World Wide Web, WWW, web -n04604644 worm fence, snake fence, snake-rail fence, Virginia fence -n04604806 worm gear -n04605057 worm wheel -n04605163 worsted -n04605321 worsted, worsted yarn -n04605446 wrap, wrapper -n04605572 wraparound -n04605726 wrapping, wrap, wrapper -n04606251 wreck -n04606574 wrench, spanner -n04607035 wrestling mat -n04607242 wringer -n04607640 wrist pad -n04607759 wrist pin, gudgeon pin -n04607869 wristwatch, wrist watch -n04607982 writing arm -n04608329 writing desk -n04608435 writing desk -n04608567 writing implement -n04608809 xerographic printer -n04608923 Xerox, xerographic copier, Xerox machine -n04609531 X-ray film -n04609651 X-ray machine -n04609811 X-ray tube -n04610013 yacht, racing yacht -n04610176 yacht chair -n04610274 yagi, Yagi aerial -n04610503 yard -n04610676 yard -n04611351 yardarm -n04611795 yard marker -n04611916 yardstick, yard measure -n04612026 yarmulke, yarmulka, yarmelke -n04612159 yashmak, yashmac -n04612257 yataghan -n04612373 yawl, dandy -n04612504 yawl -n04612840 yoke -n04613015 yoke -n04613158 yoke, coupling -n04613696 yurt -n04613939 Zamboni -n04614505 zero -n04614655 ziggurat, zikkurat, zikurat -n04614844 zill -n04615149 zip gun -n04615226 zither, cither, zithern -n04615644 zoot suit -n04682018 shading -n04950713 grain -n04950952 wood grain, woodgrain, woodiness -n04951071 graining, woodgraining -n04951186 marbleization, marbleisation, marbleizing, marbleising -n04951373 light, lightness -n04951716 aura, aureole, halo, nimbus, glory, gloriole -n04951875 sunniness -n04953296 glint -n04953678 opalescence, iridescence -n04955160 polish, gloss, glossiness, burnish -n04957356 primary color for pigments, primary colour for pigments -n04957589 primary color for light, primary colour for light -n04958634 colorlessness, colourlessness, achromatism, achromaticity -n04958865 mottle -n04959061 achromia -n04959230 shade, tint, tincture, tone -n04959672 chromatic color, chromatic colour, spectral color, spectral colour -n04960277 black, blackness, inkiness -n04960582 coal black, ebony, jet black, pitch black, sable, soot black -n04961062 alabaster -n04961331 bone, ivory, pearl, off-white -n04961691 gray, grayness, grey, greyness -n04962062 ash grey, ash gray, silver, silver grey, silver gray -n04962240 charcoal, charcoal grey, charcoal gray, oxford grey, oxford gray -n04963111 sanguine -n04963307 Turkey red, alizarine red -n04963588 crimson, ruby, deep red -n04963740 dark red -n04964001 claret -n04964799 fuschia -n04964878 maroon -n04965179 orange, orangeness -n04965451 reddish orange -n04965661 yellow, yellowness -n04966543 gamboge, lemon, lemon yellow, maize -n04966941 pale yellow, straw, wheat -n04967191 green, greenness, viridity -n04967561 greenishness -n04967674 sea green -n04967801 sage green -n04967882 bottle green -n04968056 emerald -n04968139 olive green, olive-green -n04968749 jade green, jade -n04968895 blue, blueness -n04969242 azure, cerulean, sapphire, lazuline, sky-blue -n04969540 steel blue -n04969798 greenish blue, aqua, aquamarine, turquoise, cobalt blue, peacock blue -n04969952 purplish blue, royal blue -n04970059 purple, purpleness -n04970312 Tyrian purple -n04970398 indigo -n04970470 lavender -n04970631 reddish purple, royal purple -n04970916 pink -n04971211 carnation -n04971313 rose, rosiness -n04972350 chestnut -n04972451 chocolate, coffee, deep brown, umber, burnt umber -n04972801 light brown -n04973020 tan, topaz -n04973291 beige, ecru -n04973386 reddish brown, sepia, burnt sienna, Venetian red, mahogany -n04973585 brick red -n04973669 copper, copper color -n04973816 Indian red -n04974145 puce -n04974340 olive -n04974859 ultramarine -n04975739 complementary color, complementary -n04976319 pigmentation -n04976952 complexion, skin color, skin colour -n04977412 ruddiness, rosiness -n04978561 nonsolid color, nonsolid colour, dithered color, dithered colour -n04979002 aposematic coloration, warning coloration -n04979307 cryptic coloration -n04981658 ring -n05102764 center of curvature, centre of curvature -n05218119 cadaver, corpse, stiff, clay, remains -n05233741 mandibular notch -n05235879 rib -n05238282 skin, tegument, cutis -n05239437 skin graft -n05241218 epidermal cell -n05241485 melanocyte -n05241662 prickle cell -n05242070 columnar cell, columnar epithelial cell -n05242239 spongioblast -n05242928 squamous cell -n05244421 amyloid plaque, amyloid protein plaque -n05244755 dental plaque, bacterial plaque -n05244934 macule, macula -n05245192 freckle, lentigo -n05257476 bouffant -n05257967 sausage curl -n05258051 forelock -n05258627 spit curl, kiss curl -n05259914 pigtail -n05260127 pageboy -n05260240 pompadour -n05261310 thatch -n05262422 soup-strainer, toothbrush -n05262534 mustachio, moustachio, handle-bars -n05262698 walrus mustache, walrus moustache -n05263183 stubble -n05263316 vandyke beard, vandyke -n05263448 soul patch, Attilio -n05265736 esophageal smear -n05266096 paraduodenal smear, duodenal smear -n05266879 specimen -n05278922 punctum -n05279953 glenoid fossa, glenoid cavity -n05282652 diastema -n05285623 marrow, bone marrow -n05302499 mouth, oral cavity, oral fissure, rima oris -n05314075 canthus -n05399034 milk -n05399243 mother's milk -n05399356 colostrum, foremilk -n05418717 vein, vena, venous blood vessel -n05427346 ganglion cell, gangliocyte -n05442594 X chromosome -n05447757 embryonic cell, formative cell -n05448704 myeloblast -n05448827 sideroblast -n05449196 osteocyte -n05449661 megalocyte, macrocyte -n05449959 leukocyte, leucocyte, white blood cell, white cell, white blood corpuscle, white corpuscle, WBC -n05450617 histiocyte -n05451099 fixed phagocyte -n05451384 lymphocyte, lymph cell -n05453412 monoblast -n05453657 neutrophil, neutrophile -n05453815 microphage -n05454833 sickle cell -n05454978 siderocyte -n05455113 spherocyte -n05458173 ootid -n05458576 oocyte -n05459101 spermatid -n05459457 Leydig cell, Leydig's cell -n05459769 striated muscle cell, striated muscle fiber -n05460759 smooth muscle cell -n05464534 Ranvier's nodes, nodes of Ranvier -n05467054 neuroglia, glia -n05467758 astrocyte -n05468098 protoplasmic astrocyte -n05468739 oligodendrocyte -n05469664 proprioceptor -n05469861 dendrite -n05475397 sensory fiber, afferent fiber -n05482922 subarachnoid space -n05486510 cerebral cortex, cerebral mantle, pallium, cortex -n05491154 renal cortex -n05526957 prepuce, foreskin -n05538625 head, caput -n05539947 scalp -n05541509 frontal eminence -n05542893 suture, sutura, fibrous joint -n05545879 foramen magnum -n05571341 esophagogastric junction, oesophagogastric junction -n05578095 heel -n05581932 cuticle -n05584746 hangnail, agnail -n05586759 exoskeleton -n05604434 abdominal wall -n05716342 lemon -n06008896 coordinate axis -n06209940 landscape -n06254669 medium -n06255081 vehicle -n06255613 paper -n06259898 channel, transmission channel -n06262567 film, cinema, celluloid -n06262943 silver screen -n06263202 free press -n06263369 press, public press -n06263609 print media -n06263762 storage medium, data-storage medium -n06263895 magnetic storage medium, magnetic medium, magnetic storage -n06266417 journalism, news media -n06266633 Fleet Street -n06266710 photojournalism -n06266878 news photography -n06266973 rotogravure -n06267145 newspaper, paper -n06267564 daily -n06267655 gazette -n06267758 school newspaper, school paper -n06267893 tabloid, rag, sheet -n06267991 yellow journalism, tabloid, tab -n06271778 telecommunication, telecom -n06272290 telephone, telephony -n06272612 voice mail, voicemail -n06272803 call, phone call, telephone call -n06273207 call-back -n06273294 collect call -n06273414 call forwarding -n06273555 call-in -n06273743 call waiting -n06273890 crank call -n06273986 local call -n06274092 long distance, long-distance call, trunk call -n06274292 toll call -n06274546 wake-up call -n06274760 three-way calling -n06274921 telegraphy -n06275095 cable, cablegram, overseas telegram -n06275353 wireless -n06275471 radiotelegraph, radiotelegraphy, wireless telegraphy -n06276501 radiotelephone, radiotelephony, wireless telephone -n06276697 broadcasting -n06276902 Rediffusion -n06277025 multiplex -n06277135 radio, radiocommunication, wireless -n06277280 television, telecasting, TV, video -n06278338 cable television, cable -n06278475 high-definition television, HDTV -n06281040 reception -n06281175 signal detection, detection -n06340977 Hakham -n06359193 web site, website, internet site, site -n06359467 chat room, chatroom -n06359657 portal site, portal -n06415688 jotter -n06417096 breviary -n06418693 wordbook -n06419354 desk dictionary, collegiate dictionary -n06423496 reckoner, ready reckoner -n06470073 document, written document, papers -n06591815 album, record album -n06592078 concept album -n06592281 rock opera -n06592421 tribute album, benefit album -n06595351 magazine, mag -n06596179 colour supplement -n06596364 comic book -n06596474 news magazine -n06596607 pulp, pulp magazine -n06596727 slick, slick magazine, glossy -n06596845 trade magazine -n06613686 movie, film, picture, moving picture, moving-picture show, motion picture, motion-picture show, picture show, pic, flick -n06614901 outtake -n06616216 shoot-'em-up -n06618653 spaghetti Western -n06625062 encyclical, encyclical letter -n06785654 crossword puzzle, crossword -n06793231 sign -n06794110 street sign -n06874185 traffic light, traffic signal, stoplight -n06883725 swastika, Hakenkreuz -n06892775 concert -n06998748 artwork, art, graphics, nontextual matter -n07005523 lobe -n07248320 book jacket, dust cover, dust jacket, dust wrapper -n07273802 cairn -n07461050 three-day event -n07556406 comfort food -n07556637 comestible, edible, eatable, pabulum, victual, victuals -n07556872 tuck -n07556970 course -n07557165 dainty, delicacy, goody, kickshaw, treat -n07557434 dish -n07560193 fast food -n07560331 finger food -n07560422 ingesta -n07560542 kosher -n07560652 fare -n07560903 diet -n07561112 diet -n07561590 dietary -n07561848 balanced diet -n07562017 bland diet, ulcer diet -n07562172 clear liquid diet -n07562379 diabetic diet -n07562495 dietary supplement -n07562651 carbohydrate loading, carbo loading -n07562881 fad diet -n07562984 gluten-free diet -n07563207 high-protein diet -n07563366 high-vitamin diet, vitamin-deficiency diet -n07563642 light diet -n07563800 liquid diet -n07564008 low-calorie diet -n07564101 low-fat diet -n07564292 low-sodium diet, low-salt diet, salt-free diet -n07564515 macrobiotic diet -n07564629 reducing diet, obesity diet -n07564796 soft diet, pap, spoon food -n07564971 vegetarianism -n07565083 menu -n07565161 chow, chuck, eats, grub -n07565259 board, table -n07565608 mess -n07565725 ration -n07565945 field ration -n07566092 K ration -n07566231 C-ration -n07566340 foodstuff, food product -n07566863 starches -n07567039 breadstuff -n07567139 coloring, colouring, food coloring, food colouring, food color, food colour -n07567390 concentrate -n07567611 tomato concentrate -n07567707 meal -n07567980 kibble -n07568095 cornmeal, Indian meal -n07568241 farina -n07568389 matzo meal, matzoh meal, matzah meal -n07568502 oatmeal, rolled oats -n07568625 pea flour -n07568818 roughage, fiber -n07568991 bran -n07569106 flour -n07569423 plain flour -n07569543 wheat flour -n07569644 whole wheat flour, graham flour, graham, whole meal flour -n07569873 soybean meal, soybean flour, soy flour -n07570021 semolina -n07570530 corn gluten feed -n07570720 nutriment, nourishment, nutrition, sustenance, aliment, alimentation, victuals -n07572353 commissariat, provisions, provender, viands, victuals -n07572616 larder -n07572858 frozen food, frozen foods -n07572957 canned food, canned foods, canned goods, tinned goods -n07573103 canned meat, tinned meat -n07573347 Spam -n07573453 dehydrated food, dehydrated foods -n07573563 square meal -n07573696 meal, repast -n07574176 potluck -n07574426 refection -n07574504 refreshment -n07574602 breakfast -n07574780 continental breakfast, petit dejeuner -n07574923 brunch -n07575076 lunch, luncheon, tiffin, dejeuner -n07575226 business lunch -n07575392 high tea -n07575510 tea, afternoon tea, teatime -n07575726 dinner -n07575984 supper -n07576182 buffet -n07576438 picnic -n07576577 cookout -n07576781 barbecue, barbeque -n07576969 clambake -n07577144 fish fry -n07577374 bite, collation, snack -n07577538 nosh -n07577657 nosh-up -n07577772 ploughman's lunch -n07577918 coffee break, tea break -n07578093 banquet, feast, spread -n07579575 entree, main course -n07579688 piece de resistance -n07579787 plate -n07579917 adobo -n07580053 side dish, side order, entremets -n07580253 special -n07580359 casserole -n07580470 chicken casserole -n07580592 chicken cacciatore, chicken cacciatora, hunter's chicken -n07581249 antipasto -n07581346 appetizer, appetiser, starter -n07581607 canape -n07581775 cocktail -n07581931 fruit cocktail -n07582027 crab cocktail -n07582152 shrimp cocktail -n07582277 hors d'oeuvre -n07582441 relish -n07582609 dip -n07582811 bean dip -n07582892 cheese dip -n07582970 clam dip -n07583066 guacamole -n07583197 soup -n07583865 soup du jour -n07583978 alphabet soup -n07584110 consomme -n07584228 madrilene -n07584332 bisque -n07584423 borsch, borsh, borscht, borsht, borshch, bortsch -n07584593 broth -n07584859 barley water -n07584938 bouillon -n07585015 beef broth, beef stock -n07585107 chicken broth, chicken stock -n07585208 broth, stock -n07585474 stock cube -n07585557 chicken soup -n07585644 cock-a-leekie, cocky-leeky -n07585758 gazpacho -n07585906 gumbo -n07585997 julienne -n07586099 marmite -n07586179 mock turtle soup -n07586318 mulligatawny -n07586485 oxtail soup -n07586604 pea soup -n07586718 pepper pot, Philadelphia pepper pot -n07586894 petite marmite, minestrone, vegetable soup -n07587023 potage, pottage -n07587111 pottage -n07587206 turtle soup, green turtle soup -n07587331 eggdrop soup -n07587441 chowder -n07587618 corn chowder -n07587700 clam chowder -n07587819 Manhattan clam chowder -n07587962 New England clam chowder -n07588111 fish chowder -n07588193 won ton, wonton, wonton soup -n07588299 split-pea soup -n07588419 green pea soup, potage St. Germain -n07588574 lentil soup -n07588688 Scotch broth -n07588817 vichyssoise -n07588947 stew -n07589458 bigos -n07589543 Brunswick stew -n07589724 burgoo -n07589872 burgoo -n07589967 olla podrida, Spanish burgoo -n07590068 mulligan stew, mulligan, Irish burgoo -n07590177 purloo, chicken purloo, poilu -n07590320 goulash, Hungarian goulash, gulyas -n07590502 hotchpotch -n07590611 hot pot, hotpot -n07590752 beef goulash -n07590841 pork-and-veal goulash -n07590974 porkholt -n07591049 Irish stew -n07591162 oyster stew -n07591236 lobster stew -n07591330 lobscouse, lobscuse, scouse -n07591473 fish stew -n07591586 bouillabaisse -n07591813 matelote -n07591961 paella -n07592094 fricassee -n07592317 chicken stew -n07592400 turkey stew -n07592481 beef stew -n07592656 ragout -n07592768 ratatouille -n07592922 salmi -n07593004 pot-au-feu -n07593107 slumgullion -n07593199 smorgasbord -n07593471 viand -n07593774 ready-mix -n07593972 brownie mix -n07594066 cake mix -n07594155 lemonade mix -n07594250 self-rising flour, self-raising flour -n07594737 choice morsel, tidbit, titbit -n07594840 savory, savoury -n07595051 calf's-foot jelly -n07595180 caramel, caramelized sugar -n07595368 lump sugar -n07595649 cane sugar -n07595751 castor sugar, caster sugar -n07595914 powdered sugar -n07596046 granulated sugar -n07596160 icing sugar -n07596362 corn sugar -n07596452 brown sugar -n07596566 demerara, demerara sugar -n07596684 sweet, confection -n07596967 confectionery -n07597145 confiture -n07597263 sweetmeat -n07597365 candy, confect -n07598256 candy bar -n07598529 carob bar -n07598622 hardbake -n07598734 hard candy -n07598928 barley-sugar, barley candy -n07599068 brandyball -n07599161 jawbreaker -n07599242 lemon drop -n07599383 sourball -n07599468 patty -n07599554 peppermint patty -n07599649 bonbon -n07599783 brittle, toffee, toffy -n07599911 peanut brittle -n07599998 chewing gum, gum -n07600177 gum ball -n07600285 bubble gum -n07600394 butterscotch -n07600506 candied fruit, succade, crystallized fruit -n07600696 candied apple, candy apple, taffy apple, caramel apple, toffee apple -n07600895 crystallized ginger -n07601025 grapefruit peel -n07601175 lemon peel -n07601290 orange peel -n07601407 candied citrus peel -n07601572 candy cane -n07601686 candy corn -n07601809 caramel -n07602650 center, centre -n07604956 comfit -n07605040 cotton candy, spun sugar, candyfloss -n07605198 dragee -n07605282 dragee -n07605380 fondant -n07605474 fudge -n07605597 chocolate fudge -n07605693 divinity, divinity fudge -n07605804 penuche, penoche, panoche, panocha -n07605944 gumdrop -n07606058 jujube -n07606191 honey crisp -n07606278 mint, mint candy -n07606419 horehound -n07606538 peppermint, peppermint candy -n07606669 jelly bean, jelly egg -n07606764 kiss, candy kiss -n07606933 molasses kiss -n07607027 meringue kiss -n07607138 chocolate kiss -n07607361 licorice, liquorice -n07607492 Life Saver -n07607605 lollipop, sucker, all-day sucker -n07607707 lozenge -n07607832 cachou -n07607967 cough drop, troche, pastille, pastil -n07608098 marshmallow -n07608245 marzipan, marchpane -n07608339 nougat -n07608429 nougat bar -n07608533 nut bar -n07608641 peanut bar -n07608721 popcorn ball -n07608866 praline -n07608980 rock candy -n07609083 rock candy, rock -n07609215 sugar candy -n07609316 sugarplum -n07609407 taffy -n07609549 molasses taffy -n07609632 truffle, chocolate truffle -n07609728 Turkish Delight -n07609840 dessert, sweet, afters -n07610295 ambrosia, nectar -n07610502 ambrosia -n07610620 baked Alaska -n07610746 blancmange -n07610890 charlotte -n07611046 compote, fruit compote -n07611148 dumpling -n07611267 flan -n07611358 frozen dessert -n07611733 junket -n07611839 mousse -n07611991 mousse -n07612137 pavlova -n07612273 peach melba -n07612367 whip -n07612530 prune whip -n07612632 pudding -n07612996 pudding, pud -n07613158 syllabub, sillabub -n07613266 tiramisu -n07613480 trifle -n07613671 tipsy cake -n07613815 jello, Jell-O -n07614103 apple dumpling -n07614198 ice, frappe -n07614348 water ice, sorbet -n07614500 ice cream, icecream -n07614730 ice-cream cone -n07614825 chocolate ice cream -n07615052 Neapolitan ice cream -n07615190 peach ice cream -n07615289 sherbert, sherbet -n07615460 strawberry ice cream -n07615569 tutti-frutti -n07615671 vanilla ice cream -n07615774 ice lolly, lolly, lollipop, popsicle -n07615954 ice milk -n07616046 frozen yogurt -n07616174 snowball -n07616265 snowball -n07616386 parfait -n07616487 ice-cream sundae, sundae -n07616590 split -n07616748 banana split -n07616906 frozen pudding -n07617051 frozen custard, soft ice cream -n07617188 pudding -n07617344 flummery -n07617447 fish mousse -n07617526 chicken mousse -n07617611 chocolate mousse -n07617708 plum pudding, Christmas pudding -n07617839 carrot pudding -n07617932 corn pudding -n07618029 steamed pudding -n07618119 duff, plum duff -n07618281 vanilla pudding -n07618432 chocolate pudding -n07618587 brown Betty -n07618684 Nesselrode, Nesselrode pudding -n07618871 pease pudding -n07619004 custard -n07619208 creme caramel -n07619301 creme anglais -n07619409 creme brulee -n07619508 fruit custard -n07619881 tapioca -n07620047 tapioca pudding -n07620145 roly-poly, roly-poly pudding -n07620327 suet pudding -n07620597 Bavarian cream -n07620689 maraschino, maraschino cherry -n07621264 nonpareil -n07621497 zabaglione, sabayon -n07621618 garnish -n07623136 pastry, pastry dough -n07624466 turnover -n07624666 apple turnover -n07624757 knish -n07624924 pirogi, piroshki, pirozhki -n07625061 samosa -n07625324 timbale -n07627931 puff paste, pate feuillete -n07628068 phyllo -n07628181 puff batter, pouf paste, pate a choux -n07631926 ice-cream cake, icebox cake -n07639069 doughnut, donut, sinker -n07641928 fish cake, fish ball -n07642361 fish stick, fish finger -n07642471 conserve, preserve, conserves, preserves -n07642742 apple butter -n07642833 chowchow -n07642933 jam -n07643026 lemon curd, lemon cheese -n07643200 strawberry jam, strawberry preserves -n07643306 jelly -n07643474 apple jelly -n07643577 crabapple jelly -n07643679 grape jelly -n07643764 marmalade -n07643891 orange marmalade -n07643981 gelatin, jelly -n07644244 gelatin dessert -n07648913 buffalo wing -n07648997 barbecued wing -n07650792 mess -n07650903 mince -n07651025 puree -n07654148 barbecue, barbeque -n07654298 biryani, biriani -n07655067 escalope de veau Orloff -n07655263 saute -n07663899 patty, cake -n07665438 veal parmesan, veal parmigiana -n07666176 veal cordon bleu -n07672914 margarine, margarin, oleo, oleomargarine, marge -n07678586 mincemeat -n07678729 stuffing, dressing -n07678953 turkey stuffing -n07679034 oyster stuffing, oyster dressing -n07679140 forcemeat, farce -n07679356 bread, breadstuff, staff of life -n07680168 anadama bread -n07680313 bap -n07680416 barmbrack -n07680517 breadstick, bread-stick -n07680655 grissino -n07680761 brown bread, Boston brown bread -n07680932 bun, roll -n07681264 tea bread -n07681355 caraway seed bread -n07681450 challah, hallah -n07681691 cinnamon bread -n07681805 cracked-wheat bread -n07681926 cracker -n07682197 crouton -n07682316 dark bread, whole wheat bread, whole meal bread, brown bread -n07682477 English muffin -n07682624 flatbread -n07682808 garlic bread -n07682952 gluten bread -n07683039 graham bread -n07683138 Host -n07683265 flatbrod -n07683360 bannock -n07683490 chapatti, chapati -n07683617 pita, pocket bread -n07683786 loaf of bread, loaf -n07684084 French loaf -n07684164 matzo, matzoh, matzah, unleavened bread -n07684289 nan, naan -n07684422 onion bread -n07684517 raisin bread -n07684600 quick bread -n07684938 banana bread -n07685031 date bread -n07685118 date-nut bread -n07685218 nut bread -n07685303 oatcake -n07685399 Irish soda bread -n07685546 skillet bread, fry bread -n07685730 rye bread -n07685918 black bread, pumpernickel -n07686021 Jewish rye bread, Jewish rye -n07686202 limpa -n07686299 Swedish rye bread, Swedish rye -n07686461 salt-rising bread -n07686634 simnel -n07686720 sour bread, sourdough bread -n07686873 toast -n07687053 wafer -n07687211 white bread, light bread -n07687381 baguet, baguette -n07687469 French bread -n07687626 Italian bread -n07687789 cornbread -n07688021 corn cake -n07688130 skillet corn bread -n07688265 ashcake, ash cake, corn tash -n07688412 hoecake -n07688624 cornpone, pone -n07688757 corn dab, corn dodger, dodger -n07688898 hush puppy, hushpuppy -n07689003 johnnycake, johnny cake, journey cake -n07689217 Shawnee cake -n07689313 spoon bread, batter bread -n07689490 cinnamon toast -n07689624 orange toast -n07689757 Melba toast -n07689842 zwieback, rusk, Brussels biscuit, twice-baked bread -n07690019 frankfurter bun, hotdog bun -n07690152 hamburger bun, hamburger roll -n07690273 muffin, gem -n07690431 bran muffin -n07690511 corn muffin -n07690585 Yorkshire pudding -n07690739 popover -n07690892 scone -n07691091 drop scone, griddlecake, Scotch pancake -n07691237 cross bun, hot cross bun -n07691539 brioche -n07691650 crescent roll, croissant -n07691758 hard roll, Vienna roll -n07691863 soft roll -n07691954 kaiser roll -n07692114 Parker House roll -n07692248 clover-leaf roll -n07692405 onion roll -n07692517 bialy, bialystoker -n07692614 sweet roll, coffee roll -n07692887 bear claw, bear paw -n07693048 cinnamon roll, cinnamon bun, cinnamon snail -n07693223 honey bun, sticky bun, caramel bun, schnecken -n07693439 pinwheel roll -n07693590 danish, danish pastry -n07693725 bagel, beigel -n07693889 onion bagel -n07693972 biscuit -n07694169 rolled biscuit -n07694403 baking-powder biscuit -n07694516 buttermilk biscuit, soda biscuit -n07694659 shortcake -n07694839 hardtack, pilot biscuit, pilot bread, sea biscuit, ship biscuit -n07695187 saltine -n07695284 soda cracker -n07695410 oyster cracker -n07695504 water biscuit -n07695652 graham cracker -n07695742 pretzel -n07695878 soft pretzel -n07695965 sandwich -n07696403 sandwich plate -n07696527 butty -n07696625 ham sandwich -n07696728 chicken sandwich -n07696839 club sandwich, three-decker, triple-decker -n07696977 open-face sandwich, open sandwich -n07697100 hamburger, beefburger, burger -n07697313 cheeseburger -n07697408 tunaburger -n07697537 hotdog, hot dog, red hot -n07697699 Sloppy Joe -n07697825 bomber, grinder, hero, hero sandwich, hoagie, hoagy, Cuban sandwich, Italian sandwich, poor boy, sub, submarine, submarine sandwich, torpedo, wedge, zep -n07698250 gyro -n07698401 bacon-lettuce-tomato sandwich, BLT -n07698543 Reuben -n07698672 western, western sandwich -n07698782 wrap -n07700003 spaghetti -n07703889 hasty pudding -n07704054 gruel -n07704205 congee, jook -n07704305 skilly -n07705931 edible fruit -n07707451 vegetable, veggie, veg -n07708124 julienne, julienne vegetable -n07708398 raw vegetable, rabbit food -n07708512 crudites -n07708685 celery stick -n07708798 legume -n07709046 pulse -n07709172 potherb -n07709333 greens, green, leafy vegetable -n07709701 chop-suey greens -n07709881 bean curd, tofu -n07710007 solanaceous vegetable -n07710283 root vegetable -n07710616 potato, white potato, Irish potato, murphy, spud, tater -n07710952 baked potato -n07711080 french fries, french-fried potatoes, fries, chips -n07711232 home fries, home-fried potatoes -n07711371 jacket potato -n07711569 mashed potato -n07711683 potato skin, potato peel, potato peelings -n07711799 Uruguay potato -n07711907 yam -n07712063 sweet potato -n07712267 yam -n07712382 snack food -n07712559 chip, crisp, potato chip, Saratoga chip -n07712748 corn chip -n07712856 tortilla chip -n07712959 nacho -n07713074 eggplant, aubergine, mad apple -n07713267 pieplant, rhubarb -n07713395 cruciferous vegetable -n07713763 mustard, mustard greens, leaf mustard, Indian mustard -n07713895 cabbage, chou -n07714078 kale, kail, cole -n07714188 collards, collard greens -n07714287 Chinese cabbage, celery cabbage, Chinese celery -n07714448 bok choy, bok choi -n07714571 head cabbage -n07714802 red cabbage -n07714895 savoy cabbage, savoy -n07714990 broccoli -n07715103 cauliflower -n07715221 brussels sprouts -n07715407 broccoli rabe, broccoli raab -n07715561 squash -n07715721 summer squash -n07716034 yellow squash -n07716203 crookneck, crookneck squash, summer crookneck -n07716358 zucchini, courgette -n07716504 marrow, vegetable marrow -n07716649 cocozelle -n07716750 pattypan squash -n07716906 spaghetti squash -n07717070 winter squash -n07717410 acorn squash -n07717556 butternut squash -n07717714 hubbard squash -n07717858 turban squash -n07718068 buttercup squash -n07718195 cushaw -n07718329 winter crookneck squash -n07718472 cucumber, cuke -n07718671 gherkin -n07718747 artichoke, globe artichoke -n07718920 artichoke heart -n07719058 Jerusalem artichoke, sunchoke -n07719213 asparagus -n07719330 bamboo shoot -n07719437 sprout -n07719616 bean sprout -n07719756 alfalfa sprout -n07719839 beet, beetroot -n07719980 beet green -n07720084 sugar beet -n07720185 mangel-wurzel -n07720277 chard, Swiss chard, spinach beet, leaf beet -n07720442 pepper -n07720615 sweet pepper -n07720875 bell pepper -n07721018 green pepper -n07721118 globe pepper -n07721195 pimento, pimiento -n07721325 hot pepper -n07721456 chili, chili pepper, chilli, chilly, chile -n07721678 jalapeno, jalapeno pepper -n07721833 chipotle -n07721942 cayenne, cayenne pepper -n07722052 tabasco, red pepper -n07722217 onion -n07722390 Bermuda onion -n07722485 green onion, spring onion, scallion -n07722666 Vidalia onion -n07722763 Spanish onion -n07722888 purple onion, red onion -n07723039 leek -n07723177 shallot -n07723330 salad green, salad greens -n07723559 lettuce -n07723753 butterhead lettuce -n07723968 buttercrunch -n07724078 Bibb lettuce -n07724173 Boston lettuce -n07724269 crisphead lettuce, iceberg lettuce, iceberg -n07724492 cos, cos lettuce, romaine, romaine lettuce -n07724654 leaf lettuce, loose-leaf lettuce -n07724819 celtuce -n07724943 bean, edible bean -n07725158 goa bean -n07725255 lentil -n07725376 pea -n07725531 green pea, garden pea -n07725663 marrowfat pea -n07725789 snow pea, sugar pea -n07725888 sugar snap pea -n07726009 split-pea -n07726095 chickpea, garbanzo -n07726230 cajan pea, pigeon pea, dahl -n07726386 field pea -n07726525 mushy peas -n07726672 black-eyed pea, cowpea -n07726796 common bean -n07727048 kidney bean -n07727140 navy bean, pea bean, white bean -n07727252 pinto bean -n07727377 frijole -n07727458 black bean, turtle bean -n07727578 fresh bean -n07727741 flageolet, haricot -n07727868 green bean -n07728053 snap bean, snap -n07728181 string bean -n07728284 Kentucky wonder, Kentucky wonder bean -n07728391 scarlet runner, scarlet runner bean, runner bean, English runner bean -n07728585 haricot vert, haricots verts, French bean -n07728708 wax bean, yellow bean -n07728804 shell bean -n07729000 lima bean -n07729142 Fordhooks -n07729225 sieva bean, butter bean, butterbean, civet bean -n07729384 fava bean, broad bean -n07729485 soy, soybean, soya, soya bean -n07729828 green soybean -n07729926 field soybean -n07730033 cardoon -n07730207 carrot -n07730320 carrot stick -n07730406 celery -n07730562 pascal celery, Paschal celery -n07730708 celeriac, celery root -n07730855 chicory, curly endive -n07731006 radicchio -n07731122 coffee substitute -n07731284 chicory, chicory root -n07731436 Postum -n07731587 chicory escarole, endive, escarole -n07731767 Belgian endive, French endive, witloof -n07731952 corn, edible corn -n07732168 sweet corn, green corn -n07732302 hominy -n07732433 lye hominy -n07732525 pearl hominy -n07732636 popcorn -n07732747 cress -n07732904 watercress -n07733005 garden cress -n07733124 winter cress -n07733217 dandelion green -n07733394 gumbo, okra -n07733567 kohlrabi, turnip cabbage -n07733712 lamb's-quarter, pigweed, wild spinach -n07733847 wild spinach -n07734017 tomato -n07734183 beefsteak tomato -n07734292 cherry tomato -n07734417 plum tomato -n07734555 tomatillo, husk tomato, Mexican husk tomato -n07734744 mushroom -n07734879 stuffed mushroom -n07735052 salsify -n07735179 oyster plant, vegetable oyster -n07735294 scorzonera, black salsify -n07735404 parsnip -n07735510 pumpkin -n07735687 radish -n07735803 turnip -n07735981 white turnip -n07736087 rutabaga, swede, swedish turnip, yellow turnip -n07736256 turnip greens -n07736371 sorrel, common sorrel -n07736527 French sorrel -n07736692 spinach -n07736813 taro, taro root, cocoyam, dasheen, edda -n07736971 truffle, earthnut -n07737081 edible nut -n07737594 bunya bunya -n07737745 peanut, earthnut, goober, goober pea, groundnut, monkey nut -n07738105 freestone -n07738224 cling, clingstone -n07739035 windfall -n07739125 apple -n07739344 crab apple, crabapple -n07739506 eating apple, dessert apple -n07739923 Baldwin -n07740033 Cortland -n07740115 Cox's Orange Pippin -n07740220 Delicious -n07740342 Golden Delicious, Yellow Delicious -n07740461 Red Delicious -n07740597 Empire -n07740744 Grimes' golden -n07740855 Jonathan -n07740954 McIntosh -n07741138 Macoun -n07741235 Northern Spy -n07741357 Pearmain -n07741461 Pippin -n07741623 Prima -n07741706 Stayman -n07741804 Winesap -n07741888 Stayman Winesap -n07742012 cooking apple -n07742224 Bramley's Seedling -n07742313 Granny Smith -n07742415 Lane's Prince Albert -n07742513 Newtown Wonder -n07742605 Rome Beauty -n07742704 berry -n07743224 bilberry, whortleberry, European blueberry -n07743384 huckleberry -n07743544 blueberry -n07743723 wintergreen, boxberry, checkerberry, teaberry, spiceberry -n07743902 cranberry -n07744057 lingonberry, mountain cranberry, cowberry, lowbush cranberry -n07744246 currant -n07744430 gooseberry -n07744559 black currant -n07744682 red currant -n07744811 blackberry -n07745046 boysenberry -n07745197 dewberry -n07745357 loganberry -n07745466 raspberry -n07745661 saskatoon, serviceberry, shadberry, juneberry -n07745940 strawberry -n07746038 sugarberry, hackberry -n07746186 persimmon -n07746334 acerola, barbados cherry, surinam cherry, West Indian cherry -n07746551 carambola, star fruit -n07746749 ceriman, monstera -n07746910 carissa plum, natal plum -n07747055 citrus, citrus fruit, citrous fruit -n07747607 orange -n07747811 temple orange -n07747951 mandarin, mandarin orange -n07748157 clementine -n07748276 satsuma -n07748416 tangerine -n07748574 tangelo, ugli, ugli fruit -n07748753 bitter orange, Seville orange, sour orange -n07748912 sweet orange -n07749095 Jaffa orange -n07749192 navel orange -n07749312 Valencia orange -n07749446 kumquat -n07749582 lemon -n07749731 lime -n07749870 key lime -n07749969 grapefruit -n07750146 pomelo, shaddock -n07750299 citrange -n07750449 citron -n07750586 almond -n07750736 Jordan almond -n07750872 apricot -n07751004 peach -n07751148 nectarine -n07751280 pitahaya -n07751451 plum -n07751737 damson, damson plum -n07751858 greengage, greengage plum -n07751977 beach plum -n07752109 sloe -n07752264 Victoria plum -n07752377 dried fruit -n07752514 dried apricot -n07752602 prune -n07752664 raisin -n07752782 seedless raisin, sultana -n07752874 seeded raisin -n07752966 currant -n07753113 fig -n07753275 pineapple, ananas -n07753448 anchovy pear, river pear -n07753592 banana -n07753743 passion fruit -n07753980 granadilla -n07754155 sweet calabash -n07754279 bell apple, sweet cup, water lemon, yellow granadilla -n07754451 breadfruit -n07754684 jackfruit, jak, jack -n07754894 cacao bean, cocoa bean -n07755089 cocoa -n07755262 canistel, eggfruit -n07755411 melon -n07755619 melon ball -n07755707 muskmelon, sweet melon -n07755929 cantaloup, cantaloupe -n07756096 winter melon -n07756325 honeydew, honeydew melon -n07756499 Persian melon -n07756641 net melon, netted melon, nutmeg melon -n07756838 casaba, casaba melon -n07756951 watermelon -n07757132 cherry -n07757312 sweet cherry, black cherry -n07757511 bing cherry -n07757602 heart cherry, oxheart, oxheart cherry -n07757753 blackheart, blackheart cherry -n07757874 capulin, Mexican black cherry -n07757990 sour cherry -n07758125 amarelle -n07758260 morello -n07758407 cocoa plum, coco plum, icaco -n07758582 gherkin -n07758680 grape -n07758950 fox grape -n07759194 Concord grape -n07759324 Catawba -n07759424 muscadine, bullace grape -n07759576 scuppernong -n07759691 slipskin grape -n07759816 vinifera grape -n07760070 emperor -n07760153 muscat, muscatel, muscat grape -n07760297 ribier -n07760395 sultana -n07760501 Tokay -n07760673 flame tokay -n07760755 Thompson Seedless -n07760859 custard apple -n07761141 cherimoya, cherimolla -n07761309 soursop, guanabana -n07761611 sweetsop, annon, sugar apple -n07761777 ilama -n07761954 pond apple -n07762114 papaw, pawpaw -n07762244 papaya -n07762373 kai apple -n07762534 ketembilla, kitembilla, kitambilla -n07762740 ackee, akee -n07762913 durian -n07763107 feijoa, pineapple guava -n07763290 genip, Spanish lime -n07763483 genipap, genipap fruit -n07763629 kiwi, kiwi fruit, Chinese gooseberry -n07763792 loquat, Japanese plum -n07763987 mangosteen -n07764155 mango -n07764315 sapodilla, sapodilla plum, sapota -n07764486 sapote, mammee, marmalade plum -n07764630 tamarind, tamarindo -n07764847 avocado, alligator pear, avocado pear, aguacate -n07765073 date -n07765208 elderberry -n07765361 guava -n07765517 mombin -n07765612 hog plum, yellow mombin -n07765728 hog plum, wild plum -n07765862 jaboticaba -n07765999 jujube, Chinese date, Chinese jujube -n07766173 litchi, litchi nut, litchee, lichi, leechee, lichee, lychee -n07766409 longanberry, dragon's eye -n07766530 mamey, mammee, mammee apple -n07766723 marang -n07766891 medlar -n07767002 medlar -n07767171 mulberry -n07767344 olive -n07767549 black olive, ripe olive -n07767709 green olive -n07767847 pear -n07768068 bosc -n07768139 anjou -n07768230 bartlett, bartlett pear -n07768318 seckel, seckel pear -n07768423 plantain -n07768590 plumcot -n07768694 pomegranate -n07768858 prickly pear -n07769102 Barbados gooseberry, blade apple -n07769306 quandong, quandang, quantong, native peach -n07769465 quandong nut -n07769584 quince -n07769731 rambutan, rambotan -n07769886 pulasan, pulassan -n07770034 rose apple -n07770180 sorb, sorb apple -n07770439 sour gourd, monkey bread -n07770571 edible seed -n07770763 pumpkin seed -n07770869 betel nut, areca nut -n07771082 beechnut -n07771212 walnut -n07771405 black walnut -n07771539 English walnut -n07771731 brazil nut, brazil -n07771891 butternut -n07772026 souari nut -n07772147 cashew, cashew nut -n07772274 chestnut -n07772413 chincapin, chinkapin, chinquapin -n07772788 hazelnut, filbert, cobnut, cob -n07772935 coconut, cocoanut -n07773428 coconut milk, coconut water -n07774182 grugru nut -n07774295 hickory nut -n07774479 cola extract -n07774596 macadamia nut -n07774719 pecan -n07774842 pine nut, pignolia, pinon nut -n07775050 pistachio, pistachio nut -n07775197 sunflower seed -n07783827 anchovy paste -n07785487 rollmops -n07800091 feed, provender -n07800487 cattle cake -n07800636 creep feed -n07800740 fodder -n07801007 feed grain -n07801091 eatage, forage, pasture, pasturage, grass -n07801342 silage, ensilage -n07801508 oil cake -n07801709 oil meal -n07801779 alfalfa -n07801892 broad bean, horse bean -n07802026 hay -n07802152 timothy -n07802246 stover -n07802417 grain, food grain, cereal -n07802767 grist -n07802863 groats -n07802963 millet -n07803093 barley, barleycorn -n07803213 pearl barley -n07803310 buckwheat -n07803408 bulgur, bulghur, bulgur wheat -n07803545 wheat, wheat berry -n07803779 cracked wheat -n07803895 stodge -n07803992 wheat germ -n07804152 oat -n07804323 rice -n07804543 brown rice -n07804657 white rice, polished rice -n07804771 wild rice, Indian rice -n07804900 paddy -n07805006 slop, slops, swill, pigswill, pigwash -n07805254 mash -n07805389 chicken feed, scratch -n07805478 cud, rechewed food -n07805594 bird feed, bird food, birdseed -n07805731 petfood, pet-food, pet food -n07805966 dog food -n07806043 cat food -n07806120 canary seed -n07806221 salad -n07806633 tossed salad -n07806774 green salad -n07806879 Caesar salad -n07807002 salmagundi -n07807171 salad nicoise -n07807317 combination salad -n07807472 chef's salad -n07807594 potato salad -n07807710 pasta salad -n07807834 macaroni salad -n07807922 fruit salad -n07808022 Waldorf salad -n07808166 crab Louis -n07808268 herring salad -n07808352 tuna fish salad, tuna salad -n07808479 chicken salad -n07808587 coleslaw, slaw -n07808675 aspic -n07808806 molded salad -n07808904 tabbouleh, tabooli -n07809096 ingredient, fixings -n07809368 flavorer, flavourer, flavoring, flavouring, seasoner, seasoning -n07810531 bouillon cube -n07810907 condiment -n07811416 herb -n07812046 fines herbes -n07812184 spice -n07812662 spearmint oil -n07812790 lemon oil -n07812913 wintergreen oil, oil of wintergreen -n07813107 salt, table salt, common salt -n07813324 celery salt -n07813495 onion salt -n07813579 seasoned salt -n07813717 sour salt -n07813833 five spice powder -n07814007 allspice -n07814203 cinnamon -n07814390 stick cinnamon -n07814487 clove -n07814634 cumin, cumin seed -n07814790 fennel -n07814925 ginger, gingerroot -n07815163 ginger, powdered ginger -n07815294 mace -n07815424 nutmeg -n07815588 pepper, peppercorn -n07815839 black pepper -n07815956 white pepper -n07816052 sassafras -n07816164 basil, sweet basil -n07816296 bay leaf -n07816398 borage -n07816575 hyssop -n07816726 caraway -n07816839 chervil -n07817024 chives -n07817160 comfrey, healing herb -n07817315 coriander, Chinese parsley, cilantro -n07817465 coriander, coriander seed -n07817599 costmary -n07817758 fennel, common fennel -n07817871 fennel, Florence fennel, finocchio -n07818029 fennel seed -n07818133 fenugreek, fenugreek seed -n07818277 garlic, ail -n07818422 clove, garlic clove -n07818572 garlic chive -n07818689 lemon balm -n07818825 lovage -n07818995 marjoram, oregano -n07819166 mint -n07819303 mustard seed -n07819480 mustard, table mustard -n07819682 Chinese mustard -n07819769 nasturtium -n07819896 parsley -n07820036 salad burnet -n07820145 rosemary -n07820297 rue -n07820497 sage -n07820683 clary sage -n07820814 savory, savoury -n07820960 summer savory, summer savoury -n07821107 winter savory, winter savoury -n07821260 sweet woodruff, waldmeister -n07821404 sweet cicely -n07821610 tarragon, estragon -n07821758 thyme -n07821919 turmeric -n07822053 caper -n07822197 catsup, ketchup, cetchup, tomato ketchup -n07822323 cardamom, cardamon, cardamum -n07822518 cayenne, cayenne pepper, red pepper -n07822687 chili powder -n07822845 chili sauce -n07823105 chutney, Indian relish -n07823280 steak sauce -n07823369 taco sauce -n07823460 salsa -n07823591 mint sauce -n07823698 cranberry sauce -n07823814 curry powder -n07823951 curry -n07824191 lamb curry -n07824268 duck sauce, hoisin sauce -n07824383 horseradish -n07824502 marinade -n07824702 paprika -n07824863 Spanish paprika -n07824988 pickle -n07825194 dill pickle -n07825399 bread and butter pickle -n07825496 pickle relish -n07825597 piccalilli -n07825717 sweet pickle -n07825850 applesauce, apple sauce -n07825972 soy sauce, soy -n07826091 Tabasco, Tabasco sauce -n07826250 tomato paste -n07826340 angelica -n07826453 angelica -n07826544 almond extract -n07826653 anise, aniseed, anise seed -n07826930 Chinese anise, star anise, star aniseed -n07827130 juniper berries -n07827284 saffron -n07827410 sesame seed, benniseed -n07827554 caraway seed -n07827750 poppy seed -n07827896 dill, dill weed -n07828041 dill seed -n07828156 celery seed -n07828275 lemon extract -n07828378 monosodium glutamate, MSG -n07828642 vanilla bean -n07828987 vinegar, acetum -n07829248 cider vinegar -n07829331 wine vinegar -n07829412 sauce -n07830493 anchovy sauce -n07830593 hot sauce -n07830690 hard sauce -n07830841 horseradish sauce, sauce Albert -n07830986 bolognese pasta sauce -n07831146 carbonara -n07831267 tomato sauce -n07831450 tartare sauce, tartar sauce -n07831663 wine sauce -n07831821 marchand de vin, mushroom wine sauce -n07831955 bread sauce -n07832099 plum sauce -n07832202 peach sauce -n07832307 apricot sauce -n07832416 pesto -n07832592 ravigote, ravigotte -n07832741 remoulade sauce -n07832902 dressing, salad dressing -n07833333 sauce Louis -n07833535 bleu cheese dressing, blue cheese dressing -n07833672 blue cheese dressing, Roquefort dressing -n07833816 French dressing, vinaigrette, sauce vinaigrette -n07833951 Lorenzo dressing -n07834065 anchovy dressing -n07834160 Italian dressing -n07834286 half-and-half dressing -n07834507 mayonnaise, mayo -n07834618 green mayonnaise, sauce verte -n07834774 aioli, aioli sauce, garlic sauce -n07834872 Russian dressing, Russian mayonnaise -n07835051 salad cream -n07835173 Thousand Island dressing -n07835331 barbecue sauce -n07835457 hollandaise -n07835547 bearnaise -n07835701 Bercy, Bercy butter -n07835823 bordelaise -n07835921 bourguignon, bourguignon sauce, Burgundy sauce -n07836077 brown sauce, sauce Espagnole -n07836269 Espagnole, sauce Espagnole -n07836456 Chinese brown sauce, brown sauce -n07836600 blanc -n07836731 cheese sauce -n07836838 chocolate sauce, chocolate syrup -n07837002 hot-fudge sauce, fudge sauce -n07837110 cocktail sauce, seafood sauce -n07837234 Colbert, Colbert butter -n07837362 white sauce, bechamel sauce, bechamel -n07837545 cream sauce -n07837630 Mornay sauce -n07837755 demiglace, demi-glaze -n07837912 gravy, pan gravy -n07838073 gravy -n07838233 spaghetti sauce, pasta sauce -n07838441 marinara -n07838551 mole -n07838659 hunter's sauce, sauce chausseur -n07838811 mushroom sauce -n07838905 mustard sauce -n07839055 Nantua, shrimp sauce -n07839172 Hungarian sauce, paprika sauce -n07839312 pepper sauce, Poivrade -n07839478 roux -n07839593 Smitane -n07839730 Soubise, white onion sauce -n07839864 Lyonnaise sauce, brown onion sauce -n07840027 veloute -n07840124 allemande, allemande sauce -n07840219 caper sauce -n07840304 poulette -n07840395 curry sauce -n07840520 Worcester sauce, Worcestershire, Worcestershire sauce -n07840672 coconut milk, coconut cream -n07840804 egg, eggs -n07841037 egg white, white, albumen, ovalbumin -n07841345 egg yolk, yolk -n07841495 boiled egg, coddled egg -n07841639 hard-boiled egg, hard-cooked egg -n07841800 Easter egg -n07841907 Easter egg -n07842044 chocolate egg -n07842130 candy egg -n07842202 poached egg, dropped egg -n07842308 scrambled eggs -n07842433 deviled egg, stuffed egg -n07842605 shirred egg, baked egg, egg en cocotte -n07842753 omelet, omelette -n07842972 firm omelet -n07843117 French omelet -n07843220 fluffy omelet -n07843348 western omelet -n07843464 souffle -n07843636 fried egg -n07843775 dairy product -n07844042 milk -n07844604 milk -n07844786 sour milk -n07844867 soya milk, soybean milk, soymilk -n07845087 formula -n07845166 pasteurized milk -n07845335 cows' milk -n07845421 yak's milk -n07845495 goats' milk -n07845571 acidophilus milk -n07845702 raw milk -n07845775 scalded milk -n07845863 homogenized milk -n07846014 certified milk -n07846143 powdered milk, dry milk, dried milk, milk powder -n07846274 nonfat dry milk -n07846359 evaporated milk -n07846471 condensed milk -n07846557 skim milk, skimmed milk -n07846688 semi-skimmed milk -n07846802 whole milk -n07846938 low-fat milk -n07847047 buttermilk -n07847198 cream -n07847453 clotted cream, Devonshire cream -n07847585 double creme, heavy whipping cream -n07847706 half-and-half -n07847827 heavy cream -n07847917 light cream, coffee cream, single cream -n07848093 sour cream, soured cream -n07848196 whipping cream, light whipping cream -n07848338 butter -n07848771 clarified butter, drawn butter -n07848936 ghee -n07849026 brown butter, beurre noisette -n07849186 Meuniere butter, lemon butter -n07849336 yogurt, yoghurt, yoghourt -n07849506 blueberry yogurt -n07849619 raita -n07849733 whey -n07849912 curd -n07850083 curd -n07850219 clabber -n07850329 cheese -n07851054 paring -n07851298 cream cheese -n07851443 double cream -n07851554 mascarpone -n07851641 triple cream, triple creme -n07851767 cottage cheese, pot cheese, farm cheese, farmer's cheese -n07851926 process cheese, processed cheese -n07852045 bleu, blue cheese -n07852229 Stilton -n07852302 Roquefort -n07852376 gorgonzola -n07852452 Danish blue -n07852532 Bavarian blue -n07852614 Brie -n07852712 brick cheese -n07852833 Camembert -n07852919 cheddar, cheddar cheese, Armerican cheddar, American cheese -n07853125 rat cheese, store cheese -n07853232 Cheshire cheese -n07853345 double Gloucester -n07853445 Edam -n07853560 goat cheese, chevre -n07853648 Gouda, Gouda cheese -n07853762 grated cheese -n07853852 hand cheese -n07853946 Liederkranz -n07854066 Limburger -n07854184 mozzarella -n07854266 Muenster -n07854348 Parmesan -n07854455 quark cheese, quark -n07854614 ricotta -n07854707 string cheese -n07854813 Swiss cheese -n07854982 Emmenthal, Emmental, Emmenthaler, Emmentaler -n07855105 Gruyere -n07855188 sapsago -n07855317 Velveeta -n07855413 nut butter -n07855510 peanut butter -n07855603 marshmallow fluff -n07855721 onion butter -n07855812 pimento butter -n07855907 shrimp butter -n07856045 lobster butter -n07856186 yak butter -n07856270 spread, paste -n07856756 cheese spread -n07856895 anchovy butter -n07856992 fishpaste -n07857076 garlic butter -n07857170 miso -n07857356 wasabi -n07857598 snail butter -n07857731 hummus, humus, hommos, hoummos, humous -n07857959 pate -n07858114 duck pate -n07858197 foie gras, pate de foie gras -n07858336 tapenade -n07858484 tahini -n07858595 sweetening, sweetener -n07858841 aspartame -n07858978 honey -n07859142 saccharin -n07859284 sugar, refined sugar -n07859583 syrup, sirup -n07859796 sugar syrup -n07859951 molasses -n07860103 sorghum, sorghum molasses -n07860208 treacle, golden syrup -n07860331 grenadine -n07860447 maple syrup -n07860548 corn syrup -n07860629 miraculous food, manna, manna from heaven -n07860805 batter -n07860988 dough -n07861158 bread dough -n07861247 pancake batter -n07861334 fritter batter -n07861557 coq au vin -n07861681 chicken provencale -n07861813 chicken and rice -n07861983 moo goo gai pan -n07862095 arroz con pollo -n07862244 bacon and eggs -n07862348 barbecued spareribs, spareribs -n07862461 beef Bourguignonne, boeuf Bourguignonne -n07862611 beef Wellington, filet de boeuf en croute -n07862770 bitok -n07862946 boiled dinner, New England boiled dinner -n07863107 Boston baked beans -n07863229 bubble and squeak -n07863374 pasta -n07863547 cannelloni -n07863644 carbonnade flamande, Belgian beef stew -n07863802 cheese souffle -n07863935 chicken Marengo -n07864065 chicken cordon bleu -n07864198 Maryland chicken -n07864317 chicken paprika, chicken paprikash -n07864475 chicken Tetrazzini -n07864638 Tetrazzini -n07864756 chicken Kiev -n07864934 chili, chili con carne -n07865105 chili dog -n07865196 chop suey -n07865484 chow mein -n07865575 codfish ball, codfish cake -n07865700 coquille -n07865788 coquilles Saint-Jacques -n07866015 croquette -n07866151 cottage pie -n07866277 rissole -n07866409 dolmas, stuffed grape leaves -n07866571 egg foo yong, egg fu yung -n07866723 egg roll, spring roll -n07866868 eggs Benedict -n07867021 enchilada -n07867164 falafel, felafel -n07867324 fish and chips -n07867421 fondue, fondu -n07867616 cheese fondue -n07867751 chocolate fondue -n07867883 fondue, fondu -n07868045 beef fondue, boeuf fondu bourguignon -n07868200 French toast -n07868340 fried rice, Chinese fried rice -n07868508 frittata -n07868684 frog legs -n07868830 galantine -n07868955 gefilte fish, fish ball -n07869111 haggis -n07869291 ham and eggs -n07869391 hash -n07869522 corned beef hash -n07869611 jambalaya -n07869775 kabob, kebab, shish kebab -n07869937 kedgeree -n07870069 souvlaki, souvlakia -n07870167 lasagna, lasagne -n07870313 seafood Newburg -n07870478 lobster Newburg, lobster a la Newburg -n07870620 shrimp Newburg -n07870734 Newburg sauce -n07870894 lobster thermidor -n07871065 lutefisk, lutfisk -n07871234 macaroni and cheese -n07871335 macedoine -n07871436 meatball -n07871588 porcupine ball, porcupines -n07871720 Swedish meatball -n07871810 meat loaf, meatloaf -n07872593 moussaka -n07872748 osso buco -n07873057 marrow, bone marrow -n07873198 pheasant under glass -n07873348 pigs in blankets -n07873464 pilaf, pilaff, pilau, pilaw -n07873679 bulgur pilaf -n07873807 pizza, pizza pie -n07874063 sausage pizza -n07874159 pepperoni pizza -n07874259 cheese pizza -n07874343 anchovy pizza -n07874441 Sicilian pizza -n07874531 poi -n07874674 pork and beans -n07874780 porridge -n07874995 oatmeal, burgoo -n07875086 loblolly -n07875152 potpie -n07875267 rijsttaffel, rijstaffel, rijstafel -n07875436 risotto, Italian rice -n07875560 roulade -n07875693 fish loaf -n07875835 salmon loaf -n07875926 Salisbury steak -n07876026 sauerbraten -n07876189 sauerkraut -n07876281 scallopine, scallopini -n07876460 veal scallopini -n07876550 scampi -n07876651 Scotch egg -n07876775 Scotch woodcock -n07876893 scrapple -n07877187 spaghetti and meatballs -n07877299 Spanish rice -n07877675 steak tartare, tartar steak, cannibal mound -n07877849 pepper steak -n07877961 steak au poivre, peppered steak, pepper steak -n07878145 beef Stroganoff -n07878283 stuffed cabbage -n07878479 kishke, stuffed derma -n07878647 stuffed peppers -n07878785 stuffed tomato, hot stuffed tomato -n07878926 stuffed tomato, cold stuffed tomato -n07879072 succotash -n07879174 sukiyaki -n07879350 sashimi -n07879450 sushi -n07879560 Swiss steak -n07879659 tamale -n07879821 tamale pie -n07879953 tempura -n07880080 teriyaki -n07880213 terrine -n07880325 Welsh rarebit, Welsh rabbit, rarebit -n07880458 schnitzel, Wiener schnitzel -n07880751 taco -n07880880 chicken taco -n07880968 burrito -n07881117 beef burrito -n07881205 quesadilla -n07881404 tostada -n07881525 bean tostada -n07881625 refried beans, frijoles refritos -n07881800 beverage, drink, drinkable, potable -n07882420 wish-wash -n07882497 concoction, mixture, intermixture -n07882886 mix, premix -n07883031 filling -n07883156 lekvar -n07883251 potion -n07883384 elixir -n07883510 elixir of life -n07883661 philter, philtre, love-potion, love-philter, love-philtre -n07884567 alcohol, alcoholic drink, alcoholic beverage, intoxicant, inebriant -n07885705 proof spirit -n07886057 home brew, homebrew -n07886176 hooch, hootch -n07886317 kava, kavakava -n07886463 aperitif -n07886572 brew, brewage -n07886849 beer -n07887099 draft beer, draught beer -n07887192 suds -n07887304 Munich beer, Munchener -n07887461 bock, bock beer -n07887634 lager, lager beer -n07887967 light beer -n07888058 Oktoberfest, Octoberfest -n07888229 Pilsner, Pilsener -n07888378 shebeen -n07888465 Weissbier, white beer, wheat beer -n07888816 Weizenbock -n07888909 malt -n07889193 wort -n07889274 malt, malt liquor -n07889510 ale -n07889814 bitter -n07889990 Burton -n07890068 pale ale -n07890226 porter, porter's beer -n07890352 stout -n07890540 Guinness -n07890617 kvass -n07890750 mead -n07890890 metheglin -n07890970 hydromel -n07891095 oenomel -n07891189 near beer -n07891309 ginger beer -n07891433 sake, saki, rice beer -n07891726 wine, vino -n07892418 vintage -n07892512 red wine -n07892813 white wine -n07893253 blush wine, pink wine, rose, rose wine -n07893425 altar wine, sacramental wine -n07893528 sparkling wine -n07893642 champagne, bubbly -n07893792 cold duck -n07893891 Burgundy, Burgundy wine -n07894102 Beaujolais -n07894298 Medoc -n07894451 Canary wine -n07894551 Chablis, white Burgundy -n07894703 Montrachet -n07894799 Chardonnay, Pinot Chardonnay -n07894965 Pinot noir -n07895100 Pinot blanc -n07895237 Bordeaux, Bordeaux wine -n07895435 claret, red Bordeaux -n07895595 Chianti -n07895710 Cabernet, Cabernet Sauvignon -n07895839 Merlot -n07895962 Sauvignon blanc -n07896060 California wine -n07896165 Cotes de Provence -n07896287 dessert wine -n07896422 Dubonnet -n07896560 jug wine -n07896661 macon, maconnais -n07896765 Moselle -n07896893 Muscadet -n07896994 plonk -n07897116 retsina -n07897200 Rhine wine, Rhenish, hock -n07897438 Riesling -n07897600 liebfraumilch -n07897750 Rhone wine -n07897865 Rioja -n07897975 sack -n07898117 Saint Emilion -n07898247 Soave -n07898333 zinfandel -n07898443 Sauterne, Sauternes -n07898617 straw wine -n07898745 table wine -n07898895 Tokay -n07899003 vin ordinaire -n07899108 vermouth -n07899292 sweet vermouth, Italian vermouth -n07899434 dry vermouth, French vermouth -n07899533 Chenin blanc -n07899660 Verdicchio -n07899769 Vouvray -n07899899 Yquem -n07899976 generic, generic wine -n07900225 varietal, varietal wine -n07900406 fortified wine -n07900616 Madeira -n07900734 malmsey -n07900825 port, port wine -n07900958 sherry -n07901355 Marsala -n07901457 muscat, muscatel, muscadel, muscadelle -n07901587 liquor, spirits, booze, hard drink, hard liquor, John Barleycorn, strong drink -n07902121 neutral spirits, ethyl alcohol -n07902336 aqua vitae, ardent spirits -n07902443 eau de vie -n07902520 moonshine, bootleg, corn liquor -n07902698 bathtub gin -n07902799 aquavit, akvavit -n07902937 arrack, arak -n07903101 bitters -n07903208 brandy -n07903543 applejack -n07903643 Calvados -n07903731 Armagnac -n07903841 Cognac -n07903962 grappa -n07904072 kirsch -n07904293 slivovitz -n07904395 gin -n07904637 sloe gin -n07904760 geneva, Holland gin, Hollands -n07904865 grog -n07904934 ouzo -n07905038 rum -n07905296 demerara, demerara rum -n07905386 Jamaica rum -n07905474 schnapps, schnaps -n07905618 pulque -n07905770 mescal -n07905979 tequila -n07906111 vodka -n07906284 whiskey, whisky -n07906572 blended whiskey, blended whisky -n07906718 bourbon -n07906877 corn whiskey, corn whisky, corn -n07907037 firewater -n07907161 Irish, Irish whiskey, Irish whisky -n07907342 poteen -n07907429 rye, rye whiskey, rye whisky -n07907548 Scotch, Scotch whiskey, Scotch whisky, malt whiskey, malt whisky, Scotch malt whiskey, Scotch malt whisky -n07907831 sour mash, sour mash whiskey -n07907943 liqueur, cordial -n07908411 absinth, absinthe -n07908567 amaretto -n07908647 anisette, anisette de Bordeaux -n07908812 benedictine -n07908923 Chartreuse -n07909129 coffee liqueur -n07909231 creme de cacao -n07909362 creme de menthe -n07909504 creme de fraise -n07909593 Drambuie -n07909714 Galliano -n07909811 orange liqueur -n07909954 curacao, curacoa -n07910048 triple sec -n07910152 Grand Marnier -n07910245 kummel -n07910379 maraschino, maraschino liqueur -n07910538 pastis -n07910656 Pernod -n07910799 pousse-cafe -n07910970 Kahlua -n07911061 ratafia, ratafee -n07911249 sambuca -n07911371 mixed drink -n07911677 cocktail -n07912093 Dom Pedro -n07912211 highball -n07913180 mixer -n07913300 bishop -n07913393 Bloody Mary -n07913537 Virgin Mary, bloody shame -n07913644 bullshot -n07913774 cobbler -n07913882 collins, Tom Collins -n07914006 cooler -n07914128 refresher -n07914271 smoothie -n07914413 daiquiri, rum cocktail -n07914586 strawberry daiquiri -n07914686 NADA daiquiri -n07914777 spritzer -n07914887 flip -n07914995 gimlet -n07915094 gin and tonic -n07915213 grasshopper -n07915366 Harvey Wallbanger -n07915491 julep, mint julep -n07915618 manhattan -n07915800 Rob Roy -n07915918 margarita -n07916041 martini -n07916183 gin and it -n07916319 vodka martini -n07916437 old fashioned -n07916582 pink lady -n07917133 Sazerac -n07917272 screwdriver -n07917392 sidecar -n07917507 Scotch and soda -n07917618 sling -n07917791 brandy sling -n07917874 gin sling -n07917951 rum sling -n07918028 sour -n07918193 whiskey sour, whisky sour -n07918309 stinger -n07918706 swizzle -n07918879 hot toddy, toddy -n07919165 zombie, zombi -n07919310 fizz -n07919441 Irish coffee -n07919572 cafe au lait -n07919665 cafe noir, demitasse -n07919787 decaffeinated coffee, decaf -n07919894 drip coffee -n07920052 espresso -n07920222 caffe latte, latte -n07920349 cappuccino, cappuccino coffee, coffee cappuccino -n07920540 iced coffee, ice coffee -n07920663 instant coffee -n07920872 mocha, mocha coffee -n07920989 mocha -n07921090 cassareep -n07921239 Turkish coffee -n07921360 chocolate milk -n07921455 cider, cyder -n07921615 hard cider -n07921834 scrumpy -n07921948 sweet cider -n07922041 mulled cider -n07922147 perry -n07922512 rotgut -n07922607 slug -n07922764 cocoa, chocolate, hot chocolate, drinking chocolate -n07922955 criollo -n07923748 juice -n07924033 fruit juice, fruit crush -n07924276 nectar -n07924366 apple juice -n07924443 cranberry juice -n07924560 grape juice -n07924655 must -n07924747 grapefruit juice -n07924834 orange juice -n07924955 frozen orange juice, orange-juice concentrate -n07925116 pineapple juice -n07925229 lemon juice -n07925327 lime juice -n07925423 papaya juice -n07925500 tomato juice -n07925608 carrot juice -n07925708 V-8 juice -n07925808 koumiss, kumis -n07925966 fruit drink, ade -n07926250 lemonade -n07926346 limeade -n07926442 orangeade -n07926540 malted milk -n07926785 mate -n07926920 mulled wine -n07927070 negus -n07927197 soft drink -n07927512 pop, soda, soda pop, soda water, tonic -n07927716 birch beer -n07927836 bitter lemon -n07927931 cola, dope -n07928163 cream soda -n07928264 egg cream -n07928367 ginger ale, ginger pop -n07928488 orange soda -n07928578 phosphate -n07928696 Coca Cola, Coke -n07928790 Pepsi, Pepsi Cola -n07928887 root beer -n07928998 sarsaparilla -n07929172 tonic, tonic water, quinine water -n07929351 coffee bean, coffee berry, coffee -n07929519 coffee, java -n07929940 cafe royale, coffee royal -n07930062 fruit punch -n07930205 milk punch -n07930315 mimosa, buck's fizz -n07930433 pina colada -n07930554 punch -n07930864 cup -n07931001 champagne cup -n07931096 claret cup -n07931280 wassail -n07931452 planter's punch -n07931612 White Russian -n07931733 fish house punch -n07931870 May wine -n07932039 eggnog -n07932323 cassiri -n07932454 spruce beer -n07932614 rickey -n07932762 gin rickey -n07932841 tea, tea leaf -n07933154 tea bag -n07933274 tea -n07933530 tea-like drink -n07933652 cambric tea -n07933799 cuppa, cupper -n07933891 herb tea, herbal tea, herbal -n07934032 tisane -n07934152 camomile tea -n07934282 ice tea, iced tea -n07934373 sun tea -n07934530 black tea -n07934678 congou, congo, congou tea, English breakfast tea -n07934800 Darjeeling -n07934908 orange pekoe, pekoe -n07935043 souchong, soochong -n07935152 green tea -n07935288 hyson -n07935379 oolong -n07935504 water -n07935737 bottled water -n07935878 branch water -n07936015 spring water -n07936093 sugar water -n07936263 drinking water -n07936459 ice water -n07936548 soda water, carbonated water, club soda, seltzer, sparkling water -n07936745 mineral water -n07936979 seltzer -n07937069 Vichy water -n07937344 perishable, spoilable -n07937461 couscous -n07937621 ramekin, ramequin -n07938007 multivitamin, multivitamin pill -n07938149 vitamin pill -n07938313 soul food -n07938594 mold, mould -n07942152 people -n07951464 collection, aggregation, accumulation, assemblage -n07954211 book, rule book -n07977870 library -n08079613 baseball club, ball club, club, nine -n08182379 crowd -n08238463 class, form, grade, course -n08242223 core, nucleus, core group -n08249459 concert band, military band -n08253141 dance -n08256735 wedding, wedding party -n08376250 chain, concatenation -n08385989 power breakfast -n08492354 aerie, aery, eyrie, eyry -n08492461 agora -n08494231 amusement park, funfair, pleasure ground -n08495908 aphelion -n08496334 apron -n08500819 interplanetary space -n08500989 interstellar space -n08501887 intergalactic space -n08505018 bush -n08506347 semidesert -n08511017 beam-ends -n08517010 bridgehead -n08517676 bus stop -n08518171 campsite, campground, camping site, camping ground, bivouac, encampment, camping area -n08519299 detention basin -n08521623 cemetery, graveyard, burial site, burial ground, burying ground, memorial park, necropolis -n08523340 trichion, crinion -n08524735 city, metropolis, urban center -n08539072 business district, downtown -n08539276 outskirts -n08540532 borough -n08547468 cow pasture -n08547544 crest -n08551296 eparchy, exarchate -n08554440 suburb, suburbia, suburban area -n08555333 stockbroker belt -n08555710 crawlspace, crawl space -n08558770 sheikdom, sheikhdom -n08558963 residence, abode -n08559155 domicile, legal residence -n08560295 dude ranch -n08569482 farmland, farming area -n08571275 midfield -n08571642 firebreak, fireguard -n08571898 flea market -n08573674 battlefront, front, front line -n08573842 garbage heap, junk heap, rubbish heap, scrapheap, trash heap, junk pile, trash pile, refuse heap -n08578517 benthos, benthic division, benthonic zone -n08579266 goldfield -n08579352 grainfield, grain field -n08580944 half-mast, half-staff -n08583292 hemline -n08583455 heronry -n08583554 hipline -n08583682 hipline -n08584914 hole-in-the-wall -n08586978 junkyard -n08589670 isoclinic line, isoclinal -n08596076 littoral, litoral, littoral zone, sands -n08597579 magnetic pole -n08598301 grassland -n08598568 mecca -n08599174 observer's meridian -n08599292 prime meridian -n08611339 nombril -n08611421 no-parking zone -n08613733 outdoors, out-of-doors, open air, open -n08614632 fairground -n08616050 pasture, pastureland, grazing land, lea, ley -n08618831 perihelion -n08619112 periselene, perilune -n08623676 locus of infection -n08628141 kasbah, casbah -n08633683 waterfront -n08640531 resort, resort hotel, holiday resort -n08640739 resort area, playground, vacation spot -n08640962 rough -n08643267 ashram -n08644045 harborage, harbourage -n08645104 scrubland -n08645212 weald -n08645318 wold -n08647264 schoolyard -n08648917 showplace -n08649711 bedside -n08651104 sideline, out of bounds -n08652376 ski resort -n08658309 soil horizon -n08658918 geological horizon -n08659242 coal seam -n08659331 coalface -n08659446 field -n08659861 oilfield -n08661878 Temperate Zone -n08662427 terreplein -n08663051 three-mile limit -n08663703 desktop -n08663860 top -n08673039 kampong, campong -n08674344 subtropics, semitropics -n08676253 barrio -n08677424 veld, veldt -n08677801 vertex, peak, apex, acme -n08678783 waterline, water line, water level -n08679167 high-water mark -n08679269 low-water mark -n08679562 continental divide -n08685188 zodiac -n08782627 Aegean island -n08896327 sultanate -n09032191 Swiss canton -n09186592 abyssal zone -n09189157 aerie, aery, eyrie, eyry -n09191635 air bubble -n09193551 alluvial flat, alluvial plain -n09193705 alp -n09194227 Alpine glacier, Alpine type of glacier -n09199101 anthill, formicary -n09201998 aquifer -n09203827 archipelago -n09205509 arete -n09206896 arroyo -n09206985 ascent, acclivity, rise, raise, climb, upgrade -n09208496 asterism -n09209025 asthenosphere -n09210862 atoll -n09213434 bank -n09213565 bank -n09214060 bar -n09214269 barbecue pit -n09214916 barrier reef -n09215023 baryon, heavy particle -n09215437 basin -n09217230 beach -n09218315 honeycomb -n09218494 belay -n09218641 ben -n09219233 berm -n09223487 bladder stone, cystolith -n09224725 bluff -n09226869 borrow pit -n09228055 brae -n09229709 bubble -n09230041 burrow, tunnel -n09230202 butte -n09231117 caldera -n09233446 canyon, canon -n09233603 canyonside -n09238926 cave -n09239302 cavern -n09242389 chasm -n09245515 cirque, corrie, cwm -n09246464 cliff, drop, drop-off -n09247410 cloud -n09248153 coast -n09248399 coastland -n09249034 col, gap -n09249155 collector -n09251407 comet -n09255070 continental glacier -n09256479 coral reef -n09257843 cove -n09259025 crag -n09259219 crater -n09260907 cultivated land, farmland, plowland, ploughland, tilled land, tillage, tilth -n09262690 dale -n09263912 defile, gorge -n09264803 delta -n09265620 descent, declivity, fall, decline, declination, declension, downslope -n09266604 diapir -n09267854 divot -n09268007 divot -n09269341 down -n09269472 downhill -n09269882 draw -n09270160 drey -n09270657 drumlin -n09270735 dune, sand dune -n09274152 escarpment, scarp -n09274305 esker -n09279986 fireball -n09281252 flare star -n09282208 floor -n09283193 fomite, vehicle -n09283405 foothill -n09283514 footwall -n09283767 foreland -n09283866 foreshore -n09287415 gauge boson -n09287968 geological formation, formation -n09288635 geyser -n09289331 glacier -n09289596 glen -n09290350 gopher hole -n09290444 gorge -n09294877 grotto, grot -n09295210 growler -n09295946 gulch, flume -n09300306 gully -n09300905 hail -n09302616 highland, upland -n09303008 hill -n09303528 hillside -n09304750 hole, hollow -n09305031 hollow, holler -n09305898 hot spring, thermal spring -n09308572 iceberg, berg -n09308743 icecap, ice cap -n09309046 ice field -n09309168 ice floe, floe -n09309292 ice mass -n09310616 inclined fault -n09315159 ion -n09319604 isthmus -n09325824 kidney stone, urinary calculus, nephrolith, renal calculus -n09326662 knoll, mound, hillock, hummock, hammock -n09327077 kopje, koppie -n09327538 Kuiper belt, Edgeworth-Kuiper belt -n09330378 lake bed, lake bottom -n09331251 lakefront -n09332890 lakeside, lakeshore -n09335693 landfall -n09335809 landfill -n09336555 lather -n09337048 leak -n09337253 ledge, shelf -n09338013 lepton -n09339810 lithosphere, geosphere -n09344198 lowland -n09344324 lunar crater -n09344724 maar -n09348460 massif -n09349648 meander -n09351905 mesa, table -n09352849 meteorite -n09353815 microfossil -n09354511 midstream -n09357346 molehill -n09357447 monocline -n09359803 mountain, mount -n09361517 mountainside, versant -n09362316 mouth -n09362945 mull -n09366017 natural depression, depression -n09366317 natural elevation, elevation -n09375606 nullah -n09376198 ocean -n09376526 ocean floor, sea floor, ocean bottom, seabed, sea bottom, Davy Jones's locker, Davy Jones -n09376786 oceanfront -n09381242 outcrop, outcropping, rock outcrop -n09382099 oxbow -n09384106 pallasite -n09389867 perforation -n09391386 photosphere -n09391644 piedmont -n09391774 Piedmont glacier, Piedmont type of glacier -n09392402 pinetum -n09393524 plage -n09393605 plain, field, champaign -n09396465 point -n09396608 polar glacier -n09398076 pothole, chuckhole -n09398677 precipice -n09399592 promontory, headland, head, foreland -n09400584 ptyalith -n09400987 pulsar -n09402944 quicksand -n09403086 rabbit burrow, rabbit hole -n09403211 radiator -n09403427 rainbow -n09403734 range, mountain range, range of mountains, chain, mountain chain, chain of mountains -n09405078 rangeland -n09405787 ravine -n09406793 reef -n09409512 ridge -n09409752 ridge, ridgeline -n09410224 rift valley -n09411189 riparian forest -n09411295 ripple mark -n09415584 riverbank, riverside -n09415671 riverbed, river bottom -n09416076 rock, stone -n09416890 roof -n09421031 saltpan -n09421799 sandbank -n09421951 sandbar, sand bar -n09422190 sandpit -n09422631 sanitary landfill -n09425019 sawpit -n09425344 scablands -n09428293 seashore, coast, seacoast, sea-coast -n09428628 seaside, seaboard -n09429630 seif dune -n09432283 shell -n09432990 shiner -n09433312 shoal -n09433442 shore -n09433839 shoreline -n09435739 sinkhole, sink, swallow hole -n09436444 ski slope -n09436708 sky -n09437454 slope, incline, side -n09438844 snowcap -n09438940 snowdrift -n09439032 snowfield -n09439213 soapsuds, suds, lather -n09442595 spit, tongue -n09443281 spoor -n09443641 spume -n09444783 star -n09445008 steep -n09445289 steppe -n09447666 strand -n09448690 streambed, creek bed -n09450163 sun, Sun -n09451237 supernova -n09452291 swale -n09452395 swamp, swampland -n09452760 swell -n09453008 tableland, plateau -n09454153 talus, scree -n09454412 tangle -n09454744 tar pit -n09456207 terrace, bench -n09457979 tidal basin -n09458269 tideland -n09459979 tor -n09460046 tor -n09461069 Trapezium -n09462600 troposphere -n09463226 tundra -n09464486 twinkler -n09466678 uphill -n09467696 urolith -n09468604 valley, vale -n09470027 vehicle-borne transmission -n09470222 vein, mineral vein -n09472413 volcanic crater, crater -n09472597 volcano -n09474010 wadi -n09474412 wall -n09474765 warren, rabbit warren -n09475044 wasp's nest, wasps' nest, hornet's nest, hornets' nest -n09475179 watercourse -n09475925 waterside -n09476123 water table, water level, groundwater level -n09478210 whinstone, whin -n09480959 wormcast -n09481120 xenolith -n09493983 Circe -n09495962 gryphon, griffin, griffon -n09505153 spiritual leader -n09537660 messiah, christ -n09556121 Rhea Silvia, Rea Silvia -n09605110 number one -n09606009 adventurer, venturer -n09606527 anomaly, unusual person -n09607630 appointee, appointment -n09607782 argonaut -n09607903 Ashkenazi -n09608709 benefactor, helper -n09610255 color-blind person -n09610405 commoner, common man, common person -n09611722 conservator -n09612700 contrarian -n09613118 contadino -n09613191 contestant -n09613690 cosigner, cosignatory -n09615336 discussant -n09616573 enologist, oenologist, fermentologist -n09616922 entertainer -n09617161 eulogist, panegyrist -n09617435 ex-gambler -n09617577 experimenter -n09617696 experimenter -n09618760 exponent -n09618880 ex-president -n09618957 face -n09619168 female, female person -n09619452 finisher -n09620078 inhabitant, habitant, dweller, denizen, indweller -n09620794 native, indigen, indigene, aborigine, aboriginal -n09621232 native -n09622049 juvenile, juvenile person -n09622302 lover -n09624168 male, male person -n09624559 mediator, go-between, intermediator, intermediary, intercessor -n09624899 mediatrix -n09625401 national, subject -n09626238 peer, equal, match, compeer -n09627807 prize winner, lottery winner -n09627906 recipient, receiver -n09629065 religionist -n09629246 sensualist -n09629752 traveler, traveller -n09631129 unwelcome person, persona non grata -n09632274 unskilled person -n09632518 worker -n09633969 wrongdoer, offender -n09635534 Black African -n09635635 Afrikaner, Afrikander, Boer -n09635973 Aryan -n09636339 Black, Black person, blackamoor, Negro, Negroid -n09637339 Black woman -n09638454 mulatto -n09638875 White, White person, Caucasian -n09639382 Circassian -n09639919 Semite -n09640327 Chaldean, Chaldaean, Chaldee -n09640715 Elamite -n09641002 white man -n09641578 WASP, white Anglo-Saxon Protestant -n09643799 gook, slant-eye -n09644152 Mongol, Mongolian -n09644657 Tatar, Tartar, Mongol Tatar -n09648743 Nahuatl -n09648911 Aztec -n09649067 Olmec -n09650729 Biloxi -n09650839 Blackfoot -n09650989 Brule -n09651123 Caddo -n09651968 Cheyenne -n09652149 Chickasaw -n09653144 Cocopa, Cocopah -n09653438 Comanche -n09654079 Creek -n09654518 Delaware -n09654898 Diegueno -n09655213 Esselen -n09655466 Eyeish -n09656077 Havasupai -n09657206 Hunkpapa -n09657748 Iowa, Ioway -n09658254 Kalapooia, Kalapuya, Calapooya, Calapuya -n09658398 Kamia -n09658815 Kekchi -n09658921 Kichai -n09659039 Kickapoo -n09659188 Kiliwa, Kiliwi -n09660010 Malecite -n09660240 Maricopa -n09661873 Mohican, Mahican -n09662038 Muskhogean, Muskogean -n09662661 Navaho, Navajo -n09662951 Nootka -n09663248 Oglala, Ogalala -n09663786 Osage -n09663999 Oneida -n09664556 Paiute, Piute -n09664908 Passamaquody -n09665367 Penobscot -n09665545 Penutian -n09666349 Potawatomi -n09666476 Powhatan -n09666883 kachina -n09667358 Salish -n09668199 Shahaptian, Sahaptin, Sahaptino -n09668437 Shasta -n09668562 Shawnee -n09668988 Sihasapa -n09669631 Teton, Lakota, Teton Sioux, Teton Dakota -n09670280 Taracahitian -n09670521 Tarahumara -n09670909 Tuscarora -n09671089 Tutelo -n09672590 Yana -n09672725 Yavapai -n09672840 Yokuts -n09673091 Yuma -n09674412 Gadaba -n09674786 Kolam -n09675045 Kui -n09675673 Toda -n09675799 Tulu -n09675922 Gujarati, Gujerati -n09676021 Kashmiri -n09676247 Punjabi, Panjabi -n09676884 Slav -n09677427 Anabaptist -n09678747 Adventist, Second Adventist -n09679028 gentile, non-Jew, goy -n09679170 gentile -n09679925 Catholic -n09680908 Old Catholic -n09681107 Uniat, Uniate, Uniate Christian -n09681234 Copt -n09681973 Jewess -n09683180 Jihadist -n09683757 Buddhist -n09683924 Zen Buddhist -n09684082 Mahayanist -n09684901 swami -n09685233 Hare Krishna -n09685806 Shintoist -n09686262 Eurafrican -n09686401 Eurasian -n09688233 Gael -n09688804 Frank -n09689435 Afghan, Afghanistani -n09689958 Albanian -n09690083 Algerian -n09690208 Altaic -n09690496 Andorran -n09690621 Angolan -n09690864 Anguillan -n09691604 Austrian -n09691729 Bahamian -n09691858 Bahraini, Bahreini -n09692125 Basotho -n09692915 Herero -n09693244 Luba, Chiluba -n09693982 Barbadian -n09694664 Bolivian -n09694771 Bornean -n09695019 Carioca -n09695132 Tupi -n09695514 Bruneian -n09695620 Bulgarian -n09695979 Byelorussian, Belorussian, White Russian -n09696456 Cameroonian -n09696585 Canadian -n09696763 French Canadian -n09697401 Central American -n09697986 Chilean -n09698644 Congolese -n09699020 Cypriot, Cypriote, Cyprian -n09699642 Dane -n09700125 Djiboutian -n09700964 Britisher, Briton, Brit -n09701148 English person -n09701833 Englishwoman -n09702134 Anglo-Saxon -n09702673 Angle -n09703101 West Saxon -n09703344 Lombard, Langobard -n09703485 limey, John Bull -n09703708 Cantabrigian -n09703809 Cornishman -n09703932 Cornishwoman -n09704057 Lancastrian -n09704157 Lancastrian -n09704283 Geordie -n09705003 Oxonian -n09705124 Ethiopian -n09705671 Amhara -n09705784 Eritrean -n09706029 Finn -n09706255 Komi -n09707061 Livonian -n09707289 Lithuanian -n09707735 Selkup, Ostyak-Samoyed -n09708750 Parisian -n09708889 Parisienne -n09709531 Creole -n09709673 Creole -n09710041 Gabonese -n09710164 Greek, Hellene -n09710886 Dorian -n09711132 Athenian -n09711435 Laconian -n09712324 Guyanese -n09712448 Haitian -n09712696 Malay, Malayan -n09712967 Moro -n09713108 Netherlander, Dutchman, Hollander -n09714120 Icelander -n09714694 Iraqi, Iraki -n09715165 Irishman -n09715303 Irishwoman -n09715427 Dubliner -n09716047 Italian -n09716933 Roman -n09717233 Sabine -n09718217 Japanese, Nipponese -n09718811 Jordanian -n09718936 Korean -n09719309 Kenyan -n09719794 Lao, Laotian -n09720033 Lapp, Lapplander, Sami, Saami, Same, Saame -n09720256 Latin American, Latino -n09720595 Lebanese -n09720702 Levantine -n09720842 Liberian -n09721244 Luxemburger, Luxembourger -n09721444 Macedonian -n09722064 Sabahan -n09722658 Mexican -n09722817 Chicano -n09723067 Mexican-American, Mexicano -n09723819 Namibian -n09723944 Nauruan -n09724234 Gurkha -n09724533 New Zealander, Kiwi -n09724656 Nicaraguan -n09724785 Nigerian -n09725000 Hausa, Haussa -n09725229 North American -n09725546 Nova Scotian, bluenose -n09725653 Omani -n09725772 Pakistani -n09725935 Brahui -n09726621 South American Indian -n09726811 Carib, Carib Indian -n09727440 Filipino -n09727826 Polynesian -n09728137 Qatari, Katari -n09728285 Romanian, Rumanian -n09729062 Muscovite -n09729156 Georgian -n09730077 Sarawakian -n09730204 Scandinavian, Norse, Northman -n09730824 Senegalese -n09731343 Slovene -n09731436 South African -n09731571 South American -n09732170 Sudanese -n09733459 Syrian -n09733793 Tahitian -n09734185 Tanzanian -n09734450 Tibetan -n09734535 Togolese -n09734639 Tuareg -n09735258 Turki -n09735654 Chuvash -n09736485 Turkoman, Turkmen, Turcoman -n09736798 Uzbek, Uzbeg, Uzbak, Usbek, Usbeg -n09736945 Ugandan -n09737050 Ukranian -n09737161 Yakut -n09737453 Tungus, Evenk -n09738121 Igbo -n09738400 American -n09740724 Anglo-American -n09741074 Alaska Native, Alaskan Native, Native Alaskan -n09741331 Arkansan, Arkansawyer -n09741722 Carolinian -n09741816 Coloradan -n09741904 Connecticuter -n09741999 Delawarean, Delawarian -n09742101 Floridian -n09742315 German American -n09742927 Illinoisan -n09743487 Mainer, Down Easter -n09743601 Marylander -n09743792 Minnesotan, Gopher -n09744161 Nebraskan, Cornhusker -n09744346 New Hampshirite, Granite Stater -n09744462 New Jerseyan, New Jerseyite, Garden Stater -n09744679 New Yorker -n09744834 North Carolinian, Tarheel -n09745229 Oregonian, Beaver -n09745324 Pennsylvanian, Keystone Stater -n09745834 Texan -n09745933 Utahan -n09746936 Uruguayan -n09747191 Vietnamese, Annamese -n09747495 Gambian -n09748101 East German -n09748408 Berliner -n09748648 Prussian -n09748889 Ghanian -n09749386 Guinean -n09750282 Papuan -n09750641 Walloon -n09750770 Yemeni -n09750891 Yugoslav, Jugoslav, Yugoslavian, Jugoslavian -n09751076 Serbian, Serb -n09751496 Xhosa -n09751622 Zairese, Zairean -n09751895 Zimbabwean -n09752023 Zulu -n09752519 Gemini, Twin -n09753348 Sagittarius, Archer -n09753792 Pisces, Fish -n09754152 abbe -n09754217 abbess, mother superior, prioress -n09754633 abnegator -n09754907 abridger, abbreviator -n09755086 abstractor, abstracter -n09755241 absconder -n09755555 absolver -n09755788 abecedarian -n09755893 aberrant -n09756049 abettor, abetter -n09756195 abhorrer -n09756961 abomination -n09757449 abseiler, rappeller -n09758173 abstainer, ascetic -n09758885 academic administrator -n09759501 academician -n09760290 accessory before the fact -n09760609 companion -n09760913 accompanist, accompanyist -n09761068 accomplice, confederate -n09761753 account executive, account representative, registered representative, customer's broker, customer's man -n09762011 accused -n09762385 accuser -n09763272 acid head -n09763784 acquaintance, friend -n09764201 acquirer -n09764598 aerialist -n09764732 action officer -n09764900 active -n09765118 active citizen -n09765278 actor, histrion, player, thespian, role player -n09767197 actor, doer, worker -n09769076 addict, nut, freak, junkie, junky -n09769525 adducer -n09769929 adjuster, adjustor, claims adjuster, claims adjustor, claim agent -n09770179 adjutant, aide, aide-de-camp -n09770359 adjutant general -n09771435 admirer, adorer -n09772330 adoptee -n09772746 adulterer, fornicator -n09772930 adulteress, fornicatress, hussy, jade, loose woman, slut, strumpet, trollop -n09773962 advertiser, advertizer, adman -n09774167 advisee -n09774783 advocate, advocator, proponent, exponent -n09775907 aeronautical engineer -n09776346 affiliate -n09776642 affluent -n09776807 aficionado -n09777870 buck sergeant -n09778266 agent-in-place -n09778537 aggravator, annoyance -n09778783 agitator, fomenter -n09778927 agnostic -n09779124 agnostic, doubter -n09779280 agonist -n09779461 agony aunt -n09779790 agriculturist, agriculturalist, cultivator, grower, raiser -n09780395 air attache -n09780828 air force officer, commander -n09780984 airhead -n09781398 air traveler, air traveller -n09781504 alarmist -n09781650 albino -n09782167 alcoholic, alky, dipsomaniac, boozer, lush, soaker, souse -n09782397 alderman -n09782855 alexic -n09783537 alienee, grantee -n09783776 alienor -n09783884 aliterate, aliterate person -n09784043 algebraist -n09784160 allegorizer, allegoriser -n09784564 alliterator -n09785236 almoner, medical social worker -n09785659 alpinist -n09785891 altar boy -n09786115 alto -n09787534 ambassador, embassador -n09787765 ambassador -n09788073 ambusher -n09788237 amicus curiae, friend of the court -n09789150 amoralist -n09789566 amputee -n09789898 analogist -n09790047 analphabet, analphabetic -n09790482 analyst -n09791014 industry analyst -n09791419 market strategist -n09791816 anarchist, nihilist, syndicalist -n09792125 anathema, bete noire -n09792555 ancestor, ascendant, ascendent, antecedent, root -n09792969 anchor, anchorman, anchorperson -n09793141 ancient -n09793352 anecdotist, raconteur -n09793946 angler, troller -n09794550 animator -n09794668 animist -n09795010 annotator -n09795124 announcer -n09795334 announcer -n09796809 anti -n09796974 anti-American -n09797742 anti-Semite, Jew-baiter -n09797873 Anzac -n09797998 ape-man -n09798096 aphakic -n09800469 appellant, plaintiff in error -n09800964 appointee -n09801102 apprehender -n09801275 April fool -n09801533 aspirant, aspirer, hopeful, wannabe, wannabee -n09802445 appreciator -n09802641 appropriator -n09802951 Arabist -n09804230 archaist -n09805151 archbishop -n09805324 archer, bowman -n09805475 architect, designer -n09806944 archivist -n09807075 archpriest, hierarch, high priest, prelate, primate -n09808080 Aristotelian, Aristotelean, Peripatetic -n09808591 armiger -n09809279 army attache -n09809538 army engineer, military engineer -n09809749 army officer -n09809925 arranger, adapter, transcriber -n09810166 arrival, arriver, comer -n09811568 arthritic -n09811712 articulator -n09811852 artilleryman, cannoneer, gunner, machine gunner -n09813219 artist's model, sitter -n09814252 assayer -n09814381 assemblyman -n09814488 assemblywoman -n09814567 assenter -n09814660 asserter, declarer, affirmer, asseverator, avower -n09815455 assignee -n09815790 assistant, helper, help, supporter -n09816654 assistant professor -n09816771 associate -n09817174 associate -n09817386 associate professor -n09818022 astronaut, spaceman, cosmonaut -n09819477 cosmographer, cosmographist -n09820044 atheist -n09820263 athlete, jock -n09821831 attendant, attender, tender -n09822830 attorney general -n09823153 auditor -n09823287 augur, auspex -n09823502 aunt, auntie, aunty -n09823832 au pair girl -n09824135 authoritarian, dictator -n09824609 authority -n09825096 authorizer, authoriser -n09825750 automobile mechanic, auto-mechanic, car-mechanic, mechanic, grease monkey -n09826204 aviator, aeronaut, airman, flier, flyer -n09826605 aviatrix, airwoman, aviatress -n09826821 ayah -n09827246 babu, baboo -n09827363 baby, babe, sister -n09828216 baby -n09828403 baby boomer, boomer -n09828988 baby farmer -n09830194 back -n09830400 backbencher -n09830629 backpacker, packer -n09830759 backroom boy, brain truster -n09830926 backscratcher -n09831962 bad person -n09832456 baggage -n09832633 bag lady -n09832978 bailee -n09833111 bailiff -n09833275 bailor -n09833441 bairn -n09833536 baker, bread maker -n09833751 balancer -n09833997 balker, baulker, noncompliant -n09834258 ball-buster, ball-breaker -n09834378 ball carrier, runner -n09834699 ballet dancer -n09834885 ballet master -n09835017 ballet mistress -n09835153 balletomane -n09835230 ball hawk -n09835348 balloonist -n09835506 ballplayer, baseball player -n09836160 bullfighter, toreador -n09836343 banderillero -n09836519 matador -n09836786 picador -n09837459 bandsman -n09837720 banker -n09838295 bank robber -n09838370 bankrupt, insolvent -n09838621 bantamweight -n09839702 barmaid -n09840217 baron, big businessman, business leader, king, magnate, mogul, power, top executive, tycoon -n09840435 baron -n09840520 baron -n09841188 bartender, barman, barkeep, barkeeper, mixologist -n09841515 baseball coach, baseball manager -n09841696 base runner, runner -n09842047 basketball player, basketeer, cager -n09842288 basketweaver, basketmaker -n09842395 Basket Maker -n09842528 bass, basso -n09842823 bastard, by-blow, love child, illegitimate child, illegitimate, whoreson -n09843443 bat boy -n09843602 bather -n09843716 batman -n09843824 baton twirler, twirler -n09844457 Bavarian -n09844898 beadsman, bedesman -n09845401 beard -n09845849 beatnik, beat -n09846142 beauty consultant -n09846469 Bedouin, Beduin -n09846586 bedwetter, bed wetter, wetter -n09846755 beekeeper, apiarist, apiculturist -n09846894 beer drinker, ale drinker -n09847267 beggarman -n09847344 beggarwoman -n09847543 beldam, beldame -n09848110 theist -n09848489 believer, truster -n09849167 bell founder -n09849990 benedick, benedict -n09850760 berserker, berserk -n09850974 besieger -n09851165 best, topper -n09851575 betrothed -n09853541 Big Brother -n09853645 bigot -n09853881 big shot, big gun, big wheel, big cheese, big deal, big enchilada, big fish, head honcho -n09854218 big sister -n09854421 billiard player -n09854915 biochemist -n09855433 biographer -n09856401 bird fancier -n09856671 birth -n09856827 birth-control campaigner, birth-control reformer -n09857007 bisexual, bisexual person -n09858165 black belt -n09858299 blackmailer, extortioner, extortionist -n09858733 Black Muslim -n09859152 blacksmith -n09859285 blade -n09859975 blind date -n09861287 bluecoat -n09861599 bluestocking, bas bleu -n09861863 boatbuilder -n09861946 boatman, boater, waterman -n09862183 boatswain, bos'n, bo's'n, bosun, bo'sun -n09862621 bobby -n09863031 bodyguard, escort -n09863339 boffin -n09863749 Bolshevik, Marxist, red, bolshie, bolshy -n09863936 Bolshevik, Bolshevist -n09864632 bombshell -n09864968 bondman, bondsman -n09865068 bondwoman, bondswoman, bondmaid -n09865162 bondwoman, bondswoman, bondmaid -n09865398 bond servant -n09865672 book agent -n09865744 bookbinder -n09866115 bookkeeper -n09866354 bookmaker -n09866559 bookworm -n09866661 booster, shoplifter, lifter -n09866817 bootblack, shoeblack -n09866922 bootlegger, moonshiner -n09867069 bootmaker, boot maker -n09867154 borderer -n09867311 border patrolman -n09868270 botanist, phytologist, plant scientist -n09868782 bottom feeder -n09868899 boulevardier -n09869317 bounty hunter -n09869447 bounty hunter -n09869578 Bourbon -n09870096 bowler -n09871095 slugger, slogger -n09871229 cub, lad, laddie, sonny, sonny boy -n09871681 Boy Scout -n09871867 boy scout -n09871952 boy wonder -n09872066 bragger, braggart, boaster, blowhard, line-shooter, vaunter -n09872557 brahman, brahmin -n09873348 brawler -n09873473 breadwinner -n09873769 breaststroker -n09873899 breeder, stock breeder -n09874428 brick -n09874725 bride -n09874862 bridesmaid, maid of honor -n09875025 bridge agent -n09875979 broadcast journalist -n09876701 Brother -n09877288 brother-in-law -n09877587 browser -n09877750 Brummie, Brummy -n09877951 buddy, brother, chum, crony, pal, sidekick -n09878921 bull -n09879552 bully -n09880189 bunny, bunny girl -n09880741 burglar -n09881265 bursar -n09881358 busboy, waiter's assistant -n09881895 business editor -n09883047 business traveler -n09883452 buster -n09883807 busybody, nosy-parker, nosey-parker, quidnunc -n09885059 buttinsky -n09885866 cabinetmaker, furniture maker -n09886403 caddie, golf caddie -n09886540 cadet, plebe -n09888635 caller, caller-out -n09889065 call girl -n09889170 calligrapher, calligraphist -n09889691 campaigner, candidate, nominee -n09889941 camper -n09890192 camp follower -n09890749 candidate, prospect -n09891730 canonist -n09892262 capitalist -n09892513 captain, headwaiter, maitre d'hotel, maitre d' -n09892693 captain, senior pilot -n09893191 captain -n09893344 captain, chieftain -n09893502 captive -n09893600 captive -n09894143 cardinal -n09894445 cardiologist, heart specialist, heart surgeon -n09894654 card player -n09894909 cardsharp, card sharp, cardsharper, card sharper, sharper, sharpie, sharpy, card shark -n09895222 careerist -n09895480 career man -n09895561 caregiver -n09895701 caretaker -n09895902 caretaker -n09896170 caricaturist -n09896311 carillonneur -n09896401 caroler, caroller -n09896685 carpenter -n09896826 carper, niggler -n09898020 Cartesian -n09899289 cashier -n09899671 casualty, injured party -n09899782 casualty -n09899929 casuist, sophist -n09901337 catechist -n09901502 catechumen, neophyte -n09901642 caterer -n09901786 Catholicos -n09901921 cat fancier -n09902128 Cavalier, Royalist -n09902353 cavalryman, trooper -n09902731 caveman, cave man, cave dweller, troglodyte -n09902851 celebrant -n09902954 celebrant, celebrator, celebrater -n09903153 celebrity, famous person -n09903501 cellist, violoncellist -n09903639 censor -n09903936 censor -n09904208 centenarian -n09904837 centrist, middle of the roader, moderate, moderationist -n09905050 centurion -n09905185 certified public accountant, CPA -n09905530 chachka, tsatske, tshatshke, tchotchke, tchotchkeleh -n09906293 chambermaid, fille de chambre -n09906449 chameleon -n09906704 champion, champ, title-holder -n09907804 chandler -n09908769 prison chaplain -n09909660 charcoal burner -n09909929 charge d'affaires -n09910222 charioteer -n09910374 charmer, beguiler -n09910556 chartered accountant -n09910840 chartist, technical analyst -n09911226 charwoman, char, cleaning woman, cleaning lady, woman -n09912431 male chauvinist, sexist -n09912681 cheapskate, tightwad -n09912907 Chechen -n09912995 checker -n09913329 cheerer -n09913455 cheerleader -n09913593 cheerleader -n09915434 Cheops, Khufu -n09915651 chess master -n09916348 chief executive officer, CEO, chief operating officer -n09917214 chief of staff -n09917345 chief petty officer -n09917481 Chief Secretary -n09917593 child, kid, youngster, minor, shaver, nipper, small fry, tiddler, tike, tyke, fry, nestling -n09918248 child, kid -n09918554 child, baby -n09918867 child prodigy, infant prodigy, wonder child -n09919061 chimneysweeper, chimneysweep, sweep -n09919200 chiropractor -n09919451 chit -n09919899 choker -n09920106 choragus -n09920283 choreographer -n09920901 chorus girl, showgirl, chorine -n09921034 chosen -n09923003 cicerone -n09923186 cigar smoker -n09923418 cipher, cypher, nobody, nonentity -n09923561 circus acrobat -n09923673 citizen -n09923996 city editor -n09924106 city father -n09924195 city man -n09924313 city slicker, city boy -n09924437 civic leader, civil leader -n09924996 civil rights leader, civil rights worker, civil rights activist -n09927089 cleaner -n09927451 clergyman, reverend, man of the cloth -n09928136 cleric, churchman, divine, ecclesiastic -n09928451 clerk -n09928845 clever Dick, clever clogs -n09929202 climatologist -n09929298 climber -n09929577 clinician -n09930257 closer, finisher -n09930628 closet queen -n09930876 clown, buffoon, goof, goofball, merry andrew -n09931165 clown, buffoon -n09931418 coach, private instructor, tutor -n09931640 coach, manager, handler -n09932098 pitching coach -n09932336 coachman -n09932508 coal miner, collier, pitman -n09932788 coastguardsman -n09933020 cobber -n09933098 cobbler, shoemaker -n09933842 codger, old codger -n09933972 co-beneficiary -n09934337 cog -n09934488 cognitive neuroscientist -n09934774 coiffeur -n09935107 coiner -n09935434 collaborator, cooperator, partner, pardner -n09936825 colleen -n09936892 college student, university student -n09937056 collegian, college man, college boy -n09937688 colonial -n09937802 colonialist -n09937903 colonizer, coloniser -n09938080 coloratura, coloratura soprano -n09938449 color guard -n09938991 colossus, behemoth, giant, heavyweight, titan -n09940725 comedian -n09940818 comedienne -n09941089 comer -n09941571 commander -n09941787 commander in chief, generalissimo -n09941964 commanding officer, commandant, commander -n09942697 commissar, political commissar -n09942970 commissioned officer -n09943239 commissioned military officer -n09943811 commissioner -n09944022 commissioner -n09944160 committee member -n09944430 committeewoman -n09945021 commodore -n09945223 communicant -n09945319 communist, commie -n09945603 Communist -n09945745 commuter -n09946814 compere -n09947127 complexifier -n09950457 compulsive -n09950728 computational linguist -n09951070 computer scientist -n09951274 computer user -n09951524 Comrade -n09951616 concert-goer, music lover -n09952163 conciliator, make-peace, pacifier, peacemaker, reconciler -n09953052 conductor -n09953350 confectioner, candymaker -n09953615 Confederate -n09954355 confessor -n09954639 confidant, intimate -n09955406 Confucian, Confucianist -n09955944 rep -n09956578 conqueror, vanquisher -n09957523 Conservative -n09958133 Nonconformist, chapelgoer -n09958292 Anglican -n09958447 consignee -n09958569 consigner, consignor -n09959142 constable -n09959658 constructivist -n09960688 contractor -n09961198 contralto -n09961331 contributor -n09961469 control freak -n09961605 convalescent -n09961739 convener -n09962966 convict, con, inmate, yard bird, yardbird -n09964202 copilot, co-pilot -n09964411 copycat, imitator, emulator, ape, aper -n09965515 coreligionist -n09965787 cornerback -n09966470 corporatist -n09966554 correspondent, letter writer -n09967063 cosmetician -n09967406 cosmopolitan, cosmopolite -n09967555 Cossack -n09967816 cost accountant -n09967967 co-star -n09968259 costumier, costumer, costume designer -n09968652 cotter, cottier -n09968741 cotter, cottar -n09968845 counselor, counsellor -n09970088 counterterrorist -n09970192 counterspy, mole -n09970402 countess -n09970822 compromiser -n09971273 countrywoman -n09971385 county agent, agricultural agent, extension agent -n09971839 courtier -n09972010 cousin, first cousin, cousin-german, full cousin -n09972458 cover girl, pin-up, lovely -n09972587 cow -n09974648 craftsman, artisan, journeyman, artificer -n09975425 craftsman, crafter -n09976024 crapshooter -n09976283 crazy, loony, looney, nutcase, weirdo -n09976429 creature, wight -n09976728 creditor -n09976917 creep, weirdo, weirdie, weirdy, spook -n09978442 criminologist -n09979321 critic -n09979913 Croesus -n09980458 cross-examiner, cross-questioner -n09980805 crossover voter, crossover -n09980985 croupier -n09981092 crown prince -n09981278 crown princess -n09981540 cryptanalyst, cryptographer, cryptologist -n09981939 Cub Scout -n09982152 cuckold -n09982525 cultist -n09983314 curandera -n09983572 curate, minister of religion, minister, parson, pastor, rector -n09983889 curator, conservator -n09984960 customer agent -n09985470 cutter, carver -n09985809 cyberpunk -n09985978 cyborg, bionic man, bionic woman -n09986450 cymbalist -n09986700 Cynic -n09986904 cytogeneticist -n09987045 cytologist -n09987161 czar -n09987239 czar, tsar, tzar -n09988063 dad, dada, daddy, pa, papa, pappa, pop -n09988311 dairyman -n09988493 Dalai Lama, Grand Lama -n09988703 dallier, dillydallier, dilly-dallier, mope, lounger -n09989502 dancer, professional dancer, terpsichorean -n09990415 dancer, social dancer -n09990690 clog dancer -n09990777 dancing-master, dance master -n09991740 dark horse -n09991867 darling, favorite, favourite, pet, dearie, deary, ducky -n09992538 date, escort -n09992837 daughter, girl -n09993252 dawdler, drone, laggard, lagger, trailer, poke -n09993651 day boarder -n09994400 day laborer, day labourer -n09994673 deacon, Protestant deacon -n09994808 deaconess -n09994878 deadeye -n09995829 deipnosophist -n09996039 dropout -n09996304 deadhead -n09996481 deaf person -n09997622 debtor, debitor -n09998788 deckhand, roustabout -n09999135 defamer, maligner, slanderer, vilifier, libeler, backbiter, traducer -n10000294 defense contractor -n10000459 deist, freethinker -n10000787 delegate -n10001217 deliveryman, delivery boy, deliverer -n10001481 demagogue, demagog, rabble-rouser -n10001764 demigod, superman, Ubermensch -n10002257 demographer, demographist, population scientist -n10002760 demonstrator, protester -n10003476 den mother -n10004718 department head -n10005006 depositor -n10005934 deputy -n10006177 dermatologist, skin doctor -n10006748 descender -n10007684 designated hitter -n10007809 designer, intriguer -n10007995 desk clerk, hotel desk clerk, hotel clerk -n10008123 desk officer -n10008254 desk sergeant, deskman, station keeper -n10009162 detainee, political detainee -n10009276 detective, investigator, tec, police detective -n10009484 detective -n10009671 detractor, disparager, depreciator, knocker -n10010062 developer -n10010243 deviationist -n10010632 devisee -n10010767 devisor -n10010864 devourer -n10011360 dialectician -n10011486 diarist, diary keeper, journalist -n10012484 dietician, dietitian, nutritionist -n10013811 diocesan -n10015215 director, theater director, theatre director -n10015485 director -n10015792 dirty old man -n10015897 disbeliever, nonbeliever, unbeliever -n10017272 disk jockey, disc jockey, dj -n10017422 dispatcher -n10018747 distortionist -n10018861 distributor, distributer -n10019072 district attorney, DA -n10019187 district manager -n10019406 diver, plunger -n10020366 divorcee, grass widow -n10020533 ex-wife, ex -n10020670 divorce lawyer -n10020807 docent -n10020890 doctor, doc, physician, MD, Dr., medico -n10022908 dodo, fogy, fogey, fossil -n10023264 doge -n10023506 dog in the manger -n10023656 dogmatist, doctrinaire -n10024025 dolichocephalic -n10024362 domestic partner, significant other, spousal equivalent, spouse equivalent -n10024937 Dominican -n10025060 dominus, dominie, domine, dominee -n10025295 don, father -n10025391 Donatist -n10025635 donna -n10026976 dosser, street person -n10027246 double, image, look-alike -n10027590 double-crosser, double-dealer, two-timer, betrayer, traitor -n10028402 down-and-out -n10028541 doyenne -n10029068 draftsman, drawer -n10030277 dramatist, playwright -n10032987 dreamer -n10033412 dressmaker, modiste, needlewoman, seamstress, sempstress -n10033572 dressmaker's model -n10033663 dribbler, driveller, slobberer, drooler -n10033888 dribbler -n10034201 drinker, imbiber, toper, juicer -n10034614 drinker -n10035952 drug addict, junkie, junky -n10036266 drug user, substance abuser, user -n10036444 Druid -n10036692 drum majorette, majorette -n10036929 drummer -n10037080 drunk -n10037385 drunkard, drunk, rummy, sot, inebriate, wino -n10037588 Druze, Druse -n10037922 dry, prohibitionist -n10038119 dry nurse -n10038409 duchess -n10038620 duke -n10039271 duffer -n10039946 dunker -n10040240 Dutch uncle -n10040698 dyspeptic -n10040945 eager beaver, busy bee, live wire, sharpie, sharpy -n10041373 earl -n10041887 earner, wage earner -n10042690 eavesdropper -n10042845 eccentric, eccentric person, flake, oddball, geek -n10043024 eclectic, eclecticist -n10043491 econometrician, econometrist -n10043643 economist, economic expert -n10044682 ectomorph -n10044879 editor, editor in chief -n10047199 egocentric, egoist -n10047459 egotist, egoist, swellhead -n10048117 ejaculator -n10048367 elder -n10048612 elder statesman -n10048836 elected official -n10049363 electrician, lineman, linesman -n10050043 elegist -n10050880 elocutionist -n10051026 emancipator, manumitter -n10051761 embryologist -n10051861 emeritus -n10051975 emigrant, emigre, emigree, outgoer -n10052694 emissary, envoy -n10053439 empress -n10053808 employee -n10054657 employer -n10055297 enchantress, witch -n10055410 enchantress, temptress, siren, Delilah, femme fatale -n10055566 encyclopedist, encyclopaedist -n10055730 endomorph -n10055847 enemy, foe, foeman, opposition -n10056103 energizer, energiser, vitalizer, vitaliser, animator -n10056611 end man -n10056719 end man, corner man -n10057271 endorser, indorser -n10058411 enjoyer -n10058962 enlisted woman -n10059067 enophile, oenophile -n10060075 entrant -n10060175 entrant -n10060352 entrepreneur, enterpriser -n10061043 envoy, envoy extraordinary, minister plenipotentiary -n10061195 enzymologist -n10061431 eparch -n10061882 epidemiologist -n10062042 epigone, epigon -n10062176 epileptic -n10062275 Episcopalian -n10062492 equerry -n10062594 equerry -n10062716 erotic -n10062905 escapee -n10062996 escapist, dreamer, wishful thinker -n10063635 Eskimo, Esquimau, Inuit -n10063919 espionage agent -n10064831 esthetician, aesthetician -n10064977 etcher -n10065758 ethnologist -n10066206 Etonian -n10066314 etymologist -n10067011 evangelist, revivalist, gospeler, gospeller -n10067305 Evangelist -n10067600 event planner -n10067968 examiner, inspector -n10068234 examiner, tester, quizzer -n10068425 exarch -n10069296 executant -n10069981 executive secretary -n10070108 executive vice president -n10070377 executrix -n10070449 exegete -n10070563 exhibitor, exhibitioner, shower -n10070711 exhibitionist, show-off -n10071332 exile, expatriate, expat -n10071557 existentialist, existentialist philosopher, existential philosopher -n10072054 exorcist, exorciser -n10074249 ex-spouse -n10074578 extern, medical extern -n10074735 extremist -n10074841 extrovert, extravert -n10075299 eyewitness -n10075693 facilitator -n10076224 fairy godmother -n10076483 falangist, phalangist -n10076604 falconer, hawker -n10076957 falsifier -n10077106 familiar -n10077593 fan, buff, devotee, lover -n10077879 fanatic, fiend -n10078131 fancier, enthusiast -n10078719 farm boy -n10078806 farmer, husbandman, granger, sodbuster -n10079399 farmhand, fieldhand, field hand, farm worker -n10079893 fascist -n10080117 fascista -n10080508 fatalist, determinist, predestinarian, predestinationist -n10080869 father, male parent, begetter -n10081204 Father, Padre -n10081842 father-figure -n10082043 father-in-law -n10082299 Fauntleroy, Little Lord Fauntleroy -n10082423 Fauve, fauvist -n10082562 favorite son -n10082687 featherweight -n10082997 federalist -n10083677 fellow traveler, fellow traveller -n10083823 female aristocrat -n10084043 female offspring -n10084295 female child, girl, little girl -n10085101 fence -n10085869 fiance, groom-to-be -n10086383 fielder, fieldsman -n10086744 field judge -n10087434 fighter pilot -n10087736 filer -n10088200 film director, director -n10090745 finder -n10091349 fire chief, fire marshal -n10091450 fire-eater, fire-swallower -n10091564 fire-eater, hothead -n10091651 fireman, firefighter, fire fighter, fire-eater -n10091861 fire marshall -n10091997 fire walker -n10092488 first baseman, first sacker -n10092643 firstborn, eldest -n10092794 first lady -n10092978 first lieutenant, 1st lieutenant -n10093167 first offender -n10093475 first sergeant, sergeant first class -n10093818 fishmonger, fishwife -n10094320 flagellant -n10094584 flag officer -n10094782 flak catcher, flak, flack catcher, flack -n10095265 flanker back, flanker -n10095420 flapper -n10095769 flatmate -n10095869 flatterer, adulator -n10096126 flibbertigibbet, foolish woman -n10096508 flight surgeon -n10097262 floorwalker, shopwalker -n10097477 flop, dud, washout -n10097590 Florentine -n10097842 flower girl -n10097995 flower girl -n10098245 flutist, flautist, flute player -n10098388 fly-by-night -n10098517 flyweight -n10098624 flyweight -n10098710 foe, enemy -n10098862 folk dancer -n10099002 folk poet -n10099375 follower -n10101308 football hero -n10101634 football player, footballer -n10101981 footman -n10102800 forefather, father, sire -n10103155 foremother -n10103228 foreign agent -n10103921 foreigner, outsider -n10104064 boss -n10104487 foreman -n10104756 forester, tree farmer, arboriculturist -n10104888 forewoman -n10105085 forger, counterfeiter -n10105733 forward -n10105906 foster-brother, foster brother -n10106387 foster-father, foster father -n10106509 foster-mother, foster mother -n10106995 foster-sister, foster sister -n10107173 foster-son, foster son -n10107303 founder, beginner, founding father, father -n10108018 foundress -n10108089 four-minute man -n10108464 framer -n10108832 Francophobe -n10109443 freak, monster, monstrosity, lusus naturae -n10109662 free agent, free spirit, freewheeler -n10109826 free agent -n10110093 freedom rider -n10110731 free-liver -n10110893 freeloader -n10111358 free trader -n10111779 Freudian -n10111903 friar, mendicant -n10112129 monk, monastic -n10113249 frontierswoman -n10113583 front man, front, figurehead, nominal head, straw man, strawman -n10113869 frotteur -n10114476 fucker -n10114550 fucker -n10114662 fuddy-duddy -n10115430 fullback -n10115946 funambulist, tightrope walker -n10116370 fundamentalist -n10116478 fundraiser -n10116702 futurist -n10117017 gadgeteer -n10117267 gagman, gagster, gagwriter -n10117415 gagman, standup comedian -n10117739 gainer, weight gainer -n10117851 gal -n10118301 galoot -n10118743 gambist -n10118844 gambler -n10119609 gamine -n10120330 garbage man, garbageman, garbage collector, garbage carter, garbage hauler, refuse collector, dustman -n10120671 gardener -n10121026 garment cutter -n10121246 garroter, garrotter, strangler, throttler, choker -n10121714 gasman -n10121800 gastroenterologist -n10122300 gatherer -n10122531 gawker -n10123122 gendarme -n10123844 general, full general -n10126177 generator, source, author -n10126424 geneticist -n10126708 genitor -n10127186 gent -n10127689 geologist -n10128519 geophysicist -n10128748 ghostwriter, ghost -n10129338 Gibson girl -n10129825 girl, miss, missy, young lady, young woman, fille -n10130686 girlfriend, girl, lady friend -n10130877 girlfriend -n10131151 girl wonder -n10131268 Girondist, Girondin -n10131590 gitano -n10131815 gladiator -n10132035 glassblower -n10132502 gleaner -n10134178 goat herder, goatherd -n10134396 godchild -n10134760 godfather -n10134982 godparent -n10135129 godson -n10135197 gofer -n10135297 goffer, gopher -n10136615 goldsmith, goldworker, gold-worker -n10136959 golfer, golf player, linksman -n10137825 gondolier, gondoliere -n10138369 good guy -n10138472 good old boy, good ole boy, good ol' boy -n10139077 good Samaritan -n10139651 gossip columnist -n10140051 gouger -n10140597 governor general -n10140683 grabber -n10140783 grader -n10140929 graduate nurse, trained nurse -n10141364 grammarian, syntactician -n10141732 granddaughter -n10142166 grande dame -n10142391 grandfather, gramps, granddad, grandad, granddaddy, grandpa -n10142537 Grand Inquisitor -n10142747 grandma, grandmother, granny, grannie, gran, nan, nanna -n10142946 grandmaster -n10143172 grandparent -n10143595 grantee -n10143725 granter -n10144338 grass widower, divorced man -n10145239 great-aunt, grandaunt -n10145340 great grandchild -n10145480 great granddaughter -n10145590 great grandmother -n10145774 great grandparent -n10145902 great grandson -n10146002 great-nephew, grandnephew -n10146104 great-niece, grandniece -n10146416 Green Beret -n10146816 grenadier, grenade thrower -n10146927 greeter, saluter, welcomer -n10147121 gringo -n10147262 grinner -n10147710 grocer -n10147935 groom, bridegroom -n10148035 groom, bridegroom -n10148305 grouch, grump, crank, churl, crosspatch -n10148825 group captain -n10149436 grunter -n10149867 prison guard, jailer, jailor, gaoler, screw, turnkey -n10150071 guard -n10150794 guesser -n10150940 guest, invitee -n10151133 guest -n10151261 guest of honor -n10151367 guest worker, guestworker -n10151570 guide -n10151760 guitarist, guitar player -n10152306 gunnery sergeant -n10152616 guru -n10152763 guru -n10153155 guvnor -n10153414 guy, cat, hombre, bozo -n10153594 gymnast -n10153865 gym rat -n10154013 gynecologist, gynaecologist, woman's doctor -n10154186 Gypsy, Gipsy, Romany, Rommany, Romani, Roma, Bohemian -n10154601 hack, drudge, hacker -n10155222 hacker, cyber-terrorist, cyberpunk -n10155600 haggler -n10155849 hairdresser, hairstylist, stylist, styler -n10156629 hakim, hakeem -n10156831 Hakka -n10157016 halberdier -n10157128 halfback -n10157271 half blood -n10158506 hand -n10159045 animal trainer, handler -n10159289 handyman, jack of all trades, odd-job man -n10159533 hang glider -n10160188 hardliner -n10160280 harlequin -n10160412 harmonizer, harmoniser -n10161622 hash head -n10162016 hatchet man, iceman -n10162194 hater -n10162354 hatmaker, hatter, milliner, modiste -n10164025 headman, tribal chief, chieftain, chief -n10164233 headmaster, schoolmaster, master -n10164492 head nurse -n10165448 hearer, listener, auditor, attender -n10166189 heartbreaker -n10166394 heathen, pagan, gentile, infidel -n10167152 heavyweight -n10167361 heavy -n10167565 heckler, badgerer -n10167838 hedger -n10168012 hedger, equivocator, tergiversator -n10168183 hedonist, pagan, pleasure seeker -n10168584 heir, inheritor, heritor -n10168837 heir apparent -n10169147 heiress, inheritress, inheritrix -n10169241 heir presumptive -n10169419 hellion, heller, devil -n10169796 helmsman, steersman, steerer -n10170060 hire -n10170681 hematologist, haematologist -n10170866 hemiplegic -n10171219 herald, trumpeter -n10171456 herbalist, herb doctor -n10171567 herder, herdsman, drover -n10172080 hermaphrodite, intersex, gynandromorph, androgyne, epicene, epicene person -n10173410 heroine -n10173579 heroin addict -n10173665 hero worshiper, hero worshipper -n10173771 Herr -n10174253 highbinder -n10174330 highbrow -n10174445 high commissioner -n10174589 highflier, highflyer -n10174695 Highlander, Scottish Highlander, Highland Scot -n10174971 high-muck-a-muck, pooh-bah -n10175248 high priest -n10175725 highjacker, hijacker -n10176913 hireling, pensionary -n10177150 historian, historiographer -n10178077 hitchhiker -n10178216 hitter, striker -n10179069 hobbyist -n10180580 holdout -n10180791 holdover, hangover -n10180923 holdup man, stickup man -n10181445 homeboy -n10181547 homeboy -n10181799 home buyer -n10181878 homegirl -n10182190 homeless, homeless person -n10182402 homeopath, homoeopath -n10183347 honest woman -n10183931 honor guard, guard of honor -n10184505 hooker -n10185148 hoper -n10185483 hornist -n10185793 horseman, equestrian, horseback rider -n10186068 horse trader -n10186143 horsewoman -n10186216 horse wrangler, wrangler -n10186350 horticulturist, plantsman -n10186686 hospital chaplain -n10186774 host, innkeeper, boniface -n10187130 host -n10187491 hostess -n10187990 hotelier, hotelkeeper, hotel manager, hotelman, hosteller -n10188715 housekeeper -n10188856 housemaster -n10188957 housemate -n10189278 house physician, resident, resident physician -n10189597 house sitter -n10190122 housing commissioner -n10190516 huckster, cheap-jack -n10191001 hugger -n10191388 humanist, humanitarian -n10191613 humanitarian, do-gooder, improver -n10192839 hunk -n10193650 huntress -n10194231 ex-husband, ex -n10194775 hydrologist -n10195056 hyperope -n10195155 hypertensive -n10195261 hypnotist, hypnotizer, hypnotiser, mesmerist, mesmerizer -n10195593 hypocrite, dissembler, dissimulator, phony, phoney, pretender -n10196404 iceman -n10196725 iconoclast -n10197392 ideologist, ideologue -n10198437 idol, matinee idol -n10198832 idolizer, idoliser -n10199251 imam, imaum -n10200246 imperialist -n10200781 important person, influential person, personage -n10202225 inamorato -n10202624 incumbent, officeholder -n10202763 incurable -n10203949 inductee -n10204177 industrialist -n10204833 infanticide -n10205231 inferior -n10205344 infernal -n10205457 infielder -n10205714 infiltrator -n10206173 informer, betrayer, rat, squealer, blabber -n10206506 ingenue -n10206629 ingenue -n10207077 polymath -n10207169 in-law, relative-in-law -n10208189 inquiry agent -n10208847 inspector -n10208950 inspector general -n10209082 instigator, initiator -n10209731 insurance broker, insurance agent, general agent, underwriter -n10210137 insurgent, insurrectionist, freedom fighter, rebel -n10210512 intelligence analyst -n10210648 interior designer, designer, interior decorator, house decorator, room decorator, decorator -n10210911 interlocutor, conversational partner -n10211036 interlocutor, middleman -n10211666 International Grandmaster -n10211830 internationalist -n10212231 internist -n10212501 interpreter, translator -n10212780 interpreter -n10213034 intervenor -n10213429 introvert -n10214062 invader, encroacher -n10214390 invalidator, voider, nullifier -n10215623 investigator -n10216106 investor -n10216403 invigilator -n10217208 irreligionist -n10218043 Ivy Leaguer -n10218164 Jack of all trades -n10218292 Jacksonian -n10219240 Jane Doe -n10219453 janissary -n10219879 Jat -n10220080 Javanese, Javan -n10220924 Jekyll and Hyde -n10221312 jester, fool, motley fool -n10221520 Jesuit -n10222170 jezebel -n10222259 jilt -n10222497 jobber, middleman, wholesaler -n10222716 job candidate -n10223069 Job's comforter -n10223177 jockey -n10223606 John Doe -n10224578 journalist -n10225219 judge, justice, jurist -n10225931 judge advocate -n10226413 juggler -n10227166 Jungian -n10227266 junior -n10227393 junior -n10227490 Junior, Jr, Jnr -n10227698 junior lightweight -n10227793 junior middleweight -n10227985 jurist, legal expert -n10228278 juror, juryman, jurywoman -n10228468 justice of the peace -n10228592 justiciar, justiciary -n10228712 kachina -n10229883 keyboardist -n10230216 Khedive -n10233248 kingmaker -n10235024 king, queen, world-beater -n10235269 King's Counsel -n10235385 Counsel to the Crown -n10236304 kin, kinsperson, family -n10236521 enate, matrikin, matrilineal kin, matrisib, matrilineal sib -n10236842 kink -n10237069 kinswoman -n10237196 kisser, osculator -n10237464 kitchen help -n10237556 kitchen police, KP -n10237676 Klansman, Ku Kluxer, Kluxer -n10237799 kleptomaniac -n10238272 kneeler -n10238375 knight -n10239928 knocker -n10240082 knower, apprehender -n10240235 know-it-all, know-all -n10240417 kolkhoznik -n10240821 Kshatriya -n10241024 labor coach, birthing coach, doula, monitrice -n10241300 laborer, manual laborer, labourer, jack -n10242328 Labourite -n10243137 lady -n10243273 lady-in-waiting -n10243483 lady's maid -n10243664 lama -n10243872 lamb, dear -n10244108 lame duck -n10244359 lamplighter -n10244913 land agent -n10245029 landgrave -n10245341 landlubber, lubber, landsman -n10245507 landlubber, landsman, landman -n10245639 landowner, landholder, property owner -n10245863 landscape architect, landscape gardener, landscaper, landscapist -n10246317 langlaufer -n10246395 languisher -n10246703 lapidary, lapidarist -n10247358 lass, lassie, young girl, jeune fille -n10247880 Latin -n10248008 Latin -n10248198 latitudinarian -n10248377 Jehovah's Witness -n10249191 law agent -n10249270 lawgiver, lawmaker -n10249459 lawman, law officer, peace officer -n10249869 law student -n10249950 lawyer, attorney -n10250712 lay reader -n10251329 lazybones -n10251612 leaker -n10252075 leaseholder, lessee -n10252222 lector, lecturer, reader -n10252354 lector, reader -n10252547 lecturer -n10253122 left-hander, lefty, southpaw -n10253296 legal representative -n10253479 legate, official emissary -n10253611 legatee -n10253703 legionnaire, legionary -n10255459 letterman -n10257221 liberator -n10258602 licenser -n10258786 licentiate -n10259348 lieutenant -n10259780 lieutenant colonel, light colonel -n10259997 lieutenant commander -n10260473 lieutenant junior grade, lieutenant JG -n10260706 life -n10260800 lifeguard, lifesaver -n10261211 life tenant -n10261511 light flyweight -n10261624 light heavyweight, cruiserweight -n10261862 light heavyweight -n10262343 light-o'-love, light-of-love -n10262445 lightweight -n10262561 lightweight -n10262655 lightweight -n10262880 lilliputian -n10263146 limnologist -n10263411 lineman -n10263790 line officer -n10265281 lion-hunter -n10265801 lisper -n10265891 lister -n10266016 literary critic -n10266328 literate, literate person -n10266848 litigant, litigator -n10267166 litterer, litterbug, litter lout -n10267311 little brother -n10267865 little sister -n10268629 lobbyist -n10269199 locksmith -n10269289 locum tenens, locum -n10271677 Lord, noble, nobleman -n10272782 loser -n10272913 loser, also-ran -n10273064 failure, loser, nonstarter, unsuccessful person -n10274173 Lothario -n10274318 loudmouth, blusterer -n10274815 lowerclassman, underclassman -n10275249 Lowlander, Scottish Lowlander, Lowland Scot -n10275395 loyalist, stalwart -n10275848 Luddite -n10276045 lumberman, lumberjack, logger, feller, faller -n10276477 lumper -n10276942 bedlamite -n10277027 pyromaniac -n10277638 lutist, lutanist, lutenist -n10277815 Lutheran -n10277912 lyricist, lyrist -n10278456 macebearer, mace, macer -n10279018 machinist, mechanic, shop mechanic -n10279778 madame -n10280034 maenad -n10280130 maestro, master -n10280598 magdalen -n10280674 magician, prestidigitator, conjurer, conjuror, illusionist -n10281546 magus -n10281770 maharani, maharanee -n10281896 mahatma -n10282482 maid, maiden -n10282672 maid, maidservant, housemaid, amah -n10283170 major -n10283366 major -n10283546 major-domo, seneschal -n10284064 maker, shaper -n10284871 malahini -n10284965 malcontent -n10286282 malik -n10286539 malingerer, skulker, shammer -n10286749 Malthusian -n10288964 adonis -n10289039 man -n10289176 man -n10289462 manageress -n10289766 mandarin -n10290422 maneuverer, manoeuvrer -n10290541 maniac -n10290813 Manichaean, Manichean, Manichee -n10290919 manicurist -n10291110 manipulator -n10291469 man-at-arms -n10291822 man of action, man of deeds -n10291942 man of letters -n10292316 manufacturer, producer -n10293332 marcher, parader -n10293590 marchioness, marquise -n10293861 margrave -n10294020 margrave -n10294139 Marine, devil dog, leatherneck, shipboard soldier -n10295371 marquess -n10295479 marquis, marquess -n10296176 marshal, marshall -n10296444 martinet, disciplinarian, moralist -n10297234 mascot -n10297367 masochist -n10297531 mason, stonemason -n10297841 masquerader, masker, masquer -n10298202 masseur -n10298271 masseuse -n10298647 master -n10298912 master, captain, sea captain, skipper -n10299125 master-at-arms -n10299250 master of ceremonies, emcee, host -n10299700 masturbator, onanist -n10299875 matchmaker, matcher, marriage broker -n10300041 mate, first mate -n10300154 mate -n10300303 mate -n10300500 mater -n10300654 material -n10300829 materialist -n10302576 matriarch, materfamilias -n10302700 matriarch -n10302905 matriculate -n10303037 matron -n10303814 mayor, city manager -n10304086 mayoress -n10304650 mechanical engineer -n10304914 medalist, medallist, medal winner -n10305635 medical officer, medic -n10305802 medical practitioner, medical man -n10306004 medical scientist -n10306279 medium, spiritualist, sensitive -n10306496 megalomaniac -n10306595 melancholic, melancholiac -n10306890 Melkite, Melchite -n10307114 melter -n10308066 nonmember -n10308168 board member -n10308275 clansman, clanswoman, clan member -n10308504 memorizer, memoriser -n10308653 Mendelian -n10308732 mender, repairer, fixer -n10310783 Mesoamerican -n10311506 messmate -n10311661 mestiza -n10312287 meteorologist -n10312491 meter maid -n10312600 Methodist -n10313000 Metis -n10313239 metropolitan -n10313441 mezzo-soprano, mezzo -n10313724 microeconomist, microeconomic expert -n10314054 middle-aged man -n10314182 middlebrow -n10314517 middleweight -n10314836 midwife, accoucheuse -n10315217 mikado, tenno -n10315456 Milanese -n10315561 miler -n10315730 miles gloriosus -n10316360 military attache -n10316527 military chaplain, padre, Holy Joe, sky pilot -n10316862 military leader -n10317007 military officer, officer -n10317500 military policeman, MP -n10317963 mill agent -n10318293 mill-hand, factory worker -n10318607 millionairess -n10318686 millwright -n10319313 minder -n10320484 mining engineer -n10320863 minister, government minister -n10321126 ministrant -n10321340 minor leaguer, bush leaguer -n10321632 Minuteman -n10321882 misanthrope, misanthropist -n10322238 misfit -n10323634 mistress -n10323752 mistress, kept woman, fancy woman -n10323999 mixed-blood -n10324560 model, poser -n10325549 class act -n10325774 modeler, modeller -n10326776 modifier -n10327143 molecular biologist -n10327987 Monegasque, Monacan -n10328123 monetarist -n10328328 moneygrubber -n10328437 moneymaker -n10328696 Mongoloid -n10328941 monolingual -n10329035 monologist -n10330593 moonlighter -n10330931 moralist -n10331098 morosoph -n10331167 morris dancer -n10331258 mortal enemy -n10331347 mortgagee, mortgage holder -n10331841 mortician, undertaker, funeral undertaker, funeral director -n10332110 moss-trooper -n10332385 mother, female parent -n10332861 mother -n10332953 mother -n10333044 mother figure -n10333165 mother hen -n10333317 mother-in-law -n10333439 mother's boy, mamma's boy, mama's boy -n10333601 mother's daughter -n10333838 motorcycle cop, motorcycle policeman, speed cop -n10334009 motorcyclist -n10334461 Mound Builder -n10334782 mountebank, charlatan -n10335246 mourner, griever, sorrower, lamenter -n10335801 mouthpiece, mouth -n10335931 mover -n10336411 moviegoer, motion-picture fan -n10336904 muffin man -n10337488 mugwump, independent, fencesitter -n10338231 Mullah, Mollah, Mulla -n10338391 muncher -n10339179 murderess -n10339251 murder suspect -n10339717 musher -n10340312 musician, instrumentalist, player -n10341243 musicologist -n10341343 music teacher -n10341446 musketeer -n10341573 Muslimah -n10341955 mutilator, maimer, mangler -n10342180 mutineer -n10342367 mute, deaf-mute, deaf-and-dumb person -n10342543 mutterer, mumbler, murmurer -n10342893 muzzler -n10342992 Mycenaen -n10343088 mycologist -n10343355 myope -n10343449 myrmidon -n10343554 mystic, religious mystic -n10343869 mythologist -n10344121 naif -n10344203 nailer -n10344319 namby-pamby -n10344656 name dropper -n10344774 namer -n10345015 nan -n10345100 nanny, nursemaid, nurse -n10345302 narc, nark, narcotics agent -n10345422 narcissist, narcist -n10345659 nark, copper's nark -n10346015 nationalist -n10347204 nautch girl -n10347446 naval commander -n10348526 Navy SEAL, SEAL -n10349243 obstructionist, obstructor, obstructer, resister, thwarter -n10349750 Nazarene -n10349836 Nazarene, Ebionite -n10350220 Nazi, German Nazi -n10350774 nebbish, nebbech -n10351064 necker -n10353016 neonate, newborn, newborn infant, newborn baby -n10353355 nephew -n10353928 neurobiologist -n10354265 neurologist, brain doctor -n10354754 neurosurgeon, brain surgeon -n10355142 neutral -n10355306 neutralist -n10355449 newcomer, fledgling, fledgeling, starter, neophyte, freshman, newbie, entrant -n10355688 newcomer -n10355806 New Dealer -n10356450 newspaper editor -n10356877 newsreader, news reader -n10357012 Newtonian -n10357613 niece -n10357737 niggard, skinflint, scrooge, churl -n10358032 night porter -n10358124 night rider, nightrider -n10358575 NIMBY -n10359117 niqaabi -n10359422 nitpicker -n10359546 Nobelist, Nobel Laureate -n10359659 NOC -n10360366 noncandidate -n10360747 noncommissioned officer, noncom, enlisted officer -n10361060 nondescript -n10361194 nondriver -n10361296 nonparticipant -n10361525 nonperson, unperson -n10362003 nonresident -n10362319 nonsmoker -n10362557 Northern Baptist -n10363445 noticer -n10363573 novelist -n10364198 novitiate, novice -n10364502 nuclear chemist, radiochemist -n10365514 nudger -n10366145 nullipara -n10366276 number theorist -n10366966 nurse -n10368291 nursling, nurseling, suckling -n10368528 nymph, houri -n10368624 nymphet -n10368711 nympholept -n10368798 nymphomaniac, nympho -n10369095 oarswoman -n10369317 oboist -n10369417 obscurantist -n10369528 observer, commentator -n10369699 obstetrician, accoucheur -n10369955 occupier -n10370381 occultist -n10370955 wine lover -n10371052 offerer, offeror -n10371221 office-bearer -n10371330 office boy -n10371450 officeholder, officer -n10373390 officiant -n10373525 Federal, Fed, federal official -n10374541 oilman -n10374849 oil tycoon -n10374943 old-age pensioner -n10375052 old boy -n10375314 old lady -n10375402 old man -n10376523 oldster, old person, senior citizen, golden ager -n10376890 old-timer, oldtimer, gaffer, old geezer, antique -n10377021 old woman -n10377185 oligarch -n10377291 Olympian -n10377542 omnivore -n10377633 oncologist -n10378026 onlooker, looker-on -n10378113 onomancer -n10378780 operator -n10379376 opportunist, self-seeker -n10380126 optimist -n10380499 Orangeman -n10380672 orator, speechmaker, rhetorician, public speaker, speechifier -n10381804 orderly, hospital attendant -n10381981 orderly -n10382157 orderly sergeant -n10382302 ordinand -n10382480 ordinary -n10382710 organ-grinder -n10382825 organist -n10383094 organization man -n10383237 organizer, organiser, arranger -n10383505 organizer, organiser, labor organizer -n10383816 originator, conceiver, mastermind -n10384214 ornithologist, bird watcher -n10384392 orphan -n10384496 orphan -n10385566 osteopath, osteopathist -n10386196 out-and-outer -n10386754 outdoorswoman -n10386874 outfielder -n10386984 outfielder -n10387196 right fielder -n10387324 right-handed pitcher, right-hander -n10387836 outlier -n10389865 owner-occupier -n10389976 oyabun -n10390600 packrat -n10390698 padrone -n10390807 padrone -n10391416 page, pageboy -n10393909 painter -n10394434 Paleo-American, Paleo-Amerind, Paleo-Indian -n10394786 paleontologist, palaeontologist, fossilist -n10395073 pallbearer, bearer -n10395209 palmist, palmister, chiromancer -n10395390 pamperer, spoiler, coddler, mollycoddler -n10395828 Panchen Lama -n10396106 panelist, panellist -n10396337 panhandler -n10396727 paparazzo -n10396908 paperboy -n10397001 paperhanger, paperer -n10397142 paperhanger -n10397392 papoose, pappoose -n10399130 pardoner -n10400003 paretic -n10400108 parishioner -n10400205 park commissioner -n10400437 Parliamentarian, Member of Parliament -n10400618 parliamentary agent -n10400998 parodist, lampooner -n10401204 parricide -n10401331 parrot -n10401639 partaker, sharer -n10402709 part-timer -n10402824 party -n10403633 party man, party liner -n10403876 passenger, rider -n10404426 passer -n10404998 paster -n10405540 pater -n10405694 patient -n10406266 patriarch -n10406391 patriarch -n10406765 patriarch, paterfamilias -n10407310 patriot, nationalist -n10407954 patron, sponsor, supporter -n10408809 patternmaker -n10409459 pawnbroker -n10409752 payer, remunerator -n10410246 peacekeeper -n10410996 peasant -n10411356 pedant, bookworm, scholastic -n10411551 peddler, pedlar, packman, hawker, pitchman -n10411867 pederast, paederast, child molester -n10414239 penologist -n10414768 pentathlete -n10414865 Pentecostal, Pentecostalist -n10415037 percussionist -n10416567 periodontist -n10417288 peshmerga -n10417424 personality -n10417551 personal representative -n10417682 personage -n10417843 persona grata -n10417969 persona non grata -n10418101 personification -n10418735 perspirer, sweater -n10419047 pervert, deviant, deviate, degenerate -n10419472 pessimist -n10419630 pest, blighter, cuss, pesterer, gadfly -n10419785 Peter Pan -n10420031 petitioner, suppliant, supplicant, requester -n10420277 petit juror, petty juror -n10420507 pet sitter, critter sitter -n10420649 petter, fondler -n10421016 Pharaoh, Pharaoh of Egypt -n10421470 pharmacist, druggist, chemist, apothecary, pill pusher, pill roller -n10421956 philanthropist, altruist -n10422405 philatelist, stamp collector -n10425946 philosopher -n10426454 phonetician -n10426630 phonologist -n10427223 photojournalist -n10427359 photometrist, photometrician -n10427764 physical therapist, physiotherapist -n10428004 physicist -n10431122 piano maker -n10431625 picker, chooser, selector -n10432189 picnicker, picknicker -n10432441 pilgrim -n10432875 pill -n10432957 pillar, mainstay -n10433077 pill head -n10433452 pilot -n10433610 Piltdown man, Piltdown hoax -n10433737 pimp, procurer, panderer, pander, pandar, fancy man, ponce -n10435169 pipe smoker -n10435251 pip-squeak, squirt, small fry -n10435716 pisser, urinator -n10435988 pitcher, hurler, twirler -n10436334 pitchman -n10437014 placeman, placeseeker -n10437137 placer miner -n10437262 plagiarist, plagiarizer, plagiariser, literary pirate, pirate -n10437698 plainsman -n10438172 planner, contriver, deviser -n10438619 planter, plantation owner -n10438842 plasterer -n10439373 platinum blond, platinum blonde -n10439523 platitudinarian -n10439727 playboy, man-about-town, Corinthian -n10439851 player, participant -n10441037 playmate, playfellow -n10441124 pleaser -n10441694 pledger -n10441962 plenipotentiary -n10442093 plier, plyer -n10442232 plodder, slowpoke, stick-in-the-mud, slowcoach -n10442417 plodder, slogger -n10442573 plotter, mapper -n10443032 plumber, pipe fitter -n10443659 pluralist -n10443830 pluralist -n10444194 poet -n10448322 pointsman -n10448455 point woman -n10449664 policyholder -n10450038 political prisoner -n10450161 political scientist -n10450303 politician, politico, pol, political leader -n10451450 politician -n10451590 pollster, poll taker, headcounter, canvasser -n10451858 polluter, defiler -n10453184 pool player -n10455619 portraitist, portrait painter, portrayer, limner -n10456070 poseuse -n10456138 positivist, rationalist -n10456696 postdoc, post doc -n10457214 poster girl -n10457444 postulator -n10457903 private citizen -n10458111 problem solver, solver, convergent thinker -n10458356 pro-lifer -n10458596 prosthetist -n10459882 postulant -n10460033 potboy, potman -n10461060 poultryman, poulterer -n10462588 power user -n10462751 power worker, power-station worker -n10462860 practitioner, practician -n10464052 prayer, supplicant -n10464542 preceptor, don -n10464711 predecessor -n10464870 preemptor, pre-emptor -n10465002 preemptor, pre-emptor -n10465451 premature baby, preterm baby, premature infant, preterm infant, preemie, premie -n10465831 presbyter -n10466198 presenter, sponsor -n10466564 presentist -n10466918 preserver -n10467179 president -n10467395 President of the United States, United States President, President, Chief Executive -n10468750 president, prexy -n10469611 press agent, publicity man, public relations man, PR man -n10469874 press photographer -n10470779 priest -n10471640 prima ballerina -n10471732 prima donna, diva -n10471859 prima donna -n10472129 primigravida, gravida I -n10472447 primordial dwarf, hypoplastic dwarf, true dwarf, normal dwarf -n10473453 prince charming -n10473562 prince consort -n10473789 princeling -n10473917 Prince of Wales -n10474064 princess -n10474343 princess royal -n10474446 principal, dealer -n10474645 principal, school principal, head teacher, head -n10475835 print seller -n10475940 prior -n10476467 private, buck private, common soldier -n10477713 probationer, student nurse -n10477955 processor -n10478118 process-server -n10478293 proconsul -n10478462 proconsul -n10478827 proctologist -n10478960 proctor, monitor -n10479135 procurator -n10479328 procurer, securer -n10481167 profit taker -n10481268 programmer, computer programmer, coder, software engineer -n10482054 promiser, promisor -n10482220 promoter, booster, plugger -n10482587 promulgator -n10482921 propagandist -n10483138 propagator, disseminator -n10483395 property man, propman, property master -n10483799 prophetess -n10483890 prophet -n10484858 prosecutor, public prosecutor, prosecuting officer, prosecuting attorney -n10485298 prospector -n10485883 protectionist -n10486166 protegee -n10486236 protozoologist -n10486561 provost marshal -n10487182 pruner, trimmer -n10487363 psalmist -n10487592 psephologist -n10488016 psychiatrist, head-shrinker, shrink -n10488309 psychic -n10488656 psycholinguist -n10489426 psychophysicist -n10490421 publican, tavern keeper -n10491998 pudge -n10492086 puerpera -n10492727 punching bag -n10493199 punter -n10493419 punter -n10493685 puppeteer -n10493835 puppy, pup -n10493922 purchasing agent -n10494195 puritan -n10494373 Puritan -n10495167 pursuer -n10495421 pusher, shover -n10495555 pusher, drug peddler, peddler, drug dealer, drug trafficker -n10495756 pusher, thruster -n10496393 putz -n10496489 Pygmy, Pigmy -n10497135 qadi -n10497534 quadriplegic -n10497645 quadruplet, quad -n10498046 quaker, trembler -n10498699 quarter -n10498816 quarterback, signal caller, field general -n10498986 quartermaster -n10499110 quartermaster general -n10499232 Quebecois -n10499355 queen, queen regnant, female monarch -n10499631 Queen of England -n10499857 queen -n10500217 queen -n10500419 queen consort -n10500603 queen mother -n10500824 Queen's Counsel -n10500942 question master, quizmaster -n10501453 quick study, sponge -n10501635 quietist -n10502046 quitter -n10502329 rabbi -n10502950 racist, racialist -n10503818 radiobiologist -n10504090 radiologic technologist -n10504206 radiologist, radiotherapist -n10505347 rainmaker -n10505613 raiser -n10505732 raja, rajah -n10505942 rake, rakehell, profligate, rip, blood, roue -n10506336 ramrod -n10506544 ranch hand -n10506915 ranker -n10507070 ranter, raver -n10507380 rape suspect -n10507482 rapper -n10507565 rapporteur -n10507692 rare bird, rara avis -n10508141 ratepayer -n10508379 raw recruit -n10508710 reader -n10509063 reading teacher -n10509161 realist -n10509810 real estate broker, real estate agent, estate agent, land agent, house agent -n10510245 rear admiral -n10510974 receiver -n10511771 reciter -n10512201 recruit, enlistee -n10512372 recruit, military recruit -n10512708 recruiter -n10512859 recruiting-sergeant -n10513509 redcap -n10513823 redhead, redheader, red-header, carrottop -n10513938 redneck, cracker -n10514051 reeler -n10514121 reenactor -n10514255 referral -n10514429 referee, ref -n10514784 refiner -n10515863 Reform Jew -n10516527 registered nurse, RN -n10517137 registrar -n10517283 Regius professor -n10518349 reliever, allayer, comforter -n10519126 anchorite, hermit -n10519494 religious leader -n10519984 remover -n10520286 Renaissance man, generalist -n10520544 renegade -n10520964 rentier -n10521100 repairman, maintenance man, service man -n10521662 reporter, newsman, newsperson -n10521853 newswoman -n10522035 representative -n10522324 reprobate, miscreant -n10522759 rescuer, recoverer, saver -n10523341 reservist -n10524076 resident commissioner -n10524223 respecter -n10524869 restaurateur, restauranter -n10525134 restrainer, controller -n10525436 retailer, retail merchant -n10525617 retiree, retired person -n10525878 returning officer -n10526534 revenant -n10527147 revisionist -n10527334 revolutionist, revolutionary, subversive, subverter -n10528023 rheumatologist -n10528148 Rhodesian man, Homo rhodesiensis -n10528493 rhymer, rhymester, versifier, poetizer, poetiser -n10529231 rich person, wealthy person, have -n10530150 rider -n10530383 riding master -n10530571 rifleman -n10530959 right-hander, right hander, righthander -n10531109 right-hand man, chief assistant, man Friday -n10531445 ringer -n10531838 ringleader -n10533874 roadman, road mender -n10533983 roarer, bawler, bellower, screamer, screecher, shouter, yeller -n10536134 rocket engineer, rocket scientist -n10536274 rocket scientist -n10536416 rock star -n10537708 Romanov, Romanoff -n10537906 romanticist, romantic -n10538629 ropemaker, rope-maker, roper -n10538733 roper -n10538853 roper -n10539015 ropewalker, ropedancer -n10539160 rosebud -n10539278 Rosicrucian -n10540114 Mountie -n10540252 Rough Rider -n10540656 roundhead -n10541833 civil authority, civil officer -n10542608 runner -n10542761 runner -n10542888 runner -n10543161 running back -n10543937 rusher -n10544232 rustic -n10544748 saboteur, wrecker, diversionist -n10545792 sadist -n10546428 sailing master, navigator -n10546633 sailor, crewman -n10548419 salesgirl, saleswoman, saleslady -n10548537 salesman -n10548681 salesperson, sales representative, sales rep -n10549510 salvager, salvor -n10550252 sandwichman -n10550369 sangoma -n10550468 sannup -n10551576 sapper -n10552393 Sassenach -n10553140 satrap -n10553235 saunterer, stroller, ambler -n10554024 Savoyard -n10554141 sawyer -n10554846 scalper -n10555059 scandalmonger -n10555430 scapegrace, black sheep -n10556033 scene painter -n10556518 schemer, plotter -n10556704 schizophrenic -n10556825 schlemiel, shlemiel -n10557246 schlockmeister, shlockmeister -n10557854 scholar, scholarly person, bookman, student -n10559009 scholiast -n10559288 schoolchild, school-age child, pupil -n10559508 schoolfriend -n10559683 Schoolman, medieval Schoolman -n10559996 schoolmaster -n10560106 schoolmate, classmate, schoolfellow, class fellow -n10560637 scientist -n10561222 scion -n10561320 scoffer, flouter, mocker, jeerer -n10561736 scofflaw -n10562135 scorekeeper, scorer -n10562283 scorer -n10562509 scourer -n10562968 scout, talent scout -n10563314 scoutmaster -n10563403 scrambler -n10563711 scratcher -n10564098 screen actor, movie actor -n10565502 scrutineer, canvasser -n10565667 scuba diver -n10566072 sculptor, sculpturer, carver, statue maker -n10567613 Sea Scout -n10567722 seasonal worker, seasonal -n10567848 seasoner -n10568200 second baseman, second sacker -n10568358 second cousin -n10568443 seconder -n10568608 second fiddle, second banana -n10568915 second-in-command -n10569011 second lieutenant, 2nd lieutenant -n10569179 second-rater, mediocrity -n10570019 secretary -n10570704 Secretary of Agriculture, Agriculture Secretary -n10571907 Secretary of Health and Human Services -n10572706 Secretary of State -n10572889 Secretary of the Interior, Interior Secretary -n10573957 sectarian, sectary, sectarist -n10574311 section hand -n10574538 secularist -n10574840 security consultant -n10575463 seeded player, seed -n10575594 seeder, cloud seeder -n10575787 seeker, searcher, quester -n10576223 segregate -n10576316 segregator, segregationist -n10576676 selectman -n10576818 selectwoman -n10576962 selfish person -n10577182 self-starter -n10577284 seller, marketer, vender, vendor, trafficker -n10577710 selling agent -n10577820 semanticist, semiotician -n10578021 semifinalist -n10578162 seminarian, seminarist -n10578471 senator -n10578656 sendee -n10579062 senior -n10579549 senior vice president -n10580030 separatist, separationist -n10580437 septuagenarian -n10580535 serf, helot, villein -n10581648 spree killer -n10581890 serjeant-at-law, serjeant, sergeant-at-law, sergeant -n10582604 server -n10582746 serviceman, military man, man, military personnel -n10583387 settler, colonist -n10583790 settler -n10585077 sex symbol -n10585217 sexton, sacristan -n10585628 shaheed -n10586166 Shakespearian, Shakespearean -n10586265 shanghaier, seizer -n10586444 sharecropper, cropper, sharecrop farmer -n10586903 shaver -n10586998 Shavian -n10588074 sheep -n10588357 sheik, tribal sheik, sheikh, tribal sheikh, Arab chief -n10588724 shelver -n10588965 shepherd -n10589666 ship-breaker -n10590146 shipmate -n10590239 shipowner -n10590452 shipping agent -n10590903 shirtmaker -n10591072 shogun -n10591811 shopaholic -n10592049 shop girl -n10592811 shop steward, steward -n10593521 shot putter -n10594147 shrew, termagant -n10594523 shuffler -n10594857 shyster, pettifogger -n10595164 sibling, sib -n10595647 sick person, diseased person, sufferer -n10596517 sightreader -n10596899 signaler, signaller -n10597505 signer -n10597745 signor, signior -n10597889 signora -n10598013 signore -n10598181 signorina -n10598459 silent partner, sleeping partner -n10598904 addle-head, addlehead, loon, birdbrain -n10599215 simperer -n10599806 singer, vocalist, vocalizer, vocaliser -n10601234 Sinologist -n10601362 sipper -n10602119 sirrah -n10602470 Sister -n10602985 sister, sis -n10603528 waverer, vacillator, hesitator, hesitater -n10603851 sitar player -n10604275 sixth-former -n10604380 skateboarder -n10604634 skeptic, sceptic, doubter -n10604880 sketcher -n10604979 skidder -n10605253 skier -n10605737 skinny-dipper -n10607291 skin-diver, aquanaut -n10607478 skinhead -n10609092 slasher -n10609198 slattern, slut, slovenly woman, trollop -n10610465 sleeper, slumberer -n10610850 sleeper -n10611267 sleeping beauty -n10611613 sleuth, sleuthhound -n10612210 slob, sloven, pig, slovenly person -n10612373 sloganeer -n10612518 slopseller, slop-seller -n10613996 smasher, stunner, knockout, beauty, ravisher, sweetheart, peach, lulu, looker, mantrap, dish -n10614507 smirker -n10614629 smith, metalworker -n10615179 smoothie, smoothy, sweet talker, charmer -n10615334 smuggler, runner, contrabandist, moon curser, moon-curser -n10616578 sneezer -n10617024 snob, prig, snot, snoot -n10617193 snoop, snooper -n10617397 snorer -n10618234 sob sister -n10618342 soccer player -n10618465 social anthropologist, cultural anthropologist -n10618685 social climber, climber -n10618848 socialist -n10619492 socializer, socialiser -n10619642 social scientist -n10619888 social secretary -n10620212 Socinian -n10620586 sociolinguist -n10620758 sociologist -n10621294 soda jerk, soda jerker -n10621400 sodalist -n10621514 sodomite, sodomist, sod, bugger -n10622053 soldier -n10624074 son, boy -n10624310 songster -n10624437 songstress -n10624540 songwriter, songster, ballad maker -n10625860 sorcerer, magician, wizard, necromancer, thaumaturge, thaumaturgist -n10626630 sorehead -n10627252 soul mate -n10628097 Southern Baptist -n10628644 sovereign, crowned head, monarch -n10629329 spacewalker -n10629647 Spanish American, Hispanic American, Hispanic -n10629939 sparring partner, sparring mate -n10630093 spastic -n10630188 speaker, talker, utterer, verbalizer, verbaliser -n10631131 native speaker -n10631309 Speaker -n10631654 speechwriter -n10632576 specialist, medical specialist -n10633298 specifier -n10633450 spectator, witness, viewer, watcher, looker -n10634464 speech therapist -n10634849 speedskater, speed skater -n10634990 spellbinder -n10635788 sphinx -n10636488 spinster, old maid -n10637483 split end -n10638922 sport, sportsman, sportswoman -n10639238 sport, summercater -n10639359 sporting man, outdoor man -n10639637 sports announcer, sportscaster, sports commentator -n10639817 sports editor -n10641223 sprog -n10642596 square dancer -n10642705 square shooter, straight shooter, straight arrow -n10643095 squatter -n10643837 squire -n10643937 squire -n10644598 staff member, staffer -n10645017 staff sergeant -n10645223 stage director -n10646032 stainer -n10646140 stakeholder -n10646433 stalker -n10646641 stalking-horse -n10646780 stammerer, stutterer -n10646942 stamper, stomper, tramper, trampler -n10647745 standee -n10648237 stand-in, substitute, relief, reliever, backup, backup man, fill-in -n10648696 star, principal, lead -n10649197 starlet -n10649308 starter, dispatcher -n10650162 statesman, solon, national leader -n10652605 state treasurer -n10652703 stationer, stationery seller -n10654015 stenographer, amanuensis, shorthand typist -n10654211 stentor -n10654321 stepbrother, half-brother, half brother -n10654827 stepmother -n10654932 stepparent -n10655169 stevedore, loader, longshoreman, docker, dockhand, dock worker, dockworker, dock-walloper, lumper -n10655442 steward -n10655594 steward, flight attendant -n10655730 steward -n10655986 stickler -n10656120 stiff -n10656223 stifler, smotherer -n10656969 stipendiary, stipendiary magistrate -n10657306 stitcher -n10657556 stockjobber -n10657835 stock trader -n10658304 stockist -n10659042 stoker, fireman -n10659762 stooper -n10660128 store detective -n10660621 strafer -n10660883 straight man, second banana -n10661002 stranger, alien, unknown -n10661216 stranger -n10661563 strategist, strategian -n10661732 straw boss, assistant foreman -n10663315 streetwalker, street girl, hooker, hustler, floozy, floozie, slattern -n10663549 stretcher-bearer, litter-bearer -n10665302 struggler -n10665587 stud, he-man, macho-man -n10665698 student, pupil, educatee -n10666752 stumblebum, palooka -n10667477 stylist -n10667709 subaltern -n10667863 subcontractor -n10668450 subduer, surmounter, overcomer -n10668666 subject, case, guinea pig -n10669991 subordinate, subsidiary, underling, foot soldier -n10671042 substitute, reserve, second-stringer -n10671613 successor, heir -n10671736 successor, replacement -n10671898 succorer, succourer -n10672371 Sufi -n10672540 suffragan, suffragan bishop -n10672662 suffragette -n10673296 sugar daddy -n10673776 suicide bomber -n10674130 suitor, suer, wooer -n10674713 sumo wrestler -n10675010 sunbather -n10675142 sundowner -n10675609 super heavyweight -n10676018 superior, higher-up, superordinate -n10676434 supermom -n10676569 supernumerary, spear carrier, extra -n10678937 supremo -n10679174 surgeon, operating surgeon, sawbones -n10679503 Surgeon General -n10679610 Surgeon General -n10679723 surpriser -n10680609 surveyor -n10680796 surveyor -n10681194 survivor, subsister -n10681557 sutler, victualer, victualler, provisioner -n10682713 sweeper -n10682953 sweetheart, sweetie, steady, truelove -n10683675 swinger, tramp -n10684146 switcher, whipper -n10684630 swot, grind, nerd, wonk, dweeb -n10684827 sycophant, toady, crawler, lackey, ass-kisser -n10685398 sylph -n10686073 sympathizer, sympathiser, well-wisher -n10686517 symphonist -n10686694 syncopator -n10686885 syndic -n10688356 tactician -n10688811 tagger -n10689306 tailback -n10690268 tallyman, tally clerk -n10690421 tallyman -n10690648 tanker, tank driver -n10691318 tapper, wiretapper, phone tapper -n10691937 Tartuffe, Tartufe -n10692090 Tarzan -n10692482 taster, taste tester, taste-tester, sampler -n10692883 tax assessor, assessor -n10693235 taxer -n10693334 taxi dancer -n10693824 taxonomist, taxonomer, systematist -n10694258 teacher, instructor -n10694939 teaching fellow -n10695450 tearaway -n10696101 technical sergeant -n10696508 technician -n10697135 Ted, Teddy boy -n10697282 teetotaler, teetotaller, teetotalist -n10698368 television reporter, television newscaster, TV reporter, TV newsman -n10699558 temporizer, temporiser -n10699752 tempter -n10699981 term infant -n10700105 toiler -n10700201 tenant, renter -n10700640 tenant -n10700963 tenderfoot -n10701180 tennis player -n10701644 tennis pro, professional tennis player -n10701962 tenor saxophonist, tenorist -n10702167 termer -n10702615 terror, scourge, threat -n10703221 tertigravida, gravida III -n10703336 testator, testate -n10703480 testatrix -n10703692 testee, examinee -n10704238 test-tube baby -n10704712 Texas Ranger, Ranger -n10704886 thane -n10705448 theatrical producer -n10705615 theologian, theologist, theologizer, theologiser -n10706812 theorist, theoretician, theorizer, theoriser, idealogue -n10707134 theosophist -n10707233 therapist, healer -n10707707 Thessalonian -n10708292 thinker, creative thinker, mind -n10708454 thinker -n10709529 thrower -n10710171 thurifer -n10710259 ticket collector, ticket taker -n10710778 tight end -n10710913 tiler -n10711483 timekeeper, timer -n10711766 Timorese -n10712229 tinkerer, fiddler -n10712374 tinsmith, tinner -n10712474 tinter -n10712690 tippler, social drinker -n10712835 tipster, tout -n10713254 T-man -n10713686 toastmaster, symposiarch -n10713843 toast mistress -n10714195 tobogganist -n10715030 tomboy, romp, hoyden -n10715347 toolmaker -n10715789 torchbearer -n10716576 Tory -n10716864 Tory -n10717055 tosser -n10717196 tosser, jerk-off, wanker -n10717337 totalitarian -n10718131 tourist, tourer, holidaymaker -n10718349 tout, touter -n10718509 tout, ticket tout -n10718665 tovarich, tovarisch -n10718952 towhead -n10719036 town clerk -n10719132 town crier, crier -n10719267 townsman, towner -n10719807 toxicologist -n10720197 track star -n10720453 trader, bargainer, dealer, monger -n10720964 trade unionist, unionist, union member -n10721124 traditionalist, diehard -n10721321 traffic cop -n10721612 tragedian -n10721708 tragedian -n10721819 tragedienne -n10722029 trail boss -n10722575 trainer -n10722965 traitor, treasonist -n10723230 traitress -n10723597 transactor -n10724132 transcriber -n10724372 transfer, transferee -n10724570 transferee -n10725280 translator, transcriber -n10726031 transvestite, cross-dresser -n10726786 traveling salesman, travelling salesman, commercial traveler, commercial traveller, roadman, bagman -n10727016 traverser -n10727171 trawler -n10727458 Treasury, First Lord of the Treasury -n10728117 trencher -n10728233 trend-setter, taste-maker, fashion arbiter -n10728624 tribesman -n10728998 trier, attempter, essayer -n10729330 trifler -n10730542 trooper -n10730728 trooper, state trooper -n10731013 Trotskyite, Trotskyist, Trot -n10731732 truant, hooky player -n10732010 trumpeter, cornetist -n10732521 trusty -n10732854 Tudor -n10732967 tumbler -n10733820 tutee -n10734394 twin -n10734741 two-timer -n10734891 Tyke -n10734963 tympanist, timpanist -n10735173 typist -n10735298 tyrant, autocrat, despot -n10735984 umpire, ump -n10737103 understudy, standby -n10737264 undesirable -n10738111 unicyclist -n10738215 unilateralist -n10738670 Unitarian -n10738871 Arminian -n10739135 universal donor -n10739297 UNIX guru -n10739391 Unknown Soldier -n10740594 upsetter -n10740732 upstager -n10740868 upstart, parvenu, nouveau-riche, arriviste -n10741152 upstart -n10741367 urchin -n10741493 urologist -n10742005 usherette -n10742111 usher, doorkeeper -n10742546 usurper, supplanter -n10742997 utility man -n10743124 utilizer, utiliser -n10743356 Utopian -n10744078 uxoricide -n10744164 vacationer, vacationist -n10745006 valedictorian, valedictory speaker -n10745770 valley girl -n10746931 vaulter, pole vaulter, pole jumper -n10747119 vegetarian -n10747424 vegan -n10747548 venerator -n10747965 venture capitalist -n10748142 venturer, merchant-venturer -n10748506 vermin, varmint -n10748620 very important person, VIP, high-up, dignitary, panjandrum, high muckamuck -n10749928 vibist, vibraphonist -n10750031 vicar -n10750188 vicar -n10750640 vicar-general -n10751026 vice chancellor -n10751152 vicegerent -n10751265 vice president, V.P. -n10751710 vice-regent -n10752480 victim, dupe -n10753061 Victorian -n10753182 victualer, victualler -n10753339 vigilante, vigilance man -n10753442 villager -n10753989 vintager -n10754189 vintner, wine merchant -n10754281 violator, debaucher, ravisher -n10754449 violator, lawbreaker, law offender -n10755080 violist -n10755164 virago -n10755394 virologist -n10755648 Visayan, Bisayan -n10756061 viscountess -n10756148 viscount -n10756261 Visigoth -n10756641 visionary -n10756837 visiting fireman -n10757050 visiting professor -n10757492 visualizer, visualiser -n10758337 vixen, harpy, hellcat -n10758445 vizier -n10758949 voicer -n10759151 volunteer, unpaid worker -n10759331 volunteer, military volunteer, voluntary -n10759982 votary -n10760199 votary -n10760622 vouchee -n10760951 vower -n10761190 voyager -n10761326 voyeur, Peeping Tom, peeper -n10761519 vulcanizer, vulcaniser -n10762212 waffler -n10762480 Wagnerian -n10763075 waif, street child -n10763245 wailer -n10763383 waiter, server -n10763620 waitress -n10764465 walking delegate -n10764622 walk-on -n10764719 wallah -n10765305 wally -n10765587 waltzer -n10765679 wanderer, roamer, rover, bird of passage -n10765885 Wandering Jew -n10766260 wanton -n10768148 warrantee -n10768272 warrantee -n10768903 washer -n10769084 washerman, laundryman -n10769188 washwoman, washerwoman, laundrywoman, laundress -n10769321 wassailer, carouser -n10769459 wastrel, waster -n10771066 Wave -n10772092 weatherman, weather forecaster -n10772580 weekend warrior -n10772937 weeder -n10773665 welder -n10773800 welfare case, charity case -n10774329 westerner -n10774756 West-sider -n10775003 wetter -n10775128 whaler -n10776052 Whig -n10776339 whiner, complainer, moaner, sniveller, crybaby, bellyacher, grumbler, squawker -n10776887 whipper-in -n10777299 whisperer -n10778044 whiteface -n10778148 Carmelite, White Friar -n10778711 Augustinian -n10778999 white hope, great white hope -n10779610 white supremacist -n10779897 whoremaster, whoremonger -n10779995 whoremaster, whoremonger, john, trick -n10780284 widow, widow woman -n10780632 wife, married woman -n10781236 wiggler, wriggler, squirmer -n10781817 wimp, chicken, crybaby -n10782362 wing commander -n10782471 winger -n10782791 winner -n10782940 winner, victor -n10783240 window dresser, window trimmer -n10783539 winker -n10783646 wiper -n10783734 wireman, wirer -n10784113 wise guy, smart aleck, wiseacre, wisenheimer, weisenheimer -n10784544 witch doctor -n10784922 withdrawer -n10785480 withdrawer -n10787470 woman, adult female -n10788852 woman -n10789415 wonder boy, golden boy -n10789709 wonderer -n10791115 working girl -n10791221 workman, workingman, working man, working person -n10791820 workmate -n10791890 worldling -n10792335 worshiper, worshipper -n10792506 worthy -n10792856 wrecker -n10793570 wright -n10793799 write-in candidate, write-in -n10794014 writer, author -n10801561 Wykehamist -n10801802 yakuza -n10802507 yard bird, yardbird -n10802621 yardie -n10802953 yardman -n10803031 yardmaster, trainmaster, train dispatcher -n10803282 yenta -n10803978 yogi -n10804287 young buck, young man -n10804636 young Turk -n10804732 Young Turk -n10805501 Zionist -n10806113 zoo keeper -n10994097 Genet, Edmund Charles Edouard Genet, Citizen Genet -n11100798 Kennan, George F. Kennan, George Frost Kennan -n11196627 Munro, H. H. Munro, Hector Hugh Munro, Saki -n11242849 Popper, Karl Popper, Sir Karl Raimund Popper -n11318824 Stoker, Bram Stoker, Abraham Stoker -n11346873 Townes, Charles Townes, Charles Hard Townes -n11448153 dust storm, duster, sandstorm, sirocco -n11487732 parhelion, mock sun, sundog -n11508382 snow, snowfall -n11511327 facula -n11524451 wave -n11530008 microflora -n11531193 wilding -n11531334 semi-climber -n11532682 volva -n11533212 basidiocarp -n11533999 domatium -n11536567 apomict -n11536673 aquatic -n11537327 bryophyte, nonvascular plant -n11539289 acrocarp, acrocarpous moss -n11542137 sphagnum, sphagnum moss, peat moss, bog moss -n11542640 liverwort, hepatic -n11544015 hepatica, Marchantia polymorpha -n11545350 pecopteris -n11545524 pteridophyte, nonflowering plant -n11545714 fern -n11547562 fern ally -n11547855 spore -n11548728 carpospore -n11548870 chlamydospore -n11549009 conidium, conidiospore -n11549245 oospore -n11549779 tetraspore -n11549895 zoospore -n11552133 cryptogam -n11552386 spermatophyte, phanerogam, seed plant -n11552594 seedling -n11552806 annual -n11552976 biennial -n11553240 perennial -n11553522 hygrophyte -n11596108 gymnosperm -n11597657 gnetum, Gnetum gnemon -n11598287 Catha edulis -n11598686 ephedra, joint fir -n11598886 mahuang, Ephedra sinica -n11599324 welwitschia, Welwitschia mirabilis -n11600372 cycad -n11601177 sago palm, Cycas revoluta -n11601333 false sago, fern palm, Cycas circinalis -n11601918 zamia -n11602091 coontie, Florida arrowroot, Seminole bread, Zamia pumila -n11602478 ceratozamia -n11602873 dioon -n11603246 encephalartos -n11603462 kaffir bread, Encephalartos caffer -n11603835 macrozamia -n11604046 burrawong, Macrozamia communis, Macrozamia spiralis -n11608250 pine, pine tree, true pine -n11609475 pinon, pinyon -n11609684 nut pine -n11609862 pinon pine, Mexican nut pine, Pinus cembroides -n11610047 Rocky mountain pinon, Pinus edulis -n11610215 single-leaf, single-leaf pine, single-leaf pinyon, Pinus monophylla -n11610437 bishop pine, bishop's pine, Pinus muricata -n11610602 California single-leaf pinyon, Pinus californiarum -n11610823 Parry's pinyon, Pinus quadrifolia, Pinus parryana -n11611087 spruce pine, Pinus glabra -n11611233 black pine, Pinus nigra -n11611356 pitch pine, northern pitch pine, Pinus rigida -n11611561 pond pine, Pinus serotina -n11611758 stone pine, umbrella pine, European nut pine, Pinus pinea -n11612018 Swiss pine, Swiss stone pine, arolla pine, cembra nut tree, Pinus cembra -n11612235 cembra nut, cedar nut -n11612349 Swiss mountain pine, mountain pine, dwarf mountain pine, mugho pine, mugo pine, Pinus mugo -n11612575 ancient pine, Pinus longaeva -n11612923 white pine -n11613219 American white pine, eastern white pine, weymouth pine, Pinus strobus -n11613459 western white pine, silver pine, mountain pine, Pinus monticola -n11613692 southwestern white pine, Pinus strobiformis -n11613867 limber pine, Pinus flexilis -n11614039 whitebark pine, whitebarked pine, Pinus albicaulis -n11614250 yellow pine -n11614420 ponderosa, ponderosa pine, western yellow pine, bull pine, Pinus ponderosa -n11614713 Jeffrey pine, Jeffrey's pine, black pine, Pinus jeffreyi -n11615026 shore pine, lodgepole, lodgepole pine, spruce pine, Pinus contorta -n11615259 Sierra lodgepole pine, Pinus contorta murrayana -n11615387 loblolly pine, frankincense pine, Pinus taeda -n11615607 jack pine, Pinus banksiana -n11615812 swamp pine -n11615967 longleaf pine, pitch pine, southern yellow pine, Georgia pine, Pinus palustris -n11616260 shortleaf pine, short-leaf pine, shortleaf yellow pine, Pinus echinata -n11616486 red pine, Canadian red pine, Pinus resinosa -n11616662 Scotch pine, Scots pine, Scotch fir, Pinus sylvestris -n11616852 scrub pine, Virginia pine, Jersey pine, Pinus virginiana -n11617090 Monterey pine, Pinus radiata -n11617272 bristlecone pine, Rocky Mountain bristlecone pine, Pinus aristata -n11617631 table-mountain pine, prickly pine, hickory pine, Pinus pungens -n11617878 knobcone pine, Pinus attenuata -n11618079 Japanese red pine, Japanese table pine, Pinus densiflora -n11618290 Japanese black pine, black pine, Pinus thunbergii -n11618525 Torrey pine, Torrey's pine, soledad pine, grey-leaf pine, sabine pine, Pinus torreyana -n11618861 larch, larch tree -n11619227 American larch, tamarack, black larch, Larix laricina -n11619455 western larch, western tamarack, Oregon larch, Larix occidentalis -n11619687 subalpine larch, Larix lyallii -n11619845 European larch, Larix decidua -n11620016 Siberian larch, Larix siberica, Larix russica -n11620389 golden larch, Pseudolarix amabilis -n11620673 fir, fir tree, true fir -n11621029 silver fir -n11621281 amabilis fir, white fir, Pacific silver fir, red silver fir, Christmas tree, Abies amabilis -n11621547 European silver fir, Christmas tree, Abies alba -n11621727 white fir, Colorado fir, California white fir, Abies concolor, Abies lowiana -n11621950 balsam fir, balm of Gilead, Canada balsam, Abies balsamea -n11622184 Fraser fir, Abies fraseri -n11622368 lowland fir, lowland white fir, giant fir, grand fir, Abies grandis -n11622591 Alpine fir, subalpine fir, Abies lasiocarpa -n11622771 Santa Lucia fir, bristlecone fir, Abies bracteata, Abies venusta -n11623105 cedar, cedar tree, true cedar -n11623815 cedar of Lebanon, Cedrus libani -n11623967 deodar, deodar cedar, Himalayan cedar, Cedrus deodara -n11624192 Atlas cedar, Cedrus atlantica -n11624531 spruce -n11625003 Norway spruce, Picea abies -n11625223 weeping spruce, Brewer's spruce, Picea breweriana -n11625391 Engelmann spruce, Engelmann's spruce, Picea engelmannii -n11625632 white spruce, Picea glauca -n11625804 black spruce, Picea mariana, spruce pine -n11626010 Siberian spruce, Picea obovata -n11626152 Sitka spruce, Picea sitchensis -n11626409 oriental spruce, Picea orientalis -n11626585 Colorado spruce, Colorado blue spruce, silver spruce, Picea pungens -n11626826 red spruce, eastern spruce, yellow spruce, Picea rubens -n11627168 hemlock, hemlock tree -n11627512 eastern hemlock, Canadian hemlock, spruce pine, Tsuga canadensis -n11627714 Carolina hemlock, Tsuga caroliniana -n11627908 mountain hemlock, black hemlock, Tsuga mertensiana -n11628087 western hemlock, Pacific hemlock, west coast hemlock, Tsuga heterophylla -n11628456 douglas fir -n11628793 green douglas fir, douglas spruce, douglas pine, douglas hemlock, Oregon fir, Oregon pine, Pseudotsuga menziesii -n11629047 big-cone spruce, big-cone douglas fir, Pseudotsuga macrocarpa -n11629354 Cathaya -n11630017 cedar, cedar tree -n11630489 cypress, cypress tree -n11631159 gowen cypress, Cupressus goveniana -n11631405 pygmy cypress, Cupressus pigmaea, Cupressus goveniana pigmaea -n11631619 Santa Cruz cypress, Cupressus abramsiana, Cupressus goveniana abramsiana -n11631854 Arizona cypress, Cupressus arizonica -n11631985 Guadalupe cypress, Cupressus guadalupensis -n11632167 Monterey cypress, Cupressus macrocarpa -n11632376 Mexican cypress, cedar of Goa, Portuguese cypress, Cupressus lusitanica -n11632619 Italian cypress, Mediterranean cypress, Cupressus sempervirens -n11632929 King William pine, Athrotaxis selaginoides -n11633284 Chilean cedar, Austrocedrus chilensis -n11634736 incense cedar, red cedar, Calocedrus decurrens, Libocedrus decurrens -n11635152 southern white cedar, coast white cedar, Atlantic white cedar, white cypress, white cedar, Chamaecyparis thyoides -n11635433 Oregon cedar, Port Orford cedar, Lawson's cypress, Lawson's cedar, Chamaecyparis lawsoniana -n11635830 yellow cypress, yellow cedar, Nootka cypress, Alaska cedar, Chamaecyparis nootkatensis -n11636204 Japanese cedar, Japan cedar, sugi, Cryptomeria japonica -n11636835 juniper berry -n11639084 incense cedar -n11639306 kawaka, Libocedrus plumosa -n11639445 pahautea, Libocedrus bidwillii, mountain pine -n11640132 metasequoia, dawn redwood, Metasequoia glyptostrodoides -n11643835 arborvitae -n11644046 western red cedar, red cedar, canoe cedar, Thuja plicata -n11644226 American arborvitae, northern white cedar, white cedar, Thuja occidentalis -n11644462 Oriental arborvitae, Thuja orientalis, Platycladus orientalis -n11644872 hiba arborvitae, Thujopsis dolobrata -n11645163 keteleeria -n11645590 Wollemi pine -n11645914 araucaria -n11646167 monkey puzzle, chile pine, Araucaria araucana -n11646344 norfolk island pine, Araucaria heterophylla, Araucaria excelsa -n11646517 new caledonian pine, Araucaria columnaris -n11646694 bunya bunya, bunya bunya tree, Araucaria bidwillii -n11646955 hoop pine, Moreton Bay pine, Araucaria cunninghamii -n11647306 kauri pine, dammar pine -n11647703 kauri, kaury, Agathis australis -n11647868 amboina pine, amboyna pine, Agathis dammara, Agathis alba -n11648039 dundathu pine, queensland kauri, smooth bark kauri, Agathis robusta -n11648268 red kauri, Agathis lanceolata -n11648776 plum-yew -n11649150 California nutmeg, nutmeg-yew, Torreya californica -n11649359 stinking cedar, stinking yew, Torrey tree, Torreya taxifolia -n11649878 celery pine -n11650160 celery top pine, celery-topped pine, Phyllocladus asplenifolius -n11650307 tanekaha, Phyllocladus trichomanoides -n11650430 Alpine celery pine, Phyllocladus alpinus -n11650558 yellowwood, yellowwood tree -n11650759 gymnospermous yellowwood -n11652039 podocarp -n11652217 yacca, yacca podocarp, Podocarpus coriaceus -n11652376 brown pine, Rockingham podocarp, Podocarpus elatus -n11652578 cape yellowwood, African yellowwood, Podocarpus elongatus -n11652753 South-African yellowwood, Podocarpus latifolius -n11652966 alpine totara, Podocarpus nivalis -n11653126 totara, Podocarpus totara -n11653570 common yellowwood, bastard yellowwood, Afrocarpus falcata -n11653904 kahikatea, New Zealand Dacryberry, New Zealand white pine, Dacrycarpus dacrydioides, Podocarpus dacrydioides -n11654293 rimu, imou pine, red pine, Dacrydium cupressinum -n11654438 tarwood, tar-wood, Dacrydium colensoi -n11654984 common sickle pine, Falcatifolium falciforme -n11655152 yellow-leaf sickle pine, Falcatifolium taxoides -n11655592 tarwood, tar-wood, New Zealand mountain pine, Halocarpus bidwilli, Dacrydium bidwilli -n11655974 westland pine, silver pine, Lagarostrobus colensoi -n11656123 huon pine, Lagarostrobus franklinii, Dacrydium franklinii -n11656549 Chilean rimu, Lepidothamnus fonkii -n11656771 mountain rimu, Lepidothamnus laxifolius, Dacridium laxifolius -n11657585 nagi, Nageia nagi -n11658331 miro, black pine, Prumnopitys ferruginea, Podocarpus ferruginea -n11658544 matai, black pine, Prumnopitys taxifolia, Podocarpus spicata -n11658709 plum-fruited yew, Prumnopitys andina, Prumnopitys elegans -n11659248 Prince Albert yew, Prince Albert's yew, Saxe-gothea conspicua -n11659627 Sundacarpus amara, Prumnopitys amara, Podocarpus amara -n11660300 Japanese umbrella pine, Sciadopitys verticillata -n11661372 yew -n11661909 Old World yew, English yew, Taxus baccata -n11662128 Pacific yew, California yew, western yew, Taxus brevifolia -n11662371 Japanese yew, Taxus cuspidata -n11662585 Florida yew, Taxus floridana -n11662937 New Caledonian yew, Austrotaxus spicata -n11663263 white-berry yew, Pseudotaxus chienii -n11664418 ginkgo, gingko, maidenhair tree, Ginkgo biloba -n11665372 angiosperm, flowering plant -n11666854 dicot, dicotyledon, magnoliopsid, exogen -n11668117 monocot, monocotyledon, liliopsid, endogen -n11669786 floret, floweret -n11669921 flower -n11672269 bloomer -n11672400 wildflower, wild flower -n11674019 apetalous flower -n11674332 inflorescence -n11675025 rosebud -n11675404 gynostegium -n11675738 pollinium -n11676500 pistil -n11676743 gynobase -n11676850 gynophore -n11677485 stylopodium -n11677902 carpophore -n11678010 cornstalk, corn stalk -n11678299 petiolule -n11678377 mericarp -n11679378 micropyle -n11680457 germ tube -n11680596 pollen tube -n11682659 gemma -n11683216 galbulus -n11683838 nectary, honey gland -n11684264 pericarp, seed vessel -n11684499 epicarp, exocarp -n11684654 mesocarp -n11685091 pip -n11685621 silique, siliqua -n11686195 cataphyll -n11686652 perisperm -n11686780 monocarp, monocarpic plant, monocarpous plant -n11686912 sporophyte -n11687071 gametophyte -n11687432 megasporangium, macrosporangium -n11687789 microspore -n11687964 microsporangium -n11688069 microsporophyll -n11688378 archespore, archesporium -n11689197 bonduc nut, nicker nut, nicker seed -n11689367 Job's tears -n11689483 oilseed, oil-rich seed -n11689678 castor bean -n11689815 cottonseed -n11689957 candlenut -n11690088 peach pit -n11690254 hypanthium, floral cup, calyx tube -n11690455 petal, flower petal -n11691046 corolla -n11691857 lip -n11692265 perianth, chlamys, floral envelope, perigone, perigonium -n11692792 thistledown -n11693981 custard apple, custard apple tree -n11694300 cherimoya, cherimoya tree, Annona cherimola -n11694469 ilama, ilama tree, Annona diversifolia -n11694664 soursop, prickly custard apple, soursop tree, Annona muricata -n11694866 bullock's heart, bullock's heart tree, bullock heart, Annona reticulata -n11695085 sweetsop, sweetsop tree, Annona squamosa -n11695285 pond apple, pond-apple tree, Annona glabra -n11695599 pawpaw, papaw, papaw tree, Asimina triloba -n11695974 ilang-ilang, ylang-ylang, Cananga odorata -n11696450 lancewood, lancewood tree, Oxandra lanceolata -n11696935 Guinea pepper, negro pepper, Xylopia aethiopica -n11697560 barberry -n11697802 American barberry, Berberis canadensis -n11698042 common barberry, European barberry, Berberis vulgaris -n11698245 Japanese barberry, Berberis thunbergii -n11699442 Oregon grape, Oregon holly grape, hollygrape, mountain grape, holly-leaves barberry, Mahonia aquifolium -n11699751 Oregon grape, Mahonia nervosa -n11700058 mayapple, May apple, wild mandrake, Podophyllum peltatum -n11700279 May apple -n11700864 allspice -n11701066 Carolina allspice, strawberry shrub, strawberry bush, sweet shrub, Calycanthus floridus -n11701302 spicebush, California allspice, Calycanthus occidentalis -n11702713 katsura tree, Cercidiphyllum japonicum -n11703669 laurel -n11704093 true laurel, bay, bay laurel, bay tree, Laurus nobilis -n11704620 camphor tree, Cinnamomum camphora -n11704791 cinnamon, Ceylon cinnamon, Ceylon cinnamon tree, Cinnamomum zeylanicum -n11705171 cassia, cassia-bark tree, Cinnamomum cassia -n11705387 cassia bark, Chinese cinnamon -n11705573 Saigon cinnamon, Cinnamomum loureirii -n11705776 cinnamon bark -n11706325 spicebush, spice bush, American spicebush, Benjamin bush, Lindera benzoin, Benzoin odoriferum -n11706761 avocado, avocado tree, Persea Americana -n11706942 laurel-tree, red bay, Persea borbonia -n11707229 sassafras, sassafras tree, Sassafras albidum -n11707827 California laurel, California bay tree, Oregon myrtle, pepperwood, spice tree, sassafras laurel, California olive, mountain laurel, Umbellularia californica -n11708658 anise tree -n11708857 purple anise, Illicium floridanum -n11709045 star anise, Illicium anisatum -n11709205 star anise, Chinese anise, Illicium verum -n11709674 magnolia -n11710136 southern magnolia, evergreen magnolia, large-flowering magnolia, bull bay, Magnolia grandiflora -n11710393 umbrella tree, umbrella magnolia, elkwood, elk-wood, Magnolia tripetala -n11710658 earleaved umbrella tree, Magnolia fraseri -n11710827 cucumber tree, Magnolia acuminata -n11710987 large-leaved magnolia, large-leaved cucumber tree, great-leaved macrophylla, Magnolia macrophylla -n11711289 saucer magnolia, Chinese magnolia, Magnolia soulangiana -n11711537 star magnolia, Magnolia stellata -n11711764 sweet bay, swamp bay, swamp laurel, Magnolia virginiana -n11711971 manglietia, genus Manglietia -n11712282 tulip tree, tulip poplar, yellow poplar, canary whitewood, Liriodendron tulipifera -n11713164 moonseed -n11713370 common moonseed, Canada moonseed, yellow parilla, Menispermum canadense -n11713763 Carolina moonseed, Cocculus carolinus -n11714382 nutmeg, nutmeg tree, Myristica fragrans -n11715430 water nymph, fragrant water lily, pond lily, Nymphaea odorata -n11715678 European white lily, Nymphaea alba -n11716698 southern spatterdock, Nuphar sagittifolium -n11717399 lotus, Indian lotus, sacred lotus, Nelumbo nucifera -n11717577 water chinquapin, American lotus, yanquapin, Nelumbo lutea -n11718296 water-shield, fanwort, Cabomba caroliniana -n11718681 water-shield, Brasenia schreberi, water-target -n11719286 peony, paeony -n11720353 buttercup, butterflower, butter-flower, crowfoot, goldcup, kingcup -n11720643 meadow buttercup, tall buttercup, tall crowfoot, tall field buttercup, Ranunculus acris -n11720891 water crowfoot, water buttercup, Ranunculus aquatilis -n11721337 lesser celandine, pilewort, Ranunculus ficaria -n11721642 lesser spearwort, Ranunculus flammula -n11722036 greater spearwort, Ranunculus lingua -n11722342 western buttercup, Ranunculus occidentalis -n11722466 creeping buttercup, creeping crowfoot, Ranunculus repens -n11722621 cursed crowfoot, celery-leaved buttercup, Ranunculus sceleratus -n11722982 aconite -n11723227 monkshood, helmetflower, helmet flower, Aconitum napellus -n11723452 wolfsbane, wolfbane, wolf's bane, Aconitum lycoctonum -n11723770 baneberry, cohosh, herb Christopher -n11723986 baneberry -n11724109 red baneberry, redberry, red-berry, snakeberry, Actaea rubra -n11724660 pheasant's-eye, Adonis annua -n11725015 anemone, windflower -n11725311 Alpine anemone, mountain anemone, Anemone tetonensis -n11725480 Canada anemone, Anemone Canadensis -n11725623 thimbleweed, Anemone cylindrica -n11725821 wood anemone, Anemone nemorosa -n11725973 wood anemone, snowdrop, Anemone quinquefolia -n11726145 longheaded thimbleweed, Anemone riparia -n11726269 snowdrop anemone, snowdrop windflower, Anemone sylvestris -n11726433 Virginia thimbleweed, Anemone virginiana -n11726707 rue anemone, Anemonella thalictroides -n11727091 columbine, aquilegia, aquilege -n11727358 meeting house, honeysuckle, Aquilegia canadensis -n11727540 blue columbine, Aquilegia caerulea, Aquilegia scopulorum calcarea -n11727738 granny's bonnets, Aquilegia vulgaris -n11728099 marsh marigold, kingcup, meadow bright, May blob, cowslip, water dragon, Caltha palustris -n11728769 American bugbane, summer cohosh, Cimicifuga americana -n11728945 black cohosh, black snakeroot, rattle-top, Cimicifuga racemosa -n11729142 fetid bugbane, foetid bugbane, Cimicifuga foetida -n11729478 clematis -n11729860 pine hyacinth, Clematis baldwinii, Viorna baldwinii -n11730015 blue jasmine, blue jessamine, curly clematis, marsh clematis, Clematis crispa -n11730458 golden clematis, Clematis tangutica -n11730602 scarlet clematis, Clematis texensis -n11730750 leather flower, Clematis versicolor -n11730933 leather flower, vase-fine, vase vine, Clematis viorna -n11731157 virgin's bower, old man's beard, devil's darning needle, Clematis virginiana -n11731659 purple clematis, purple virgin's bower, mountain clematis, Clematis verticillaris -n11732052 goldthread, golden thread, Coptis groenlandica, Coptis trifolia groenlandica -n11732567 rocket larkspur, Consolida ambigua, Delphinium ajacis -n11733054 delphinium -n11733312 larkspur -n11733548 winter aconite, Eranthis hyemalis -n11734493 lenten rose, black hellebore, Helleborus orientalis -n11734698 green hellebore, Helleborus viridis -n11735053 hepatica, liverleaf -n11735570 goldenseal, golden seal, yellow root, turmeric root, Hydrastis Canadensis -n11735977 false rue anemone, false rue, Isopyrum biternatum -n11736362 giant buttercup, Laccopetalum giganteum -n11736694 nigella -n11736851 love-in-a-mist, Nigella damascena -n11737009 fennel flower, Nigella hispanica -n11737125 black caraway, nutmeg flower, Roman coriander, Nigella sativa -n11737534 pasqueflower, pasque flower -n11738547 meadow rue -n11738997 false bugbane, Trautvetteria carolinensis -n11739365 globeflower, globe flower -n11739978 winter's bark, winter's bark tree, Drimys winteri -n11740414 pepper shrub, Pseudowintera colorata, Wintera colorata -n11741175 sweet gale, Scotch gale, Myrica gale -n11741350 wax myrtle -n11741575 bay myrtle, puckerbush, Myrica cerifera -n11741797 bayberry, candleberry, swamp candleberry, waxberry, Myrica pensylvanica -n11742310 sweet fern, Comptonia peregrina, Comptonia asplenifolia -n11742878 corkwood, corkwood tree, Leitneria floridana -n11744011 jointed rush, Juncus articulatus -n11744108 toad rush, Juncus bufonius -n11744471 slender rush, Juncus tenuis -n11745817 zebrawood, zebrawood tree -n11746600 Connarus guianensis -n11747468 legume, leguminous plant -n11748002 legume -n11748811 peanut -n11749112 granadilla tree, granadillo, Brya ebenus -n11749603 arariba, Centrolobium robustum -n11750173 tonka bean, coumara nut -n11750508 courbaril, Hymenaea courbaril -n11750989 melilotus, melilot, sweet clover -n11751765 darling pea, poison bush -n11751974 smooth darling pea, Swainsona galegifolia -n11752578 clover, trefoil -n11752798 alpine clover, Trifolium alpinum -n11752937 hop clover, shamrock, lesser yellow trefoil, Trifolium dubium -n11753143 crimson clover, Italian clover, Trifolium incarnatum -n11753355 red clover, purple clover, Trifolium pratense -n11753562 buffalo clover, Trifolium reflexum, Trifolium stoloniferum -n11753700 white clover, dutch clover, shamrock, Trifolium repens -n11754893 mimosa -n11756092 acacia -n11756329 shittah, shittah tree -n11756669 wattle -n11756870 black wattle, Acacia auriculiformis -n11757017 gidgee, stinking wattle, Acacia cambegei -n11757190 catechu, Jerusalem thorn, Acacia catechu -n11757653 silver wattle, mimosa, Acacia dealbata -n11757851 huisache, cassie, mimosa bush, sweet wattle, sweet acacia, scented wattle, flame tree, Acacia farnesiana -n11758122 lightwood, Acacia melanoxylon -n11758276 golden wattle, Acacia pycnantha -n11758483 fever tree, Acacia xanthophloea -n11758799 coralwood, coral-wood, red sandalwood, Barbados pride, peacock flower fence, Adenanthera pavonina -n11759224 albizzia, albizia -n11759404 silk tree, Albizia julibrissin, Albizzia julibrissin -n11759609 siris, siris tree, Albizia lebbeck, Albizzia lebbeck -n11759853 rain tree, saman, monkeypod, monkey pod, zaman, zamang, Albizia saman -n11760785 calliandra -n11761202 conacaste, elephant's ear, Enterolobium cyclocarpa -n11761650 inga -n11761836 ice-cream bean, Inga edulis -n11762018 guama, Inga laurina -n11762433 lead tree, white popinac, Leucaena glauca, Leucaena leucocephala -n11762927 wild tamarind, Lysiloma latisiliqua, Lysiloma bahamensis -n11763142 sabicu, Lysiloma sabicu -n11763625 nitta tree -n11763874 Parkia javanica -n11764478 manila tamarind, camachile, huamachil, wild tamarind, Pithecellobium dulce -n11764814 cat's-claw, catclaw, black bead, Pithecellodium unguis-cati -n11765568 honey mesquite, Western honey mesquite, Prosopis glandulosa -n11766046 algarroba, algarrobilla, algarobilla -n11766189 screw bean, screwbean, tornillo, screwbean mesquite, Prosopis pubescens -n11766432 screw bean -n11767354 dogbane -n11767877 Indian hemp, rheumatism weed, Apocynum cannabinum -n11768816 bushman's poison, ordeal tree, Acocanthera oppositifolia, Acocanthera venenata -n11769176 impala lily, mock azalia, desert rose, kudu lily, Adenium obesum, Adenium multiflorum -n11769621 allamanda -n11769803 common allamanda, golden trumpet, Allamanda cathartica -n11770256 dita, dita bark, devil tree, Alstonia scholaris -n11771147 Nepal trumpet flower, Easter lily vine, Beaumontia grandiflora -n11771539 carissa -n11771746 hedge thorn, natal plum, Carissa bispinosa -n11771924 natal plum, amatungulu, Carissa macrocarpa, Carissa grandiflora -n11772408 periwinkle, rose periwinkle, Madagascar periwinkle, old maid, Cape periwinkle, red periwinkle, cayenne jasmine, Catharanthus roseus, Vinca rosea -n11772879 ivory tree, conessi, kurchi, kurchee, Holarrhena pubescens, Holarrhena antidysenterica -n11773408 white dipladenia, Mandevilla boliviensis, Dipladenia boliviensis -n11773628 Chilean jasmine, Mandevilla laxa -n11773987 oleander, rose bay, Nerium oleander -n11774513 frangipani, frangipanni -n11774972 West Indian jasmine, pagoda tree, Plumeria alba -n11775340 rauwolfia, rauvolfia -n11775626 snakewood, Rauwolfia serpentina -n11776234 Strophanthus kombe -n11777080 yellow oleander, Thevetia peruviana, Thevetia neriifolia -n11778092 myrtle, Vinca minor -n11778257 large periwinkle, Vinca major -n11779300 arum, aroid -n11780148 cuckoopint, lords-and-ladies, jack-in-the-pulpit, Arum maculatum -n11780424 black calla, Arum palaestinum -n11781176 calamus -n11782036 alocasia, elephant's ear, elephant ear -n11782266 giant taro, Alocasia macrorrhiza -n11782761 amorphophallus -n11782878 pungapung, telingo potato, elephant yam, Amorphophallus paeonifolius, Amorphophallus campanulatus -n11783162 devil's tongue, snake palm, umbrella arum, Amorphophallus rivieri -n11783920 anthurium, tailflower, tail-flower -n11784126 flamingo flower, flamingo plant, Anthurium andraeanum, Anthurium scherzerianum -n11784497 jack-in-the-pulpit, Indian turnip, wake-robin, Arisaema triphyllum, Arisaema atrorubens -n11785276 friar's-cowl, Arisarum vulgare -n11785668 caladium -n11785875 Caladium bicolor -n11786131 wild calla, water arum, Calla palustris -n11786539 taro, taro plant, dalo, dasheen, Colocasia esculenta -n11786843 taro, cocoyam, dasheen, eddo -n11787190 cryptocoryne, water trumpet -n11788039 dracontium -n11788727 golden pothos, pothos, ivy arum, Epipremnum aureum, Scindapsus aureus -n11789066 skunk cabbage, Lysichiton americanum -n11789438 monstera -n11789589 ceriman, Monstera deliciosa -n11789962 nephthytis -n11790089 Nephthytis afzelii -n11790788 arrow arum -n11790936 green arrow arum, tuckahoe, Peltandra virginica -n11791341 philodendron -n11791569 pistia, water lettuce, water cabbage, Pistia stratiotes, Pistia stratoites -n11792029 pothos -n11792341 spathiphyllum, peace lily, spathe flower -n11792742 skunk cabbage, polecat weed, foetid pothos, Symplocarpus foetidus -n11793403 yautia, tannia, spoonflower, malanga, Xanthosoma sagittifolium, Xanthosoma atrovirens -n11793779 calla lily, calla, arum lily, Zantedeschia aethiopica -n11794024 pink calla, Zantedeschia rehmanii -n11794139 golden calla -n11794519 duckweed -n11795049 common duckweed, lesser duckweed, Lemna minor -n11795216 star-duckweed, Lemna trisulca -n11795580 great duckweed, water flaxseed, Spirodela polyrrhiza -n11796005 watermeal -n11796188 common wolffia, Wolffia columbiana -n11797321 aralia -n11797508 American angelica tree, devil's walking stick, Hercules'-club, Aralia spinosa -n11797981 American spikenard, petty morel, life-of-man, Aralia racemosa -n11798270 bristly sarsaparilla, bristly sarsparilla, dwarf elder, Aralia hispida -n11798496 Japanese angelica tree, Aralia elata -n11798688 Chinese angelica, Chinese angelica tree, Aralia stipulata -n11798978 ivy, common ivy, English ivy, Hedera helix -n11799331 puka, Meryta sinclairii -n11799732 ginseng, nin-sin, Panax ginseng, Panax schinseng, Panax pseudoginseng -n11800236 ginseng -n11800565 umbrella tree, Schefflera actinophylla, Brassaia actinophylla -n11801392 birthwort, Aristolochia clematitis -n11801665 Dutchman's-pipe, pipe vine, Aristolochia macrophylla, Aristolochia durior -n11801891 Virginia snakeroot, Virginia serpentaria, Virginia serpentary, Aristolochia serpentaria -n11802410 Canada ginger, black snakeroot, Asarum canadense -n11802586 heartleaf, heart-leaf, Asarum virginicum -n11802800 heartleaf, heart-leaf, Asarum shuttleworthii -n11802995 asarabacca, Asarum europaeum -n11805255 caryophyllaceous plant -n11805544 corn cockle, corn campion, crown-of-the-field, Agrostemma githago -n11805956 sandwort -n11806219 mountain sandwort, mountain starwort, mountain daisy, Arenaria groenlandica -n11806369 pine-barren sandwort, longroot, Arenaria caroliniana -n11806521 seabeach sandwort, Arenaria peploides -n11806679 rock sandwort, Arenaria stricta -n11806814 thyme-leaved sandwort, Arenaria serpyllifolia -n11807108 mouse-ear chickweed, mouse eared chickweed, mouse ear, clammy chickweed, chickweed -n11807525 snow-in-summer, love-in-a-mist, Cerastium tomentosum -n11807696 Alpine mouse-ear, Arctic mouse-ear, Cerastium alpinum -n11807979 pink, garden pink -n11808299 sweet William, Dianthus barbatus -n11808468 carnation, clove pink, gillyflower, Dianthus caryophyllus -n11808721 china pink, rainbow pink, Dianthus chinensis -n11808932 Japanese pink, Dianthus chinensis heddewigii -n11809094 maiden pink, Dianthus deltoides -n11809271 cheddar pink, Diangus gratianopolitanus -n11809437 button pink, Dianthus latifolius -n11809594 cottage pink, grass pink, Dianthus plumarius -n11809754 fringed pink, Dianthus supurbus -n11810030 drypis -n11810358 baby's breath, babies'-breath, Gypsophila paniculata -n11811059 coral necklace, Illecebrum verticullatum -n11811473 lychnis, catchfly -n11811706 ragged robin, cuckoo flower, Lychnis flos-cuculi, Lychins floscuculi -n11811921 scarlet lychnis, maltese cross, Lychins chalcedonica -n11812094 mullein pink, rose campion, gardener's delight, dusty miller, Lychnis coronaria -n11812910 sandwort, Moehringia lateriflora -n11813077 sandwort, Moehringia mucosa -n11814584 soapwort, hedge pink, bouncing Bet, bouncing Bess, Saponaria officinalis -n11814996 knawel, knawe, Scleranthus annuus -n11815491 silene, campion, catchfly -n11815721 moss campion, Silene acaulis -n11815918 wild pink, Silene caroliniana -n11816121 red campion, red bird's eye, Silene dioica, Lychnis dioica -n11816336 white campion, evening lychnis, white cockle, bladder campion, Silene latifolia, Lychnis alba -n11816649 fire pink, Silene virginica -n11816829 bladder campion, Silene uniflora, Silene vulgaris -n11817160 corn spurry, corn spurrey, Spergula arvensis -n11817501 sand spurry, sea spurry, Spergularia rubra -n11817914 chickweed -n11818069 common chickweed, Stellaria media -n11818636 cowherb, cow cockle, Vaccaria hispanica, Vaccaria pyramidata, Saponaria vaccaria -n11819509 Hottentot fig, Hottentot's fig, sour fig, Carpobrotus edulis, Mesembryanthemum edule -n11819912 livingstone daisy, Dorotheanthus bellidiformis -n11820965 fig marigold, pebble plant -n11821184 ice plant, icicle plant, Mesembryanthemum crystallinum -n11822300 New Zealand spinach, Tetragonia tetragonioides, Tetragonia expansa -n11823043 amaranth -n11823305 amaranth -n11823436 tumbleweed, Amaranthus albus, Amaranthus graecizans -n11823756 prince's-feather, gentleman's-cane, prince's-plume, red amaranth, purple amaranth, Amaranthus cruentus, Amaranthus hybridus hypochondriacus, Amaranthus hybridus erythrostachys -n11824146 pigweed, Amaranthus hypochondriacus -n11824344 thorny amaranth, Amaranthus spinosus -n11824747 alligator weed, alligator grass, Alternanthera philoxeroides -n11825351 cockscomb, common cockscomb, Celosia cristata, Celosia argentea cristata -n11825749 cottonweed -n11826198 globe amaranth, bachelor's button, Gomphrena globosa -n11826569 bloodleaf -n11827541 saltwort, Batis maritima -n11828577 lamb's-quarters, pigweed, wild spinach, Chenopodium album -n11828973 good-king-henry, allgood, fat hen, wild spinach, Chenopodium bonus-henricus -n11829205 Jerusalem oak, feather geranium, Mexican tea, Chenopodium botrys, Atriplex mexicana -n11829672 oak-leaved goosefoot, oakleaf goosefoot, Chenopodium glaucum -n11829922 sowbane, red goosefoot, Chenopodium hybridum -n11830045 nettle-leaved goosefoot, nettleleaf goosefoot, Chenopodium murale -n11830252 red goosefoot, French spinach, Chenopodium rubrum -n11830400 stinking goosefoot, Chenopodium vulvaria -n11830714 orach, orache -n11830906 saltbush -n11831100 garden orache, mountain spinach, Atriplex hortensis -n11831297 desert holly, Atriplex hymenelytra -n11831521 quail bush, quail brush, white thistle, Atriplex lentiformis -n11832214 beet, common beet, Beta vulgaris -n11832480 beetroot, Beta vulgaris rubra -n11832671 chard, Swiss chard, spinach beet, leaf beet, chard plant, Beta vulgaris cicla -n11832899 mangel-wurzel, mangold-wurzel, mangold, Beta vulgaris vulgaris -n11833373 winged pigweed, tumbleweed, Cycloloma atriplicifolium -n11833749 halogeton, Halogeton glomeratus -n11834272 glasswort, samphire, Salicornia europaea -n11834654 saltwort, barilla, glasswort, kali, kelpwort, Salsola kali, Salsola soda -n11834890 Russian thistle, Russian tumbleweed, Russian cactus, tumbleweed, Salsola kali tenuifolia -n11835251 greasewood, black greasewood, Sarcobatus vermiculatus -n11836327 scarlet musk flower, Nyctaginia capitata -n11836722 sand verbena -n11837204 sweet sand verbena, Abronia fragrans -n11837351 yellow sand verbena, Abronia latifolia -n11837562 beach pancake, Abronia maritima -n11837743 beach sand verbena, pink sand verbena, Abronia umbellata -n11837970 desert sand verbena, Abronia villosa -n11838413 trailing four o'clock, trailing windmills, Allionia incarnata -n11838916 bougainvillea -n11839460 umbrellawort -n11839568 four o'clock -n11839823 common four-o'clock, marvel-of-Peru, Mirabilis jalapa, Mirabilis uniflora -n11840067 California four o'clock, Mirabilis laevis, Mirabilis californica -n11840246 sweet four o'clock, maravilla, Mirabilis longiflora -n11840476 desert four o'clock, Colorado four o'clock, maravilla, Mirabilis multiflora -n11840764 mountain four o'clock, Mirabilis oblongifolia -n11841247 cockspur, Pisonia aculeata -n11843441 rattail cactus, rat's-tail cactus, Aporocactus flagelliformis -n11844371 saguaro, sahuaro, Carnegiea gigantea -n11844892 night-blooming cereus -n11845557 echinocactus, barrel cactus -n11845793 hedgehog cactus -n11845913 golden barrel cactus, Echinocactus grusonii -n11846312 hedgehog cereus -n11846425 rainbow cactus -n11846765 epiphyllum, orchid cactus -n11847169 barrel cactus -n11848479 night-blooming cereus -n11848867 chichipe, Lemaireocereus chichipe -n11849271 mescal, mezcal, peyote, Lophophora williamsii -n11849467 mescal button, sacred mushroom, magic mushroom -n11849871 mammillaria -n11849983 feather ball, Mammillaria plumosa -n11850521 garambulla, garambulla cactus, Myrtillocactus geometrizans -n11850918 Knowlton's cactus, Pediocactus knowltonii -n11851258 nopal -n11851578 prickly pear, prickly pear cactus -n11851839 cholla, Opuntia cholla -n11852028 nopal, Opuntia lindheimeri -n11852148 tuna, Opuntia tuna -n11852531 Barbados gooseberry, Barbados-gooseberry vine, Pereskia aculeata -n11853079 mistletoe cactus -n11853356 Christmas cactus, Schlumbergera buckleyi, Schlumbergera baridgesii -n11853813 night-blooming cereus -n11854479 crab cactus, Thanksgiving cactus, Zygocactus truncatus, Schlumbergera truncatus -n11855274 pokeweed -n11855435 Indian poke, Phytolacca acinosa -n11855553 poke, pigeon berry, garget, scoke, Phytolacca americana -n11855842 ombu, bella sombra, Phytolacca dioica -n11856573 bloodberry, blood berry, rougeberry, rouge plant, Rivina humilis -n11857696 portulaca -n11857875 rose moss, sun plant, Portulaca grandiflora -n11858077 common purslane, pussley, pussly, verdolagas, Portulaca oleracea -n11858703 rock purslane -n11858814 red maids, redmaids, Calandrinia ciliata -n11859275 Carolina spring beauty, Claytonia caroliniana -n11859472 spring beauty, Clatonia lanceolata -n11859737 Virginia spring beauty, Claytonia virginica -n11860208 siskiyou lewisia, Lewisia cotyledon -n11860555 bitterroot, Lewisia rediviva -n11861238 broad-leaved montia, Montia cordifolia -n11861487 blinks, blinking chickweed, water chickweed, Montia lamprosperma -n11861641 toad lily, Montia chamissoi -n11861853 winter purslane, miner's lettuce, Cuban spinach, Montia perfoliata -n11862835 flame flower, flame-flower, flameflower, Talinum aurantiacum -n11863467 pigmy talinum, Talinum brevifolium -n11863877 jewels-of-opar, Talinum paniculatum -n11865071 caper -n11865276 native pomegranate, Capparis arborea -n11865429 caper tree, Jamaica caper tree, Capparis cynophallophora -n11865574 caper tree, bay-leaved caper, Capparis flexuosa -n11865874 common caper, Capparis spinosa -n11866248 spiderflower, cleome -n11866706 Rocky Mountain bee plant, stinking clover, Cleome serrulata -n11867311 clammyweed, Polanisia graveolens, Polanisia dodecandra -n11868814 crucifer, cruciferous plant -n11869351 cress, cress plant -n11869689 watercress -n11870044 stonecress, stone cress -n11870418 garlic mustard, hedge garlic, sauce-alone, jack-by-the-hedge, Alliaria officinalis -n11870747 alyssum, madwort -n11871059 rose of Jericho, resurrection plant, Anastatica hierochuntica -n11871496 Arabidopsis thaliana, mouse-ear cress -n11871748 Arabidopsis lyrata -n11872146 rock cress, rockcress -n11872324 sicklepod, Arabis Canadensis -n11872658 tower mustard, tower cress, Turritis glabra, Arabis glabra -n11873182 horseradish, horseradish root -n11873612 winter cress, St. Barbara's herb, scurvy grass -n11874081 yellow rocket, rockcress, rocket cress, Barbarea vulgaris, Sisymbrium barbarea -n11874423 hoary alison, hoary alyssum, Berteroa incana -n11874878 buckler mustard, Biscutalla laevigata -n11875523 wild cabbage, Brassica oleracea -n11875691 cabbage, cultivated cabbage, Brassica oleracea -n11875938 head cabbage, head cabbage plant, Brassica oleracea capitata -n11876204 savoy cabbage -n11876432 brussels sprout, Brassica oleracea gemmifera -n11876634 cauliflower, Brassica oleracea botrytis -n11876803 broccoli, Brassica oleracea italica -n11877193 collard -n11877283 kohlrabi, Brassica oleracea gongylodes -n11877473 turnip plant -n11877646 turnip, white turnip, Brassica rapa -n11877860 rutabaga, turnip cabbage, swede, Swedish turnip, rutabaga plant, Brassica napus napobrassica -n11878101 broccoli raab, broccoli rabe, Brassica rapa ruvo -n11878283 mustard -n11878633 chinese mustard, indian mustard, leaf mustard, gai choi, Brassica juncea -n11879054 bok choy, bok choi, pakchoi, pak choi, Chinese white cabbage, Brassica rapa chinensis -n11879722 rape, colza, Brassica napus -n11879895 rapeseed -n11881189 shepherd's purse, shepherd's pouch, Capsella bursa-pastoris -n11882074 lady's smock, cuckooflower, cuckoo flower, meadow cress, Cardamine pratensis -n11882237 coral-root bittercress, coralroot, coralwort, Cardamine bulbifera, Dentaria bulbifera -n11882426 crinkleroot, crinkle-root, crinkle root, pepper root, toothwort, Cardamine diphylla, Dentaria diphylla -n11882636 American watercress, mountain watercress, Cardamine rotundifolia -n11882821 spring cress, Cardamine bulbosa -n11882972 purple cress, Cardamine douglasii -n11883328 wallflower, Cheiranthus cheiri, Erysimum cheiri -n11883628 prairie rocket -n11883945 scurvy grass, common scurvy grass, Cochlearia officinalis -n11884384 sea kale, sea cole, Crambe maritima -n11884967 tansy mustard, Descurainia pinnata -n11885856 draba -n11887119 wallflower -n11887310 prairie rocket -n11887476 Siberian wall flower, Erysimum allionii, Cheiranthus allionii -n11887750 western wall flower, Erysimum asperum, Cheiranthus asperus, Erysimum arkansanum -n11888061 wormseed mustard, Erysimum cheiranthoides -n11888424 heliophila -n11888800 damask violet, Dame's violet, sweet rocket, Hesperis matronalis -n11889205 tansy-leaved rocket, Hugueninia tanacetifolia, Sisymbrium tanacetifolia -n11889619 candytuft -n11890022 woad -n11890150 dyer's woad, Isatis tinctoria -n11890884 bladderpod -n11891175 sweet alyssum, sweet alison, Lobularia maritima -n11892029 Malcolm stock, stock -n11892181 Virginian stock, Virginia stock, Malcolmia maritima -n11892637 stock, gillyflower -n11892817 brompton stock, Matthiola incana -n11893640 bladderpod -n11893916 chamois cress, Pritzelago alpina, Lepidium alpina -n11894327 radish plant, radish -n11894558 jointed charlock, wild radish, wild rape, runch, Raphanus raphanistrum -n11894770 radish, Raphanus sativus -n11895092 radish, daikon, Japanese radish, Raphanus sativus longipinnatus -n11895472 marsh cress, yellow watercress, Rorippa islandica -n11895714 great yellowcress, Rorippa amphibia, Nasturtium amphibium -n11896141 schizopetalon, Schizopetalon walkeri -n11896722 field mustard, wild mustard, charlock, chadlock, Brassica kaber, Sinapis arvensis -n11897116 hedge mustard, Sisymbrium officinale -n11897466 desert plume, prince's-plume, Stanleya pinnata, Cleome pinnata -n11898639 pennycress -n11898775 field pennycress, French weed, fanweed, penny grass, stinkweed, mithridate mustard, Thlaspi arvense -n11899223 fringepod, lacepod -n11899762 bladderpod -n11899921 wasabi -n11900569 poppy -n11901294 Iceland poppy, Papaver alpinum -n11901452 western poppy, Papaver californicum -n11901597 prickly poppy, Papaver argemone -n11901759 Iceland poppy, arctic poppy, Papaver nudicaule -n11901977 oriental poppy, Papaver orientale -n11902200 corn poppy, field poppy, Flanders poppy, Papaver rhoeas -n11902389 opium poppy, Papaver somniferum -n11902709 prickly poppy, argemone, white thistle, devil's fig -n11902982 Mexican poppy, Argemone mexicana -n11903333 bocconia, tree celandine, Bocconia frutescens -n11903671 celandine, greater celandine, swallowwort, swallow wort, Chelidonium majus -n11904109 corydalis -n11904274 climbing corydalis, Corydalis claviculata, Fumaria claviculata -n11905392 California poppy, Eschscholtzia californica -n11905749 horn poppy, horned poppy, yellow horned poppy, sea poppy, Glaucium flavum -n11906127 golden cup, Mexican tulip poppy, Hunnemania fumariifolia -n11906514 plume poppy, bocconia, Macleaya cordata -n11906917 blue poppy, Meconopsis betonicifolia -n11907100 Welsh poppy, Meconopsis cambrica -n11907405 creamcups, Platystemon californicus -n11907689 matilija poppy, California tree poppy, Romneya coulteri -n11908549 wind poppy, flaming poppy, Stylomecon heterophyllum, Papaver heterophyllum -n11908846 celandine poppy, wood poppy, Stylophorum diphyllum -n11909864 climbing fumitory, Allegheny vine, Adlumia fungosa, Fumaria fungosa -n11910271 bleeding heart, lyreflower, lyre-flower, Dicentra spectabilis -n11910460 Dutchman's breeches, Dicentra cucullaria -n11910666 squirrel corn, Dicentra canadensis -n11915214 composite, composite plant -n11915658 compass plant, compass flower -n11915899 everlasting, everlasting flower -n11916467 achillea -n11916696 yarrow, milfoil, Achillea millefolium -n11917407 pink-and-white everlasting, pink paper daisy, Acroclinium roseum -n11917835 white snakeroot, white sanicle, Ageratina altissima, Eupatorium rugosum -n11918286 ageratum -n11918473 common ageratum, Ageratum houstonianum -n11918808 sweet sultan, Amberboa moschata, Centaurea moschata -n11919447 ragweed, ambrosia, bitterweed -n11919761 common ragweed, Ambrosia artemisiifolia -n11919975 great ragweed, Ambrosia trifida -n11920133 western ragweed, perennial ragweed, Ambrosia psilostachya -n11920498 ammobium -n11920663 winged everlasting, Ammobium alatum -n11920998 pellitory, pellitory-of-Spain, Anacyclus pyrethrum -n11921395 pearly everlasting, cottonweed, Anaphalis margaritacea -n11921792 andryala -n11922661 plantain-leaved pussytoes -n11922755 field pussytoes -n11922839 solitary pussytoes -n11922926 mountain everlasting -n11923174 mayweed, dog fennel, stinking mayweed, stinking chamomile, Anthemis cotula -n11923397 yellow chamomile, golden marguerite, dyers' chamomile, Anthemis tinctoria -n11923637 corn chamomile, field chamomile, corn mayweed, Anthemis arvensis -n11924014 woolly daisy, dwarf daisy, Antheropeas wallacei, Eriophyllum wallacei -n11924445 burdock, clotbur -n11924849 great burdock, greater burdock, cocklebur, Arctium lappa -n11925303 African daisy -n11925450 blue-eyed African daisy, Arctotis stoechadifolia, Arctotis venusta -n11925898 marguerite, marguerite daisy, Paris daisy, Chrysanthemum frutescens, Argyranthemum frutescens -n11926365 silversword, Argyroxiphium sandwicense -n11926833 arnica -n11926976 heartleaf arnica, Arnica cordifolia -n11927215 Arnica montana -n11927740 lamb succory, dwarf nipplewort, Arnoseris minima -n11928352 artemisia -n11928858 mugwort -n11929743 sweet wormwood, Artemisia annua -n11930038 field wormwood, Artemisia campestris -n11930203 tarragon, estragon, Artemisia dracunculus -n11930353 sand sage, silvery wormwood, Artemisia filifolia -n11930571 wormwood sage, prairie sagewort, Artemisia frigida -n11930788 western mugwort, white sage, cudweed, prairie sage, Artemisia ludoviciana, Artemisia gnaphalodes -n11930994 Roman wormwood, Artemis pontica -n11931135 bud brush, bud sagebrush, Artemis spinescens -n11931540 common mugwort, Artemisia vulgaris -n11931918 aster -n11932745 wood aster -n11932927 whorled aster, Aster acuminatus -n11933099 heath aster, Aster arenosus -n11933257 heart-leaved aster, Aster cordifolius -n11933387 white wood aster, Aster divaricatus -n11933546 bushy aster, Aster dumosus -n11933728 heath aster, Aster ericoides -n11933903 white prairie aster, Aster falcatus -n11934041 stiff aster, Aster linarifolius -n11934239 goldilocks, goldilocks aster, Aster linosyris, Linosyris vulgaris -n11934463 large-leaved aster, Aster macrophyllus -n11934616 New England aster, Aster novae-angliae -n11934807 Michaelmas daisy, New York aster, Aster novi-belgii -n11935027 upland white aster, Aster ptarmicoides -n11935187 Short's aster, Aster shortii -n11935330 sea aster, sea starwort, Aster tripolium -n11935469 prairie aster, Aster turbinellis -n11935627 annual salt-marsh aster -n11935715 aromatic aster -n11935794 arrow leaved aster -n11935877 azure aster -n11935953 bog aster -n11936027 crooked-stemmed aster -n11936113 Eastern silvery aster -n11936199 flat-topped white aster -n11936287 late purple aster -n11936369 panicled aster -n11936448 perennial salt marsh aster -n11936539 purple-stemmed aster -n11936624 rough-leaved aster -n11936707 rush aster -n11936782 Schreiber's aster -n11936864 small white aster -n11936946 smooth aster -n11937023 southern aster -n11937102 starved aster, calico aster -n11937195 tradescant's aster -n11937278 wavy-leaved aster -n11937360 Western silvery aster -n11937446 willow aster -n11937692 ayapana, Ayapana triplinervis, Eupatorium aya-pana -n11938556 mule fat, Baccharis viminea -n11939180 balsamroot -n11939491 daisy -n11939699 common daisy, English daisy, Bellis perennis -n11940006 bur marigold, burr marigold, beggar-ticks, beggar's-ticks, sticktight -n11940349 Spanish needles, Bidens bipinnata -n11940599 tickseed sunflower, Bidens coronata, Bidens trichosperma -n11940750 European beggar-ticks, trifid beggar-ticks, trifid bur marigold, Bidens tripartita -n11941094 slender knapweed -n11941478 false chamomile -n11941924 Swan River daisy, Brachycome Iberidifolia -n11942659 woodland oxeye, Buphthalmum salicifolium -n11943133 Indian plantain -n11943407 calendula -n11943660 common marigold, pot marigold, ruddles, Scotch marigold, Calendula officinalis -n11943992 China aster, Callistephus chinensis -n11944196 thistle -n11944751 welted thistle, Carduus crispus -n11944954 musk thistle, nodding thistle, Carduus nutans -n11945367 carline thistle -n11945514 stemless carline thistle, Carlina acaulis -n11945783 common carline thistle, Carlina vulgaris -n11946051 safflower, false saffron, Carthamus tinctorius -n11946313 safflower seed -n11946727 catananche -n11946918 blue succory, cupid's dart, Catananche caerulea -n11947251 centaury -n11947629 dusty miller, Centaurea cineraria, Centaurea gymnocarpa -n11947802 cornflower, bachelor's button, bluebottle, Centaurea cyanus -n11948044 star-thistle, caltrop, Centauria calcitrapa -n11948264 knapweed -n11948469 sweet sultan, Centaurea imperialis -n11948864 great knapweed, greater knapweed, Centaurea scabiosa -n11949015 Barnaby's thistle, yellow star-thistle, Centaurea solstitialis -n11949402 chamomile, camomile, Chamaemelum nobilis, Anthemis nobilis -n11949857 chaenactis -n11950345 chrysanthemum -n11950686 corn marigold, field marigold, Chrysanthemum segetum -n11950877 crown daisy, Chrysanthemum coronarium -n11951052 chop-suey greens, tong ho, shun giku, Chrysanthemum coronarium spatiosum -n11951511 golden aster -n11951820 Maryland golden aster, Chrysopsis mariana -n11952346 goldenbush -n11952541 rabbit brush, rabbit bush, Chrysothamnus nauseosus -n11953038 chicory, succory, chicory plant, Cichorium intybus -n11953339 endive, witloof, Cichorium endivia -n11953610 chicory, chicory root -n11953884 plume thistle, plumed thistle -n11954161 Canada thistle, creeping thistle, Cirsium arvense -n11954345 field thistle, Cirsium discolor -n11954484 woolly thistle, Cirsium flodmanii -n11954642 European woolly thistle, Cirsium eriophorum -n11954798 melancholy thistle, Cirsium heterophylum, Cirsium helenioides -n11955040 brook thistle, Cirsium rivulare -n11955153 bull thistle, boar thistle, spear thistle, Cirsium vulgare, Cirsium lanceolatum -n11955532 blessed thistle, sweet sultan, Cnicus benedictus -n11955896 mistflower, mist-flower, ageratum, Conoclinium coelestinum, Eupatorium coelestinum -n11956348 horseweed, Canadian fleabane, fleabane, Conyza canadensis, Erigeron canadensis -n11956850 coreopsis, tickseed, tickweed, tick-weed -n11957317 giant coreopsis, Coreopsis gigantea -n11957514 sea dahlia, Coreopsis maritima -n11957678 calliopsis, Coreopsis tinctoria -n11958080 cosmos, cosmea -n11958499 brass buttons, Cotula coronopifolia -n11958888 billy buttons -n11959259 hawk's-beard, hawk's-beards -n11959632 artichoke, globe artichoke, artichoke plant, Cynara scolymus -n11959862 cardoon, Cynara cardunculus -n11960245 dahlia, Dahlia pinnata -n11960673 German ivy, Delairea odorata, Senecio milkanioides -n11961100 florist's chrysanthemum, florists' chrysanthemum, mum, Dendranthema grandifloruom, Chrysanthemum morifolium -n11961446 cape marigold, sun marigold, star of the veldt -n11961871 leopard's-bane, leopardbane -n11962272 coneflower -n11962667 globe thistle -n11962994 elephant's-foot -n11963572 tassel flower, Emilia sagitta -n11963932 brittlebush, brittle bush, incienso, Encelia farinosa -n11964446 sunray, Enceliopsis nudicaulis -n11964848 engelmannia -n11965218 fireweed, Erechtites hieracifolia -n11965627 fleabane -n11965962 blue fleabane, Erigeron acer -n11966083 daisy fleabane, Erigeron annuus -n11966215 orange daisy, orange fleabane, Erigeron aurantiacus -n11966385 spreading fleabane, Erigeron divergens -n11966617 seaside daisy, beach aster, Erigeron glaucous -n11966896 Philadelphia fleabane, Erigeron philadelphicus -n11967142 robin's plantain, Erigeron pulchellus -n11967315 showy daisy, Erigeron speciosus -n11967744 woolly sunflower -n11967878 golden yarrow, Eriophyllum lanatum -n11968519 dog fennel, Eupatorium capillifolium -n11968704 Joe-Pye weed, spotted Joe-Pye weed, Eupatorium maculatum -n11968931 boneset, agueweed, thoroughwort, Eupatorium perfoliatum -n11969166 Joe-Pye weed, purple boneset, trumpet weed, marsh milkweed, Eupatorium purpureum -n11969607 blue daisy, blue marguerite, Felicia amelloides -n11969806 kingfisher daisy, Felicia bergeriana -n11970101 cotton rose, cudweed, filago -n11970298 herba impia, Filago germanica -n11970586 gaillardia -n11971248 gazania -n11971406 treasure flower, Gazania rigens -n11971783 African daisy -n11971927 Barberton daisy, Transvaal daisy, Gerbera jamesonii -n11972291 desert sunflower, Gerea canescens -n11972759 cudweed -n11972959 chafeweed, wood cudweed, Gnaphalium sylvaticum -n11973341 gumweed, gum plant, tarweed, rosinweed -n11973634 Grindelia robusta -n11973749 curlycup gumweed, Grindelia squarrosa -n11974373 little-head snakeweed, Gutierrezia microcephala -n11974557 rabbitweed, rabbit-weed, snakeweed, broom snakeweed, broom snakeroot, turpentine weed, Gutierrezia sarothrae -n11974888 broomweed, broom-weed, Gutierrezia texana -n11975254 velvet plant, purple velvet plant, royal velvet plant, Gynura aurantiaca -n11976170 goldenbush -n11976314 camphor daisy, Haplopappus phyllocephalus -n11976511 yellow spiny daisy, Haplopappus spinulosus -n11976933 hoary golden bush, Hazardia cana -n11977303 sneezeweed -n11977660 orange sneezeweed, owlclaws, Helenium hoopesii -n11977887 rosilla, Helenium puberulum -n11978233 sunflower, helianthus -n11978551 swamp sunflower, Helianthus angustifolius -n11978713 common sunflower, mirasol, Helianthus annuus -n11978961 giant sunflower, tall sunflower, Indian potato, Helianthus giganteus -n11979187 showy sunflower, Helianthus laetiflorus -n11979354 Maximilian's sunflower, Helianthus maximilianii -n11979527 prairie sunflower, Helianthus petiolaris -n11979715 Jerusalem artichoke, girasol, Jerusalem artichoke sunflower, Helianthus tuberosus -n11979964 Jerusalem artichoke -n11980318 strawflower, golden everlasting, yellow paper daisy, Helichrysum bracteatum -n11980682 heliopsis, oxeye -n11981192 strawflower -n11981475 hairy golden aster, prairie golden aster, Heterotheca villosa, Chrysopsis villosa -n11982115 hawkweed -n11982545 rattlesnake weed, Hieracium venosum -n11982939 alpine coltsfoot, Homogyne alpina, Tussilago alpina -n11983375 alpine gold, alpine hulsea, Hulsea algida -n11983606 dwarf hulsea, Hulsea nana -n11984144 cat's-ear, California dandelion, capeweed, gosmore, Hypochaeris radicata -n11984542 inula -n11985053 marsh elder, iva -n11985321 burweed marsh elder, false ragweed, Iva xanthifolia -n11985739 krigia -n11985903 dwarf dandelion, Krigia dandelion, Krigia bulbosa -n11986511 garden lettuce, common lettuce, Lactuca sativa -n11986729 cos lettuce, romaine lettuce, Lactuca sativa longifolia -n11987126 leaf lettuce, Lactuca sativa crispa -n11987349 celtuce, stem lettuce, Lactuca sativa asparagina -n11987511 prickly lettuce, horse thistle, Lactuca serriola, Lactuca scariola -n11988132 goldfields, Lasthenia chrysostoma -n11988596 tidytips, tidy tips, Layia platyglossa -n11988893 hawkbit -n11989087 fall dandelion, arnica bud, Leontodon autumnalis -n11989393 edelweiss, Leontopodium alpinum -n11989869 oxeye daisy, ox-eyed daisy, marguerite, moon daisy, white daisy, Leucanthemum vulgare, Chrysanthemum leucanthemum -n11990167 oxeye daisy, Leucanthemum maximum, Chrysanthemum maximum -n11990313 shasta daisy, Leucanthemum superbum, Chrysanthemum maximum maximum -n11990627 Pyrenees daisy, Leucanthemum lacustre, Chrysanthemum lacustre -n11990920 north island edelweiss, Leucogenes leontopodium -n11991263 blazing star, button snakeroot, gayfeather, gay-feather, snakeroot -n11991549 dotted gayfeather, Liatris punctata -n11991777 dense blazing star, Liatris pycnostachya -n11992479 Texas star, Lindheimera texana -n11992806 African daisy, yellow ageratum, Lonas inodora, Lonas annua -n11993203 tahoka daisy, tansy leaf aster, Machaeranthera tanacetifolia -n11993444 sticky aster, Machaeranthera bigelovii -n11993675 Mojave aster, Machaeranthera tortifoloia -n11994150 tarweed -n11995092 sweet false chamomile, wild chamomile, German chamomile, Matricaria recutita, Matricaria chamomilla -n11995396 pineapple weed, rayless chamomile, Matricaria matricarioides -n11996251 climbing hempweed, climbing boneset, wild climbing hempweed, climbing hemp-vine, Mikania scandens -n11996677 mutisia -n11997032 rattlesnake root -n11997160 white lettuce, cankerweed, Nabalus alba, Prenanthes alba -n11997969 daisybush, daisy-bush, daisy bush -n11998492 New Zealand daisybush, Olearia haastii -n11998888 cotton thistle, woolly thistle, Scotch thistle, Onopordum acanthium, Onopordon acanthium -n11999278 othonna -n11999656 cascade everlasting, Ozothamnus secundiflorus, Helichrysum secundiflorum -n12000191 butterweed -n12001294 American feverfew, wild quinine, prairie dock, Parthenium integrifolium -n12001707 cineraria, Pericallis cruenta, Senecio cruentus -n12001924 florest's cineraria, Pericallis hybrida -n12002428 butterbur, bog rhubarb, Petasites hybridus, Petasites vulgaris -n12002651 winter heliotrope, sweet coltsfoot, Petasites fragrans -n12002826 sweet coltsfoot, Petasites sagitattus -n12003167 oxtongue, bristly oxtongue, bitterweed, bugloss, Picris echioides -n12003696 hawkweed -n12004120 mouse-ear hawkweed, Pilosella officinarum, Hieracium pilocella -n12004547 stevia -n12004987 rattlesnake root, Prenanthes purpurea -n12005656 fleabane, feabane mullet, Pulicaria dysenterica -n12006306 sheep plant, vegetable sheep, Raoulia lutescens, Raoulia australis -n12006766 coneflower -n12006930 Mexican hat, Ratibida columnaris -n12007196 long-head coneflower, prairie coneflower, Ratibida columnifera -n12007406 prairie coneflower, Ratibida tagetes -n12007766 Swan River everlasting, rhodanthe, Rhodanthe manglesii, Helipterum manglesii -n12008252 coneflower -n12008487 black-eyed Susan, Rudbeckia hirta, Rudbeckia serotina -n12008749 cutleaved coneflower, Rudbeckia laciniata -n12009047 golden glow, double gold, hortensia, Rudbeckia laciniata hortensia -n12009420 lavender cotton, Santolina chamaecyparissus -n12009792 creeping zinnia, Sanvitalia procumbens -n12010628 golden thistle -n12010815 Spanish oyster plant, Scolymus hispanicus -n12011370 nodding groundsel, Senecio bigelovii -n12011620 dusty miller, Senecio cineraria, Cineraria maritima -n12012111 butterweed, ragwort, Senecio glabellus -n12012253 ragwort, tansy ragwort, ragweed, benweed, Senecio jacobaea -n12012510 arrowleaf groundsel, Senecio triangularis -n12013035 black salsify, viper's grass, scorzonera, Scorzonera hispanica -n12013511 white-topped aster -n12013701 narrow-leaved white-topped aster -n12014085 silver sage, silver sagebrush, grey sage, gray sage, Seriphidium canum, Artemisia cana -n12014355 sea wormwood, Seriphidium maritimum, Artemisia maritima -n12014923 sawwort, Serratula tinctoria -n12015221 rosinweed, Silphium laciniatum -n12015525 milk thistle, lady's thistle, Our Lady's mild thistle, holy thistle, blessed thistle, Silybum marianum -n12015959 goldenrod -n12016434 silverrod, Solidago bicolor -n12016567 meadow goldenrod, Canadian goldenrod, Solidago canadensis -n12016777 Missouri goldenrod, Solidago missouriensis -n12016914 alpine goldenrod, Solidago multiradiata -n12017127 grey goldenrod, gray goldenrod, Solidago nemoralis -n12017326 Blue Mountain tea, sweet goldenrod, Solidago odora -n12017511 dyer's weed, Solidago rugosa -n12017664 seaside goldenrod, beach goldenrod, Solidago sempervirens -n12017853 narrow goldenrod, Solidago spathulata -n12018014 Boott's goldenrod -n12018100 Elliott's goldenrod -n12018188 Ohio goldenrod -n12018271 rough-stemmed goldenrod -n12018363 showy goldenrod -n12018447 tall goldenrod -n12018530 zigzag goldenrod, broad leaved goldenrod -n12018760 sow thistle, milk thistle -n12019035 milkweed, Sonchus oleraceus -n12019827 stevia -n12020184 stokes' aster, cornflower aster, Stokesia laevis -n12020507 marigold -n12020736 African marigold, big marigold, Aztec marigold, Tagetes erecta -n12020941 French marigold, Tagetes patula -n12022054 painted daisy, pyrethrum, Tanacetum coccineum, Chrysanthemum coccineum -n12022382 pyrethrum, Dalmatian pyrethrum, Dalmatia pyrethrum, Tanacetum cinerariifolium, Chrysanthemum cinerariifolium -n12022821 northern dune tansy, Tanacetum douglasii -n12023108 feverfew, Tanacetum parthenium, Chrysanthemum parthenium -n12023407 dusty miller, silver-lace, silver lace, Tanacetum ptarmiciflorum, Chrysanthemum ptarmiciflorum -n12023726 tansy, golden buttons, scented fern, Tanacetum vulgare -n12024176 dandelion, blowball -n12024445 common dandelion, Taraxacum ruderalia, Taraxacum officinale -n12024690 dandelion green -n12024805 Russian dandelion, kok-saghyz, kok-sagyz, Taraxacum kok-saghyz -n12025220 stemless hymenoxys, Tetraneuris acaulis, Hymenoxys acaulis -n12026018 Mexican sunflower, tithonia -n12026476 Easter daisy, stemless daisy, Townsendia Exscapa -n12026981 yellow salsify, Tragopogon dubius -n12027222 salsify, oyster plant, vegetable oyster, Tragopogon porrifolius -n12027658 meadow salsify, goatsbeard, shepherd's clock, Tragopogon pratensis -n12028424 scentless camomile, scentless false camomile, scentless mayweed, scentless hayweed, corn mayweed, Tripleurospermum inodorum, Matricaria inodorum -n12029039 turfing daisy, Tripleurospermum tchihatchewii, Matricaria tchihatchewii -n12029635 coltsfoot, Tussilago farfara -n12030092 ursinia -n12030654 crownbeard, crown-beard, crown beard -n12030908 wingstem, golden ironweed, yellow ironweed, golden honey plant, Verbesina alternifolia, Actinomeris alternifolia -n12031139 cowpen daisy, golden crownbeard, golden crown beard, butter daisy, Verbesina encelioides, Ximenesia encelioides -n12031388 gravelweed, Verbesina helianthoides -n12031547 Virginia crownbeard, frostweed, frost-weed, Verbesina virginica -n12031927 ironweed, vernonia -n12032429 mule's ears, Wyethia amplexicaulis -n12032686 white-rayed mule's ears, Wyethia helianthoides -n12033139 cocklebur, cockle-bur, cockleburr, cockle-burr -n12033504 xeranthemum -n12033709 immortelle, Xeranthemum annuum -n12034141 zinnia, old maid, old maid flower -n12034384 white zinnia, Zinnia acerosa -n12034594 little golden zinnia, Zinnia grandiflora -n12035631 blazing star, Mentzelia livicaulis, Mentzelia laevicaulis -n12035907 bartonia, Mentzelia lindleyi -n12036067 achene -n12036226 samara, key fruit, key -n12036939 campanula, bellflower -n12037499 creeping bellflower, Campanula rapunculoides -n12037691 Canterbury bell, cup and saucer, Campanula medium -n12038038 tall bellflower, Campanula americana -n12038208 marsh bellflower, Campanula aparinoides -n12038406 clustered bellflower, Campanula glomerata -n12038585 peach bells, peach bell, willow bell, Campanula persicifolia -n12038760 chimney plant, chimney bellflower, Campanula pyramidalis -n12038898 rampion, rampion bellflower, Campanula rapunculus -n12039317 tussock bellflower, spreading bellflower, Campanula carpatica -n12041446 orchid, orchidaceous plant -n12043444 orchis -n12043673 male orchis, early purple orchid, Orchis mascula -n12043836 butterfly orchid, butterfly orchis, Orchis papilionaceae -n12044041 showy orchis, purple orchis, purple-hooded orchis, Orchis spectabilis -n12044467 aerides -n12044784 angrecum -n12045157 jewel orchid -n12045514 puttyroot, adam-and-eve, Aplectrum hyemale -n12045860 arethusa -n12046028 bog rose, wild pink, dragon's mouth, Arethusa bulbosa -n12046428 bletia -n12046815 Bletilla striata, Bletia striata -n12047345 brassavola -n12047884 spider orchid, Brassia lawrenceana -n12048056 spider orchid, Brassia verrucosa -n12048399 caladenia -n12048928 calanthe -n12049282 grass pink, Calopogon pulchellum, Calopogon tuberosum -n12049562 calypso, fairy-slipper, Calypso bulbosa -n12050533 cattleya -n12050959 helleborine -n12051103 red helleborine, Cephalanthera rubra -n12051514 spreading pogonia, funnel-crest rosebud orchid, Cleistes divaricata, Pogonia divaricata -n12051792 rosebud orchid, Cleistes rosea, Pogonia rosea -n12052267 satyr orchid, Coeloglossum bracteatum -n12052447 frog orchid, Coeloglossum viride -n12052787 coelogyne -n12053405 coral root -n12053690 spotted coral root, Corallorhiza maculata -n12053962 striped coral root, Corallorhiza striata -n12054195 early coral root, pale coral root, Corallorhiza trifida -n12055073 swan orchid, swanflower, swan-flower, swanneck, swan-neck -n12055516 cymbid, cymbidium -n12056099 cypripedia -n12056217 lady's slipper, lady-slipper, ladies' slipper, slipper orchid -n12056601 moccasin flower, nerveroot, Cypripedium acaule -n12056758 common lady's-slipper, showy lady's-slipper, showy lady slipper, Cypripedium reginae, Cypripedium album -n12056990 ram's-head, ram's-head lady's slipper, Cypripedium arietinum -n12057211 yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum -n12057447 large yellow lady's slipper, Cypripedium calceolus pubescens -n12057660 California lady's slipper, Cypripedium californicum -n12057895 clustered lady's slipper, Cypripedium fasciculatum -n12058192 mountain lady's slipper, Cypripedium montanum -n12058630 marsh orchid -n12058822 common spotted orchid, Dactylorhiza fuchsii, Dactylorhiza maculata fuchsii -n12059314 dendrobium -n12059625 disa -n12060546 phantom orchid, snow orchid, Eburophyton austinae -n12061104 tulip orchid, Encyclia citrina, Cattleya citrina -n12061380 butterfly orchid, Encyclia tampensis, Epidendrum tampense -n12061614 butterfly orchid, butterfly orchis, Epidendrum venosum, Encyclia venosa -n12062105 epidendron -n12062468 helleborine -n12062626 Epipactis helleborine -n12062781 stream orchid, chatterbox, giant helleborine, Epipactis gigantea -n12063211 tongueflower, tongue-flower -n12063639 rattlesnake plantain, helleborine -n12064389 fragrant orchid, Gymnadenia conopsea -n12064591 short-spurred fragrant orchid, Gymnadenia odoratissima -n12065316 fringed orchis, fringed orchid -n12065649 frog orchid -n12065777 rein orchid, rein orchis -n12066018 bog rein orchid, bog candles, Habenaria dilatata -n12066261 white fringed orchis, white fringed orchid, Habenaria albiflora -n12066451 elegant Habenaria, Habenaria elegans -n12066630 purple-fringed orchid, purple-fringed orchis, Habenaria fimbriata -n12066821 coastal rein orchid, Habenaria greenei -n12067029 Hooker's orchid, Habenaria hookeri -n12067193 ragged orchid, ragged orchis, ragged-fringed orchid, green fringed orchis, Habenaria lacera -n12067433 prairie orchid, prairie white-fringed orchis, Habenaria leucophaea -n12067672 snowy orchid, Habenaria nivea -n12067817 round-leaved rein orchid, Habenaria orbiculata -n12068138 purple fringeless orchid, purple fringeless orchis, Habenaria peramoena -n12068432 purple-fringed orchid, purple-fringed orchis, Habenaria psycodes -n12068615 Alaska rein orchid, Habenaria unalascensis -n12069009 crested coral root, Hexalectris spicata -n12069217 Texas purple spike, Hexalectris warnockii -n12069679 lizard orchid, Himantoglossum hircinum -n12070016 laelia -n12070381 liparis -n12070583 twayblade -n12070712 fen orchid, fen orchis, Liparis loeselii -n12071259 broad-leaved twayblade, Listera convallarioides -n12071477 lesser twayblade, Listera cordata -n12071744 twayblade, Listera ovata -n12072210 green adder's mouth, Malaxis-unifolia, Malaxis ophioglossoides -n12072722 masdevallia -n12073217 maxillaria -n12073554 pansy orchid -n12073991 odontoglossum -n12074408 oncidium, dancing lady orchid, butterfly plant, butterfly orchid -n12074867 bee orchid, Ophrys apifera -n12075010 fly orchid, Ophrys insectifera, Ophrys muscifera -n12075151 spider orchid -n12075299 early spider orchid, Ophrys sphegodes -n12075830 Venus' slipper, Venus's slipper, Venus's shoe -n12076223 phaius -n12076577 moth orchid, moth plant -n12076852 butterfly plant, Phalaenopsis amabilis -n12077244 rattlesnake orchid -n12077944 lesser butterfly orchid, Platanthera bifolia, Habenaria bifolia -n12078172 greater butterfly orchid, Platanthera chlorantha, Habenaria chlorantha -n12078451 prairie white-fringed orchid, Platanthera leucophea -n12078747 tangle orchid -n12079120 Indian crocus -n12079523 pleurothallis -n12079963 pogonia -n12080395 butterfly orchid -n12080588 Psychopsis krameriana, Oncidium papilio kramerianum -n12080820 Psychopsis papilio, Oncidium papilio -n12081215 helmet orchid, greenhood -n12081649 foxtail orchid -n12082131 orange-blossom orchid, Sarcochilus falcatus -n12083113 sobralia -n12083591 ladies' tresses, lady's tresses -n12083847 screw augur, Spiranthes cernua -n12084158 hooded ladies' tresses, Spiranthes romanzoffiana -n12084400 western ladies' tresses, Spiranthes porrifolia -n12084555 European ladies' tresses, Spiranthes spiralis -n12084890 stanhopea -n12085267 stelis -n12085664 fly orchid -n12086012 vanda -n12086192 blue orchid, Vanda caerulea -n12086539 vanilla -n12086778 vanilla orchid, Vanilla planifolia -n12087961 yam, yam plant -n12088223 yam -n12088327 white yam, water yam, Dioscorea alata -n12088495 cinnamon vine, Chinese yam, Dioscorea batata -n12088909 elephant's-foot, tortoise plant, Hottentot bread vine, Hottentot's bread vine, Dioscorea elephantipes -n12089320 wild yam, Dioscorea paniculata -n12089496 cush-cush, Dioscorea trifida -n12089846 black bryony, black bindweed, Tamus communis -n12090890 primrose, primula -n12091213 English primrose, Primula vulgaris -n12091377 cowslip, paigle, Primula veris -n12091550 oxlip, paigle, Primula elatior -n12091697 Chinese primrose, Primula sinensis -n12091953 polyanthus, Primula polyantha -n12092262 pimpernel -n12092417 scarlet pimpernel, red pimpernel, poor man's weatherglass, Anagallis arvensis -n12092629 bog pimpernel, Anagallis tenella -n12092930 chaffweed, bastard pimpernel, false pimpernel -n12093329 cyclamen, Cyclamen purpurascens -n12093600 sowbread, Cyclamen hederifolium, Cyclamen neopolitanum -n12093885 sea milkwort, sea trifoly, black saltwort, Glaux maritima -n12094244 featherfoil, feather-foil -n12094401 water gillyflower, American featherfoil, Hottonia inflata -n12094612 water violet, Hottonia palustris -n12095020 loosestrife -n12095281 gooseneck loosestrife, Lysimachia clethroides Duby -n12095412 yellow pimpernel, Lysimachia nemorum -n12095543 fringed loosestrife, Lysimachia ciliatum -n12095647 moneywort, creeping Jenny, creeping Charlie, Lysimachia nummularia -n12095934 swamp candles, Lysimachia terrestris -n12096089 whorled loosestrife, Lysimachia quadrifolia -n12096395 water pimpernel -n12096563 brookweed, Samolus valerandii -n12096674 brookweed, Samolus parviflorus, Samolus floribundus -n12097396 coralberry, spiceberry, Ardisia crenata -n12097556 marlberry, Ardisia escallonoides, Ardisia paniculata -n12098403 plumbago -n12098524 leadwort, Plumbago europaea -n12098827 thrift -n12099342 sea lavender, marsh rosemary, statice -n12100187 barbasco, joewood, Jacquinia keyensis -n12101870 gramineous plant, graminaceous plant -n12102133 grass -n12103680 midgrass -n12103894 shortgrass, short-grass -n12104104 sword grass -n12104238 tallgrass, tall-grass -n12104501 herbage, pasturage -n12104734 goat grass, Aegilops triuncalis -n12105125 wheatgrass, wheat-grass -n12105353 crested wheatgrass, crested wheat grass, fairway crested wheat grass, Agropyron cristatum -n12105828 bearded wheatgrass, Agropyron subsecundum -n12105981 western wheatgrass, bluestem wheatgrass, Agropyron smithii -n12106134 intermediate wheatgrass, Agropyron intermedium, Elymus hispidus -n12106323 slender wheatgrass, Agropyron trachycaulum, Agropyron pauciflorum, Elymus trachycaulos -n12107002 velvet bent, velvet bent grass, brown bent, Rhode Island bent, dog bent, Agrostis canina -n12107191 cloud grass, Agrostis nebulosa -n12107710 meadow foxtail, Alopecurus pratensis -n12107970 foxtail, foxtail grass -n12108432 broom grass -n12108613 broom sedge, Andropogon virginicus -n12108871 tall oat grass, tall meadow grass, evergreen grass, false oat, French rye, Arrhenatherum elatius -n12109365 toetoe, toitoi, Arundo conspicua, Chionochloa conspicua -n12109827 oat -n12110085 cereal oat, Avena sativa -n12110236 wild oat, wild oat grass, Avena fatua -n12110352 slender wild oat, Avena barbata -n12110475 wild red oat, animated oat, Avene sterilis -n12110778 brome, bromegrass -n12111238 chess, cheat, Bromus secalinus -n12111627 field brome, Bromus arvensis -n12112008 grama, grama grass, gramma, gramma grass -n12112337 black grama, Bouteloua eriopoda -n12112609 buffalo grass, Buchloe dactyloides -n12112918 reed grass -n12113195 feather reed grass, feathertop, Calamagrostis acutiflora -n12113323 Australian reed grass, Calamagrostic quadriseta -n12113657 burgrass, bur grass -n12114010 buffel grass, Cenchrus ciliaris, Pennisetum cenchroides -n12114590 Rhodes grass, Chloris gayana -n12115180 pampas grass, Cortaderia selloana -n12116058 giant star grass, Cynodon plectostachyum -n12116429 orchard grass, cocksfoot, cockspur, Dactylis glomerata -n12116734 Egyptian grass, crowfoot grass, Dactyloctenium aegypticum -n12117017 crabgrass, crab grass, finger grass -n12117235 smooth crabgrass, Digitaria ischaemum -n12117326 large crabgrass, hairy finger grass, Digitaria sanguinalis -n12117695 barnyard grass, barn grass, barn millet, Echinochloa crusgalli -n12117912 Japanese millet, billion-dollar grass, Japanese barnyard millet, sanwa millet, Echinochloa frumentacea -n12118414 yardgrass, yard grass, wire grass, goose grass, Eleusine indica -n12118661 finger millet, ragi, ragee, African millet, coracan, corakan, kurakkan, Eleusine coracana -n12119099 lyme grass -n12119238 wild rye -n12119390 giant ryegrass, Elymus condensatus, Leymus condensatus -n12119539 sea lyme grass, European dune grass, Elymus arenarius, Leymus arenaria -n12119717 Canada wild rye, Elymus canadensis -n12120347 teff, teff grass, Eragrostis tef, Eragrostic abyssinica -n12120578 weeping love grass, African love grass, Eragrostis curvula -n12121033 plume grass -n12121187 Ravenna grass, wool grass, Erianthus ravennae -n12121610 fescue, fescue grass, meadow fescue, Festuca elatior -n12122442 reed meadow grass, Glyceria grandis -n12122725 velvet grass, Yorkshire fog, Holcus lanatus -n12122918 creeping soft grass, Holcus mollis -n12123648 barleycorn -n12123741 barley grass, wall barley, Hordeum murinum -n12124172 little barley, Hordeum pusillum -n12124627 rye grass, ryegrass -n12124818 perennial ryegrass, English ryegrass, Lolium perenne -n12125001 Italian ryegrass, Italian rye, Lolium multiflorum -n12125183 darnel, tare, bearded darnel, cheat, Lolium temulentum -n12125584 nimblewill, nimble Will, Muhlenbergia schreberi -n12126084 cultivated rice, Oryza sativa -n12126360 ricegrass, rice grass -n12126736 smilo, smilo grass, Oryzopsis miliacea -n12127460 switch grass, Panicum virgatum -n12127575 broomcorn millet, hog millet, Panicum miliaceum -n12127768 goose grass, Texas millet, Panicum Texanum -n12128071 dallisgrass, dallis grass, paspalum, Paspalum dilatatum -n12128306 Bahia grass, Paspalum notatum -n12128490 knotgrass, Paspalum distichum -n12129134 fountain grass, Pennisetum ruppelii, Pennisetum setaceum -n12129738 reed canary grass, gardener's garters, lady's laces, ribbon grass, Phalaris arundinacea -n12129986 canary grass, birdseed grass, Phalaris canariensis -n12130549 timothy, herd's grass, Phleum pratense -n12131405 bluegrass, blue grass -n12131550 meadowgrass, meadow grass -n12132092 wood meadowgrass, Poa nemoralis, Agrostis alba -n12132956 noble cane -n12133151 munj, munja, Saccharum bengalense, Saccharum munja -n12133462 broom beard grass, prairie grass, wire grass, Andropogon scoparius, Schizachyrium scoparium -n12133682 bluestem, blue stem, Andropogon furcatus, Andropogon gerardii -n12134025 rye, Secale cereale -n12134486 bristlegrass, bristle grass -n12134695 giant foxtail -n12134836 yellow bristlegrass, yellow bristle grass, yellow foxtail, glaucous bristlegrass, Setaria glauca -n12135049 green bristlegrass, green foxtail, rough bristlegrass, bottle-grass, bottle grass, Setaria viridis -n12135576 Siberian millet, Setaria italica rubrofructa -n12135729 German millet, golden wonder millet, Setaria italica stramineofructa -n12135898 millet -n12136392 rattan, rattan cane -n12136581 malacca -n12136720 reed -n12137120 sorghum -n12137569 grain sorghum -n12137791 durra, doura, dourah, Egyptian corn, Indian millet, Guinea corn -n12137954 feterita, federita, Sorghum vulgare caudatum -n12138110 hegari -n12138248 kaoliang -n12138444 milo, milo maize -n12138578 shallu, Sorghum vulgare rosburghii -n12139196 broomcorn, Sorghum vulgare technicum -n12139575 cordgrass, cord grass -n12139793 salt reed grass, Spartina cynosuroides -n12139921 prairie cordgrass, freshwater cordgrass, slough grass, Spartina pectinmata -n12140511 smut grass, blackseed, carpet grass, Sporobolus poiretii -n12140759 sand dropseed, Sporobolus cryptandrus -n12140903 rush grass, rush-grass -n12141167 St. Augustine grass, Stenotaphrum secundatum, buffalo grass -n12141385 grain -n12141495 cereal, cereal grass -n12142085 wheat -n12142357 wheat berry -n12142450 durum, durum wheat, hard wheat, Triticum durum, Triticum turgidum, macaroni wheat -n12143065 spelt, Triticum spelta, Triticum aestivum spelta -n12143215 emmer, starch wheat, two-grain spelt, Triticum dicoccum -n12143405 wild wheat, wild emmer, Triticum dicoccum dicoccoides -n12143676 corn, maize, Indian corn, Zea mays -n12144313 mealie -n12144580 corn -n12144987 dent corn, Zea mays indentata -n12145148 flint corn, flint maize, Yankee corn, Zea mays indurata -n12145477 popcorn, Zea mays everta -n12146311 zoysia -n12146488 Manila grass, Japanese carpet grass, Zoysia matrella -n12146654 Korean lawn grass, Japanese lawn grass, Zoysia japonica -n12147226 bamboo -n12147835 common bamboo, Bambusa vulgaris -n12148757 giant bamboo, kyo-chiku, Dendrocalamus giganteus -n12150722 umbrella plant, umbrella sedge, Cyperus alternifolius -n12150969 chufa, yellow nutgrass, earth almond, ground almond, rush nut, Cyperus esculentus -n12151170 galingale, galangal, Cyperus longus -n12151615 nutgrass, nut grass, nutsedge, nut sedge, Cyperus rotundus -n12152031 sand sedge, sand reed, Carex arenaria -n12152251 cypress sedge, Carex pseudocyperus -n12152532 cotton grass, cotton rush -n12152722 common cotton grass, Eriophorum angustifolium -n12153033 hardstem bulrush, hardstemmed bulrush, Scirpus acutus -n12153224 wool grass, Scirpus cyperinus -n12153580 spike rush -n12153741 water chestnut, Chinese water chestnut, Eleocharis dulcis -n12153914 needle spike rush, needle rush, slender spike rush, hair grass, Eleocharis acicularis -n12154114 creeping spike rush, Eleocharis palustris -n12154773 pandanus, screw pine -n12155009 textile screw pine, lauhala, Pandanus tectorius -n12155583 cattail -n12155773 cat's-tail, bullrush, bulrush, nailrod, reed mace, reedmace, Typha latifolia -n12156679 bur reed -n12156819 grain, caryopsis -n12157056 kernel -n12157179 rye -n12157769 gourd, gourd vine -n12158031 gourd -n12158443 pumpkin, pumpkin vine, autumn pumpkin, Cucurbita pepo -n12158798 squash, squash vine -n12159055 summer squash, summer squash vine, Cucurbita pepo melopepo -n12159388 yellow squash -n12159555 marrow, marrow squash, vegetable marrow -n12159804 zucchini, courgette -n12159942 cocozelle, Italian vegetable marrow -n12160125 cymling, pattypan squash -n12160303 spaghetti squash -n12160490 winter squash, winter squash plant -n12160857 acorn squash -n12161056 hubbard squash, Cucurbita maxima -n12161285 turban squash, Cucurbita maxima turbaniformis -n12161577 buttercup squash -n12161744 butternut squash, Cucurbita maxima -n12161969 winter crookneck, winter crookneck squash, Cucurbita moschata -n12162181 cushaw, Cucurbita mixta, Cucurbita argyrosperma -n12162425 prairie gourd, prairie gourd vine, Missouri gourd, wild pumpkin, buffalo gourd, calabazilla, Cucurbita foetidissima -n12162758 prairie gourd -n12163035 bryony, briony -n12163279 white bryony, devil's turnip, Bryonia alba -n12164363 sweet melon, muskmelon, sweet melon vine, Cucumis melo -n12164656 cantaloupe, cantaloup, cantaloupe vine, cantaloup vine, Cucumis melo cantalupensis -n12164881 winter melon, Persian melon, honeydew melon, winter melon vine, Cucumis melo inodorus -n12165170 net melon, netted melon, nutmeg melon, Cucumis melo reticulatus -n12165384 cucumber, cucumber vine, Cucumis sativus -n12165758 squirting cucumber, exploding cucumber, touch-me-not, Ecballium elaterium -n12166128 bottle gourd, calabash, Lagenaria siceraria -n12166424 luffa, dishcloth gourd, sponge gourd, rag gourd, strainer vine -n12166793 loofah, vegetable sponge, Luffa cylindrica -n12166929 angled loofah, sing-kwa, Luffa acutangula -n12167075 loofa, loofah, luffa, loufah sponge -n12167436 balsam apple, Momordica balsamina -n12167602 balsam pear, Momordica charantia -n12168565 lobelia -n12169099 water lobelia, Lobelia dortmanna -n12170585 mallow -n12171098 musk mallow, mus rose, Malva moschata -n12171316 common mallow, Malva neglecta -n12171966 okra, gumbo, okra plant, lady's-finger, Abelmoschus esculentus, Hibiscus esculentus -n12172364 okra -n12172481 abelmosk, musk mallow, Abelmoschus moschatus, Hibiscus moschatus -n12172906 flowering maple -n12173069 velvetleaf, velvet-leaf, velvetweed, Indian mallow, butter-print, China jute, Abutilon theophrasti -n12173664 hollyhock -n12173912 rose mallow, Alcea rosea, Althea rosea -n12174311 althea, althaea, hollyhock -n12174521 marsh mallow, white mallow, Althea officinalis -n12174926 poppy mallow -n12175181 fringed poppy mallow, Callirhoe digitata -n12175370 purple poppy mallow, Callirhoe involucrata -n12175598 clustered poppy mallow, Callirhoe triangulata -n12176453 sea island cotton, tree cotton, Gossypium barbadense -n12176709 Levant cotton, Gossypium herbaceum -n12176953 upland cotton, Gossypium hirsutum -n12177129 Peruvian cotton, Gossypium peruvianum -n12177455 wild cotton, Arizona wild cotton, Gossypium thurberi -n12178129 kenaf, kanaf, deccan hemp, bimli, bimli hemp, Indian hemp, Bombay hemp, Hibiscus cannabinus -n12178780 sorrel tree, Hibiscus heterophyllus -n12178896 rose mallow, swamp mallow, common rose mallow, swamp rose mallow, Hibiscus moscheutos -n12179122 cotton rose, Confederate rose, Confederate rose mallow, Hibiscus mutabilis -n12179632 roselle, rozelle, sorrel, red sorrel, Jamaica sorrel, Hibiscus sabdariffa -n12180168 mahoe, majagua, mahagua, balibago, purau, Hibiscus tiliaceus -n12180456 flower-of-an-hour, flowers-of-an-hour, bladder ketmia, black-eyed Susan, Hibiscus trionum -n12180885 lacebark, ribbonwood, houhere, Hoheria populnea -n12181352 wild hollyhock, Iliamna remota, Sphaeralcea remota -n12181612 mountain hollyhock, Iliamna ruvularis, Iliamna acerifolia -n12182049 seashore mallow -n12182276 salt marsh mallow, Kosteletzya virginica -n12183026 chaparral mallow, Malacothamnus fasciculatus, Sphaeralcea fasciculata -n12183452 malope, Malope trifida -n12183816 false mallow -n12184095 waxmallow, wax mallow, sleeping hibiscus -n12184468 glade mallow, Napaea dioica -n12184912 pavonia -n12185254 ribbon tree, ribbonwood, Plagianthus regius, Plagianthus betulinus -n12185859 bush hibiscus, Radyera farragei, Hibiscus farragei -n12186352 Virginia mallow, Sida hermaphrodita -n12186554 Queensland hemp, jellyleaf, Sida rhombifolia -n12186839 Indian mallow, Sida spinosa -n12187247 checkerbloom, wild hollyhock, Sidalcea malviflora -n12187663 globe mallow, false mallow -n12187891 prairie mallow, red false mallow, Sphaeralcea coccinea, Malvastrum coccineum -n12188289 tulipwood tree -n12188635 portia tree, bendy tree, seaside mahoe, Thespesia populnea -n12189429 red silk-cotton tree, simal, Bombax ceiba, Bombax malabarica -n12189779 cream-of-tartar tree, sour gourd, Adansonia gregorii -n12189987 baobab, monkey-bread tree, Adansonia digitata -n12190410 kapok, ceiba tree, silk-cotton tree, white silk-cotton tree, Bombay ceiba, God tree, Ceiba pentandra -n12190869 durian, durion, durian tree, Durio zibethinus -n12191240 Montezuma -n12192132 shaving-brush tree, Pseudobombax ellipticum -n12192877 quandong, quandong tree, Brisbane quandong, silver quandong tree, blue fig, Elaeocarpus grandis -n12193334 quandong, blue fig -n12193665 makomako, New Zealand wine berry, wineberry, Aristotelia serrata, Aristotelia racemosa -n12194147 Jamaican cherry, calabur tree, calabura, silk wood, silkwood, Muntingia calabura -n12194613 breakax, breakaxe, break-axe, Sloanea jamaicensis -n12195391 sterculia -n12195533 Panama tree, Sterculia apetala -n12195734 kalumpang, Java olives, Sterculia foetida -n12196129 bottle-tree, bottle tree -n12196336 flame tree, flame durrajong, Brachychiton acerifolius, Sterculia acerifolia -n12196527 flame tree, broad-leaved bottletree, Brachychiton australis -n12196694 kurrajong, currajong, Brachychiton populneus -n12196954 Queensland bottletree, narrow-leaved bottletree, Brachychiton rupestris, Sterculia rupestris -n12197359 kola, kola nut, kola nut tree, goora nut, Cola acuminata -n12197601 kola nut, cola nut -n12198286 Chinese parasol tree, Chinese parasol, Japanese varnish tree, phoenix tree, Firmiana simplex -n12198793 flannelbush, flannel bush, California beauty -n12199266 screw tree -n12199399 nut-leaved screw tree, Helicteres isora -n12199790 red beech, brown oak, booyong, crow's foot, stave wood, silky elm, Heritiera trifoliolata, Terrietia trifoliolata -n12199982 looking glass tree, Heritiera macrophylla -n12200143 looking-glass plant, Heritiera littoralis -n12200504 honey bell, honeybells, Hermannia verticillata, Mahernia verticillata -n12200905 mayeng, maple-leaved bayur, Pterospermum acerifolium -n12201331 silver tree, Tarrietia argyrodendron -n12201580 cacao, cacao tree, chocolate tree, Theobroma cacao -n12201938 obeche, obechi, arere, samba, Triplochiton scleroxcylon -n12202936 linden, linden tree, basswood, lime, lime tree -n12203529 American basswood, American lime, Tilia americana -n12203699 small-leaved linden, small-leaved lime, Tilia cordata -n12203896 white basswood, cottonwood, Tilia heterophylla -n12204032 Japanese linden, Japanese lime, Tilia japonica -n12204175 silver lime, silver linden, Tilia tomentosa -n12204730 corchorus -n12205460 African hemp, Sparmannia africana -n12205694 herb, herbaceous plant -n12214789 protea -n12215022 honeypot, king protea, Protea cynaroides -n12215210 honeyflower, honey-flower, Protea mellifera -n12215579 banksia -n12215824 honeysuckle, Australian honeysuckle, coast banksia, Banksia integrifolia -n12216215 smoke bush -n12216628 Chilean firebush, Chilean flameflower, Embothrium coccineum -n12216968 Chilean nut, Chile nut, Chile hazel, Chilean hazelnut, Guevina heterophylla, Guevina avellana -n12217453 grevillea -n12217851 red-flowered silky oak, Grevillea banksii -n12218274 silky oak, Grevillea robusta -n12218490 beefwood, Grevillea striata -n12218868 cushion flower, pincushion hakea, Hakea laurina -n12219668 rewa-rewa, New Zealand honeysuckle -n12220019 honeyflower, honey-flower, mountain devil, Lambertia formosa -n12220496 silver tree, Leucadendron argenteum -n12220829 lomatia -n12221191 macadamia, macadamia tree -n12221368 Macadamia integrifolia -n12221522 macadamia nut, macadamia nut tree, Macadamia ternifolia -n12221801 Queensland nut, Macadamia tetraphylla -n12222090 prickly ash, Orites excelsa -n12222493 geebung -n12222900 wheel tree, firewheel tree, Stenocarpus sinuatus -n12223160 scrub beefwood, beefwood, Stenocarpus salignus -n12223569 waratah, Telopea Oreades -n12223764 waratah, Telopea speciosissima -n12224978 casuarina -n12225222 she-oak -n12225349 beefwood -n12225563 Australian pine, Casuarina equisetfolia -n12226932 heath -n12227658 tree heath, briar, brier, Erica arborea -n12227909 briarroot -n12228229 winter heath, spring heath, Erica carnea -n12228387 bell heather, heather bell, fine-leaved heath, Erica cinerea -n12228689 Cornish heath, Erica vagans -n12228886 Spanish heath, Portuguese heath, Erica lusitanica -n12229111 Prince-of-Wales'-heath, Prince of Wales heath, Erica perspicua -n12229651 bog rosemary, moorwort, Andromeda glaucophylla -n12229887 marsh andromeda, common bog rosemary, Andromeda polifolia -n12230540 madrona, madrono, manzanita, Arbutus menziesii -n12230794 strawberry tree, Irish strawberry, Arbutus unedo -n12231192 bearberry -n12231709 alpine bearberry, black bearberry, Arctostaphylos alpina -n12232114 heartleaf manzanita, Arctostaphylos andersonii -n12232280 Parry manzanita, Arctostaphylos manzanita -n12232851 spike heath, Bruckenthalia spiculifolia -n12233249 bryanthus -n12234318 leatherleaf, Chamaedaphne calyculata -n12234669 Connemara heath, St. Dabeoc's heath, Daboecia cantabrica -n12235051 trailing arbutus, mayflower, Epigaea repens -n12235479 creeping snowberry, moxie plum, maidenhair berry, Gaultheria hispidula -n12236160 salal, shallon, Gaultheria shallon -n12236546 huckleberry -n12236768 black huckleberry, Gaylussacia baccata -n12236977 dangleberry, dangle-berry, Gaylussacia frondosa -n12237152 box huckleberry, Gaylussacia brachycera -n12237486 kalmia -n12237641 mountain laurel, wood laurel, American laurel, calico bush, Kalmia latifolia -n12237855 swamp laurel, bog laurel, bog kalmia, Kalmia polifolia -n12238756 trapper's tea, glandular Labrador tea -n12238913 wild rosemary, marsh tea, Ledum palustre -n12239240 sand myrtle, Leiophyllum buxifolium -n12239647 leucothoe -n12239880 dog laurel, dog hobble, switch-ivy, Leucothoe fontanesiana, Leucothoe editorum -n12240150 sweet bells, Leucothoe racemosa -n12240477 alpine azalea, mountain azalea, Loiseleuria procumbens -n12240965 staggerbush, stagger bush, Lyonia mariana -n12241192 maleberry, male berry, privet andromeda, he-huckleberry, Lyonia ligustrina -n12241426 fetterbush, fetter bush, shiny lyonia, Lyonia lucida -n12241880 false azalea, fool's huckleberry, Menziesia ferruginea -n12242123 minniebush, minnie bush, Menziesia pilosa -n12242409 sorrel tree, sourwood, titi, Oxydendrum arboreum -n12242850 mountain heath, Phyllodoce caerulea, Bryanthus taxifolius -n12243109 purple heather, Brewer's mountain heather, Phyllodoce breweri -n12243693 fetterbush, mountain fetterbush, mountain andromeda, Pieris floribunda -n12244153 rhododendron -n12244458 coast rhododendron, Rhododendron californicum -n12244650 rosebay, Rhododendron maxima -n12244819 swamp azalea, swamp honeysuckle, white honeysuckle, Rhododendron viscosum -n12245319 azalea -n12245695 cranberry -n12245885 American cranberry, large cranberry, Vaccinium macrocarpon -n12246037 European cranberry, small cranberry, Vaccinium oxycoccus -n12246232 blueberry, blueberry bush -n12246773 farkleberry, sparkleberry, Vaccinium arboreum -n12246941 low-bush blueberry, low blueberry, Vaccinium angustifolium, Vaccinium pennsylvanicum -n12247202 rabbiteye blueberry, rabbit-eye blueberry, rabbiteye, Vaccinium ashei -n12247407 dwarf bilberry, dwarf blueberry, Vaccinium caespitosum -n12247963 evergreen blueberry, Vaccinium myrsinites -n12248141 evergreen huckleberry, Vaccinium ovatum -n12248359 bilberry, thin-leaved bilberry, mountain blue berry, Viccinium membranaceum -n12248574 bilberry, whortleberry, whinberry, blaeberry, Viccinium myrtillus -n12248780 bog bilberry, bog whortleberry, moor berry, Vaccinium uliginosum alpinum -n12248941 dryland blueberry, dryland berry, Vaccinium pallidum -n12249122 grouseberry, grouse-berry, grouse whortleberry, Vaccinium scoparium -n12249294 deerberry, squaw huckleberry, Vaccinium stamineum -n12249542 cowberry, mountain cranberry, lingonberry, lingenberry, lingberry, foxberry, Vaccinium vitis-idaea -n12251001 diapensia -n12251278 galax, galaxy, wandflower, beetleweed, coltsfoot, Galax urceolata -n12251740 pyxie, pixie, pixy, Pyxidanthera barbulata -n12252168 shortia -n12252383 oconee bells, Shortia galacifolia -n12252866 Australian heath -n12253229 epacris -n12253487 common heath, Epacris impressa -n12253664 common heath, blunt-leaf heath, Epacris obtusifolia -n12253835 Port Jackson heath, Epacris purpurascens -n12254168 native cranberry, groundberry, ground-berry, cranberry heath, Astroloma humifusum, Styphelia humifusum -n12255225 pink fivecorner, Styphelia triflora -n12256112 wintergreen, pyrola -n12256325 false wintergreen, Pyrola americana, Pyrola rotundifolia americana -n12256522 lesser wintergreen, Pyrola minor -n12256708 wild lily of the valley, shinleaf, Pyrola elliptica -n12256920 wild lily of the valley, Pyrola rotundifolia -n12257570 pipsissewa, prince's pine -n12257725 love-in-winter, western prince's pine, Chimaphila umbellata, Chimaphila corymbosa -n12258101 one-flowered wintergreen, one-flowered pyrola, Moneses uniflora, Pyrola uniflora -n12258885 Indian pipe, waxflower, Monotropa uniflora -n12259316 pinesap, false beachdrops, Monotropa hypopithys -n12260799 beech, beech tree -n12261359 common beech, European beech, Fagus sylvatica -n12261571 copper beech, purple beech, Fagus sylvatica atropunicea, Fagus purpurea, Fagus sylvatica purpurea -n12261808 American beech, white beech, red beech, Fagus grandifolia, Fagus americana -n12262018 weeping beech, Fagus pendula, Fagus sylvatica pendula -n12262185 Japanese beech -n12262553 chestnut, chestnut tree -n12263038 American chestnut, American sweet chestnut, Castanea dentata -n12263204 European chestnut, sweet chestnut, Spanish chestnut, Castanea sativa -n12263410 Chinese chestnut, Castanea mollissima -n12263588 Japanese chestnut, Castanea crenata -n12263738 Allegheny chinkapin, eastern chinquapin, chinquapin, dwarf chestnut, Castanea pumila -n12263987 Ozark chinkapin, Ozark chinquapin, chinquapin, Castanea ozarkensis -n12264512 oak chestnut -n12264786 giant chinkapin, golden chinkapin, Chrysolepis chrysophylla, Castanea chrysophylla, Castanopsis chrysophylla -n12265083 dwarf golden chinkapin, Chrysolepis sempervirens -n12265394 tanbark oak, Lithocarpus densiflorus -n12265600 Japanese oak, Lithocarpus glabra, Lithocarpus glaber -n12266217 southern beech, evergreen beech -n12266528 myrtle beech, Nothofagus cuninghamii -n12266644 Coigue, Nothofagus dombeyi -n12266796 New Zealand beech -n12266984 silver beech, Nothofagus menziesii -n12267133 roble beech, Nothofagus obliqua -n12267265 rauli beech, Nothofagus procera -n12267411 black beech, Nothofagus solanderi -n12267534 hard beech, Nothofagus truncata -n12267677 acorn -n12267931 cupule, acorn cup -n12268246 oak, oak tree -n12269241 live oak -n12269406 coast live oak, California live oak, Quercus agrifolia -n12269652 white oak -n12270027 American white oak, Quercus alba -n12270278 Arizona white oak, Quercus arizonica -n12270460 swamp white oak, swamp oak, Quercus bicolor -n12270741 European turkey oak, turkey oak, Quercus cerris -n12270946 canyon oak, canyon live oak, maul oak, iron oak, Quercus chrysolepis -n12271187 scarlet oak, Quercus coccinea -n12271451 jack oak, northern pin oak, Quercus ellipsoidalis -n12271643 red oak -n12271933 southern red oak, swamp red oak, turkey oak, Quercus falcata -n12272239 Oregon white oak, Oregon oak, Garry oak, Quercus garryana -n12272432 holm oak, holm tree, holly-leaved oak, evergreen oak, Quercus ilex -n12272735 bear oak, Quercus ilicifolia -n12272883 shingle oak, laurel oak, Quercus imbricaria -n12273114 bluejack oak, turkey oak, Quercus incana -n12273344 California black oak, Quercus kelloggii -n12273515 American turkey oak, turkey oak, Quercus laevis -n12273768 laurel oak, pin oak, Quercus laurifolia -n12273939 California white oak, valley oak, valley white oak, roble, Quercus lobata -n12274151 overcup oak, Quercus lyrata -n12274358 bur oak, burr oak, mossy-cup oak, mossycup oak, Quercus macrocarpa -n12274630 scrub oak -n12274863 blackjack oak, blackjack, jack oak, Quercus marilandica -n12275131 swamp chestnut oak, Quercus michauxii -n12275317 Japanese oak, Quercus mongolica, Quercus grosseserrata -n12275489 chestnut oak -n12275675 chinquapin oak, chinkapin oak, yellow chestnut oak, Quercus muehlenbergii -n12275888 myrtle oak, seaside scrub oak, Quercus myrtifolia -n12276110 water oak, possum oak, Quercus nigra -n12276314 Nuttall oak, Nuttall's oak, Quercus nuttalli -n12276477 durmast, Quercus petraea, Quercus sessiliflora -n12276628 basket oak, cow oak, Quercus prinus, Quercus montana -n12276872 pin oak, swamp oak, Quercus palustris -n12277150 willow oak, Quercus phellos -n12277334 dwarf chinkapin oak, dwarf chinquapin oak, dwarf oak, Quercus prinoides -n12277578 common oak, English oak, pedunculate oak, Quercus robur -n12277800 northern red oak, Quercus rubra, Quercus borealis -n12278107 Shumard oak, Shumard red oak, Quercus shumardii -n12278371 post oak, box white oak, brash oak, iron oak, Quercus stellata -n12278650 cork oak, Quercus suber -n12278865 Spanish oak, Quercus texana -n12279060 huckleberry oak, Quercus vaccinifolia -n12279293 Chinese cork oak, Quercus variabilis -n12279458 black oak, yellow oak, quercitron, quercitron oak, Quercus velutina -n12279772 southern live oak, Quercus virginiana -n12280060 interior live oak, Quercus wislizenii, Quercus wizlizenii -n12280364 mast -n12281241 birch, birch tree -n12281788 yellow birch, Betula alleghaniensis, Betula leutea -n12281974 American white birch, paper birch, paperbark birch, canoe birch, Betula cordifolia, Betula papyrifera -n12282235 grey birch, gray birch, American grey birch, American gray birch, Betula populifolia -n12282527 silver birch, common birch, European white birch, Betula pendula -n12282737 downy birch, white birch, Betula pubescens -n12282933 black birch, river birch, red birch, Betula nigra -n12283147 sweet birch, cherry birch, black birch, Betula lenta -n12283395 Yukon white birch, Betula neoalaskana -n12283542 swamp birch, water birch, mountain birch, Western paper birch, Western birch, Betula fontinalis -n12283790 Newfoundland dwarf birch, American dwarf birch, Betula glandulosa -n12284262 alder, alder tree -n12284821 common alder, European black alder, Alnus glutinosa, Alnus vulgaris -n12285049 grey alder, gray alder, Alnus incana -n12285195 seaside alder, Alnus maritima -n12285369 white alder, mountain alder, Alnus rhombifolia -n12285512 red alder, Oregon alder, Alnus rubra -n12285705 speckled alder, Alnus rugosa -n12285900 smooth alder, hazel alder, Alnus serrulata -n12286068 green alder, Alnus veridis -n12286197 green alder, Alnus veridis crispa, Alnus crispa -n12286826 hornbeam -n12286988 European hornbeam, Carpinus betulus -n12287195 American hornbeam, Carpinus caroliniana -n12287642 hop hornbeam -n12287836 Old World hop hornbeam, Ostrya carpinifolia -n12288005 Eastern hop hornbeam, ironwood, ironwood tree, Ostrya virginiana -n12288823 hazelnut, hazel, hazelnut tree -n12289310 American hazel, Corylus americana -n12289433 cobnut, filbert, Corylus avellana, Corylus avellana grandis -n12289585 beaked hazelnut, Corylus cornuta -n12290748 centaury -n12290975 rosita, Centaurium calycosum -n12291143 lesser centaury, Centaurium minus -n12291459 seaside centaury -n12291671 slender centaury -n12291959 prairie gentian, tulip gentian, bluebell, Eustoma grandiflorum -n12292463 Persian violet, Exacum affine -n12292877 columbo, American columbo, deer's-ear, deer's-ears, pyramid plant, American gentian -n12293723 gentian -n12294124 gentianella, Gentiana acaulis -n12294331 closed gentian, blind gentian, bottle gentian, Gentiana andrewsii -n12294542 explorer's gentian, Gentiana calycosa -n12294723 closed gentian, blind gentian, Gentiana clausa -n12294871 great yellow gentian, Gentiana lutea -n12295033 marsh gentian, calathian violet, Gentiana pneumonanthe -n12295237 soapwort gentian, Gentiana saponaria -n12295429 striped gentian, Gentiana villosa -n12295796 agueweed, ague weed, five-flowered gentian, stiff gentian, Gentianella quinquefolia, Gentiana quinquefolia -n12296045 felwort, gentianella amarella -n12296432 fringed gentian -n12296735 Gentianopsis crinita, Gentiana crinita -n12296929 Gentianopsis detonsa, Gentiana detonsa -n12297110 Gentianopsid procera, Gentiana procera -n12297280 Gentianopsis thermalis, Gentiana thermalis -n12297507 tufted gentian, Gentianopsis holopetala, Gentiana holopetala -n12297846 spurred gentian -n12298165 sabbatia -n12299640 toothbrush tree, mustard tree, Salvadora persica -n12300840 olive tree -n12301180 olive, European olive tree, Olea europaea -n12301445 olive -n12301613 black maire, Olea cunninghamii -n12301766 white maire, Olea lanceolata -n12302071 fringe tree -n12302248 fringe bush, Chionanthus virginicus -n12302565 forestiera -n12303083 forsythia -n12303462 ash, ash tree -n12304115 white ash, Fraxinus Americana -n12304286 swamp ash, Fraxinus caroliniana -n12304420 flowering ash, Fraxinus cuspidata -n12304703 European ash, common European ash, Fraxinus excelsior -n12304899 Oregon ash, Fraxinus latifolia, Fraxinus oregona -n12305089 black ash, basket ash, brown ash, hoop ash, Fraxinus nigra -n12305293 manna ash, flowering ash, Fraxinus ornus -n12305475 red ash, downy ash, Fraxinus pennsylvanica -n12305654 green ash, Fraxinus pennsylvanica subintegerrima -n12305819 blue ash, Fraxinus quadrangulata -n12305986 mountain ash, Fraxinus texensis -n12306089 pumpkin ash, Fraxinus tomentosa -n12306270 Arizona ash, Fraxinus velutina -n12306717 jasmine -n12306938 primrose jasmine, Jasminum mesnyi -n12307076 winter jasmine, Jasminum nudiflorum -n12307240 common jasmine, true jasmine, jessamine, Jasminum officinale -n12307756 privet -n12308112 Amur privet, Ligustrum amurense -n12308447 Japanese privet, Ligustrum japonicum -n12308907 Ligustrum obtusifolium -n12309277 common privet, Ligustrum vulgare -n12309630 devilwood, American olive, Osmanthus americanus -n12310021 mock privet -n12310349 lilac -n12310638 Himalayan lilac, Syringa emodi -n12311045 Persian lilac, Syringa persica -n12311224 Japanese tree lilac, Syringa reticulata, Syringa amurensis japonica -n12311413 Japanese lilac, Syringa villosa -n12311579 common lilac, Syringa vulgaris -n12312110 bloodwort -n12312728 kangaroo paw, kangaroo's paw, kangaroo's-foot, kangaroo-foot plant, Australian sword lily, Anigozanthus manglesii -n12315060 Virginian witch hazel, Hamamelis virginiana -n12315245 vernal witch hazel, Hamamelis vernalis -n12315598 winter hazel, flowering hazel -n12315999 fothergilla, witch alder -n12316444 liquidambar -n12316572 sweet gum, sweet gum tree, bilsted, red gum, American sweet gum, Liquidambar styraciflua -n12317296 iron tree, iron-tree, ironwood, ironwood tree -n12318378 walnut, walnut tree -n12318782 California black walnut, Juglans californica -n12318965 butternut, butternut tree, white walnut, Juglans cinerea -n12319204 black walnut, black walnut tree, black hickory, Juglans nigra -n12319414 English walnut, English walnut tree, Circassian walnut, Persian walnut, Juglans regia -n12320010 hickory, hickory tree -n12320414 water hickory, bitter pecan, water bitternut, Carya aquatica -n12320627 pignut, pignut hickory, brown hickory, black hickory, Carya glabra -n12320806 bitternut, bitternut hickory, bitter hickory, bitter pignut, swamp hickory, Carya cordiformis -n12321077 pecan, pecan tree, Carya illinoensis, Carya illinoinsis -n12321395 big shellbark, big shellbark hickory, big shagbark, king nut, king nut hickory, Carya laciniosa -n12321669 nutmeg hickory, Carya myristicaeformis, Carya myristiciformis -n12321873 shagbark, shagbark hickory, shellbark, shellbark hickory, Carya ovata -n12322099 mockernut, mockernut hickory, black hickory, white-heart hickory, big-bud hickory, Carya tomentosa -n12322501 wing nut, wing-nut -n12322699 Caucasian walnut, Pterocarya fraxinifolia -n12323665 dhawa, dhava -n12324056 combretum -n12324222 hiccup nut, hiccough nut, Combretum bracteosum -n12324388 bush willow, Combretum appiculatum -n12324558 bush willow, Combretum erythrophyllum -n12324906 button tree, button mangrove, Conocarpus erectus -n12325234 white mangrove, Laguncularia racemosa -n12325787 oleaster -n12327022 water milfoil -n12327528 anchovy pear, anchovy pear tree, Grias cauliflora -n12327846 brazil nut, brazil-nut tree, Bertholletia excelsa -n12328398 loosestrife -n12328567 purple loosestrife, spiked loosestrife, Lythrum salicaria -n12328801 grass poly, hyssop loosestrife, Lythrum hyssopifolia -n12329260 crape myrtle, crepe myrtle, crepe flower, Lagerstroemia indica -n12329473 Queen's crape myrtle, pride-of-India, Lagerstroemia speciosa -n12330239 myrtaceous tree -n12330469 myrtle -n12330587 common myrtle, Myrtus communis -n12330891 bayberry, bay-rum tree, Jamaica bayberry, wild cinnamon, Pimenta acris -n12331066 allspice, allspice tree, pimento tree, Pimenta dioica -n12331263 allspice tree, Pimenta officinalis -n12331655 sour cherry, Eugenia corynantha -n12331788 nakedwood, Eugenia dicrana -n12332030 Surinam cherry, pitanga, Eugenia uniflora -n12332218 rose apple, rose-apple tree, jambosa, Eugenia jambos -n12332555 feijoa, feijoa bush -n12333053 jaboticaba, jaboticaba tree, Myrciaria cauliflora -n12333530 guava, true guava, guava bush, Psidium guajava -n12333771 guava, strawberry guava, yellow cattley guava, Psidium littorale -n12333961 cattley guava, purple strawberry guava, Psidium cattleianum, Psidium littorale longipes -n12334153 Brazilian guava, Psidium guineense -n12334293 gum tree, gum -n12334891 eucalyptus, eucalypt, eucalyptus tree -n12335483 flooded gum -n12335664 mallee -n12335800 stringybark -n12335937 smoothbark -n12336092 red gum, peppermint, peppermint gum, Eucalyptus amygdalina -n12336224 red gum, marri, Eucalyptus calophylla -n12336333 river red gum, river gum, Eucalyptus camaldulensis, Eucalyptus rostrata -n12336586 mountain swamp gum, Eucalyptus camphora -n12336727 snow gum, ghost gum, white ash, Eucalyptus coriacea, Eucalyptus pauciflora -n12336973 alpine ash, mountain oak, Eucalyptus delegatensis -n12337131 white mallee, congoo mallee, Eucalyptus dumosa -n12337246 white stringybark, thin-leaved stringybark, Eucalyptusd eugenioides -n12337391 white mountain ash, Eucalyptus fraxinoides -n12337617 blue gum, fever tree, Eucalyptus globulus -n12337800 rose gum, Eucalypt grandis -n12337922 cider gum, Eucalypt gunnii -n12338034 swamp gum, Eucalypt ovata -n12338146 spotted gum, Eucalyptus maculata -n12338258 lemon-scented gum, Eucalyptus citriodora, Eucalyptus maculata citriodora -n12338454 black mallee, black sally, black gum, Eucalytus stellulata -n12338655 forest red gum, Eucalypt tereticornis -n12338796 mountain ash, Eucalyptus regnans -n12338979 manna gum, Eucalyptus viminalis -n12339526 clove, clove tree, Syzygium aromaticum, Eugenia aromaticum, Eugenia caryophyllatum -n12339831 clove -n12340383 tupelo, tupelo tree -n12340581 water gum, Nyssa aquatica -n12340755 sour gum, black gum, pepperidge, Nyssa sylvatica -n12341542 enchanter's nightshade -n12341931 Circaea lutetiana -n12342299 willowherb -n12342498 fireweed, giant willowherb, rosebay willowherb, wickup, Epilobium angustifolium -n12342852 California fuchsia, humming bird's trumpet, Epilobium canum canum, Zauschneria californica -n12343480 fuchsia -n12343753 lady's-eardrop, ladies'-eardrop, lady's-eardrops, ladies'-eardrops, Fuchsia coccinea -n12344283 evening primrose -n12344483 common evening primrose, German rampion, Oenothera biennis -n12344700 sundrops, Oenothera fruticosa -n12344837 Missouri primrose, Ozark sundrops, Oenothera macrocarpa -n12345280 pomegranate, pomegranate tree, Punica granatum -n12345899 mangrove, Rhizophora mangle -n12346578 daphne -n12346813 garland flower, Daphne cneorum -n12346986 spurge laurel, wood laurel, Daphne laureola -n12347158 mezereon, February daphne, Daphne mezereum -n12349315 Indian rhododendron, Melastoma malabathricum -n12349711 Medinilla magnifica -n12350032 deer grass, meadow beauty -n12350758 canna -n12351091 achira, indian shot, arrowroot, Canna indica, Canna edulis -n12351790 arrowroot, American arrowroot, obedience plant, Maranta arundinaceae -n12352287 banana, banana tree -n12352639 dwarf banana, Musa acuminata -n12352844 Japanese banana, Musa basjoo -n12352990 plantain, plantain tree, Musa paradisiaca -n12353203 edible banana, Musa paradisiaca sapientum -n12353431 abaca, Manila hemp, Musa textilis -n12353754 Abyssinian banana, Ethiopian banana, Ensete ventricosum, Musa ensete -n12355760 ginger -n12356023 common ginger, Canton ginger, stem ginger, Zingiber officinale -n12356395 turmeric, Curcuma longa, Curcuma domestica -n12356960 galangal, Alpinia galanga -n12357485 shellflower, shall-flower, shell ginger, Alpinia Zerumbet, Alpinia speciosa, Languas speciosa -n12357968 grains of paradise, Guinea grains, Guinea pepper, melagueta pepper, Aframomum melegueta -n12358293 cardamom, cardamon, Elettaria cardamomum -n12360108 begonia -n12360534 fibrous-rooted begonia -n12360684 tuberous begonia -n12360817 rhizomatous begonia -n12360958 Christmas begonia, blooming-fool begonia, Begonia cheimantha -n12361135 angel-wing begonia, Begonia cocchinea -n12361560 beefsteak begonia, kidney begonia, Begonia erythrophylla, Begonia feastii -n12361754 star begonia, star-leaf begonia, Begonia heracleifolia -n12361946 rex begonia, king begonia, painted-leaf begonia, beefsteak geranium, Begonia rex -n12362274 wax begonia, Begonia semperflorens -n12362514 Socotra begonia, Begonia socotrana -n12362668 hybrid tuberous begonia, Begonia tuberhybrida -n12363301 dillenia -n12363768 guinea gold vine, guinea flower -n12364604 poon -n12364940 calaba, Santa Maria tree, Calophyllum calaba -n12365158 Maria, Calophyllum longifolium -n12365285 laurelwood, lancewood tree, Calophyllum candidissimum -n12365462 Alexandrian laurel, Calophyllum inophyllum -n12365900 clusia -n12366053 wild fig, Clusia flava -n12366186 waxflower, Clusia insignis -n12366313 pitch apple, strangler fig, Clusia rosea, Clusia major -n12366675 mangosteen, mangosteen tree, Garcinia mangostana -n12366870 gamboge tree, Garcinia hanburyi, Garcinia cambogia, Garcinia gummi-gutta -n12367611 St John's wort -n12368028 common St John's wort, tutsan, Hypericum androsaemum -n12368257 great St John's wort, Hypericum ascyron, Hypericum pyramidatum -n12368451 creeping St John's wort, Hypericum calycinum -n12369066 low St Andrew's cross, Hypericum hypericoides -n12369309 klammath weed, Hypericum perforatum -n12369476 shrubby St John's wort, Hypericum prolificum, Hypericum spathulatum -n12369665 St Peter's wort, Hypericum tetrapterum, Hypericum maculatum -n12369845 marsh St-John's wort, Hypericum virginianum -n12370174 mammee apple, mammee, mamey, mammee tree, Mammea americana -n12370549 rose chestnut, ironwood, ironwood tree, Mesua ferrea -n12371202 bower actinidia, tara vine, Actinidia arguta -n12371439 Chinese gooseberry, kiwi, kiwi vine, Actinidia chinensis, Actinidia deliciosa -n12371704 silvervine, silver vine, Actinidia polygama -n12372233 wild cinnamon, white cinnamon tree, Canella winterana, Canella-alba -n12373100 papaya, papaia, pawpaw, papaya tree, melon tree, Carica papaya -n12373739 souari, souari nut, souari tree, Caryocar nuciferum -n12374418 rockrose, rock rose -n12374705 white-leaved rockrose, Cistus albidus -n12374862 common gum cistus, Cistus ladanifer, Cistus ladanum -n12375769 frostweed, frost-weed, frostwort, Helianthemum canadense, Crocanthemum canadense -n12377198 dipterocarp -n12377494 red lauan, red lauan tree, Shorea teysmanniana -n12378249 governor's plum, governor plum, Madagascar plum, ramontchi, batoko palm, Flacourtia indica -n12378753 kei apple, kei apple bush, Dovyalis caffra -n12378963 ketembilla, kitembilla, kitambilla, ketembilla tree, Ceylon gooseberry, Dovyalis hebecarpa -n12379531 chaulmoogra, chaulmoogra tree, chaulmugra, Hydnocarpus kurzii, Taraktagenos kurzii, Taraktogenos kurzii -n12380761 wild peach, Kiggelaria africana -n12381511 candlewood -n12382233 boojum tree, cirio, Fouquieria columnaris, Idria columnaris -n12382875 bird's-eye bush, Ochna serrulata -n12383737 granadilla, purple granadillo, Passiflora edulis -n12383894 granadilla, sweet granadilla, Passiflora ligularis -n12384037 granadilla, giant granadilla, Passiflora quadrangularis -n12384227 maypop, Passiflora incarnata -n12384375 Jamaica honeysuckle, yellow granadilla, Passiflora laurifolia -n12384569 banana passion fruit, Passiflora mollissima -n12384680 sweet calabash, Passiflora maliformis -n12384839 love-in-a-mist, running pop, wild water lemon, Passiflora foetida -n12385429 reseda -n12385566 mignonette, sweet reseda, Reseda odorata -n12385830 dyer's rocket, dyer's mignonette, weld, Reseda luteola -n12386945 false tamarisk, German tamarisk, Myricaria germanica -n12387103 halophyte -n12387633 viola -n12387839 violet -n12388143 field pansy, heartsease, Viola arvensis -n12388293 American dog violet, Viola conspersa -n12388858 dog violet, heath violet, Viola canina -n12388989 horned violet, tufted pansy, Viola cornuta -n12389130 two-eyed violet, heartsease, Viola ocellata -n12389501 bird's-foot violet, pansy violet, Johnny-jump-up, wood violet, Viola pedata -n12389727 downy yellow violet, Viola pubescens -n12389932 long-spurred violet, Viola rostrata -n12390099 pale violet, striped violet, cream violet, Viola striata -n12390314 hedge violet, wood violet, Viola sylvatica, Viola reichenbachiana -n12392070 nettle -n12392549 stinging nettle, Urtica dioica -n12392765 Roman nettle, Urtica pipulifera -n12393269 ramie, ramee, Chinese silk plant, China grass, Boehmeria nivea -n12394118 wood nettle, Laportea canadensis -n12394328 Australian nettle, Australian nettle tree -n12394638 pellitory-of-the-wall, wall pellitory, pellitory, Parietaria difussa -n12395068 richweed, clearweed, dead nettle, Pilea pumilla -n12395289 artillery plant, Pilea microphylla -n12395463 friendship plant, panamica, panamiga, Pilea involucrata -n12395906 Queensland grass-cloth plant, Pipturus argenteus -n12396091 Pipturus albidus -n12396924 cannabis, hemp -n12397431 Indian hemp, Cannabis indica -n12399132 mulberry, mulberry tree -n12399384 white mulberry, Morus alba -n12399534 black mulberry, Morus nigra -n12399656 red mulberry, Morus rubra -n12399899 osage orange, bow wood, mock orange, Maclura pomifera -n12400489 breadfruit, breadfruit tree, Artocarpus communis, Artocarpus altilis -n12400720 jackfruit, jackfruit tree, Artocarpus heterophyllus -n12400924 marang, marang tree, Artocarpus odoratissima -n12401335 fig tree -n12401684 fig, common fig, common fig tree, Ficus carica -n12401893 caprifig, Ficus carica sylvestris -n12402051 golden fig, Florida strangler fig, strangler fig, wild fig, Ficus aurea -n12402348 banyan, banyan tree, banian, banian tree, Indian banyan, East Indian fig tree, Ficus bengalensis -n12402596 pipal, pipal tree, pipul, peepul, sacred fig, bo tree, Ficus religiosa -n12402840 India-rubber tree, India-rubber plant, India-rubber fig, rubber plant, Assam rubber, Ficus elastica -n12403075 mistletoe fig, mistletoe rubber plant, Ficus diversifolia, Ficus deltoidea -n12403276 Port Jackson fig, rusty rig, little-leaf fig, Botany Bay fig, Ficus rubiginosa -n12403513 sycamore, sycamore fig, mulberry fig, Ficus sycomorus -n12403994 paper mulberry, Broussonetia papyrifera -n12404729 trumpetwood, trumpet-wood, trumpet tree, snake wood, imbauba, Cecropia peltata -n12405714 elm, elm tree -n12406304 winged elm, wing elm, Ulmus alata -n12406488 American elm, white elm, water elm, rock elm, Ulmus americana -n12406715 smooth-leaved elm, European field elm, Ulmus carpinifolia -n12406902 cedar elm, Ulmus crassifolia -n12407079 witch elm, wych elm, Ulmus glabra -n12407222 Dutch elm, Ulmus hollandica -n12407396 Huntingdon elm, Ulmus hollandica vegetata -n12407545 water elm, Ulmus laevis -n12407715 Chinese elm, Ulmus parvifolia -n12407890 English elm, European elm, Ulmus procera -n12408077 Siberian elm, Chinese elm, dwarf elm, Ulmus pumila -n12408280 slippery elm, red elm, Ulmus rubra -n12408466 Jersey elm, guernsey elm, wheately elm, Ulmus sarniensis, Ulmus campestris sarniensis, Ulmus campestris wheatleyi -n12408717 September elm, red elm, Ulmus serotina -n12408873 rock elm, Ulmus thomasii -n12409231 hackberry, nettle tree -n12409470 European hackberry, Mediterranean hackberry, Celtis australis -n12409651 American hackberry, Celtis occidentalis -n12409840 sugarberry, Celtis laevigata -n12411461 iridaceous plant -n12412355 bearded iris -n12412606 beardless iris -n12412987 orrisroot, orris -n12413165 dwarf iris, Iris cristata -n12413301 Dutch iris, Iris filifolia -n12413419 Florentine iris, orris, Iris germanica florentina, Iris florentina -n12413642 stinking iris, gladdon, gladdon iris, stinking gladwyn, roast beef plant, Iris foetidissima -n12413880 German iris, Iris germanica -n12414035 Japanese iris, Iris kaempferi -n12414159 German iris, Iris kochii -n12414329 Dalmatian iris, Iris pallida -n12414449 Persian iris, Iris persica -n12414818 Dutch iris, Iris tingitana -n12414932 dwarf iris, vernal iris, Iris verna -n12415595 Spanish iris, xiphium iris, Iris xiphium -n12416073 blackberry-lily, leopard lily, Belamcanda chinensis -n12416423 crocus -n12416703 saffron, saffron crocus, Crocus sativus -n12417836 corn lily -n12418221 blue-eyed grass -n12418507 wandflower, Sparaxis tricolor -n12419037 amaryllis -n12419878 salsilla, Bomarea edulis -n12420124 salsilla, Bomarea salsilla -n12420535 blood lily -n12420722 Cape tulip, Haemanthus coccineus -n12421137 hippeastrum, Hippeastrum puniceum -n12421467 narcissus -n12421683 daffodil, Narcissus pseudonarcissus -n12421917 jonquil, Narcissus jonquilla -n12422129 jonquil -n12422559 Jacobean lily, Aztec lily, Strekelia formosissima -n12425281 liliaceous plant -n12426623 mountain lily, Lilium auratum -n12426749 Canada lily, wild yellow lily, meadow lily, wild meadow lily, Lilium canadense -n12427184 tiger lily, leopard lily, pine lily, Lilium catesbaei -n12427391 Columbia tiger lily, Oregon lily, Lilium columbianum -n12427566 tiger lily, devil lily, kentan, Lilium lancifolium -n12427757 Easter lily, Bermuda lily, white trumpet lily, Lilium longiflorum -n12427946 coast lily, Lilium maritinum -n12428076 Turk's-cap, martagon, Lilium martagon -n12428242 Michigan lily, Lilium michiganense -n12428412 leopard lily, panther lily, Lilium pardalinum -n12428747 Turk's-cap, Turk's cap-lily, Lilium superbum -n12429352 African lily, African tulip, blue African lily, Agapanthus africanus -n12430198 colicroot, colic root, crow corn, star grass, unicorn root -n12430471 ague root, ague grass, Aletris farinosa -n12430675 yellow colicroot, Aletris aurea -n12431434 alliaceous plant -n12432069 Hooker's onion, Allium acuminatum -n12432356 wild leek, Levant garlic, kurrat, Allium ampeloprasum -n12432574 Canada garlic, meadow leek, rose leek, Allium canadense -n12432707 keeled garlic, Allium carinatum -n12433081 onion -n12433178 shallot, eschalot, multiplier onion, Allium cepa aggregatum, Allium ascalonicum -n12433769 nodding onion, nodding wild onion, lady's leek, Allium cernuum -n12433952 Welsh onion, Japanese leek, Allium fistulosum -n12434106 red-skinned onion, Allium haematochiton -n12434483 daffodil garlic, flowering onion, Naples garlic, Allium neopolitanum -n12434634 few-flowered leek, Allium paradoxum -n12434775 garlic, Allium sativum -n12434985 sand leek, giant garlic, Spanish garlic, rocambole, Allium scorodoprasum -n12435152 chives, chive, cive, schnittlaugh, Allium schoenoprasum -n12435486 crow garlic, false garlic, field garlic, stag's garlic, wild garlic, Allium vineale -n12435649 wild garlic, wood garlic, Ramsons, Allium ursinum -n12435777 garlic chive, Chinese chive, Oriental garlic, Allium tuberosum -n12435965 round-headed leek, Allium sphaerocephalum -n12436090 three-cornered leek, triquetrous leek, Allium triquetrum -n12436907 cape aloe, Aloe ferox -n12437513 kniphofia, tritoma, flame flower, flame-flower, flameflower -n12437769 poker plant, Kniphofia uvaria -n12437930 red-hot poker, Kniphofia praecox -n12439154 fly poison, Amianthum muscaetoxicum, Amianthum muscitoxicum -n12439830 amber lily, Anthericum torreyi -n12441183 asparagus, edible asparagus, Asparagus officinales -n12441390 asparagus fern, Asparagus setaceous, Asparagus plumosus -n12441552 smilax, Asparagus asparagoides -n12441958 asphodel -n12442548 Jacob's rod -n12443323 aspidistra, cast-iron plant, bar-room plant, Aspidistra elatio -n12443736 coral drops, Bessera elegans -n12444095 Christmas bells -n12444898 climbing onion, Bowiea volubilis -n12446200 mariposa, mariposa tulip, mariposa lily -n12446519 globe lily, fairy lantern -n12446737 cat's-ear -n12446908 white globe lily, white fairy lantern, Calochortus albus -n12447121 yellow globe lily, golden fairy lantern, Calochortus amabilis -n12447346 rose globe lily, Calochortus amoenus -n12447581 star tulip, elegant cat's ears, Calochortus elegans -n12447891 desert mariposa tulip, Calochortus kennedyi -n12448136 yellow mariposa tulip, Calochortus luteus -n12448361 sagebrush mariposa tulip, Calochortus macrocarpus -n12448700 sego lily, Calochortus nuttallii -n12449296 camas, camass, quamash, camosh, camash -n12449526 common camas, Camassia quamash -n12449784 Leichtlin's camas, Camassia leichtlinii -n12449934 wild hyacinth, indigo squill, Camassia scilloides -n12450344 dogtooth violet, dogtooth, dog's-tooth violet -n12450607 white dogtooth violet, white dog's-tooth violet, blonde lilian, Erythronium albidum -n12450840 yellow adder's tongue, trout lily, amberbell, Erythronium americanum -n12451070 European dogtooth, Erythronium dens-canis -n12451240 fawn lily, Erythronium californicum -n12451399 glacier lily, snow lily, Erythronium grandiflorum -n12451566 avalanche lily, Erythronium montanum -n12451915 fritillary, checkered lily -n12452256 mission bells, rice-grain fritillary, Fritillaria affinis, Fritillaria lanceolata, Fritillaria mutica -n12452480 mission bells, black fritillary, Fritillaria biflora -n12452673 stink bell, Fritillaria agrestis -n12452836 crown imperial, Fritillaria imperialis -n12453018 white fritillary, Fritillaria liliaceae -n12453186 snake's head fritillary, guinea-hen flower, checkered daffodil, leper lily, Fritillaria meleagris -n12453714 adobe lily, pink fritillary, Fritillaria pluriflora -n12453857 scarlet fritillary, Fritillaria recurva -n12454159 tulip -n12454436 dwarf tulip, Tulipa armena, Tulipa suaveolens -n12454556 lady tulip, candlestick tulip, Tulipa clusiana -n12454705 Tulipa gesneriana -n12454793 cottage tulip -n12454949 Darwin tulip -n12455950 gloriosa, glory lily, climbing lily, creeping lily, Gloriosa superba -n12457091 lemon lily, Hemerocallis lilio-asphodelus, Hemerocallis flava -n12458550 common hyacinth, Hyacinthus orientalis -n12458713 Roman hyacinth, Hyacinthus orientalis albulus -n12458874 summer hyacinth, cape hyacinth, Hyacinthus candicans, Galtonia candicans -n12459629 star-of-Bethlehem -n12460146 bath asparagus, Prussian asparagus, Ornithogalum pyrenaicum -n12460697 grape hyacinth -n12460957 common grape hyacinth, Muscari neglectum -n12461109 tassel hyacinth, Muscari comosum -n12461466 scilla, squill -n12461673 spring squill, Scilla verna, sea onion -n12462032 false asphodel -n12462221 Scotch asphodel, Tofieldia pusilla -n12462582 sea squill, sea onion, squill, Urginea maritima -n12462805 squill -n12463134 butcher's broom, Ruscus aculeatus -n12463743 bog asphodel -n12463975 European bog asphodel, Narthecium ossifragum -n12464128 American bog asphodel, Narthecium americanum -n12464476 hellebore, false hellebore -n12464649 white hellebore, American hellebore, Indian poke, bugbane, Veratrum viride -n12465557 squaw grass, bear grass, Xerophyllum tenax -n12466727 death camas, zigadene -n12467018 alkali grass, Zigadenus elegans -n12467197 white camas, Zigadenus glaucus -n12467433 poison camas, Zigadenus nuttalli -n12467592 grassy death camas, Zigadenus venenosus, Zigadenus venenosus gramineus -n12468545 prairie wake-robin, prairie trillium, Trillium recurvatum -n12468719 dwarf-white trillium, snow trillium, early wake-robin -n12469517 herb Paris, Paris quadrifolia -n12470092 sarsaparilla -n12470512 bullbrier, greenbrier, catbrier, horse brier, horse-brier, brier, briar, Smilax rotundifolia -n12470907 rough bindweed, Smilax aspera -n12472024 clintonia, Clinton's lily -n12473608 false lily of the valley, Maianthemum canadense -n12473840 false lily of the valley, Maianthemum bifolium -n12474167 Solomon's-seal -n12474418 great Solomon's-seal, Polygonatum biflorum, Polygonatum commutatum -n12475035 bellwort, merry bells, wild oats -n12475242 strawflower, cornflower, Uvularia grandiflora -n12475774 pia, Indian arrowroot, Tacca leontopetaloides, Tacca pinnatifida -n12476510 agave, century plant, American aloe -n12477163 American agave, Agave americana -n12477401 sisal, Agave sisalana -n12477583 maguey, cantala, Agave cantala -n12477747 maguey, Agave atrovirens -n12477983 Agave tequilana -n12478768 cabbage tree, grass tree, Cordyline australis -n12479537 dracaena -n12480456 tuberose, Polianthes tuberosa -n12480895 sansevieria, bowstring hemp -n12481150 African bowstring hemp, African hemp, Sansevieria guineensis -n12481289 Ceylon bowstring hemp, Sansevieria zeylanica -n12481458 mother-in-law's tongue, snake plant, Sansevieria trifasciata -n12482437 Spanish bayonet, Yucca aloifolia -n12482668 Spanish bayonet, Yucca baccata -n12482893 Joshua tree, Yucca brevifolia -n12483282 soapweed, soap-weed, soap tree, Yucca elata -n12483427 Adam's needle, Adam's needle-and-thread, spoonleaf yucca, needle palm, Yucca filamentosa -n12483625 bear grass, Yucca glauca -n12483841 Spanish dagger, Yucca gloriosa -n12484244 Our Lord's candle, Yucca whipplei -n12484784 water shamrock, buckbean, bogbean, bog myrtle, marsh trefoil, Menyanthes trifoliata -n12485653 butterfly bush, buddleia -n12485981 yellow jasmine, yellow jessamine, Carolina jasmine, evening trumpet flower, Gelsemium sempervirens -n12486574 flax -n12487058 calabar bean, ordeal bean -n12488454 bonduc, bonduc tree, Caesalpinia bonduc, Caesalpinia bonducella -n12488709 divi-divi, Caesalpinia coriaria -n12489046 Mysore thorn, Caesalpinia decapetala, Caesalpinia sepiaria -n12489676 brazilian ironwood, Caesalpinia ferrea -n12489815 bird of paradise, poinciana, Caesalpinia gilliesii, Poinciana gilliesii -n12490490 shingle tree, Acrocarpus fraxinifolius -n12491017 mountain ebony, orchid tree, Bauhinia variegata -n12491435 msasa, Brachystegia speciformis -n12491826 cassia -n12492106 golden shower tree, drumstick tree, purging cassia, pudding pipe tree, canafistola, canafistula, Cassia fistula -n12492460 pink shower, pink shower tree, horse cassia, Cassia grandis -n12492682 rainbow shower, Cassia javonica -n12492900 horse cassia, Cassia roxburghii, Cassia marginata -n12493208 carob, carob tree, carob bean tree, algarroba, Ceratonia siliqua -n12493426 carob, carob bean, algarroba bean, algarroba, locust bean, locust pod -n12493868 paloverde -n12494794 royal poinciana, flamboyant, flame tree, peacock flower, Delonix regia, Poinciana regia -n12495146 locust tree, locust -n12495670 water locust, swamp locust, Gleditsia aquatica -n12495895 honey locust, Gleditsia triacanthos -n12496427 Kentucky coffee tree, bonduc, chicot, Gymnocladus dioica -n12496949 logwood, logwood tree, campeachy, bloodwood tree, Haematoxylum campechianum -n12497669 Jerusalem thorn, horsebean, Parkinsonia aculeata -n12498055 palo verde, Parkinsonia florida, Cercidium floridum -n12498457 Dalmatian laburnum, Petteria ramentacea, Cytisus ramentaceus -n12499163 senna -n12499757 avaram, tanner's cassia, Senna auriculata, Cassia auriculata -n12499979 Alexandria senna, Alexandrian senna, true senna, tinnevelly senna, Indian senna, Senna alexandrina, Cassia acutifolia, Cassia augustifolia -n12500309 wild senna, Senna marilandica, Cassia marilandica -n12500518 sicklepod, Senna obtusifolia, Cassia tora -n12500751 coffee senna, mogdad coffee, styptic weed, stinking weed, Senna occidentalis, Cassia occidentalis -n12501202 tamarind, tamarind tree, tamarindo, Tamarindus indica -n12504570 false indigo, bastard indigo, Amorpha californica -n12504783 false indigo, bastard indigo, Amorpha fruticosa -n12505253 hog peanut, wild peanut, Amphicarpaea bracteata, Amphicarpa bracteata -n12506181 angelim, andelmin -n12506341 cabbage bark, cabbage-bark tree, cabbage tree, Andira inermis -n12506991 kidney vetch, Anthyllis vulneraria -n12507379 groundnut, groundnut vine, Indian potato, potato bean, wild bean, Apios americana, Apios tuberosa -n12507823 rooibos, Aspalathus linearis, Aspalathus cedcarbergensis -n12508309 milk vetch, milk-vetch -n12508618 alpine milk vetch, Astragalus alpinus -n12508762 purple milk vetch, Astragalus danicus -n12509109 camwood, African sandalwood, Baphia nitida -n12509476 wild indigo, false indigo -n12509665 blue false indigo, Baptisia australis -n12509821 white false indigo, Baptisia lactea -n12509993 indigo broom, horsefly weed, rattle weed, Baptisia tinctoria -n12510343 dhak, dak, palas, Butea frondosa, Butea monosperma -n12510774 pigeon pea, pigeon-pea plant, cajan pea, catjang pea, red gram, dhal, dahl, Cajanus cajan -n12511488 sword bean, Canavalia gladiata -n12511856 pea tree, caragana -n12512095 Siberian pea tree, Caragana arborescens -n12512294 Chinese pea tree, Caragana sinica -n12512674 Moreton Bay chestnut, Australian chestnut -n12513172 butterfly pea, Centrosema virginianum -n12513613 Judas tree, love tree, Circis siliquastrum -n12513933 redbud, Cercis canadensis -n12514138 western redbud, California redbud, Cercis occidentalis -n12514592 tagasaste, Chamaecytisus palmensis, Cytesis proliferus -n12514992 weeping tree broom -n12515393 flame pea -n12515711 chickpea, chickpea plant, Egyptian pea, Cicer arietinum -n12515925 chickpea, garbanzo -n12516165 Kentucky yellowwood, gopherwood, Cladrastis lutea, Cladrastis kentukea -n12516584 glory pea, clianthus -n12516828 desert pea, Sturt pea, Sturt's desert pea, Clianthus formosus, Clianthus speciosus -n12517077 parrot's beak, parrot's bill, Clianthus puniceus -n12517445 butterfly pea, Clitoria mariana -n12517642 blue pea, butterfly pea, Clitoria turnatea -n12518013 telegraph plant, semaphore plant, Codariocalyx motorius, Desmodium motorium, Desmodium gyrans -n12518481 bladder senna, Colutea arborescens -n12519089 axseed, crown vetch, Coronilla varia -n12519563 crotalaria, rattlebox -n12520406 guar, cluster bean, Cyamopsis tetragonolobus, Cyamopsis psoraloides -n12521186 white broom, white Spanish broom, Cytisus albus, Cytisus multiflorus -n12521394 common broom, Scotch broom, green broom, Cytisus scoparius -n12522188 rosewood, rosewood tree -n12522678 Indian blackwood, East Indian rosewood, East India rosewood, Indian rosewood, Dalbergia latifolia -n12522894 sissoo, sissu, sisham, Dalbergia sissoo -n12523141 kingwood, kingwood tree, Dalbergia cearensis -n12523475 Brazilian rosewood, caviuna wood, jacaranda, Dalbergia nigra -n12523850 cocobolo, Dalbergia retusa -n12524188 blackwood, blackwood tree -n12525168 bitter pea -n12525513 derris -n12525753 derris root, tuba root, Derris elliptica -n12526178 prairie mimosa, prickle-weed, Desmanthus ilinoensis -n12526516 tick trefoil, beggar lice, beggar's lice -n12526754 beggarweed, Desmodium tortuosum, Desmodium purpureum -n12527081 Australian pea, Dipogon lignosus, Dolichos lignosus -n12527738 coral tree, erythrina -n12528109 kaffir boom, Cape kafferboom, Erythrina caffra -n12528382 coral bean tree, Erythrina corallodendrum -n12528549 ceibo, crybaby tree, cry-baby tree, common coral tree, Erythrina crista-galli -n12528768 kaffir boom, Transvaal kafferboom, Erythrina lysistemon -n12528974 Indian coral tree, Erythrina variegata, Erythrina Indica -n12529220 cork tree, Erythrina vespertilio -n12529500 goat's rue, goat rue, Galega officinalis -n12529905 poison bush, poison pea, gastrolobium -n12530629 Spanish broom, Spanish gorse, Genista hispanica -n12530818 woodwaxen, dyer's greenweed, dyer's-broom, dyeweed, greenweed, whin, woadwaxen, Genista tinctoria -n12531328 chanar, chanal, Geoffroea decorticans -n12531727 gliricidia -n12532564 soy, soybean, soya bean -n12532886 licorice, liquorice, Glycyrrhiza glabra -n12533190 wild licorice, wild liquorice, American licorice, American liquorice, Glycyrrhiza lepidota -n12533437 licorice root -n12534208 Western Australia coral pea, Hardenbergia comnptoniana -n12534625 sweet vetch, Hedysarum boreale -n12534862 French honeysuckle, sulla, Hedysarum coronarium -n12536291 anil, Indigofera suffruticosa, Indigofera anil -n12537253 scarlet runner, running postman, Kennedia prostrata -n12537569 hyacinth bean, bonavist, Indian bean, Egyptian bean, Lablab purpureus, Dolichos lablab -n12538209 Scotch laburnum, Alpine golden chain, Laburnum alpinum -n12539074 vetchling -n12539306 wild pea -n12539832 everlasting pea -n12540250 beach pea, sea pea, Lathyrus maritimus, Lathyrus japonicus -n12540647 grass vetch, grass vetchling, Lathyrus nissolia -n12540966 marsh pea, Lathyrus palustris -n12541157 common vetchling, meadow pea, yellow vetchling, Lathyrus pratensis -n12541403 grass pea, Indian pea, khesari, Lathyrus sativus -n12542043 Tangier pea, Tangier peavine, Lalthyrus tingitanus -n12542240 heath pea, earth-nut pea, earthnut pea, tuberous vetch, Lathyrus tuberosus -n12543186 bicolor lespediza, ezo-yama-hagi, Lespedeza bicolor -n12543455 japanese clover, japan clover, jap clover, Lespedeza striata -n12543639 Korean lespedeza, Lespedeza stipulacea -n12543826 sericea lespedeza, Lespedeza sericea, Lespedeza cuneata -n12544240 lentil, lentil plant, Lens culinaris -n12544539 lentil -n12545232 prairie bird's-foot trefoil, compass plant, prairie lotus, prairie trefoil, Lotus americanus -n12545635 bird's foot trefoil, bird's foot clover, babies' slippers, bacon and eggs, Lotus corniculatus -n12545865 winged pea, asparagus pea, Lotus tetragonolobus -n12546183 lupine, lupin -n12546420 white lupine, field lupine, wolf bean, Egyptian lupine, Lupinus albus -n12546617 tree lupine, Lupinus arboreus -n12546962 wild lupine, sundial lupine, Indian beet, old-maid's bonnet, Lupinus perennis -n12547215 bluebonnet, buffalo clover, Texas bluebonnet, Lupinus subcarnosus -n12547503 Texas bluebonnet, Lupinus texensis -n12548280 medic, medick, trefoil -n12548564 moon trefoil, Medicago arborea -n12548804 sickle alfalfa, sickle lucerne, sickle medick, Medicago falcata -n12549005 Calvary clover, Medicago intertexta, Medicago echinus -n12549192 black medick, hop clover, yellow trefoil, nonesuch clover, Medicago lupulina -n12549420 alfalfa, lucerne, Medicago sativa -n12549799 millettia -n12550210 mucuna -n12550408 cowage, velvet bean, Bengal bean, Benghal bean, Florida bean, Mucuna pruriens utilis, Mucuna deeringiana, Mucuna aterrima, Stizolobium deeringiana -n12551173 tolu tree, tolu balsam tree, Myroxylon balsamum, Myroxylon toluiferum -n12551457 Peruvian balsam, Myroxylon pereirae, Myroxylon balsamum pereirae -n12552309 sainfoin, sanfoin, holy clover, esparcet, Onobrychis viciifolia, Onobrychis viciaefolia -n12552893 restharrow, rest-harrow, Ononis repens -n12553742 bead tree, jumby bean, jumby tree, Ormosia monosperma -n12554029 jumby bead, jumbie bead, Ormosia coarctata -n12554526 locoweed, crazyweed, crazy weed -n12554729 purple locoweed, purple loco, Oxytropis lambertii -n12554911 tumbleweed -n12555255 yam bean, Pachyrhizus erosus -n12555859 shamrock pea, Parochetus communis -n12556656 pole bean -n12557064 kidney bean, frijol, frijole -n12557438 haricot -n12557556 wax bean -n12557681 scarlet runner, scarlet runner bean, Dutch case-knife bean, runner bean, Phaseolus coccineus, Phaseolus multiflorus -n12558230 lima bean, lima bean plant, Phaseolus limensis -n12558425 sieva bean, butter bean, butter-bean plant, lima bean, Phaseolus lunatus -n12558680 tepary bean, Phaseolus acutifolius latifolius -n12559044 chaparral pea, stingaree-bush, Pickeringia montana -n12559518 Jamaica dogwood, fish fuddle, Piscidia piscipula, Piscidia erythrina -n12560282 pea -n12560621 garden pea -n12560775 edible-pod pea, edible-podded pea, Pisum sativum macrocarpon -n12561169 sugar snap pea, snap pea -n12561309 field pea, field-pea plant, Austrian winter pea, Pisum sativum arvense, Pisum arvense -n12561594 field pea -n12562141 common flat pea, native holly, Playlobium obtusangulum -n12562577 quira -n12562785 roble, Platymiscium trinitatis -n12563045 Panama redwood tree, Panama redwood, Platymiscium pinnatum -n12563702 Indian beech, Pongamia glabra -n12564083 winged bean, winged pea, goa bean, goa bean vine, Manila bean, Psophocarpus tetragonolobus -n12564613 breadroot, Indian breadroot, pomme blanche, pomme de prairie, Psoralea esculenta -n12565102 bloodwood tree, kiaat, Pterocarpus angolensis -n12565912 kino, Pterocarpus marsupium -n12566331 red sandalwood, red sanders, red sanderswood, red saunders, Pterocarpus santalinus -n12566954 kudzu, kudzu vine, Pueraria lobata -n12567950 bristly locust, rose acacia, moss locust, Robinia hispida -n12568186 black locust, yellow locust, Robinia pseudoacacia -n12568649 clammy locust, Robinia viscosa -n12569037 carib wood, Sabinea carinalis -n12569616 Colorado River hemp, Sesbania exaltata -n12569851 scarlet wisteria tree, vegetable hummingbird, Sesbania grandiflora -n12570394 Japanese pagoda tree, Chinese scholartree, Chinese scholar tree, Sophora japonica, Sophora sinensis -n12570703 mescal bean, coral bean, frijolito, frijolillo, Sophora secundiflora -n12570972 kowhai, Sophora tetraptera -n12571781 jade vine, emerald creeper, Strongylodon macrobotrys -n12572546 hoary pea -n12572759 bastard indigo, Tephrosia purpurea -n12572858 catgut, goat's rue, wild sweet pea, Tephrosia virginiana -n12573256 bush pea -n12573474 false lupine, golden pea, yellow pea, Thermopsis macrophylla -n12573647 Carolina lupine, Thermopsis villosa -n12573911 tipu, tipu tree, yellow jacaranda, pride of Bolivia -n12574320 bird's foot trefoil, Trigonella ornithopodioides -n12574470 fenugreek, Greek clover, Trigonella foenumgraecum -n12574866 gorse, furze, whin, Irish gorse, Ulex europaeus -n12575322 vetch -n12575812 tufted vetch, bird vetch, Calnada pea, Vicia cracca -n12576323 broad bean, fava bean, horsebean -n12576451 bitter betch, Vicia orobus -n12576695 bush vetch, Vicia sepium -n12577362 moth bean, Vigna aconitifolia, Phaseolus aconitifolius -n12577895 snailflower, snail-flower, snail flower, snail bean, corkscrew flower, Vigna caracalla, Phaseolus caracalla -n12578255 mung, mung bean, green gram, golden gram, Vigna radiata, Phaseolus aureus -n12578626 cowpea, cowpea plant, black-eyed pea, Vigna unguiculata, Vigna sinensis -n12578916 cowpea, black-eyed pea -n12579038 asparagus bean, yard-long bean, Vigna unguiculata sesquipedalis, Vigna sesquipedalis -n12579404 swamp oak, Viminaria juncea, Viminaria denudata -n12579822 keurboom, Virgilia capensis, Virgilia oroboides -n12580012 keurboom, Virgilia divaricata -n12580654 Japanese wistaria, Wisteria floribunda -n12580786 Chinese wistaria, Wisteria chinensis -n12580896 American wistaria, American wisteria, Wisteria frutescens -n12581110 silky wisteria, Wisteria venusta -n12582231 palm, palm tree -n12582665 sago palm -n12582846 feather palm -n12583126 fan palm -n12583401 palmetto -n12583681 coyol, coyol palm, Acrocomia vinifera -n12583855 grugru, gri-gri, grugru palm, macamba, Acrocomia aculeata -n12584191 areca -n12584365 betel palm, Areca catechu -n12584715 sugar palm, gomuti, gomuti palm, Arenga pinnata -n12585137 piassava palm, pissaba palm, Bahia piassava, bahia coquilla, Attalea funifera -n12585373 coquilla nut -n12585629 palmyra, palmyra palm, toddy palm, wine palm, lontar, longar palm, Borassus flabellifer -n12586298 calamus -n12586499 rattan, rattan palm, Calamus rotang -n12586725 lawyer cane, Calamus australis -n12586989 fishtail palm -n12587132 wine palm, jaggery palm, kitul, kittul, kitul tree, toddy palm, Caryota urens -n12587487 wax palm, Ceroxylon andicola, Ceroxylon alpinum -n12587803 coconut, coconut palm, coco palm, coco, cocoa palm, coconut tree, Cocos nucifera -n12588320 carnauba, carnauba palm, wax palm, Copernicia prunifera, Copernicia cerifera -n12588780 caranday, caranda, caranda palm, wax palm, Copernicia australis, Copernicia alba -n12589142 corozo, corozo palm -n12589458 gebang palm, Corypha utan, Corypha gebanga -n12589687 latanier, latanier palm -n12589841 talipot, talipot palm, Corypha umbraculifera -n12590232 oil palm -n12590499 African oil palm, Elaeis guineensis -n12590600 American oil palm, Elaeis oleifera -n12590715 palm nut, palm kernel -n12591017 cabbage palm, Euterpe oleracea -n12591351 cabbage palm, cabbage tree, Livistona australis -n12591702 true sago palm, Metroxylon sagu -n12592058 nipa palm, Nipa fruticans -n12592544 babassu, babassu palm, coco de macao, Orbignya phalerata, Orbignya spesiosa, Orbignya martiana -n12592839 babassu nut -n12593122 cohune palm, Orbignya cohune, cohune -n12593341 cohune nut -n12593994 date palm, Phoenix dactylifera -n12594324 ivory palm, ivory-nut palm, ivory plant, Phytelephas macrocarpa -n12594989 raffia palm, Raffia farinifera, Raffia ruffia -n12595699 bamboo palm, Raffia vinifera -n12595964 lady palm -n12596148 miniature fan palm, bamboo palm, fern rhapis, Rhapis excelsa -n12596345 reed rhapis, slender lady palm, Rhapis humilis -n12596709 royal palm, Roystonea regia -n12596849 cabbage palm, Roystonea oleracea -n12597134 cabbage palmetto, cabbage palm, Sabal palmetto -n12597466 saw palmetto, scrub palmetto, Serenoa repens -n12597798 thatch palm, thatch tree, silver thatch, broom palm, Thrinax parviflora -n12598027 key palm, silvertop palmetto, silver thatch, Thrinax microcarpa, Thrinax morrisii, Thrinax keyensis -n12599185 English plantain, narrow-leaved plantain, ribgrass, ribwort, ripple-grass, buckthorn, Plantago lanceolata -n12599435 broad-leaved plantain, common plantain, white-man's foot, whiteman's foot, cart-track plant, Plantago major -n12599661 hoary plantain, Plantago media -n12599874 fleawort, psyllium, Spanish psyllium, Plantago psyllium -n12600095 rugel's plantain, broad-leaved plantain, Plantago rugelii -n12600267 hoary plantain, Plantago virginica -n12601494 buckwheat, Polygonum fagopyrum, Fagopyrum esculentum -n12601805 prince's-feather, princess feather, kiss-me-over-the-garden-gate, prince's-plume, Polygonum orientale -n12602262 eriogonum -n12602434 umbrella plant, Eriogonum allenii -n12602612 wild buckwheat, California buckwheat, Erigonum fasciculatum -n12602980 rhubarb, rhubarb plant -n12603273 Himalayan rhubarb, Indian rhubarb, red-veined pie plant, Rheum australe, Rheum emodi -n12603449 pie plant, garden rhubarb, Rheum cultorum, Rheum rhabarbarum, Rheum rhaponticum -n12603672 Chinese rhubarb, Rheum palmatum -n12604228 sour dock, garden sorrel, Rumex acetosa -n12604460 sheep sorrel, sheep's sorrel, Rumex acetosella -n12604639 bitter dock, broad-leaved dock, yellow dock, Rumex obtusifolius -n12604845 French sorrel, garden sorrel, Rumex scutatus -n12605683 yellow-eyed grass -n12606438 commelina -n12606545 spiderwort, dayflower -n12607456 pineapple, pineapple plant, Ananas comosus -n12609379 pipewort, Eriocaulon aquaticum -n12610328 water hyacinth, water orchid, Eichhornia crassipes, Eichhornia spesiosa -n12610740 water star grass, mud plantain, Heteranthera dubia -n12611640 naiad, water nymph -n12612170 water plantain, Alisma plantago-aquatica -n12612811 narrow-leaved water plantain -n12613706 hydrilla, Hydrilla verticillata -n12614096 American frogbit, Limnodium spongia -n12614477 waterweed -n12614625 Canadian pondweed, Elodea canadensis -n12615232 tape grass, eelgrass, wild celery, Vallisneria spiralis -n12615710 pondweed -n12616248 curled leaf pondweed, curly pondweed, Potamogeton crispus -n12616630 loddon pondweed, Potamogeton nodosus, Potamogeton americanus -n12616996 frog's lettuce -n12617559 arrow grass, Triglochin maritima -n12618146 horned pondweed, Zannichellia palustris -n12618727 eelgrass, grass wrack, sea wrack, Zostera marina -n12620196 rose, rosebush -n12620546 hip, rose hip, rosehip -n12620969 banksia rose, Rosa banksia -n12621410 damask rose, summer damask rose, Rosa damascena -n12621619 sweetbrier, sweetbriar, brier, briar, eglantine, Rosa eglanteria -n12621945 Cherokee rose, Rosa laevigata -n12622297 musk rose, Rosa moschata -n12622875 agrimonia, agrimony -n12623077 harvest-lice, Agrimonia eupatoria -n12623211 fragrant agrimony, Agrimonia procera -n12623818 alderleaf Juneberry, alder-leaved serviceberry, Amelanchier alnifolia -n12624381 flowering quince -n12624568 japonica, maule's quince, Chaenomeles japonica -n12625003 coco plum, coco plum tree, cocoa plum, icaco, Chrysobalanus icaco -n12625383 cotoneaster -n12625670 Cotoneaster dammeri -n12625823 Cotoneaster horizontalis -n12626674 parsley haw, parsley-leaved thorn, Crataegus apiifolia, Crataegus marshallii -n12626878 scarlet haw, Crataegus biltmoreana -n12627119 blackthorn, pear haw, pear hawthorn, Crataegus calpodendron, Crataegus tomentosa -n12627347 cockspur thorn, cockspur hawthorn, Crataegus crus-galli -n12627526 mayhaw, summer haw, Crataegus aestivalis -n12628356 red haw, downy haw, Crataegus mollis, Crataegus coccinea mollis -n12628705 red haw, Crataegus pedicellata, Crataegus coccinea -n12628986 quince, quince bush, Cydonia oblonga -n12629305 mountain avens, Dryas octopetala -n12629666 loquat, loquat tree, Japanese medlar, Japanese plum, Eriobotrya japonica -n12630763 beach strawberry, Chilean strawberry, Fragaria chiloensis -n12630999 Virginia strawberry, scarlet strawberry, Fragaria virginiana -n12631331 avens -n12631637 yellow avens, Geum alleppicum strictum, Geum strictum -n12631932 yellow avens, Geum macrophyllum -n12632335 prairie smoke, purple avens, Geum triflorum -n12632733 bennet, white avens, Geum virginianum -n12633061 toyon, tollon, Christmasberry, Christmas berry, Heteromeles arbutifolia, Photinia arbutifolia -n12633638 apple tree -n12633994 apple, orchard apple tree, Malus pumila -n12634211 wild apple, crab apple, crabapple -n12634429 crab apple, crabapple, cultivated crab apple -n12634734 Siberian crab, Siberian crab apple, cherry apple, cherry crab, Malus baccata -n12634986 wild crab, Malus sylvestris -n12635151 American crab apple, garland crab, Malus coronaria -n12635359 Oregon crab apple, Malus fusca -n12635532 Southern crab apple, flowering crab, Malus angustifolia -n12635744 Iowa crab, Iowa crab apple, prairie crab, western crab apple, Malus ioensis -n12635955 Bechtel crab, flowering crab -n12636224 medlar, medlar tree, Mespilus germanica -n12636885 cinquefoil, five-finger -n12637123 silverweed, goose-tansy, goose grass, Potentilla anserina -n12637485 salad burnet, burnet bloodwort, pimpernel, Poterium sanguisorba -n12638218 plum, plum tree -n12638556 wild plum, wild plum tree -n12638753 Allegheny plum, Alleghany plum, sloe, Prunus alleghaniensis -n12638964 American red plum, August plum, goose plum, Prunus americana -n12639168 chickasaw plum, hog plum, hog plum bush, Prunus angustifolia -n12639376 beach plum, beach plum bush, Prunus maritima -n12639584 common plum, Prunus domestica -n12639736 bullace, Prunus insititia -n12639910 damson plum, damson plum tree, Prunus domestica insititia -n12640081 big-tree plum, Prunus mexicana -n12640284 Canada plum, Prunus nigra -n12640435 plumcot, plumcot tree -n12640607 apricot, apricot tree -n12640839 Japanese apricot, mei, Prunus mume -n12641007 common apricot, Prunus armeniaca -n12641180 purple apricot, black apricot, Prunus dasycarpa -n12641413 cherry, cherry tree -n12641931 wild cherry, wild cherry tree -n12642090 wild cherry -n12642200 sweet cherry, Prunus avium -n12642435 heart cherry, oxheart, oxheart cherry -n12642600 gean, mazzard, mazzard cherry -n12642964 capulin, capulin tree, Prunus capuli -n12643113 cherry laurel, laurel cherry, mock orange, wild orange, Prunus caroliniana -n12643313 cherry plum, myrobalan, myrobalan plum, Prunus cerasifera -n12643473 sour cherry, sour cherry tree, Prunus cerasus -n12643688 amarelle, Prunus cerasus caproniana -n12643877 morello, Prunus cerasus austera -n12644283 marasca -n12644902 almond tree -n12645174 almond, sweet almond, Prunus dulcis, Prunus amygdalus, Amygdalus communis -n12645530 bitter almond, Prunus dulcis amara, Amygdalus communis amara -n12646072 jordan almond -n12646197 dwarf flowering almond, Prunus glandulosa -n12646397 holly-leaved cherry, holly-leaf cherry, evergreen cherry, islay, Prunus ilicifolia -n12646605 fuji, fuji cherry, Prunus incisa -n12646740 flowering almond, oriental bush cherry, Prunus japonica -n12646950 cherry laurel, laurel cherry, Prunus laurocerasus -n12647231 Catalina cherry, Prunus lyonii -n12647376 bird cherry, bird cherry tree -n12647560 hagberry tree, European bird cherry, common bird cherry, Prunus padus -n12647787 hagberry -n12647893 pin cherry, Prunus pensylvanica -n12648045 peach, peach tree, Prunus persica -n12648196 nectarine, nectarine tree, Prunus persica nectarina -n12648424 sand cherry, Prunus pumila, Prunus pumilla susquehanae, Prunus susquehanae, Prunus cuneata -n12648693 Japanese plum, Prunus salicina -n12648888 black cherry, black cherry tree, rum cherry, Prunus serotina -n12649065 flowering cherry -n12649317 oriental cherry, Japanese cherry, Japanese flowering cherry, Prunus serrulata -n12649539 Japanese flowering cherry, Prunus sieboldii -n12649866 Sierra plum, Pacific plum, Prunus subcordata -n12650038 rosebud cherry, winter flowering cherry, Prunus subhirtella -n12650229 Russian almond, dwarf Russian almond, Prunus tenella -n12650379 flowering almond, Prunus triloba -n12650556 chokecherry, chokecherry tree, Prunus virginiana -n12650805 chokecherry -n12650915 western chokecherry, Prunus virginiana demissa, Prunus demissa -n12651229 Pyracantha, pyracanth, fire thorn, firethorn -n12651611 pear, pear tree, Pyrus communis -n12651821 fruit tree -n12653218 bramble bush -n12653436 lawyerbush, lawyer bush, bush lawyer, Rubus cissoides, Rubus australis -n12653633 stone bramble, Rubus saxatilis -n12654227 sand blackberry, Rubus cuneifolius -n12654857 boysenberry, boysenberry bush -n12655062 loganberry, Rubus loganobaccus, Rubus ursinus loganobaccus -n12655245 American dewberry, Rubus canadensis -n12655351 Northern dewberry, American dewberry, Rubus flagellaris -n12655498 Southern dewberry, Rubus trivialis -n12655605 swamp dewberry, swamp blackberry, Rubus hispidus -n12655726 European dewberry, Rubus caesius -n12655869 raspberry, raspberry bush -n12656369 wild raspberry, European raspberry, framboise, Rubus idaeus -n12656528 American raspberry, Rubus strigosus, Rubus idaeus strigosus -n12656685 black raspberry, blackcap, blackcap raspberry, thimbleberry, Rubus occidentalis -n12656909 salmonberry, Rubus spectabilis -n12657082 salmonberry, salmon berry, thimbleberry, Rubus parviflorus -n12657755 wineberry, Rubus phoenicolasius -n12658118 mountain ash -n12658308 rowan, rowan tree, European mountain ash, Sorbus aucuparia -n12658481 rowanberry -n12658603 American mountain ash, Sorbus americana -n12658715 Western mountain ash, Sorbus sitchensis -n12658846 service tree, sorb apple, sorb apple tree, Sorbus domestica -n12659064 wild service tree, Sorbus torminalis -n12659356 spirea, spiraea -n12659539 bridal wreath, bridal-wreath, Saint Peter's wreath, St. Peter's wreath, Spiraea prunifolia -n12660601 madderwort, rubiaceous plant -n12661045 Indian madder, munjeet, Rubia cordifolia -n12661227 madder, Rubia tinctorum -n12661538 woodruff -n12662074 dagame, lemonwood tree, Calycophyllum candidissimum -n12662379 blolly, West Indian snowberry, Chiococca alba -n12662772 coffee, coffee tree -n12663023 Arabian coffee, Coffea arabica -n12663254 Liberian coffee, Coffea liberica -n12663359 robusta coffee, Rio Nunez coffee, Coffea robusta, Coffea canephora -n12663804 cinchona, chinchona -n12664005 Cartagena bark, Cinchona cordifolia, Cinchona lancifolia -n12664187 calisaya, Cinchona officinalis, Cinchona ledgeriana, Cinchona calisaya -n12664469 cinchona tree, Cinchona pubescens -n12664710 cinchona, cinchona bark, Peruvian bark, Jesuit's bark -n12665048 bedstraw -n12665271 sweet woodruff, waldmeister, woodruff, fragrant bedstraw, Galium odoratum, Asperula odorata -n12665659 Northern bedstraw, Northern snow bedstraw, Galium boreale -n12665857 yellow bedstraw, yellow cleavers, Our Lady's bedstraw, Galium verum -n12666050 wild licorice, Galium lanceolatum -n12666159 cleavers, clivers, goose grass, catchweed, spring cleavers, Galium aparine -n12666369 wild madder, white madder, white bedstraw, infant's-breath, false baby's breath, Galium mollugo -n12666965 cape jasmine, cape jessamine, Gardenia jasminoides, Gardenia augusta -n12667406 genipa -n12667582 genipap fruit, jagua, marmalade box, Genipa Americana -n12667964 hamelia -n12668131 scarlet bush, scarlet hamelia, coloradillo, Hamelia patens, Hamelia erecta -n12669803 lemonwood, lemon-wood, lemonwood tree, lemon-wood tree, Psychotria capensis -n12670334 negro peach, Sarcocephalus latifolius, Sarcocephalus esculentus -n12670758 wild medlar, wild medlar tree, medlar, Vangueria infausta -n12670962 Spanish tamarind, Vangueria madagascariensis -n12671651 abelia -n12672289 bush honeysuckle, Diervilla sessilifolia -n12673588 American twinflower, Linnaea borealis americana -n12674120 honeysuckle -n12674685 American fly honeysuckle, fly honeysuckle, Lonicera canadensis -n12674895 Italian honeysuckle, Italian woodbine, Lonicera caprifolium -n12675299 yellow honeysuckle, Lonicera flava -n12675515 hairy honeysuckle, Lonicera hirsuta -n12675876 Japanese honeysuckle, Lonicera japonica -n12676134 Hall's honeysuckle, Lonicera japonica halliana -n12676370 Morrow's honeysuckle, Lonicera morrowii -n12676534 woodbine, Lonicera periclymenum -n12676703 trumpet honeysuckle, coral honeysuckle, trumpet flower, trumpet vine, Lonicera sempervirens -n12677120 European fly honeysuckle, European honeysuckle, Lonicera xylosteum -n12677331 swamp fly honeysuckle -n12677612 snowberry, common snowberry, waxberry, Symphoricarpos alba -n12677841 coralberry, Indian currant, Symphoricarpos orbiculatus -n12678794 blue elder, blue elderberry, Sambucus caerulea -n12679023 dwarf elder, danewort, Sambucus ebulus -n12679432 American red elder, red-berried elder, stinking elder, Sambucus pubens -n12679593 European red elder, red-berried elder, Sambucus racemosa -n12679876 feverroot, horse gentian, tinker's root, wild coffee, Triostium perfoliatum -n12680402 cranberry bush, cranberry tree, American cranberry bush, highbush cranberry, Viburnum trilobum -n12680652 wayfaring tree, twist wood, twistwood, Viburnum lantana -n12680864 guelder rose, European cranberrybush, European cranberry bush, crampbark, cranberry tree, Viburnum opulus -n12681376 arrow wood, Viburnum recognitum -n12681579 black haw, Viburnum prunifolium -n12681893 weigela, Weigela florida -n12682411 teasel, teazel, teasle -n12682668 common teasel, Dipsacus fullonum -n12682882 fuller's teasel, Dipsacus sativus -n12683096 wild teasel, Dipsacus sylvestris -n12683407 scabious, scabiosa -n12683571 sweet scabious, pincushion flower, mournful widow, Scabiosa atropurpurea -n12683791 field scabious, Scabiosa arvensis -n12684379 jewelweed, lady's earrings, orange balsam, celandine, touch-me-not, Impatiens capensis -n12685431 geranium -n12685831 cranesbill, crane's bill -n12686077 wild geranium, spotted cranesbill, Geranium maculatum -n12686274 meadow cranesbill, Geranium pratense -n12686496 Richardson's geranium, Geranium richardsonii -n12686676 herb robert, herbs robert, herb roberts, Geranium robertianum -n12686877 sticky geranium, Geranium viscosissimum -n12687044 dove's foot geranium, Geranium molle -n12687462 rose geranium, sweet-scented geranium, Pelargonium graveolens -n12687698 fish geranium, bedding geranium, zonal pelargonium, Pelargonium hortorum -n12687957 ivy geranium, ivy-leaved geranium, hanging geranium, Pelargonium peltatum -n12688187 apple geranium, nutmeg geranium, Pelargonium odoratissimum -n12688372 lemon geranium, Pelargonium limoneum -n12688716 storksbill, heron's bill -n12689305 musk clover, muskus grass, white-stemmed filaree, Erodium moschatum -n12690653 incense tree -n12691428 elephant tree, Bursera microphylla -n12691661 gumbo-limbo, Bursera simaruba -n12692024 Boswellia carteri -n12692160 salai, Boswellia serrata -n12692521 balm of gilead, Commiphora meccanensis -n12692714 myrrh tree, Commiphora myrrha -n12693244 Protium heptaphyllum -n12693352 Protium guianense -n12693865 water starwort -n12694486 barbados cherry, acerola, Surinam cherry, West Indian cherry, Malpighia glabra -n12695144 mahogany, mahogany tree -n12695975 chinaberry, chinaberry tree, China tree, Persian lilac, pride-of-India, azederach, azedarach, Melia azederach, Melia azedarach -n12696492 neem, neem tree, nim tree, margosa, arishth, Azadirachta indica, Melia Azadirachta -n12696830 neem seed -n12697152 Spanish cedar, Spanish cedar tree, Cedrela odorata -n12697514 satinwood, satinwood tree, Chloroxylon swietenia -n12698027 African scented mahogany, cedar mahogany, sapele mahogany, Entandrophragma cylindricum -n12698435 silver ash -n12698598 native beech, flindosa, flindosy, Flindersia australis -n12698774 bunji-bunji, Flindersia schottiana -n12699031 African mahogany -n12699301 lanseh tree, langsat, langset, Lansium domesticum -n12699922 true mahogany, Cuban mahogany, Dominican mahogany, Swietinia mahogani -n12700088 Honduras mahogany, Swietinia macrophylla -n12700357 Philippine mahogany, Philippine cedar, kalantas, Toona calantas, Cedrela calantas -n12702124 caracolito, Ruptiliocarpon caracolito -n12703190 common wood sorrel, cuckoo bread, shamrock, Oxalis acetosella -n12703383 Bermuda buttercup, English-weed, Oxalis pes-caprae, Oxalis cernua -n12703557 creeping oxalis, creeping wood sorrel, Oxalis corniculata -n12703716 goatsfoot, goat's foot, Oxalis caprina -n12703856 violet wood sorrel, Oxalis violacea -n12704041 oca, oka, Oxalis tuberosa, Oxalis crenata -n12704343 carambola, carambola tree, Averrhoa carambola -n12704513 bilimbi, Averrhoa bilimbi -n12705013 milkwort -n12705220 senega, Polygala alba -n12705458 orange milkwort, yellow milkwort, candyweed, yellow bachelor's button, Polygala lutea -n12705698 flowering wintergreen, gaywings, bird-on-the-wing, fringed polygala, Polygala paucifolia -n12705978 Seneca snakeroot, Seneka snakeroot, senga root, senega root, senega snakeroot, Polygala senega -n12706410 common milkwort, gand flower, Polygala vulgaris -n12707199 rue, herb of grace, Ruta graveolens -n12707781 citrus, citrus tree -n12708293 orange, orange tree -n12708654 sour orange, Seville orange, bitter orange, bitter orange tree, bigarade, marmalade orange, Citrus aurantium -n12708941 bergamot, bergamot orange, Citrus bergamia -n12709103 pomelo, pomelo tree, pummelo, shaddock, Citrus maxima, Citrus grandis, Citrus decumana -n12709349 citron, citron tree, Citrus medica -n12709688 grapefruit, Citrus paradisi -n12709901 mandarin, mandarin orange, mandarin orange tree, Citrus reticulata -n12710295 tangerine, tangerine tree -n12710415 clementine, clementine tree -n12710577 satsuma, satsuma tree -n12710693 sweet orange, sweet orange tree, Citrus sinensis -n12710917 temple orange, temple orange tree, tangor, king orange, Citrus nobilis -n12711182 tangelo, tangelo tree, ugli fruit, Citrus tangelo -n12711398 rangpur, rangpur lime, lemanderin, Citrus limonia -n12711596 lemon, lemon tree, Citrus limon -n12711817 sweet lemon, sweet lime, Citrus limetta -n12711984 lime, lime tree, Citrus aurantifolia -n12712320 citrange, citrange tree, Citroncirus webberi -n12712626 fraxinella, dittany, burning bush, gas plant, Dictamnus alba -n12713063 kumquat, cumquat, kumquat tree -n12713358 marumi, marumi kumquat, round kumquat, Fortunella japonica -n12713521 nagami, nagami kumquat, oval kumquat, Fortunella margarita -n12713866 cork tree, Phellodendron amurense -n12714254 trifoliate orange, trifoliata, wild orange, Poncirus trifoliata -n12714755 prickly ash -n12714949 toothache tree, sea ash, Zanthoxylum americanum, Zanthoxylum fraxineum -n12715195 Hercules'-club, Hercules'-clubs, Hercules-club, Zanthoxylum clava-herculis -n12715914 bitterwood tree -n12716400 marupa, Simarouba amara -n12716594 paradise tree, bitterwood, Simarouba glauca -n12717072 ailanthus -n12717224 tree of heaven, tree of the gods, Ailanthus altissima -n12717644 wild mango, dika, wild mango tree, Irvingia gabonensis -n12718074 pepper tree, Kirkia wilmsii -n12718483 Jamaica quassia, bitterwood, Picrasma excelsa, Picrasma excelsum -n12718995 quassia, bitterwood, Quassia amara -n12719684 nasturtium -n12719944 garden nasturtium, Indian cress, Tropaeolum majus -n12720200 bush nasturtium, Tropaeolum minus -n12720354 canarybird flower, canarybird vine, canary creeper, Tropaeolum peregrinum -n12721122 bean caper, Syrian bean caper, Zygophyllum fabago -n12721477 palo santo, Bulnesia sarmienti -n12722071 lignum vitae, Guaiacum officinale -n12723062 creosote bush, coville, hediondilla, Larrea tridentata -n12723610 caltrop, devil's weed, Tribulus terestris -n12724942 willow, willow tree -n12725521 osier -n12725738 white willow, Huntingdon willow, Salix alba -n12725940 silver willow, silky willow, Salix alba sericea, Salix sericea -n12726159 golden willow, Salix alba vitellina, Salix vitellina -n12726357 cricket-bat willow, Salix alba caerulea -n12726528 arctic willow, Salix arctica -n12726670 weeping willow, Babylonian weeping willow, Salix babylonica -n12726902 Wisconsin weeping willow, Salix pendulina, Salix blanda, Salix pendulina blanda -n12727101 pussy willow, Salix discolor -n12727301 sallow -n12727518 goat willow, florist's willow, pussy willow, Salix caprea -n12727729 peachleaf willow, peach-leaved willow, almond-leaves willow, Salix amygdaloides -n12727960 almond willow, black Hollander, Salix triandra, Salix amygdalina -n12728164 hoary willow, sage willow, Salix candida -n12728322 crack willow, brittle willow, snap willow, Salix fragilis -n12728508 prairie willow, Salix humilis -n12728656 dwarf willow, Salix herbacea -n12728864 grey willow, gray willow, Salix cinerea -n12729023 arroyo willow, Salix lasiolepis -n12729164 shining willow, Salix lucida -n12729315 swamp willow, black willow, Salix nigra -n12729521 bay willow, laurel willow, Salix pentandra -n12729729 purple willow, red willow, red osier, basket willow, purple osier, Salix purpurea -n12729950 balsam willow, Salix pyrifolia -n12730143 creeping willow, Salix repens -n12730370 Sitka willow, silky willow, Salix sitchensis -n12730544 dwarf grey willow, dwarf gray willow, sage willow, Salix tristis -n12730776 bearberry willow, Salix uva-ursi -n12731029 common osier, hemp willow, velvet osier, Salix viminalis -n12731401 poplar, poplar tree -n12731835 balsam poplar, hackmatack, tacamahac, Populus balsamifera -n12732009 white poplar, white aspen, abele, aspen poplar, silver-leaved poplar, Populus alba -n12732252 grey poplar, gray poplar, Populus canescens -n12732491 black poplar, Populus nigra -n12732605 Lombardy poplar, Populus nigra italica -n12732756 cottonwood -n12732966 Eastern cottonwood, necklace poplar, Populus deltoides -n12733218 black cottonwood, Western balsam poplar, Populus trichocarpa -n12733428 swamp cottonwood, black cottonwood, downy poplar, swamp poplar, Populus heterophylla -n12733647 aspen -n12733870 quaking aspen, European quaking aspen, Populus tremula -n12734070 American quaking aspen, American aspen, Populus tremuloides -n12734215 Canadian aspen, bigtooth aspen, bigtoothed aspen, big-toothed aspen, large-toothed aspen, large tooth aspen, Populus grandidentata -n12735160 sandalwood tree, true sandalwood, Santalum album -n12736603 quandong, quandang, quandong tree, Eucarya acuminata, Fusanus acuminatus -n12736999 rabbitwood, buffalo nut, Pyrularia pubera -n12737383 Loranthaceae, family Loranthaceae, mistletoe family -n12737898 mistletoe, Loranthus europaeus -n12738259 American mistletoe, Arceuthobium pusillum -n12739332 mistletoe, Viscum album, Old World mistletoe -n12739966 American mistletoe, Phoradendron serotinum, Phoradendron flavescens -n12740967 aalii -n12741222 soapberry, soapberry tree -n12741586 wild China tree, Sapindus drumondii, Sapindus marginatus -n12741792 China tree, false dogwood, jaboncillo, chinaberry, Sapindus saponaria -n12742290 akee, akee tree, Blighia sapida -n12742741 soapberry vine -n12742878 heartseed, Cardiospermum grandiflorum -n12743009 balloon vine, heart pea, Cardiospermum halicacabum -n12743352 longan, lungen, longanberry, Dimocarpus longan, Euphorbia litchi, Nephelium longana -n12743823 harpullia -n12743976 harpulla, Harpullia cupanioides -n12744142 Moreton Bay tulipwood, Harpullia pendula -n12744387 litchi, lichee, litchi tree, Litchi chinensis, Nephelium litchi -n12744850 Spanish lime, Spanish lime tree, honey berry, mamoncillo, genip, ginep, Melicocca bijuga, Melicocca bijugatus -n12745386 rambutan, rambotan, rambutan tree, Nephelium lappaceum -n12745564 pulasan, pulassan, pulasan tree, Nephelium mutabile -n12746884 pachysandra -n12747120 Allegheny spurge, Allegheny mountain spurge, Pachysandra procumbens -n12748248 bittersweet, American bittersweet, climbing bittersweet, false bittersweet, staff vine, waxwork, shrubby bittersweet, Celastrus scandens -n12749049 spindle tree, spindleberry, spindleberry tree -n12749456 winged spindle tree, Euonymous alatus -n12749679 wahoo, burning bush, Euonymus atropurpureus -n12749852 strawberry bush, wahoo, Euonymus americanus -n12750076 evergreen bittersweet, Euonymus fortunei radicans, Euonymus radicans vegetus -n12750767 cyrilla, leatherwood, white titi, Cyrilla racemiflora -n12751172 titi, buckwheat tree, Cliftonia monophylla -n12751675 crowberry -n12752205 maple -n12753007 silver maple, Acer saccharinum -n12753245 sugar maple, rock maple, Acer saccharum -n12753573 red maple, scarlet maple, swamp maple, Acer rubrum -n12753762 moosewood, moose-wood, striped maple, striped dogwood, goosefoot maple, Acer pennsylvanicum -n12754003 Oregon maple, big-leaf maple, Acer macrophyllum -n12754174 dwarf maple, Rocky-mountain maple, Acer glabrum -n12754311 mountain maple, mountain alder, Acer spicatum -n12754468 vine maple, Acer circinatum -n12754648 hedge maple, field maple, Acer campestre -n12754781 Norway maple, Acer platanoides -n12754981 sycamore, great maple, scottish maple, Acer pseudoplatanus -n12755225 box elder, ash-leaved maple, Acer negundo -n12755387 California box elder, Acer negundo Californicum -n12755559 pointed-leaf maple, Acer argutum -n12755727 Japanese maple, full moon maple, Acer japonicum -n12755876 Japanese maple, Acer palmatum -n12756457 holly -n12757115 Chinese holly, Ilex cornuta -n12757303 bearberry, possum haw, winterberry, Ilex decidua -n12757458 inkberry, gallberry, gall-berry, evergreen winterberry, Ilex glabra -n12757668 mate, Paraguay tea, Ilex paraguariensis -n12757816 American holly, Christmas holly -n12757930 low gallberry holly -n12758014 tall gallberry holly -n12758099 yaupon holly -n12758176 deciduous holly -n12758250 juneberry holly -n12758325 largeleaf holly -n12758399 Geogia holly -n12758471 common winterberry holly -n12758555 smooth winterberry holly -n12759273 cashew, cashew tree, Anacardium occidentale -n12759668 goncalo alves, Astronium fraxinifolium -n12760539 Venetian sumac, wig tree, Cotinus coggygria -n12760875 laurel sumac, Malosma laurina, Rhus laurina -n12761284 mango, mango tree, Mangifera indica -n12761702 pistachio, Pistacia vera, pistachio tree -n12761905 terebinth, Pistacia terebinthus -n12762049 mastic, mastic tree, lentisk, Pistacia lentiscus -n12762405 Australian sumac, Rhodosphaera rhodanthema, Rhus rhodanthema -n12762896 sumac, sumach, shumac -n12763529 smooth sumac, scarlet sumac, vinegar tree, Rhus glabra -n12764008 sugar-bush, sugar sumac, Rhus ovata -n12764202 staghorn sumac, velvet sumac, Virginian sumac, vinegar tree, Rhus typhina -n12764507 squawbush, squaw-bush, skunkbush, Rhus trilobata -n12764978 aroeira blanca, Schinus chichita -n12765115 pepper tree, molle, Peruvian mastic tree, Schinus molle -n12765402 Brazilian pepper tree, Schinus terebinthifolius -n12765846 hog plum, yellow mombin, yellow mombin tree, Spondias mombin -n12766043 mombin, mombin tree, jocote, Spondias purpurea -n12766595 poison ash, poison dogwood, poison sumac, Toxicodendron vernix, Rhus vernix -n12766869 poison ivy, markweed, poison mercury, poison oak, Toxicodendron radicans, Rhus radicans -n12767208 western poison oak, Toxicodendron diversilobum, Rhus diversiloba -n12767423 eastern poison oak, Toxicodendron quercifolium, Rhus quercifolia, Rhus toxicodenedron -n12767648 varnish tree, lacquer tree, Chinese lacquer tree, Japanese lacquer tree, Japanese varnish tree, Japanese sumac, Toxicodendron vernicifluum, Rhus verniciflua -n12768369 horse chestnut, buckeye, Aesculus hippocastanum -n12768682 buckeye, horse chestnut, conker -n12768809 sweet buckeye -n12768933 Ohio buckeye -n12769065 dwarf buckeye, bottlebrush buckeye -n12769219 red buckeye -n12769318 particolored buckeye -n12770529 ebony, ebony tree, Diospyros ebenum -n12770892 marblewood, marble-wood, Andaman marble, Diospyros kurzii -n12771085 marblewood, marble-wood -n12771192 persimmon, persimmon tree -n12771390 Japanese persimmon, kaki, Diospyros kaki -n12771597 American persimmon, possumwood, Diospyros virginiana -n12771890 date plum, Diospyros lotus -n12772753 buckthorn -n12772908 southern buckthorn, shittimwood, shittim, mock orange, Bumelia lycioides -n12773142 false buckthorn, chittamwood, chittimwood, shittimwood, black haw, Bumelia lanuginosa -n12773651 star apple, caimito, Chrysophyllum cainito -n12773917 satinleaf, satin leaf, caimitillo, damson plum, Chrysophyllum oliviforme -n12774299 balata, balata tree, beefwood, bully tree, Manilkara bidentata -n12774641 sapodilla, sapodilla tree, Manilkara zapota, Achras zapota -n12775070 gutta-percha tree, Palaquium gutta -n12775393 gutta-percha tree -n12775717 canistel, canistel tree, Pouteria campechiana nervosa -n12775919 marmalade tree, mammee, sapote, Pouteria zapota, Calocarpum zapota -n12776558 sweetleaf, Symplocus tinctoria -n12776774 Asiatic sweetleaf, sapphire berry, Symplocus paniculata -n12777436 styrax -n12777680 snowbell, Styrax obassia -n12777778 Japanese snowbell, Styrax japonicum -n12777892 Texas snowbell, Texas snowbells, Styrax texana -n12778398 silver-bell tree, silverbell tree, snowdrop tree, opossum wood, Halesia carolina, Halesia tetraptera -n12778605 carnivorous plant -n12779603 pitcher plant -n12779851 common pitcher plant, huntsman's cup, huntsman's cups, Sarracenia purpurea -n12780325 hooded pitcher plant, Sarracenia minor -n12780563 huntsman's horn, huntsman's horns, yellow trumpet, yellow pitcher plant, trumpets, Sarracenia flava -n12781940 tropical pitcher plant -n12782530 sundew, sundew plant, daily dew -n12782915 Venus's flytrap, Venus's flytraps, Dionaea muscipula -n12783316 waterwheel plant, Aldrovanda vesiculosa -n12783730 Drosophyllum lusitanicum -n12784371 roridula -n12784889 Australian pitcher plant, Cephalotus follicularis -n12785724 sedum -n12785889 stonecrop -n12786273 rose-root, midsummer-men, Sedum rosea -n12786464 orpine, orpin, livelong, live-forever, Sedum telephium -n12786836 pinwheel, Aeonium haworthii -n12787364 Christmas bush, Christmas tree, Ceratopetalum gummiferum -n12788854 hortensia, Hydrangea macrophylla hortensis -n12789054 fall-blooming hydrangea, Hydrangea paniculata -n12789554 carpenteria, Carpenteria californica -n12789977 decumary, Decumaria barbata, Decumaria barbara -n12790430 deutzia -n12791064 philadelphus -n12791329 mock orange, syringa, Philadelphus coronarius -n12793015 saxifrage, breakstone, rockfoil -n12793284 yellow mountain saxifrage, Saxifraga aizoides -n12793494 meadow saxifrage, fair-maids-of-France, Saxifraga granulata -n12793695 mossy saxifrage, Saxifraga hypnoides -n12793886 western saxifrage, Saxifraga occidentalis -n12794135 purple saxifrage, Saxifraga oppositifolia -n12794367 star saxifrage, starry saxifrage, Saxifraga stellaris -n12794568 strawberry geranium, strawberry saxifrage, mother-of-thousands, Saxifraga stolonifera, Saxifraga sarmentosam -n12794985 astilbe -n12795209 false goatsbeard, Astilbe biternata -n12795352 dwarf astilbe, Astilbe chinensis pumila -n12795555 spirea, spiraea, Astilbe japonica -n12796022 bergenia -n12796385 coast boykinia, Boykinia elata, Boykinia occidentalis -n12796849 golden saxifrage, golden spleen -n12797368 umbrella plant, Indian rhubarb, Darmera peltata, Peltiphyllum peltatum -n12797860 bridal wreath, bridal-wreath, Francoa ramosa -n12798284 alumroot, alumbloom -n12798910 coralbells, Heuchera sanguinea -n12799269 leatherleaf saxifrage, Leptarrhena pyrolifolia -n12799776 woodland star, Lithophragma affine, Lithophragma affinis, Tellima affinis -n12800049 prairie star, Lithophragma parviflorum -n12800586 miterwort, mitrewort, bishop's cap -n12801072 five-point bishop's cap, Mitella pentandra -n12801520 parnassia, grass-of-Parnassus -n12801781 bog star, Parnassia palustris -n12801966 fringed grass of Parnassus, Parnassia fimbriata -n12803226 false alumroot, fringe cups, Tellima grandiflora -n12803754 foamflower, coolwart, false miterwort, false mitrewort, Tiarella cordifolia -n12803958 false miterwort, false mitrewort, Tiarella unifoliata -n12804352 pickaback plant, piggyback plant, youth-on-age, Tolmiea menziesii -n12805146 currant, currant bush -n12805561 black currant, European black currant, Ribes nigrum -n12805762 white currant, Ribes sativum -n12806015 gooseberry, gooseberry bush, Ribes uva-crispa, Ribes grossularia -n12806732 plane tree, sycamore, platan -n12807251 London plane, Platanus acerifolia -n12807409 American sycamore, American plane, buttonwood, Platanus occidentalis -n12807624 oriental plane, Platanus orientalis -n12807773 California sycamore, Platanus racemosa -n12808007 Arizona sycamore, Platanus wrightii -n12809868 Greek valerian, Polemonium reptans -n12810007 northern Jacob's ladder, Polemonium boreale -n12810151 skunkweed, skunk-weed, Polemonium viscosum -n12810595 phlox -n12811027 moss pink, mountain phlox, moss phlox, dwarf phlox, Phlox subulata -n12811713 evening-snow, Linanthus dichotomus -n12812235 acanthus -n12812478 bear's breech, bear's breeches, sea holly, Acanthus mollis -n12812801 caricature plant, Graptophyllum pictum -n12813189 black-eyed Susan, black-eyed Susan vine, Thunbergia alata -n12814643 catalpa, Indian bean -n12814857 Catalpa bignioides -n12814960 Catalpa speciosa -n12815198 desert willow, Chilopsis linearis -n12815668 calabash, calabash tree, Crescentia cujete -n12815838 calabash -n12816508 borage, tailwort, Borago officinalis -n12816942 common amsinckia, Amsinckia intermedia -n12817464 anchusa -n12817694 bugloss, alkanet, Anchusa officinalis -n12817855 cape forget-me-not, Anchusa capensis -n12818004 cape forget-me-not, Anchusa riparia -n12818346 Spanish elm, Equador laurel, salmwood, cypre, princewood, Cordia alliodora -n12818601 princewood, Spanish elm, Cordia gerascanthus -n12818966 Chinese forget-me-not, Cynoglossum amabile -n12819141 hound's-tongue, Cynoglossum officinale -n12819354 hound's-tongue, Cynoglossum virginaticum -n12819728 blueweed, blue devil, blue thistle, viper's bugloss, Echium vulgare -n12820113 beggar's lice, beggar lice -n12820669 gromwell, Lithospermum officinale -n12820853 puccoon, Lithospermum caroliniense -n12821505 Virginia bluebell, Virginia cowslip, Mertensia virginica -n12821895 garden forget-me-not, Myosotis sylvatica -n12822115 forget-me-not, mouse ear, Myosotis scorpiodes -n12822466 false gromwell -n12822769 comfrey, cumfrey -n12822955 common comfrey, boneset, Symphytum officinale -n12823717 convolvulus -n12823859 bindweed -n12824053 field bindweed, wild morning-glory, Convolvulus arvensis -n12824289 scammony, Convolvulus scammonia -n12824735 silverweed -n12825497 dodder -n12826143 dichondra, Dichondra micrantha -n12827270 cypress vine, star-glory, Indian pink, Ipomoea quamoclit, Quamoclit pennata -n12827537 moonflower, belle de nuit, Ipomoea alba -n12827907 wild potato vine, wild sweet potato vine, man-of-the-earth, manroot, scammonyroot, Ipomoea panurata, Ipomoea fastigiata -n12828220 red morning-glory, star ipomoea, Ipomoea coccinea -n12828379 man-of-the-earth, Ipomoea leptophylla -n12828520 scammony, Ipomoea orizabensis -n12828791 Japanese morning glory, Ipomoea nil -n12828977 imperial Japanese morning glory, Ipomoea imperialis -n12829582 gesneriad -n12829975 gesneria -n12830222 achimenes, hot water plant -n12830568 aeschynanthus -n12831141 lace-flower vine, Alsobia dianthiflora, Episcia dianthiflora -n12831535 columnea -n12831932 episcia -n12832315 gloxinia -n12832538 Canterbury bell, Gloxinia perennis -n12832822 kohleria -n12833149 African violet, Saintpaulia ionantha -n12833985 streptocarpus -n12834190 Cape primrose -n12834798 waterleaf -n12834938 Virginia waterleaf, Shawnee salad, shawny, Indian salad, John's cabbage, Hydrophyllum virginianum -n12835331 yellow bells, California yellow bells, whispering bells, Emmanthe penduliflora -n12835766 yerba santa, Eriodictyon californicum -n12836212 nemophila -n12836337 baby blue-eyes, Nemophila menziesii -n12836508 five-spot, Nemophila maculata -n12836862 scorpionweed, scorpion weed, phacelia -n12837052 California bluebell, Phacelia campanularia -n12837259 California bluebell, whitlavia, Phacelia minor, Phacelia whitlavia -n12837466 fiddleneck, Phacelia tanacetifolia -n12837803 fiesta flower, Pholistoma auritum, Nemophila aurita -n12839574 basil thyme, basil balm, mother of thyme, Acinos arvensis, Satureja acinos -n12839979 giant hyssop -n12840168 yellow giant hyssop, Agastache nepetoides -n12840362 anise hyssop, Agastache foeniculum -n12840502 Mexican hyssop, Agastache mexicana -n12840749 bugle, bugleweed -n12841007 creeping bugle, Ajuga reptans -n12841193 erect bugle, blue bugle, Ajuga genevensis -n12841354 pyramid bugle, Ajuga pyramidalis -n12842302 wood mint -n12842519 hairy wood mint, Blephilia hirsuta -n12842642 downy wood mint, Blephilia celiata -n12842887 calamint -n12843144 common calamint, Calamintha sylvatica, Satureja calamintha officinalis -n12843316 large-flowered calamint, Calamintha grandiflora, Clinopodium grandiflorum, Satureja grandiflora -n12843557 lesser calamint, field balm, Calamintha nepeta, Calamintha nepeta glantulosa, Satureja nepeta, Satureja calamintha glandulosa -n12843970 wild basil, cushion calamint, Clinopodium vulgare, Satureja vulgaris -n12844409 horse balm, horseweed, stoneroot, stone-root, richweed, stone root, Collinsonia canadensis -n12844939 coleus, flame nettle -n12845187 country borage, Coleus aromaticus, Coleus amboinicus, Plectranthus amboinicus -n12845413 painted nettle, Joseph's coat, Coleus blumei, Solenostemon blumei, Solenostemon scutellarioides -n12845908 Apalachicola rosemary, Conradina glabra -n12846335 dragonhead, dragon's head, Dracocephalum parviflorum -n12846690 elsholtzia -n12847008 hemp nettle, dead nettle, Galeopsis tetrahit -n12847374 ground ivy, alehoof, field balm, gill-over-the-ground, runaway robin, Glechoma hederaceae, Nepeta hederaceae -n12847927 pennyroyal, American pennyroyal, Hedeoma pulegioides -n12848499 hyssop, Hyssopus officinalis -n12849061 dead nettle -n12849279 white dead nettle, Lamium album -n12849416 henbit, Lamium amplexicaule -n12849952 English lavender, Lavandula angustifolia, Lavandula officinalis -n12850168 French lavender, Lavandula stoechas -n12850336 spike lavender, French lavender, Lavandula latifolia -n12850906 dagga, Cape dagga, red dagga, wilde dagga, Leonotis leonurus -n12851094 lion's-ear, Leonotis nepetaefolia, Leonotis nepetifolia -n12851469 motherwort, Leonurus cardiaca -n12851860 pitcher sage, Lepechinia calycina, Sphacele calycina -n12852234 bugleweed, Lycopus virginicus -n12852428 water horehound, Lycopus americanus -n12852570 gipsywort, gypsywort, Lycopus europaeus -n12853080 origanum -n12853287 oregano, marjoram, pot marjoram, wild marjoram, winter sweet, Origanum vulgare -n12853482 sweet marjoram, knotted marjoram, Origanum majorana, Majorana hortensis -n12854048 horehound -n12854193 common horehound, white horehound, Marrubium vulgare -n12854600 lemon balm, garden balm, sweet balm, bee balm, beebalm, Melissa officinalis -n12855365 corn mint, field mint, Mentha arvensis -n12855494 water-mint, water mint, Mentha aquatica -n12855710 bergamot mint, lemon mint, eau de cologne mint, Mentha citrata -n12855886 horsemint, Mentha longifolia -n12856091 peppermint, Mentha piperita -n12856287 spearmint, Mentha spicata -n12856479 apple mint, applemint, Mentha rotundifolia, Mentha suaveolens -n12856680 pennyroyal, Mentha pulegium -n12857204 yerba buena, Micromeria chamissonis, Micromeria douglasii, Satureja douglasii -n12857779 molucca balm, bells of Ireland, Molucella laevis -n12858150 monarda, wild bergamot -n12858397 bee balm, beebalm, bergamot mint, oswego tea, Monarda didyma -n12858618 horsemint, Monarda punctata -n12858871 bee balm, beebalm, Monarda fistulosa -n12858987 lemon mint, horsemint, Monarda citriodora -n12859153 plains lemon monarda, Monarda pectinata -n12859272 basil balm, Monarda clinopodia -n12859679 mustang mint, Monardella lanceolata -n12859986 catmint, catnip, Nepeta cataria -n12860365 basil -n12860978 beefsteak plant, Perilla frutescens crispa -n12861345 phlomis -n12861541 Jerusalem sage, Phlomis fruticosa -n12861892 physostegia -n12862512 plectranthus -n12862828 patchouli, patchouly, pachouli, Pogostemon cablin -n12863234 self-heal, heal all, Prunella vulgaris -n12863624 mountain mint -n12864160 rosemary, Rosmarinus officinalis -n12865037 clary sage, Salvia clarea -n12865562 purple sage, chaparral sage, Salvia leucophylla -n12865708 cancerweed, cancer weed, Salvia lyrata -n12865824 common sage, ramona, Salvia officinalis -n12866002 meadow clary, Salvia pratensis -n12866162 clary, Salvia sclarea -n12866333 pitcher sage, Salvia spathacea -n12866459 Mexican mint, Salvia divinorum -n12866635 wild sage, wild clary, vervain sage, Salvia verbenaca -n12866968 savory -n12867184 summer savory, Satureja hortensis, Satureia hortensis -n12867449 winter savory, Satureja montana, Satureia montana -n12867826 skullcap, helmetflower -n12868019 blue pimpernel, blue skullcap, mad-dog skullcap, mad-dog weed, Scutellaria lateriflora -n12868880 hedge nettle, dead nettle, Stachys sylvatica -n12869061 hedge nettle, Stachys palustris -n12869478 germander -n12869668 American germander, wood sage, Teucrium canadense -n12870048 cat thyme, marum, Teucrium marum -n12870225 wood sage, Teucrium scorodonia -n12870535 thyme -n12870682 common thyme, Thymus vulgaris -n12870891 wild thyme, creeping thyme, Thymus serpyllum -n12871272 blue curls -n12871696 turpentine camphor weed, camphorweed, vinegarweed, Trichostema lanceolatum -n12871859 bastard pennyroyal, Trichostema dichotomum -n12872458 bladderwort -n12872914 butterwort -n12873341 genlisea -n12873984 martynia, Martynia annua -n12875269 common unicorn plant, devil's claw, common devil's claw, elephant-tusk, proboscis flower, ram's horn, Proboscidea louisianica -n12875697 sand devil's claw, Proboscidea arenaria, Martynia arenaria -n12875861 sweet unicorn plant, Proboscidea fragrans, Martynia fragrans -n12876899 figwort -n12877244 snapdragon -n12877493 white snapdragon, Antirrhinum coulterianum -n12877637 yellow twining snapdragon, Antirrhinum filipes -n12877838 Mediterranean snapdragon, Antirrhinum majus -n12878169 kitten-tails -n12878325 Alpine besseya, Besseya alpina -n12878784 false foxglove, Aureolaria pedicularia, Gerardia pedicularia -n12879068 false foxglove, Aureolaria virginica, Gerardia virginica -n12879527 calceolaria, slipperwort -n12879963 Indian paintbrush, painted cup -n12880244 desert paintbrush, Castilleja chromosa -n12880462 giant red paintbrush, Castilleja miniata -n12880638 great plains paintbrush, Castilleja sessiliflora -n12880799 sulfur paintbrush, Castilleja sulphurea -n12881105 shellflower, shell-flower, turtlehead, snakehead, snake-head, Chelone glabra -n12881913 maiden blue-eyed Mary, Collinsia parviflora -n12882158 blue-eyed Mary, Collinsia verna -n12882779 foxglove, digitalis -n12882945 common foxglove, fairy bell, fingerflower, finger-flower, fingerroot, finger-root, Digitalis purpurea -n12883265 yellow foxglove, straw foxglove, Digitalis lutea -n12883628 gerardia -n12884100 blue toadflax, old-field toadflax, Linaria canadensis -n12884260 toadflax, butter-and-eggs, wild snapdragon, devil's flax, Linaria vulgaris -n12885045 golden-beard penstemon, Penstemon barbatus -n12885265 scarlet bugler, Penstemon centranthifolius -n12885510 red shrubby penstemon, redwood penstemon -n12885754 Platte River penstemon, Penstemon cyananthus -n12886185 hot-rock penstemon, Penstemon deustus -n12886402 Jones' penstemon, Penstemon dolius -n12886600 shrubby penstemon, lowbush penstemon, Penstemon fruticosus -n12886831 narrow-leaf penstemon, Penstemon linarioides -n12887293 balloon flower, scented penstemon, Penstemon palmeri -n12887532 Parry's penstemon, Penstemon parryi -n12887713 rock penstemon, cliff penstemon, Penstemon rupicola -n12888016 Rydberg's penstemon, Penstemon rydbergii -n12888234 cascade penstemon, Penstemon serrulatus -n12888457 Whipple's penstemon, Penstemon whippleanus -n12889219 moth mullein, Verbascum blattaria -n12889412 white mullein, Verbascum lychnitis -n12889579 purple mullein, Verbascum phoeniceum -n12889713 common mullein, great mullein, Aaron's rod, flannel mullein, woolly mullein, torch, Verbascum thapsus -n12890265 veronica, speedwell -n12890490 field speedwell, Veronica agrestis -n12890685 brooklime, American brooklime, Veronica americana -n12890928 corn speedwell, Veronica arvensis -n12891093 brooklime, European brooklime, Veronica beccabunga -n12891305 germander speedwell, bird's eye, Veronica chamaedrys -n12891469 water speedwell, Veronica michauxii, Veronica anagallis-aquatica -n12891643 common speedwell, gypsyweed, Veronica officinalis -n12891824 purslane speedwell, Veronica peregrina -n12892013 thyme-leaved speedwell, Veronica serpyllifolia -n12893463 nightshade -n12893993 horse nettle, ball nettle, bull nettle, ball nightshade, Solanum carolinense -n12895298 African holly, Solanum giganteum -n12895811 potato vine, Solanum jasmoides -n12896615 garden huckleberry, wonderberry, sunberry, Solanum nigrum guineese, Solanum melanocerasum, Solanum burbankii -n12897118 naranjilla, Solanum quitoense -n12897788 potato vine, giant potato creeper, Solanum wendlandii -n12897999 potato tree, Brazilian potato tree, Solanum wrightii, Solanum macranthum -n12898342 belladonna, belladonna plant, deadly nightshade, Atropa belladonna -n12898774 bush violet, browallia -n12899166 lady-of-the-night, Brunfelsia americana -n12899537 angel's trumpet, maikoa, Brugmansia arborea, Datura arborea -n12899752 angel's trumpet, Brugmansia suaveolens, Datura suaveolens -n12899971 red angel's trumpet, Brugmansia sanguinea, Datura sanguinea -n12900783 cone pepper, Capsicum annuum conoides -n12901724 bird pepper, Capsicum frutescens baccatum, Capsicum baccatum -n12902466 day jessamine, Cestrum diurnum -n12902662 night jasmine, night jessamine, Cestrum nocturnum -n12903014 tree tomato, tamarillo -n12903367 thorn apple -n12903503 jimsonweed, jimson weed, Jamestown weed, common thorn apple, apple of Peru, Datura stramonium -n12903964 pichi, Fabiana imbricata -n12904314 henbane, black henbane, stinking nightshade, Hyoscyamus niger -n12904562 Egyptian henbane, Hyoscyamus muticus -n12904938 matrimony vine, boxthorn -n12905135 common matrimony vine, Duke of Argyll's tea tree, Lycium barbarum, Lycium halimifolium -n12905412 Christmasberry, Christmas berry, Lycium carolinianum -n12906214 plum tomato -n12906498 mandrake, devil's apples, Mandragora officinarum -n12906771 mandrake root, mandrake -n12907057 apple of Peru, shoo fly, Nicandra physaloides -n12907671 flowering tobacco, Jasmine tobacco, Nicotiana alata -n12907857 common tobacco, Nicotiana tabacum -n12908093 wild tobacco, Indian tobacco, Nicotiana rustica -n12908645 cupflower, nierembergia -n12908854 whitecup, Nierembergia repens, Nierembergia rivularis -n12909421 petunia -n12909614 large white petunia, Petunia axillaris -n12909759 violet-flowered petunia, Petunia integrifolia -n12909917 hybrid petunia, Petunia hybrida -n12911079 cape gooseberry, purple ground cherry, Physalis peruviana -n12911264 strawberry tomato, dwarf cape gooseberry, Physalis pruinosa -n12911440 tomatillo, jamberry, Mexican husk tomato, Physalis ixocarpa -n12911673 tomatillo, miltomate, purple ground cherry, jamberry, Physalis philadelphica -n12911914 yellow henbane, Physalis viscosa -n12912274 cock's eggs, Salpichroa organifolia, Salpichroa rhomboidea -n12912670 salpiglossis -n12912801 painted tongue, Salpiglossis sinuata -n12913144 butterfly flower, poor man's orchid, schizanthus -n12913524 Scopolia carniolica -n12913791 chalice vine, trumpet flower, cupflower, Solandra guttata -n12914923 verbena, vervain -n12915140 lantana -n12915568 black mangrove, Avicennia marina -n12915811 white mangrove, Avicennia officinalis -n12916179 black mangrove, Aegiceras majus -n12916511 teak, Tectona grandis -n12917901 spurge -n12918609 sun spurge, wartweed, wartwort, devil's milk, Euphorbia helioscopia -n12918810 petty spurge, devil's milk, Euphorbia peplus -n12918991 medusa's head, Euphorbia medusae, Euphorbia caput-medusae -n12919195 wild spurge, flowering spurge, tramp's spurge, Euphorbia corollata -n12919403 snow-on-the-mountain, snow-in-summer, ghost weed, Euphorbia marginata -n12919646 cypress spurge, Euphorbia cyparissias -n12919847 leafy spurge, wolf's milk, Euphorbia esula -n12920043 hairy spurge, Euphorbia hirsuta -n12920204 poinsettia, Christmas star, Christmas flower, lobster plant, Mexican flameleaf, painted leaf, Euphorbia pulcherrima -n12920521 Japanese poinsettia, mole plant, paint leaf, Euphorbia heterophylla -n12920719 fire-on-the-mountain, painted leaf, Mexican fire plant, Euphorbia cyathophora -n12920955 wood spurge, Euphorbia amygdaloides -n12921315 dwarf spurge, Euphorbia exigua -n12921499 scarlet plume, Euphorbia fulgens -n12921660 naboom, cactus euphorbia, Euphorbia ingens -n12921868 crown of thorns, Christ thorn, Christ plant, Euphorbia milii -n12922119 toothed spurge, Euphorbia dentata -n12922458 three-seeded mercury, Acalypha virginica -n12922763 croton, Croton tiglium -n12923108 cascarilla, Croton eluteria -n12923257 cascarilla bark, eleuthera bark, sweetwood bark -n12924623 castor-oil plant, castor bean plant, palma christi, palma christ, Ricinus communis -n12925179 spurge nettle, tread-softly, devil nettle, pica-pica, Cnidoscolus urens, Jatropha urens, Jatropha stimulosus -n12925583 physic nut, Jatropha curcus -n12926039 Para rubber tree, caoutchouc tree, Hevea brasiliensis -n12926480 cassava, casava -n12926689 bitter cassava, manioc, mandioc, mandioca, tapioca plant, gari, Manihot esculenta, Manihot utilissima -n12927013 cassava, manioc -n12927194 sweet cassava, Manihot dulcis -n12927494 candlenut, varnish tree, Aleurites moluccana -n12927758 tung tree, tung, tung-oil tree, Aleurites fordii -n12928071 slipper spurge, slipper plant -n12928307 candelilla, Pedilanthus bracteatus, Pedilanthus pavonis -n12928491 Jewbush, Jew-bush, Jew bush, redbird cactus, redbird flower, Pedilanthus tithymaloides -n12928819 jumping bean, jumping seed, Mexican jumping bean -n12929403 camellia, camelia -n12929600 japonica, Camellia japonica -n12930778 umbellifer, umbelliferous plant -n12930951 wild parsley -n12931231 fool's parsley, lesser hemlock, Aethusa cynapium -n12931542 dill, Anethum graveolens -n12931906 angelica, angelique -n12932173 garden angelica, archangel, Angelica Archangelica -n12932365 wild angelica, Angelica sylvestris -n12932706 chervil, beaked parsley, Anthriscus cereifolium -n12932966 cow parsley, wild chervil, Anthriscus sylvestris -n12933274 wild celery, Apium graveolens -n12934036 astrantia, masterwort -n12934174 greater masterwort, Astrantia major -n12934479 caraway, Carum carvi -n12934685 whorled caraway -n12934985 water hemlock, Cicuta verosa -n12935166 spotted cowbane, spotted hemlock, spotted water hemlock -n12935609 hemlock, poison hemlock, poison parsley, California fern, Nebraska fern, winter fern, Conium maculatum -n12936155 earthnut, Conopodium denudatum -n12936826 cumin, Cuminum cyminum -n12937130 wild carrot, Queen Anne's lace, Daucus carota -n12938081 eryngo, eringo -n12938193 sea holly, sea holm, sea eryngium, Eryngium maritimum -n12938445 button snakeroot, Eryngium aquaticum -n12938667 rattlesnake master, rattlesnake's master, button snakeroot, Eryngium yuccifolium -n12939104 fennel -n12939282 common fennel, Foeniculum vulgare -n12939479 Florence fennel, Foeniculum dulce, Foeniculum vulgare dulce -n12939874 cow parsnip, hogweed, Heracleum sphondylium -n12940226 lovage, Levisticum officinale -n12940609 sweet cicely, Myrrhis odorata -n12941220 water fennel, Oenanthe aquatica -n12941536 parsnip, Pastinaca sativa -n12941717 cultivated parsnip -n12942025 wild parsnip, madnep -n12942395 parsley, Petroselinum crispum -n12942572 Italian parsley, flat-leaf parsley, Petroselinum crispum neapolitanum -n12942729 Hamburg parsley, turnip-rooted parsley, Petroselinum crispum tuberosum -n12943049 anise, anise plant, Pimpinella anisum -n12943443 sanicle, snakeroot -n12943912 purple sanicle, Sanicula bipinnatifida -n12944095 European sanicle, Sanicula Europaea -n12945177 water parsnip, Sium suave -n12945366 greater water parsnip, Sium latifolium -n12945549 skirret, Sium sisarum -n12946849 dogwood, dogwood tree, cornel -n12947313 common white dogwood, eastern flowering dogwood, Cornus florida -n12947544 red osier, red osier dogwood, red dogwood, American dogwood, redbrush, Cornus stolonifera -n12947756 silky dogwood, Cornus obliqua -n12947895 silky cornel, silky dogwood, Cornus amomum -n12948053 common European dogwood, red dogwood, blood-twig, pedwood, Cornus sanguinea -n12948251 bunchberry, dwarf cornel, crackerberry, pudding berry, Cornus canadensis -n12948495 cornelian cherry, Cornus mas -n12949160 puka, Griselinia lucida -n12949361 kapuka, Griselinia littoralis -n12950126 valerian -n12950314 common valerian, garden heliotrope, Valeriana officinalis -n12950796 common corn salad, lamb's lettuce, Valerianella olitoria, Valerianella locusta -n12951146 red valerian, French honeysuckle, Centranthus ruber -n12951835 filmy fern, film fern -n12952165 bristle fern, filmy fern -n12952469 hare's-foot bristle fern, Trichomanes boschianum -n12952590 Killarney fern, Trichomanes speciosum -n12952717 kidney fern, Trichomanes reniforme -n12953206 flowering fern, osmund -n12953484 royal fern, royal osmund, king fern, ditch fern, French bracken, Osmunda regalis -n12953712 interrupted fern, Osmunda clatonia -n12954353 crape fern, Prince-of-Wales fern, Prince-of-Wales feather, Prince-of-Wales plume, Leptopteris superba, Todea superba -n12954799 crepe fern, king fern, Todea barbara -n12955414 curly grass, curly grass fern, Schizaea pusilla -n12955840 pine fern, Anemia adiantifolia -n12956170 climbing fern -n12956367 creeping fern, Hartford fern, Lygodium palmatum -n12956588 climbing maidenhair, climbing maidenhair fern, snake fern, Lygodium microphyllum -n12956922 scented fern, Mohria caffrorum -n12957608 clover fern, pepperwort -n12957803 nardoo, nardo, common nardoo, Marsilea drummondii -n12957924 water clover, Marsilea quadrifolia -n12958261 pillwort, Pilularia globulifera -n12958615 regnellidium, Regnellidium diphyllum -n12959074 floating-moss, Salvinia rotundifolia, Salvinia auriculata -n12959538 mosquito fern, floating fern, Carolina pond fern, Azolla caroliniana -n12960378 adder's tongue, adder's tongue fern -n12960552 ribbon fern, Ophioglossum pendulum -n12960863 grape fern -n12961242 daisyleaf grape fern, daisy-leaved grape fern, Botrychium matricariifolium -n12961393 leathery grape fern, Botrychium multifidum -n12961536 rattlesnake fern, Botrychium virginianum -n12961879 flowering fern, Helminthostachys zeylanica -n12963628 powdery mildew -n12964920 Dutch elm fungus, Ceratostomella ulmi -n12965626 ergot, Claviceps purpurea -n12965951 rye ergot -n12966804 black root rot fungus, Xylaria mali -n12966945 dead-man's-fingers, dead-men's-fingers, Xylaria polymorpha -n12968136 sclerotinia -n12968309 brown cup -n12969131 earthball, false truffle, puffball, hard-skinned puffball -n12969425 Scleroderma citrinum, Scleroderma aurantium -n12969670 Scleroderma flavidium, star earthball -n12969927 Scleroderma bovista, smooth earthball -n12970193 Podaxaceae -n12970293 stalked puffball -n12970733 stalked puffball -n12971400 false truffle -n12971804 Rhizopogon idahoensis -n12972136 Truncocolumella citrina -n12973443 mucor -n12973791 rhizopus -n12973937 bread mold, Rhizopus nigricans -n12974987 slime mold, slime mould -n12975804 true slime mold, acellular slime mold, plasmodial slime mold, myxomycete -n12976198 cellular slime mold -n12976554 dictostylium -n12978076 pond-scum parasite -n12979316 potato wart fungus, Synchytrium endobioticum -n12979829 white fungus, Saprolegnia ferax -n12980080 water mold -n12980840 downy mildew, false mildew -n12981086 blue mold fungus, Peronospora tabacina -n12981301 onion mildew, Peronospora destructor -n12981443 tobacco mildew, Peronospora hyoscyami -n12981954 white rust -n12982468 pythium -n12982590 damping off fungus, Pythium debaryanum -n12982915 Phytophthora citrophthora -n12983048 Phytophthora infestans -n12983654 clubroot fungus, Plasmodiophora brassicae -n12983873 Geglossaceae -n12983961 Sarcosomataceae -n12984267 Rufous rubber cup -n12984489 devil's cigar -n12984595 devil's urn -n12985420 truffle, earthnut, earth-ball -n12985773 club fungus -n12985857 coral fungus -n12986227 tooth fungus -n12987056 lichen -n12987423 ascolichen -n12987535 basidiolichen -n12988158 lecanora -n12988341 manna lichen -n12988572 archil, orchil -n12989007 roccella, Roccella tinctoria -n12989938 beard lichen, beard moss, Usnea barbata -n12990597 horsehair lichen, horsetail lichen -n12991184 reindeer moss, reindeer lichen, arctic moss, Cladonia rangiferina -n12991837 crottle, crottal, crotal -n12992177 Iceland moss, Iceland lichen, Cetraria islandica -n12992868 fungus -n12994892 promycelium -n12995601 true fungus -n12997654 basidiomycete, basidiomycetous fungi -n12997919 mushroom -n12998815 agaric -n13000891 mushroom -n13001041 mushroom -n13001206 toadstool -n13001366 horse mushroom, Agaricus arvensis -n13001529 meadow mushroom, field mushroom, Agaricus campestris -n13001930 shiitake, shiitake mushroom, Chinese black mushroom, golden oak mushroom, Oriental black mushroom, Lentinus edodes -n13002209 scaly lentinus, Lentinus lepideus -n13002750 royal agaric, Caesar's agaric, Amanita caesarea -n13002925 false deathcap, Amanita mappa -n13003061 fly agaric, Amanita muscaria -n13003254 death cap, death cup, death angel, destroying angel, Amanita phalloides -n13003522 blushing mushroom, blusher, Amanita rubescens -n13003712 destroying angel, Amanita verna -n13004423 chanterelle, chantarelle, Cantharellus cibarius -n13004640 floccose chanterelle, Cantharellus floccosus -n13004826 pig's ears, Cantharellus clavatus -n13004992 cinnabar chanterelle, Cantharellus cinnabarinus -n13005329 jack-o-lantern fungus, jack-o-lantern, jack-a-lantern, Omphalotus illudens -n13005984 inky cap, inky-cap mushroom, Coprinus atramentarius -n13006171 shaggymane, shaggy cap, shaggymane mushroom, Coprinus comatus -n13006631 milkcap, Lactarius delicioso -n13006894 fairy-ring mushroom, Marasmius oreades -n13007034 fairy ring, fairy circle -n13007417 oyster mushroom, oyster fungus, oyster agaric, Pleurotus ostreatus -n13007629 olive-tree agaric, Pleurotus phosphoreus -n13008157 Pholiota astragalina -n13008315 Pholiota aurea, golden pholiota -n13008485 Pholiota destruens -n13008689 Pholiota flammans -n13008839 Pholiota flavida -n13009085 nameko, viscid mushroom, Pholiota nameko -n13009244 Pholiota squarrosa-adiposa -n13009429 Pholiota squarrosa, scaly pholiota -n13009656 Pholiota squarrosoides -n13010694 Stropharia ambigua -n13010951 Stropharia hornemannii -n13011221 Stropharia rugoso-annulata -n13011595 gill fungus -n13012253 Entoloma lividum, Entoloma sinuatum -n13012469 Entoloma aprile -n13012973 Chlorophyllum molybdites -n13013534 lepiota -n13013764 parasol mushroom, Lepiota procera -n13013965 poisonous parasol, Lepiota morgani -n13014097 Lepiota naucina -n13014265 Lepiota rhacodes -n13014409 American parasol, Lepiota americana -n13014581 Lepiota rubrotincta -n13014741 Lepiota clypeolaria -n13014879 onion stem, Lepiota cepaestipes -n13015509 pink disease fungus, Corticium salmonicolor -n13015688 bottom rot fungus, Corticium solani -n13016076 potato fungus, Pellicularia filamentosa, Rhizoctinia solani -n13016289 coffee fungus, Pellicularia koleroga -n13017102 blewits, Clitocybe nuda -n13017240 sandy mushroom, Tricholoma populinum -n13017439 Tricholoma pessundatum -n13017610 Tricholoma sejunctum -n13017789 man-on-a-horse, Tricholoma flavovirens -n13017979 Tricholoma venenata -n13018088 Tricholoma pardinum -n13018232 Tricholoma vaccinum -n13018407 Tricholoma aurantium -n13018906 Volvaria bombycina -n13019496 Pluteus aurantiorugosus -n13019643 Pluteus magnus, sawdust mushroom -n13019835 deer mushroom, Pluteus cervinus -n13020191 straw mushroom, Chinese mushroom, Volvariella volvacea -n13020481 Volvariella bombycina -n13020964 Clitocybe clavipes -n13021166 Clitocybe dealbata -n13021332 Clitocybe inornata -n13021543 Clitocybe robusta, Clytocybe alba -n13021689 Clitocybe irina, Tricholoma irinum, Lepista irina -n13021867 Clitocybe subconnexa -n13022210 winter mushroom, Flammulina velutipes -n13022709 mycelium -n13022903 sclerotium -n13023134 sac fungus -n13024012 ascomycete, ascomycetous fungus -n13024500 Clavicipitaceae, grainy club mushrooms -n13024653 grainy club -n13025647 yeast -n13025854 baker's yeast, brewer's yeast, Saccharomyces cerevisiae -n13026015 wine-maker's yeast, Saccharomyces ellipsoides -n13027557 Aspergillus fumigatus -n13027879 brown root rot fungus, Thielavia basicola -n13028611 discomycete, cup fungus -n13028937 Leotia lubrica -n13029122 Mitrula elegans -n13029326 Sarcoscypha coccinea, scarlet cup -n13029610 Caloscypha fulgens -n13029760 Aleuria aurantia, orange peel fungus -n13030337 elf cup -n13030616 Peziza domicilina -n13030852 blood cup, fairy cup, Peziza coccinea -n13031193 Urnula craterium, urn fungus -n13031323 Galiella rufa -n13031474 Jafnea semitosta -n13032115 morel -n13032381 common morel, Morchella esculenta, sponge mushroom, sponge morel -n13032618 Disciotis venosa, cup morel -n13032923 Verpa, bell morel -n13033134 Verpa bohemica, early morel -n13033396 Verpa conica, conic Verpa -n13033577 black morel, Morchella conica, conic morel, Morchella angusticeps, narrowhead morel -n13033879 Morchella crassipes, thick-footed morel -n13034062 Morchella semilibera, half-free morel, cow's head -n13034555 Wynnea americana -n13034788 Wynnea sparassoides -n13035241 false morel -n13035389 lorchel -n13035707 helvella -n13035925 Helvella crispa, miter mushroom -n13036116 Helvella acetabulum -n13036312 Helvella sulcata -n13036804 discina -n13037406 gyromitra -n13037585 Gyromitra californica, California false morel -n13037805 Gyromitra sphaerospora, round-spored gyromitra -n13038068 Gyromitra esculenta, brain mushroom, beefsteak morel -n13038376 Gyromitra infula, saddled-shaped false morel -n13038577 Gyromitra fastigiata, Gyromitra brunnea -n13038744 Gyromitra gigas -n13039349 gasteromycete, gastromycete -n13040303 stinkhorn, carrion fungus -n13040629 common stinkhorn, Phallus impudicus -n13040796 Phallus ravenelii -n13041312 dog stinkhorn, Mutinus caninus -n13041943 Calostoma lutescens -n13042134 Calostoma cinnabarina -n13042316 Calostoma ravenelii -n13042982 stinky squid, Pseudocolus fusiformis -n13043926 puffball, true puffball -n13044375 giant puffball, Calvatia gigantea -n13044778 earthstar -n13045210 Geastrum coronatum -n13045594 Radiigera fuscogleba -n13045975 Astreus pteridis -n13046130 Astreus hygrometricus -n13046669 bird's-nest fungus -n13047862 Gastrocybe lateritia -n13048447 Macowanites americanus -n13049953 polypore, pore fungus, pore mushroom -n13050397 bracket fungus, shelf fungus -n13050705 Albatrellus dispansus -n13050940 Albatrellus ovinus, sheep polypore -n13051346 Neolentinus ponderosus -n13052014 Oligoporus leucospongia -n13052248 Polyporus tenuiculus -n13052670 hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa -n13052931 Polyporus squamosus, scaly polypore -n13053608 beefsteak fungus, Fistulina hepatica -n13054073 agaric, Fomes igniarius -n13054560 bolete -n13055423 Boletus chrysenteron -n13055577 Boletus edulis -n13055792 Frost's bolete, Boletus frostii -n13055949 Boletus luridus -n13056135 Boletus mirabilis -n13056349 Boletus pallidus -n13056607 Boletus pulcherrimus -n13056799 Boletus pulverulentus -n13057054 Boletus roxanae -n13057242 Boletus subvelutipes -n13057422 Boletus variipes -n13057639 Boletus zelleri -n13058037 Fuscoboletinus paluster -n13058272 Fuscoboletinus serotinus -n13058608 Leccinum fibrillosum -n13059298 Suillus albivelatus -n13059657 old-man-of-the-woods, Strobilomyces floccopus -n13060017 Boletellus russellii -n13060190 jelly fungus -n13061172 snow mushroom, Tremella fuciformis -n13061348 witches' butter, Tremella lutescens -n13061471 Tremella foliacea -n13061704 Tremella reticulata -n13062421 Jew's-ear, Jew's-ears, ear fungus, Auricularia auricula -n13063269 rust, rust fungus -n13063514 aecium -n13064111 flax rust, flax rust fungus, Melampsora lini -n13064457 blister rust, Cronartium ribicola -n13065089 wheat rust, Puccinia graminis -n13065514 apple rust, cedar-apple rust, Gymnosporangium juniperi-virginianae -n13066129 smut, smut fungus -n13066448 covered smut -n13066979 loose smut -n13067191 cornsmut, corn smut -n13067330 boil smut, Ustilago maydis -n13067532 Sphacelotheca, genus Sphacelotheca -n13067672 head smut, Sphacelotheca reiliana -n13068255 bunt, Tilletia caries -n13068434 bunt, stinking smut, Tilletia foetida -n13068735 onion smut, Urocystis cepulae -n13068917 flag smut fungus -n13069224 wheat flag smut, Urocystis tritici -n13069773 felt fungus, Septobasidium pseudopedicellatum -n13070308 waxycap -n13070875 Hygrocybe acutoconica, conic waxycap -n13071371 Hygrophorus borealis -n13071553 Hygrophorus caeruleus -n13071815 Hygrophorus inocybiformis -n13072031 Hygrophorus kauffmanii -n13072209 Hygrophorus marzuolus -n13072350 Hygrophorus purpurascens -n13072528 Hygrophorus russula -n13072706 Hygrophorus sordidus -n13072863 Hygrophorus tennesseensis -n13073055 Hygrophorus turundus -n13073703 Neohygrophorus angelesianus -n13074619 Cortinarius armillatus -n13074814 Cortinarius atkinsonianus -n13075020 Cortinarius corrugatus -n13075272 Cortinarius gentilis -n13075441 Cortinarius mutabilis, purple-staining Cortinarius -n13075684 Cortinarius semisanguineus -n13075847 Cortinarius subfoetidus -n13076041 Cortinarius violaceus -n13076405 Gymnopilus spectabilis -n13076643 Gymnopilus validipes -n13076831 Gymnopilus ventricosus -n13077033 mold, mould -n13077295 mildew -n13078021 verticillium -n13079073 monilia -n13079419 candida -n13079567 Candida albicans, Monilia albicans -n13080306 blastomycete -n13080866 yellow spot fungus, Cercospora kopkei -n13081229 green smut fungus, Ustilaginoidea virens -n13081999 dry rot -n13082568 rhizoctinia -n13083023 houseplant -n13083461 bedder, bedding plant -n13084184 succulent -n13084834 cultivar -n13085113 weed -n13085747 wort -n13090018 brier -n13090871 aril -n13091620 sporophyll, sporophyl -n13091774 sporangium, spore case, spore sac -n13091982 sporangiophore -n13092078 ascus -n13092240 ascospore -n13092385 arthrospore -n13092987 eusporangium -n13093275 tetrasporangium -n13093629 gametangium -n13094145 sorus -n13094273 sorus -n13095013 partial veil -n13096779 lignum -n13098515 vascular ray, medullary ray -n13098962 phloem, bast -n13099833 evergreen, evergreen plant -n13099999 deciduous plant -n13100156 poisonous plant -n13100677 vine -n13102648 creeper -n13102775 tendril -n13103023 root climber -n13103660 lignosae -n13103750 arborescent plant -n13103877 snag -n13104059 tree -n13107694 timber tree -n13107807 treelet -n13107891 arbor -n13108131 bean tree -n13108323 pollard -n13108481 sapling -n13108545 shade tree -n13108662 gymnospermous tree -n13108841 conifer, coniferous tree -n13109733 angiospermous tree, flowering tree -n13110915 nut tree -n13111174 spice tree -n13111340 fever tree -n13111504 stump, tree stump -n13111881 bonsai -n13112035 ming tree -n13112201 ming tree -n13118330 undershrub -n13118707 subshrub, suffrutex -n13119870 bramble -n13120211 liana -n13120958 geophyte -n13121104 desert plant, xerophyte, xerophytic plant, xerophile, xerophilous plant -n13121349 mesophyte, mesophytic plant -n13122364 marsh plant, bog plant, swamp plant -n13123309 hemiepiphyte, semiepiphyte -n13123431 strangler, strangler tree -n13123841 lithophyte, lithophytic plant -n13124358 saprobe -n13124654 autophyte, autophytic plant, autotroph, autotrophic organism -n13125117 root -n13126050 taproot -n13126856 prop root -n13127001 prophyll -n13127303 rootstock -n13127666 quickset -n13127843 stolon, runner, offset -n13128278 tuberous plant -n13128582 rhizome, rootstock, rootstalk -n13128976 rachis -n13129078 caudex -n13130014 cladode, cladophyll, phylloclad, phylloclade -n13130161 receptacle -n13130726 scape, flower stalk -n13131028 umbel -n13131618 petiole, leafstalk -n13132034 peduncle -n13132156 pedicel, pedicle -n13132338 flower cluster -n13132486 raceme -n13132656 panicle -n13132756 thyrse, thyrsus -n13132940 cyme -n13133140 cymule -n13133233 glomerule -n13133316 scorpioid cyme -n13133613 ear, spike, capitulum -n13133932 spadix -n13134302 bulbous plant -n13134531 bulbil, bulblet -n13134844 cormous plant -n13134947 fruit -n13135692 fruitlet -n13135832 seed -n13136316 bean -n13136556 nut -n13136781 nutlet -n13137010 kernel, meat -n13137225 syconium -n13137409 berry -n13137672 aggregate fruit, multiple fruit, syncarp -n13137951 simple fruit, bacca -n13138155 acinus -n13138308 drupe, stone fruit -n13138658 drupelet -n13138842 pome, false fruit -n13139055 pod, seedpod -n13139321 loment -n13139482 pyxidium, pyxis -n13139647 husk -n13139837 cornhusk -n13140049 pod, cod, seedcase -n13140367 accessory fruit, pseudocarp -n13141141 buckthorn -n13141415 buckthorn berry, yellow berry -n13141564 cascara buckthorn, bearberry, bearwood, chittamwood, chittimwood, Rhamnus purshianus -n13141797 cascara, cascara sagrada, chittam bark, chittem bark -n13141972 Carolina buckthorn, indian cherry, Rhamnus carolinianus -n13142182 coffeeberry, California buckthorn, California coffee, Rhamnus californicus -n13142504 redberry, red-berry, Rhamnus croceus -n13142907 nakedwood -n13143285 jujube, jujube bush, Christ's-thorn, Jerusalem thorn, Ziziphus jujuba -n13143758 Christ's-thorn, Jerusalem thorn, Paliurus spina-christi -n13144084 hazel, hazel tree, Pomaderris apetala -n13145040 fox grape, Vitis labrusca -n13145250 muscadine, Vitis rotundifolia -n13145444 vinifera, vinifera grape, common grape vine, Vitis vinifera -n13146403 Pinot blanc -n13146583 Sauvignon grape -n13146928 Sauvignon blanc -n13147153 Muscadet -n13147270 Riesling -n13147386 Zinfandel -n13147532 Chenin blanc -n13147689 malvasia -n13147918 Verdicchio -n13148208 Boston ivy, Japanese ivy, Parthenocissus tricuspidata -n13148384 Virginia creeper, American ivy, woodbine, Parthenocissus quinquefolia -n13149296 true pepper, pepper vine -n13149970 betel, betel pepper, Piper betel -n13150378 cubeb -n13150592 schizocarp -n13150894 peperomia -n13151082 watermelon begonia, Peperomia argyreia, Peperomia sandersii -n13152339 yerba mansa, Anemopsis californica -n13154388 pinna, pinnule -n13154494 frond -n13154841 bract -n13155095 bracteole, bractlet -n13155305 involucre -n13155611 glume -n13156986 palmate leaf -n13157137 pinnate leaf -n13157346 bijugate leaf, bijugous leaf, twice-pinnate -n13157481 decompound leaf -n13157684 acuminate leaf -n13157971 deltoid leaf -n13158167 ensiform leaf -n13158512 linear leaf, elongate leaf -n13158605 lyrate leaf -n13158714 obtuse leaf -n13158815 oblanceolate leaf -n13159357 pandurate leaf, panduriform leaf -n13159691 reniform leaf -n13159890 spatulate leaf -n13160116 even-pinnate leaf, abruptly-pinnate leaf -n13160254 odd-pinnate leaf -n13160365 pedate leaf -n13160604 crenate leaf -n13160831 dentate leaf -n13160938 denticulate leaf -n13161151 erose leaf -n13161254 runcinate leaf -n13161904 prickly-edged leaf -n13163553 deadwood -n13163649 haulm, halm -n13163991 branchlet, twig, sprig -n13164501 osier -n13170840 giant scrambling fern, Diplopterygium longissimum -n13171210 umbrella fern, fan fern, Sticherus flabellatus, Gleichenia flabellata -n13171797 floating fern, water sprite, Ceratopteris pteridioides -n13172923 polypody -n13173132 licorice fern, Polypodium glycyrrhiza -n13173259 grey polypody, gray polypody, resurrection fern, Polypodium polypodioides -n13173488 leatherleaf, leathery polypody, coast polypody, Polypodium scouleri -n13173697 rock polypody, rock brake, American wall fern, Polypodium virgianum -n13173882 common polypody, adder's fern, wall fern, golden maidenhair, golden polypody, sweet fern, Polypodium vulgare -n13174354 bear's-paw fern, Aglaomorpha meyeniana -n13174670 strap fern -n13174823 Florida strap fern, cow-tongue fern, hart's-tongue fern -n13175682 basket fern, Drynaria rigidula -n13176363 snake polypody, Microgramma-piloselloides -n13176714 climbing bird's nest fern, Microsorium punctatum -n13177048 golden polypody, serpent fern, rabbit's-foot fern, Phlebodium aureum, Polypodium aureum -n13177529 staghorn fern -n13177768 South American staghorn, Platycerium andinum -n13177884 common staghorn fern, elkhorn fern, Platycerium bifurcatum, Platycerium alcicorne -n13178284 felt fern, tongue fern, Pyrrosia lingua, Cyclophorus lingua -n13178707 potato fern, Solanopteris bifrons -n13179056 myrmecophyte -n13179804 grass fern, ribbon fern, Vittaria lineata -n13180534 spleenwort -n13180875 black spleenwort, Asplenium adiantum-nigrum -n13181055 bird's nest fern, Asplenium nidus -n13181244 ebony spleenwort, Scott's Spleenwort, Asplenium platyneuron -n13181406 black-stem spleenwort, black-stemmed spleenwort, little ebony spleenwort -n13181811 walking fern, walking leaf, Asplenium rhizophyllum, Camptosorus rhizophyllus -n13182164 green spleenwort, Asplenium viride -n13182338 mountain spleenwort, Asplenium montanum -n13182799 lobed spleenwort, Asplenium pinnatifidum -n13182937 lanceolate spleenwort, Asplenium billotii -n13183056 hart's-tongue, hart's-tongue fern, Asplenium scolopendrium, Phyllitis scolopendrium -n13183489 scale fern, scaly fern, Asplenium ceterach, Ceterach officinarum -n13184394 scolopendrium -n13185269 deer fern, Blechnum spicant -n13185658 doodia, rasp fern -n13186388 chain fern -n13186546 Virginia chain fern, Woodwardia virginica -n13187367 silver tree fern, sago fern, black tree fern, Cyathea medullaris -n13188096 davallia -n13188268 hare's-foot fern -n13188462 Canary Island hare's foot fern, Davallia canariensis -n13188767 squirrel's-foot fern, ball fern, Davalia bullata, Davalia bullata mariesii, Davallia Mariesii -n13190060 bracken, Pteridium esculentum -n13190747 soft tree fern, Dicksonia antarctica -n13191148 Scythian lamb, Cibotium barometz -n13191620 false bracken, Culcita dubia -n13191884 thyrsopteris, Thyrsopteris elegans -n13192625 shield fern, buckler fern -n13193143 broad buckler-fern, Dryopteris dilatata -n13193269 fragrant cliff fern, fragrant shield fern, fragrant wood fern, Dryopteris fragrans -n13193466 Goldie's fern, Goldie's shield fern, goldie's wood fern, Dryopteris goldiana -n13193642 wood fern, wood-fern, woodfern -n13193856 male fern, Dryopteris filix-mas -n13194036 marginal wood fern, evergreen wood fern, leatherleaf wood fern, Dryopteris marginalis -n13194212 mountain male fern, Dryopteris oreades -n13194572 lady fern, Athyrium filix-femina -n13194758 Alpine lady fern, Athyrium distentifolium -n13194918 silvery spleenwort, glade fern, narrow-leaved spleenwort, Athyrium pycnocarpon, Diplazium pycnocarpon -n13195341 holly fern, Cyrtomium aculeatum, Polystichum aculeatum -n13195761 bladder fern -n13196003 brittle bladder fern, brittle fern, fragile fern, Cystopteris fragilis -n13196234 mountain bladder fern, Cystopteris montana -n13196369 bulblet fern, bulblet bladder fern, berry fern, Cystopteris bulbifera -n13196738 silvery spleenwort, Deparia acrostichoides, Athyrium thelypteroides -n13197274 oak fern, Gymnocarpium dryopteris, Thelypteris dryopteris -n13197507 limestone fern, northern oak fern, Gymnocarpium robertianum -n13198054 ostrich fern, shuttlecock fern, fiddlehead, Matteuccia struthiopteris, Pteretis struthiopteris, Onoclea struthiopteris -n13198482 hart's-tongue, hart's-tongue fern, Olfersia cervina, Polybotrya cervina, Polybotria cervina -n13198914 sensitive fern, bead fern, Onoclea sensibilis -n13199717 Christmas fern, canker brake, dagger fern, evergreen wood fern, Polystichum acrostichoides -n13199970 holly fern -n13200193 Braun's holly fern, prickly shield fern, Polystichum braunii -n13200542 western holly fern, Polystichum scopulinum -n13200651 soft shield fern, Polystichum setiferum -n13200986 leather fern, leatherleaf fern, ten-day fern, Rumohra adiantiformis, Polystichum adiantiformis -n13201423 button fern, Tectaria cicutaria -n13201566 Indian button fern, Tectaria macrodonta -n13201969 woodsia -n13202125 rusty woodsia, fragrant woodsia, oblong woodsia, Woodsia ilvensis -n13202355 Alpine woodsia, northern woodsia, flower-cup fern, Woodsia alpina -n13202602 smooth woodsia, Woodsia glabella -n13205058 Boston fern, Nephrolepis exaltata, Nephrolepis exaltata bostoniensis -n13205249 basket fern, toothed sword fern, Nephrolepis pectinata -n13206178 golden fern, leather fern, Acrostichum aureum -n13206817 maidenhair, maidenhair fern -n13207094 common maidenhair, Venushair, Venus'-hair fern, southern maidenhair, Venus maidenhair, Adiantum capillus-veneris -n13207335 American maidenhair fern, five-fingered maidenhair fern, Adiantum pedatum -n13207572 Bermuda maidenhair, Bermuda maidenhair fern, Adiantum bellum -n13207736 brittle maidenhair, brittle maidenhair fern, Adiantum tenerum -n13207923 Farley maidenhair, Farley maidenhair fern, Barbados maidenhair, glory fern, Adiantum tenerum farleyense -n13208302 annual fern, Jersey fern, Anogramma leptophylla -n13208705 lip fern, lipfern -n13208965 smooth lip fern, Alabama lip fern, Cheilanthes alabamensis -n13209129 lace fern, Cheilanthes gracillima -n13209270 wooly lip fern, hairy lip fern, Cheilanthes lanosa -n13209460 southwestern lip fern, Cheilanthes eatonii -n13209808 bamboo fern, Coniogramme japonica -n13210350 American rock brake, American parsley fern, Cryptogramma acrostichoides -n13210597 European parsley fern, mountain parsley fern, Cryptogramma crispa -n13211020 hand fern, Doryopteris pedata -n13211790 cliff brake, cliff-brake, rock brake -n13212025 coffee fern, Pellaea andromedifolia -n13212175 purple rock brake, Pellaea atropurpurea -n13212379 bird's-foot fern, Pellaea mucronata, Pellaea ornithopus -n13212559 button fern, Pellaea rotundifolia -n13213066 silver fern, Pityrogramma argentea -n13213397 golden fern, Pityrogramma calomelanos aureoflava -n13213577 gold fern, Pityrogramma chrysophylla -n13214217 Pteris cretica -n13214340 spider brake, spider fern, Pteris multifida -n13214485 ribbon fern, spider fern, Pteris serrulata -n13215258 potato fern, Marattia salicina -n13215586 angiopteris, giant fern, Angiopteris evecta -n13217005 skeleton fork fern, Psilotum nudum -n13219422 horsetail -n13219833 common horsetail, field horsetail, Equisetum arvense -n13219976 swamp horsetail, water horsetail, Equisetum fluviatile -n13220122 scouring rush, rough horsetail, Equisetum hyemale, Equisetum hyemale robustum, Equisetum robustum -n13220355 marsh horsetail, Equisetum palustre -n13220525 wood horsetail, Equisetum Sylvaticum -n13220663 variegated horsetail, variegated scouring rush, Equisetum variegatum -n13221529 club moss, club-moss, lycopod -n13222877 shining clubmoss, Lycopodium lucidulum -n13222985 alpine clubmoss, Lycopodium alpinum -n13223090 fir clubmoss, mountain clubmoss, little clubmoss, Lycopodium selago -n13223588 ground cedar, staghorn moss, Lycopodium complanatum -n13223710 ground fir, princess pine, tree clubmoss, Lycopodium obscurum -n13223843 foxtail grass, Lycopodium alopecuroides -n13224673 spikemoss, spike moss, little club moss -n13224922 meadow spikemoss, basket spikemoss, Selaginella apoda -n13225244 desert selaginella, Selaginella eremophila -n13225365 resurrection plant, rose of Jericho, Selaginella lepidophylla -n13225617 florida selaginella, Selaginella eatonii -n13226320 quillwort -n13226871 earthtongue, earth-tongue -n13228017 snuffbox fern, meadow fern, Thelypteris palustris pubescens, Dryopteris thelypteris pubescens -n13228536 christella -n13229543 mountain fern, Oreopteris limbosperma, Dryopteris oreopteris -n13229951 New York fern, Parathelypteris novae-boracensis, Dryopteris noveboracensis -n13230190 Massachusetts fern, Parathelypteris simulata, Thelypteris simulata -n13230662 beech fern -n13230843 broad beech fern, southern beech fern, Phegopteris hexagonoptera, Dryopteris hexagonoptera, Thelypteris hexagonoptera -n13231078 long beech fern, narrow beech fern, northern beech fern, Phegopteris connectilis, Dryopteris phegopteris, Thelypteris phegopteris -n13231678 shoestring fungus -n13231919 Armillaria caligata, booted armillaria -n13232106 Armillaria ponderosa, white matsutake -n13232363 Armillaria zelleri -n13232779 honey mushroom, honey fungus, Armillariella mellea -n13233727 milkweed, silkweed -n13234114 white milkweed, Asclepias albicans -n13234519 poke milkweed, Asclepias exaltata -n13234678 swamp milkweed, Asclepias incarnata -n13234857 Mead's milkweed, Asclepias meadii, Asclepia meadii -n13235011 purple silkweed, Asclepias purpurascens -n13235159 showy milkweed, Asclepias speciosa -n13235319 poison milkweed, horsetail milkweed, Asclepias subverticillata -n13235503 butterfly weed, orange milkweed, chigger flower, chiggerflower, pleurisy root, tuber root, Indian paintbrush, Asclepias tuberosa -n13235766 whorled milkweed, Asclepias verticillata -n13236100 cruel plant, Araujia sericofera -n13237188 wax plant, Hoya carnosa -n13237508 silk vine, Periploca graeca -n13238375 stapelia, carrion flower, starfish flower -n13238654 Stapelias asterias -n13238988 stephanotis -n13239177 Madagascar jasmine, waxflower, Stephanotis floribunda -n13239736 negro vine, Vincetoxicum hirsutum, Vincetoxicum negrum -n13239921 zygospore -n13240362 tree of knowledge -n13252672 orangery -n13354021 pocketbook -n13555775 shit, dump -n13579829 cordage -n13650447 yard, pace -n13653902 extremum, peak -n13862407 leaf shape, leaf form -n13862552 equilateral -n13862780 figure -n13863020 pencil -n13863186 plane figure, two-dimensional figure -n13863473 solid figure, three-dimensional figure -n13863771 line -n13864035 bulb -n13864153 convex shape, convexity -n13864965 concave shape, concavity, incurvation, incurvature -n13865298 cylinder -n13865483 round shape -n13865904 heart -n13866144 polygon, polygonal shape -n13866626 convex polygon -n13866827 concave polygon -n13867005 reentrant polygon, reentering polygon -n13867492 amorphous shape -n13868248 closed curve -n13868371 simple closed curve, Jordan curve -n13868515 S-shape -n13868944 wave, undulation -n13869045 extrados -n13869547 hook, crotchet -n13869788 envelope -n13869896 bight -n13871717 diameter -n13872592 cone, conoid, cone shape -n13872822 funnel, funnel shape -n13873361 oblong -n13873502 circle -n13873917 circle -n13874073 equator -n13874558 scallop, crenation, crenature, crenel, crenelle -n13875392 ring, halo, annulus, doughnut, anchor ring -n13875571 loop -n13875884 bight -n13876561 helix, spiral -n13877547 element of a cone -n13877667 element of a cylinder -n13878306 ellipse, oval -n13879049 quadrate -n13879320 triangle, trigon, trilateral -n13879816 acute triangle, acute-angled triangle -n13880199 isosceles triangle -n13880415 obtuse triangle, obtuse-angled triangle -n13880551 right triangle, right-angled triangle -n13880704 scalene triangle -n13880994 parallel -n13881512 trapezoid -n13881644 star -n13882201 pentagon -n13882276 hexagon -n13882487 heptagon -n13882563 octagon -n13882639 nonagon -n13882713 decagon -n13882961 rhombus, rhomb, diamond -n13883603 spherical polygon -n13883763 spherical triangle -n13884261 convex polyhedron -n13884384 concave polyhedron -n13884930 cuboid -n13885011 quadrangular prism -n13886260 bell, bell shape, campana -n13888491 angular distance -n13889066 true anomaly -n13889331 spherical angle -n13891547 angle of refraction -n13891937 acute angle -n13893786 groove, channel -n13894154 rut -n13894434 bulge, bump, hump, swelling, gibbosity, gibbousness, jut, prominence, protuberance, protrusion, extrusion, excrescence -n13895262 belly -n13896100 bow, arc -n13896217 crescent -n13897198 ellipsoid -n13897528 hypotenuse -n13897996 balance, equilibrium, equipoise, counterbalance -n13898207 conformation -n13898315 symmetry, proportion -n13898645 spheroid, ellipsoid of revolution -n13899735 spherule -n13900287 toroid -n13900422 column, tower, pillar -n13901211 barrel, drum -n13901321 pipe, tube -n13901423 pellet -n13901490 bolus -n13901858 dewdrop -n13902048 ridge -n13902336 rim -n13902793 taper -n13903079 boundary, edge, bound -n13905121 incisure, incisura -n13905275 notch -n13905792 wrinkle, furrow, crease, crinkle, seam, line -n13906484 dermatoglyphic -n13906669 frown line -n13906767 line of life, life line, lifeline -n13906936 line of heart, heart line, love line, mensal line -n13907272 crevice, cranny, crack, fissure, chap -n13908201 cleft -n13908580 roulette, line roulette -n13911045 node -n13912260 tree, tree diagram -n13912540 stemma -n13914141 brachium -n13914265 fork, crotch -n13914608 block, cube -n13915023 ovoid -n13915113 tetrahedron -n13915209 pentahedron -n13915305 hexahedron -n13915999 regular polyhedron, regular convex solid, regular convex polyhedron, Platonic body, Platonic solid, ideal solid -n13916363 polyhedral angle -n13916721 cube, regular hexahedron -n13917690 truncated pyramid -n13917785 truncated cone -n13918274 tail, tail end -n13918387 tongue, knife -n13918717 trapezohedron -n13919547 wedge, wedge shape, cuneus -n13919919 keel -n13926786 place, shoes -n14131950 herpes -n14175579 chlamydia -n14564779 wall -n14582716 micronutrient -n14583400 chyme -n14585392 ragweed pollen -n14592309 pina cloth -n14603798 chlorobenzylidenemalononitrile, CS gas -n14633206 carbon, C, atomic number 6 -n14685296 charcoal, wood coal -n14696793 rock, stone -n14698884 gravel, crushed rock -n14714645 aflatoxin -n14720833 alpha-tocopheral -n14765422 leopard -n14785065 bricks and mortar -n14786943 lagging -n14804958 hydraulic cement, Portland cement -n14810561 choline -n14820180 concrete -n14821852 glass wool -n14844693 soil, dirt -n14853210 high explosive -n14858292 litter -n14867545 fish meal -n14891255 Greek fire -n14899328 culture medium, medium -n14900184 agar, nutrient agar -n14900342 blood agar -n14908027 hip tile, hipped tile -n14909584 hyacinth, jacinth -n14914945 hydroxide ion, hydroxyl ion -n14915184 ice, water ice -n14919819 inositol -n14938389 linoleum, lino -n14941787 lithia water -n14942411 lodestone, loadstone -n14973585 pantothenic acid, pantothen -n14974264 paper -n14975598 papyrus -n14976759 pantile -n14976871 blacktop, blacktopping -n14977188 tarmacadam, tarmac -n14977504 paving, pavement, paving material -n14992287 plaster -n14993378 poison gas -n15005577 ridge tile -n15006012 roughcast -n15019030 sand -n15048888 spackle, spackling compound -n15060326 render -n15060688 wattle and daub -n15062057 stucco -n15067877 tear gas, teargas, lacrimator, lachrymator -n15075141 toilet tissue, toilet paper, bathroom tissue -n15086247 linseed, flaxseed -n15089258 vitamin -n15089472 fat-soluble vitamin -n15089645 water-soluble vitamin -n15089803 vitamin A, antiophthalmic factor, axerophthol, A -n15090065 vitamin A1, retinol -n15090238 vitamin A2, dehydroretinol -n15090742 B-complex vitamin, B complex, vitamin B complex, vitamin B, B vitamin, B -n15091129 vitamin B1, thiamine, thiamin, aneurin, antiberiberi factor -n15091304 vitamin B12, cobalamin, cyanocobalamin, antipernicious anemia factor -n15091473 vitamin B2, vitamin G, riboflavin, lactoflavin, ovoflavin, hepatoflavin -n15091669 vitamin B6, pyridoxine, pyridoxal, pyridoxamine, adermin -n15091846 vitamin Bc, vitamin M, folate, folic acid, folacin, pteroylglutamic acid, pteroylmonoglutamic acid -n15092059 niacin, nicotinic acid -n15092227 vitamin D, calciferol, viosterol, ergocalciferol, cholecalciferol, D -n15092409 vitamin E, tocopherol, E -n15092650 biotin, vitamin H -n15092751 vitamin K, naphthoquinone, antihemorrhagic factor -n15092942 vitamin K1, phylloquinone, phytonadione -n15093049 vitamin K3, menadione -n15093137 vitamin P, bioflavinoid, citrin -n15093298 vitamin C, C, ascorbic acid -n15102359 planking -n15102455 chipboard, hardboard -n15102894 knothole diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py deleted file mode 100644 index 0b1677bf185..00000000000 --- a/tensorflow_serving/example/inception_client.py +++ /dev/null @@ -1,53 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -#!/usr/bin/env python2.7 - -"""Send JPEG image to inception_inference server for classification. -""" - -# This is a placeholder for a Google-internal import. - -from grpc.beta import implementations -import tensorflow as tf - -from tensorflow_serving.example import inception_inference_pb2 - - -tf.app.flags.DEFINE_string('server', 'localhost:9000', - 'inception_inference service host:port') -tf.app.flags.DEFINE_string('image', '', 'path to image in JPEG format') -FLAGS = tf.app.flags.FLAGS - - -NUM_CLASSES = 5 - - -def main(_): - host, port = FLAGS.server.split(':') - channel = implementations.insecure_channel(host, int(port)) - stub = inception_inference_pb2.beta_create_InceptionService_stub(channel) - # Send request - with open(FLAGS.image, 'rb') as f: - # See inception_inference.proto for gRPC request/response details. - data = f.read() - request = inception_inference_pb2.InceptionRequest() - request.jpeg_encoded = data - result = stub.Classify(request, 10.0) # 10 secs timeout - print result - - -if __name__ == '__main__': - tf.app.run() diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py deleted file mode 100644 index a30617d0fc5..00000000000 --- a/tensorflow_serving/example/inception_export.py +++ /dev/null @@ -1,148 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -#!/usr/bin/env python2.7 -"""Export inception model given existing training checkpoints. -""" - -import os.path - -# This is a placeholder for a Google-internal import. - -import tensorflow as tf - -from tensorflow.contrib.session_bundle import exporter -from inception import inception_model - - -tf.app.flags.DEFINE_string('checkpoint_dir', '/tmp/inception_train', - """Directory where to read training checkpoints.""") -tf.app.flags.DEFINE_string('export_dir', '/tmp/inception_export', - """Directory where to export inference model.""") -tf.app.flags.DEFINE_integer('image_size', 299, - """Needs to provide same value as in training.""") -FLAGS = tf.app.flags.FLAGS - - -NUM_CLASSES = 1000 -NUM_TOP_CLASSES = 5 - -WORKING_DIR = os.path.dirname(os.path.realpath(__file__)) -SYNSET_FILE = os.path.join(WORKING_DIR, 'imagenet_lsvrc_2015_synsets.txt') -METADATA_FILE = os.path.join(WORKING_DIR, 'imagenet_metadata.txt') - - -def export(): - # Create index->synset mapping - synsets = [] - with open(SYNSET_FILE) as f: - synsets = f.read().splitlines() - # Create synset->metadata mapping - texts = {} - with open(METADATA_FILE) as f: - for line in f.read().splitlines(): - parts = line.split('\t') - assert len(parts) == 2 - texts[parts[0]] = parts[1] - - with tf.Graph().as_default(): - # Build inference model. - # Please refer to Tensorflow inception model for details. - - # Input transformation. - jpegs = tf.placeholder(tf.string) - images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32) - - # Run inference. - logits, _ = inception_model.inference(images, NUM_CLASSES + 1) - - # Transform output to topK result. - values, indices = tf.nn.top_k(logits, NUM_TOP_CLASSES) - - # Create a constant string Tensor where the i'th element is - # the human readable class description for the i'th index. - # Note that the 0th index is an unused background class - # (see inception model definition code). - class_descriptions = ['unused background'] - for s in synsets: - class_descriptions.append(texts[s]) - class_tensor = tf.constant(class_descriptions) - - classes = tf.contrib.lookup.index_to_string(tf.to_int64(indices), - mapping=class_tensor) - - # Restore variables from training checkpoint. - variable_averages = tf.train.ExponentialMovingAverage( - inception_model.MOVING_AVERAGE_DECAY) - variables_to_restore = variable_averages.variables_to_restore() - saver = tf.train.Saver(variables_to_restore) - with tf.Session() as sess: - # Restore variables from training checkpoints. - ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) - if ckpt and ckpt.model_checkpoint_path: - saver.restore(sess, ckpt.model_checkpoint_path) - # Assuming model_checkpoint_path looks something like: - # /my-favorite-path/imagenet_train/model.ckpt-0, - # extract global_step from it. - global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] - print('Successfully loaded model from %s at step=%s.' % - (ckpt.model_checkpoint_path, global_step)) - else: - print('No checkpoint file found at %s' % FLAGS.checkpoint_dir) - return - - # Export inference model. - init_op = tf.group(tf.initialize_all_tables(), name='init_op') - model_exporter = exporter.Exporter(saver) - signature = exporter.classification_signature( - input_tensor=jpegs, classes_tensor=classes, scores_tensor=values) - model_exporter.init(default_graph_signature=signature, init_op=init_op) - model_exporter.export(FLAGS.export_dir, tf.constant(global_step), sess) - print('Successfully exported model to %s' % FLAGS.export_dir) - - -def preprocess_image(image_buffer): - """Preprocess JPEG encoded bytes to 3D float Tensor.""" - - # Decode the string as an RGB JPEG. - # Note that the resulting image contains an unknown height and width - # that is set dynamically by decode_jpeg. In other words, the height - # and width of image is unknown at compile-time. - image = tf.image.decode_jpeg(image_buffer, channels=3) - # After this point, all image pixels reside in [0,1) - # until the very end, when they're rescaled to (-1, 1). The various - # adjust_* ops all require this range for dtype float. - image = tf.image.convert_image_dtype(image, dtype=tf.float32) - # Crop the central region of the image with an area containing 87.5% of - # the original image. - image = tf.image.central_crop(image, central_fraction=0.875) - # Resize the image to the original height and width. - image = tf.expand_dims(image, 0) - image = tf.image.resize_bilinear(image, - [FLAGS.image_size, FLAGS.image_size], - align_corners=False) - image = tf.squeeze(image, [0]) - # Finally, rescale to [-1,1] instead of [0, 1) - image = tf.sub(image, 0.5) - image = tf.mul(image, 2.0) - return image - - -def main(unused_argv=None): - export() - - -if __name__ == '__main__': - tf.app.run() diff --git a/tensorflow_serving/example/inception_inference.cc b/tensorflow_serving/example/inception_inference.cc deleted file mode 100644 index 4697025909a..00000000000 --- a/tensorflow_serving/example/inception_inference.cc +++ /dev/null @@ -1,393 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A gRPC server that serves inception model exported by inception_export.py. -// Given each request with an image as JPEG encoded byte stream, the server -// responds with kNumTopClasses integer values as indexes of top k matched -// categories and kNumTopClasses float values as the corresponding -// probabilities. - -#include -#include -#include -#include -#include - -#include - -#include "grpc++/completion_queue.h" -#include "grpc++/security/server_credentials.h" -#include "grpc++/server.h" -#include "grpc++/server_builder.h" -#include "grpc++/server_context.h" -#include "grpc++/support/async_unary_call.h" -#include "grpc++/support/status.h" -#include "grpc++/support/status_code_enum.h" -#include "grpc/grpc.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" -#include "tensorflow_serving/batching/batch_scheduler.h" -#include "tensorflow_serving/core/manager.h" -#include "tensorflow_serving/core/servable_handle.h" -#include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/example/inception_inference.grpc.pb.h" -#include "tensorflow_serving/example/inception_inference.pb.h" -#include "tensorflow_serving/servables/tensorflow/simple_servers.h" - -using grpc::InsecureServerCredentials; -using grpc::Server; -using grpc::ServerAsyncResponseWriter; -using grpc::ServerBuilder; -using grpc::ServerContext; -using grpc::ServerCompletionQueue; -using grpc::Status; -using grpc::StatusCode; -using tensorflow::serving::InceptionRequest; -using tensorflow::serving::InceptionResponse; -using tensorflow::serving::InceptionService; -using tensorflow::string; -using tensorflow::Tensor; -using tensorflow::serving::ClassificationSignature; - -const int kNumTopClasses = 5; - -namespace { -class InceptionServiceImpl; - -// Class encompassing the state and logic needed to serve a request. -class CallData { - public: - CallData(InceptionServiceImpl* service_impl, - InceptionService::AsyncService* service, - ServerCompletionQueue* cq); - - void Proceed(); - - void Finish(Status status); - - const InceptionRequest& request() { return request_; } - - InceptionResponse* mutable_response() { return &response_; } - - private: - // Service implementation. - InceptionServiceImpl* service_impl_; - - // The means of communication with the gRPC runtime for an asynchronous - // server. - InceptionService::AsyncService* service_; - // The producer-consumer queue where for asynchronous server notifications. - ServerCompletionQueue* cq_; - // Context for the rpc, allowing to tweak aspects of it such as the use - // of compression, authentication, as well as to send metadata back to the - // client. - ServerContext ctx_; - - // What we get from the client. - InceptionRequest request_; - // What we send back to the client. - InceptionResponse response_; - - // The means to get back to the client. - ServerAsyncResponseWriter responder_; - - // Let's implement a tiny state machine with the following states. - enum CallStatus { CREATE, PROCESS, FINISH }; - CallStatus status_; // The current serving state. -}; - -// A Task holds all of the information for a single inference request. -struct Task : public tensorflow::serving::BatchTask { - ~Task() override = default; - size_t size() const override { return 1; } - - Task(CallData* calldata_arg) - : calldata(calldata_arg) {} - - CallData* calldata; -}; - -class InceptionServiceImpl final { - public: - InceptionServiceImpl(const string& servable_name, - std::unique_ptr manager); - - void Classify(CallData* call_data); - - // Produces classifications for a batch of requests and associated responses. - void DoClassifyInBatch( - std::unique_ptr> batch); - - // Name of the servable to use for inference. - const string servable_name_; - // Manager in charge of loading and unloading servables. - std::unique_ptr manager_; - // A scheduler for batching multiple request calls into single calls to - // Session->Run(). - std::unique_ptr> - batch_scheduler_; -}; - -// Take in the "service" instance (in this case representing an asynchronous -// server) and the completion queue "cq" used for asynchronous communication -// with the gRPC runtime. -CallData::CallData(InceptionServiceImpl* service_impl, - InceptionService::AsyncService* service, - ServerCompletionQueue* cq) - : service_impl_(service_impl), - service_(service), cq_(cq), responder_(&ctx_), status_(CREATE) { - // Invoke the serving logic right away. - Proceed(); -} - -void CallData::Proceed() { - if (status_ == CREATE) { - // As part of the initial CREATE state, we *request* that the system - // start processing Classify requests. In this request, "this" acts are - // the tag uniquely identifying the request (so that different CallData - // instances can serve different requests concurrently), in this case - // the memory address of this CallData instance. - service_->RequestClassify(&ctx_, &request_, &responder_, cq_, cq_, this); - // Make this instance progress to the PROCESS state. - status_ = PROCESS; - } else if (status_ == PROCESS) { - // Spawn a new CallData instance to serve new clients while we process - // the one for this CallData. The instance will deallocate itself as - // part of its FINISH state. - new CallData(service_impl_, service_, cq_); - // Start processing. - service_impl_->Classify(this); - } else { - GPR_ASSERT(status_ == FINISH); - // Once in the FINISH state, deallocate ourselves (CallData). - delete this; - } -} - -void CallData::Finish(Status status) { - status_ = FINISH; - responder_.Finish(response_, status, this); -} - -InceptionServiceImpl::InceptionServiceImpl( - const string& servable_name, - std::unique_ptr manager) - : servable_name_(servable_name), manager_(std::move(manager)) { - // Setup a batcher used to combine multiple requests (tasks) into a single - // graph run for efficiency. - // The batcher queues tasks until, - // (a) the next task would cause the batch to exceed the size target; - // (b) waiting for more tasks to be added would exceed the timeout. - // at which point it processes the entire batch. - // - // Use the default batch-size, timeout and thread options. In general - // the numbers are extremely performance critical and should be tuned based - // specific graph structure and usage. - tensorflow::serving::BasicBatchScheduler::Options scheduler_options; - scheduler_options.thread_pool_name = "inception_service_batch_threads"; - // Use a very large queue, to avoid rejecting requests. (Note: a production - // server with load balancing may want to use the default, much smaller, - // value.) - scheduler_options.max_enqueued_batches = 1000; - scheduler_options.max_batch_size = 10; - TF_CHECK_OK(tensorflow::serving::BasicBatchScheduler::Create( - scheduler_options, - [this](std::unique_ptr> batch) { - this->DoClassifyInBatch(std::move(batch)); - }, - &batch_scheduler_)); -} - -// Creates a gRPC Status from a TensorFlow Status. -Status ToGRPCStatus(const tensorflow::Status& status) { - return Status(static_cast(status.code()), - status.error_message()); -} - -void InceptionServiceImpl::Classify(CallData* calldata) { - // Create and submit a task to the batch scheduler. - std::unique_ptr task(new Task(calldata)); - tensorflow::Status status = batch_scheduler_->Schedule(&task); - - if (!status.ok()) { - calldata->Finish(ToGRPCStatus(status)); - return; - } -} - -// Produces classifications for a batch of requests and associated responses. -void InceptionServiceImpl::DoClassifyInBatch( - std::unique_ptr> batch) { - batch->WaitUntilClosed(); - if (batch->empty()) { - return; - } - const int batch_size = batch->num_tasks(); - - // Replies to each task with the given error status. - auto complete_with_error = [&batch](StatusCode code, const string& msg) { - Status status(code, msg); - for (int i = 0; i < batch->num_tasks(); i++) { - Task* task = batch->mutable_task(i); - task->calldata->Finish(status); - } - }; - - // Get a handle to the SessionBundle. The handle ensures the Manager does - // not reload this while it is in use. - auto handle_request = - tensorflow::serving::ServableRequest::Latest(servable_name_); - tensorflow::serving::ServableHandle - bundle; - const tensorflow::Status lookup_status = - manager_->GetServableHandle(handle_request, &bundle); - if (!lookup_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - lookup_status.error_message()); - return; - } - - // Get the default signature of the graph. Expected to be a - // classification signature. - tensorflow::serving::ClassificationSignature signature; - const tensorflow::Status signature_status = - GetClassificationSignature(bundle->meta_graph_def, &signature); - if (!signature_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - signature_status.error_message()); - return; - } - - // Transform protobuf input to inference input tensor. - tensorflow::Tensor batched_input(tensorflow::DT_STRING, {batch_size}); - for (int i = 0; i < batch_size; ++i) { - batched_input.vec()(i) = - batch->mutable_task(i)->calldata->request().jpeg_encoded(); - } - - // Run classification. - tensorflow::Tensor batched_classes; - tensorflow::Tensor batched_scores; - const tensorflow::Status run_status = - RunClassification(signature, batched_input, bundle->session.get(), - &batched_classes, &batched_scores); - if (!run_status.ok()) { - complete_with_error(StatusCode::INTERNAL, run_status.error_message()); - return; - } - - // Transform inference output tensor to protobuf output. - for (int i = 0; i < batch_size; ++i) { - auto calldata = batch->mutable_task(i)->calldata; - auto classes = calldata->mutable_response()->mutable_classes(); - auto scores = calldata->mutable_response()->mutable_scores(); - for (int j = 0; j < kNumTopClasses; j++) { - *classes->Add() = batched_classes.matrix()(i, j); - scores->Add(batched_scores.matrix()(i, j)); - } - calldata->Finish(Status::OK); - } -} - -void HandleRpcs(InceptionServiceImpl* service_impl, - InceptionService::AsyncService* service, - ServerCompletionQueue* cq) { - // Spawn a new CallData instance to serve new clients. - new CallData(service_impl, service, cq); - void* tag; // uniquely identifies a request. - bool ok = false; - while (true) { - // Block waiting to read the next event from the completion queue. The - // event is uniquely identified by its tag, which in this case is the - // memory address of a CallData instance. - if (!cq->Next(&tag, &ok)) { - break; // server shutting down - } - if (!ok) { - continue; // irregular event - } - static_cast(tag)->Proceed(); - } -} - -// Runs InceptionService server until shutdown. -void RunServer(const int port, const string& servable_name, - std::unique_ptr manager) { - // "0.0.0.0" is the way to listen on localhost in gRPC. - const string server_address = "0.0.0.0:" + std::to_string(port); - - InceptionService::AsyncService service; - ServerBuilder builder; - std::shared_ptr creds = InsecureServerCredentials(); - builder.AddListeningPort(server_address, creds); - builder.RegisterService(&service); - std::unique_ptr cq = builder.AddCompletionQueue(); - std::unique_ptr server(builder.BuildAndStart()); - LOG(INFO) << "Running..."; - - InceptionServiceImpl service_impl(servable_name, std::move(manager)); - HandleRpcs(&service_impl, &service, cq.get()); -} - -} // namespace - -int main(int argc, char** argv) { - // Parse command-line options. - tensorflow::int32 port = 0; - const bool parse_result = - tensorflow::ParseFlags(&argc, argv, {tensorflow::Flag("port", &port)}); - if (!parse_result) { - LOG(FATAL) << "Error parsing command line flags."; - } - - if (argc != 2) { - LOG(ERROR) << "Usage: inception_inference --port=9000 /path/to/exports"; - return -1; - } - const string export_base_path(argv[1]); - tensorflow::port::InitMain(argv[0], &argc, &argv); - - std::unique_ptr manager; - tensorflow::Status status = tensorflow::serving::simple_servers:: - CreateSingleTFModelManagerFromBasePath(export_base_path, &manager); - - TF_CHECK_OK(status) << "Error creating manager"; - - // Wait until at least one model is loaded. - std::vector ready_ids; - // TODO(b/25545573): Create a more streamlined startup mechanism than polling. - do { - LOG(INFO) << "Waiting for models to be loaded..."; - tensorflow::Env::Default()->SleepForMicroseconds(1 * 1000 * 1000 /*1 sec*/); - ready_ids = manager->ListAvailableServableIds(); - } while (ready_ids.empty()); - - // Run the service. - RunServer(port, ready_ids[0].name, std::move(manager)); - - return 0; -} diff --git a/tensorflow_serving/example/inception_inference.proto b/tensorflow_serving/example/inception_inference.proto deleted file mode 100644 index 64bdc4f6baa..00000000000 --- a/tensorflow_serving/example/inception_inference.proto +++ /dev/null @@ -1,22 +0,0 @@ -// Protobuf definition of inception v3 model inference server. - -syntax = "proto3"; - -package tensorflow.serving; - -message InceptionRequest { - // JPEG encoded stream of the image to be classified. - bytes jpeg_encoded = 1; -}; - -message InceptionResponse { - // Human readable descriptions of the classes, in scores descending order. - repeated string classes = 3; - // Scores of top matches, in same order as classes. - repeated float scores = 2; -}; - -service InceptionService { - // Classifies an JPEG image into classes. - rpc Classify(InceptionRequest) returns (InceptionResponse); -} diff --git a/tensorflow_serving/example/inception_inference_pb2.py b/tensorflow_serving/example/inception_inference_pb2.py deleted file mode 100644 index 31344e39bc2..00000000000 --- a/tensorflow_serving/example/inception_inference_pb2.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: inception_inference.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='inception_inference.proto', - package='tensorflow.serving', - syntax='proto3', - serialized_pb=_b('\n\x19inception_inference.proto\x12\x12tensorflow.serving\"(\n\x10InceptionRequest\x12\x14\n\x0cjpeg_encoded\x18\x01 \x01(\x0c\"4\n\x11InceptionResponse\x12\x0f\n\x07\x63lasses\x18\x03 \x03(\t\x12\x0e\n\x06scores\x18\x02 \x03(\x02\x32k\n\x10InceptionService\x12W\n\x08\x43lassify\x12$.tensorflow.serving.InceptionRequest\x1a%.tensorflow.serving.InceptionResponseb\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_INCEPTIONREQUEST = _descriptor.Descriptor( - name='InceptionRequest', - full_name='tensorflow.serving.InceptionRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='jpeg_encoded', full_name='tensorflow.serving.InceptionRequest.jpeg_encoded', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=49, - serialized_end=89, -) - - -_INCEPTIONRESPONSE = _descriptor.Descriptor( - name='InceptionResponse', - full_name='tensorflow.serving.InceptionResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='classes', full_name='tensorflow.serving.InceptionResponse.classes', index=0, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scores', full_name='tensorflow.serving.InceptionResponse.scores', index=1, - number=2, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=91, - serialized_end=143, -) - -DESCRIPTOR.message_types_by_name['InceptionRequest'] = _INCEPTIONREQUEST -DESCRIPTOR.message_types_by_name['InceptionResponse'] = _INCEPTIONRESPONSE - -InceptionRequest = _reflection.GeneratedProtocolMessageType('InceptionRequest', (_message.Message,), dict( - DESCRIPTOR = _INCEPTIONREQUEST, - __module__ = 'inception_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.InceptionRequest) - )) -_sym_db.RegisterMessage(InceptionRequest) - -InceptionResponse = _reflection.GeneratedProtocolMessageType('InceptionResponse', (_message.Message,), dict( - DESCRIPTOR = _INCEPTIONRESPONSE, - __module__ = 'inception_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.InceptionResponse) - )) -_sym_db.RegisterMessage(InceptionResponse) - - -import abc -import six -from grpc.beta import implementations as beta_implementations -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities - -class BetaInceptionServiceServicer(six.with_metaclass(abc.ABCMeta, object)): - """""" - @abc.abstractmethod - def Classify(self, request, context): - raise NotImplementedError() - -class BetaInceptionServiceStub(six.with_metaclass(abc.ABCMeta, object)): - """The interface to which stubs will conform.""" - @abc.abstractmethod - def Classify(self, request, timeout): - raise NotImplementedError() - Classify.future = None - -def beta_create_InceptionService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import inception_inference_pb2 - import inception_inference_pb2 - request_deserializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionRequest.FromString, - } - response_serializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionResponse.SerializeToString, - } - method_implementations = { - ('tensorflow.serving.InceptionService', 'Classify'): face_utilities.unary_unary_inline(servicer.Classify), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_InceptionService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import inception_inference_pb2 - import inception_inference_pb2 - request_serializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionRequest.SerializeToString, - } - response_deserializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionResponse.FromString, - } - cardinalities = { - 'Classify': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.InceptionService', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/example/inception_k8s.json b/tensorflow_serving/example/inception_k8s.json deleted file mode 100644 index 7877af436d1..00000000000 --- a/tensorflow_serving/example/inception_k8s.json +++ /dev/null @@ -1,61 +0,0 @@ -{ - "apiVersion": "v1", - "kind": "ReplicationController", - "metadata": { - "name": "inception-controller" - }, - "spec": { - "replicas": 3, - "selector": { - "worker": "inception-pod" - }, - "template": { - "metadata": { - "labels": { - "worker": "inception-pod" - } - }, - "spec": { - "containers": [ - { - "name": "inception-container", - "image": "gcr.io/tensorflow-serving/inception", - "command": [ - "/bin/sh", - "-c" - ], - "args": [ - "/serving/bazel-bin/tensorflow_serving/example/inception_inference --port=9000 /serving/inception-export" - ], - "ports": [ - { - "containerPort": 9000 - } - ] - } - ], - "restartPolicy": "Always" - } - } - } -} - -{ - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "inception-service" - }, - "spec": { - "ports": [ - { - "port": 9000, - "targetPort": 9000 - } - ], - "selector": { - "worker": "inception-pod" - }, - "type": "LoadBalancer" - } -} \ No newline at end of file diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index ab43bcc8bf8..fd90af69b2d 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -15,43 +15,116 @@ #!/usr/bin/env python2.7 -"""A client that talks to mnist_inference service. +"""A client that talks to tensorflow_model_server loaded with mnist model. The client downloads test images of mnist data set, queries the service with -such test images to get classification, and calculates the inference error rate. -Please see mnist_inference.proto for details. +such test images to get predictions, and calculates the inference error rate. Typical usage example: mnist_client.py --num_tests=100 --server=localhost:9000 """ +from __future__ import print_function + import sys import threading -# This is a placeholder for a Google-internal import. - -from grpc.beta import implementations +import grpc import numpy import tensorflow as tf -from tensorflow_serving.example import mnist_inference_pb2 -from tensorflow_serving.example import mnist_input_data +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_service_pb2_grpc +import mnist_input_data + + +tf.compat.v1.app.flags.DEFINE_integer( + 'concurrency', 1, 'maximum number of concurrent inference requests') +tf.compat.v1.app.flags.DEFINE_integer('num_tests', 100, 'Number of test images') +tf.compat.v1.app.flags.DEFINE_string('server', '', + 'PredictionService host:port') +tf.compat.v1.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory. ') +FLAGS = tf.compat.v1.app.flags.FLAGS + + +class _ResultCounter(object): + """Counter for the prediction results.""" + + def __init__(self, num_tests, concurrency): + self._num_tests = num_tests + self._concurrency = concurrency + self._error = 0 + self._done = 0 + self._active = 0 + self._condition = threading.Condition() + + def inc_error(self): + with self._condition: + self._error += 1 + def inc_done(self): + with self._condition: + self._done += 1 + self._condition.notify() -tf.app.flags.DEFINE_integer('concurrency', 1, - 'maximum number of concurrent inference requests') -tf.app.flags.DEFINE_integer('num_tests', 100, 'Number of test images') -tf.app.flags.DEFINE_string('server', '', 'mnist_inference service host:port') -tf.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory. ') -FLAGS = tf.app.flags.FLAGS + def dec_active(self): + with self._condition: + self._active -= 1 + self._condition.notify() + + def get_error_rate(self): + with self._condition: + while self._done != self._num_tests: + self._condition.wait() + return self._error / float(self._num_tests) + + def throttle(self): + with self._condition: + while self._active == self._concurrency: + self._condition.wait() + self._active += 1 + + +def _create_rpc_callback(label, result_counter): + """Creates RPC callback function. + + Args: + label: The correct label for the predicted example. + result_counter: Counter for the prediction result. + Returns: + The callback function. + """ + def _callback(result_future): + """Callback function. + + Calculates the statistics for the prediction result. + + Args: + result_future: Result future of the RPC. + """ + exception = result_future.exception() + if exception: + result_counter.inc_error() + print(exception) + else: + sys.stdout.write('.') + sys.stdout.flush() + response = numpy.array( + result_future.result().outputs['scores'].float_val) + prediction = numpy.argmax(response) + if label != prediction: + result_counter.inc_error() + result_counter.inc_done() + result_counter.dec_active() + return _callback def do_inference(hostport, work_dir, concurrency, num_tests): - """Tests mnist_inference service with concurrent requests. + """Tests PredictionService with concurrent requests. Args: - hostport: Host:port address of the mnist_inference service. + hostport: Host:port address of the PredictionService. work_dir: The full path of working directory for test data set. concurrency: Maximum number of concurrent requests. num_tests: Number of test images to use. @@ -63,60 +136,34 @@ def do_inference(hostport, work_dir, concurrency, num_tests): IOError: An error occurred processing test data set. """ test_data_set = mnist_input_data.read_data_sets(work_dir).test - host, port = hostport.split(':') - channel = implementations.insecure_channel(host, int(port)) - stub = mnist_inference_pb2.beta_create_MnistService_stub(channel) - cv = threading.Condition() - result = {'active': 0, 'error': 0, 'done': 0} - def done(result_future, label): - with cv: - # Workaround for gRPC issue https://github.com/grpc/grpc/issues/7133 - try: - exception = result_future.exception() - except AttributeError: - exception = None - if exception: - result['error'] += 1 - print exception - else: - sys.stdout.write('.') - sys.stdout.flush() - response = numpy.array(result_future.result().value) - prediction = numpy.argmax(response) - if label != prediction: - result['error'] += 1 - result['done'] += 1 - result['active'] -= 1 - cv.notify() + channel = grpc.insecure_channel(hostport) + stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + result_counter = _ResultCounter(num_tests, concurrency) for _ in range(num_tests): - request = mnist_inference_pb2.MnistRequest() + request = predict_pb2.PredictRequest() + request.model_spec.name = 'mnist' + request.model_spec.signature_name = 'predict_images' image, label = test_data_set.next_batch(1) - for pixel in image[0]: - request.image_data.append(pixel.item()) - with cv: - while result['active'] == concurrency: - cv.wait() - result['active'] += 1 - result_future = stub.Classify.future(request, 5.0) # 5 seconds + request.inputs['images'].CopyFrom( + tf.make_tensor_proto(image[0], shape=[1, image[0].size])) + result_counter.throttle() + result_future = stub.Predict.future(request, 5.0) # 5 seconds result_future.add_done_callback( - lambda result_future, l=label[0]: done(result_future, l)) # pylint: disable=cell-var-from-loop - with cv: - while result['done'] != num_tests: - cv.wait() - return result['error'] / float(num_tests) + _create_rpc_callback(label[0], result_counter)) + return result_counter.get_error_rate() def main(_): if FLAGS.num_tests > 10000: - print 'num_tests should not be greater than 10k' + print('num_tests should not be greater than 10k') return if not FLAGS.server: - print 'please specify server host:port' + print('please specify server host:port') return error_rate = do_inference(FLAGS.server, FLAGS.work_dir, FLAGS.concurrency, FLAGS.num_tests) - print '\nInference error rate: %s%%' % (error_rate * 100) + print('\nInference error rate: %s%%' % (error_rate * 100)) if __name__ == '__main__': - tf.app.run() + tf.compat.v1.app.run() diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py deleted file mode 100644 index a4d22656b56..00000000000 --- a/tensorflow_serving/example/mnist_export.py +++ /dev/null @@ -1,93 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -#!/usr/bin/env python2.7 - -"""Train and export a simple Softmax Regression TensorFlow model. - -The model is from the TensorFlow "MNIST For ML Beginner" tutorial. This program -simply follows all its training instructions, and uses TensorFlow Serving -exporter to export the trained model. - -Usage: mnist_export.py [--training_iteration=x] [--export_version=y] export_dir -""" - -import sys - -# This is a placeholder for a Google-internal import. - -import tensorflow as tf - -from tensorflow.contrib.session_bundle import exporter -from tensorflow_serving.example import mnist_input_data - -tf.app.flags.DEFINE_integer('training_iteration', 1000, - 'number of training iterations.') -tf.app.flags.DEFINE_integer('export_version', 1, 'version number of the model.') -tf.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory.') -FLAGS = tf.app.flags.FLAGS - - -def main(_): - if len(sys.argv) < 2 or sys.argv[-1].startswith('-'): - print('Usage: mnist_export.py [--training_iteration=x] ' - '[--export_version=y] export_dir') - sys.exit(-1) - if FLAGS.training_iteration <= 0: - print 'Please specify a positive value for training iteration.' - sys.exit(-1) - if FLAGS.export_version <= 0: - print 'Please specify a positive value for version number.' - sys.exit(-1) - - # Train model - print 'Training model...' - mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) - sess = tf.InteractiveSession() - x = tf.placeholder('float', shape=[None, 784]) - y_ = tf.placeholder('float', shape=[None, 10]) - w = tf.Variable(tf.zeros([784, 10])) - b = tf.Variable(tf.zeros([10])) - sess.run(tf.initialize_all_variables()) - y = tf.nn.softmax(tf.matmul(x, w) + b) - cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) - train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) - for _ in range(FLAGS.training_iteration): - batch = mnist.train.next_batch(50) - train_step.run(feed_dict={x: batch[0], y_: batch[1]}) - correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) - accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) - print 'training accuracy %g' % sess.run(accuracy, - feed_dict={x: mnist.test.images, - y_: mnist.test.labels}) - print 'Done training!' - - # Export model - # WARNING(break-tutorial-inline-code): The following code snippet is - # in-lined in tutorials, please update tutorial documents accordingly - # whenever code changes. - export_path = sys.argv[-1] - print 'Exporting trained model to', export_path - saver = tf.train.Saver(sharded=True) - model_exporter = exporter.Exporter(saver) - signature = exporter.classification_signature(input_tensor=x, scores_tensor=y) - model_exporter.init(sess.graph.as_graph_def(), - default_graph_signature=signature) - model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess) - print 'Done exporting!' - - -if __name__ == '__main__': - tf.app.run() diff --git a/tensorflow_serving/example/mnist_inference.cc b/tensorflow_serving/example/mnist_inference.cc deleted file mode 100644 index ec2631a9183..00000000000 --- a/tensorflow_serving/example/mnist_inference.cc +++ /dev/null @@ -1,219 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A gRPC server that classifies images into digit 0-9. -// Given each request with an image pixels encoded as floats, the server -// responds with 10 float values as probabilities for digit 0-9 respectively. -// The classification is done by running image data through a simple softmax -// regression network trained and exported by mnist_export.py. -// The intention of this example to demonstrate usage of Tensorflow -// APIs in an end-to-end scenario. - -#include -#include -#include -#include -#include - -#include "grpc++/security/server_credentials.h" -#include "grpc++/server.h" -#include "grpc++/server_builder.h" -#include "grpc++/server_context.h" -#include "grpc++/support/status.h" -#include "grpc++/support/status_code_enum.h" -#include "grpc/grpc.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/public/session_options.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow_serving/example/mnist_inference.grpc.pb.h" -#include "tensorflow_serving/example/mnist_inference.pb.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_factory.h" - -using grpc::InsecureServerCredentials; -using grpc::Server; -using grpc::ServerBuilder; -using grpc::ServerContext; -using grpc::Status; -using grpc::StatusCode; -using tensorflow::serving::ClassificationSignature; -using tensorflow::serving::MnistRequest; -using tensorflow::serving::MnistResponse; -using tensorflow::serving::MnistService; -using tensorflow::serving::BatchingParameters; -using tensorflow::serving::SessionBundle; -using tensorflow::serving::SessionBundleConfig; -using tensorflow::serving::SessionBundleFactory; -using tensorflow::string; -using tensorflow::Tensor; -using tensorflow::TensorShape; - -namespace { -const int kImageSize = 28; -const int kNumChannels = 1; -const int kImageDataSize = kImageSize * kImageSize * kNumChannels; -const int kNumLabels = 10; - -// Creates a gRPC Status from a TensorFlow Status. -Status ToGRPCStatus(const tensorflow::Status& status) { - return Status(static_cast(status.code()), - status.error_message()); -} - -class MnistServiceImpl final : public MnistService::Service { - public: - explicit MnistServiceImpl(std::unique_ptr bundle) - : bundle_(std::move(bundle)) { - signature_status_ = tensorflow::serving::GetClassificationSignature( - bundle_->meta_graph_def, &signature_); - } - - Status Classify(ServerContext* context, const MnistRequest* request, - MnistResponse* response) override { - // Verify protobuf input. - if (request->image_data_size() != kImageDataSize) { - return Status(StatusCode::INVALID_ARGUMENT, - tensorflow::strings::StrCat("expected image_data of size ", - kImageDataSize, ", got ", - request->image_data_size())); - } - - // Transform protobuf input to inference input tensor and create - // output tensor placeholder. - // See mnist_export.py for details. - Tensor input(tensorflow::DT_FLOAT, {1, kImageDataSize}); - std::copy_n(request->image_data().begin(), kImageDataSize, - input.flat().data()); - std::vector outputs; - - // Run inference. - if (!signature_status_.ok()) { - return ToGRPCStatus(signature_status_); - } - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - const tensorflow::Status status = bundle_->session->Run( - {{signature_.input().tensor_name(), input}}, - {signature_.scores().tensor_name()}, {}, &outputs); - if (!status.ok()) { - return ToGRPCStatus(status); - } - - // Transform inference output tensor to protobuf output. - // See mnist_export.py for details. - if (outputs.size() != 1) { - return Status(StatusCode::INTERNAL, - tensorflow::strings::StrCat( - "expected one model output, got ", outputs.size())); - } - const Tensor& score_tensor = outputs[0]; - const TensorShape expected_shape({1, kNumLabels}); - if (!score_tensor.shape().IsSameSize(expected_shape)) { - return Status( - StatusCode::INTERNAL, - tensorflow::strings::StrCat("expected output of size ", - expected_shape.DebugString(), ", got ", - score_tensor.shape().DebugString())); - } - const auto score_flat = outputs[0].flat(); - for (int i = 0; i < score_flat.size(); ++i) { - response->add_value(score_flat(i)); - } - - return Status::OK; - } - - private: - std::unique_ptr bundle_; - tensorflow::Status signature_status_; - ClassificationSignature signature_; -}; - -void RunServer(int port, std::unique_ptr bundle) { - // "0.0.0.0" is the way to listen on localhost in gRPC. - const string server_address = "0.0.0.0:" + std::to_string(port); - MnistServiceImpl service(std::move(bundle)); - ServerBuilder builder; - std::shared_ptr creds = InsecureServerCredentials(); - builder.AddListeningPort(server_address, creds); - builder.RegisterService(&service); - std::unique_ptr server(builder.BuildAndStart()); - LOG(INFO) << "Running..."; - server->Wait(); -} - -} // namespace - -int main(int argc, char** argv) { - tensorflow::int32 port = 0; - const bool parse_result = - tensorflow::ParseFlags(&argc, argv, {tensorflow::Flag("port", &port)}); - if (!parse_result) { - LOG(FATAL) << "Error parsing command line flags."; - } - - if (argc != 2) { - LOG(FATAL) << "Usage: mnist_inference --port=9000 /path/to/export"; - } - const string bundle_path(argv[1]); - - tensorflow::port::InitMain(argv[0], &argc, &argv); - - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - - SessionBundleConfig session_bundle_config; - - ////// - // Request batching, keeping default values for the tuning parameters. - // - // (If you prefer to disable batching, simply omit the following lines of code - // such that session_bundle_config.batching_parameters remains unset.) - BatchingParameters* batching_parameters = - session_bundle_config.mutable_batching_parameters(); - batching_parameters->mutable_thread_pool_name()->set_value( - "mnist_service_batch_threads"); - // Use a very large queue, to avoid rejecting requests. (Note: a production - // server with load balancing may want to use the default, much smaller, - // value.) - batching_parameters->mutable_max_enqueued_batches()->set_value(1000); - ////// - - std::unique_ptr bundle_factory; - TF_QCHECK_OK( - SessionBundleFactory::Create(session_bundle_config, &bundle_factory)); - std::unique_ptr bundle(new SessionBundle); - TF_QCHECK_OK(bundle_factory->CreateSessionBundle(bundle_path, &bundle)); - - // END WARNING(break-tutorial-inline-code) - - RunServer(port, std::move(bundle)); - - return 0; -} diff --git a/tensorflow_serving/example/mnist_inference.proto b/tensorflow_serving/example/mnist_inference.proto deleted file mode 100644 index 54e9a5e1485..00000000000 --- a/tensorflow_serving/example/mnist_inference.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Protobuf definition of MNIST model inference server. -// The server classifies 28x28 greyscale image into digit 0-9. -// See mnist_export.py for model details. -// See mnist_inference.cc for server implementation. - -syntax = "proto3"; - -package tensorflow.serving; - -message MnistRequest { - // Row-major encoding of a 28x28 image, each byte being the grayscale value - // of a pixel rescaled from [0, 255] down to [-0.5, 0.5]. - repeated float image_data = 1 [packed = true]; -}; - -message MnistResponse { - // Matching probability of digit 0-9 in range [0.0, 1.0]. - repeated float value = 1 [packed = true]; -}; - -service MnistService { - // Classifies image into digits. - rpc Classify(MnistRequest) returns (MnistResponse); -} diff --git a/tensorflow_serving/example/mnist_inference_2.cc b/tensorflow_serving/example/mnist_inference_2.cc deleted file mode 100644 index b6d003028fd..00000000000 --- a/tensorflow_serving/example/mnist_inference_2.cc +++ /dev/null @@ -1,432 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A gRPC server that classifies images into digit 0-9. -// Given each request with an image pixels encoded as floats, the server -// responds with 10 float values as probabilities for digit 0-9 respectively. -// The classification is done by running image data through a convolutional -// network trained and exported by mnist_model.py. -// The server constantly monitors a file system storage path for models. -// Whenever a new version of model is available, it eagerly unloads older -// version before loading the new one. The server also batches multiple -// requests together and does batched inference for efficiency. -// The intention of this example to demonstrate usage of DymanicManager, -// VersionPolicy and BasicBatchScheduler. - -#include -#include -#include -#include -#include - -#include "grpc++/completion_queue.h" -#include "grpc++/security/server_credentials.h" -#include "grpc++/server.h" -#include "grpc++/server_builder.h" -#include "grpc++/server_context.h" -#include "grpc++/support/async_unary_call.h" -#include "grpc++/support/status.h" -#include "grpc++/support/status_code_enum.h" -#include "grpc/grpc.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" -#include "tensorflow_serving/batching/batch_scheduler.h" -#include "tensorflow_serving/core/manager.h" -#include "tensorflow_serving/core/servable_handle.h" -#include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/example/mnist_inference.grpc.pb.h" -#include "tensorflow_serving/example/mnist_inference.pb.h" -#include "tensorflow_serving/servables/tensorflow/simple_servers.h" - -using grpc::InsecureServerCredentials; -using grpc::Server; -using grpc::ServerAsyncResponseWriter; -using grpc::ServerBuilder; -using grpc::ServerContext; -using grpc::ServerCompletionQueue; -using grpc::Status; -using grpc::StatusCode; -using tensorflow::serving::MnistRequest; -using tensorflow::serving::MnistResponse; -using tensorflow::serving::MnistService; -using tensorflow::string; -using tensorflow::Tensor; -using tensorflow::serving::ClassificationSignature; - -namespace { -const int kImageSize = 28; -const int kNumChannels = 1; -const int kImageDataSize = kImageSize * kImageSize * kNumChannels; -const int kNumLabels = 10; - -class MnistServiceImpl; - -// Class encompassing the state and logic needed to serve a request. -class CallData { - public: - CallData(MnistServiceImpl* service_impl, - MnistService::AsyncService* service, - ServerCompletionQueue* cq); - - void Proceed(); - - void Finish(Status status); - - const MnistRequest& request() { return request_; } - - MnistResponse* mutable_response() { return &response_; } - - private: - // Service implementation. - MnistServiceImpl* service_impl_; - - // The means of communication with the gRPC runtime for an asynchronous - // server. - MnistService::AsyncService* service_; - // The producer-consumer queue where for asynchronous server notifications. - ServerCompletionQueue* cq_; - // Context for the rpc, allowing to tweak aspects of it such as the use - // of compression, authentication, as well as to send metadata back to the - // client. - ServerContext ctx_; - - // What we get from the client. - MnistRequest request_; - // What we send back to the client. - MnistResponse response_; - - // The means to get back to the client. - ServerAsyncResponseWriter responder_; - - // Let's implement a tiny state machine with the following states. - enum CallStatus { CREATE, PROCESS, FINISH }; - CallStatus status_; // The current serving state. -}; - -// A Task holds all of the information for a single inference request. -struct Task : public tensorflow::serving::BatchTask { - ~Task() override = default; - size_t size() const override { return 1; } - - Task(CallData* calldata_arg) - : calldata(calldata_arg) {} - - CallData* calldata; -}; - -class MnistServiceImpl final { - public: - MnistServiceImpl(const string& servable_name, - std::unique_ptr manager); - - void Classify(CallData* call_data); - - // Produces classifications for a batch of requests and associated responses. - void DoClassifyInBatch( - std::unique_ptr> batch); - - // Name of the servable to use for inference. - const string servable_name_; - // Manager in charge of loading and unloading servables. - std::unique_ptr manager_; - // A scheduler for batching multiple request calls into single calls to - // Session->Run(). - std::unique_ptr> - batch_scheduler_; -}; - -// Take in the "service" instance (in this case representing an asynchronous -// server) and the completion queue "cq" used for asynchronous communication -// with the gRPC runtime. -CallData::CallData(MnistServiceImpl* service_impl, - MnistService::AsyncService* service, - ServerCompletionQueue* cq) - : service_impl_(service_impl), - service_(service), cq_(cq), responder_(&ctx_), status_(CREATE) { - // Invoke the serving logic right away. - Proceed(); -} - -void CallData::Proceed() { - if (status_ == CREATE) { - // As part of the initial CREATE state, we *request* that the system - // start processing Classify requests. In this request, "this" acts are - // the tag uniquely identifying the request (so that different CallData - // instances can serve different requests concurrently), in this case - // the memory address of this CallData instance. - service_->RequestClassify(&ctx_, &request_, &responder_, cq_, cq_, this); - // Make this instance progress to the PROCESS state. - status_ = PROCESS; - } else if (status_ == PROCESS) { - // Spawn a new CallData instance to serve new clients while we process - // the one for this CallData. The instance will deallocate itself as - // part of its FINISH state. - new CallData(service_impl_, service_, cq_); - // Start processing. - service_impl_->Classify(this); - } else { - GPR_ASSERT(status_ == FINISH); - // Once in the FINISH state, deallocate ourselves (CallData). - delete this; - } -} - -void CallData::Finish(Status status) { - status_ = FINISH; - responder_.Finish(response_, status, this); -} - -MnistServiceImpl::MnistServiceImpl( - const string& servable_name, - std::unique_ptr manager) - : servable_name_(servable_name), manager_(std::move(manager)) { - // Setup a batcher used to combine multiple requests (tasks) into a single - // graph run for efficiency. - // The batcher queues tasks until, - // (a) the next task would cause the batch to exceed the size target; - // (b) waiting for more tasks to be added would exceed the timeout. - // at which point it processes the entire batch. - // - // Use the default batch-size, timeout and thread options. In general - // the numbers are extremely performance critical and should be tuned based - // specific graph structure and usage. - tensorflow::serving::BasicBatchScheduler::Options scheduler_options; - scheduler_options.thread_pool_name = "mnist_service_batch_threads"; - // Use a very large queue, to avoid rejecting requests. (Note: a production - // server with load balancing may want to use the default, much smaller, - // value.) - scheduler_options.max_enqueued_batches = 1000; - TF_CHECK_OK(tensorflow::serving::BasicBatchScheduler::Create( - scheduler_options, - [this](std::unique_ptr> batch) { - this->DoClassifyInBatch(std::move(batch)); - }, - &batch_scheduler_)); -} - -// Creates a gRPC Status from a TensorFlow Status. -Status ToGRPCStatus(const tensorflow::Status& status) { - return Status(static_cast(status.code()), - status.error_message()); -} - -// WARNING(break-tutorial-inline-code): The following code snippet is -// in-lined in tutorials, please update tutorial documents accordingly -// whenever code changes. -void MnistServiceImpl::Classify(CallData* calldata) { - // Verify input. - if (calldata->request().image_data_size() != kImageDataSize) { - calldata->Finish( - Status(StatusCode::INVALID_ARGUMENT, - tensorflow::strings::StrCat( - "expected image_data of size ", kImageDataSize, - ", got ", calldata->request().image_data_size()))); - return; - } - - // Create and submit a task to the batch scheduler. - std::unique_ptr task(new Task(calldata)); - tensorflow::Status status = batch_scheduler_->Schedule(&task); - - if (!status.ok()) { - calldata->Finish(ToGRPCStatus(status)); - return; - } -} - -// Produces classifications for a batch of requests and associated responses. -void MnistServiceImpl::DoClassifyInBatch( - std::unique_ptr> batch) { - batch->WaitUntilClosed(); - if (batch->empty()) { - return; - } - const int batch_size = batch->num_tasks(); - - // Replies to each task with the given error status. - auto complete_with_error = [&batch](StatusCode code, const string& msg) { - Status status(code, msg); - for (int i = 0; i < batch->num_tasks(); i++) { - Task* task = batch->mutable_task(i); - task->calldata->Finish(status); - } - }; - - // Get a handle to the SessionBundle. The handle ensures the Manager does - // not reload this while it is in use. - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - auto handle_request = - tensorflow::serving::ServableRequest::Latest(servable_name_); - tensorflow::serving::ServableHandle - bundle; - const tensorflow::Status lookup_status = - manager_->GetServableHandle(handle_request, &bundle); - if (!lookup_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - lookup_status.error_message()); - return; - } - - // Get the default signature of the graph. Expected to be a - // classification signature. - tensorflow::serving::ClassificationSignature signature; - const tensorflow::Status signature_status = - GetClassificationSignature(bundle->meta_graph_def, &signature); - if (!signature_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - signature_status.error_message()); - return; - } - - // Transform protobuf input to inference input tensor. - // See mnist_model.py for details. - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - Tensor input(tensorflow::DT_FLOAT, {batch_size, kImageDataSize}); - auto dst = input.flat_outer_dims().data(); - for (int i = 0; i < batch_size; ++i) { - std::copy_n( - batch->mutable_task(i)->calldata->request().image_data().begin(), - kImageDataSize, dst); - dst += kImageDataSize; - } - - // Run classification. - tensorflow::Tensor scores; - const tensorflow::Status run_status = - RunClassification(signature, input, bundle->session.get(), - nullptr /* classes */, &scores); - if (!run_status.ok()) { - complete_with_error(StatusCode::INTERNAL, run_status.error_message()); - return; - } - if (scores.dtype() != tensorflow::DT_FLOAT) { - complete_with_error( - StatusCode::INTERNAL, - tensorflow::strings::StrCat( - "Expected output Tensor of DT_FLOAT. Got: ", - tensorflow::DataType_Name(scores.dtype()))); - return; - } - if (scores.dim_size(1) != kNumLabels) { - complete_with_error( - StatusCode::INTERNAL, - tensorflow::strings::StrCat( - "Expected ", kNumLabels, " labels in each output. Got: ", - scores.dim_size(1))); - return; - } - - // Transform inference output tensor to protobuf output. - // See mnist_model.py for details. - const auto& scores_mat = scores.matrix(); - for (int i = 0; i < batch_size; ++i) { - auto calldata = batch->mutable_task(i)->calldata; - for (int c = 0; c < scores.dim_size(1); ++c) { - calldata->mutable_response()->add_value(scores_mat(i, c)); - } - calldata->Finish(Status::OK); - } -} - -void HandleRpcs(MnistServiceImpl* service_impl, - MnistService::AsyncService* service, - ServerCompletionQueue* cq) { - // Spawn a new CallData instance to serve new clients. - new CallData(service_impl, service, cq); - void* tag; // uniquely identifies a request. - bool ok; - while (true) { - // Block waiting to read the next event from the completion queue. The - // event is uniquely identified by its tag, which in this case is the - // memory address of a CallData instance. - cq->Next(&tag, &ok); - GPR_ASSERT(ok); - static_cast(tag)->Proceed(); - } -} - -// Runs MnistService server until shutdown. -void RunServer(const int port, const string& servable_name, - std::unique_ptr manager) { - // "0.0.0.0" is the way to listen on localhost in gRPC. - const string server_address = "0.0.0.0:" + std::to_string(port); - - MnistService::AsyncService service; - ServerBuilder builder; - std::shared_ptr creds = InsecureServerCredentials(); - builder.AddListeningPort(server_address, creds); - builder.RegisterService(&service); - std::unique_ptr cq = builder.AddCompletionQueue(); - std::unique_ptr server(builder.BuildAndStart()); - LOG(INFO) << "Running..."; - - MnistServiceImpl service_impl(servable_name, std::move(manager)); - HandleRpcs(&service_impl, &service, cq.get()); -} - -} // namespace - -int main(int argc, char** argv) { - // Parse command-line options. - tensorflow::int32 port = 0; - const bool parse_result = - tensorflow::ParseFlags(&argc, argv, {tensorflow::Flag("port", &port)}); - if (!parse_result) { - LOG(FATAL) << "Error parsing command line flags."; - } - if (argc != 2) { - LOG(FATAL) << "Usage: mnist_inference_2 --port=9000 /path/to/exports"; - } - const string export_base_path(argv[1]); - tensorflow::port::InitMain(argv[0], &argc, &argv); - - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - std::unique_ptr manager; - tensorflow::Status status = tensorflow::serving::simple_servers:: - CreateSingleTFModelManagerFromBasePath(export_base_path, &manager); - - TF_CHECK_OK(status) << "Error creating manager"; - - // Wait until at least one model is loaded. - std::vector ready_ids; - // TODO(b/25545573): Create a more streamlined startup mechanism than polling. - do { - LOG(INFO) << "Waiting for models to be loaded..."; - tensorflow::Env::Default()->SleepForMicroseconds(1 * 1000 * 1000 /*1 sec*/); - ready_ids = manager->ListAvailableServableIds(); - } while (ready_ids.empty()); - - // Run the service. - RunServer(port, ready_ids[0].name, std::move(manager)); - - return 0; -} diff --git a/tensorflow_serving/example/mnist_inference_pb2.py b/tensorflow_serving/example/mnist_inference_pb2.py deleted file mode 100644 index 2c5354c1b97..00000000000 --- a/tensorflow_serving/example/mnist_inference_pb2.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mnist_inference.proto - -import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mnist_inference.proto', - package='tensorflow.serving', - syntax='proto3', - serialized_pb= - _b('\n\x15mnist_inference.proto\x12\x12tensorflow.serving\"&\n\x0cMnistRequest\x12\x16\n\nimage_data\x18\x01 \x03(\x02\x42\x02\x10\x01\"\"\n\rMnistResponse\x12\x11\n\x05value\x18\x01 \x03(\x02\x42\x02\x10\x01\x32_\n\x0cMnistService\x12O\n\x08\x43lassify\x12 .tensorflow.serving.MnistRequest\x1a!.tensorflow.serving.MnistResponseb\x06proto3')) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_MNISTREQUEST = _descriptor.Descriptor( - name='MnistRequest', - full_name='tensorflow.serving.MnistRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='image_data', full_name='tensorflow.serving.MnistRequest.image_data', index=0, - number=1, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), _b('\020\001'))), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=45, - serialized_end=83, -) - - -_MNISTRESPONSE = _descriptor.Descriptor( - name='MnistResponse', - full_name='tensorflow.serving.MnistResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='tensorflow.serving.MnistResponse.value', index=0, - number=1, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), _b('\020\001'))), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=85, - serialized_end=119, -) - -DESCRIPTOR.message_types_by_name['MnistRequest'] = _MNISTREQUEST -DESCRIPTOR.message_types_by_name['MnistResponse'] = _MNISTRESPONSE - -MnistRequest = _reflection.GeneratedProtocolMessageType('MnistRequest', (_message.Message,), dict( - DESCRIPTOR = _MNISTREQUEST, - __module__ = 'mnist_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.MnistRequest) - )) -_sym_db.RegisterMessage(MnistRequest) - -MnistResponse = _reflection.GeneratedProtocolMessageType('MnistResponse', (_message.Message,), dict( - DESCRIPTOR = _MNISTRESPONSE, - __module__ = 'mnist_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.MnistResponse) - )) -_sym_db.RegisterMessage(MnistResponse) - - -_MNISTREQUEST.fields_by_name['image_data'].has_options = True -_MNISTREQUEST.fields_by_name['image_data']._options = _descriptor._ParseOptions( - descriptor_pb2.FieldOptions(), _b('\020\001')) -_MNISTRESPONSE.fields_by_name['value'].has_options = True -_MNISTRESPONSE.fields_by_name['value']._options = _descriptor._ParseOptions( - descriptor_pb2.FieldOptions(), _b('\020\001')) -import abc -from grpc.beta import implementations as beta_implementations -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities - -class BetaMnistServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def Classify(self, request, context): - raise NotImplementedError() - -class BetaMnistServiceStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def Classify(self, request, timeout): - raise NotImplementedError() - Classify.future = None - -def beta_create_MnistService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import mnist_inference_pb2 - import mnist_inference_pb2 - request_deserializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistRequest.FromString, - } - response_serializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistResponse.SerializeToString, - } - method_implementations = { - ('tensorflow.serving.MnistService', 'Classify'): face_utilities.unary_unary_inline(servicer.Classify), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_MnistService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import mnist_inference_pb2 - import mnist_inference_pb2 - request_serializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistRequest.SerializeToString, - } - response_deserializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistResponse.FromString, - } - cardinalities = { - 'Classify': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.MnistService', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/example/mnist_input_data.py b/tensorflow_serving/example/mnist_input_data.py index 66693e059d2..3e8021a2435 100644 --- a/tensorflow_serving/example/mnist_input_data.py +++ b/tensorflow_serving/example/mnist_input_data.py @@ -17,13 +17,16 @@ """Functions for downloading and reading MNIST data.""" +from __future__ import print_function + import gzip import os import numpy from six.moves import urllib -SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' +# CVDF mirror of http://yann.lecun.com/exdb/mnist/ +SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/' TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' TEST_IMAGES = 't10k-images-idx3-ubyte.gz' @@ -39,7 +42,7 @@ def maybe_download(filename, work_directory): if not os.path.exists(filepath): filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath) statinfo = os.stat(filepath) - print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') + print('Successfully downloaded %s %d bytes.' % (filename, statinfo.st_size)) return filepath @@ -50,7 +53,7 @@ def _read32(bytestream): def extract_images(filename): """Extract the images into a 4D uint8 numpy array [index, y, x, depth].""" - print('Extracting', filename) + print('Extracting %s' % filename) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2051: @@ -77,7 +80,7 @@ def dense_to_one_hot(labels_dense, num_classes=10): def extract_labels(filename, one_hot=False): """Extract the labels into a 1D uint8 numpy array [index].""" - print('Extracting', filename) + print('Extracting %s' % filename) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2049: @@ -144,8 +147,9 @@ def next_batch(self, batch_size, fake_data=False): fake_label = [1] + [0] * 9 else: fake_label = 0 - return [fake_image for _ in xrange(batch_size)], [ - fake_label for _ in xrange(batch_size)] + return [fake_image for _ in range(batch_size)], [ + fake_label for _ in range(batch_size) + ] start = self._index_in_epoch self._index_in_epoch += batch_size if self._index_in_epoch > self._num_examples: diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py new file mode 100644 index 00000000000..0bb3053893c --- /dev/null +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -0,0 +1,159 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +#! /usr/bin/env python +r"""Train and export a simple Softmax Regression TensorFlow model. + +The model is from the TensorFlow "MNIST For ML Beginner" tutorial. This program +simply follows all its training instructions, and uses TensorFlow SavedModel to +export the trained model with proper signatures that can be loaded by standard +tensorflow_model_server. + +Usage: mnist_saved_model.py [--training_iteration=x] [--model_version=y] \ + export_dir +""" + +from __future__ import print_function + +import os +import sys + +import tensorflow as tf + +from tensorflow.python.ops import lookup_ops + +import mnist_input_data + +tf.compat.v1.app.flags.DEFINE_integer('training_iteration', 1000, + 'number of training iterations.') +tf.compat.v1.app.flags.DEFINE_integer('model_version', 1, + 'version number of the model.') +tf.compat.v1.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory.') +FLAGS = tf.compat.v1.app.flags.FLAGS + +tf.compat.v1.disable_eager_execution() + + +def main(_): + if len(sys.argv) < 2 or sys.argv[-1].startswith('-'): + print('Usage: mnist_saved_model.py [--training_iteration=x] ' + '[--model_version=y] export_dir') + sys.exit(-1) + if FLAGS.training_iteration <= 0: + print('Please specify a positive value for training iteration.') + sys.exit(-1) + if FLAGS.model_version <= 0: + print('Please specify a positive value for version number.') + sys.exit(-1) + + # Train model + print('Training model...') + mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) + sess = tf.compat.v1.InteractiveSession() + serialized_tf_example = tf.compat.v1.placeholder(tf.string, name='tf_example') + feature_configs = { + 'x': tf.io.FixedLenFeature(shape=[784], dtype=tf.float32), + } + tf_example = tf.io.parse_example(serialized_tf_example, feature_configs) + x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name + y_ = tf.compat.v1.placeholder('float', shape=[None, 10]) + w = tf.Variable(tf.zeros([784, 10])) + b = tf.Variable(tf.zeros([10])) + sess.run(tf.compat.v1.global_variables_initializer()) + y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') + cross_entropy = -tf.math.reduce_sum(y_ * tf.math.log(y)) + train_step = tf.compat.v1.train.GradientDescentOptimizer(0.01).minimize( + cross_entropy) + values, indices = tf.nn.top_k(y, 10) + table = lookup_ops.index_to_string_table_from_tensor( + tf.constant([str(i) for i in range(10)])) + prediction_classes = table.lookup(tf.dtypes.cast(indices, tf.int64)) + for _ in range(FLAGS.training_iteration): + batch = mnist.train.next_batch(50) + train_step.run(feed_dict={x: batch[0], y_: batch[1]}) + correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) + accuracy = tf.math.reduce_mean(tf.cast(correct_prediction, 'float')) + print('training accuracy %g' % sess.run( + accuracy, feed_dict={ + x: mnist.test.images, + y_: mnist.test.labels + })) + print('Done training!') + + # Export model + # WARNING(break-tutorial-inline-code): The following code snippet is + # in-lined in tutorials, please update tutorial documents accordingly + # whenever code changes. + export_path_base = sys.argv[-1] + export_path = os.path.join( + tf.compat.as_bytes(export_path_base), + tf.compat.as_bytes(str(FLAGS.model_version))) + print('Exporting trained model to', export_path) + builder = tf.compat.v1.saved_model.builder.SavedModelBuilder(export_path) + + # Build the signature_def_map. + classification_inputs = tf.compat.v1.saved_model.utils.build_tensor_info( + serialized_tf_example) + classification_outputs_classes = tf.compat.v1.saved_model.utils.build_tensor_info( + prediction_classes) + classification_outputs_scores = tf.compat.v1.saved_model.utils.build_tensor_info( + values) + + classification_signature = ( + tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={ + tf.compat.v1.saved_model.signature_constants.CLASSIFY_INPUTS: + classification_inputs + }, + outputs={ + tf.compat.v1.saved_model.signature_constants + .CLASSIFY_OUTPUT_CLASSES: + classification_outputs_classes, + tf.compat.v1.saved_model.signature_constants + .CLASSIFY_OUTPUT_SCORES: + classification_outputs_scores + }, + method_name=tf.compat.v1.saved_model.signature_constants + .CLASSIFY_METHOD_NAME)) + + tensor_info_x = tf.compat.v1.saved_model.utils.build_tensor_info(x) + tensor_info_y = tf.compat.v1.saved_model.utils.build_tensor_info(y) + + prediction_signature = ( + tf.compat.v1.saved_model.signature_def_utils.build_signature_def( + inputs={'images': tensor_info_x}, + outputs={'scores': tensor_info_y}, + method_name=tf.compat.v1.saved_model.signature_constants + .PREDICT_METHOD_NAME)) + + builder.add_meta_graph_and_variables( + sess, [tf.compat.v1.saved_model.tag_constants.SERVING], + signature_def_map={ + 'predict_images': + prediction_signature, + tf.compat.v1.saved_model.signature_constants + .DEFAULT_SERVING_SIGNATURE_DEF_KEY: + classification_signature, + }, + main_op=tf.compat.v1.tables_initializer(), + strip_default_attrs=True) + + builder.save() + + print('Done exporting!') + + +if __name__ == '__main__': + tf.compat.v1.app.run() diff --git a/tensorflow_serving/example/resnet_client.cc b/tensorflow_serving/example/resnet_client.cc new file mode 100644 index 00000000000..edcf01b91e5 --- /dev/null +++ b/tensorflow_serving/example/resnet_client.cc @@ -0,0 +1,211 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include + +#include +#include +#include +#include + +#include "grpcpp/create_channel.h" +#include "grpcpp/security/credentials.h" +#include "google/protobuf/map.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/jpeg.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/command_line_flags.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" + +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; + +using tensorflow::serving::PredictRequest; +using tensorflow::serving::PredictResponse; +using tensorflow::serving::PredictionService; + +typedef google::protobuf::Map OutMap; + +struct tf_jpeg_error_mgr { + struct jpeg_error_mgr pub; + jmp_buf setjmp_buffer; +}; + +typedef struct tf_jpeg_error_mgr* tf_jpeg_error_ptr; + +METHODDEF(void) +tf_jpeg_error_exit(j_common_ptr cinfo) { + tf_jpeg_error_ptr tf_jpeg_err = (tf_jpeg_error_ptr)cinfo->err; + + (*cinfo->err->output_message)(cinfo); + + longjmp(tf_jpeg_err->setjmp_buffer, 1); +} + +class ServingClient { + public: + // JPEG decompression code following libjpeg-turbo documentation: + // https://github.com/libjpeg-turbo/libjpeg-turbo/blob/main/example.txt + int readJPEG(const char* file_name, tensorflow::TensorProto* proto) { + struct tf_jpeg_error_mgr jerr; + FILE* infile; + JSAMPARRAY buffer; + int row_stride; + struct jpeg_decompress_struct cinfo; + + if ((infile = fopen(file_name, "rb")) == NULL) { + fprintf(stderr, "can't open %s\n", file_name); + return -1; + } + + cinfo.err = jpeg_std_error(&jerr.pub); + jerr.pub.error_exit = tf_jpeg_error_exit; + if (setjmp(jerr.setjmp_buffer)) { + jpeg_destroy_decompress(&cinfo); + fclose(infile); + return -1; + } + + jpeg_create_decompress(&cinfo); + jpeg_stdio_src(&cinfo, infile); + + (void)jpeg_read_header(&cinfo, TRUE); + + (void)jpeg_start_decompress(&cinfo); + row_stride = cinfo.output_width * cinfo.output_components; + CHECK(cinfo.output_components == 3) + << "Only 3-channel (RGB) JPEG files are supported"; + + buffer = (*cinfo.mem->alloc_sarray)((j_common_ptr)&cinfo, JPOOL_IMAGE, + row_stride, 1); + + proto->set_dtype(tensorflow::DataType::DT_FLOAT); + while (cinfo.output_scanline < cinfo.output_height) { + (void)jpeg_read_scanlines(&cinfo, buffer, 1); + for (size_t i = 0; i < cinfo.output_width; i++) { + proto->add_float_val(buffer[0][i * 3] / 255.0); + proto->add_float_val(buffer[0][i * 3 + 1] / 255.0); + proto->add_float_val(buffer[0][i * 3 + 2] / 255.0); + } + } + + proto->mutable_tensor_shape()->add_dim()->set_size(1); + proto->mutable_tensor_shape()->add_dim()->set_size(cinfo.output_height); + proto->mutable_tensor_shape()->add_dim()->set_size(cinfo.output_width); + proto->mutable_tensor_shape()->add_dim()->set_size(cinfo.output_components); + + (void)jpeg_finish_decompress(&cinfo); + + jpeg_destroy_decompress(&cinfo); + fclose(infile); + return 0; + } + + ServingClient(std::shared_ptr channel) + : stub_(PredictionService::NewStub(channel)) {} + + tensorflow::string callPredict(const tensorflow::string& model_name, + const tensorflow::string& model_signature_name, + const tensorflow::string& input_name, + const tensorflow::string& file_path) { + PredictRequest predictRequest; + PredictResponse response; + ClientContext context; + + predictRequest.mutable_model_spec()->set_name(model_name); + predictRequest.mutable_model_spec()->set_signature_name( + model_signature_name); + + google::protobuf::Map& inputs = + *predictRequest.mutable_inputs(); + + tensorflow::TensorProto proto; + + const char* infile = file_path.c_str(); + + if (readJPEG(infile, &proto)) { + std::cout << "error constructing the protobuf"; + return "execution failed"; + } + + inputs[input_name] = proto; + + Status status = stub_->Predict(&context, predictRequest, &response); + + if (status.ok()) { + std::cout << "call predict ok" << std::endl; + std::cout << "outputs size is " << response.outputs_size() << std::endl; + OutMap& map_outputs = *response.mutable_outputs(); + OutMap::iterator iter; + int output_index = 0; + + for (iter = map_outputs.begin(); iter != map_outputs.end(); ++iter) { + tensorflow::TensorProto& result_tensor_proto = iter->second; + tensorflow::Tensor tensor; + bool converted = tensor.FromProto(result_tensor_proto); + if (converted) { + std::cout << "the result tensor[" << output_index + << "] is:" << std::endl + << tensor.SummarizeValue(1001) << std::endl; + } else { + std::cout << "the result tensor[" << output_index + << "] convert failed." << std::endl; + } + ++output_index; + } + return "Done."; + } else { + std::cout << "gRPC call return code: " << status.error_code() << ": " + << status.error_message() << std::endl; + return "gRPC failed."; + } + } + + private: + std::unique_ptr stub_; +}; + +int main(int argc, char** argv) { + tensorflow::string server_port = "localhost:8500"; + tensorflow::string image_file = ""; + tensorflow::string model_name = "resnet"; + tensorflow::string model_signature_name = "serving_default"; + tensorflow::string input_name = "input_1"; + std::vector flag_list = { + tensorflow::Flag("server_port", &server_port, + "the IP and port of the server"), + tensorflow::Flag("image_file", &image_file, "the path to the image"), + tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("model_signature_name", &model_signature_name, + "name of model signature"), + tensorflow::Flag("input_name", &input_name, "name of input tensor")}; + + tensorflow::string usage = tensorflow::Flags::Usage(argv[0], flag_list); + const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); + if (!parse_result || image_file.empty()) { + std::cout << usage; + return -1; + } + + ServingClient guide( + grpc::CreateChannel(server_port, grpc::InsecureChannelCredentials())); + std::cout << "calling predict using file: " << image_file << " ..." + << std::endl; + std::cout << guide.callPredict(model_name, model_signature_name, input_name, + image_file) + << std::endl; + return 0; +} diff --git a/tensorflow_serving/example/resnet_client.py b/tensorflow_serving/example/resnet_client.py new file mode 100644 index 00000000000..b531166a91d --- /dev/null +++ b/tensorflow_serving/example/resnet_client.py @@ -0,0 +1,87 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""A client that performs inferences on a ResNet model using the REST API. + +The client downloads a test image of a cat, queries the server over the REST API +with the test image repeatedly and measures how long it takes to respond. + +The client expects a TensorFlow Serving ModelServer running a ResNet SavedModel +from: + +https://github.com/tensorflow/models/tree/master/official/vision/image_classification/resnet#pretrained-models + +Typical usage example: + + resnet_client.py +""" + +from __future__ import print_function + +import base64 +import io +import json + +import numpy as np +from PIL import Image +import requests + +# The server URL specifies the endpoint of your server running the ResNet +# model with the name "resnet" and using the predict interface. +SERVER_URL = 'http://localhost:8501/v1/models/resnet:predict' + +# The image URL is the location of the image we should send to the server +IMAGE_URL = 'https://tensorflow.org/images/blogs/serving/cat.jpg' + +# Current Resnet model in TF Model Garden (as of 7/2021) does not accept JPEG +# as input +MODEL_ACCEPT_JPG = False + + +def main(): + # Download the image + dl_request = requests.get(IMAGE_URL, stream=True) + dl_request.raise_for_status() + + if MODEL_ACCEPT_JPG: + # Compose a JSON Predict request (send JPEG image in base64). + jpeg_bytes = base64.b64encode(dl_request.content).decode('utf-8') + predict_request = '{"instances" : [{"b64": "%s"}]}' % jpeg_bytes + else: + # Compose a JOSN Predict request (send the image tensor). + jpeg_rgb = Image.open(io.BytesIO(dl_request.content)) + # Normalize and batchify the image + jpeg_rgb = np.expand_dims(np.array(jpeg_rgb) / 255.0, 0).tolist() + predict_request = json.dumps({'instances': jpeg_rgb}) + + # Send few requests to warm-up the model. + for _ in range(3): + response = requests.post(SERVER_URL, data=predict_request) + response.raise_for_status() + + # Send few actual requests and report average latency. + total_time = 0 + num_requests = 10 + for _ in range(num_requests): + response = requests.post(SERVER_URL, data=predict_request) + response.raise_for_status() + total_time += response.elapsed.total_seconds() + prediction = response.json()['predictions'][0] + + print('Prediction class: {}, avg latency: {} ms'.format( + np.argmax(prediction), (total_time * 1000) / num_requests)) + + +if __name__ == '__main__': + main() diff --git a/tensorflow_serving/example/resnet_client_grpc.py b/tensorflow_serving/example/resnet_client_grpc.py new file mode 100644 index 00000000000..96e22ee8d2c --- /dev/null +++ b/tensorflow_serving/example/resnet_client_grpc.py @@ -0,0 +1,78 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Send JPEG image to tensorflow_model_server loaded with ResNet model. + +""" + +from __future__ import print_function + +import io + +import grpc +import numpy as np +from PIL import Image +import requests +import tensorflow as tf + +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_service_pb2_grpc + +# The image URL is the location of the image we should send to the server +IMAGE_URL = 'https://tensorflow.org/images/blogs/serving/cat.jpg' + +tf.compat.v1.app.flags.DEFINE_string('server', 'localhost:8500', + 'PredictionService host:port') +tf.compat.v1.app.flags.DEFINE_string('image', '', + 'path to image in JPEG format') +FLAGS = tf.compat.v1.app.flags.FLAGS + +# Current Resnet model in TF Model Garden (as of 7/2021) does not accept JPEG +# as input +MODEL_ACCEPT_JPG = False + + +def main(_): + if FLAGS.image: + with open(FLAGS.image, 'rb') as f: + data = f.read() + else: + # Download the image since we weren't given one + dl_request = requests.get(IMAGE_URL, stream=True) + dl_request.raise_for_status() + data = dl_request.content + + if not MODEL_ACCEPT_JPG: + data = Image.open(io.BytesIO(dl_request.content)) + # Normalize and batchify the image + data = np.array(data) / 255.0 + data = np.expand_dims(data, 0) + data = data.astype(np.float32) + + channel = grpc.insecure_channel(FLAGS.server) + stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + # Send request + # See prediction_service.proto for gRPC request/response details. + request = predict_pb2.PredictRequest() + request.model_spec.name = 'resnet' + request.model_spec.signature_name = 'serving_default' + request.inputs['input_1'].CopyFrom( + tf.make_tensor_proto(data)) + result = stub.Predict(request, 10.0) # 10 secs timeout + result = result.outputs['activation_49'].float_val + print('Prediction class: {}'.format(np.argmax(result))) + + +if __name__ == '__main__': + tf.compat.v1.app.run() diff --git a/tensorflow_serving/example/resnet_k8s.yaml b/tensorflow_serving/example/resnet_k8s.yaml new file mode 100644 index 00000000000..d3359a498f0 --- /dev/null +++ b/tensorflow_serving/example/resnet_k8s.yaml @@ -0,0 +1,48 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +apiVersion: apps/v1 +kind: Deployment +metadata: + name: resnet-deployment +spec: + selector: + matchLabels: + app: resnet-server + replicas: 3 + template: + metadata: + labels: + app: resnet-server + spec: + containers: + - name: resnet-container + image: gcr.io/tensorflow-serving/resnet + ports: + - containerPort: 8500 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: resnet-service + name: resnet-service +spec: + ports: + - port: 8500 + targetPort: 8500 + selector: + app: resnet-server + type: LoadBalancer diff --git a/tensorflow_serving/example/resnet_warmup.py b/tensorflow_serving/example/resnet_warmup.py new file mode 100644 index 00000000000..c7a6af151fb --- /dev/null +++ b/tensorflow_serving/example/resnet_warmup.py @@ -0,0 +1,107 @@ +# Copyright 2019 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Creates the tf_serving_warmup_requests file to warm up a ResNet SavedModel. + + 1. Invoke this script passing in the saved_model directory (including version + folder, the folder containing saved_model.pb) as an argument. + 2. Restart tensorflow_model_server. + + If unsure of the model directory, look for the output: + 'No warmup data file found at' in the tensorflow_model_server + startup log + + After the script is run, and tensorflow_model_server is restarted, to verify + it is working look for the output: + 'Starting to read warmup data for model at' and 'Finished reading warmup data + for model at' in the tensorflow_model_server startup log + + Usage example: + python resnet_warmup.py saved_model_dir +""" + +from __future__ import print_function + +import io +import os +import sys + +import numpy as np +from PIL import Image +import requests +import tensorflow as tf +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_log_pb2 + + +# IMAGE_URLS are the locations of the images we use to warmup the model +IMAGE_URLS = ['https://tensorflow.org/images/blogs/serving/cat.jpg', + # pylint: disable=g-line-too-long + 'https://storage.googleapis.com/download.tensorflow.org/example_images/320px-Felis_catus-cat_on_snow.jpg', + # pylint: enable=g-line-too-long + ] + +# Current Resnet model in TF Model Garden (as of 7/2021) does not accept JPEG +# as input +MODEL_ACCEPT_JPG = False + + +def main(): + if len(sys.argv) != 2 or sys.argv[-1].startswith('-'): + print('Usage: resnet_warmup.py saved_model_dir') + sys.exit(-1) + + model_dir = sys.argv[-1] + if not os.path.isdir(model_dir): + print('The saved model directory: %s does not exist. ' + 'Specify the path of an existing model.' % model_dir) + sys.exit(-1) + + # Create the assets.extra directory, assuming model_dir is the versioned + # directory containing the SavedModel + assets_dir = os.path.join(model_dir, 'assets.extra') + if not os.path.exists(assets_dir): + os.mkdir(assets_dir) + + warmup_file = os.path.join(assets_dir, 'tf_serving_warmup_requests') + with tf.io.TFRecordWriter(warmup_file) as writer: + for image in IMAGE_URLS: + # Download the image + dl_request = requests.get(image, stream=True) + dl_request.raise_for_status() + data = dl_request.content + + if not MODEL_ACCEPT_JPG: + data = Image.open(io.BytesIO(dl_request.content)) + # Normalize and batchify the image + data = np.array(data) / 255.0 + data = np.expand_dims(data, 0) + data = data.astype(np.float32) + + # Create the inference request + request = predict_pb2.PredictRequest() + request.model_spec.name = 'resnet' + request.model_spec.signature_name = 'serving_default' + request.inputs['input_1'].CopyFrom( + tf.make_tensor_proto(data)) + + log = prediction_log_pb2.PredictionLog( + predict_log=prediction_log_pb2.PredictLog(request=request)) + writer.write(log.SerializeToString()) + + print('Created the file \'%s\', restart tensorflow_model_server to warmup ' + 'the ResNet SavedModel.' % warmup_file) + +if __name__ == '__main__': + main() diff --git a/tensorflow_serving/experimental/example/BUILD b/tensorflow_serving/experimental/example/BUILD new file mode 100644 index 00000000000..770a269be48 --- /dev/null +++ b/tensorflow_serving/experimental/example/BUILD @@ -0,0 +1,58 @@ +# Description: Tensorflow Serving examples. + +# Placeholder: load py_binary + +package( + default_visibility = ["//tensorflow_serving:internal"], + features = ["no_layering_check"], +) + +licenses(["notice"]) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), +) + +py_binary( + name = "remote_predict_client", + srcs = ["remote_predict_client.py"], + python_version = "PY3", + tags = [ + "manual", + ], + deps = ["//tensorflow_serving/experimental/tensorflow/ops/remote_predict:remote_predict_py"], +) + +py_binary( + name = "half_plus_two_with_rpop", + srcs = [ + "half_plus_two_with_rpop.py", + ], + python_version = "PY3", + srcs_version = "PY3", + tags = [ + "manual", + ], + deps = ["//tensorflow_serving/experimental/tensorflow/ops/remote_predict:remote_predict_py"], +) + +py_binary( + name = "half_plus_two_with_rpop_client", + srcs = ["half_plus_two_with_rpop_client.py"], + python_version = "PY3", + tags = [ + "manual", + ], + deps = [ + "//tensorflow_serving/apis:predict_proto_py_pb2", + "//tensorflow_serving/apis:prediction_service_proto_py_pb2", + "//tensorflow_serving/experimental/tensorflow/ops/remote_predict:remote_predict_py", + ], +) diff --git a/tensorflow_serving/experimental/example/half_plus_two_with_rpop.py b/tensorflow_serving/experimental/example/half_plus_two_with_rpop.py new file mode 100644 index 00000000000..07fe685d139 --- /dev/null +++ b/tensorflow_serving/experimental/example/half_plus_two_with_rpop.py @@ -0,0 +1,139 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Exports an example inference graph with RemotePredictOp. + +Exports a TensorFlow graph to `/tmp/half_plus_two_with_rpop` based on the +`SavedModel` format. + +This graph calculates, + +\\( + y = a*x + b +\\) + +where `a` is variable with `a=0.5`, and `b` is the first output tensors from +RemotePredictOp. +""" + +import tensorflow.compat.v1 as tf + +from tensorflow_serving.experimental.tensorflow.ops.remote_predict.python.ops import remote_predict_ops + +tf.app.flags.DEFINE_string("output_dir", "/tmp/half_plus_two_with_rpop/1/", + "Savedmodel export path") +tf.app.flags.DEFINE_string("target_address", "", + "Target address for the RemotePredictOp.") +tf.app.flags.DEFINE_string("remote_model_name", "half_plus_two", + "Model name for the RemotePredictOp.") +tf.app.flags.DEFINE_integer("remote_model_version", -1, + "Model version for the RemotePredictOp.") + +FLAGS = tf.app.flags.FLAGS + + +def _generate_saved_model_for_half_plus_two(export_dir, target_address, + remote_model_name): + """Generates SavedModel for half plus two with RemotePredictOp. + + Args: + export_dir: The directory to which the SavedModel should be written. + target_address: The target_address for RemotePredictOp. + remote_model_name: The model name for RemotePredictOp. + """ + builder = tf.saved_model.builder.SavedModelBuilder(export_dir) + + device_name = "/cpu:0" + + with tf.Session( + graph=tf.Graph(), + config=tf.ConfigProto(log_device_placement=True)) as sess: + with tf.device(device_name): + a = tf.Variable(0.5, name="a") + + model_name = remote_model_name + input_tensor_aliases = tf.constant(["x"]) + input_tensors = [tf.constant(2.0, tf.float32)] + + output_tensor_aliases = tf.constant(["y"]) + output_types = [tf.float32] + + results = remote_predict_ops.run_returning_status( + input_tensor_aliases, + input_tensors, + output_tensor_aliases, + target_address=target_address, + model_name=model_name, + model_version=FLAGS.remote_model_version, + output_types=output_types) + b = results.output_tensors[0] + + # Create a placeholder for serialized tensorflow.Example messages to be + # fed. + serialized_tf_example = tf.placeholder(tf.string, name="tf_example") + + # Parse the tensorflow.Example looking for a feature named "x" with a + # single floating point value. + feature_configs = {"x": tf.FixedLenFeature([1], dtype=tf.float32)} + # parse_example only works on CPU + with tf.device("/cpu:0"): + tf_example = tf.parse_example(serialized_tf_example, feature_configs) + # Use tf.identity() to assign name + x = tf.identity(tf_example["x"], name="x") + y = tf.add(tf.multiply(a, x), b, name="y") + + # Set up the signature for Predict with input and output tensor + # specification. + predict_input_tensor = tf.saved_model.utils.build_tensor_info(x) + predict_signature_inputs = {"x": predict_input_tensor} + + predict_output_tensor = tf.saved_model.utils.build_tensor_info(y) + predict_signature_outputs = {"y": predict_output_tensor} + predict_signature_def = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs=predict_signature_inputs, + outputs=predict_signature_outputs, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)) + + signature_def_map = { + tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + predict_signature_def + } + # Initialize all variables and then save the SavedModel. + sess.run(tf.global_variables_initializer()) + + builder.add_meta_graph_and_variables( + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map=signature_def_map, + assets_collection=tf.get_collection(tf.GraphKeys.ASSET_FILEPATHS), + main_op=tf.compat.v1.tables_initializer(), + strip_default_attrs=True) + builder.save(False) + + +def main(_): + _generate_saved_model_for_half_plus_two(FLAGS.output_dir, + FLAGS.target_address, + FLAGS.remote_model_name) + print( + "SavedModel generated at: %(dir)s with target_address: %(target_address)s" + ", remote_model_name: %(remote_model_name)s. " % { + "dir": FLAGS.output_dir, + "target_address": FLAGS.target_address, + "remote_model_name": FLAGS.remote_model_name + }) + + +if __name__ == "__main__": + tf.app.run() diff --git a/tensorflow_serving/experimental/example/half_plus_two_with_rpop_client.py b/tensorflow_serving/experimental/example/half_plus_two_with_rpop_client.py new file mode 100644 index 00000000000..261c8a029dc --- /dev/null +++ b/tensorflow_serving/experimental/example/half_plus_two_with_rpop_client.py @@ -0,0 +1,65 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Remote Predict Op client example. + +This client sample code will send a request to query model +/tmp/half_plus_two_with_rpop which contains Remote Predict Op. The Remote +Predict Op will send a request to query /tmp/half_plus_two. For more details +about /tmp/half_plus_two_with_rpop, please refer to half_plus_two_with_rpop.py. + +To run this client example locally, please create a server config file like: + model_config_list { + config: { + name: "half_plus_two" + base_path: "/tmp/half_plus_two" + model_platform: "tensorflow" + } + config: { + name: "half_plus_two_with_rpop" + base_path: "/tmp/half_plus_two_with_rpop" + model_platform: "tensorflow" + } + } +And then run +tensorflow_model_server --port=8500 --model_config_file=/tmp/config_file.txt + +""" +from __future__ import print_function + +import grpc +import tensorflow.compat.v1 as tf + +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_service_pb2_grpc + +tf.app.flags.DEFINE_string('server', 'localhost:8500', + 'PredictionService host:port') +FLAGS = tf.app.flags.FLAGS + + +def main(_): + channel = grpc.insecure_channel(FLAGS.server) + stub = prediction_service_pb2_grpc.PredictionServiceStub(channel) + # Send request + request = predict_pb2.PredictRequest() + request.model_spec.name = 'half_plus_two_with_rpop' + request.model_spec.signature_name = 'serving_default' + request.inputs['x'].CopyFrom(tf.make_tensor_proto([10.0], shape=[1])) + result = stub.Predict(request, 30) + print(result) + + +if __name__ == '__main__': + tf.compat.v1.app.run() diff --git a/tensorflow_serving/experimental/example/remote_predict_client.py b/tensorflow_serving/experimental/example/remote_predict_client.py new file mode 100644 index 00000000000..ccf98a12c2c --- /dev/null +++ b/tensorflow_serving/experimental/example/remote_predict_client.py @@ -0,0 +1,60 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +r"""Remote Predict Op client example. + +Example client code which calls the Remote Predict Op directly. +""" + +from __future__ import print_function + +import tensorflow.compat.v1 as tf + +from tensorflow_serving.experimental.tensorflow.ops.remote_predict.python.ops import remote_predict_ops + +tf.app.flags.DEFINE_string("input_tensor_aliases", "x", + "Aliases of input tensors") +tf.app.flags.DEFINE_float("input_value", 1.0, "input value") +tf.app.flags.DEFINE_string("output_tensor_aliases", "y", + "Aliases of output tensors") + +tf.app.flags.DEFINE_string("target_address", "localhost:8500", + "PredictionService address host:port") +tf.app.flags.DEFINE_string("model_name", "half_plus_two", "Name of the model") +tf.app.flags.DEFINE_integer("model_version", -1, "Version of the model") +tf.app.flags.DEFINE_boolean("fail_op_on_rpc_error", True, "Failure handling") +tf.app.flags.DEFINE_integer("rpc_deadline_millis", 30000, + "rpc deadline in milliseconds") + +FLAGS = tf.app.flags.FLAGS + + +def main(unused_argv): + print("Call remote_predict_op") + results = remote_predict_ops.run( + [FLAGS.input_tensor_aliases], + [tf.constant(FLAGS.input_value, dtype=tf.float32)], + [FLAGS.output_tensor_aliases], + target_address=FLAGS.target_address, + model_name=FLAGS.model_name, + model_version=FLAGS.model_version, + fail_op_on_rpc_error=FLAGS.fail_op_on_rpc_error, + max_rpc_deadline_millis=FLAGS.rpc_deadline_millis, + output_types=[tf.float32]) + print("Done remote_predict_op") + print("Returned Result:", results.output_tensors[0].numpy()) + + +if __name__ == "__main__": + tf.app.run() diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/BUILD b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/BUILD new file mode 100644 index 00000000000..24c2027f729 --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/BUILD @@ -0,0 +1,195 @@ +# Description: +# Remote Predict Op Prototype + +# buildifier: disable=same-origin-load +# Placeholder: load py_test +load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_custom_op_library", "tf_gen_op_libs", "tf_gen_op_wrapper_cc", "tf_gen_op_wrapper_py", "tf_kernel_library") +load("@org_tensorflow//tensorflow:tensorflow.bzl", "tf_custom_op_py_library") +# buildifier: enable=same-origin-load + +licenses(["notice"]) + +cc_library( + name = "remote_predict_op_kernel_lib", + srcs = [ + "kernels/prediction_service_grpc.cc", + "kernels/remote_predict_op_kernel.cc", + ], + hdrs = [ + "kernels/prediction_service_grpc.h", + "kernels/remote_predict_op_kernel.h", + ], + deps = [ + "//tensorflow_serving/apis:model_cc_proto", + "//tensorflow_serving/apis:predict_cc_proto", + "//tensorflow_serving/apis:prediction_service_cc_proto", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/status", + "@com_google_absl//absl/time", + "@com_google_protobuf//:protobuf_lite", + "@org_tensorflow//tensorflow/core:framework_headers_lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core/kernels:ops_util_hdrs", + "@org_tensorflow//tensorflow/core/platform:statusor", + ], + alwayslink = 1, +) + +tf_gen_op_libs( + op_lib_names = ["remote_predict_op"], + deps = [ + "@org_tensorflow//tensorflow/core:lib", + ], +) + +tf_gen_op_wrapper_cc( + name = "gen_cc_remote_predict_op", + out_ops_file = "cc/ops/remote_predict_op", + deps = [":remote_predict_op_op_lib"], +) + +cc_library( + name = "remote_predict_ops_cc", + srcs = ["cc/ops/remote_predict_op.cc"], + hdrs = ["cc/ops/remote_predict_op.h"], + deps = [ + ":remote_predict_op_op_lib", + "@org_tensorflow//tensorflow/cc:const_op", + "@org_tensorflow//tensorflow/cc:ops", + "@org_tensorflow//tensorflow/cc:scope", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +tf_gen_op_wrapper_py( + name = "gen_remote_predict_op", + out = "ops/gen_remote_predict_op.py", + deps = [":remote_predict_op_op_lib"], +) + +tf_kernel_library( + name = "remote_predict_op_kernel", + srcs = [ + "kernels/prediction_service_grpc.cc", + "kernels/remote_predict_op_kernel.cc", + ], + hdrs = [ + "kernels/prediction_service_grpc.h", + "kernels/remote_predict_op_kernel.h", + ], + deps = [ + ":remote_predict_op_op_lib", + "//tensorflow_serving/apis:model_cc_proto", + "//tensorflow_serving/apis:predict_cc_proto", + "//tensorflow_serving/apis:prediction_service_cc_proto", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/status", + "@com_google_absl//absl/time", + "@com_google_protobuf//:cc_wkt_protos", + "@com_google_protobuf//:protobuf_lite", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core/kernels:ops_util", + "@org_tensorflow//tensorflow/core/kernels:split_lib", + "@org_tensorflow//tensorflow/core/platform:statusor", + ], +) + +tf_custom_op_library( + name = "python/ops/_remote_predict_op.so", + srcs = [ + "ops/remote_predict_op.cc", + ], + deps = [":remote_predict_op_kernel_lib"], +) + +tf_custom_op_py_library( + name = "remote_predict_py", + srcs = glob(["python/ops/*.py"]) + ["__init__.py"], + dso = [":python/ops/_remote_predict_op.so"], + kernels = [ + ":remote_predict_op_kernel", + ":remote_predict_op_op_lib", + ], + srcs_version = "PY2AND3", + tags = [ + "manual", + ], + visibility = ["//tensorflow_serving/experimental/example:__subpackages__"], + deps = [ + ":gen_remote_predict_op", + "@org_tensorflow//tensorflow/python/framework:for_generated_wrappers", + ], +) + +cc_test( + name = "remote_predict_ops_cc_test", + size = "small", + srcs = [ + "kernels/remote_predict_op_kernel.h", + "kernels/remote_predict_op_kernel_test.cc", + ], + deps = [ + ":remote_predict_ops_cc", + "//tensorflow_serving/apis:model_cc_proto", + "//tensorflow_serving/apis:predict_cc_proto", + "//tensorflow_serving/apis:prediction_service_cc_proto", + "//tensorflow_serving/core/test_util:test_main", + "@com_google_absl//absl/status", + "@com_google_absl//absl/time", + "@com_google_protobuf//:cc_wkt_protos", + "@com_google_protobuf//:protobuf_lite", + "@org_tensorflow//tensorflow/cc:client_session", + "@org_tensorflow//tensorflow/cc:const_op", + "@org_tensorflow//tensorflow/core:all_kernels", + "@org_tensorflow//tensorflow/core:direct_session", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:test", + "@org_tensorflow//tensorflow/core:testlib", + "@org_tensorflow//tensorflow/core/kernels:ops_util", + ], +) + +cc_library( + name = "prediction_service_grpc", + srcs = ["kernels/prediction_service_grpc.cc"], + hdrs = ["kernels/prediction_service_grpc.h"], + deps = [ + "//tensorflow_serving/apis:prediction_service_cc_proto", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/status", + "@com_google_absl//absl/time", + "@org_tensorflow//tensorflow/core/platform:statusor", + ], +) + +cc_test( + name = "prediction_service_grpc_test", + size = "small", + srcs = [ + "kernels/prediction_service_grpc_test.cc", + ], + deps = [ + ":prediction_service_grpc", + "//tensorflow_serving/core/test_util:test_main", + "@com_github_grpc_grpc//:grpc++", + "@com_google_absl//absl/time", + "@org_tensorflow//tensorflow/core:testlib", + ], +) + +filegroup( + name = "all_files", + srcs = glob( + ["**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), + visibility = ["//visibility:private"], +) diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/README.md b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/README.md new file mode 100644 index 00000000000..107489cd8fc --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/README.md @@ -0,0 +1,186 @@ +# Remote Predict TensorFlow Operator + +_Warning: This is an experimental feature, not yet supported by TensorFlow +Serving team and may change or be removed at any point. While we would love to +hear about your interest and discuss your use cases for this op via +[Github Issues](https://github.com/tensorflow/serving/issues), we cannot offer +debugging help and support for this feature at the moment as many pieces needed +to productionize its usage are not currently available. We look forward to +hearing your preliminary thoughts and feedback as we continue to assess the need +for this feature and iterate on its design._ + +## Motivation and Use Case + +The Remote Predict Op (RPOp) is an experimental TensorFlow operator that enables +users to make a +[Predict RPC](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto#L23) +from within a TensorFlow graph executing on machine A to another graph hosted by +TensorFlow Serving on machine B. This capability unlocks the following two +use-cases: + +1) *Serving large models*: When model size grows beyond what can be loaded into +the host machine's RAM, we can split the model into subgraphs, hosted on +multiple machines, and use this op to distribute the inference over the +subgraphs on the various machines. + +2) *Re-using model sub-graphs*: In cases where we have N models that share the +same base (as in Transfer Learning use cases) or otherwise have significant +common subgraphs or shared embeddings, instead of embedding the shared component +in each graph, we can host them centrally, and have the different consumer +graphs use the RPOp to remote call into them. + +For this guide, we will do a deeper walk-through of the first use case. + +### Serving Large Models + +Many machine learning models have to make inferences over items in large +corpuses of potentially billions of items. The elements of these corpuses are +often represented using large embedding matrices, which in the most extreme +cases are too large to fit in the RAM of the host machine. + +Using RPOp, we can shard the embedding matrix and split it over multiple +machines, all running TensorFlow Serving to serve their embedding shard. We +could then have a master graph that processes the input request, farms out +requests to the various leaf shards, post-processes their responses and returns +the final response to the user. + +## Configuration + +This op uses a combination of attributes (set at model export time) and inputs +(set at inference time) to configure remote inference. + +Some notable configurable attributes include: + +* `target_address`: Address of the server hosting the remote graph. +* `model_name`: The name of the remote TF graph. +* `model_version`: the target model version for the Predict call. When unset, + the default value (-1) implies the latest available version should be used. +* `max_rpc_deadline_millis`: The rpc deadline for remote predict. Of course if + the incoming RPC times out before this deadline is reached, the client + should timeout the incoming RPC to this server. +* `output_types`: A list, equal in length to output_tensors, of types of the + output tensors. + +The inputs that the op expects at inference time are: + +* `input_tensor_aliases`: Tensor of strings for the input tensor alias names + to supply to the RemotePredict call. +* `input_tensors`: List of tensors to provide as input, which should be equal + in length to 'input_tensor_aliases'. +* `output_tensor_aliases`: Tensor of strings for the output tensor alias names + to supply to the Predict call. + +And the outputs that the op will provide are: + +* `status_code`: Returns the status code of the RPC, converting + tensorflow::error::Code to its int value; 0 means OK. +* `status_error_message`: Returns the error message contained in the RPC + status. +* `output_tensors`: Tensors returned by the Predict call on the remote graph. + +It's worth noting that the RPOp kernel implementation is templated on a +PredictionServiceStubType, which allows users to easily extend it to support RPC +frameworks other than gRPC, for which this op comes with out of the box support. + +## Usage + +To demonstrate usage of this feature, we'll serve a modified version of the +[half plus two model](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two.py) +where instead of directly computing `y = a * x + b; a = 0.5, b = 2`, we rely on +a remote graph (a vanilla half plus two model) to supply us the value for `b` in +the first tensor in its output. The code for this example model can be found +[here](../../../example/half_plus_two_with_rpop.py). + +To ease the set up and not get into networking-environment-specific concerns, +we'll demonstrate deploying both of these on the same TF Serving instance, +utilizing the host's loopback interface. + +### Export Models and Model Config + +Note: The following assumes your current working directory is the root of the +repository. + +First, use [this script](../../../example/half_plus_two_with_rpop.py) to export +the model containing the RPOp. By default, it will be exported to +`/tmp/half_plus_two_with_rpop`. Read through the script to see how to configure +the parameters of the RPOp: + +`tools/run_in_docker.sh bazel run +tensorflow_serving/experimental/example:half_plus_two_with_rpop -- +--target_address=localhost:8500` + +Then, copy over the vanilla half plus two +[model](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_cpu). + +`cp -r +tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_cpu/ +/tmp/half_plus_two` + +Finally, create a model config file `/tmp/config_file.txt` containing the +following: + +``` +model_config_list { + config: { + name: "half_plus_two" + base_path: "/tmp/half_plus_two" + model_platform: "tensorflow" + } + config: { + name: "half_plus_two_with_rpop" + base_path: "/tmp/half_plus_two_with_rpop" + model_platform: "tensorflow" + } +} +``` + +### Build and Run TF Serving + +You will then need to build TF Serving with the RPOp op and kernel linked in. + +Add the following two lines to the list of deps of the `server_lib` C++ library +build rule in model server's +[BUILD file](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/BUILD) +to link the op and kernel into the model server binary. + +``` +cc_library( + name = "server_lib", + ... + deps = [ ... + "//tensorflow_serving/experimental/tensorflow/ops/remote_predict:remote_predict_op_kernel", + "//tensorflow_serving/experimental/tensorflow/ops/remote_predict:remote_predict_ops_cc", + ] + SUPPORTED_TENSORFLOW_OPS, +) +``` + +Then use this command to build the TF Serving binary, with the op linked in, +using Docker: + +`tools/run_in_docker.sh bazel build +tensorflow_serving/model_servers:tensorflow_model_server` + +### Deploy Models + +Now, you can start TF Serving: + +`bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=8500 +--rest_api_port=8501 --model_config_file=/tmp/config_file.txt` + +### Make Inference Calls + +You may now send inference requests to this model: + +`curl -d '{"instances": [1.0, 2.0, 5.0]}' -X POST +http://localhost:8501/v1/models/half_plus_two_with_rpop:predict` + +## Caveat & Feedback + +As mentioned in the warning, this op is only a part of a larger set of +infrastructure needed to enable distributed serving in production settings, +including support for determining reasonable graph partition points, executing +the graph splits, injection of the RPOp(s), coordinating new model version +rollouts, load balancing, autoscaling and more. We look forward to hearing your +feedback and thoughts on whether this op is a good fit for your production use +cases to help us better understand the value in releasing the additional tooling +and infrastructure needed to productionize such a set up. diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/WORKSPACE b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/WORKSPACE new file mode 100644 index 00000000000..e69de29bb2d diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/__init__.py b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/__init__.py new file mode 100644 index 00000000000..c1597decea2 --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/__init__.py @@ -0,0 +1,25 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Ops and modules related to RemotePredict. + +@@run +""" +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function +from tensorflow.python.util.all_util import remove_undocumented +from tensorflow_serving.experimental.tensorflow.ops.remote_predict.python.ops.remote_predict_ops import run + +remove_undocumented(__name__) diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.cc b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.cc new file mode 100644 index 00000000000..98234fc99f8 --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.cc @@ -0,0 +1,69 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.h" + +#include +#include + +#include "grpcpp/create_channel.h" +#include "grpcpp/security/credentials.h" +#include "absl/time/clock.h" + +using namespace tensorflow; // NOLINT(build/namespaces) +namespace tensorflow { +namespace serving { +namespace { + +absl::Status FromGrpcStatus(const ::grpc::Status& s) { + if (s.ok()) { + return absl::Status(); + } + return absl::Status(static_cast(s.error_code()), + s.error_message()); +} + +} // namespace + +PredictionServiceGrpc::PredictionServiceGrpc( + const std::string& target_address) { + // TODO(b/159739577): Set security channel from incoming rpc request. + auto channel = ::grpc::CreateChannel(target_address, + ::grpc::InsecureChannelCredentials()); + stub_ = tensorflow::serving::PredictionService::NewStub(channel); +} + +absl::StatusOr< ::grpc::ClientContext*> PredictionServiceGrpc::CreateRpc( + absl::Duration max_rpc_deadline) { + ::grpc::ClientContext* rpc = new ::grpc::ClientContext(); + // TODO(b/300069508): Set deadline as the min value between + // the incoming rpc deadline and max_rpc_deadline_millis. + rpc->set_deadline(std::chrono::system_clock::now() + + absl::ToChronoSeconds(max_rpc_deadline)); + return rpc; +} + +void PredictionServiceGrpc::Predict( + ::grpc::ClientContext* rpc, PredictRequest* request, + PredictResponse* response, + std::function callback) { + std::function wrapped_callback = + [callback](::grpc::Status status) { callback(FromGrpcStatus(status)); }; + + stub_->experimental_async()->Predict(rpc, request, response, + wrapped_callback); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.h b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.h new file mode 100644 index 00000000000..2d31d658d00 --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.h @@ -0,0 +1,51 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef THIRD_PARTY_TENSORFLOW_SERVING_EXPERIMENTAL_TENSORFLOW_OPS_REMOTE_PREDICT_KERNELS_PREDICTION_SERVICE_GRPC_H_ +#define THIRD_PARTY_TENSORFLOW_SERVING_EXPERIMENTAL_TENSORFLOW_OPS_REMOTE_PREDICT_KERNELS_PREDICTION_SERVICE_GRPC_H_ +#include + +#include "absl/status/status.h" +#include "absl/time/time.h" +#include "tensorflow/core/platform/statusor.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" + +namespace tensorflow { +namespace serving { + +// gRPC based communication point with PredictionService. +class PredictionServiceGrpc { + public: + // Creates a new instance. Returns an error if the creation fails. + static absl::Status Create(const std::string& target_address, + std::unique_ptr* service) { + service->reset(new PredictionServiceGrpc(target_address)); + return ::absl::OkStatus(); + } + + StatusOr<::grpc::ClientContext*> CreateRpc(absl::Duration max_rpc_deadline); + + void Predict(::grpc::ClientContext* rpc, PredictRequest* request, + PredictResponse* response, + std::function callback); + + private: + PredictionServiceGrpc(const std::string& target_address); + std::unique_ptr stub_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // THIRD_PARTY_TENSORFLOW_SERVING_EXPERIMENTAL_TENSORFLOW_OPS_REMOTE_PREDICT_KERNELS_PREDICTION_SERVICE_GRPC_H_ diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc_test.cc b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc_test.cc new file mode 100644 index 00000000000..6c15afedcc9 --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc_test.cc @@ -0,0 +1,50 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.h" + +#include + +#include "absl/time/clock.h" +#include "tensorflow/core/framework/tensor_testutil.h" + +namespace tensorflow { +namespace serving { +namespace { + +class PredictionServiceGrpcTest : public ::testing::Test { + protected: + virtual void SetUp() { + auto prediction_service_status = + PredictionServiceGrpc::Create("target_address", &grpc_stub_); + } + std::unique_ptr grpc_stub_; + std::unique_ptr<::grpc::ClientContext> rpc_; +}; + +TEST_F(PredictionServiceGrpcTest, TestSetDeadline) { + const absl::Duration deadline = absl::Milliseconds(30000); + auto rpc_or = grpc_stub_->CreateRpc(deadline); + ASSERT_TRUE(rpc_or.ok()); + rpc_.reset(rpc_or.value()); + + EXPECT_NEAR(absl::ToDoubleMilliseconds(deadline), + absl::ToDoubleMilliseconds(absl::FromChrono(rpc_->deadline()) - + absl::Now()), + 10); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.cc b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.cc new file mode 100644 index 00000000000..1b97d99da3f --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.cc @@ -0,0 +1,28 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.h" + +#include "tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/prediction_service_grpc.h" + +namespace tensorflow { +namespace serving { +namespace { + +REGISTER_KERNEL_BUILDER(Name("TfServingRemotePredict").Device(DEVICE_CPU), + RemotePredictOp); + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.h b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.h new file mode 100644 index 00000000000..beb4b5e9bf2 --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.h @@ -0,0 +1,206 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#ifndef TENSORFLOW_SERVING_EXPERIMENTAL_TENSORFLOW_OPS_REMOTE_PREDICT_KERNELS_REMOTE_PREDICT_OP_KERNEL_H_ +#define TENSORFLOW_SERVING_EXPERIMENTAL_TENSORFLOW_OPS_REMOTE_PREDICT_KERNELS_REMOTE_PREDICT_OP_KERNEL_H_ + +#include "google/protobuf/wrappers.pb.h" +#include "google/protobuf/map.h" +#include "absl/status/status.h" +#include "absl/time/time.h" +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op_kernel.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/resource_mgr.h" +#include "tensorflow/core/framework/shape_inference.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/kernels/ops_util.h" +#include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/lib/gtl/cleanup.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow/core/protobuf/named_tensor.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" + +namespace tensorflow { +namespace serving { + +typedef google::protobuf::Map AliasTensorMap; + +// Remote Predict Op kernel implementation class templated on different +// PredictionServiceStubTypes. +template +class RemotePredictOp : public AsyncOpKernel { + public: + explicit RemotePredictOp(OpKernelConstruction* context) + : AsyncOpKernel(context) { + string target_address; + OP_REQUIRES_OK(context, + context->GetAttr("target_address", &target_address)); + OP_REQUIRES_OK(context, context->GetAttr("model_name", &model_name_)); + OP_REQUIRES_OK(context, context->GetAttr("model_version", &model_version_)); + OP_REQUIRES_OK(context, context->GetAttr("max_rpc_deadline_millis", + &max_rpc_deadline_millis_)); + OP_REQUIRES_OK(context, context->GetAttr("fail_op_on_rpc_error", + &fail_op_on_rpc_error_)); + OP_REQUIRES_OK(context, + context->GetAttr("signature_name", &signature_name_)); + absl::Status prediction_service_status = + PredictionServiceStubType::Create(target_address, &prediction_service_); + OP_REQUIRES(context, prediction_service_status.ok(), + tensorflow::Status(static_cast( + prediction_service_status.code()), + prediction_service_status.message())); + } + + void ComputeAsync(OpKernelContext* context, DoneCallback done) override { + // Get the input tensor alias names. + const auto& input_tensor_aliases = context->input(0).flat(); + + // Get the input tensors. + OpInputList input_tensors; + OP_REQUIRES_OK_ASYNC( + context, context->input_list("input_tensors", &input_tensors), done); + // Get the output tensor alias names. + // Directly index to output_tensor_aliases by moving past all the input + // before it, including the input_tensor_aliases and input_tensors. + auto output_tensor_aliases = + context->input(1 + input_tensors.size()).flat(); + + // Build the PredictRequest. + PredictRequest* request = new PredictRequest(); + + request->mutable_model_spec()->set_name(model_name_); + + request->mutable_model_spec()->set_signature_name(signature_name_); + + if (model_version_ >= 0) { + request->mutable_model_spec()->mutable_version()->set_value( + model_version_); + } + + AliasTensorMap& inputs = *request->mutable_inputs(); + for (int i = 0; i < input_tensor_aliases.size(); ++i) { + tensorflow::TensorProto proto; + input_tensors[i].AsProtoField(&proto); + inputs[input_tensor_aliases(i)] = proto; + } + + for (int i = 0; i < output_tensor_aliases.size(); ++i) { + request->add_output_filter(tensorflow::string(output_tensor_aliases(i))); + } + + PredictResponse* response = new PredictResponse(); + + auto rpc_or = prediction_service_->CreateRpc( + absl::Milliseconds(max_rpc_deadline_millis_)); + OP_REQUIRES_ASYNC(context, rpc_or.ok(), + tensorflow::Status(static_cast( + rpc_or.status().code()), + rpc_or.status().message()), + [&]() { + delete request; + delete response; + done(); + }); + auto rpc = rpc_or.value(); + auto callback = [this, context, rpc, request, response, + output_tensor_aliases, done](const absl::Status& status) { + PostProcessResponse(context, response, status, fail_op_on_rpc_error_, + output_tensor_aliases, [&]() { + delete rpc; + delete request; + delete response; + done(); + }); + }; + // Make the RPC call. + prediction_service_->Predict(rpc, request, response, callback); + } + + void PostProcessResponse(OpKernelContext* context, PredictResponse* response, + const absl::Status& rpc_status, + bool fail_op_on_rpc_error, + TTypes::Flat output_tensor_aliases, + DoneCallback rpc_done) { + auto rpc_cleaner = gtl::MakeCleanup([&] { rpc_done(); }); + Tensor* status_code; + OP_REQUIRES_OK_ASYNC( + context, context->allocate_output(0, TensorShape({}), &status_code), + rpc_cleaner.release()); + status_code->scalar()() = static_cast(rpc_status.code()); + Tensor* status_error_message; + OP_REQUIRES_OK_ASYNC( + context, + context->allocate_output(1, TensorShape({}), &status_error_message), + rpc_cleaner.release()); + status_error_message->scalar()() = rpc_status.message(); + OpOutputList output_tensors_list; + OP_REQUIRES_OK_ASYNC( + context, context->output_list("output_tensors", &output_tensors_list), + rpc_cleaner.release()); + // Process the response. + if (!rpc_status.ok()) { + if (fail_op_on_rpc_error) { + OP_REQUIRES_OK_ASYNC( + context, + tensorflow::Status( + static_cast(rpc_status.code()), + rpc_status.message()), + rpc_cleaner.release()); + } else { + // Allocate some empty output for the output_tensors. + for (int i = 0; i < output_tensors_list.size(); ++i) { + Tensor* unused; + OP_REQUIRES_OK_ASYNC( + context, + output_tensors_list.allocate(i, TensorShape({}), &unused), + rpc_cleaner.release()); + } + return; + } + } + OP_REQUIRES_ASYNC( + context, output_tensors_list.size() == output_tensor_aliases.size(), + errors::Internal( + "Response doesn't have the right number of outputs; actual: ", + output_tensors_list.size(), + " expected: ", output_tensor_aliases.size()), + rpc_cleaner.release()); + AliasTensorMap& outputs = *response->mutable_outputs(); + for (int i = 0; i < output_tensor_aliases.size(); i++) { + Tensor output_tensor; + OP_REQUIRES_ASYNC( + context, output_tensor.FromProto(outputs[output_tensor_aliases(i)]), + errors::Internal("Response tensor proto: ", + tensorflow::string(output_tensor_aliases(i)), + " cannot be converted back to a tensor."), + rpc_cleaner.release()); + output_tensors_list.set(i, output_tensor); + } + } + + private: + string model_name_; + int64_t model_version_; + bool fail_op_on_rpc_error_; + int64_t max_rpc_deadline_millis_; + string signature_name_; + std::unique_ptr prediction_service_; +}; + +} // namespace serving +} // namespace tensorflow +#endif // TENSORFLOW_SERVING_EXPERIMENTAL_TENSORFLOW_OPS_REMOTE_PREDICT_KERNELS_REMOTE_PREDICT_OP_KERNEL_H_ diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel_test.cc b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel_test.cc new file mode 100644 index 00000000000..334d3af9fec --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel_test.cc @@ -0,0 +1,177 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow_serving/experimental/tensorflow/ops/remote_predict/kernels/remote_predict_op_kernel.h" + +#include +#include +#include +#include + +#include "absl/status/status.h" +#include "absl/time/time.h" +#include "tensorflow/cc/client/client_session.h" +#include "tensorflow/cc/ops/const_op.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/status.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#include "tensorflow_serving/experimental/tensorflow/ops/remote_predict/cc/ops/remote_predict_op.h" + +namespace tensorflow { +namespace serving { +namespace { + +// Empty mock rpc class. +class MockRpc {}; + +// Mock class for RemotePredict Op kernel test. +class MockPredictionService { + public: + static absl::Status Create(const string& target_address, + std::unique_ptr* service) { + service->reset(new MockPredictionService(target_address)); + return ::absl::OkStatus(); + } + + absl::StatusOr CreateRpc(absl::Duration max_rpc_deadline) { + return new MockRpc; + } + + // The model_name in request determines response and/or status. + void Predict(MockRpc* rpc, PredictRequest* request, PredictResponse* response, + std::function callback); + + static constexpr char kGoodModel[] = "good_model"; + static constexpr char kBadModel[] = "bad_model"; + + private: + MockPredictionService(const string& target_address); +}; + +constexpr char MockPredictionService::kGoodModel[]; +constexpr char MockPredictionService::kBadModel[]; + +typedef google::protobuf::Map AliasTensorMap; + +MockPredictionService::MockPredictionService(const string& target_address) {} + +void MockPredictionService::Predict( + MockRpc* rpc, PredictRequest* request, PredictResponse* response, + std::function callback) { + // Use model name to specify the behavior of each test. + std::string model_name = request->model_spec().name(); + if (model_name == kGoodModel) { + *(response->mutable_model_spec()) = request->model_spec(); + AliasTensorMap& inputs = *request->mutable_inputs(); + AliasTensorMap& outputs = *response->mutable_outputs(); + outputs["output0"] = inputs["input0"]; + outputs["output1"] = inputs["input1"]; + callback(::absl::OkStatus()); + } + + if (model_name == kBadModel) { + callback(absl::Status(absl::StatusCode::kAborted, "Aborted")); + } +} + +REGISTER_KERNEL_BUILDER(Name("TfServingRemotePredict").Device(DEVICE_CPU), + RemotePredictOp); + +using RemotePredict = ops::TfServingRemotePredict; + +// Use model_name to specify the behavior of different tests. +absl::Status RunRemotePredict( + const string& model_name, std::vector* outputs, + const DataTypeSlice& output_types = {DT_INT32, DT_INT32}, + const absl::optional<::absl::Duration> deadline = absl::nullopt, + bool fail_on_rpc_error = true, + const string& target_address = "target_address", + int64_t target_model_version = -1, const string& signature_name = "") { + const Scope scope = Scope::DisabledShapeInferenceScope(); + // Model_name will decide the result of the RPC. + auto input_tensor_aliases = ops::Const( + scope.WithOpName("input_tensor_aliases"), {"input0", "input1"}); + auto input_tensors0 = ops::Const(scope.WithOpName("input_tensors0"), {1, 2}); + auto input_tensors1 = ops::Const(scope.WithOpName("input_tensors1"), {3, 4}); + auto output_tensor_aliases = ops::Const( + scope.WithOpName("output_tensor_aliases"), {"output0", "output1"}); + std::vector fetch_outputs; + RemotePredict::Attrs attrs = RemotePredict::Attrs() + .TargetAddress(target_address) + .ModelName(model_name) + .SignatureName(signature_name); + + if (target_model_version >= 0) { + attrs = attrs.ModelVersion(target_model_version); + } + if (deadline.has_value()) { + attrs = attrs.MaxRpcDeadlineMillis(absl::ToInt64Seconds(deadline.value()) * + 1000); + } + attrs = attrs.FailOpOnRpcError(fail_on_rpc_error); + + auto remote_predict = RemotePredict( + scope, input_tensor_aliases, {input_tensors0, input_tensors1}, + output_tensor_aliases, output_types, attrs); + + fetch_outputs = {remote_predict.status_code, + remote_predict.status_error_message}; + fetch_outputs.insert(fetch_outputs.end(), + remote_predict.output_tensors.begin(), + remote_predict.output_tensors.end()); + TF_RETURN_IF_ERROR(scope.status()); + + ClientSession session(scope); + return session.Run(fetch_outputs, outputs); +} + +TEST(RemotePredictTest, TestSimple) { + std::vector outputs; + TF_ASSERT_OK(RunRemotePredict( + /*model_name=*/MockPredictionService::kGoodModel, &outputs)); + ASSERT_EQ(4, outputs.size()); + // Checks whether the status code is 0 and there is no error message. + EXPECT_EQ(0, outputs[0].scalar()()); + EXPECT_EQ("", outputs[1].scalar()()); + test::ExpectTensorEqual(outputs[2], test::AsTensor({1, 2})); + test::ExpectTensorEqual(outputs[3], test::AsTensor({3, 4})); +} + +TEST(RemotePredictTest, TestRpcError) { + std::vector outputs; + const auto status = RunRemotePredict( + /*model_name=*/MockPredictionService::kBadModel, &outputs); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(error::Code::ABORTED, status.code()); + EXPECT_THAT(status.message(), ::testing::HasSubstr("Aborted")); +} + +TEST(RemotePredictTest, TestRpcErrorReturnStatus) { + std::vector outputs; + // Specifying output_types to float solves + // "MemorySanitizer: use-of-uninitialized-value" + const auto status = RunRemotePredict( + /*model_name=*/MockPredictionService::kBadModel, &outputs, + {DT_FLOAT, DT_FLOAT}, /*deadline=*/absl::nullopt, + /*fail_on_rpc_error=*/false); + + EXPECT_TRUE(status.ok()); + EXPECT_EQ(static_cast(error::Code::ABORTED), outputs[0].scalar()()); + EXPECT_EQ("Aborted", outputs[1].scalar()()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/ops/remote_predict_op.cc b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/ops/remote_predict_op.cc new file mode 100644 index 00000000000..def975ac60d --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/ops/remote_predict_op.cc @@ -0,0 +1,111 @@ +/* Copyright 2020 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include + +#include "tensorflow/core/framework/common_shape_fns.h" +#include "tensorflow/core/framework/op.h" +#include "tensorflow/core/framework/shape_inference.h" + +namespace tensorflow { + +REGISTER_OP("TfServingRemotePredict") + .Attr("T: list(type)") + .Attr("target_address: string = ''") + .Attr("model_name: string = ''") + .Attr("model_version: int = -1") + .Attr("fail_op_on_rpc_error: bool = true") + .Attr("max_rpc_deadline_millis: int = 30000") + .Attr("signature_name: string = 'serving_default'") + .Input("input_tensor_aliases: string") + .Input("input_tensors: T") + .Input("output_tensor_aliases: string") + .Output("status_code: int32") + .Output("status_error_message: string") + .Output("output_tensors: output_types") + .Attr("output_types: list(type)") + .SetShapeFn([](shape_inference::InferenceContext* c) { + shape_inference::ShapeHandle unused; + // Checks the length of input_tensor_aliases with that of input_tensors. + std::vector input_aliases_handle; + TF_RETURN_IF_ERROR( + c->input("input_tensor_aliases", &input_aliases_handle)); + TF_RETURN_IF_ERROR(c->WithRank(input_aliases_handle[0], 1, &unused)); + std::vector inputs_handle; + TF_RETURN_IF_ERROR(c->input("input_tensors", &inputs_handle)); + if (c->Value(c->NumElements(input_aliases_handle[0])) != + inputs_handle.size()) { + return errors::InvalidArgument( + "'input_tensors' should be equal in length to " + "'input_tensor_aliases'. Length of 'input_tensors': ", + inputs_handle.size(), ", length of 'input_tensor_aliases': ", + c->Value(c->NumElements(input_aliases_handle[0]))); + } + + // Checks the length of output_tensor_aliases with that of output_types. + DataTypeVector output_types; + TF_RETURN_IF_ERROR(c->GetAttr("output_types", &output_types)); + std::vector output_aliases_handle; + TF_RETURN_IF_ERROR( + c->input("output_tensor_aliases", &output_aliases_handle)); + if (c->Value(c->NumElements(output_aliases_handle[0])) != + output_types.size()) { + return errors::InvalidArgument( + "'output_types' should be equal in length to " + "'output_tensor_aliases'. Length of 'output_types': ", + output_types.size(), ", length of 'output_tensor_aliases': ", + c->Value(c->NumElements(output_aliases_handle[0]))); + } + + // We know the shape of the first 2 outputs, but not the rest. + TF_RETURN_IF_ERROR(c->set_output("status_code", {c->Scalar()})); + TF_RETURN_IF_ERROR(c->set_output("status_error_message", {c->Scalar()})); + for (int i = 2; i < c->num_outputs(); ++i) { + c->set_output(i, c->UnknownShape()); + } + + return absl::Status(); + }) + .SetIsStateful() + .SetIsDistributedCommunication() + .Doc(R"doc( +Invokes Predict on a remote graph. +fail_op_on_rpc_error: If set true, the Op fails if the rpc fails, and returns + the status code as 0 and an empty status_message. Otherwise the + Op returns the status of the rpc call, along with the output tensors, if any. + Set true by default. +max_rpc_deadline_millis: The rpc deadline for remote predict. The actual +deadline is min(incoming_rpc_deadline, max_rpc_deadline_millis). +signature_name: the signature def for remote graph inference, defaulting to +"serving_default". +target_address: Address of the server hosting the remote graph. +model_name: Model name of the remote TF graph. +model_version: the target version for the Predict call. When unset, the + default value (-1) implies the latest available version should be used. +input_tensor_aliases: Tensor of strings for the input tensor alias names to supply + to the RemotePredict call. +input_tensors: List of tensors to provide as input. Should be equal in length + to 'input_tensor_aliases'. +output_tensor_aliases: Tensor of strings for the output tensor alias names to + supply to the Predict call. +status_code: Returns the status code of the rpc call; basically converting + tensorflow::error::Code to it's int value, so 0 means OK. +status_error_message: Returns the error message in the rpc status. +output_tensors: Tensors returned by the Predict call on the remote graph, which + are in the same order as output_tensor_aliases. +output_types: A list of types of the output tensors. Length of this list should + be equal to the length of 'output_tensor_aliases'. +)doc"); + +} // namespace tensorflow diff --git a/tensorflow_serving/experimental/tensorflow/ops/remote_predict/python/ops/remote_predict_ops.py b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/python/ops/remote_predict_ops.py new file mode 100644 index 00000000000..b2854503459 --- /dev/null +++ b/tensorflow_serving/experimental/tensorflow/ops/remote_predict/python/ops/remote_predict_ops.py @@ -0,0 +1,124 @@ +# Copyright 2020 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Operations for RemotePredict.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +import os.path +import tensorflow.compat.v1 as tf + +from tensorflow_serving.experimental.tensorflow.ops.remote_predict.ops import gen_remote_predict_op +# pylint: disable=wildcard-import +from tensorflow_serving.experimental.tensorflow.ops.remote_predict.ops.gen_remote_predict_op import * +# pylint: enable=wildcard-import + +_remote_predict_op_module = tf.load_op_library( + os.path.join(tf.compat.v1.resource_loader.get_data_files_path(), + '_remote_predict_op.so')) + + +# Aliases +def run(input_tensor_alias, + input_tensors, + output_tensor_alias, + target_address, + model_name, + model_version=-1, + max_rpc_deadline_millis=3000, + output_types=None, + name=None, + signature_name='serving_default'): + """Runs a predict in remote process through rpc. + + Args: + input_tensor_alias: input tensor alias for Predict + input_tensors: input tensors for Predict + output_tensor_alias: output tensor alias for Predict + target_address: target_address where the rpc is sent to + model_name: model_name that the Predict is running on + model_version: the model version for the Predict call. If unset, the highest + version available for serving will be targeted. + max_rpc_deadline_millis: rpc deadline in millis + output_types: output types for Predict + name: name for the op in the graph + signature_name: the signature def for remote graph inference + + Returns: + output_tensors as a result of the Predict. + + Raises ValueError if model_name value is missing. + """ + if model_name is None: + raise ValueError('model_name must be specified.') + return (gen_remote_predict_op.tf_serving_remote_predict( + input_tensor_alias, + input_tensors, + output_tensor_alias, + target_address=target_address, + model_name=model_name, + model_version=model_version, + fail_op_on_rpc_error=True, + max_rpc_deadline_millis=max_rpc_deadline_millis, + signature_name=signature_name, + output_types=output_types, + name=name))[2] + + +def run_returning_status(input_tensor_alias, + input_tensors, + output_tensor_alias, + target_address, + model_name, + model_version=-1, + max_rpc_deadline_millis=3000, + output_types=None, + name=None, + signature_name='serving_default'): + """Runs a predict in remote process through rpc. + + Args: + input_tensor_alias: input tensor alias for Predict + input_tensors: input tensors for Predict + output_tensor_alias: output tensor alias for Predict + target_address: target_address where the rpc is sent to + model_name: model_name that the Predict is running on + model_version: the model version for the Predict call. If unset, the highest + version available for serving will be targeted. + max_rpc_deadline_millis: rpc deadline in millis + output_types: output types for Predict + name: name for the op in the graph + signature_name: the signature def for remote graph inference + + Returns: + status_code, status_error_message and output_tensors. + + Raises ValueError if model_name value is missing. + """ + if model_name is None: + raise ValueError('model_name must be specified.') + return (gen_remote_predict_op.tf_serving_remote_predict( + input_tensor_alias, + input_tensors, + output_tensor_alias, + target_address=target_address, + model_name=model_name, + model_version=model_version, + fail_op_on_rpc_error=False, + max_rpc_deadline_millis=max_rpc_deadline_millis, + signature_name=signature_name, + output_types=output_types, + name=name)) diff --git a/tensorflow_serving/g3doc/METADATA b/tensorflow_serving/g3doc/METADATA deleted file mode 100644 index 8fb0d57fc32..00000000000 --- a/tensorflow_serving/g3doc/METADATA +++ /dev/null @@ -1,5 +0,0 @@ -name: "TensorFlow Serving" -g3doc: { - sitemap_file: "_includes/nav.md" -} - diff --git a/tensorflow_serving/g3doc/_config.yml b/tensorflow_serving/g3doc/_config.yml deleted file mode 100644 index 71a84622bd6..00000000000 --- a/tensorflow_serving/g3doc/_config.yml +++ /dev/null @@ -1,23 +0,0 @@ -# Jekyll configuration file for Github Pages. If you are running a local -# Jekyll server, note that his file is *NOT* reloaded automatically when you change it. If -# you change this file, please restart your local server process. - -# Site settings -title: TensorFlow Serving -email: discuss@tensorflow.org -description: > # this means to ignore newlines until "baseurl:" - TensorFlow Serving is a flexible, high-performance serving system for - machine learning models, designed for production environments. -baseurl: "/serving" # the subpath of your site, e.g. /blog -url: "https://tensorflow.github.io" # the base hostname & protocol for your site -twitter_username: googleresearch -github_username: tensorflow - -# Build settings -markdown: kramdown -defaults: - - - scope: - path: "" - values: - layout: "default" diff --git a/tensorflow_serving/g3doc/_includes/footer.html b/tensorflow_serving/g3doc/_includes/footer.html deleted file mode 100644 index d51240ac00f..00000000000 --- a/tensorflow_serving/g3doc/_includes/footer.html +++ /dev/null @@ -1,50 +0,0 @@ -
- -
- -

{{ site.title }}

- -
-
- -
- -
- -
- -
-

{{ site.description }}

-
-
- -
- - -
diff --git a/tensorflow_serving/g3doc/_includes/head.html b/tensorflow_serving/g3doc/_includes/head.html deleted file mode 100644 index 41340ae57b4..00000000000 --- a/tensorflow_serving/g3doc/_includes/head.html +++ /dev/null @@ -1,12 +0,0 @@ - - - - - - {% if page.title %}{{ page.title }}{% else %}{{ site.title }}{% endif %} - - - - - - diff --git a/tensorflow_serving/g3doc/_includes/header.html b/tensorflow_serving/g3doc/_includes/header.html deleted file mode 100644 index 1cc2fd8e3c1..00000000000 --- a/tensorflow_serving/g3doc/_includes/header.html +++ /dev/null @@ -1,7 +0,0 @@ -
- -
- {{ site.title }} -
- -
diff --git a/tensorflow_serving/g3doc/_includes/icon-github.html b/tensorflow_serving/g3doc/_includes/icon-github.html deleted file mode 100644 index e501a16b187..00000000000 --- a/tensorflow_serving/g3doc/_includes/icon-github.html +++ /dev/null @@ -1 +0,0 @@ -{% include icon-github.svg %}{{ include.username }} diff --git a/tensorflow_serving/g3doc/_includes/icon-github.svg b/tensorflow_serving/g3doc/_includes/icon-github.svg deleted file mode 100644 index 4422c4f5dcc..00000000000 --- a/tensorflow_serving/g3doc/_includes/icon-github.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tensorflow_serving/g3doc/_includes/icon-twitter.html b/tensorflow_serving/g3doc/_includes/icon-twitter.html deleted file mode 100644 index e623dbd6efc..00000000000 --- a/tensorflow_serving/g3doc/_includes/icon-twitter.html +++ /dev/null @@ -1 +0,0 @@ -{{ include.username }} diff --git a/tensorflow_serving/g3doc/_includes/icon-twitter.svg b/tensorflow_serving/g3doc/_includes/icon-twitter.svg deleted file mode 100644 index dcf660e7bb3..00000000000 --- a/tensorflow_serving/g3doc/_includes/icon-twitter.svg +++ /dev/null @@ -1 +0,0 @@ - diff --git a/tensorflow_serving/g3doc/_includes/nav.md b/tensorflow_serving/g3doc/_includes/nav.md deleted file mode 100644 index 93fabb571b0..00000000000 --- a/tensorflow_serving/g3doc/_includes/nav.md +++ /dev/null @@ -1,21 +0,0 @@ -## Get Started - -* [Architecture Overview](architecture_overview) -* [Installation](setup) - -## Tutorials - -* [TensorFlow Serving Basics](serving_basic) -* [TensorFlow Serving Advanced](serving_advanced) -* [Serving Inception Model with TensorFlow Serving and Kubernetes](serving_inception) - -## How To - -* [Create a Custom Servable](custom_servable) -* [Create a Custom Source](custom_source) - -## Resources - -* [Source code](https://github.com/tensorflow/serving) -* [TensorFlow site](http://tensorflow.org) -* [Docker images](docker) diff --git a/tensorflow_serving/g3doc/_layouts/default.html b/tensorflow_serving/g3doc/_layouts/default.html deleted file mode 100644 index 21ef154988d..00000000000 --- a/tensorflow_serving/g3doc/_layouts/default.html +++ /dev/null @@ -1,27 +0,0 @@ - - - - {% include head.html %} - - - - {% include header.html %} -
-
-
- {% capture sidebar %}{% include nav.md %}{% endcapture %} - {{ sidebar | markdownify }} -
-
- -
-
- {{ content }} -
-
-
- {% include footer.html %} - - - - diff --git a/tensorflow_serving/g3doc/_sass/_base.scss b/tensorflow_serving/g3doc/_sass/_base.scss deleted file mode 100644 index d2aecc18798..00000000000 --- a/tensorflow_serving/g3doc/_sass/_base.scss +++ /dev/null @@ -1,225 +0,0 @@ -// Reset some basic elements -body, -h1, -h2, -h3, -h4, -h5, -h6, -p, -blockquote, -pre, -hr, -dl, -dd, -ol, -ul, -figure { - margin: 0; - padding: 0; -} - - - -// Basic styling -body { - font: $base-font-weight #{$base-font-size}/#{$base-line-height} $base-font-family; - color: $text-color; - background-color: $background-color; - -webkit-text-size-adjust: 100%; - -webkit-font-feature-settings: "kern" 1; - -moz-font-feature-settings: "kern" 1; - -o-font-feature-settings: "kern" 1; - font-feature-settings: "kern" 1; - font-kerning: normal; -} - - - -// Set `margin-bottom` to maintain vertical rhythm -h1, -h2, -h3, -h4, -h5, -h6, -p, -blockquote, -pre, -ul, -ol, -dl, -figure, -%vertical-rhythm { - margin-bottom: $spacing-unit / 2; -} - -// Links -a:visited { - color: #000; -} - -// Images -img { - max-width: 100%; - vertical-align: middle; -} - - - -// Figures -figure > img { - display: block; -} - -figcaption { - font-size: $small-font-size; -} - - - -// Lists -ul, -ol { - margin-left: $spacing-unit; -} - -li { - > ul, - > ol { - margin-bottom: 0; - } -} - - - -// Headings -h1, -h2, -h3, -h4, -h5, -h6 { - font-weight: $base-font-weight; -} - - -// Links -a { - color: $brand-color; - text-decoration: none; - - &:visited { - color: darken($brand-color, -15%); - } - - &:hover { - color: $text-color; - text-decoration: underline; - } -} - - - -// Blockquotes -blockquote { - color: $grey-color; - border-left: 4px solid $grey-color-light; - padding-left: $spacing-unit / 2; - font-size: 18px; - letter-spacing: -1px; - font-style: italic; - - > :last-child { - margin-bottom: 0; - } -} - - - -// Code formatting -pre, -code { - font-size: 15px; - border: 1px solid $grey-color-light; - border-radius: 3px; - background-color: #eef; -} - -code { - padding: 1px 5px; -} - -pre { - padding: 8px 12px; - overflow-x: auto; - - > code { - border: 0; - padding-right: 0; - padding-left: 0; - } -} - - - -// Wrapper -.wrapper { - max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit} * 2)); - max-width: calc(#{$content-width} - (#{$spacing-unit} * 2)); - margin-right: auto; - padding-right: $spacing-unit; - padding-left: $spacing-unit; - @extend %clearfix; - - @include media-query($on-laptop) { - max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit})); - max-width: calc(#{$content-width} - (#{$spacing-unit})); - padding-right: $spacing-unit / 2; - padding-left: $spacing-unit / 2; - } -} - -.footer-wrapper { - max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit} * 2)); - max-width: calc(#{$content-width} - (#{$spacing-unit} * 2)); - margin-left: auto; - margin-right: auto; - padding-right: $spacing-unit; - padding-left: $spacing-unit; - @extend %clearfix; - - @include media-query($on-laptop) { - max-width: -webkit-calc(#{$content-width} - (#{$spacing-unit})); - max-width: calc(#{$content-width} - (#{$spacing-unit})); - padding-right: $spacing-unit / 2; - padding-left: $spacing-unit / 2; - } -} - -// Clearfix -%clearfix { - - &:after { - content: ''""''; - display: table; - clear: both; - } -} - - -// Icons -.icon { - - > svg { - display: inline-block; - width: 16px; - height: 16px; - vertical-align: middle; - - path { - fill: $grey-color; - } - } -} diff --git a/tensorflow_serving/g3doc/_sass/_layout.scss b/tensorflow_serving/g3doc/_sass/_layout.scss deleted file mode 100644 index ba1a54107b0..00000000000 --- a/tensorflow_serving/g3doc/_sass/_layout.scss +++ /dev/null @@ -1,275 +0,0 @@ -// scss-lint:disable all - -/** - * Site header - */ -.site-header { - //border-top: 5px solid $grey-color-dark; - //border-bottom: 1px solid $grey-color-light; - height: $header-height; - line-height: 80px; - background-color: #f6911e; - background-image: -webkit-linear-gradient(-360deg, #f6911e,#febd36); - background-image: linear-gradient(90deg,#f6911e,#febd36); - - // Positioning context for the mobile navigation icon - position: relative; -} - -.site-title { - color: #000; - font-size: 30px; - font-weight: 400; - line-height: $header-height; // Equals header height to center. - letter-spacing: -1px; - margin-bottom: 0; - float: left; -} - -a.site-title, a.site-title:visited { - color: #ffffff; -} - -.site-nav { - float: right; - line-height: 56px; - - .menu-icon { - display: none; - } - - .page-link { - color: $text-color; - line-height: $base-line-height; - - // Gaps between nav items, but not on the last one - &:not(:last-child) { - margin-right: 20px; - } - } - - @include media-query($on-palm) { - position: absolute; - top: 9px; - right: $spacing-unit / 2; - background-color: $background-color; - border: 1px solid $grey-color-light; - border-radius: 5px; - text-align: right; - - .menu-icon { - display: block; - float: right; - width: 36px; - height: 26px; - line-height: 0; - padding-top: 10px; - text-align: center; - - > svg { - width: 18px; - height: 15px; - - path { - fill: $grey-color-dark; - } - } - } - - .trigger { - clear: both; - display: none; - } - - &:hover .trigger { - display: block; - padding-bottom: 5px; - } - - .page-link { - display: block; - padding: 5px 10px; - - &:not(:last-child) { - margin-right: 0; - } - margin-left: 20px; - } - } -} - - - -/** - * Site footer - */ -.site-footer { - border-top: 1px solid $grey-color-light; - padding: $spacing-unit 0; -} - -.footer-heading { - font-size: 18px; - margin-bottom: $spacing-unit / 2; -} - -.contact-list, -.social-media-list { - list-style: none; - margin-left: 0; -} - -.footer-col-wrapper { - font-size: 15px; - color: $grey-color; - margin-left: -$spacing-unit / 2; - @extend %clearfix; -} - -.footer-col { - float: left; - margin-bottom: $spacing-unit / 2; - padding-left: $spacing-unit / 2; -} - -.footer-col-1 { - width: -webkit-calc(35% - (#{$spacing-unit} / 2)); - width: calc(35% - (#{$spacing-unit} / 2)); -} - -.footer-col-2 { - width: -webkit-calc(20% - (#{$spacing-unit} / 2)); - width: calc(20% - (#{$spacing-unit} / 2)); -} - -.footer-col-3 { - width: -webkit-calc(45% - (#{$spacing-unit} / 2)); - width: calc(45% - (#{$spacing-unit} / 2)); -} - -@include media-query($on-laptop) { - .footer-col-1, - .footer-col-2 { - width: -webkit-calc(50% - (#{$spacing-unit} / 2)); - width: calc(50% - (#{$spacing-unit} / 2)); - } - - .footer-col-3 { - width: -webkit-calc(100% - (#{$spacing-unit} / 2)); - width: calc(100% - (#{$spacing-unit} / 2)); - } -} - -@include media-query($on-palm) { - .footer-col { - float: none; - width: -webkit-calc(100% - (#{$spacing-unit} / 2)); - width: calc(100% - (#{$spacing-unit} / 2)); - } -} - -/** - * Main page block. - */ -#page-main { - clear:both; -} - -#page-main:after { - content: ""; - clear: both; - display: table; -} - -/** - * Sidebar - */ -.sidebar { - width:250px; - margin-top: 27px; - padding:0; - vertical-align:top; - float:left; - - a { - color: $text-color; - } -} - -/** - * Page content - */ -.page-content { - padding: $spacing-unit 0; - float:left -} - -.page-heading { - font-size: 20px; -} - -.post-list { - margin-left: 0; - list-style: none; - - > li { - margin-bottom: $spacing-unit; - } -} - -.post-meta { - font-size: $small-font-size; - color: $grey-color; -} - -.post-link { - display: block; - font-size: 24px; -} - - - -/** - * Posts - */ -.post-header { - margin-bottom: $spacing-unit; -} - -.post-title { - font-size: 42px; - letter-spacing: -1px; - line-height: 1; - - @include media-query($on-laptop) { - font-size: 36px; - } -} - -.post-content { - margin-bottom: $spacing-unit; - - h2 { - font-size: 32px; - - @include media-query($on-laptop) { - font-size: 28px; - } - } - - h3 { - font-size: 26px; - - @include media-query($on-laptop) { - font-size: 22px; - } - } - - h4 { - font-size: 20px; - - @include media-query($on-laptop) { - font-size: 18px; - } - } -} diff --git a/tensorflow_serving/g3doc/_sass/_syntax-highlighting.scss b/tensorflow_serving/g3doc/_sass/_syntax-highlighting.scss deleted file mode 100644 index 8fac59776d5..00000000000 --- a/tensorflow_serving/g3doc/_sass/_syntax-highlighting.scss +++ /dev/null @@ -1,71 +0,0 @@ -/** - * Syntax highlighting styles - */ -.highlight { - background: #fff; - @extend %vertical-rhythm; - - .highlighter-rouge & { - background: #eef; - } - - .c { color: #998; font-style: italic } // Comment - .err { color: #a61717; background-color: #e3d2d2 } // Error - .k { font-weight: bold } // Keyword - .o { font-weight: bold } // Operator - .cm { color: #998; font-style: italic } // Comment.Multiline - .cp { color: #999; font-weight: bold } // Comment.Preproc - .c1 { color: #998; font-style: italic } // Comment.Single - .cs { color: #999; font-weight: bold; font-style: italic } // Comment.Special - .gd { color: #000; background-color: #fdd } // Generic.Deleted - .gd .x { color: #000; background-color: #faa } // Generic.Deleted.Specific - .ge { font-style: italic } // Generic.Emph - .gr { color: #a00 } // Generic.Error - .gh { color: #999 } // Generic.Heading - .gi { color: #000; background-color: #dfd } // Generic.Inserted - .gi .x { color: #000; background-color: #afa } // Generic.Inserted.Specific - .go { color: #888 } // Generic.Output - .gp { color: #555 } // Generic.Prompt - .gs { font-weight: bold } // Generic.Strong - .gu { color: #aaa } // Generic.Subheading - .gt { color: #a00 } // Generic.Traceback - .kc { font-weight: bold } // Keyword.Constant - .kd { font-weight: bold } // Keyword.Declaration - .kp { font-weight: bold } // Keyword.Pseudo - .kr { font-weight: bold } // Keyword.Reserved - .kt { color: #458; font-weight: bold } // Keyword.Type - .m { color: #099 } // Literal.Number - .s { color: #d14 } // Literal.String - .na { color: #008080 } // Name.Attribute - .nb { color: #0086B3 } // Name.Builtin - .nc { color: #458; font-weight: bold } // Name.Class - .no { color: #008080 } // Name.Constant - .ni { color: #800080 } // Name.Entity - .ne { color: #900; font-weight: bold } // Name.Exception - .nf { color: #900; font-weight: bold } // Name.Function - .nn { color: #555 } // Name.Namespace - .nt { color: #000080 } // Name.Tag - .nv { color: #008080 } // Name.Variable - .ow { font-weight: bold } // Operator.Word - .w { color: #bbb } // Text.Whitespace - .mf { color: #099 } // Literal.Number.Float - .mh { color: #099 } // Literal.Number.Hex - .mi { color: #099 } // Literal.Number.Integer - .mo { color: #099 } // Literal.Number.Oct - .sb { color: #d14 } // Literal.String.Backtick - .sc { color: #d14 } // Literal.String.Char - .sd { color: #d14 } // Literal.String.Doc - .s2 { color: #d14 } // Literal.String.Double - .se { color: #d14 } // Literal.String.Escape - .sh { color: #d14 } // Literal.String.Heredoc - .si { color: #d14 } // Literal.String.Interpol - .sx { color: #d14 } // Literal.String.Other - .sr { color: #009926 } // Literal.String.Regex - .s1 { color: #d14 } // Literal.String.Single - .ss { color: #990073 } // Literal.String.Symbol - .bp { color: #999 } // Name.Builtin.Pseudo - .vc { color: #008080 } // Name.Variable.Class - .vg { color: #008080 } // Name.Variable.Global - .vi { color: #008080 } // Name.Variable.Instance - .il { color: #099 } // Literal.Number.Integer.Long -} diff --git a/tensorflow_serving/g3doc/_toc.yaml b/tensorflow_serving/g3doc/_toc.yaml new file mode 100644 index 00000000000..6a1bc8a165c --- /dev/null +++ b/tensorflow_serving/g3doc/_toc.yaml @@ -0,0 +1,23 @@ +toc: +- title: TensorFlow Serving with Docker + path: /tfx/serving/docker +- title: Installation + path: /tfx/serving/setup +- title: Serve a TensorFlow model + path: /tfx/serving/serving_basic +- title: Architecture + path: /tfx/serving/architecture +- title: Advanced model server configuration + path: /tfx/serving/serving_config +- title: Build a TensorFlow ModelServer + path: /tfx/serving/serving_advanced +- title: Use TensorFlow Serving with Kubernetes + path: /tfx/serving/serving_kubernetes +- title: Create a new kind of servable + path: /tfx/serving/custom_servable +- title: Create a module that discovers new servable paths + path: /tfx/serving/custom_source +- title: Serving TensorFlow models with custom ops + path: /tfx/serving/custom_op +- title: SignatureDefs in SavedModel for TensorFlow Serving + path: /tfx/serving/signature_defs diff --git a/tensorflow_serving/g3doc/api_rest.md b/tensorflow_serving/g3doc/api_rest.md new file mode 100644 index 00000000000..8edd1d3acbc --- /dev/null +++ b/tensorflow_serving/g3doc/api_rest.md @@ -0,0 +1,499 @@ +# RESTful API + +In addition to +[gRPC APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto) +TensorFlow ModelServer also supports RESTful APIs. This page describes these API +endpoints and an end-to-end [example](#example) on usage. + +The request and response is a JSON object. The composition of this object +depends on the request type or verb. See the API specific sections below for +details. + +In case of error, all APIs will return a JSON object in the response body with +`error` as key and the error message as the value: + +```javascript +{ + "error": +} +``` + +## Model status API + +This API closely follows the +[`ModelService.GetModelStatus`](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/model_service.proto#L17) +gRPC API. It returns the status of a model in the ModelServer. + +### URL + +``` +GET http://host:port/v1/models/${MODEL_NAME}[/versions/${VERSION}|/labels/${LABEL}] +``` + +Including `/versions/${VERSION}` or `/labels/${LABEL}` is optional. If omitted +status for all versions is returned in the response. + +### Response format + +If successful, returns a JSON representation of +[`GetModelStatusResponse`](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/get_model_status.proto#L64) +protobuf. + +## Model Metadata API + +This API closely follows the +[`PredictionService.GetModelMetadata`](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/prediction_service.proto#L29) +gRPC API. It returns the metadata of a model in the ModelServer. + +### URL + +``` +GET http://host:port/v1/models/${MODEL_NAME}[/versions/${VERSION}|/labels/${LABEL}]/metadata +``` + +Including `/versions/${VERSION}` or `/labels/${LABEL}` is optional. If omitted +the model metadata for the latest version is returned in the response. + +### Response format + +If successful, returns a JSON representation of +[`GetModelMetadataResponse`](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/get_model_metadata.proto#L23) +protobuf. + +## Classify and Regress API + +This API closely follows the `Classify` and `Regress` methods of +[`PredictionService`](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/prediction_service.proto#L15) +gRPC API. + +### URL + +``` +POST http://host:port/v1/models/${MODEL_NAME}[/versions/${VERSION}|/labels/${LABEL}]:(classify|regress) +``` + +Including `/versions/${VERSION}` or `/labels/${LABEL}` is optional. If omitted +the latest version is used. + +### Request format + +The request body for the `classify` and `regress` APIs must be a JSON object +formatted as follows: + +```javascript +{ + // Optional: serving signature to use. + // If unspecifed default serving signature is used. + "signature_name": , + + // Optional: Common context shared by all examples. + // Features that appear here MUST NOT appear in examples (below). + "context": { + "": | + "": | + }, + + // List of Example objects + "examples": [ + { + // Example 1 + "": |, + "": |, + ... + }, + { + // Example 2 + "": |, + "": |, + ... + } + ... + ] +} +``` + +`` is a JSON number (whole or decimal), JSON string, or a JSON object +that represents binary data (see the [Encoding binary values](#encoding-binary-values) +section below for details). `` is a list of such values. This +format is similar to gRPC's `ClassificationRequest` and `RegressionRequest` +protos. Both versions accept list of +[`Example`](https://github.com/tensorflow/tensorflow/blob/92e6c3e4f5c1cabfda1e61547a6a1b268ef95fa5/tensorflow/core/example/example.proto#L13) +objects. + +### Response format + +A `classify` request returns a JSON object in the response body, formatted as +follows: + +```javascript +{ + "result": [ + // List of class label/score pairs for first Example (in request) + [ [, ], [, ], ... ], + + // List of class label/score pairs for next Example (in request) + [ [, ], [, ], ... ], + ... + ] +} +``` + +`