From e13d3c9f46249eae67f17ba42fe333bfd1e36038 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 8 Sep 2016 14:17:43 -0800 Subject: [PATCH 0001/8554] Upgrade LoaderHarness logging level from VLOG(1) to LOG(INFO). Change: 132607268 --- tensorflow_serving/core/loader_harness.cc | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow_serving/core/loader_harness.cc b/tensorflow_serving/core/loader_harness.cc index 68b4dcf12e4..0eebabb5014 100644 --- a/tensorflow_serving/core/loader_harness.cc +++ b/tensorflow_serving/core/loader_harness.cc @@ -67,7 +67,7 @@ Status LoaderHarness::LoadApproved() { " instead of state: ", StateDebugString(State::kLoadRequested)); } state_ = State::kLoadApproved; - VLOG(1) << "Approving load for servable version " << id_; + LOG(INFO) << "Approving load for servable version " << id_; return Status::OK(); } @@ -82,7 +82,7 @@ Status LoaderHarness::Load(const ResourceAllocation& available_resources) { StateDebugString(State::kLoadApproved)); } state_ = State::kLoading; - VLOG(1) << "Loading servable version " << id_; + LOG(INFO) << "Loading servable version " << id_; } const Status status = [&]() { @@ -115,7 +115,7 @@ Status LoaderHarness::Load(const ResourceAllocation& available_resources) { DCHECK_EQ(State::kLoading, state_); if (status.ok()) { state_ = State::kReady; - VLOG(1) << "Successfully loaded servable version " << id_; + LOG(INFO) << "Successfully loaded servable version " << id_; } else { ErrorInternal(status); } @@ -153,7 +153,7 @@ void LoaderHarness::Unload() { mutex_lock l(mu_); DCHECK_EQ(state_, State::kQuiesced); state_ = State::kUnloading; - VLOG(1) << "Unloading servable version " << id_; + LOG(INFO) << "Unloading servable version " << id_; } loader_->Unload(); @@ -162,7 +162,7 @@ void LoaderHarness::Unload() { mutex_lock l(mu_); DCHECK_EQ(state_, State::kUnloading); state_ = State::kDisabled; - VLOG(1) << "Done unloading servable version " << id_; + LOG(INFO) << "Done unloading servable version " << id_; } } @@ -175,7 +175,7 @@ Status LoaderHarness::StartQuiescing() { StateDebugString(State::kUnloadRequested)); } state_ = State::kQuiescing; - VLOG(1) << "Quiescing servable version " << id_; + LOG(INFO) << "Quiescing servable version " << id_; return Status::OK(); } @@ -183,14 +183,14 @@ void LoaderHarness::DoneQuiescing() { mutex_lock l(mu_); DCHECK_EQ(state_, State::kQuiescing); state_ = State::kQuiesced; - VLOG(1) << "Done quiescing servable version " << id_; + LOG(INFO) << "Done quiescing servable version " << id_; } void LoaderHarness::ErrorInternal(const Status status) { state_ = State::kError; status_ = status; - VLOG(1) << "Encountered an error for servable version " << id_ << ": " - << status_; + LOG(INFO) << "Encountered an error for servable version " << id_ << ": " + << status_; } void LoaderHarness::Error(const Status status) { From 156e52b9835ed2d84eda7da31845a5deb86b1abe Mon Sep 17 00:00:00 2001 From: Fangwei Li Date: Thu, 8 Sep 2016 14:26:15 -0800 Subject: [PATCH 0002/8554] Change examples and tutorials to use standard tensorflow_model_server. Change: 132608510 --- tensorflow_serving/example/BUILD | 90 +--- .../example/inception_client.py | 15 +- .../example/inception_export.py | 10 +- .../example/inception_inference.cc | 393 ---------------- .../example/inception_inference.proto | 22 - .../example/inception_inference_pb2.py | 178 -------- tensorflow_serving/example/mnist_client.py | 21 +- tensorflow_serving/example/mnist_export.py | 11 +- tensorflow_serving/example/mnist_inference.cc | 219 --------- .../example/mnist_inference.proto | 24 - .../example/mnist_inference_2.cc | 432 ------------------ .../example/mnist_inference_pb2.py | 178 -------- tensorflow_serving/g3doc/serving_advanced.md | 183 ++++---- tensorflow_serving/g3doc/serving_basic.md | 196 ++------ tensorflow_serving/g3doc/serving_inception.md | 112 +++-- 15 files changed, 238 insertions(+), 1846 deletions(-) delete mode 100644 tensorflow_serving/example/inception_inference.cc delete mode 100644 tensorflow_serving/example/inception_inference.proto delete mode 100644 tensorflow_serving/example/inception_inference_pb2.py delete mode 100644 tensorflow_serving/example/mnist_inference.cc delete mode 100644 tensorflow_serving/example/mnist_inference.proto delete mode 100644 tensorflow_serving/example/mnist_inference_2.cc delete mode 100644 tensorflow_serving/example/mnist_inference_pb2.py diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 9ea4a93c5c1..0c63af1585f 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -50,73 +50,19 @@ py_binary( ], ) -cc_binary( - name = "mnist_inference", - srcs = [ - "mnist_inference.cc", - ], - linkopts = ["-lm"], - deps = [ - ":mnist_inference_proto", - "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", - "//tensorflow_serving/servables/tensorflow:session_bundle_factory", - "@grpc//:grpc++", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:core_cpu", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - -cc_binary( - name = "mnist_inference_2", - srcs = [ - "mnist_inference_2.cc", - ], - linkopts = ["-lm"], - deps = [ - ":mnist_inference_proto", - "//tensorflow_serving/batching:basic_batch_scheduler", - "//tensorflow_serving/batching:batch_scheduler", - "//tensorflow_serving/core:manager", - "//tensorflow_serving/core:servable_handle", - "//tensorflow_serving/core:servable_id", - "//tensorflow_serving/servables/tensorflow:simple_servers", - "@grpc//:grpc++", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - py_binary( name = "mnist_client", srcs = [ "mnist_client.py", - "mnist_inference_pb2.py", ], deps = [ ":mnist_input_data", + "//tensorflow_serving/apis:predict_proto_py_pb2", + "//tensorflow_serving/apis:prediction_service_proto_py_pb2", "@org_tensorflow//tensorflow:tensorflow_py", ], ) -serving_proto_library( - name = "inception_inference_proto", - srcs = ["inception_inference.proto"], - has_services = 1, - cc_api_version = 2, - cc_grpc_version = 1, -) - py_binary( name = "inception_export", srcs = [ @@ -129,36 +75,14 @@ py_binary( ], ) -cc_binary( - name = "inception_inference", - srcs = [ - "inception_inference.cc", - ], - linkopts = ["-lm"], - deps = [ - ":inception_inference_proto", - "//tensorflow_serving/batching:basic_batch_scheduler", - "//tensorflow_serving/batching:batch_scheduler", - "//tensorflow_serving/core:manager", - "//tensorflow_serving/core:servable_handle", - "//tensorflow_serving/core:servable_id", - "//tensorflow_serving/servables/tensorflow:simple_servers", - "@grpc//:grpc++", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - py_binary( name = "inception_client", srcs = [ "inception_client.py", - "inception_inference_pb2.py", ], - deps = ["@org_tensorflow//tensorflow:tensorflow_py"], + deps = [ + "//tensorflow_serving/apis:predict_proto_py_pb2", + "//tensorflow_serving/apis:prediction_service_proto_py_pb2", + "@org_tensorflow//tensorflow:tensorflow_py", + ], ) diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py index 0b1677bf185..b5d9790bf5e 100644 --- a/tensorflow_serving/example/inception_client.py +++ b/tensorflow_serving/example/inception_client.py @@ -15,7 +15,7 @@ #!/usr/bin/env python2.7 -"""Send JPEG image to inception_inference server for classification. +"""Send JPEG image to tensorflow_model_server loaded with inception model. """ # This is a placeholder for a Google-internal import. @@ -23,7 +23,8 @@ from grpc.beta import implementations import tensorflow as tf -from tensorflow_serving.example import inception_inference_pb2 +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_service_pb2 tf.app.flags.DEFINE_string('server', 'localhost:9000', @@ -38,14 +39,16 @@ def main(_): host, port = FLAGS.server.split(':') channel = implementations.insecure_channel(host, int(port)) - stub = inception_inference_pb2.beta_create_InceptionService_stub(channel) + stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) # Send request with open(FLAGS.image, 'rb') as f: # See inception_inference.proto for gRPC request/response details. data = f.read() - request = inception_inference_pb2.InceptionRequest() - request.jpeg_encoded = data - result = stub.Classify(request, 10.0) # 10 secs timeout + request = predict_pb2.PredictRequest() + request.model_spec.name = 'inception' + request.inputs['images'].CopyFrom( + tf.contrib.util.make_tensor_proto(data, shape=[1])) + result = stub.Predict(request, 10.0) # 10 secs timeout print result diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index a30617d0fc5..f658f4733a0 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -15,6 +15,9 @@ #!/usr/bin/env python2.7 """Export inception model given existing training checkpoints. + +The model is exported with proper signatures that can be loaded by standard +tensorflow_model_server. """ import os.path @@ -106,9 +109,10 @@ def export(): # Export inference model. init_op = tf.group(tf.initialize_all_tables(), name='init_op') model_exporter = exporter.Exporter(saver) - signature = exporter.classification_signature( - input_tensor=jpegs, classes_tensor=classes, scores_tensor=values) - model_exporter.init(default_graph_signature=signature, init_op=init_op) + model_exporter.init(init_op=init_op, named_graph_signatures={ + 'inputs': exporter.generic_signature({'images': jpegs}), + 'outputs': exporter.generic_signature({'classes': classes, + 'scores': values})}) model_exporter.export(FLAGS.export_dir, tf.constant(global_step), sess) print('Successfully exported model to %s' % FLAGS.export_dir) diff --git a/tensorflow_serving/example/inception_inference.cc b/tensorflow_serving/example/inception_inference.cc deleted file mode 100644 index 4697025909a..00000000000 --- a/tensorflow_serving/example/inception_inference.cc +++ /dev/null @@ -1,393 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A gRPC server that serves inception model exported by inception_export.py. -// Given each request with an image as JPEG encoded byte stream, the server -// responds with kNumTopClasses integer values as indexes of top k matched -// categories and kNumTopClasses float values as the corresponding -// probabilities. - -#include -#include -#include -#include -#include - -#include - -#include "grpc++/completion_queue.h" -#include "grpc++/security/server_credentials.h" -#include "grpc++/server.h" -#include "grpc++/server_builder.h" -#include "grpc++/server_context.h" -#include "grpc++/support/async_unary_call.h" -#include "grpc++/support/status.h" -#include "grpc++/support/status_code_enum.h" -#include "grpc/grpc.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" -#include "tensorflow_serving/batching/batch_scheduler.h" -#include "tensorflow_serving/core/manager.h" -#include "tensorflow_serving/core/servable_handle.h" -#include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/example/inception_inference.grpc.pb.h" -#include "tensorflow_serving/example/inception_inference.pb.h" -#include "tensorflow_serving/servables/tensorflow/simple_servers.h" - -using grpc::InsecureServerCredentials; -using grpc::Server; -using grpc::ServerAsyncResponseWriter; -using grpc::ServerBuilder; -using grpc::ServerContext; -using grpc::ServerCompletionQueue; -using grpc::Status; -using grpc::StatusCode; -using tensorflow::serving::InceptionRequest; -using tensorflow::serving::InceptionResponse; -using tensorflow::serving::InceptionService; -using tensorflow::string; -using tensorflow::Tensor; -using tensorflow::serving::ClassificationSignature; - -const int kNumTopClasses = 5; - -namespace { -class InceptionServiceImpl; - -// Class encompassing the state and logic needed to serve a request. -class CallData { - public: - CallData(InceptionServiceImpl* service_impl, - InceptionService::AsyncService* service, - ServerCompletionQueue* cq); - - void Proceed(); - - void Finish(Status status); - - const InceptionRequest& request() { return request_; } - - InceptionResponse* mutable_response() { return &response_; } - - private: - // Service implementation. - InceptionServiceImpl* service_impl_; - - // The means of communication with the gRPC runtime for an asynchronous - // server. - InceptionService::AsyncService* service_; - // The producer-consumer queue where for asynchronous server notifications. - ServerCompletionQueue* cq_; - // Context for the rpc, allowing to tweak aspects of it such as the use - // of compression, authentication, as well as to send metadata back to the - // client. - ServerContext ctx_; - - // What we get from the client. - InceptionRequest request_; - // What we send back to the client. - InceptionResponse response_; - - // The means to get back to the client. - ServerAsyncResponseWriter responder_; - - // Let's implement a tiny state machine with the following states. - enum CallStatus { CREATE, PROCESS, FINISH }; - CallStatus status_; // The current serving state. -}; - -// A Task holds all of the information for a single inference request. -struct Task : public tensorflow::serving::BatchTask { - ~Task() override = default; - size_t size() const override { return 1; } - - Task(CallData* calldata_arg) - : calldata(calldata_arg) {} - - CallData* calldata; -}; - -class InceptionServiceImpl final { - public: - InceptionServiceImpl(const string& servable_name, - std::unique_ptr manager); - - void Classify(CallData* call_data); - - // Produces classifications for a batch of requests and associated responses. - void DoClassifyInBatch( - std::unique_ptr> batch); - - // Name of the servable to use for inference. - const string servable_name_; - // Manager in charge of loading and unloading servables. - std::unique_ptr manager_; - // A scheduler for batching multiple request calls into single calls to - // Session->Run(). - std::unique_ptr> - batch_scheduler_; -}; - -// Take in the "service" instance (in this case representing an asynchronous -// server) and the completion queue "cq" used for asynchronous communication -// with the gRPC runtime. -CallData::CallData(InceptionServiceImpl* service_impl, - InceptionService::AsyncService* service, - ServerCompletionQueue* cq) - : service_impl_(service_impl), - service_(service), cq_(cq), responder_(&ctx_), status_(CREATE) { - // Invoke the serving logic right away. - Proceed(); -} - -void CallData::Proceed() { - if (status_ == CREATE) { - // As part of the initial CREATE state, we *request* that the system - // start processing Classify requests. In this request, "this" acts are - // the tag uniquely identifying the request (so that different CallData - // instances can serve different requests concurrently), in this case - // the memory address of this CallData instance. - service_->RequestClassify(&ctx_, &request_, &responder_, cq_, cq_, this); - // Make this instance progress to the PROCESS state. - status_ = PROCESS; - } else if (status_ == PROCESS) { - // Spawn a new CallData instance to serve new clients while we process - // the one for this CallData. The instance will deallocate itself as - // part of its FINISH state. - new CallData(service_impl_, service_, cq_); - // Start processing. - service_impl_->Classify(this); - } else { - GPR_ASSERT(status_ == FINISH); - // Once in the FINISH state, deallocate ourselves (CallData). - delete this; - } -} - -void CallData::Finish(Status status) { - status_ = FINISH; - responder_.Finish(response_, status, this); -} - -InceptionServiceImpl::InceptionServiceImpl( - const string& servable_name, - std::unique_ptr manager) - : servable_name_(servable_name), manager_(std::move(manager)) { - // Setup a batcher used to combine multiple requests (tasks) into a single - // graph run for efficiency. - // The batcher queues tasks until, - // (a) the next task would cause the batch to exceed the size target; - // (b) waiting for more tasks to be added would exceed the timeout. - // at which point it processes the entire batch. - // - // Use the default batch-size, timeout and thread options. In general - // the numbers are extremely performance critical and should be tuned based - // specific graph structure and usage. - tensorflow::serving::BasicBatchScheduler::Options scheduler_options; - scheduler_options.thread_pool_name = "inception_service_batch_threads"; - // Use a very large queue, to avoid rejecting requests. (Note: a production - // server with load balancing may want to use the default, much smaller, - // value.) - scheduler_options.max_enqueued_batches = 1000; - scheduler_options.max_batch_size = 10; - TF_CHECK_OK(tensorflow::serving::BasicBatchScheduler::Create( - scheduler_options, - [this](std::unique_ptr> batch) { - this->DoClassifyInBatch(std::move(batch)); - }, - &batch_scheduler_)); -} - -// Creates a gRPC Status from a TensorFlow Status. -Status ToGRPCStatus(const tensorflow::Status& status) { - return Status(static_cast(status.code()), - status.error_message()); -} - -void InceptionServiceImpl::Classify(CallData* calldata) { - // Create and submit a task to the batch scheduler. - std::unique_ptr task(new Task(calldata)); - tensorflow::Status status = batch_scheduler_->Schedule(&task); - - if (!status.ok()) { - calldata->Finish(ToGRPCStatus(status)); - return; - } -} - -// Produces classifications for a batch of requests and associated responses. -void InceptionServiceImpl::DoClassifyInBatch( - std::unique_ptr> batch) { - batch->WaitUntilClosed(); - if (batch->empty()) { - return; - } - const int batch_size = batch->num_tasks(); - - // Replies to each task with the given error status. - auto complete_with_error = [&batch](StatusCode code, const string& msg) { - Status status(code, msg); - for (int i = 0; i < batch->num_tasks(); i++) { - Task* task = batch->mutable_task(i); - task->calldata->Finish(status); - } - }; - - // Get a handle to the SessionBundle. The handle ensures the Manager does - // not reload this while it is in use. - auto handle_request = - tensorflow::serving::ServableRequest::Latest(servable_name_); - tensorflow::serving::ServableHandle - bundle; - const tensorflow::Status lookup_status = - manager_->GetServableHandle(handle_request, &bundle); - if (!lookup_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - lookup_status.error_message()); - return; - } - - // Get the default signature of the graph. Expected to be a - // classification signature. - tensorflow::serving::ClassificationSignature signature; - const tensorflow::Status signature_status = - GetClassificationSignature(bundle->meta_graph_def, &signature); - if (!signature_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - signature_status.error_message()); - return; - } - - // Transform protobuf input to inference input tensor. - tensorflow::Tensor batched_input(tensorflow::DT_STRING, {batch_size}); - for (int i = 0; i < batch_size; ++i) { - batched_input.vec()(i) = - batch->mutable_task(i)->calldata->request().jpeg_encoded(); - } - - // Run classification. - tensorflow::Tensor batched_classes; - tensorflow::Tensor batched_scores; - const tensorflow::Status run_status = - RunClassification(signature, batched_input, bundle->session.get(), - &batched_classes, &batched_scores); - if (!run_status.ok()) { - complete_with_error(StatusCode::INTERNAL, run_status.error_message()); - return; - } - - // Transform inference output tensor to protobuf output. - for (int i = 0; i < batch_size; ++i) { - auto calldata = batch->mutable_task(i)->calldata; - auto classes = calldata->mutable_response()->mutable_classes(); - auto scores = calldata->mutable_response()->mutable_scores(); - for (int j = 0; j < kNumTopClasses; j++) { - *classes->Add() = batched_classes.matrix()(i, j); - scores->Add(batched_scores.matrix()(i, j)); - } - calldata->Finish(Status::OK); - } -} - -void HandleRpcs(InceptionServiceImpl* service_impl, - InceptionService::AsyncService* service, - ServerCompletionQueue* cq) { - // Spawn a new CallData instance to serve new clients. - new CallData(service_impl, service, cq); - void* tag; // uniquely identifies a request. - bool ok = false; - while (true) { - // Block waiting to read the next event from the completion queue. The - // event is uniquely identified by its tag, which in this case is the - // memory address of a CallData instance. - if (!cq->Next(&tag, &ok)) { - break; // server shutting down - } - if (!ok) { - continue; // irregular event - } - static_cast(tag)->Proceed(); - } -} - -// Runs InceptionService server until shutdown. -void RunServer(const int port, const string& servable_name, - std::unique_ptr manager) { - // "0.0.0.0" is the way to listen on localhost in gRPC. - const string server_address = "0.0.0.0:" + std::to_string(port); - - InceptionService::AsyncService service; - ServerBuilder builder; - std::shared_ptr creds = InsecureServerCredentials(); - builder.AddListeningPort(server_address, creds); - builder.RegisterService(&service); - std::unique_ptr cq = builder.AddCompletionQueue(); - std::unique_ptr server(builder.BuildAndStart()); - LOG(INFO) << "Running..."; - - InceptionServiceImpl service_impl(servable_name, std::move(manager)); - HandleRpcs(&service_impl, &service, cq.get()); -} - -} // namespace - -int main(int argc, char** argv) { - // Parse command-line options. - tensorflow::int32 port = 0; - const bool parse_result = - tensorflow::ParseFlags(&argc, argv, {tensorflow::Flag("port", &port)}); - if (!parse_result) { - LOG(FATAL) << "Error parsing command line flags."; - } - - if (argc != 2) { - LOG(ERROR) << "Usage: inception_inference --port=9000 /path/to/exports"; - return -1; - } - const string export_base_path(argv[1]); - tensorflow::port::InitMain(argv[0], &argc, &argv); - - std::unique_ptr manager; - tensorflow::Status status = tensorflow::serving::simple_servers:: - CreateSingleTFModelManagerFromBasePath(export_base_path, &manager); - - TF_CHECK_OK(status) << "Error creating manager"; - - // Wait until at least one model is loaded. - std::vector ready_ids; - // TODO(b/25545573): Create a more streamlined startup mechanism than polling. - do { - LOG(INFO) << "Waiting for models to be loaded..."; - tensorflow::Env::Default()->SleepForMicroseconds(1 * 1000 * 1000 /*1 sec*/); - ready_ids = manager->ListAvailableServableIds(); - } while (ready_ids.empty()); - - // Run the service. - RunServer(port, ready_ids[0].name, std::move(manager)); - - return 0; -} diff --git a/tensorflow_serving/example/inception_inference.proto b/tensorflow_serving/example/inception_inference.proto deleted file mode 100644 index 64bdc4f6baa..00000000000 --- a/tensorflow_serving/example/inception_inference.proto +++ /dev/null @@ -1,22 +0,0 @@ -// Protobuf definition of inception v3 model inference server. - -syntax = "proto3"; - -package tensorflow.serving; - -message InceptionRequest { - // JPEG encoded stream of the image to be classified. - bytes jpeg_encoded = 1; -}; - -message InceptionResponse { - // Human readable descriptions of the classes, in scores descending order. - repeated string classes = 3; - // Scores of top matches, in same order as classes. - repeated float scores = 2; -}; - -service InceptionService { - // Classifies an JPEG image into classes. - rpc Classify(InceptionRequest) returns (InceptionResponse); -} diff --git a/tensorflow_serving/example/inception_inference_pb2.py b/tensorflow_serving/example/inception_inference_pb2.py deleted file mode 100644 index 31344e39bc2..00000000000 --- a/tensorflow_serving/example/inception_inference_pb2.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: inception_inference.proto - -import sys -_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='inception_inference.proto', - package='tensorflow.serving', - syntax='proto3', - serialized_pb=_b('\n\x19inception_inference.proto\x12\x12tensorflow.serving\"(\n\x10InceptionRequest\x12\x14\n\x0cjpeg_encoded\x18\x01 \x01(\x0c\"4\n\x11InceptionResponse\x12\x0f\n\x07\x63lasses\x18\x03 \x03(\t\x12\x0e\n\x06scores\x18\x02 \x03(\x02\x32k\n\x10InceptionService\x12W\n\x08\x43lassify\x12$.tensorflow.serving.InceptionRequest\x1a%.tensorflow.serving.InceptionResponseb\x06proto3') -) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_INCEPTIONREQUEST = _descriptor.Descriptor( - name='InceptionRequest', - full_name='tensorflow.serving.InceptionRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='jpeg_encoded', full_name='tensorflow.serving.InceptionRequest.jpeg_encoded', index=0, - number=1, type=12, cpp_type=9, label=1, - has_default_value=False, default_value=_b(""), - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=49, - serialized_end=89, -) - - -_INCEPTIONRESPONSE = _descriptor.Descriptor( - name='InceptionResponse', - full_name='tensorflow.serving.InceptionResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='classes', full_name='tensorflow.serving.InceptionResponse.classes', index=0, - number=3, type=9, cpp_type=9, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - _descriptor.FieldDescriptor( - name='scores', full_name='tensorflow.serving.InceptionResponse.scores', index=1, - number=2, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=None), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=91, - serialized_end=143, -) - -DESCRIPTOR.message_types_by_name['InceptionRequest'] = _INCEPTIONREQUEST -DESCRIPTOR.message_types_by_name['InceptionResponse'] = _INCEPTIONRESPONSE - -InceptionRequest = _reflection.GeneratedProtocolMessageType('InceptionRequest', (_message.Message,), dict( - DESCRIPTOR = _INCEPTIONREQUEST, - __module__ = 'inception_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.InceptionRequest) - )) -_sym_db.RegisterMessage(InceptionRequest) - -InceptionResponse = _reflection.GeneratedProtocolMessageType('InceptionResponse', (_message.Message,), dict( - DESCRIPTOR = _INCEPTIONRESPONSE, - __module__ = 'inception_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.InceptionResponse) - )) -_sym_db.RegisterMessage(InceptionResponse) - - -import abc -import six -from grpc.beta import implementations as beta_implementations -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities - -class BetaInceptionServiceServicer(six.with_metaclass(abc.ABCMeta, object)): - """""" - @abc.abstractmethod - def Classify(self, request, context): - raise NotImplementedError() - -class BetaInceptionServiceStub(six.with_metaclass(abc.ABCMeta, object)): - """The interface to which stubs will conform.""" - @abc.abstractmethod - def Classify(self, request, timeout): - raise NotImplementedError() - Classify.future = None - -def beta_create_InceptionService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import inception_inference_pb2 - import inception_inference_pb2 - request_deserializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionRequest.FromString, - } - response_serializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionResponse.SerializeToString, - } - method_implementations = { - ('tensorflow.serving.InceptionService', 'Classify'): face_utilities.unary_unary_inline(servicer.Classify), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_InceptionService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import inception_inference_pb2 - import inception_inference_pb2 - request_serializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionRequest.SerializeToString, - } - response_deserializers = { - ('tensorflow.serving.InceptionService', 'Classify'): inception_inference_pb2.InceptionResponse.FromString, - } - cardinalities = { - 'Classify': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.InceptionService', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index ab43bcc8bf8..f74ba6eb7c6 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -15,11 +15,10 @@ #!/usr/bin/env python2.7 -"""A client that talks to mnist_inference service. +"""A client that talks to tensorflow_model_server loaded with mnist model. The client downloads test images of mnist data set, queries the service with -such test images to get classification, and calculates the inference error rate. -Please see mnist_inference.proto for details. +such test images to get predictions, and calculates the inference error rate. Typical usage example: @@ -35,7 +34,8 @@ import numpy import tensorflow as tf -from tensorflow_serving.example import mnist_inference_pb2 +from tensorflow_serving.apis import predict_pb2 +from tensorflow_serving.apis import prediction_service_pb2 from tensorflow_serving.example import mnist_input_data @@ -65,7 +65,7 @@ def do_inference(hostport, work_dir, concurrency, num_tests): test_data_set = mnist_input_data.read_data_sets(work_dir).test host, port = hostport.split(':') channel = implementations.insecure_channel(host, int(port)) - stub = mnist_inference_pb2.beta_create_MnistService_stub(channel) + stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) cv = threading.Condition() result = {'active': 0, 'error': 0, 'done': 0} def done(result_future, label): @@ -81,7 +81,7 @@ def done(result_future, label): else: sys.stdout.write('.') sys.stdout.flush() - response = numpy.array(result_future.result().value) + response = numpy.array(result_future.result().outputs['scores']) prediction = numpy.argmax(response) if label != prediction: result['error'] += 1 @@ -89,15 +89,16 @@ def done(result_future, label): result['active'] -= 1 cv.notify() for _ in range(num_tests): - request = mnist_inference_pb2.MnistRequest() + request = predict_pb2.PredictRequest() + request.model_spec.name = 'mnist' image, label = test_data_set.next_batch(1) - for pixel in image[0]: - request.image_data.append(pixel.item()) + request.inputs['images'].CopyFrom( + tf.contrib.util.make_tensor_proto(image[0], shape=[1, image[0].size])) with cv: while result['active'] == concurrency: cv.wait() result['active'] += 1 - result_future = stub.Classify.future(request, 5.0) # 5 seconds + result_future = stub.Predict.future(request, 5.0) # 5 seconds result_future.add_done_callback( lambda result_future, l=label[0]: done(result_future, l)) # pylint: disable=cell-var-from-loop with cv: diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index a4d22656b56..9aa82bd0c41 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -19,7 +19,8 @@ The model is from the TensorFlow "MNIST For ML Beginner" tutorial. This program simply follows all its training instructions, and uses TensorFlow Serving -exporter to export the trained model. +exporter to export the trained model with proper signatures that can be +loaded by standard tensorflow_model_server. Usage: mnist_export.py [--training_iteration=x] [--export_version=y] export_dir """ @@ -82,9 +83,11 @@ def main(_): print 'Exporting trained model to', export_path saver = tf.train.Saver(sharded=True) model_exporter = exporter.Exporter(saver) - signature = exporter.classification_signature(input_tensor=x, scores_tensor=y) - model_exporter.init(sess.graph.as_graph_def(), - default_graph_signature=signature) + model_exporter.init( + sess.graph.as_graph_def(), + named_graph_signatures={ + 'inputs': exporter.generic_signature({'images': x}), + 'outputs': exporter.generic_signature({'scores': y})}) model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess) print 'Done exporting!' diff --git a/tensorflow_serving/example/mnist_inference.cc b/tensorflow_serving/example/mnist_inference.cc deleted file mode 100644 index ec2631a9183..00000000000 --- a/tensorflow_serving/example/mnist_inference.cc +++ /dev/null @@ -1,219 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A gRPC server that classifies images into digit 0-9. -// Given each request with an image pixels encoded as floats, the server -// responds with 10 float values as probabilities for digit 0-9 respectively. -// The classification is done by running image data through a simple softmax -// regression network trained and exported by mnist_export.py. -// The intention of this example to demonstrate usage of Tensorflow -// APIs in an end-to-end scenario. - -#include -#include -#include -#include -#include - -#include "grpc++/security/server_credentials.h" -#include "grpc++/server.h" -#include "grpc++/server_builder.h" -#include "grpc++/server_context.h" -#include "grpc++/support/status.h" -#include "grpc++/support/status_code_enum.h" -#include "grpc/grpc.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_shape.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow/core/public/session_options.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow_serving/example/mnist_inference.grpc.pb.h" -#include "tensorflow_serving/example/mnist_inference.pb.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_factory.h" - -using grpc::InsecureServerCredentials; -using grpc::Server; -using grpc::ServerBuilder; -using grpc::ServerContext; -using grpc::Status; -using grpc::StatusCode; -using tensorflow::serving::ClassificationSignature; -using tensorflow::serving::MnistRequest; -using tensorflow::serving::MnistResponse; -using tensorflow::serving::MnistService; -using tensorflow::serving::BatchingParameters; -using tensorflow::serving::SessionBundle; -using tensorflow::serving::SessionBundleConfig; -using tensorflow::serving::SessionBundleFactory; -using tensorflow::string; -using tensorflow::Tensor; -using tensorflow::TensorShape; - -namespace { -const int kImageSize = 28; -const int kNumChannels = 1; -const int kImageDataSize = kImageSize * kImageSize * kNumChannels; -const int kNumLabels = 10; - -// Creates a gRPC Status from a TensorFlow Status. -Status ToGRPCStatus(const tensorflow::Status& status) { - return Status(static_cast(status.code()), - status.error_message()); -} - -class MnistServiceImpl final : public MnistService::Service { - public: - explicit MnistServiceImpl(std::unique_ptr bundle) - : bundle_(std::move(bundle)) { - signature_status_ = tensorflow::serving::GetClassificationSignature( - bundle_->meta_graph_def, &signature_); - } - - Status Classify(ServerContext* context, const MnistRequest* request, - MnistResponse* response) override { - // Verify protobuf input. - if (request->image_data_size() != kImageDataSize) { - return Status(StatusCode::INVALID_ARGUMENT, - tensorflow::strings::StrCat("expected image_data of size ", - kImageDataSize, ", got ", - request->image_data_size())); - } - - // Transform protobuf input to inference input tensor and create - // output tensor placeholder. - // See mnist_export.py for details. - Tensor input(tensorflow::DT_FLOAT, {1, kImageDataSize}); - std::copy_n(request->image_data().begin(), kImageDataSize, - input.flat().data()); - std::vector outputs; - - // Run inference. - if (!signature_status_.ok()) { - return ToGRPCStatus(signature_status_); - } - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - const tensorflow::Status status = bundle_->session->Run( - {{signature_.input().tensor_name(), input}}, - {signature_.scores().tensor_name()}, {}, &outputs); - if (!status.ok()) { - return ToGRPCStatus(status); - } - - // Transform inference output tensor to protobuf output. - // See mnist_export.py for details. - if (outputs.size() != 1) { - return Status(StatusCode::INTERNAL, - tensorflow::strings::StrCat( - "expected one model output, got ", outputs.size())); - } - const Tensor& score_tensor = outputs[0]; - const TensorShape expected_shape({1, kNumLabels}); - if (!score_tensor.shape().IsSameSize(expected_shape)) { - return Status( - StatusCode::INTERNAL, - tensorflow::strings::StrCat("expected output of size ", - expected_shape.DebugString(), ", got ", - score_tensor.shape().DebugString())); - } - const auto score_flat = outputs[0].flat(); - for (int i = 0; i < score_flat.size(); ++i) { - response->add_value(score_flat(i)); - } - - return Status::OK; - } - - private: - std::unique_ptr bundle_; - tensorflow::Status signature_status_; - ClassificationSignature signature_; -}; - -void RunServer(int port, std::unique_ptr bundle) { - // "0.0.0.0" is the way to listen on localhost in gRPC. - const string server_address = "0.0.0.0:" + std::to_string(port); - MnistServiceImpl service(std::move(bundle)); - ServerBuilder builder; - std::shared_ptr creds = InsecureServerCredentials(); - builder.AddListeningPort(server_address, creds); - builder.RegisterService(&service); - std::unique_ptr server(builder.BuildAndStart()); - LOG(INFO) << "Running..."; - server->Wait(); -} - -} // namespace - -int main(int argc, char** argv) { - tensorflow::int32 port = 0; - const bool parse_result = - tensorflow::ParseFlags(&argc, argv, {tensorflow::Flag("port", &port)}); - if (!parse_result) { - LOG(FATAL) << "Error parsing command line flags."; - } - - if (argc != 2) { - LOG(FATAL) << "Usage: mnist_inference --port=9000 /path/to/export"; - } - const string bundle_path(argv[1]); - - tensorflow::port::InitMain(argv[0], &argc, &argv); - - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - - SessionBundleConfig session_bundle_config; - - ////// - // Request batching, keeping default values for the tuning parameters. - // - // (If you prefer to disable batching, simply omit the following lines of code - // such that session_bundle_config.batching_parameters remains unset.) - BatchingParameters* batching_parameters = - session_bundle_config.mutable_batching_parameters(); - batching_parameters->mutable_thread_pool_name()->set_value( - "mnist_service_batch_threads"); - // Use a very large queue, to avoid rejecting requests. (Note: a production - // server with load balancing may want to use the default, much smaller, - // value.) - batching_parameters->mutable_max_enqueued_batches()->set_value(1000); - ////// - - std::unique_ptr bundle_factory; - TF_QCHECK_OK( - SessionBundleFactory::Create(session_bundle_config, &bundle_factory)); - std::unique_ptr bundle(new SessionBundle); - TF_QCHECK_OK(bundle_factory->CreateSessionBundle(bundle_path, &bundle)); - - // END WARNING(break-tutorial-inline-code) - - RunServer(port, std::move(bundle)); - - return 0; -} diff --git a/tensorflow_serving/example/mnist_inference.proto b/tensorflow_serving/example/mnist_inference.proto deleted file mode 100644 index 54e9a5e1485..00000000000 --- a/tensorflow_serving/example/mnist_inference.proto +++ /dev/null @@ -1,24 +0,0 @@ -// Protobuf definition of MNIST model inference server. -// The server classifies 28x28 greyscale image into digit 0-9. -// See mnist_export.py for model details. -// See mnist_inference.cc for server implementation. - -syntax = "proto3"; - -package tensorflow.serving; - -message MnistRequest { - // Row-major encoding of a 28x28 image, each byte being the grayscale value - // of a pixel rescaled from [0, 255] down to [-0.5, 0.5]. - repeated float image_data = 1 [packed = true]; -}; - -message MnistResponse { - // Matching probability of digit 0-9 in range [0.0, 1.0]. - repeated float value = 1 [packed = true]; -}; - -service MnistService { - // Classifies image into digits. - rpc Classify(MnistRequest) returns (MnistResponse); -} diff --git a/tensorflow_serving/example/mnist_inference_2.cc b/tensorflow_serving/example/mnist_inference_2.cc deleted file mode 100644 index b6d003028fd..00000000000 --- a/tensorflow_serving/example/mnist_inference_2.cc +++ /dev/null @@ -1,432 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// A gRPC server that classifies images into digit 0-9. -// Given each request with an image pixels encoded as floats, the server -// responds with 10 float values as probabilities for digit 0-9 respectively. -// The classification is done by running image data through a convolutional -// network trained and exported by mnist_model.py. -// The server constantly monitors a file system storage path for models. -// Whenever a new version of model is available, it eagerly unloads older -// version before loading the new one. The server also batches multiple -// requests together and does batched inference for efficiency. -// The intention of this example to demonstrate usage of DymanicManager, -// VersionPolicy and BasicBatchScheduler. - -#include -#include -#include -#include -#include - -#include "grpc++/completion_queue.h" -#include "grpc++/security/server_credentials.h" -#include "grpc++/server.h" -#include "grpc++/server_builder.h" -#include "grpc++/server_context.h" -#include "grpc++/support/async_unary_call.h" -#include "grpc++/support/status.h" -#include "grpc++/support/status_code_enum.h" -#include "grpc/grpc.h" -#include "tensorflow/contrib/session_bundle/manifest.pb.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_types.h" -#include "tensorflow/core/framework/types.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/util/command_line_flags.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" -#include "tensorflow_serving/batching/batch_scheduler.h" -#include "tensorflow_serving/core/manager.h" -#include "tensorflow_serving/core/servable_handle.h" -#include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/example/mnist_inference.grpc.pb.h" -#include "tensorflow_serving/example/mnist_inference.pb.h" -#include "tensorflow_serving/servables/tensorflow/simple_servers.h" - -using grpc::InsecureServerCredentials; -using grpc::Server; -using grpc::ServerAsyncResponseWriter; -using grpc::ServerBuilder; -using grpc::ServerContext; -using grpc::ServerCompletionQueue; -using grpc::Status; -using grpc::StatusCode; -using tensorflow::serving::MnistRequest; -using tensorflow::serving::MnistResponse; -using tensorflow::serving::MnistService; -using tensorflow::string; -using tensorflow::Tensor; -using tensorflow::serving::ClassificationSignature; - -namespace { -const int kImageSize = 28; -const int kNumChannels = 1; -const int kImageDataSize = kImageSize * kImageSize * kNumChannels; -const int kNumLabels = 10; - -class MnistServiceImpl; - -// Class encompassing the state and logic needed to serve a request. -class CallData { - public: - CallData(MnistServiceImpl* service_impl, - MnistService::AsyncService* service, - ServerCompletionQueue* cq); - - void Proceed(); - - void Finish(Status status); - - const MnistRequest& request() { return request_; } - - MnistResponse* mutable_response() { return &response_; } - - private: - // Service implementation. - MnistServiceImpl* service_impl_; - - // The means of communication with the gRPC runtime for an asynchronous - // server. - MnistService::AsyncService* service_; - // The producer-consumer queue where for asynchronous server notifications. - ServerCompletionQueue* cq_; - // Context for the rpc, allowing to tweak aspects of it such as the use - // of compression, authentication, as well as to send metadata back to the - // client. - ServerContext ctx_; - - // What we get from the client. - MnistRequest request_; - // What we send back to the client. - MnistResponse response_; - - // The means to get back to the client. - ServerAsyncResponseWriter responder_; - - // Let's implement a tiny state machine with the following states. - enum CallStatus { CREATE, PROCESS, FINISH }; - CallStatus status_; // The current serving state. -}; - -// A Task holds all of the information for a single inference request. -struct Task : public tensorflow::serving::BatchTask { - ~Task() override = default; - size_t size() const override { return 1; } - - Task(CallData* calldata_arg) - : calldata(calldata_arg) {} - - CallData* calldata; -}; - -class MnistServiceImpl final { - public: - MnistServiceImpl(const string& servable_name, - std::unique_ptr manager); - - void Classify(CallData* call_data); - - // Produces classifications for a batch of requests and associated responses. - void DoClassifyInBatch( - std::unique_ptr> batch); - - // Name of the servable to use for inference. - const string servable_name_; - // Manager in charge of loading and unloading servables. - std::unique_ptr manager_; - // A scheduler for batching multiple request calls into single calls to - // Session->Run(). - std::unique_ptr> - batch_scheduler_; -}; - -// Take in the "service" instance (in this case representing an asynchronous -// server) and the completion queue "cq" used for asynchronous communication -// with the gRPC runtime. -CallData::CallData(MnistServiceImpl* service_impl, - MnistService::AsyncService* service, - ServerCompletionQueue* cq) - : service_impl_(service_impl), - service_(service), cq_(cq), responder_(&ctx_), status_(CREATE) { - // Invoke the serving logic right away. - Proceed(); -} - -void CallData::Proceed() { - if (status_ == CREATE) { - // As part of the initial CREATE state, we *request* that the system - // start processing Classify requests. In this request, "this" acts are - // the tag uniquely identifying the request (so that different CallData - // instances can serve different requests concurrently), in this case - // the memory address of this CallData instance. - service_->RequestClassify(&ctx_, &request_, &responder_, cq_, cq_, this); - // Make this instance progress to the PROCESS state. - status_ = PROCESS; - } else if (status_ == PROCESS) { - // Spawn a new CallData instance to serve new clients while we process - // the one for this CallData. The instance will deallocate itself as - // part of its FINISH state. - new CallData(service_impl_, service_, cq_); - // Start processing. - service_impl_->Classify(this); - } else { - GPR_ASSERT(status_ == FINISH); - // Once in the FINISH state, deallocate ourselves (CallData). - delete this; - } -} - -void CallData::Finish(Status status) { - status_ = FINISH; - responder_.Finish(response_, status, this); -} - -MnistServiceImpl::MnistServiceImpl( - const string& servable_name, - std::unique_ptr manager) - : servable_name_(servable_name), manager_(std::move(manager)) { - // Setup a batcher used to combine multiple requests (tasks) into a single - // graph run for efficiency. - // The batcher queues tasks until, - // (a) the next task would cause the batch to exceed the size target; - // (b) waiting for more tasks to be added would exceed the timeout. - // at which point it processes the entire batch. - // - // Use the default batch-size, timeout and thread options. In general - // the numbers are extremely performance critical and should be tuned based - // specific graph structure and usage. - tensorflow::serving::BasicBatchScheduler::Options scheduler_options; - scheduler_options.thread_pool_name = "mnist_service_batch_threads"; - // Use a very large queue, to avoid rejecting requests. (Note: a production - // server with load balancing may want to use the default, much smaller, - // value.) - scheduler_options.max_enqueued_batches = 1000; - TF_CHECK_OK(tensorflow::serving::BasicBatchScheduler::Create( - scheduler_options, - [this](std::unique_ptr> batch) { - this->DoClassifyInBatch(std::move(batch)); - }, - &batch_scheduler_)); -} - -// Creates a gRPC Status from a TensorFlow Status. -Status ToGRPCStatus(const tensorflow::Status& status) { - return Status(static_cast(status.code()), - status.error_message()); -} - -// WARNING(break-tutorial-inline-code): The following code snippet is -// in-lined in tutorials, please update tutorial documents accordingly -// whenever code changes. -void MnistServiceImpl::Classify(CallData* calldata) { - // Verify input. - if (calldata->request().image_data_size() != kImageDataSize) { - calldata->Finish( - Status(StatusCode::INVALID_ARGUMENT, - tensorflow::strings::StrCat( - "expected image_data of size ", kImageDataSize, - ", got ", calldata->request().image_data_size()))); - return; - } - - // Create and submit a task to the batch scheduler. - std::unique_ptr task(new Task(calldata)); - tensorflow::Status status = batch_scheduler_->Schedule(&task); - - if (!status.ok()) { - calldata->Finish(ToGRPCStatus(status)); - return; - } -} - -// Produces classifications for a batch of requests and associated responses. -void MnistServiceImpl::DoClassifyInBatch( - std::unique_ptr> batch) { - batch->WaitUntilClosed(); - if (batch->empty()) { - return; - } - const int batch_size = batch->num_tasks(); - - // Replies to each task with the given error status. - auto complete_with_error = [&batch](StatusCode code, const string& msg) { - Status status(code, msg); - for (int i = 0; i < batch->num_tasks(); i++) { - Task* task = batch->mutable_task(i); - task->calldata->Finish(status); - } - }; - - // Get a handle to the SessionBundle. The handle ensures the Manager does - // not reload this while it is in use. - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - auto handle_request = - tensorflow::serving::ServableRequest::Latest(servable_name_); - tensorflow::serving::ServableHandle - bundle; - const tensorflow::Status lookup_status = - manager_->GetServableHandle(handle_request, &bundle); - if (!lookup_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - lookup_status.error_message()); - return; - } - - // Get the default signature of the graph. Expected to be a - // classification signature. - tensorflow::serving::ClassificationSignature signature; - const tensorflow::Status signature_status = - GetClassificationSignature(bundle->meta_graph_def, &signature); - if (!signature_status.ok()) { - complete_with_error(StatusCode::INTERNAL, - signature_status.error_message()); - return; - } - - // Transform protobuf input to inference input tensor. - // See mnist_model.py for details. - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - Tensor input(tensorflow::DT_FLOAT, {batch_size, kImageDataSize}); - auto dst = input.flat_outer_dims().data(); - for (int i = 0; i < batch_size; ++i) { - std::copy_n( - batch->mutable_task(i)->calldata->request().image_data().begin(), - kImageDataSize, dst); - dst += kImageDataSize; - } - - // Run classification. - tensorflow::Tensor scores; - const tensorflow::Status run_status = - RunClassification(signature, input, bundle->session.get(), - nullptr /* classes */, &scores); - if (!run_status.ok()) { - complete_with_error(StatusCode::INTERNAL, run_status.error_message()); - return; - } - if (scores.dtype() != tensorflow::DT_FLOAT) { - complete_with_error( - StatusCode::INTERNAL, - tensorflow::strings::StrCat( - "Expected output Tensor of DT_FLOAT. Got: ", - tensorflow::DataType_Name(scores.dtype()))); - return; - } - if (scores.dim_size(1) != kNumLabels) { - complete_with_error( - StatusCode::INTERNAL, - tensorflow::strings::StrCat( - "Expected ", kNumLabels, " labels in each output. Got: ", - scores.dim_size(1))); - return; - } - - // Transform inference output tensor to protobuf output. - // See mnist_model.py for details. - const auto& scores_mat = scores.matrix(); - for (int i = 0; i < batch_size; ++i) { - auto calldata = batch->mutable_task(i)->calldata; - for (int c = 0; c < scores.dim_size(1); ++c) { - calldata->mutable_response()->add_value(scores_mat(i, c)); - } - calldata->Finish(Status::OK); - } -} - -void HandleRpcs(MnistServiceImpl* service_impl, - MnistService::AsyncService* service, - ServerCompletionQueue* cq) { - // Spawn a new CallData instance to serve new clients. - new CallData(service_impl, service, cq); - void* tag; // uniquely identifies a request. - bool ok; - while (true) { - // Block waiting to read the next event from the completion queue. The - // event is uniquely identified by its tag, which in this case is the - // memory address of a CallData instance. - cq->Next(&tag, &ok); - GPR_ASSERT(ok); - static_cast(tag)->Proceed(); - } -} - -// Runs MnistService server until shutdown. -void RunServer(const int port, const string& servable_name, - std::unique_ptr manager) { - // "0.0.0.0" is the way to listen on localhost in gRPC. - const string server_address = "0.0.0.0:" + std::to_string(port); - - MnistService::AsyncService service; - ServerBuilder builder; - std::shared_ptr creds = InsecureServerCredentials(); - builder.AddListeningPort(server_address, creds); - builder.RegisterService(&service); - std::unique_ptr cq = builder.AddCompletionQueue(); - std::unique_ptr server(builder.BuildAndStart()); - LOG(INFO) << "Running..."; - - MnistServiceImpl service_impl(servable_name, std::move(manager)); - HandleRpcs(&service_impl, &service, cq.get()); -} - -} // namespace - -int main(int argc, char** argv) { - // Parse command-line options. - tensorflow::int32 port = 0; - const bool parse_result = - tensorflow::ParseFlags(&argc, argv, {tensorflow::Flag("port", &port)}); - if (!parse_result) { - LOG(FATAL) << "Error parsing command line flags."; - } - if (argc != 2) { - LOG(FATAL) << "Usage: mnist_inference_2 --port=9000 /path/to/exports"; - } - const string export_base_path(argv[1]); - tensorflow::port::InitMain(argv[0], &argc, &argv); - - // WARNING(break-tutorial-inline-code): The following code snippet is - // in-lined in tutorials, please update tutorial documents accordingly - // whenever code changes. - std::unique_ptr manager; - tensorflow::Status status = tensorflow::serving::simple_servers:: - CreateSingleTFModelManagerFromBasePath(export_base_path, &manager); - - TF_CHECK_OK(status) << "Error creating manager"; - - // Wait until at least one model is loaded. - std::vector ready_ids; - // TODO(b/25545573): Create a more streamlined startup mechanism than polling. - do { - LOG(INFO) << "Waiting for models to be loaded..."; - tensorflow::Env::Default()->SleepForMicroseconds(1 * 1000 * 1000 /*1 sec*/); - ready_ids = manager->ListAvailableServableIds(); - } while (ready_ids.empty()); - - // Run the service. - RunServer(port, ready_ids[0].name, std::move(manager)); - - return 0; -} diff --git a/tensorflow_serving/example/mnist_inference_pb2.py b/tensorflow_serving/example/mnist_inference_pb2.py deleted file mode 100644 index 2c5354c1b97..00000000000 --- a/tensorflow_serving/example/mnist_inference_pb2.py +++ /dev/null @@ -1,178 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -# Generated by the protocol buffer compiler. DO NOT EDIT! -# source: mnist_inference.proto - -import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) -from google.protobuf import descriptor as _descriptor -from google.protobuf import message as _message -from google.protobuf import reflection as _reflection -from google.protobuf import symbol_database as _symbol_database -from google.protobuf import descriptor_pb2 -# @@protoc_insertion_point(imports) - -_sym_db = _symbol_database.Default() - - - - -DESCRIPTOR = _descriptor.FileDescriptor( - name='mnist_inference.proto', - package='tensorflow.serving', - syntax='proto3', - serialized_pb= - _b('\n\x15mnist_inference.proto\x12\x12tensorflow.serving\"&\n\x0cMnistRequest\x12\x16\n\nimage_data\x18\x01 \x03(\x02\x42\x02\x10\x01\"\"\n\rMnistResponse\x12\x11\n\x05value\x18\x01 \x03(\x02\x42\x02\x10\x01\x32_\n\x0cMnistService\x12O\n\x08\x43lassify\x12 .tensorflow.serving.MnistRequest\x1a!.tensorflow.serving.MnistResponseb\x06proto3')) -_sym_db.RegisterFileDescriptor(DESCRIPTOR) - - - - -_MNISTREQUEST = _descriptor.Descriptor( - name='MnistRequest', - full_name='tensorflow.serving.MnistRequest', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='image_data', full_name='tensorflow.serving.MnistRequest.image_data', index=0, - number=1, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), _b('\020\001'))), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=45, - serialized_end=83, -) - - -_MNISTRESPONSE = _descriptor.Descriptor( - name='MnistResponse', - full_name='tensorflow.serving.MnistResponse', - filename=None, - file=DESCRIPTOR, - containing_type=None, - fields=[ - _descriptor.FieldDescriptor( - name='value', full_name='tensorflow.serving.MnistResponse.value', index=0, - number=1, type=2, cpp_type=6, label=3, - has_default_value=False, default_value=[], - message_type=None, enum_type=None, containing_type=None, - is_extension=False, extension_scope=None, - options=_descriptor._ParseOptions(descriptor_pb2.FieldOptions(), _b('\020\001'))), - ], - extensions=[ - ], - nested_types=[], - enum_types=[ - ], - options=None, - is_extendable=False, - syntax='proto3', - extension_ranges=[], - oneofs=[ - ], - serialized_start=85, - serialized_end=119, -) - -DESCRIPTOR.message_types_by_name['MnistRequest'] = _MNISTREQUEST -DESCRIPTOR.message_types_by_name['MnistResponse'] = _MNISTRESPONSE - -MnistRequest = _reflection.GeneratedProtocolMessageType('MnistRequest', (_message.Message,), dict( - DESCRIPTOR = _MNISTREQUEST, - __module__ = 'mnist_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.MnistRequest) - )) -_sym_db.RegisterMessage(MnistRequest) - -MnistResponse = _reflection.GeneratedProtocolMessageType('MnistResponse', (_message.Message,), dict( - DESCRIPTOR = _MNISTRESPONSE, - __module__ = 'mnist_inference_pb2' - # @@protoc_insertion_point(class_scope:tensorflow.serving.MnistResponse) - )) -_sym_db.RegisterMessage(MnistResponse) - - -_MNISTREQUEST.fields_by_name['image_data'].has_options = True -_MNISTREQUEST.fields_by_name['image_data']._options = _descriptor._ParseOptions( - descriptor_pb2.FieldOptions(), _b('\020\001')) -_MNISTRESPONSE.fields_by_name['value'].has_options = True -_MNISTRESPONSE.fields_by_name['value']._options = _descriptor._ParseOptions( - descriptor_pb2.FieldOptions(), _b('\020\001')) -import abc -from grpc.beta import implementations as beta_implementations -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities - -class BetaMnistServiceServicer(object): - """""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def Classify(self, request, context): - raise NotImplementedError() - -class BetaMnistServiceStub(object): - """The interface to which stubs will conform.""" - __metaclass__ = abc.ABCMeta - @abc.abstractmethod - def Classify(self, request, timeout): - raise NotImplementedError() - Classify.future = None - -def beta_create_MnistService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - import mnist_inference_pb2 - import mnist_inference_pb2 - request_deserializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistRequest.FromString, - } - response_serializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistResponse.SerializeToString, - } - method_implementations = { - ('tensorflow.serving.MnistService', 'Classify'): face_utilities.unary_unary_inline(servicer.Classify), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - -def beta_create_MnistService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - import mnist_inference_pb2 - import mnist_inference_pb2 - request_serializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistRequest.SerializeToString, - } - response_deserializers = { - ('tensorflow.serving.MnistService', 'Classify'): mnist_inference_pb2.MnistResponse.FromString, - } - cardinalities = { - 'Classify': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.MnistService', cardinalities, options=stub_options) -# @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 2c3eb50e376..c99413b608a 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -1,15 +1,10 @@ -# Serving Dynamically Updated TensorFlow Model with Asynchronous Batching - -This tutorial shows you how to use TensorFlow Serving components to build a -server that dynamically discovers and serves new versions of a trained -TensorFlow model. You'll also learn how to use TensorFlow Serving's more -flexible, lower-level batch scheduling API. One advantage of the lower-level API -is its asynchronous behavior, which allows you to reduce the number of client -threads and thus use less memory without compromising throughput. The code -examples in this tutorial focus on the discovery, asynchronous batching, and -serving logic. If you just want to use TensorFlow Serving to serve a single -version model, and are fine with synchronous batching (relying on many client -threads that block), see [TensorFlow Serving basic tutorial](serving_basic.md). +# Building Standard TensorFlow Model Server + +This tutorial shows you how to use TensorFlow Serving components to build the +standard TensorFlow model server that dynamically discovers and serves new +versions of a trained TensorFlow model. If you just want to use the standard +server to serve your models, see +[TensorFlow Serving basic tutorial](serving_basic.md). This tutorial uses the simple Softmax Regression model introduced in the TensorFlow tutorial for handwritten image (MNIST data) classification. If you @@ -24,16 +19,16 @@ The code for this tutorial consists of two parts: that trains and exports multiple versions of the model. * A C++ file - [mnist_inference_2.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_inference_2.cc) - that discovers new exported models and runs a [gRPC](http://www.grpc.io) - service for serving them. + [main.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc) + which is the standard TensorFlow model server that discovers new exported + models and runs a [gRPC](http://www.grpc.io) service for serving them. This tutorial steps through the following tasks: 1. Train and export a TensorFlow model. - 2. Manage model versioning with TensorFlow Serving manager. - 3. Handle request batching with TensorFlow Serving batch scheduler. - 4. Serve request with TensorFlow Serving manager. + 2. Manage model versioning with TensorFlow Serving `ServerCore`. + 3. Configure batching using `SessionBundleSourceAdapterConfig`. + 4. Serve request with TensorFlow Serving `ServerCore`. 5. Run and test the service. Before getting started, please complete the [prerequisites](setup.md#prerequisites). @@ -73,7 +68,7 @@ $>ls /tmp/mnist_model 00000001 00000002 ~~~ -## Manager +## ServerCore Now imagine v1 and v2 of the model are dynamically generated at runtime, as new algorithms are being experimented with, or as the model is trained with a new @@ -87,38 +82,68 @@ good for minimizing resource usage (e.g. RAM). TensorFlow Serving `Manager` does exactly that. It handles the full lifecycle of TensorFlow models including loading, serving and unloading them as well as version transitions. In this tutorial, you will build your server on top of a -TensorFlow Serving `Manager` (see `mnist_inference_2.cc`). +TensorFlow Serving `ServerCore`, which internally wraps an +`AspiredVersionsManager`. ~~~c++ int main(int argc, char** argv) { ... - UniquePtrWithDeps manager; - tensorflow::Status status = tensorflow::serving::simple_servers:: - CreateSingleTFModelManagerFromBasePath(export_base_path, &manager); - - ... - - RunServer(FLAGS_port, ready_ids[0].name, std::move(manager)); + std::unique_ptr core; + TF_CHECK_OK(ServerCore::Create( + config, std::bind(CreateSourceAdapter, source_adapter_config, + std::placeholders::_1, std::placeholders::_2), + &CreateServableStateMonitor, &LoadDynamicModelConfig, &core)); + RunServer(port, std::move(core)); return 0; } ~~~ -`CreateSingleTFModelManagerFromBasePath()` internally does the following: - - * Instantiate a `FileSystemStoragePathSource` that monitors new files (model - export directory) in `export_base_path`. - * Instantiate a `SessionBundleSourceAdapter` that creates a new - `SessionBundle` for each new model export. - * Instantiate a specific implementation of `Manager` called - `AspiredVersionsManager` that manages all `SessionBundle` instances created by +`ServerCore::Create()` takes four parameters: + + * `ModelServerConfig` that specifies models to be loaded. Models are declared + either through `model_config_list`, which declares a static list of models, or + through `dynamic_model_config`, which declares a dynamic list of models that + may get updated at runtime. + * `SourceAdapterCreator` that creates the `SourceAdapter`, which adapts + `StoragePath` (the path where a model version is discovered) to model + `Loader` (loads the model version from storage path and provides state + transition interfaces to the `Manager`). In this case, `CreateSourceAdapter` + creates `SessionBundleSourceAdapter`, which we will explain later. + * `ServableStateMonitorCreator` that creates `ServableStateMonitor`, which + keeps track for `Servable` (model version) state transition and provides a + query interface to the user. In this case, `CreateServableStateMonitor` + creates the base `ServableStateMonitor`, which keeps track of servable states + in memory. You can extend it to add state tracking capabilities (e.g. persists + state change to disk, remote server, etc.) + * `DynamicModelConfigLoader` that loads models from `dynamic_model_config`. + The standard TensorFlow model server supports only `model_config_list` for + now and therefore `LoadDynamicModelConfig` CHECK-fails when called. You can + extend it to add dynamic model discovery/loading capabilities (e.g. through + RPC, external service, etc.) + +`SessionBundle` is a key component of TensorFlow Serving. It represents a +TensorFlow model loaded from a given path and provides the same `Session::Run` +interface as TensorFlow to run inference. +`SessionBundleSourceAdapter` adapts storage path to `Loader` +so that model lifetime can be managed by `Manager`. + +With all these, `ServerCore` internally does the following: + + * Instantiates a `FileSystemStoragePathSource` that monitors model export + paths declared in `model_config_list`. + * Instantiates a `SourceAdapter` using the `SourceAdapterCreator` with the + model type declared in `model_config_list` and connects the + `FileSystemStoragePathSource` to it. This way, whenever a new model version is + discovered under the export path, the `SessionBundleSourceAdapter` adapts it + to a `Loader`. + * Instantiates a specific implementation of `Manager` called + `AspiredVersionsManager` that manages all such `Loader` instances created by the `SessionBundleSourceAdapter`. Whenever a new version is available, this `AspiredVersionsManager` always -unloads the old version and replaces it with the new one. Note that -`CreateSingleTFModelManagerFromBasePath()` intentionally lacks any config -parameters, because it is intended for a very first deployment. If you want to +unloads the old version and replaces it with the new one. If you want to start customizing, you are encouraged to understand the components that it creates internally, and how to configure them. @@ -140,42 +165,26 @@ Modern hardware accelerators (GPUs, etc.) used to do machine learning inference usually achieve best computation efficiency when inference requests are run in large batches. -TensorFlow Serving `BatchScheduler` provides such functionality. A specific -implementation of it -- `BasicBatchScheduler` -- enqueues tasks (requests) -until either of the following occur: - - * The next task would cause the batch to exceed the size target. - * Waiting for more tasks to be added would exceed the timeout. - -When either of these occur, the `BasicBatchScheduler` processes the entire -batch by executing a callback and passing it the current batch of tasks. - -Initializing `BasicBatchScheduler` is straightforward and is done in a -`MnistServiceImpl` constructor. A `BasicBatchScheduler::Options` is given to -configure the batch scheduler, and a callback is given to be executed when the -batch is full or timeout exceeds. - -The default `BasicBatchScheduler::Options` has batch size set to 32 and -timeout set to 10 milliseconds. These parameters are typically extremely -performance critical and should be tuned based on a specific model/scenario in -production. In this tutorial, you will not bother tuning them. - -For each incoming request, instead of immediately processing it, you always -submit a task, which encapsulates gRPC async call data, to `BatchScheduler`: +Batching can be turned on by providing proper `SessionBundleSourceAdapterConfig` +when creating the `SessionBundleSourceAdapter`. In this case we set the +`BatchingParameters` with pretty much default values. Batching can be fine-tuned +by setting custom timeout, batch_size, etc. values. For details, please refer +to `BatchingParameters`. ~~~c++ -void MnistServiceImpl::Classify(CallData* calldata) { - ... - - std::unique_ptr task(new Task(calldata)); - tensorflow::Status status = batch_scheduler_->Schedule(std::move(task)); +SessionBundleSourceAdapterConfig source_adapter_config; +// Batching config +if (enable_batching) { + BatchingParameters* batching_parameters = + source_adapter_config.mutable_config()->mutable_batching_parameters(); + batching_parameters->mutable_thread_pool_name()->set_value( + "model_server_batch_threads"); } ~~~ -Upon reaching timeout or full batch, the given callback `DoClassifyInBatch()` -will be executed. `DoClassifyInBatch()`, as we will explain later, merges tasks -into a single large tensor, and invokes a single `tensorflow::Session::Run()` -call (which is where the actual efficiency gain on GPUs comes from). +Upon reaching full batch, inference requests are merged internally into a +single large request (tensor), and `tensorflow::Session::Run()` is invoked +(which is where the actual efficiency gain on GPUs comes from). # Serve with Manager @@ -231,34 +240,12 @@ To put all these into the context of this tutorial: * `AspiredVersionsManager` monitors the export stream, and manages lifecycle of all SessionBundle` servables dynamically. -`DoClassifyInBatch` then just requests `SessionBundle` from the manager and uses -it to run inference. Most of the logic and flow is very similar to the logic and -flow described in the [TensorFlow Serving basic tutorial](serving_basic.md), -with just a few key changes: - - * The input tensor now has its first dimension set to variable batch size at - runtime, because you are batching multiple inference requests in a single - input. +`TensorflowPredictImpl::Predict` then just: -~~~c++ - Tensor input(tensorflow::DT_FLOAT, {batch_size, kImageDataSize}); -~~~ - - * You are calling `GetServableHandle` to request the `SessionBundle` of a - version of the model. With `ServableRequest::Latest()` you create a - `ServableRequest` that tells the `Manager` to return the latest version of the - servable for a given name. You can also specify a version. Note that - `SessionBundle` is wrapped in a `ServableHandle`. This is due to the fact - that the lifetime of `SessionBundle` is now managed by the `Manager`, - therefore a handle/reference is returned instead. - -~~~c++ - auto handle_request = - tensorflow::serving::ServableRequest::Latest(servable_name_); - tensorflow::serving::ServableHandle bundle; - const tensorflow::Status lookup_status = - manager_->GetServableHandle(handle_request, &bundle); -~~~ + * Requests `SessionBundle` from the manager (through ServerCore). + * Uses the `generic signatures` to map logical tensor names in `PredictRequest` + to real tensor names and bind values to tensors. + * Runs inference. ## Test and Run The Server @@ -268,8 +255,8 @@ server. ~~~shell $>mkdir /tmp/monitored $>cp -r /tmp/mnist_model/00000001 /tmp/monitored -$>bazel build //tensorflow_serving/example:mnist_inference_2 -$>bazel-bin/tensorflow_serving/example/mnist_inference_2 --port=9000 /tmp/monitored +$>bazel build //tensorflow_serving/model_servers/tensorflow_model_server +$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored ~~~ The server will emit log messages every one second that say diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 6a8a33be10f..516703da251 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -1,14 +1,9 @@ # Serving a TensorFlow Model This tutorial shows you how to use TensorFlow Serving components to export a -trained TensorFlow model and build a server to serve the exported model. The -server you will build in this tutorial is relatively simple: it serves a single -static TensorFlow model, it handles inference requests, and it calculates an -aggregate inference error rate. If you are already familiar with TensorFlow -Serving, and you want to create a more sophisticated server that handles -inference requests asynchronously (without blocking client threads), and -discovers and serves new versions of a TensorFlow model that is being -dynamically updated, see the +trained TensorFlow model and use the standard tensorflow_model_server to serve +it. If you are already familiar with TensorFlow Serving, and you want to know +more about how the server internals work, see the [TensorFlow Serving advanced tutorial](serving_advanced.md). This tutorial uses the simple Softmax Regression model introduced in the @@ -24,9 +19,9 @@ The code for this tutorial consists of two parts: that trains and exports the model. * A C++ file -([mnist_inference.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_inference.cc)) -that loads the exported model and runs a [gRPC](http://www.grpc.io) service to -serve it. +[main.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc) +which is the standard TensorFlow model server that discovers new exported +models and runs a [gRPC](http://www.grpc.io) service for serving them. Before getting started, please complete the [prerequisites](setup.md#prerequisites). @@ -48,9 +43,11 @@ export_path = sys.argv[-1] print 'Exporting trained model to', export_path saver = tf.train.Saver(sharded=True) model_exporter = exporter.Exporter(saver) -signature = exporter.classification_signature(input_tensor=x, scores_tensor=y) -model_exporter.init(sess.graph.as_graph_def(), - default_graph_signature=signature) +model_exporter.init( + sess.graph.as_graph_def(), + named_graph_signatures={ + 'inputs': exporter.generic_signature({'images': x}), + 'outputs': exporter.generic_signature({'scores': y})}) model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess) ~~~ @@ -68,22 +65,25 @@ export only the variables that will be used for inference. `export` will serialize the protobuf to the model export so that the TensorFlow graph can be properly restored later. - * `default_graph_signature=signature` specifies a model export **signature**. + * `named_graph_signatures=...` specifies a model export **signature**. Signature specifies what type of model is being exported, and the input/output tensors to bind to when running inference. In this case, you use - `exporter.classification_signature` to specify that the model is a - classification model: + `inputs` and `outputs` as keys for `exporter.generic_signature` as such a + signature is supported by the standard `tensorflow_model_server`: - * `input_tensor=x` specifies the input tensor binding. + * `{'images': x}` specifies the input tensor binding. - * `scores_tensor=y` specifies the scores tensor binding. + * `{'scores': y}` specifies the scores tensor binding. - * Typically, you should also override the `classes_tensor=None` argument to - specify class tensor binding. As an example, for a classification model that - returns the top 10 suggested videos, the output will consist of both videos - (classes) and scores for each video. In this case, however, the model always - returns Softmax scores for matching digit 0-9 in order. Therefore, - overriding `classes_tensor` is not necessary. + * `images` and `scores` are tensor alias names. They can be whatever + unique strings you want, and they will become the logical names of tensor + `x` and `y` that you refer to for tensor binding when sending prediction + requests later. For instance, if `x` refers to the tensor with name + 'long_tensor_name_foo' and `y` refers to the tensor with name + 'generated_tensor_name_bar', `exporter.generic_signature` will store + tensor logical name to real name mapping ('images' -> 'long_tensor_name_foo' + and 'scores' -> 'generated_tensor_name_bar') and allow user to refer to + these tensors with their logical names when running inference. `Exporter.export()` takes the following arguments: @@ -146,151 +146,11 @@ Each version sub-directory contains the following files: With that, your TensorFlow model is exported and ready to be loaded! -## Load Exported TensorFlow Model - -The C++ code for loading the exported TensorFlow model is in the `main()` -function in mnist_inference.cc, and is simple. The basic code is: - -~~~c++ -int main(int argc, char** argv) { - ... - - SessionBundleConfig session_bundle_config; - ... (ignoring batching for now; see below) - std::unique_ptr bundle_factory; - TF_QCHECK_OK( - SessionBundleFactory::Create(session_bundle_config, &bundle_factory)); - std::unique_ptr bundle(new SessionBundle); - TF_QCHECK_OK(bundle_factory->CreateSessionBundle(bundle_path, &bundle)); - ... - - RunServer(FLAGS_port, std::move(bundle)); - - return 0; -} -~~~ - -It uses the `SessionBundle` component of TensorFlow Serving. -`SessionBundleFactory::CreateSessionBundle()` loads an exported TensorFlow model -at the given path and creates a `SessionBundle` object for running inference -with the model. Typically, a default `tensorflow::SessionOptions` proto is given -when loading the model; a custom one can be passed via `SessionBundleConfig` if -desired. - -With a small amount of extra code, you can arrange for the server to batch -groups of inference requests together into larger tensors, which tends to -improve throughput, especially on GPUs. To enable batching you simply populate -the `BatchingParameters` sub-message of the `SessionBundleConfig`, like so: - -~~~c++ -int main(int argc, char** argv) { - ... - BatchingParameters* batching_parameters = - session_bundle_config.mutable_batching_parameters(); - batching_parameters->mutable_thread_pool_name()->set_value( - "mnist_service_batch_threads"); - // Use a very large queue, to avoid rejecting requests. - batching_parameters->mutable_max_enqueued_batches()->set_value(1000); - ... -} -~~~ - -This example sticks with default tuning parameters for batching; if you want to -adjust the maximum batch size, timeout threshold or the number of background -threads used for batched inference, you can do so by setting more values in -`BatchingParameters`. Note that the (simplified) batching API offered by -`SessionBundleFactory` requires a client thread to block while awaiting other -peer threads with which to form a batch -- gRPC promises to adjust the number of -client threads to keep things flowing smoothly. Lastly, the batcher's timeout -threshold bounds the amount of time a given request spends in the blocked state, -so a low request volume does not compromise latency. For more information about -batching, see the [Batching Guide](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/batching/README.md). - -Whether or not we enable batching, we wind up with a `SessionBundle`; let's look -at its definition in -[session_bundle.h](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/session_bundle/session_bundle.h): - -~~~c++ -struct SessionBundle { - std::unique_ptr session; - tensorflow::MetaGraphDef meta_graph_def; -}; -~~~ - -`session` is, guess what, a TensorFlow session that has the original graph -with the needed variables properly restored. In other words, the trained model -is now held in `session` and is ready for running inference! - -All you need to do now is bind inference input and output to the proper tensors -in the graph and invoke `session->run()`. But how do you know which tensors to -bind to? As you may have probably guessed, the answer is in the -`meta_graph_def`. - -`tensorflow::MetaGraphDef` is the protobuf de-serialized from the `export.meta` -file above (see [meta_graph.proto](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/core/protobuf/meta_graph.proto)). -We add all needed description and metadata of a TensorFlow model export to its -extensible `collection_def`. In particular, it contains `Signatures` (see -[manifest.proto](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/session_bundle/manifest.proto)) -that specifies the tensor to use. - -~~~proto -// Signatures of model export. -message Signatures { - // Default signature of the graph. - Signature default_signature = 1; - - // Named signatures of the graph. - map named_signatures = 2; -}; -~~~ - -Remember how we specified the model signature in `export` above? The -information eventually gets encoded here: - -~~~proto -message ClassificationSignature { - TensorBinding input = 1; - TensorBinding classes = 2; - TensorBinding scores = 3; -}; -~~~ - -`TensorBinding` contains the tensor name that can be used for `session->run()`. -With that, we can run inference given a `SessionBundle`! - -## Bring Up Inference Service - -As you can see in `mnist_inference.cc`, `RunServer` in `main` brings up a gRPC -server that exports a single `Classify()` API. The implementation of the method -is straightforward, as each inference request is processed in the following -steps: - - 1. Verify the input -- the server expects exactly one MNIST-format image for - each inference request. - - 2. Transform protobuf input to inference input tensor and create output - tensor placeholder. - - 3. Run inference. Note that in the `MnistServiceImpl` constructor you use - `GetClassificationSignature()` to extract the signature of the model from - 'meta_graph_def` and verify that it is a classification signature as expected. - With the extracted signature, the server can bind the input and output tensors - properly and run the session. - -~~~c++ -const tensorflow::Status status = - bundle_->session->Run({{signature_.input().tensor_name(), input}}, - {signature_.scores().tensor_name()}, {}, - &outputs); -~~~ - - 4. Transform the inference output tensor to protobuf output. - -To run it: +## Load Exported Model With Standard TensorFlow Model Server ~~~shell -$>bazel build //tensorflow_serving/example:mnist_inference -$>bazel-bin/tensorflow_serving/example/mnist_inference --port=9000 /tmp/mnist_model/00000001 +$>bazel build //tensorflow_serving/model_servers:tensorflow_model_server +$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ ~~~ ## Test The Server diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 0df67540a85..fe3245f3cf1 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -20,7 +20,7 @@ To learn more about TensorFlow Inception model, we recommend ## Part 0: Create a Docker image Please refer to [Using TensorFlow Serving via Docker](docker.md) for details -of building Tensorflow Serving Docker image. +about building a TensorFlow Serving Docker image. ### Run container @@ -33,10 +33,10 @@ $ docker build --pull -t $USER/tensorflow-serving-devel -f tensorflow_serving/to $ docker run --name=inception_container -it $USER/tensorflow-serving-devel ``` -### Clone, configure and build Tensorflow Serving in container +### Clone, configure, and build TensorFlow Serving in a container -In the running container, we clone, configure and build Tensorflow Serving. -Then test run [inception_inference.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/inception_inference.cc). +In the running container, we clone, configure and build TensorFlow Serving. +Then test run [tensorflow_model_server](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc). ```shell root@c97d8e820ced:/# git clone --recurse-submodules https://github.com/tensorflow/serving @@ -47,8 +47,8 @@ root@c97d8e820ced:/serving# bazel build -c opt tensorflow_serving/... root@c97d8e820ced:/serving# ls AUTHORS LICENSE RELEASE.md bazel-bin bazel-out bazel-testlogs tensorflow zlib.BUILD CONTRIBUTING.md README.md WORKSPACE bazel-genfiles bazel-serving grpc tensorflow_serving -root@c97d8e820ced:/serving# bazel-bin/tensorflow_serving/example/inception_inference -E tensorflow_serving/example/inception_inference.cc:362] Usage: inception_inference --port=9000 /path/to/exports +root@c97d8e820ced:/serving# bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server +Usage: model_server [--port=8500] [--enable_batching] [--model_name=my_name] --model_base_path=/path/to/export ``` ### Export Inception model in container @@ -95,11 +95,11 @@ $ docker run -it $USER/inception_serving ### Start the server -Run the [gRPC]( http://www.grpc.io/) server in the container. +Run the [gRPC]( http://www.grpc.io/) `tensorflow_model_server` in the container. ```shell root@f07eec53fd95:/# cd serving -root@f07eec53fd95:/serving# bazel-bin/tensorflow_serving/example/inception_inference --port=9000 inception-export &> inception_log & +root@f07eec53fd95:/serving# bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=inception --model_base_path=inception-export &> inception_log & [1] 45 ``` @@ -112,16 +112,44 @@ over gRPC for classification into human readable descriptions of the ```shell root@f07eec53fd95:/serving# bazel-bin/tensorflow_serving/example/inception_client --server=localhost:9000 --image=/path/to/my_cat_image.jpg -scores: 9.41664886475 -scores: 8.14928436279 -scores: 7.6565990448 -scores: 3.85941624641 -scores: 2.82326698303 -classes: "tiger cat" -classes: "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor" -classes: "Persian cat" -classes: "leopard, Panthera pardus" -classes: "Egyptian cat" +outputs { + key: "classes" + value { + dtype: DT_STRING + tensor_shape { + dim { + size: 1 + } + dim { + size: 5 + } + } + string_val: "tiger cat" + string_val: "Egyptian cat" + string_val: "tabby, tabby cat" + string_val: "lynx, catamount" + string_val: "Cardigan, Cardigan Welsh corgi" + } +} +outputs { + key: "scores" + value { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 5 + } + } + float_val: 9.5486907959 + float_val: 8.52025032043 + float_val: 8.05995368958 + float_val: 4.30645561218 + float_val: 3.93207240105 + } +} root@f07eec53fd95:/serving# exit ``` @@ -261,16 +289,44 @@ We can now query the service at its external address from our local host. ```shell $ bazel-bin/tensorflow_serving/example/inception_client --server=146.148.88.232:9000 --image=/path/to/my_cat_image.jpg -scores: 9.41664886475 -scores: 8.14928436279 -scores: 7.6565990448 -scores: 3.85941624641 -scores: 2.82326698303 -classes: "tiger cat" -classes: "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor" -classes: "Persian cat" -classes: "leopard, Panthera pardus" -classes: "Egyptian cat" +outputs { + key: "classes" + value { + dtype: DT_STRING + tensor_shape { + dim { + size: 1 + } + dim { + size: 5 + } + } + string_val: "tiger cat" + string_val: "Egyptian cat" + string_val: "tabby, tabby cat" + string_val: "lynx, catamount" + string_val: "Cardigan, Cardigan Welsh corgi" + } +} +outputs { + key: "scores" + value { + dtype: DT_FLOAT + tensor_shape { + dim { + size: 1 + } + dim { + size: 5 + } + } + float_val: 9.5486907959 + float_val: 8.52025032043 + float_val: 8.05995368958 + float_val: 4.30645561218 + float_val: 3.93207240105 + } +} ``` You have successfully deployed Inception model serving as a service in From e1d419b6e67e4e3d6b08ced238dd8f798ad00e86 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 9 Sep 2016 15:31:42 -0800 Subject: [PATCH 0003/8554] Remove the no longer needed BUILD target mnist_inference_proto. Change: 132730062 --- tensorflow_serving/example/BUILD | 8 -------- 1 file changed, 8 deletions(-) diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 0c63af1585f..d1718218dae 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -25,14 +25,6 @@ filegroup( ), ) -serving_proto_library( - name = "mnist_inference_proto", - srcs = ["mnist_inference.proto"], - has_services = 1, - cc_api_version = 2, - cc_grpc_version = 1, -) - py_library( name = "mnist_input_data", srcs = ["mnist_input_data.py"], From 9c347e568952218d5f9e6056445080d29eef3e6c Mon Sep 17 00:00:00 2001 From: Damien Martin-Guillerez Date: Fri, 26 Aug 2016 14:46:00 +0200 Subject: [PATCH 0004/8554] Make tensorflow new cuda auto-configuration works This is not a complete change since it requires to update the dependency to tensorflow itself. --- WORKSPACE | 9 +++++++-- tensorflow | 2 +- tensorflow_serving/workspace.bzl | 15 +++++---------- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 5a7e12f68ad..2bf7b11a735 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,14 @@ workspace(name = "tf_serving") +local_repository( + name = "org_tensorflow", + path = "tensorflow", +) + # Please add all new TensorFlow Serving dependencies in workspace.bzl. load('//tensorflow_serving:workspace.bzl', 'tf_serving_workspace') -tf_serving_workspace(__workspace_dir__) +tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") -check_version("0.3.0") \ No newline at end of file +check_version("0.3.0") diff --git a/tensorflow b/tensorflow index 83fe2a5b732..bb02c7069a7 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 83fe2a5b7328e1b754b53cbbcf9b313450a2f863 +Subproject commit bb02c7069a787bee79d73c31fcf55e5a29fabeac diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index cff062cf521..58b276f1f5b 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -1,28 +1,23 @@ # TensorFlow Serving external dependencies that can be loaded in WORKSPACE # files. -load('//tensorflow/tensorflow:workspace.bzl', 'tf_workspace') +load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') # All TensorFlow Serving external dependencies. # workspace_dir is the absolute path to the TensorFlow Serving repo. If linked # as a submodule, it'll likely be '__workspace_dir__ + "/serving"' -def tf_serving_workspace(workspace_dir): - native.local_repository( - name = "org_tensorflow", - path = workspace_dir + "/tensorflow", - ) - +def tf_serving_workspace(): native.local_repository( name = "inception_model", - path = workspace_dir + "/tf_models/inception", + path = "tf_models/inception", ) - tf_workspace("tensorflow/", "@org_tensorflow") + tf_workspace() # ===== gRPC dependencies ===== native.bind( name = "libssl", - actual = "@boringssl_git//:ssl", + actual = "@boringssl//:ssl", ) native.bind( From b511b23e4a70d8f6ac2c5afbf1627bc32e7d2292 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 14 Sep 2016 08:02:59 -0800 Subject: [PATCH 0005/8554] Clarify semantics of not using a load/unload thread pool in AspiredVersionsManager. Change: 133133104 --- tensorflow_serving/core/BUILD | 7 +------ tensorflow_serving/core/aspired_versions_manager.h | 4 ++-- tensorflow_serving/tools/docker/Dockerfile.devel | 4 ++-- tensorflow_serving/util/BUILD | 9 ++------- 4 files changed, 7 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index b7f16be192e..d96f2e4b281 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -34,7 +34,6 @@ cc_library( deps = [ "//tensorflow_serving/util:hash", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -56,7 +55,6 @@ cc_library( deps = [ ":servable_id", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -82,7 +80,6 @@ cc_library( "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/util:any_ptr", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -99,7 +96,6 @@ cc_library( "//tensorflow_serving/util:any_ptr", "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -274,7 +270,7 @@ cc_library( deps = [ ":loader", "//tensorflow_serving/util:any_ptr", - "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:lib", ], ) @@ -392,7 +388,6 @@ cc_library( ":servable_id", "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index f78e2300933..143d05fa8af 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -89,8 +89,8 @@ class AspiredVersionsManager : public Manager, // The number of threads in the thread-pool used to load and unload // servables. // - // If set as 0, we don't use a thread-pool, and the {Load,Unload}Servable() - // methods block. + // If set as 0, we don't use a thread-pool, and servable loads/unloads are + // performed serially in the manager's main work loop. uint32 num_load_unload_threads = 0; // Maximum number of times we retry loading a servable, after the first diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index bce69208c77..81c244c5c4d 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -28,8 +28,8 @@ RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ # Set up grpc -RUN pip install enum34 futures six && \ - pip install --pre protobuf>=3.0.0a3 && \ +RUN pip install enum34 futures mock six && \ + pip install --pre 'protobuf>=3.0.0a3' && \ pip install -i https://testpypi.python.org/simple --pre grpcio # Set up Bazel. diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index da21091121c..013179d3303 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -35,7 +35,6 @@ cc_library( visibility = ["//visibility:public"], deps = [ "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -54,7 +53,7 @@ cc_library( srcs = ["hash.cc"], hdrs = ["hash.h"], deps = [ - "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:lib", ], ) @@ -62,7 +61,7 @@ cc_library( name = "observer", hdrs = ["observer.h"], deps = [ - "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:lib", ], ) @@ -94,7 +93,6 @@ cc_library( hdrs = ["fast_read_dynamic_ptr.h"], deps = [ "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -128,7 +126,6 @@ cc_library( visibility = ["//visibility:public"], deps = [ "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -151,7 +148,6 @@ cc_library( hdrs = ["periodic_function.h"], deps = [ "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -182,7 +178,6 @@ cc_library( hdrs = ["cleanup.h"], deps = [ "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) From f25a49a29085b4b8aa54fcf4a644d0f9e5be9e59 Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Mon, 19 Sep 2016 18:03:23 -0400 Subject: [PATCH 0006/8554] Address issue with serving build errors on Bazel 0.3.0 (#183) Change check for Bazel 0.3.1 and update documentation. Relevant issues: https://github.com/tensorflow/tensorflow/issues/4343, https://github.com/tensorflow/tensorflow/issues/4319 --- WORKSPACE | 2 +- tensorflow_serving/g3doc/setup.md | 8 ++++---- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 2bf7b11a735..39f4bcd6e71 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,4 +11,4 @@ tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") -check_version("0.3.0") +check_version("0.3.1") diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 10a252a61ed..4f4667455c5 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -6,7 +6,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel -TensorFlow Serving requires Bazel 0.3.0 or higher. You can find the Bazel +TensorFlow Serving requires Bazel 0.3.1 or higher. You can find the Bazel installation instructions [here](http://bazel.io/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the @@ -14,13 +14,13 @@ following steps: 1. Download the relevant binary from [here](https://github.com/bazelbuild/bazel/releases). - Let's say you downloaded bazel-0.3.0-installer-linux-x86_64.sh. You would + Let's say you downloaded bazel-0.3.1-installer-linux-x86_64.sh. You would execute: ~~~shell cd ~/Downloads - chmod +x bazel-0.3.0-installer-linux-x86_64.sh - ./bazel-0.3.0-installer-linux-x86_64.sh --user + chmod +x bazel-0.3.1-installer-linux-x86_64.sh + ./bazel-0.3.1-installer-linux-x86_64.sh --user ~~~ 2. Set up your environment. Put this in your ~/.bashrc. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 81c244c5c4d..f4d65c43ace 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -56,7 +56,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.3.0 +ENV BAZEL_VERSION 0.3.1 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ From 8c89d72259a7c15f0a9711a67eecf284d9460cca Mon Sep 17 00:00:00 2001 From: Lucas Kacher Date: Tue, 20 Sep 2016 17:35:16 -0400 Subject: [PATCH 0007/8554] Ignore built /bazel-tensorflow_serving file (#184) --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 150500ce279..fd22d60859d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ node_modules /bazel-out /bazel-serving /bazel-tensorflow +/bazel-tensorflow_serving /bazel-testlogs /bazel-tf /bazel-workspace From 3db03a58226b1f32146730f7233c6ae0d6091820 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 16 Sep 2016 07:32:33 -0800 Subject: [PATCH 0008/8554] Annotate Subscribe() with TF_MUST_USE_RESULT. This is just a suggestion, but it might make the API less error-prone. If you let the result go out of scope, you will silently not get alerted. Change: 133387329 --- tensorflow_serving/util/event_bus.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/util/event_bus.h b/tensorflow_serving/util/event_bus.h index 4ea41a4bf33..167547e095e 100644 --- a/tensorflow_serving/util/event_bus.h +++ b/tensorflow_serving/util/event_bus.h @@ -129,7 +129,7 @@ class EventBus : public std::enable_shared_from_this> { // (1) Unsubscribe. // (2) Tear down anything that the callback references. std::unique_ptr Subscribe(const Callback& callback) - LOCKS_EXCLUDED(mutex_); + LOCKS_EXCLUDED(mutex_) TF_MUST_USE_RESULT; // Publishes an event to all subscribers. void Publish(const E& event) LOCKS_EXCLUDED(mutex_); From d945d0fe27eca70b297aea3db04b4f0ad1e84029 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Fri, 16 Sep 2016 07:49:05 -0800 Subject: [PATCH 0009/8554] Merge changes from github. Change: 133388804 --- WORKSPACE | 9 +++++++-- tensorflow_serving/workspace.bzl | 15 +++++---------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 5a7e12f68ad..2bf7b11a735 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,9 +1,14 @@ workspace(name = "tf_serving") +local_repository( + name = "org_tensorflow", + path = "tensorflow", +) + # Please add all new TensorFlow Serving dependencies in workspace.bzl. load('//tensorflow_serving:workspace.bzl', 'tf_serving_workspace') -tf_serving_workspace(__workspace_dir__) +tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") -check_version("0.3.0") \ No newline at end of file +check_version("0.3.0") diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index cff062cf521..58b276f1f5b 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -1,28 +1,23 @@ # TensorFlow Serving external dependencies that can be loaded in WORKSPACE # files. -load('//tensorflow/tensorflow:workspace.bzl', 'tf_workspace') +load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') # All TensorFlow Serving external dependencies. # workspace_dir is the absolute path to the TensorFlow Serving repo. If linked # as a submodule, it'll likely be '__workspace_dir__ + "/serving"' -def tf_serving_workspace(workspace_dir): - native.local_repository( - name = "org_tensorflow", - path = workspace_dir + "/tensorflow", - ) - +def tf_serving_workspace(): native.local_repository( name = "inception_model", - path = workspace_dir + "/tf_models/inception", + path = "tf_models/inception", ) - tf_workspace("tensorflow/", "@org_tensorflow") + tf_workspace() # ===== gRPC dependencies ===== native.bind( name = "libssl", - actual = "@boringssl_git//:ssl", + actual = "@boringssl//:ssl", ) native.bind( From 74a34f4e34224bb46da05fc07e3b4d361cfdf699 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 20 Sep 2016 07:52:59 -0800 Subject: [PATCH 0010/8554] Fix bazel build command in advanced tutorial. Change: 133710845 --- tensorflow_serving/g3doc/serving_advanced.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index c99413b608a..5173297a86d 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -255,7 +255,7 @@ server. ~~~shell $>mkdir /tmp/monitored $>cp -r /tmp/mnist_model/00000001 /tmp/monitored -$>bazel build //tensorflow_serving/model_servers/tensorflow_model_server +$>bazel build //tensorflow_serving/model_servers:tensorflow_model_server $>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored ~~~ From e80ed1795355e5f444170b643b320c06743f7d6c Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 22 Sep 2016 10:15:09 -0800 Subject: [PATCH 0011/8554] Eliminate dependency on regexp library. We were only using it in StaticSourceRouter, and we've downgraded that to do sub-string matching. Change: 133976007 --- .../core/static_source_router.h | 41 +++++++++---------- .../core/static_source_router_test.cc | 2 +- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/tensorflow_serving/core/static_source_router.h b/tensorflow_serving/core/static_source_router.h index 6dfb1003358..b54e49954ed 100644 --- a/tensorflow_serving/core/static_source_router.h +++ b/tensorflow_serving/core/static_source_router.h @@ -21,25 +21,24 @@ limitations under the License. #include #include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/regexp.h" #include "tensorflow_serving/core/source_router.h" namespace tensorflow { namespace serving { // A SourceRouter with N statically-configured output ports. Items are routed to -// output ports based on regular-expression matching against the servable name. -// The router is configured with N-1 regular expressions (regexps), with "fall- -// through" semantics. In particular: The regexps are numbered 0, 1, ..., N-2. -// Items whose servable name matches regexp 0 are sent to port 0; items that -// fail to match regexp 0 but do match regexp 1 are sent to port 1; and so on. -// Items that match none of the regexps are sent to port N-1. +// output ports based on substring matching against the servable name. The +// router is configured with N-1 substrings, with "fall-through" semantics. In +// particular: The substrings are numbered 0, 1, ..., N-2. Items whose servable +// name matches substring 0 are sent to port 0; items that fail to match +// substring 0 but do match substring 1 are sent to port 1; and so on. Items +// that match none of the substrings are sent to port N-1. template class StaticSourceRouter final : public SourceRouter { public: - // Creates a StaticSourceRouter with 'route_regexps.size() + 1' output ports, - // based on cascading regular expression matching as described above. - static Status Create(const std::vector& route_regexps, + // Creates a StaticSourceRouter with 'route_substrings.size() + 1' output + // ports, based on cascading substring matching as described above. + static Status Create(const std::vector& route_substrings, std::unique_ptr>* result); ~StaticSourceRouter() override; @@ -52,10 +51,11 @@ class StaticSourceRouter final : public SourceRouter { const std::vector>& versions) override; private: - explicit StaticSourceRouter(const std::vector& route_regexps); + explicit StaticSourceRouter(const std::vector& route_substrings); - // The regexps of the first N-1 routes (the Nth route is the default route). - std::vector> routes_except_default_; + // The substrings of the first N-1 routes (the Nth route is the default + // route). + std::vector routes_except_default_; TF_DISALLOW_COPY_AND_ASSIGN(StaticSourceRouter); }; @@ -65,9 +65,9 @@ class StaticSourceRouter final : public SourceRouter { template Status StaticSourceRouter::Create( - const std::vector& route_regexps, + const std::vector& route_substrings, std::unique_ptr>* result) { - result->reset(new StaticSourceRouter(route_regexps)); + result->reset(new StaticSourceRouter(route_substrings)); return Status::OK(); } @@ -80,13 +80,13 @@ template int StaticSourceRouter::Route(const StringPiece servable_name, const std::vector>& versions) { for (int i = 0; i < routes_except_default_.size(); ++i) { - if (RE2::FullMatch(servable_name.ToString(), *routes_except_default_[i])) { + if (servable_name.contains(routes_except_default_[i])) { LOG(INFO) << "Routing servable(s) from stream " << servable_name << " to route " << i; return i; } } - // None of the regexps matched, so return the "default" Nth route. + // None of the substrings matched, so return the "default" Nth route. LOG(INFO) << "Routing servable(s) from stream " << servable_name << " to default route " << routes_except_default_.size(); return routes_except_default_.size(); @@ -94,11 +94,8 @@ int StaticSourceRouter::Route(const StringPiece servable_name, template StaticSourceRouter::StaticSourceRouter( - const std::vector& route_regexps) { - for (const string& route_regexp : route_regexps) { - routes_except_default_.emplace_back(new RE2(route_regexp)); - } -} + const std::vector& route_substrings) + : routes_except_default_(route_substrings) {} } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/static_source_router_test.cc b/tensorflow_serving/core/static_source_router_test.cc index 91dd3127988..094ee122abb 100644 --- a/tensorflow_serving/core/static_source_router_test.cc +++ b/tensorflow_serving/core/static_source_router_test.cc @@ -39,7 +39,7 @@ namespace serving { namespace { TEST(StaticSourceRouterTest, Basic) { - const std::vector regexps = {"0th.*", ".*1st"}; + const std::vector regexps = {"0th", "1st"}; std::unique_ptr> router; TF_ASSERT_OK(StaticSourceRouter::Create(regexps, &router)); std::vector*> output_ports = router->GetOutputPorts(); From 2a92300b0f0a563b8a0a2a67b736dd4ebb16dc09 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 22 Sep 2016 13:33:00 -0800 Subject: [PATCH 0012/8554] Fix failing tensorflow_model_server_test. Change: 134000672 --- .../model_servers/tensorflow_model_server_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 666ba422e3a..79f4427d9e6 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -29,6 +29,7 @@ from grpc import * from grpc.beta import implementations +from grpc.beta import interfaces as beta_interfaces from grpc.framework.interfaces.face import face import tensorflow as tf @@ -122,7 +123,7 @@ def testBadModel(self): time.sleep(5) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address) - self.assertIs(StatusCode.FAILED_PRECONDITION, + self.assertIs(beta_interfaces.StatusCode.FAILED_PRECONDITION, error.exception.code) self.assertTrue(error.exception.details.startswith( 'Expected exactly one signatures proto')) From 28aa92dc327ac2f94a40bc82bd80a8498d9b5f94 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 22 Sep 2016 15:55:38 -0800 Subject: [PATCH 0013/8554] Export classification signature for MNIST and inception model export. Change: 134018020 --- .../example/inception_client.py | 7 +- .../example/inception_export.py | 30 +++-- tensorflow_serving/example/mnist_client.py | 113 ++++++++++++------ tensorflow_serving/example/mnist_export.py | 17 ++- 4 files changed, 120 insertions(+), 47 deletions(-) diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py index b5d9790bf5e..e4e8a880210 100644 --- a/tensorflow_serving/example/inception_client.py +++ b/tensorflow_serving/example/inception_client.py @@ -28,21 +28,18 @@ tf.app.flags.DEFINE_string('server', 'localhost:9000', - 'inception_inference service host:port') + 'PredictionService host:port') tf.app.flags.DEFINE_string('image', '', 'path to image in JPEG format') FLAGS = tf.app.flags.FLAGS -NUM_CLASSES = 5 - - def main(_): host, port = FLAGS.server.split(':') channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) # Send request with open(FLAGS.image, 'rb') as f: - # See inception_inference.proto for gRPC request/response details. + # See prediction_service.proto for gRPC request/response details. data = f.read() request = predict_pb2.PredictRequest() request.model_spec.name = 'inception' diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index f658f4733a0..551bfc011ac 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -65,7 +65,13 @@ def export(): # Please refer to Tensorflow inception model for details. # Input transformation. - jpegs = tf.placeholder(tf.string) + serialized_tf_example = tf.placeholder(tf.string, name='tf_example') + feature_configs = { + 'jpeg_encoded': tf.FixedLenFeature( + shape=[], dtype=tf.string, default_value=''), + } + tf_example = tf.parse_example(serialized_tf_example, feature_configs) + jpegs = tf_example['jpeg_encoded'] images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32) # Run inference. @@ -103,18 +109,28 @@ def export(): print('Successfully loaded model from %s at step=%s.' % (ckpt.model_checkpoint_path, global_step)) else: - print('No checkpoint file found at %s' % FLAGS.checkpoint_dir) + print 'No checkpoint file found at %s' % FLAGS.checkpoint_dir return # Export inference model. init_op = tf.group(tf.initialize_all_tables(), name='init_op') - model_exporter = exporter.Exporter(saver) - model_exporter.init(init_op=init_op, named_graph_signatures={ + classification_signature = exporter.classification_signature( + input_tensor=serialized_tf_example, + classes_tensor=classes, + scores_tensor=values) + named_graph_signature = { 'inputs': exporter.generic_signature({'images': jpegs}), - 'outputs': exporter.generic_signature({'classes': classes, - 'scores': values})}) + 'outputs': exporter.generic_signature({ + 'classes': classes, + 'scores': values + })} + model_exporter = exporter.Exporter(saver) + model_exporter.init( + init_op=init_op, + default_graph_signature=classification_signature, + named_graph_signatures=named_graph_signature) model_exporter.export(FLAGS.export_dir, tf.constant(global_step), sess) - print('Successfully exported model to %s' % FLAGS.export_dir) + print 'Successfully exported model to %s' % FLAGS.export_dir def preprocess_image(image_buffer): diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index f74ba6eb7c6..38899eb52c7 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -42,16 +42,88 @@ tf.app.flags.DEFINE_integer('concurrency', 1, 'maximum number of concurrent inference requests') tf.app.flags.DEFINE_integer('num_tests', 100, 'Number of test images') -tf.app.flags.DEFINE_string('server', '', 'mnist_inference service host:port') +tf.app.flags.DEFINE_string('server', '', 'PredictionService host:port') tf.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory. ') FLAGS = tf.app.flags.FLAGS +class _ResultCounter(object): + """Counter for the prediction results.""" + + def __init__(self, num_tests, concurrency): + self._num_tests = num_tests + self._concurrency = concurrency + self._error = 0 + self._done = 0 + self._active = 0 + self._condition = threading.Condition() + + def inc_error(self): + with self._condition: + self._error += 1 + + def inc_done(self): + with self._condition: + self._done += 1 + self._condition.notify() + + def dec_active(self): + with self._condition: + self._active -= 1 + self._condition.notify() + + def get_error_rate(self): + with self._condition: + while self._done != self._num_tests: + self._condition.wait() + return self._error / float(self._num_tests) + + def throttle(self): + with self._condition: + while self._active == self._concurrency: + self._condition.wait() + self._active += 1 + + +def _create_rpc_callback(label, result_counter): + """Creates RPC callback function. + + Args: + label: The correct label for the predicted example. + result_counter: Counter for the prediction result. + Returns: + The callback function. + """ + def _callback(result_future): + """Callback function. + + Calculates the statistics for the prediction result. + + Args: + result_future: Result future of the RPC. + """ + exception = result_future.exception() + if exception: + result_counter.inc_error() + print exception + else: + sys.stdout.write('.') + sys.stdout.flush() + response = numpy.array( + result_future.result().outputs['scores'].float_val) + prediction = numpy.argmax(response) + if label != prediction: + result_counter.inc_error() + result_counter.inc_done() + result_counter.dec_active() + return _callback + + def do_inference(hostport, work_dir, concurrency, num_tests): - """Tests mnist_inference service with concurrent requests. + """Tests PredictionService with concurrent requests. Args: - hostport: Host:port address of the mnist_inference service. + hostport: Host:port address of the PredictionService. work_dir: The full path of working directory for test data set. concurrency: Maximum number of concurrent requests. num_tests: Number of test images to use. @@ -66,45 +138,18 @@ def do_inference(hostport, work_dir, concurrency, num_tests): host, port = hostport.split(':') channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) - cv = threading.Condition() - result = {'active': 0, 'error': 0, 'done': 0} - def done(result_future, label): - with cv: - # Workaround for gRPC issue https://github.com/grpc/grpc/issues/7133 - try: - exception = result_future.exception() - except AttributeError: - exception = None - if exception: - result['error'] += 1 - print exception - else: - sys.stdout.write('.') - sys.stdout.flush() - response = numpy.array(result_future.result().outputs['scores']) - prediction = numpy.argmax(response) - if label != prediction: - result['error'] += 1 - result['done'] += 1 - result['active'] -= 1 - cv.notify() + result_counter = _ResultCounter(num_tests, concurrency) for _ in range(num_tests): request = predict_pb2.PredictRequest() request.model_spec.name = 'mnist' image, label = test_data_set.next_batch(1) request.inputs['images'].CopyFrom( tf.contrib.util.make_tensor_proto(image[0], shape=[1, image[0].size])) - with cv: - while result['active'] == concurrency: - cv.wait() - result['active'] += 1 + result_counter.throttle() result_future = stub.Predict.future(request, 5.0) # 5 seconds result_future.add_done_callback( - lambda result_future, l=label[0]: done(result_future, l)) # pylint: disable=cell-var-from-loop - with cv: - while result['done'] != num_tests: - cv.wait() - return result['error'] / float(num_tests) + _create_rpc_callback(label[0], result_counter)) + return result_counter.get_error_rate() def main(_): diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 9aa82bd0c41..511bb0c780c 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -57,7 +57,12 @@ def main(_): print 'Training model...' mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) sess = tf.InteractiveSession() - x = tf.placeholder('float', shape=[None, 784]) + serialized_tf_example = tf.placeholder(tf.string, name='tf_example') + feature_configs = { + 'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32), + } + tf_example = tf.parse_example(serialized_tf_example, feature_configs) + x = tf_example['x'] y_ = tf.placeholder('float', shape=[None, 10]) w = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) @@ -65,6 +70,10 @@ def main(_): y = tf.nn.softmax(tf.matmul(x, w) + b) cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) + values, indices = tf.nn.top_k(y, 10) + prediction_classes = tf.contrib.lookup.index_to_string( + tf.to_int64(indices), + mapping=tf.constant([str(i) for i in xrange(10)])) for _ in range(FLAGS.training_iteration): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x: batch[0], y_: batch[1]}) @@ -81,10 +90,16 @@ def main(_): # whenever code changes. export_path = sys.argv[-1] print 'Exporting trained model to', export_path + init_op = tf.group(tf.initialize_all_tables(), name='init_op') saver = tf.train.Saver(sharded=True) model_exporter = exporter.Exporter(saver) model_exporter.init( sess.graph.as_graph_def(), + init_op=init_op, + default_graph_signature=exporter.classification_signature( + input_tensor=serialized_tf_example, + classes_tensor=prediction_classes, + scores_tensor=values), named_graph_signatures={ 'inputs': exporter.generic_signature({'images': x}), 'outputs': exporter.generic_signature({'scores': y})}) From 309e074a197e0da04646b992bdf6f3d58f1164c6 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 23 Sep 2016 15:29:18 -0700 Subject: [PATCH 0014/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 83fe2a5b732..bb717a69fc8 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 83fe2a5b7328e1b754b53cbbcf9b313450a2f863 +Subproject commit bb717a69fc8e28af92d4a557f3f3f78f66de8129 diff --git a/tf_models b/tf_models index fb41088f854..85ccd4a3697 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit fb41088f854abfdb3824adf9cd3d3569f9ebaabe +Subproject commit 85ccd4a369711b64052a85e7fff999510e829c1e From 4c109fdca6aa0bcd3d2c883ae193a025838573d5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 26 Sep 2016 16:02:38 -0800 Subject: [PATCH 0015/8554] Assign explicit names to MNIST op. Change: 134345909 --- tensorflow_serving/example/mnist_export.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 511bb0c780c..08fae18e0a5 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -62,12 +62,12 @@ def main(_): 'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32), } tf_example = tf.parse_example(serialized_tf_example, feature_configs) - x = tf_example['x'] + x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name y_ = tf.placeholder('float', shape=[None, 10]) w = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) sess.run(tf.initialize_all_variables()) - y = tf.nn.softmax(tf.matmul(x, w) + b) + y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) values, indices = tf.nn.top_k(y, 10) From 9f2fb7a15bfac8b8793a962f319c7fd1012f72ee Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 27 Sep 2016 13:24:30 -0800 Subject: [PATCH 0016/8554] Add logging associated with calls to MallocExtension_ReleaseToSystem(). Change: 134454261 --- tensorflow_serving/core/simple_loader.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_serving/core/simple_loader.h b/tensorflow_serving/core/simple_loader.h index acce1ae1903..a38a1db0cea 100644 --- a/tensorflow_serving/core/simple_loader.h +++ b/tensorflow_serving/core/simple_loader.h @@ -224,6 +224,8 @@ void SimpleLoader::Unload() { resource_estimate.resource_quantities()) { if (entry.resource().device() == device_types::kMain && entry.resource().kind() == resource_kinds::kRamBytes) { + LOG(INFO) << "Calling MallocExtension_ReleaseToSystem() with " + << entry.quantity(); ::tensorflow::port::MallocExtension_ReleaseToSystem(entry.quantity()); } } From 92bfaad3bf1d22e08137d0cbbe458fc3daeaa29c Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 27 Sep 2016 14:50:21 -0800 Subject: [PATCH 0017/8554] Set session bundle size estimate padding to zero. (The size estimate is crude and temporary anyway; we hope to replace it with something more rigorous soon.) Change: 134465632 --- .../servables/tensorflow/session_bundle_factory.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h index a7b3011c0e8..b976a652f7c 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h @@ -46,8 +46,8 @@ class SessionBundleFactory { public: // Constants used in the resource estimation heuristic. See the documentation // on EstimateResourceRequirements(). - static constexpr double kResourceEstimateRAMMultiplier = 1.5; - static constexpr int kResourceEstimateRAMPadBytes = 10 << 20 /* 10 MB */; + static constexpr double kResourceEstimateRAMMultiplier = 1.0; + static constexpr int kResourceEstimateRAMPadBytes = 0; static Status Create(const SessionBundleConfig& config, std::unique_ptr* factory); From 4f41f533d696f05d8b8d837e424d49c393c7f0b6 Mon Sep 17 00:00:00 2001 From: Fangwei Li Date: Wed, 28 Sep 2016 15:18:27 -0800 Subject: [PATCH 0018/8554] 1. Add dynamic config reloading capability to ServerCore 2. Add backward-compatibility to ModelServerConfig and ServerCore regarding model type configuration. 3. Add corresponding test cases Change: 134596074 --- .../config/model_server_config.proto | 21 +- .../test_util/fake_loader_source_adapter.cc | 4 +- .../test_util/fake_loader_source_adapter.h | 1 + tensorflow_serving/model_servers/BUILD | 14 +- tensorflow_serving/model_servers/main.cc | 19 +- .../model_servers/model_platform_types.h | 22 ++ .../model_servers/server_core.cc | 238 +++++++++++++----- .../model_servers/server_core.h | 144 +++++++---- .../model_servers/server_core_test.cc | 141 ++++++----- .../model_servers/test_util/BUILD | 14 ++ .../test_util/mock_server_core.h | 16 +- .../test_util/server_core_test_util.cc | 45 ++++ .../test_util/server_core_test_util.h | 23 ++ 13 files changed, 502 insertions(+), 200 deletions(-) create mode 100644 tensorflow_serving/model_servers/model_platform_types.h diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index d25275c831b..f5bbb3b655e 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -5,6 +5,14 @@ option cc_enable_arenas = true; import "google/protobuf/any.proto"; +// The type of model. +// TODO(b/31336131): DEPRECATED. +enum ModelType { + MODEL_TYPE_UNSPECIFIED = 0 [deprecated = true]; + TENSORFLOW = 1 [deprecated = true]; + OTHER = 2 [deprecated = true]; +}; + // Common configuration for loading a model being served. message ModelConfig { // Name of the model. @@ -15,8 +23,12 @@ message ModelConfig { // base path is /foo/bar/my_model. string base_path = 2; + // Type of model. + // TODO(b/31336131): DEPRECATED. Please use 'model_platform' instead. + ModelType model_type = 3 [deprecated = true]; + // Type of model (e.g. "tensorflow"). - string model_type = 3; + string model_platform = 4; } // Static list of models to be loaded for serving. @@ -26,12 +38,11 @@ message ModelConfigList { // ModelServer config. message ModelServerConfig { - // ModelServer takes either a static file-based model config list or an Any - // proto representing custom model config that is loaded dynamically at + // proto representing custom model config that is fetched dynamically at // runtime (through network RPC, custom service, etc.). oneof config { ModelConfigList model_config_list = 1; - google.protobuf.Any dynamic_model_config = 2; + google.protobuf.Any custom_model_config = 2; } -} \ No newline at end of file +} diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc index 53fadba8aa8..42c0937b84e 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc @@ -43,9 +43,11 @@ FakeLoaderSourceAdapter::~FakeLoaderSourceAdapter() { } std::function>>*)> FakeLoaderSourceAdapter::GetCreator() { - return [](std::unique_ptr>>* source) { source->reset(new FakeLoaderSourceAdapter); return Status::OK(); diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h index 6c460c153d7..1097569e70e 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h @@ -45,6 +45,7 @@ class FakeLoaderSourceAdapter final // Returns a function to create a fake source adapter. static std::function>>*)> GetCreator(); diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 768227c52c1..d3ec2c0760c 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -25,14 +25,23 @@ filegroup( ), ) +cc_library( + name = "model_platform_types", + hdrs = ["model_platform_types.h"], + visibility = [ + "//visibility:public", + ], +) + cc_library( name = "server_core", srcs = ["server_core.cc"], hdrs = ["server_core.h"], visibility = [ - "//tensorflow_serving:internal", + "//visibility:public", ], deps = [ + ":model_platform_types", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:aspired_versions_manager", @@ -59,10 +68,8 @@ cc_test( ], deps = [ ":server_core", - "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:servable_state", "//tensorflow_serving/core/test_util:availability_test_util", - "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/model_servers/test_util:server_core_test_util", "//tensorflow_serving/test_util", @@ -88,6 +95,7 @@ cc_binary( ], visibility = ["//tensorflow_serving:internal"], deps = [ + ":model_platform_types", ":server_core", "@protobuf//:cc_wkt_protos", "@org_tensorflow//tensorflow/core:lib", diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 4312fb524ca..f4ad0dd66b9 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -58,6 +58,7 @@ limitations under the License. #include "tensorflow_serving/apis/prediction_service.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/core/servable_state_monitor.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" @@ -69,6 +70,7 @@ using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; using tensorflow::serving::ServableStateMonitor; using tensorflow::serving::ServerCore; +using tensorflow::serving::ServerCoreConfig; using tensorflow::serving::SessionBundleSourceAdapter; using tensorflow::serving::SessionBundleSourceAdapterConfig; using tensorflow::serving::Target; @@ -88,12 +90,11 @@ using tensorflow::serving::PredictionService; namespace { -constexpr char kTensorFlowModelType[] = "tensorflow"; - tensorflow::Status CreateSourceAdapter( - const SessionBundleSourceAdapterConfig& config, const string& model_type, + const SessionBundleSourceAdapterConfig& config, + const string& model_platform, std::unique_ptr* adapter) { - CHECK(model_type == kTensorFlowModelType) // Crash ok + CHECK(model_platform == kTensorFlowModelPlatform) // Crash ok << "ModelServer supports only TensorFlow model."; std::unique_ptr typed_adapter; TF_RETURN_IF_ERROR( @@ -109,11 +110,12 @@ tensorflow::Status CreateServableStateMonitor( return tensorflow::Status::OK(); } -tensorflow::Status LoadDynamicModelConfig( +tensorflow::Status LoadCustomModelConfig( const ::google::protobuf::Any& any, + EventBus* servable_event_bus, Target>* target) { CHECK(false) // Crash ok - << "ModelServer does not yet support dynamic model config."; + << "ModelServer does not yet support custom model config."; } ModelServerConfig BuildSingleModelConfig(const string& model_name, @@ -126,7 +128,7 @@ ModelServerConfig BuildSingleModelConfig(const string& model_name, config.mutable_model_config_list()->add_config(); single_model->set_name(model_name); single_model->set_base_path(model_base_path); - single_model->set_model_type(kTensorFlowModelType); + single_model->set_model_platform(kTensorFlowModelPlatform); return config; } @@ -211,7 +213,8 @@ int main(int argc, char** argv) { TF_CHECK_OK(ServerCore::Create( config, std::bind(CreateSourceAdapter, source_adapter_config, std::placeholders::_1, std::placeholders::_2), - &CreateServableStateMonitor, &LoadDynamicModelConfig, &core)); + &CreateServableStateMonitor, &LoadCustomModelConfig, ServerCoreConfig(), + &core)); RunServer(port, std::move(core)); return 0; diff --git a/tensorflow_serving/model_servers/model_platform_types.h b/tensorflow_serving/model_servers/model_platform_types.h new file mode 100644 index 00000000000..457ebbfe779 --- /dev/null +++ b/tensorflow_serving/model_servers/model_platform_types.h @@ -0,0 +1,22 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ +#define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ + +constexpr char kTensorFlowModelPlatform[] = "tensorflow"; +constexpr char kOtherModelPlatform[] = "other"; + +#endif // TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index b12a89af857..aef178dee06 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -21,25 +21,74 @@ limitations under the License. #include "google/protobuf/wrappers.pb.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" namespace tensorflow { namespace serving { +// ************************************************************************ +// Local Helper Methods. +// ************************************************************************ + +namespace { + +// Returns an error if it is not the case that all ModelConfigList models have +// the same model type, otherwise returns OK and sets 'model_type' to the type. +Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, + string* model_platform) { + for (const auto& model : config.model_config_list().config()) { + // Get platform (with backward compatibility) + string platform; + if (model.model_type() != ModelType::MODEL_TYPE_UNSPECIFIED) { + if (!model.model_platform().empty()) { + return errors::InvalidArgument( + "Illegal setting both model_type(deprecated) and model_platform."); + } + if (model.model_type() == ModelType::TENSORFLOW) { + platform = kTensorFlowModelPlatform; + } else { + platform = kOtherModelPlatform; + } + } else { + platform = model.model_platform(); + } + + if (platform.empty()) { + return errors::InvalidArgument( + "Illegal setting neither model_type(deprecated) nor model_platform."); + } + + // Check if matches found_platform (so far) + if (model_platform->empty()) { + *model_platform = platform; + } + // Error if not, continue if true + if (platform != *model_platform) { + return errors::InvalidArgument( + "Expect all models to have the same type."); + } + } + return Status::OK(); +} + +} // namespace + // ************************************************************************ // Public Methods. // ************************************************************************ Status ServerCore::Create( const ModelServerConfig& config, - SourceAdapterCreator source_adapter_creator, - ServableStateMonitorCreator servable_state_monitor_creator, - DynamicModelConfigLoader dynamic_state_config_loader, + const SourceAdapterCreator& source_adapter_creator, + const ServableStateMonitorCreator& servable_state_monitor_creator, + const CustomModelConfigLoader& custom_model_config_loader, + const ServerCoreConfig& server_core_config, std::unique_ptr* server_core) { - server_core->reset(new ServerCore(source_adapter_creator, - servable_state_monitor_creator, - dynamic_state_config_loader)); + server_core->reset( + new ServerCore(source_adapter_creator, servable_state_monitor_creator, + custom_model_config_loader, server_core_config)); TF_RETURN_IF_ERROR((*server_core)->Initialize()); return (*server_core)->ReloadConfig(config); } @@ -49,12 +98,14 @@ Status ServerCore::Create( // ************************************************************************ ServerCore::ServerCore( - SourceAdapterCreator source_adapter_creator, - ServableStateMonitorCreator servable_state_monitor_creator, - DynamicModelConfigLoader dynamic_state_config_loader) + const SourceAdapterCreator& source_adapter_creator, + const ServableStateMonitorCreator& servable_state_monitor_creator, + const CustomModelConfigLoader& custom_model_config_loader, + const ServerCoreConfig& server_core_config) : source_adapter_creator_(source_adapter_creator), servable_state_monitor_creator_(servable_state_monitor_creator), - dynamic_model_config_loader_(dynamic_state_config_loader), + custom_model_config_loader_(custom_model_config_loader), + server_core_config_(server_core_config), servable_event_bus_(EventBus::CreateEventBus()) {} Status ServerCore::Initialize() { @@ -70,28 +121,17 @@ Status ServerCore::Initialize() { return Status::OK(); } -Status ServerCore::AddModelsViaModelConfigList( - const ModelServerConfig& config) { +Status ServerCore::WaitUntilConfiguredModelsAvailable() { std::vector awaited_models; - for (const auto& model : config.model_config_list().config()) { - LOG(INFO) << " Adding model: " << model.name(); + for (const auto& model : config_.model_config_list().config()) { awaited_models.push_back(ServableRequest::Latest(model.name())); - std::unique_ptr>> - source_adapter; - TF_RETURN_IF_ERROR(CreateSourceAdapter(model.model_type(), manager_.get(), - &source_adapter)); - std::unique_ptr> path_source; - TF_RETURN_IF_ERROR(CreateStoragePathSource( - model.base_path(), model.name(), source_adapter.get(), &path_source)); - manager_.AddDependency(std::move(source_adapter)); - manager_.AddDependency(std::move(path_source)); } std::map states_reached; - const bool all_configured_models_available = + const bool all_models_available = servable_state_monitor_->WaitUntilServablesReachState( awaited_models, ServableState::ManagerState::kAvailable, &states_reached); - if (!all_configured_models_available) { + if (!all_models_available) { string message = "Some models did not become available: {"; for (const auto& id_and_state : states_reached) { if (id_and_state.second != ServableState::ManagerState::kAvailable) { @@ -99,40 +139,90 @@ Status ServerCore::AddModelsViaModelConfigList( } } strings::StrAppend(&message, "}"); - return Status(error::UNKNOWN, message); + return errors::Unknown(message); } return Status::OK(); } -Status ServerCore::AddModelsViaDynamicModelConfig( - const ModelServerConfig& config) { - return dynamic_model_config_loader_(config.dynamic_model_config(), - manager_.get()); +Status ServerCore::AddModelsViaModelConfigList() { + // Config validation. + string model_platform; + TF_RETURN_IF_ERROR( + ValidateAllListedModelsAreOfSamePlatform(config_, &model_platform)); + + // Create the source adapter if we haven't done so. + bool is_first_config = storage_path_source_ == nullptr; + ModelServerSourceAdapter* source_adapter = nullptr; + if (is_first_config) { + model_platform_ = model_platform; + std::unique_ptr new_source_adapter; + TF_RETURN_IF_ERROR(CreateSourceAdapter(model_platform_, manager_.get(), + &new_source_adapter)); + source_adapter = new_source_adapter.get(); + manager_.AddDependency(std::move(new_source_adapter)); + } + + // Determine if config transition is legal. + if (!is_first_config && model_platform_ != model_platform) { + return errors::FailedPrecondition( + "Cannot transition to requested model platform. It is only legal to " + "transition to the same model platform."); + } + + // Create/reload file system storage path source. + const FileSystemStoragePathSourceConfig source_config = + CreateStoragePathSourceConfig(config_); + if (is_first_config) { + TF_RETURN_IF_ERROR( + CreateFileSystemStoragePathSource(source_config, source_adapter)); + } else { + TF_RETURN_IF_ERROR(ReloadFileSystemStoragePathSourceConfig(source_config)); + } + return Status::OK(); } -Status ServerCore::ReloadConfig(const ModelServerConfig& config) { - { - mutex_lock m(seen_models_mu_); - if (seen_models_) { - return errors::Internal("Repeated ReloadConfig calls not supported."); - } - if (config.config_case() == ModelServerConfig::CONFIG_NOT_SET) { - // Nothing to load. In this case we allow a future call with a non-empty - // config. - LOG(INFO) << "Taking no action for empty config. Future Reloads " - << "are allowed."; - return Status::OK(); - } - seen_models_ = true; +Status ServerCore::AddModelsViaCustomModelConfig() { + return custom_model_config_loader_(config_.custom_model_config(), + servable_event_bus_.get(), manager_.get()); +} + +Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { + mutex_lock l(config_mu_); + + // Determine whether to accept this config transition. + const bool is_first_config = + config_.config_case() == ModelServerConfig::CONFIG_NOT_SET; + const bool accept_transition = + is_first_config || + (config_.config_case() == ModelServerConfig::kModelConfigList && + new_config.config_case() == ModelServerConfig::kModelConfigList); + if (!accept_transition) { + return errors::FailedPrecondition( + "Cannot transition to requested config. It is only legal to transition " + "from one ModelConfigList to another."); } - LOG(INFO) << "Adding models to manager."; - switch (config.config_case()) { - case ModelServerConfig::kModelConfigList: - TF_RETURN_IF_ERROR(AddModelsViaModelConfigList(config)); + if (new_config.config_case() == ModelServerConfig::CONFIG_NOT_SET) { + // Nothing to load. In this case we allow a future call with a non-empty + // config. + LOG(INFO) << "Taking no action for empty config."; + return Status::OK(); + } + config_ = new_config; + + LOG(INFO) << "Adding/updating models."; + switch (config_.config_case()) { + case ModelServerConfig::kModelConfigList: { + TF_RETURN_IF_ERROR(AddModelsViaModelConfigList()); + TF_RETURN_IF_ERROR(WaitUntilConfiguredModelsAvailable()); break; - case ModelServerConfig::kDynamicModelConfig: - TF_RETURN_IF_ERROR(AddModelsViaDynamicModelConfig(config)); + } + case ModelServerConfig::kCustomModelConfig: { + // We've already verified this invariant above, so this check should + // always pass. + CHECK(is_first_config); // Crash ok. + TF_RETURN_IF_ERROR(AddModelsViaCustomModelConfig()); break; + } default: return errors::InvalidArgument("Invalid ServerModelConfig"); } @@ -141,35 +231,55 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& config) { } Status ServerCore::CreateSourceAdapter( - const string& model_type, Target>* target, + const string& model_platform, Target>* target, std::unique_ptr* adapter) { - TF_RETURN_IF_ERROR(source_adapter_creator_(model_type, adapter)); + TF_RETURN_IF_ERROR(source_adapter_creator_(model_platform, adapter)); ConnectSourceToTarget(adapter->get(), target); return Status::OK(); } -Status ServerCore::CreateStoragePathSource( - const string& base_path, const string& servable_name, - Target* target, - std::unique_ptr>* path_source) { - FileSystemStoragePathSourceConfig config; - config.set_servable_name(servable_name); - config.set_base_path(base_path); - config.set_file_system_poll_wait_seconds(30); +FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( + const ModelServerConfig& config) const { + FileSystemStoragePathSourceConfig source_config; + source_config.set_file_system_poll_wait_seconds( + server_core_config_.file_system_poll_wait_seconds); + for (const auto& model : config.model_config_list().config()) { + LOG(INFO) << " (Re-)adding model: " << model.name(); + FileSystemStoragePathSourceConfig::ServableToMonitor* servable = + source_config.add_servables(); + servable->set_servable_name(model.name()); + servable->set_base_path(model.base_path()); + } + return source_config; +} - std::unique_ptr file_system_source; +Status ServerCore::CreateFileSystemStoragePathSource( + const FileSystemStoragePathSourceConfig& source_config, + Target* target) { + std::unique_ptr storage_path_source; TF_RETURN_IF_ERROR( - FileSystemStoragePathSource::Create(config, &file_system_source)); - ConnectSourceToTarget(file_system_source.get(), target); - *path_source = std::move(file_system_source); + FileSystemStoragePathSource::Create(source_config, &storage_path_source)); + ConnectSourceToTarget(storage_path_source.get(), target); + storage_path_source_ = storage_path_source.get(); + manager_.AddDependency(std::move(storage_path_source)); return Status::OK(); } +Status ServerCore::ReloadFileSystemStoragePathSourceConfig( + const FileSystemStoragePathSourceConfig& source_config) { + return storage_path_source_->UpdateConfig(source_config); +} + Status ServerCore::CreateAspiredVersionsManager( std::unique_ptr* const manager) { + std::unique_ptr aspired_versions_manager; AspiredVersionsManager::Options manager_options; manager_options.servable_event_bus = servable_event_bus_.get(); manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + manager_options.num_load_unload_threads = + server_core_config_.num_load_unload_threads; + manager_options.max_num_load_retries = + server_core_config_.max_num_load_retries; return AspiredVersionsManager::Create(std::move(manager_options), manager); } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index a52f381321b..a761a498c2a 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -18,10 +18,9 @@ limitations under the License. // ServerCore is independent of any domain specific APIs and independent of // platforms. // -// In terms of state, ServerCore bootstraps with an AspiredVersionsManager to -// support efficient serving. It will soon support (re)loading of -// ModelServerConfig, from which it (re)creates auxiliary data structures to -// load model from custom sources. +// In terms of state, ServerCore is initialized with and retains a static +// ModelServerConfig, from which it bootstraps an AspiredVersionsManager and +// auxiliary data structures to support efficient serving. // // Interfaces built above ServerCore, e.g. RPC service implementations, will // remain stateless and will perform all lookups of servables (models) via @@ -46,12 +45,26 @@ limitations under the License. #include "tensorflow_serving/core/source.h" #include "tensorflow_serving/core/source_adapter.h" #include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/util/event_bus.h" #include "tensorflow_serving/util/unique_ptr_with_deps.h" namespace tensorflow { namespace serving { +// ServerCore tuning parameters. +struct ServerCoreConfig { + // Time interval between file-system polls, in seconds. + int32 file_system_poll_wait_seconds = 30; + // The number of threads used to load and unload models. If set to 0, then + // no thread pool is used and the loads/unloads are performed serially in + // the manager thread. + int32 num_load_unload_threads = 0; + // Maximum number of times we retry loading a model, after the first failure, + // before we give up" + int32 max_num_load_retries = 5; +}; + namespace test_util { class ServerCoreTestAccess; } // namespace test_util @@ -64,46 +77,40 @@ class ServerCore { SourceAdapter>; using SourceAdapterCreator = - std::function* adapter)>; using ServableStateMonitorCreator = std::function* event_bus, std::unique_ptr* monitor)>; - using DynamicModelConfigLoader = std::function* event_bus, Target>* target)>; - // Creates server core and loads the given config. - // - // The config is allowed to be empty, in which case user should call - // ReloadConfig later to actually start the server. + // Creates a ServerCore instance with all the models and sources per the + // ModelServerConfig. // - // source_adapter_creator is used, upon each ReloadConfig, to (re)create the - // single instance of global source adapter that adapts all Sources to - // platform-specific Loaders for the global AspiredVersionsManager. - // servable_state_monitor_creator is used once to create the - // ServableStateMonitor for the global AspiredVersionsManager. - // dynamic_model_config_loader is used, upon each ReloadConfig, to (re)create - // Sources defined in dynamic_model_config Any proto and connect them to the - // global source adapter. + // For models statically configured with ModelConfigList, waits for them + // to be made available (or hit an error) for serving before returning. + // Returns an error status if any such model fails to load. static Status Create( const ModelServerConfig& config, - SourceAdapterCreator source_adapter_creator, - ServableStateMonitorCreator servable_state_monitor_creator, - DynamicModelConfigLoader dynamic_model_config_loader, + const SourceAdapterCreator& source_adapter_creator, + const ServableStateMonitorCreator& servable_state_monitor_creator, + const CustomModelConfigLoader& custom_model_config_loader, + const ServerCoreConfig& server_core_config, std::unique_ptr* core); - // Updates the server core with all the models/sources per the - // ModelServerConfig. + // Updates the server core with all the models and sources per the + // ModelServerConfig. Like Create(), waits for all statically configured + // servables to be made available before returning, and returns an error if + // any such model fails to load. // - // For static config given as ModelConfigList, it waits for the models to be - // made available for serving before returning from this method. - // - // TODO(b/29012372): Note: this method may be called only when the server - // currently contains no models. - virtual Status ReloadConfig(const ModelServerConfig& config); + // IMPORTANT: It is only legal to call this method more than once if using + // ModelConfigList (versus custom model config). + virtual Status ReloadConfig(const ModelServerConfig& config) + LOCKS_EXCLUDED(config_mu_); // Returns ServableStateMonitor that can be used to query servable states. virtual const ServableStateMonitor* servable_state_monitor() const { @@ -127,10 +134,10 @@ class ServerCore { } protected: - explicit ServerCore( - SourceAdapterCreator source_adapter_creator_, - ServableStateMonitorCreator servable_state_monitor_creator, - DynamicModelConfigLoader dynamic_model_config_loader); + ServerCore(const SourceAdapterCreator& source_adapter_creator, + const ServableStateMonitorCreator& servable_state_monitor_creator, + const CustomModelConfigLoader& custom_model_config_loader, + const ServerCoreConfig& server_core_config); private: friend class test_util::ServerCoreTestAccess; @@ -143,30 +150,43 @@ class ServerCore { // Must be run once and only once per ServerCore instance. Status Initialize(); - // Creates a platform-specific Loader Source by adapting the underlying - // FileSystemStoragePathSource and connects it to the supplied target. + // Creates a AspiredVersionsManager with the EagerLoadPolicy. + Status CreateAspiredVersionsManager( + std::unique_ptr* manager); + + // Creates a platform-specific Loader Source and connects it to the supplied + // target. Status CreateSourceAdapter( - const string& model_type, Target>* target, + const string& model_platform, Target>* target, std::unique_ptr* adapter); - // Creates a Source that monitors a filesystem's base_path for - // new directories and connects it to the supplied target. - // The servable_name param simply allows this source to create all - // AspiredVersions for the target with the same servable_name. - Status CreateStoragePathSource( - const string& base_path, const string& servable_name, - Target* target, - std::unique_ptr>* path_source); + // Creates a FileSystemStoragePathSourceConfig from the ModelConfigList of + // 'config'. + FileSystemStoragePathSourceConfig CreateStoragePathSourceConfig( + const ModelServerConfig& config) const; - // Creates a AspiredVersionsManager with the EagerLoadPolicy. - Status CreateAspiredVersionsManager( - std::unique_ptr* manager); + // Waits for all models from the ModelConfigList in 'config_' to be loaded. + // Returns an error if any configured model fails to load. + Status WaitUntilConfiguredModelsAvailable() + EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + + // Creates a FileSystemStoragePathSource, connects it to the supplied + // target, stores the pointer in 'storage_path_source_' and transfers the + // ownership to 'manager_'. + Status CreateFileSystemStoragePathSource( + const FileSystemStoragePathSourceConfig& source_config, + Target* target) EXCLUSIVE_LOCKS_REQUIRED(config_mu_); - // Adds models through ModelConfigList, and waits for them to be loaded. - Status AddModelsViaModelConfigList(const ModelServerConfig& config); + // Updates the 'storage_path_source_' config. + Status ReloadFileSystemStoragePathSourceConfig( + const FileSystemStoragePathSourceConfig& source_config) + EXCLUSIVE_LOCKS_REQUIRED(config_mu_); - // Adds models through dynamic model config defined in Any proto. - Status AddModelsViaDynamicModelConfig(const ModelServerConfig& config); + // Adds/reloads models through ModelConfigList of 'config_'. + Status AddModelsViaModelConfigList() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + + // Adds/reloads models through custom model config of 'config_'. + Status AddModelsViaCustomModelConfig() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); // ************************************************************************ // Request Processing. @@ -185,14 +205,28 @@ class ServerCore { SourceAdapterCreator source_adapter_creator_; ServableStateMonitorCreator servable_state_monitor_creator_; - DynamicModelConfigLoader dynamic_model_config_loader_; + CustomModelConfigLoader custom_model_config_loader_; + ServerCoreConfig server_core_config_; std::shared_ptr> servable_event_bus_; std::shared_ptr servable_state_monitor_; UniquePtrWithDeps manager_; - bool seen_models_ GUARDED_BY(seen_models_mu_) = false; - mutable mutex seen_models_mu_; + // The most recent config supplied to ReloadConfig(). + ModelServerConfig config_ GUARDED_BY(config_mu_); + + // Model platform of the source adapter created for ModelConfigList. + // Empty if the source adapter is not yet created. + string model_platform_ GUARDED_BY(config_mu_); + + // If the configuration uses a file-system source, this is populated with a + // pointer to the source (to enable reconfiguration later). The source is + // owned by 'manager_'. + FileSystemStoragePathSource* storage_path_source_ GUARDED_BY(config_mu_) = + nullptr; + + // A mutex for reconfiguration, used by ReloadConfig(). + mutex config_mu_; }; } // namespace serving diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 51ce13400ae..3a4e54a597f 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -15,14 +15,9 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" -#include -#include #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" -#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" #include "tensorflow_serving/test_util/test_util.h" @@ -30,87 +25,109 @@ namespace tensorflow { namespace serving { namespace { -using ::testing::Eq; +using test_util::ServerCoreTest; -tensorflow::Status CreateSourceAdapter( - const string& model_type, - std::unique_ptr* adapter) { - adapter->reset(new test_util::FakeLoaderSourceAdapter); - return Status::OK(); -} - -tensorflow::Status CreateServableStateMonitor( - EventBus* event_bus, - std::unique_ptr* monitor) { - monitor->reset(new ServableStateMonitor(event_bus)); - return tensorflow::Status::OK(); -} - -tensorflow::Status LoadDynamicModelConfig( - const ::google::protobuf::Any& any, - Target>* target) { - CHECK(false); -} +TEST_F(ServerCoreTest, CreateWaitsTillModelsAvailable) { + std::unique_ptr server_core; + TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfig(), &server_core)); -ModelServerConfig CreateModelServerConfig() { - ModelServerConfig config; - auto model = config.mutable_model_config_list()->add_config(); - model->set_name("test_model"); - model->set_base_path(test_util::TestSrcDirPath( - "/servables/tensorflow/testdata/half_plus_two")); - model->set_model_type("tensorflow"); - return config; + const std::vector available_servables = + test_util::ServerCoreTestAccess(server_core.get()) + .ListAvailableServableIds(); + ASSERT_EQ(available_servables.size(), 1); + const ServableId expected_id = {test_util::kTestModelName, + test_util::kTestModelVersion}; + EXPECT_EQ(available_servables.at(0), expected_id); } -// TODO(b/29012372): Currently we only support a single config reload. -// Verify multiple calls result in an error. -TEST(ServerCoreTest, MultipleLoadConfigs) { - // Create with an empty config. +TEST_F(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { + // Create a server with no models, initially. std::unique_ptr server_core; - TF_ASSERT_OK(ServerCore::Create(ModelServerConfig(), &CreateSourceAdapter, - &CreateServableStateMonitor, - &LoadDynamicModelConfig, &server_core)); - // Reload with a populated config. This is allowed since the previous config - // was empty. - TF_ASSERT_OK(server_core->ReloadConfig(CreateModelServerConfig())); - - // Second reload fails since the previous config has models. - EXPECT_THAT( - server_core->ReloadConfig({}).ToString(), - ::testing::HasSubstr("Repeated ReloadConfig calls not supported")); -} + TF_ASSERT_OK(CreateServerCore(ModelServerConfig(), &server_core)); -TEST(ServerCoreTest, CreateWaitsTillModelsAvailable) { - std::unique_ptr server_core; - TF_ASSERT_OK(ServerCore::Create( - CreateModelServerConfig(), &CreateSourceAdapter, - &CreateServableStateMonitor, &LoadDynamicModelConfig, &server_core)); + // Reconfigure it to load our test model. + TF_ASSERT_OK(server_core->ReloadConfig(GetTestModelServerConfig())); const std::vector available_servables = test_util::ServerCoreTestAccess(server_core.get()) .ListAvailableServableIds(); ASSERT_EQ(available_servables.size(), 1); - const ServableId expected_id = {"test_model", 123}; + const ServableId expected_id = {test_util::kTestModelName, + test_util::kTestModelVersion}; EXPECT_EQ(available_servables.at(0), expected_id); } -TEST(ServerCoreTest, ErroringModel) { +TEST_F(ServerCoreTest, ErroringModel) { std::unique_ptr server_core; - const Status status = ServerCore::Create( - CreateModelServerConfig(), - [](const string& model_type, - std::unique_ptr>>* - source_adapter) -> Status { + Status status = CreateServerCore( + GetTestModelServerConfig(), + [](const string& model_platform, + std::unique_ptr* source_adapter) + -> Status { source_adapter->reset( new ErrorInjectingSourceAdapter>( Status(error::CANCELLED, ""))); return Status::OK(); }, - &CreateServableStateMonitor, &LoadDynamicModelConfig, &server_core); + &server_core); EXPECT_FALSE(status.ok()); } +TEST_F(ServerCoreTest, IllegalReconfigurationToCustomConfig) { + // Create a ServerCore with ModelConfigList config. + std::unique_ptr server_core; + TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfig(), &server_core)); + + // Reload with a custom config. This is not allowed since the server was + // first configured with TensorFlow model platform. + ModelServerConfig config; + config.mutable_custom_model_config(); + EXPECT_THAT(server_core->ReloadConfig(config).ToString(), + ::testing::HasSubstr("Cannot transition to requested config")); +} + +TEST_F(ServerCoreTest, IllegalReconfigurationFromCustomConfig) { + // Create a ServerCore with custom config. + std::unique_ptr server_core; + ModelServerConfig config; + config.mutable_custom_model_config(); + TF_ASSERT_OK(CreateServerCore(config, &server_core)); + + // Reload with a ModelConfigList config. This is not allowed, since the + // server was first configured with a custom config. + EXPECT_THAT(server_core->ReloadConfig(GetTestModelServerConfig()).ToString(), + ::testing::HasSubstr("Cannot transition to requested config")); +} + +TEST_F(ServerCoreTest, IllegalConfigModelTypeAndPlatformSet) { + // Create a ServerCore with both model_type and model_platform set. + std::unique_ptr server_core; + ModelServerConfig config = GetTestModelServerConfig(); + config.mutable_model_config_list()->mutable_config(0)->set_model_type( + ModelType::TENSORFLOW); + EXPECT_THAT(CreateServerCore(config, &server_core).ToString(), + ::testing::HasSubstr("Illegal setting both")); +} + +TEST_F(ServerCoreTest, DeprecatedModelTypeConfig) { + // Create a ServerCore with deprecated config. + std::unique_ptr server_core; + ModelServerConfig config = GetTestModelServerConfig(); + config.mutable_model_config_list()->mutable_config(0)->set_model_platform(""); + config.mutable_model_config_list()->mutable_config(0)->set_model_type( + ModelType::TENSORFLOW); + TF_ASSERT_OK(CreateServerCore(config, &server_core)); + + const std::vector available_servables = + test_util::ServerCoreTestAccess(server_core.get()) + .ListAvailableServableIds(); + ASSERT_EQ(available_servables.size(), 1); + const ServableId expected_id = {test_util::kTestModelName, + test_util::kTestModelVersion}; + EXPECT_EQ(available_servables.at(0), expected_id); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 6c07a934352..0125c392035 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -29,6 +29,9 @@ cc_library( name = "mock_server_core", testonly = 1, hdrs = ["mock_server_core.h"], + visibility = [ + "//visibility:public", + ], deps = [ "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:model_server_config_proto", @@ -45,8 +48,19 @@ cc_library( testonly = 1, srcs = ["server_core_test_util.cc"], hdrs = ["server_core_test_util.h"], + data = [ + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + ], + visibility = [ + "//visibility:public", + ], deps = [ "//tensorflow_serving/core:servable_id", + "//tensorflow_serving/core/test_util:fake_loader_source_adapter", + "//tensorflow_serving/model_servers:model_platform_types", "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:test", ], ) diff --git a/tensorflow_serving/model_servers/test_util/mock_server_core.h b/tensorflow_serving/model_servers/test_util/mock_server_core.h index be73c5b2d85..31de46c920a 100644 --- a/tensorflow_serving/model_servers/test_util/mock_server_core.h +++ b/tensorflow_serving/model_servers/test_util/mock_server_core.h @@ -32,8 +32,20 @@ namespace test_util { class MockServerCore : public ServerCore { public: - explicit MockServerCore(SourceAdapterCreator source_adapter_creator) - : ServerCore(source_adapter_creator) {} + explicit MockServerCore(const SourceAdapterCreator& source_adapter_creator) + : ServerCore( + source_adapter_creator, // ServerCore::SourceAdapterCreator + [](EventBus* event_bus, + std::unique_ptr* monitor) -> Status { + monitor->reset(new ServableStateMonitor(event_bus)); + return Status::OK(); + }, // ServerCore::ServableStateMonitorCreator + [](const ::google::protobuf::Any& any, + EventBus* event_bus, + Target>* target) -> Status { + return Status::OK(); + }, // ServerCore::CustomModelConfigLoader + ServerCoreConfig()) {} MOCK_CONST_METHOD0(servable_state_monitor, ServableStateMonitor*()); MOCK_METHOD1(ReloadConfig, Status(const ModelServerConfig&)); diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index eebf9a5e294..1dd546892c5 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -15,6 +15,10 @@ limitations under the License. #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/test_util/test_util.h" + namespace tensorflow { namespace serving { namespace test_util { @@ -23,6 +27,47 @@ std::vector ServerCoreTestAccess::ListAvailableServableIds() const { return core_->ListAvailableServableIds(); } +ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { + ModelServerConfig config; + auto model = config.mutable_model_config_list()->add_config(); + model->set_name(kTestModelName); + model->set_base_path(test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/half_plus_two")); + model->set_model_platform(kTensorFlowModelPlatform); + return config; +} + +ServerCoreConfig ServerCoreTest::GetTestServerCoreConfig() { + ServerCoreConfig config; + config.file_system_poll_wait_seconds = 0; + return config; +} + +Status ServerCoreTest::CreateServerCore( + const ModelServerConfig& config, std::unique_ptr* server_core) { + return CreateServerCore( + config, test_util::FakeLoaderSourceAdapter::GetCreator(), server_core); +} + +Status ServerCoreTest::CreateServerCore( + const ModelServerConfig& config, + const ServerCore::SourceAdapterCreator& source_adapter_creator, + std::unique_ptr* server_core) { + return ServerCore::Create( + config, source_adapter_creator, // ServerCore::SourceAdapterCreator + [](EventBus* event_bus, + std::unique_ptr* monitor) -> Status { + monitor->reset(new ServableStateMonitor(event_bus)); + return Status::OK(); + }, // ServerCore::ServableStateMonitor + [](const ::google::protobuf::Any& any, EventBus* event_bus, + Target>* target) -> Status { + return Status::OK(); + }, // ServerCore::CustomModelConfigLoader + GetTestServerCoreConfig(), + server_core); +} + } // namespace test_util } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index 1951182678b..29ca8851898 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ #define TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ +#include #include "tensorflow_serving/core/servable_id.h" #include "tensorflow_serving/model_servers/server_core.h" @@ -36,6 +37,28 @@ class ServerCoreTestAccess { ServerCore* const core_; }; +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; + +class ServerCoreTest : public ::testing::Test { + protected: + // Returns ModelServerConfig that contains test model. + ModelServerConfig GetTestModelServerConfig(); + + // Returns ServerCoreConfig that uses continuous polling, to speed up testing. + ServerCoreConfig GetTestServerCoreConfig(); + + // Create a ServerCore object configured to use FakeLoaderSourceAdapter. + Status CreateServerCore(const ModelServerConfig& config, + std::unique_ptr* server_core); + + // Create a ServerCore object with the supplied SourceAdapterCreator. + Status CreateServerCore( + const ModelServerConfig& config, + const ServerCore::SourceAdapterCreator& source_adapter_creator, + std::unique_ptr* server_core); +}; + } // namespace test_util } // namespace serving } // namespace tensorflow From c9b053da4295d400b7be7fe9bea78f281a1e8c14 Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Thu, 29 Sep 2016 13:32:47 -0700 Subject: [PATCH 0019/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index bb717a69fc8..0db279f16ce 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit bb717a69fc8e28af92d4a557f3f3f78f66de8129 +Subproject commit 0db279f16ce21568c414694ea55e2dec07ad6972 diff --git a/tf_models b/tf_models index 85ccd4a3697..f98c5ded31d 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 85ccd4a369711b64052a85e7fff999510e829c1e +Subproject commit f98c5ded31d7da0c2d127c28b2c16f0307a368f0 From 90e31ba8e2b9091c8e335dcbd5196e2365c8e77c Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 29 Sep 2016 13:32:33 -0800 Subject: [PATCH 0020/8554] Merge changes from github. Change: 134709612 --- .gitignore | 1 + WORKSPACE | 2 +- tensorflow_serving/g3doc/setup.md | 8 ++++---- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 150500ce279..fd22d60859d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,7 @@ node_modules /bazel-out /bazel-serving /bazel-tensorflow +/bazel-tensorflow_serving /bazel-testlogs /bazel-tf /bazel-workspace diff --git a/WORKSPACE b/WORKSPACE index 2bf7b11a735..39f4bcd6e71 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,4 +11,4 @@ tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") -check_version("0.3.0") +check_version("0.3.1") diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 10a252a61ed..4f4667455c5 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -6,7 +6,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel -TensorFlow Serving requires Bazel 0.3.0 or higher. You can find the Bazel +TensorFlow Serving requires Bazel 0.3.1 or higher. You can find the Bazel installation instructions [here](http://bazel.io/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the @@ -14,13 +14,13 @@ following steps: 1. Download the relevant binary from [here](https://github.com/bazelbuild/bazel/releases). - Let's say you downloaded bazel-0.3.0-installer-linux-x86_64.sh. You would + Let's say you downloaded bazel-0.3.1-installer-linux-x86_64.sh. You would execute: ~~~shell cd ~/Downloads - chmod +x bazel-0.3.0-installer-linux-x86_64.sh - ./bazel-0.3.0-installer-linux-x86_64.sh --user + chmod +x bazel-0.3.1-installer-linux-x86_64.sh + ./bazel-0.3.1-installer-linux-x86_64.sh --user ~~~ 2. Set up your environment. Put this in your ~/.bashrc. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 81c244c5c4d..f4d65c43ace 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -56,7 +56,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.3.0 +ENV BAZEL_VERSION 0.3.1 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ From 884f826a04acf59278caa792d3b84e48a574c185 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 30 Sep 2016 11:26:32 -0800 Subject: [PATCH 0021/8554] Add a StopManagingServable() method to BasicManager and let higher level managers decide when to forget about a servable. Change: 134813148 --- .../core/aspired_versions_manager.cc | 17 ++++ .../core/aspired_versions_manager.h | 6 ++ .../core/aspired_versions_manager_test.cc | 68 ++++++++++++++++ tensorflow_serving/core/basic_manager.cc | 74 +++++++++++------ tensorflow_serving/core/basic_manager.h | 21 ++--- tensorflow_serving/core/basic_manager_test.cc | 79 ++++++++++++++++--- .../core/test_util/manager_test_util.cc | 4 + .../core/test_util/manager_test_util.h | 3 + 8 files changed, 227 insertions(+), 45 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index e50efd21ea5..c3a0bef0792 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -170,6 +170,7 @@ AspiredVersionsManager::AspiredVersionsManager( pf_options.thread_name_prefix = "AspiredVersionsManager_ManageState_Thread"; manage_state_thread_.reset(new PeriodicFunction( [this]() { + this->FlushServables(); this->HandlePendingAspiredVersionsRequests(); this->InvokePolicyAndExecuteAction(); }, @@ -344,6 +345,22 @@ void AspiredVersionsManager::PerformAction( } } +void AspiredVersionsManager::FlushServables() { + mutex_lock l(basic_manager_read_modify_write_mu_); + for (const string& servable_name : + basic_manager_->GetManagedServableNames()) { + for (const ServableStateSnapshot& state_snapshot : + basic_manager_->GetManagedServableStateSnapshots( + servable_name)) { + if ((state_snapshot.state == LoaderHarness::State::kDisabled || + state_snapshot.state == LoaderHarness::State::kError) && + !state_snapshot.additional_state->is_aspired) { + basic_manager_->StopManagingServable(state_snapshot.id); + } + } + } +} + void AspiredVersionsManager::HandlePendingAspiredVersionsRequests() { mutex_lock l(basic_manager_read_modify_write_mu_); mutex_lock l2(pending_aspired_versions_requests_mu_); diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 143d05fa8af..a8254c390ac 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -215,6 +215,12 @@ class AspiredVersionsManager : public Manager, optional GetNextAction() EXCLUSIVE_LOCKS_REQUIRED(basic_manager_read_modify_write_mu_); + // Checks for servables that are not aspired and at some final state and tells + // 'basic_manager_' to forget about them. This method is intended to be + // invoked periodically, interleaved with InvokePolicyAndExecuteAction() and + // HandlePendingAspiredVersionsRequests(). + void FlushServables() LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_); + // Handles enqueued aspired-versions requests. This method is intended to be // invoked periodically, interleaved with InvokePolicyAndExecuteAction(). void HandlePendingAspiredVersionsRequests() diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 7464965f212..867c0cc3a8e 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -124,6 +124,11 @@ class AspiredVersionsManagerTest : public ::testing::TestWithParam { } } + void FlushServables() { + test_util::AspiredVersionsManagerTestAccess(manager_.get()) + .FlushServables(); + } + void HandlePendingAspiredVersionsRequests() { test_util::AspiredVersionsManagerTestAccess(manager_.get()) .HandlePendingAspiredVersionsRequests(); @@ -347,6 +352,7 @@ TEST_P(AspiredVersionsManagerTest, AspiredRemovedFull) { WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 1}, {ServableState::ManagerState::kEnd}); + FlushServables(); const int num_fake_loaders_after = FakeLoader::num_fake_loaders(); EXPECT_EQ(kNumVersionsPerServable, num_fake_loaders_before - num_fake_loaders_after); @@ -555,6 +561,7 @@ TEST_P(AspiredVersionsManagerTest, DestructOnNonServingThread) { WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, {kServableName, 0}, {ServableState::ManagerState::kEnd}); + FlushServables(); // The servable has been deleted in this thread if there is no // thread-pool for load/unload. if (num_load_unload_threads_ == 0) { @@ -832,6 +839,7 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { std::unique_ptr reaspire_thread( Env::Default()->StartThread(ThreadOptions(), "ReaspireThread", [&]() { while (!second_load_called.HasBeenNotified()) { + FlushServables(); HandlePendingAspiredVersionsRequests(); InvokePolicyAndExecuteAction(); Env::Default()->SleepForMicroseconds(1000 /* 1 ms */); @@ -848,6 +856,66 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { second_load_called.WaitForNotification(); } +TEST_P(AspiredVersionsManagerTest, + UnaspireFailedServableThenImmediatelyReaspire) { + // Like UnaspireThenImmediatelyReaspire, but covers the case in which the + // servable fails to load the first time it is aspired. + + const ServableId id = {kServableName, 7}; + + std::vector>> first_aspired_versions; + test_util::MockLoader* first_loader = new NiceMock(); + first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); + EXPECT_CALL(*first_loader, Load(_)) + .WillRepeatedly(Return(Status(error::UNKNOWN, "first load failing"))); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(first_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + + // Now, we'll un-aspire the servable, and then re-aspire it with a new loader. + // The manager should wait until it is able to flush the first loader, then + // bring up the second loader. + + std::vector>> empty_aspired_versions; + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(empty_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Re-aspire the servable with a fresh loader. + std::vector>> second_aspired_versions; + test_util::MockLoader* second_loader = new NiceMock(); + second_aspired_versions.push_back( + {id, std::unique_ptr(second_loader)}); + Notification second_load_called; + EXPECT_CALL(*second_loader, Load(_)).WillOnce(InvokeWithoutArgs([&]() { + second_load_called.Notify(); + return Status::OK(); + })); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(second_aspired_versions)); + + // Run the manager's background logic in a loop, but sans FlushServables(). + // Nothing should happen for now because the first loader isn't flushed. + std::unique_ptr reaspire_thread( + Env::Default()->StartThread(ThreadOptions(), "ReaspireThread", [&]() { + while (!second_load_called.HasBeenNotified()) { + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + Env::Default()->SleepForMicroseconds(1000 /* 1 ms */); + } + })); + Env::Default()->SleepForMicroseconds(50 * 1000 /* 50 ms */); + EXPECT_FALSE(second_load_called.HasBeenNotified()); + + // Flush the first loader. The manager should finally bring up the second + // loader. + FlushServables(); + second_load_called.WaitForNotification(); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 7515c5abe1f..4b3cb122617 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -318,8 +318,8 @@ Status BasicManager::ManageServableInternal( } else { PublishOnEventBus({harness->id(), ServableState::ManagerState::kStart, harness->status()}); - managed_map_.emplace(servable.id().name, harness); } + managed_map_.emplace(servable.id().name, harness); return Status::OK(); } @@ -334,6 +334,28 @@ Status BasicManager::ManageServable( }); } +Status BasicManager::StopManagingServable(const ServableId& id) { + mutex_lock l(mu_); + const auto it = FindHarnessInMap(id); + if (it == managed_map_.end()) { + LOG(ERROR) << "Request to delete harness for " << id + << ", but no such harness found in managed_map_"; + return errors::FailedPrecondition("This servable is not being managed: ", + id.DebugString()); + } + const auto state = it->second->state(); + if (state != LoaderHarness::State::kError && + state != LoaderHarness::State::kDisabled) { + LOG(ERROR) << "Request to delete harness for " << id + << ", but it is not in an end state. State: " << state; + return errors::FailedPrecondition( + "This servable is not in an end state and we cannot stop managing it: ", + id.DebugString(), " ", LoaderHarness::StateDebugString(state)); + } + managed_map_.erase(it); + return Status::OK(); +} + Status BasicManager::GetHealthyHarness(const ServableId& id, LoaderHarness** harness) { // Look up the request servable's harness. @@ -410,22 +432,24 @@ std::vector BasicManager::GetManagedServableNames() const { Status BasicManager::ExecuteLoad(LoaderHarness* harness) { PublishOnEventBus({harness->id(), ServableState::ManagerState::kLoading, harness->status()}); - const auto load_status = harness->Load(ResourceAllocation()); + // We save the id and the status of the harness so that we can publish them + // after Load(). We can't query harness again after Load() as it may be + // deleted by another thread that called StopManagingServable(). We don't hold + // the lock while calling Load() as the latter may block. + const ServableId id = harness->id(); + const Status load_status = harness->Load(ResourceAllocation()); + + if (!load_status.ok()) { + PublishOnEventBus({id, ServableState::ManagerState::kEnd, load_status}); + return load_status; + } { mutex_lock l(mu_); - - if (!load_status.ok()) { - PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, - harness->status()}); - DeleteHarness(harness->id()); - return load_status; - } - UpdateServingMap(); - PublishOnEventBus({harness->id(), ServableState::ManagerState::kAvailable, - harness->status()}); } + + PublishOnEventBus({id, ServableState::ManagerState::kAvailable, load_status}); return Status::OK(); } @@ -448,25 +472,24 @@ void BasicManager::CancelLoadServableRetry(const ServableId& id) { } Status BasicManager::ExecuteUnload(LoaderHarness* harness) { - { + // We save the id and the status of the harness so that we can publish them + // after Unload(). Unload() always succeeds, and hence doesn't affect + // harness->status(). We can't query harness again after Unload() as it may be + // deleted by another thread that called StopManagingServable(). We don't hold + // the lock while calling Unload() as the latter may block. + const ServableId id = harness->id(); + const Status pre_unload_status = [&] { // StartQuiescing() would have been already called. mutex_lock l(mu_); - PublishOnEventBus({harness->id(), ServableState::ManagerState::kUnloading, - harness->status()}); + PublishOnEventBus( + {id, ServableState::ManagerState::kUnloading, harness->status()}); UpdateServingMap(); harness->DoneQuiescing(); - } + return harness->status(); + }(); harness->Unload(); - - { - mutex_lock l(mu_); - auto iter = FindHarnessInMap(harness->id()); - PublishOnEventBus({iter->second->id(), ServableState::ManagerState::kEnd, - iter->second->status()}); - // This erase will lead to the LoaderHarness being deleted. - managed_map_.erase(iter); - } + PublishOnEventBus({id, ServableState::ManagerState::kEnd, pre_unload_status}); return Status::OK(); } @@ -603,7 +626,6 @@ Status BasicManager::ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) { harness->Error(error); PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, harness->status()}); - DeleteHarness(harness->id()); return error; } else { // Wait until at least one load/unload request finishes, then retry. diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index 0b1bf63b40c..a69339382b7 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -51,8 +51,11 @@ namespace serving { // can go on to load the servable after this by calling LoadServable. Loading // will also make the servable available to serve. Once you decide to unload it, // you can call UnloadServable on it, which will make it unavailable to serve, -// then unload the servable and delete it. After unload, or after hitting an -// error, the servable is no longer managed by the manager. +// then unload the servable. +// +// Servables are retained until StopManagingServable() is called. This allows a +// higher level manager with more information to decide when it's safe to forget +// about a servable. // // BasicManager tracks the resources (e.g. RAM) used by loaded servables, and // only allows loading new servables that fit within the overall resource pool. @@ -71,7 +74,8 @@ namespace serving { // // REQUIRES: // 1. Order of method calls - -// ManageServable*() -> LoadServable() -> UnloadServable(). +// ManageServable*() -> LoadServable() -> UnloadServable() -> +// StopManagingServable(). // 2. Do not schedule concurrent load and unloads of the same servable. // 3. Do not call load or unload multiple times on the same servable. // @@ -93,6 +97,7 @@ namespace serving { // ... // // TF_CHECK_OK(manager.UnloadServable(id)); +// TF_CHECK_OK(manager.UnmanagerServable(id)); class BasicManager : public Manager { public: struct Options { @@ -161,6 +166,10 @@ class BasicManager : public Manager { ServableData> servable, std::unique_ptr additional_state); + // Tells the manager to stop managing this servable. Requires that the + // servable is currently being managed and that its state is kEnd. + Status StopManagingServable(const ServableId& id); + // Returns the names of all the servables managed by this manager. The names // will be duplicate-free and not in any particular order. std::vector GetManagedServableNames() const; @@ -302,9 +311,6 @@ class BasicManager : public Manager { // other things, that prevents a subsequent load request from proceeding // concurrently. // - // If it fails, removes 'harness' from 'managed_map_' (which causes 'harness' - // to be deleted). - // // Argument 'mu_lock' is a lock held on 'mu_'. It is released temporarily via // 'num_ongoing_load_unload_executions_cv_'. Status ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) @@ -320,9 +326,6 @@ class BasicManager : public Manager { // // Upon completion (and regardless of the outcome), signals exit of the // execution phase by decrementing 'num_ongoing_load_unload_executions_'. - // - // If it fails, removes 'harness' from 'managed_map_' (which causes 'harness' - // to be deleted). Status ExecuteLoadOrUnload(const LoadOrUnloadRequest& request, LoaderHarness* harness); diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 85ab636809a..c26c390aa1e 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -58,6 +58,14 @@ constexpr int kNumVersionsPerServable = 2; constexpr int kNumThreads = 10; +MATCHER_P(EqualsServableState, servable_state, servable_state.DebugString()) { + if (arg == servable_state) { + return true; + } + *result_listener << arg.DebugString(); + return false; +} + // Creates a ServableData around a FakeLoader. ServableData> CreateServable( const ServableId& id, const Status load_status = Status::OK()) { @@ -167,6 +175,59 @@ TEST_P(BasicManagerTest, ServableHandleLatestVersionIsZero) { EXPECT_EQ(id, handle.id()); } +TEST_P(BasicManagerTest, StopManagingUnknownId) { + const ServableId id = {kServableName3, 1}; + EXPECT_FALSE(basic_manager_->StopManagingServable(id).ok()); +} + +TEST_P(BasicManagerTest, StopManagingActiveServable) { + const ServableId id = {kServableName3, 1}; + basic_manager_->ManageServable(CreateServable(id)); + EXPECT_FALSE(basic_manager_->StopManagingServable(id).ok()); +} + +TEST_P(BasicManagerTest, StopManagingDisabledServable) { + const ServableId id = {kServableName3, 1}; + basic_manager_->ManageServable(CreateServable(id)); + basic_manager_->LoadServable( + id, [](const Status& status) { TF_EXPECT_OK(status); }); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + basic_manager_->UnloadServable( + id, [](const Status& status) { TF_EXPECT_OK(status); }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + const optional> snapshot = + basic_manager_->GetManagedServableStateSnapshot(id); + EXPECT_EQ(LoaderHarness::State::kDisabled, snapshot->state); + const ServableState expected_state = {id, ServableState::ManagerState::kEnd, + Status::OK()}; + EXPECT_THAT(*servable_state_monitor_.GetState(id), + EqualsServableState(expected_state)); + + TF_ASSERT_OK(basic_manager_->StopManagingServable(id)); + EXPECT_FALSE(basic_manager_->GetManagedServableStateSnapshot(id)); +} + +TEST_P(BasicManagerTest, DontStopManagingOnError) { + const ServableId id = {kServableName, 7}; + const Status error_status = errors::Internal("An error."); + std::unique_ptr loader(new FakeLoader(7, error_status)); + basic_manager_->ManageServable({id, std::move(loader)}); + basic_manager_->LoadServable(id, [error_status](const Status& status) { + EXPECT_EQ(error_status, status); + }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + const optional> snapshot = + basic_manager_->GetManagedServableStateSnapshot(id); + EXPECT_EQ(LoaderHarness::State::kError, snapshot->state); + const ServableState expected_error_state = { + id, ServableState::ManagerState::kEnd, error_status}; + EXPECT_THAT(*servable_state_monitor_.GetState(id), + EqualsServableState(expected_error_state)); +} + TEST_P(BasicManagerTest, ServableHandleSpecificVersion) { ServableHandle handle; const ServableId id = {kServableName2, 1}; @@ -456,6 +517,7 @@ TEST_P(BasicManagerTest, DestructOnNonServingThread) { id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); + basic_manager_->StopManagingServable(id); // The servable has been deleted in this thread if there is no // thread-pool for load/unload. if (num_load_unload_threads_ == 0) { @@ -527,8 +589,8 @@ TEST_P(BasicManagerTest, MultipleUnloadServables) { {ServableState::ManagerState::kEnd}); basic_manager_->UnloadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); - EXPECT_EQ(error::NOT_FOUND, status.code()); - EXPECT_THAT(status.error_message(), HasSubstr("is not being managed")); + EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); + EXPECT_THAT(status.error_message(), HasSubstr("cannot be transitioned")); }); } @@ -552,14 +614,6 @@ TEST_P(BasicManagerTest, UnloadWithoutLoad) { }); } -MATCHER_P(EqualsServableState, servable_state, servable_state.DebugString()) { - if (arg == servable_state) { - return true; - } - *result_listener << arg.DebugString(); - return false; -} - TEST_P(BasicManagerTest, EventBusErroneousVersion) { const ServableId id = {kServableName, 3}; basic_manager_->ManageServable( @@ -1038,6 +1092,11 @@ TEST_F(ResourceConstrainedBasicManagerTest, InsufficientResources) { rejected_id, ServableState::ManagerState::kEnd, rejected_status}; EXPECT_THAT(*servable_state_monitor_.GetState(rejected_id), EqualsServableState(expected_error_state)); + + // Make sure we're still managing the rejected servable. + const optional> snapshot = + basic_manager_->GetManagedServableStateSnapshot(rejected_id); + EXPECT_EQ(LoaderHarness::State::kError, snapshot->state); } TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { diff --git a/tensorflow_serving/core/test_util/manager_test_util.cc b/tensorflow_serving/core/test_util/manager_test_util.cc index 086f750bce6..9576ab0322d 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.cc +++ b/tensorflow_serving/core/test_util/manager_test_util.cc @@ -23,6 +23,10 @@ AspiredVersionsManagerTestAccess::AspiredVersionsManagerTestAccess( AspiredVersionsManager* manager) : manager_(manager) {} +void AspiredVersionsManagerTestAccess::FlushServables() { + manager_->FlushServables(); +} + void AspiredVersionsManagerTestAccess::HandlePendingAspiredVersionsRequests() { manager_->HandlePendingAspiredVersionsRequests(); } diff --git a/tensorflow_serving/core/test_util/manager_test_util.h b/tensorflow_serving/core/test_util/manager_test_util.h index d84a312e2e4..f545487677a 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.h +++ b/tensorflow_serving/core/test_util/manager_test_util.h @@ -29,6 +29,9 @@ class AspiredVersionsManagerTestAccess { public: explicit AspiredVersionsManagerTestAccess(AspiredVersionsManager* manager); + // Invokes FlushServables() on the manager. + void FlushServables(); + // Invokes HandlePendingAspiredVersionsRequests() on the manager. void HandlePendingAspiredVersionsRequests(); From 42d2f07c28bf4d3be421a495b6b95c29b62153d0 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 30 Sep 2016 16:22:29 -0800 Subject: [PATCH 0022/8554] Change ServerCore's CustomModelConfigLoader signature to take in a UniquePtrWithDeps instead of a Target> manager so it can add new deps that need to persist after the end of the function. Change: 134846730 --- tensorflow_serving/model_servers/main.cc | 3 ++- tensorflow_serving/model_servers/server_core.cc | 2 +- tensorflow_serving/model_servers/server_core.h | 7 ++++++- .../model_servers/test_util/mock_server_core.h | 2 +- .../model_servers/test_util/server_core_test_util.cc | 2 +- 5 files changed, 11 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index f4ad0dd66b9..04be75e5342 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -63,6 +63,7 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/predict_impl.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" +using tensorflow::serving::AspiredVersionsManager; using tensorflow::serving::BatchingParameters; using tensorflow::serving::EventBus; using tensorflow::serving::Loader; @@ -113,7 +114,7 @@ tensorflow::Status CreateServableStateMonitor( tensorflow::Status LoadCustomModelConfig( const ::google::protobuf::Any& any, EventBus* servable_event_bus, - Target>* target) { + UniquePtrWithDeps* manager) { CHECK(false) // Crash ok << "ModelServer does not yet support custom model config."; } diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index aef178dee06..22e3ec5a6ef 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -183,7 +183,7 @@ Status ServerCore::AddModelsViaModelConfigList() { Status ServerCore::AddModelsViaCustomModelConfig() { return custom_model_config_loader_(config_.custom_model_config(), - servable_event_bus_.get(), manager_.get()); + servable_event_bus_.get(), &manager_); } Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index a761a498c2a..b06972dc80e 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -84,9 +84,14 @@ class ServerCore { std::function* event_bus, std::unique_ptr* monitor)>; + // A function that's responsible for instantiating and connecting the + // necessary custom sources and source adapters to the manager based on a + // passed in config (any). + // The expected pattern is that ownership of the created sources/source + // adapters can be transferred to the manager. using CustomModelConfigLoader = std::function* event_bus, - Target>* target)>; + UniquePtrWithDeps* manager)>; // Creates a ServerCore instance with all the models and sources per the // ModelServerConfig. diff --git a/tensorflow_serving/model_servers/test_util/mock_server_core.h b/tensorflow_serving/model_servers/test_util/mock_server_core.h index 31de46c920a..a1251c8320b 100644 --- a/tensorflow_serving/model_servers/test_util/mock_server_core.h +++ b/tensorflow_serving/model_servers/test_util/mock_server_core.h @@ -42,7 +42,7 @@ class MockServerCore : public ServerCore { }, // ServerCore::ServableStateMonitorCreator [](const ::google::protobuf::Any& any, EventBus* event_bus, - Target>* target) -> Status { + UniquePtrWithDeps* manager) -> Status { return Status::OK(); }, // ServerCore::CustomModelConfigLoader ServerCoreConfig()) {} diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 1dd546892c5..cdad830bbd7 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -61,7 +61,7 @@ Status ServerCoreTest::CreateServerCore( return Status::OK(); }, // ServerCore::ServableStateMonitor [](const ::google::protobuf::Any& any, EventBus* event_bus, - Target>* target) -> Status { + UniquePtrWithDeps* manager) -> Status { return Status::OK(); }, // ServerCore::CustomModelConfigLoader GetTestServerCoreConfig(), From d598caf99046d6196a906d8b234100d3becca626 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Tue, 4 Oct 2016 14:30:45 -0800 Subject: [PATCH 0023/8554] Adds private methods to set and get the number of load/unload threads used in the BasicManager and the AspiredVersionsManager. - When setting the threads, we block all new load/unload requests while we destruct the old executor, create a new one and swap it with the old one. Change: 135160140 --- tensorflow_serving/core/BUILD | 1 + .../core/aspired_versions_manager.cc | 9 ++ .../core/aspired_versions_manager.h | 8 ++ tensorflow_serving/core/basic_manager.cc | 79 ++++++++--- tensorflow_serving/core/basic_manager.h | 27 +++- tensorflow_serving/core/basic_manager_test.cc | 131 ++++++++++++++++++ .../core/test_util/manager_test_util.cc | 21 +++ .../core/test_util/manager_test_util.h | 19 +++ 8 files changed, 271 insertions(+), 24 deletions(-) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index d96f2e4b281..3b22d6169f5 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -440,6 +440,7 @@ cc_test( ":servable_state_monitor", "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:fake_loader", + "//tensorflow_serving/core/test_util:manager_test_util", "//tensorflow_serving/core/test_util:mock_loader", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:any_ptr", diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index c3a0bef0792..c8c3af0cb52 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -398,5 +398,14 @@ void AspiredVersionsManager::InvokePolicyAndExecuteAction() { PerformAction(*next_action); } +void AspiredVersionsManager::SetNumLoadUnloadThreads( + const uint32 num_load_unload_threads) { + basic_manager_->SetNumLoadUnloadThreads(num_load_unload_threads); +} + +uint32 AspiredVersionsManager::num_load_unload_threads() const { + return basic_manager_->num_load_unload_threads(); +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index a8254c390ac..8d0e342ee1e 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -232,6 +232,14 @@ class AspiredVersionsManager : public Manager, void InvokePolicyAndExecuteAction() LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_); + // Sets the number of load/unload threads. + // + // We immediately block all new load/unload requests while the current + // executor is destructed, a new one is created and then swapped with the + // current one. + void SetNumLoadUnloadThreads(uint32 num_load_unload_threads); + uint32 num_load_unload_threads() const; + std::unique_ptr aspired_version_policy_; // Aspired-versions requests pending to be processed, keyed by servable name. diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 4b3cb122617..e16b4e605a2 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -37,6 +37,26 @@ limitations under the License. namespace tensorflow { namespace serving { +namespace { + +std::unique_ptr CreateLoadUnloadExecutor( + Env* const env, const uint32 num_load_unload_threads) { + std::unique_ptr load_unload_executor; + if (num_load_unload_threads == 0) { + LOG(INFO) << "Creating InlineExecutor for BasicManager."; + load_unload_executor.reset(new InlineExecutor()); + } else { + LOG(INFO) << "Creating ThreadPoolExecutor for BasicManager with " + "num_load_unload_threads: " + << num_load_unload_threads; + load_unload_executor.reset(new ThreadPoolExecutor( + env, "BasicManager_LoadUnload_ThreadPool", num_load_unload_threads)); + } + return load_unload_executor; +} + +} // namespace + struct BasicManager::ServingMap::EqRequest { bool operator()(const ServableRequest& lhs, const ServableRequest& rhs) const { @@ -185,42 +205,38 @@ void BasicManager::ServingMap::Update(const ManagedMap& managed_map) { Status BasicManager::Create(Options options, std::unique_ptr* manager) { - std::unique_ptr load_unload_executor; - if (options.num_load_unload_threads == 0) { - LOG(INFO) << "Using InlineExecutor for BasicManager."; - load_unload_executor.reset(new InlineExecutor()); - } else { - LOG(INFO) << "Using ThreadPoolExecutor for BasicManager with " - "num_load_unload_threads: " - << options.num_load_unload_threads; - load_unload_executor.reset(new ThreadPoolExecutor( - options.env, "BasicManager_LoadUnload_ThreadPool", - options.num_load_unload_threads)); - } - LoaderHarness::Options harness_options; harness_options.max_num_load_retries = options.max_num_load_retries; harness_options.load_retry_interval_micros = options.load_retry_interval_micros; - manager->reset(new BasicManager(std::move(load_unload_executor), + manager->reset(new BasicManager(options.env, options.num_load_unload_threads, std::move(options.resource_tracker), options.servable_event_bus, harness_options)); return Status::OK(); } -BasicManager::BasicManager(std::unique_ptr load_unload_executor, +BasicManager::BasicManager(Env* const env, const uint32 num_load_unload_threads, std::unique_ptr resource_tracker, EventBus* servable_event_bus, const LoaderHarness::Options& harness_options) : harness_options_(harness_options), - servable_event_bus_(servable_event_bus) { - load_unload_executor_ = std::move(load_unload_executor); + servable_event_bus_(servable_event_bus), + env_(env), + num_load_unload_threads_(num_load_unload_threads) { + { + mutex_lock l(num_load_unload_threads_mu_); + load_unload_executor_ = + CreateLoadUnloadExecutor(env_, num_load_unload_threads_); + } resource_tracker_ = std::move(resource_tracker); } BasicManager::~BasicManager() { - // Reset the executor first to finish all pending loads/unloads. - load_unload_executor_.reset(); + { + mutex_lock l(num_load_unload_threads_mu_); + // Reset the executor first to finish all pending loads/unloads. + load_unload_executor_.reset(); + } UnloadAllServables(); } @@ -523,6 +539,22 @@ Status BasicManager::ExecuteLoadOrUnload(const LoadOrUnloadRequest& request, return execution_status; } +void BasicManager::SetNumLoadUnloadThreads( + const uint32 num_load_unload_threads) { + mutex_lock l(num_load_unload_threads_mu_); + + load_unload_executor_.reset(); + num_load_unload_threads_ = num_load_unload_threads; + load_unload_executor_ = + CreateLoadUnloadExecutor(env_, num_load_unload_threads_); +} + +uint32 BasicManager::num_load_unload_threads() const { + mutex_lock l(num_load_unload_threads_mu_); + + return num_load_unload_threads_; +} + void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, DoneCallback done_callback) { const Status status = [&]() { @@ -545,9 +577,12 @@ void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, done_callback(status); return; } - load_unload_executor_->Schedule([this, request, done_callback]() { - HandleLoadOrUnloadRequest(request, done_callback); - }); + { + mutex_lock l(num_load_unload_threads_mu_); + load_unload_executor_->Schedule([this, request, done_callback]() { + HandleLoadOrUnloadRequest(request, done_callback); + }); + } } void BasicManager::HandleLoadOrUnloadRequest(const LoadOrUnloadRequest& request, diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index a69339382b7..d580ade21bf 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -44,6 +44,10 @@ limitations under the License. namespace tensorflow { namespace serving { +namespace test_util { +class BasicManagerTestAccess; +} // namespace test_util + // Helps manage the lifecycle of servables including loading, serving and // unloading them. The manager accepts servables in the form of Loaders. // @@ -241,7 +245,10 @@ class BasicManager : public Manager { void UnloadServable(const ServableId& id, DoneCallback done_callback); private: - BasicManager(std::unique_ptr load_unload_executor, + friend class AspiredVersionsManager; + friend class test_util::BasicManagerTestAccess; + + BasicManager(Env* env, uint32 num_load_unload_threads, std::unique_ptr resource_tracker, EventBus* servable_event_bus, const LoaderHarness::Options& harness_options); @@ -342,6 +349,17 @@ class BasicManager : public Manager { // are ready to be served. void UpdateServingMap() EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Sets the number of load/unload threads. + // + // We block all new load/unload requests while the old thread pool is + // destructed, a new one is created and then swapped with the old one. Note + // that destructing the old thread pool blocks until all threads are done, so + // it could block for a long time. + void SetNumLoadUnloadThreads(uint32 num_load_unload_threads) + LOCKS_EXCLUDED(num_load_unload_threads_mu_); + uint32 num_load_unload_threads() const + LOCKS_EXCLUDED(num_load_unload_threads_mu_); + struct HashString { uint64 operator()(const string& str) const { return Hash64(str); } }; @@ -441,8 +459,13 @@ class BasicManager : public Manager { // serially, which guarantees that request i’s decision phase can complete // before considering request i+1's so there’s no starvation. + Env* const env_; + + mutable mutex num_load_unload_threads_mu_; + uint32 num_load_unload_threads_ GUARDED_BY(num_load_unload_threads_mu_); // The executor used for executing load and unload of servables. - std::unique_ptr load_unload_executor_; + std::unique_ptr load_unload_executor_ + GUARDED_BY(num_load_unload_threads_mu_); // Used to serialize the decision phases of the load/unload requests. mutable mutex load_unload_decision_phase_mu_; diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index c26c390aa1e..0d74cc69e94 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader.h" +#include "tensorflow_serving/core/test_util/manager_test_util.h" #include "tensorflow_serving/core/test_util/mock_loader.h" #include "tensorflow_serving/util/any_ptr.h" #include "tensorflow_serving/util/event_bus.h" @@ -791,6 +792,136 @@ TEST_P(BasicManagerTest, InterleavedLoadsAndUnloads) { } } +class SetNumLoadUnloadThreadsBasicManagerTest : public ::testing::Test { + protected: + SetNumLoadUnloadThreadsBasicManagerTest() { + BasicManager::Options options; + options.num_load_unload_threads = 0; + options.max_num_load_retries = 10; + options.load_retry_interval_micros = 0; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager_)); + } + + std::unique_ptr basic_manager_; +}; + +TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, ThreadPoolSwapped) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + manager_test_access.SetNumLoadUnloadThreads(2); + EXPECT_EQ(2, manager_test_access.num_load_unload_threads()); + + const auto load_done_fn = [&](const Status& status) { + TF_ASSERT_OK(status); + // Tests whether the threadpools are actually swapped in + // SetNumLoadUnloadThreads(). + static thread_local int per_thread_load_ctr = 0; + ++per_thread_load_ctr; + EXPECT_EQ(1, per_thread_load_ctr); + }; + + const ServableId id0 = {kServableName3, 0}; + basic_manager_->ManageServable(CreateServable(id0)); + basic_manager_->LoadServable(id0, load_done_fn); + + manager_test_access.SetNumLoadUnloadThreads(0); + EXPECT_EQ(0, manager_test_access.num_load_unload_threads()); + + const ServableId id1 = {kServableName3, 1}; + basic_manager_->ManageServable(CreateServable(id1)); + basic_manager_->LoadServable(id1, load_done_fn); + + // Force the manager to finish before deleting the notifications. + basic_manager_.reset(); +} + +TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, + ThreadPoolsNotAliveSimultaneously) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + manager_test_access.SetNumLoadUnloadThreads(1); + EXPECT_EQ(1, manager_test_access.num_load_unload_threads()); + + std::set data_race_set; + const auto data_race_fn = [&](const Status& status) { + // This line will cause a data race if both the loads happen simultaneously + // on different threads. This will be caught by the ThreadSanitizer, causing + // the test to fail. + data_race_set.insert("string"); + }; + + const ServableId id0 = {kServableName3, 0}; + basic_manager_->ManageServable(CreateServable(id0)); + Notification notify_for_setting; + Notification continue_load; + basic_manager_->LoadServable(id0, [&](const Status& status) { + notify_for_setting.Notify(); + continue_load.WaitForNotification(); + data_race_fn(status); + }); + + { + ThreadPoolExecutor executor(Env::Default(), "SetNumLoadUnloadThreads", + kNumThreads); + executor.Schedule([&]() { + notify_for_setting.WaitForNotification(); + manager_test_access.SetNumLoadUnloadThreads(1); + EXPECT_EQ(1, manager_test_access.num_load_unload_threads()); + }); + + executor.Schedule([&]() { + const ServableId id1 = {kServableName3, 1}; + basic_manager_->ManageServable(CreateServable(id1)); + continue_load.Notify(); + basic_manager_->LoadServable( + id1, [&](const Status& status) { data_race_fn(status); }); + }); + } + + // Force the manager to finish before deleting the notifications. + basic_manager_.reset(); +} + +// Tests whether the fast-load scenario works. In the fast-load scenario we try +// to load a bunch of servables as fast as possible using a lot of threads. +TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, FastLoad) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + const uint32 prev_num_load_unload_threads = + manager_test_access.num_load_unload_threads(); + manager_test_access.SetNumLoadUnloadThreads(32); + EXPECT_EQ(32, manager_test_access.num_load_unload_threads()); + + { + ThreadPoolExecutor executor(Env::Default(), "FirstThreadPoolLoads", + kNumThreads); + for (int i = 0; i < 20; ++i) { + executor.Schedule([this, i]() { + const ServableId id = {kServableName3, i}; + basic_manager_->ManageServable(CreateServable(id)); + basic_manager_->LoadServable( + id, [](const Status& status) { TF_ASSERT_OK(status); }); + // We don't wait for load to be done here because we want to test that + // SetNumLoadUnloadThreads() waits properly till all queued loads are + // finished. If a queued load hasn't been finished the corresponding + // UnloadServable() will fail. + }); + } + } + + manager_test_access.SetNumLoadUnloadThreads(prev_num_load_unload_threads); + EXPECT_EQ(prev_num_load_unload_threads, + manager_test_access.num_load_unload_threads()); + + { + ThreadPoolExecutor executor(Env::Default(), "Unloads", kNumThreads); + for (int i = 0; i < 20; ++i) { + executor.Schedule([this, i]() { + const ServableId id = {kServableName3, i}; + basic_manager_->UnloadServable( + id, [](const Status& status) { TF_ASSERT_OK(status); }); + }); + } + } +} + TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { const ServableId id = {kServableName3, 0}; mutex status_mu; diff --git a/tensorflow_serving/core/test_util/manager_test_util.cc b/tensorflow_serving/core/test_util/manager_test_util.cc index 9576ab0322d..60b2fbf48c8 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.cc +++ b/tensorflow_serving/core/test_util/manager_test_util.cc @@ -35,6 +35,27 @@ void AspiredVersionsManagerTestAccess::InvokePolicyAndExecuteAction() { manager_->InvokePolicyAndExecuteAction(); } +void AspiredVersionsManagerTestAccess::SetNumLoadUnloadThreads( + const uint32 num_load_unload_threads) { + manager_->SetNumLoadUnloadThreads(num_load_unload_threads); +} + +uint32 AspiredVersionsManagerTestAccess::num_load_unload_threads() const { + return manager_->num_load_unload_threads(); +} + +BasicManagerTestAccess::BasicManagerTestAccess(BasicManager* manager) + : manager_(manager) {} + +void BasicManagerTestAccess::SetNumLoadUnloadThreads( + const uint32 num_load_unload_threads) { + manager_->SetNumLoadUnloadThreads(num_load_unload_threads); +} + +uint32 BasicManagerTestAccess::num_load_unload_threads() const { + return manager_->num_load_unload_threads(); +} + CachingManagerTestAccess::CachingManagerTestAccess(CachingManager* manager) : manager_(manager) {} diff --git a/tensorflow_serving/core/test_util/manager_test_util.h b/tensorflow_serving/core/test_util/manager_test_util.h index f545487677a..2b50a6c05c6 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.h +++ b/tensorflow_serving/core/test_util/manager_test_util.h @@ -38,12 +38,31 @@ class AspiredVersionsManagerTestAccess { // Invokes InvokePolicyAndExecuteAction() on the manager. void InvokePolicyAndExecuteAction(); + void SetNumLoadUnloadThreads(uint32 num_load_unload_threads); + + uint32 num_load_unload_threads() const; + private: AspiredVersionsManager* const manager_; TF_DISALLOW_COPY_AND_ASSIGN(AspiredVersionsManagerTestAccess); }; +// A test utility that provides access to private BasicManager members. +class BasicManagerTestAccess { + public: + explicit BasicManagerTestAccess(BasicManager* manager); + + void SetNumLoadUnloadThreads(uint32 num_load_unload_threads); + + uint32 num_load_unload_threads() const; + + private: + BasicManager* const manager_; + + TF_DISALLOW_COPY_AND_ASSIGN(BasicManagerTestAccess); +}; + // A test utility that provides access to private CachingManager members. class CachingManagerTestAccess { public: From a9b828bf7a03e3e0ee6a0f85ae4e24207bcea520 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Oct 2016 14:50:50 -0800 Subject: [PATCH 0024/8554] Fix AspiredVersionsManager to pass the full set of managed versions to the AspiredVersionPolicy class. Change: 135163428 --- .../core/aspired_versions_manager.cc | 7 +-- .../core/aspired_versions_manager_test.cc | 56 ++++++++++++++++--- 2 files changed, 51 insertions(+), 12 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index c8c3af0cb52..887ff94befc 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -302,19 +302,18 @@ bool AspiredVersionsManager::ContainsAnyReaspiredVersions( optional AspiredVersionsManager::GetNextAction() { std::vector> actions; - std::vector aspired_state_snapshots; for (const string& servable_name : basic_manager_->GetManagedServableNames()) { - aspired_state_snapshots.clear(); + std::vector aspired_state_snapshots; for (const ServableStateSnapshot& state_snapshot : basic_manager_->GetManagedServableStateSnapshots( servable_name)) { aspired_state_snapshots.push_back( {state_snapshot.id, state_snapshot.state, state_snapshot.additional_state->is_aspired}); - actions.emplace_back( - aspired_version_policy_->GetNextAction(aspired_state_snapshots)); } + actions.emplace_back( + aspired_version_policy_->GetNextAction(aspired_state_snapshots)); } std::sort(actions.begin(), actions.end(), CompareActions()); diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 867c0cc3a8e..4a1f3fa9971 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -39,6 +39,7 @@ namespace serving { namespace { using ::testing::_; +using ::testing::DoAll; using ::testing::Invoke; using ::testing::InvokeWithoutArgs; using ::testing::NiceMock; @@ -52,6 +53,14 @@ constexpr char kServableName2[] = "kServableName2"; constexpr int kNumVersionsPerServable = 2; constexpr int kNumTotalVersions = 4; +// Creates an aspired-versions entry with 'id' and a FakeLoader whose servable +// is id.version. +ServableData> CreateAspiredVersion( + const ServableId& id) { + std::unique_ptr loader(new FakeLoader(id.version)); + return CreateServableData(id, std::move(loader)); +} + class AspiredVersionsManagerTest : public ::testing::TestWithParam { protected: AspiredVersionsManagerTest() @@ -74,14 +83,6 @@ class AspiredVersionsManagerTest : public ::testing::TestWithParam { AspiredVersionsManager::Create(std::move(manager_options), &manager_)); } - // Creates an aspired-versions entry with 'id' and a FakeLoader whose servable - // is id.version. - ServableData> CreateAspiredVersion( - const ServableId& id) { - std::unique_ptr loader(new FakeLoader(id.version)); - return CreateServableData(id, std::move(loader)); - } - // Creates an aspired-versions entry with 'id' and an error (and no loader). ServableData> CreateErroneousAspiredVersion( const ServableId& id) { @@ -916,6 +917,45 @@ TEST_P(AspiredVersionsManagerTest, second_load_called.WaitForNotification(); } +class MockAspiredVersionPolicy : public AspiredVersionPolicy { + public: + MOCK_CONST_METHOD1(GetNextAction, + optional( + const std::vector&)); +}; + +TEST(AspiredVersionsManagerTest, CallPolicyWithAllVersions) { + std::unique_ptr manager; + AspiredVersionsManager::Options manager_options; + MockAspiredVersionPolicy* policy = new MockAspiredVersionPolicy; + manager_options.aspired_version_policy = + std::unique_ptr(policy); + TF_CHECK_OK( + AspiredVersionsManager::Create(std::move(manager_options), &manager)); + std::set servables; + std::vector>> aspired_versions; + for (int i = 0; i < kNumVersionsPerServable; ++i) { + const ServableId id = {kServableName, i}; + aspired_versions.push_back(CreateAspiredVersion(id)); + servables.insert(id); + } + manager->GetAspiredVersionsCallback()(kServableName, + std::move(aspired_versions)); + test_util::AspiredVersionsManagerTestAccess(manager.get()) + .HandlePendingAspiredVersionsRequests(); + + std::vector all_versions; + EXPECT_CALL(*policy, GetNextAction(_)) + .WillOnce(Invoke([&all_versions]( + const std::vector& snapshots) { + all_versions = snapshots; + return nullopt; + })); + test_util::AspiredVersionsManagerTestAccess(manager.get()) + .InvokePolicyAndExecuteAction(); + EXPECT_EQ(kNumVersionsPerServable, all_versions.size()); +} + } // namespace } // namespace serving } // namespace tensorflow From 19deb7354e4747be81b13e45026a8dc707a1249d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 4 Oct 2016 15:53:37 -0800 Subject: [PATCH 0025/8554] Fix flakiness in aspired_versions_manager_test. Do not let the background thread run. Change: 135171601 --- tensorflow_serving/core/aspired_versions_manager_test.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 4a1f3fa9971..fa3bab86c45 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -928,6 +928,8 @@ TEST(AspiredVersionsManagerTest, CallPolicyWithAllVersions) { std::unique_ptr manager; AspiredVersionsManager::Options manager_options; MockAspiredVersionPolicy* policy = new MockAspiredVersionPolicy; + // The state manager thread won't be run automatically. + manager_options.manage_state_interval_micros = -1; manager_options.aspired_version_policy = std::unique_ptr(policy); TF_CHECK_OK( From 202ace3dbbf60ec9f3c2065fa459d3e5ba793a24 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 5 Oct 2016 09:02:30 -0800 Subject: [PATCH 0026/8554] Improve EagerLoad and EagerUnload policies to be aware of versions that are in transition and make smarter decision about loading/unloading new versions. The new algorithms are: EagerLoad 1 - Always allow loading first. 2 - If all aspired versions are ready, we can unload not-aspired versions. EagerUnload 1 - Always allow unloading first. 2 - If all not-aspired versions are either kDisabled or kError, we can load new aspired versions. Change: 135244768 --- .../core/aspired_versions_manager_test.cc | 77 ++++++++++++++++++- tensorflow_serving/core/eager_load_policy.cc | 15 +++- .../core/eager_load_policy_test.cc | 53 +++++++++++-- .../core/eager_unload_policy.cc | 15 +++- .../core/eager_unload_policy_test.cc | 48 ++++++++++++ 5 files changed, 199 insertions(+), 9 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index fa3bab86c45..05adee0132d 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -254,6 +254,12 @@ TEST_P(AspiredVersionsManagerTest, ListAvailableServableIds) { } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); + + manager_->GetAspiredVersionsCallback()(kServableName, {}); + HandlePendingAspiredVersionsRequests(); + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 0}, {ServableState::ManagerState::kEnd}); @@ -301,13 +307,18 @@ TEST_P(AspiredVersionsManagerTest, GetAvailableServableHandles) { } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); + + manager_->GetAspiredVersionsCallback()(kServableName, {}); + HandlePendingAspiredVersionsRequests(); + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 0}, {ServableState::ManagerState::kEnd}); WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 1}, {ServableState::ManagerState::kEnd}); - { const std::map> handles_after = manager_->GetAvailableServableHandles(); @@ -773,6 +784,70 @@ TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallyFails) { EqualsServableState(error_state)); } +// Tests the interaction between AspiredVersionsManager and the EagerLoadPolicy. +// Specifically, we want to make sure that the manager will not try to unload a +// serving version that is no longer aspired if the new aspired version was not +// able to start serving. +TEST_P(AspiredVersionsManagerTest, AspireErrorDontUnload) { + const std::vector expected_before = {{kServableName, 0}, + {kServableName, 1}, + {kServableName2, 0}, + {kServableName2, 1}}; + EXPECT_THAT(manager_->ListAvailableServableIds(), + UnorderedElementsAreArray(expected_before)); + + // Set stream kServableName to have servable 7. + // This causes 0 & 1 to be set to not aspired and 7 to be loaded, but 7 errors + // on load, so never moves to a loaded state. + { + std::vector>> aspired_versions; + const ServableId id = {kServableName, 7}; + std::unique_ptr loader( + new FakeLoader(7, errors::Internal("An error."))); + aspired_versions.push_back({id, std::move(loader)}); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Will try to load version 7 and fail. + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + } + + // The same servables as before as we can't unload the current servables after + // the failure to load the new one. + EXPECT_THAT(manager_->ListAvailableServableIds(), + UnorderedElementsAreArray(expected_before)); + + // Now successfully loading a new version should allow the older versions to + // be unloaded. + { + std::vector>> aspired_versions; + const ServableId id = {kServableName, 8}; + std::unique_ptr loader(new FakeLoader(8)); + aspired_versions.push_back({id, std::move(loader)}); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Will try to load version 8 and succeed. + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + + // Will unload version 0 and 1. + InvokePolicyAndExecuteAction(); + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 1}, + {ServableState::ManagerState::kEnd}); + } +} + TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { // This test exercises a scenario in which a servable has been unaspired, and // while it is still being managed (e.g. loading, serving or unloading) it diff --git a/tensorflow_serving/core/eager_load_policy.cc b/tensorflow_serving/core/eager_load_policy.cc index b4ff9f1ac97..53c8f56990b 100644 --- a/tensorflow_serving/core/eager_load_policy.cc +++ b/tensorflow_serving/core/eager_load_policy.cc @@ -14,6 +14,7 @@ limitations under the License. ==============================================================================*/ #include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/loader_harness.h" namespace tensorflow { namespace serving { @@ -27,6 +28,18 @@ optional EagerLoadPolicy::GetNextAction( } } + // Second, check if there are any aspired versions that are not ready. In that + // case we can't unload any versions. + const bool aspired_not_serving = + std::any_of(all_versions.begin(), all_versions.end(), + [](const AspiredServableStateSnapshot& version) { + return version.is_aspired && + version.state != LoaderHarness::State::kReady; + }); + if (aspired_not_serving) { + return nullopt; + } + // If there is no new aspired version, but a not-aspired version, unload the // latter. for (const auto& version : all_versions) { @@ -34,7 +47,7 @@ optional EagerLoadPolicy::GetNextAction( return {{Action::kUnload, version.id}}; } } - return {}; + return nullopt; } } // namespace serving diff --git a/tensorflow_serving/core/eager_load_policy_test.cc b/tensorflow_serving/core/eager_load_policy_test.cc index 1e84dc67e11..3b069e38846 100644 --- a/tensorflow_serving/core/eager_load_policy_test.cc +++ b/tensorflow_serving/core/eager_load_policy_test.cc @@ -40,19 +40,18 @@ TEST(EagerLoadPolicy, LoadsFirstAspired) { // Test that the first non-aspired version is unloaded when there are none to // load. -TEST(EagerLoadPolicy, UnLoadsFirstNonAspiredWhenNoneToLoad) { +TEST(EagerLoadPolicy, UnLoadsFirstNonAspiredWhenNoneLoading) { std::vector versions; versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 4}, LoaderHarness::State::kDisabled, false}); - versions.push_back({{"test", 5}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kDisabled, false}); + versions.push_back({{"test", 4}, LoaderHarness::State::kReady, false}); EagerLoadPolicy policy; const auto action = policy.GetNextAction(versions); ASSERT_TRUE(action); EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); - EXPECT_EQ(3, action->id.version); + EXPECT_EQ(2, action->id.version); } // Test that no action is returned (empty optional) when there are no versions @@ -70,6 +69,48 @@ TEST(EagerLoadPolicy, ReturnsNoActionWhenNone) { EXPECT_FALSE(action); } +TEST(EagerLoadPolicy, DoesNotUnloadWhenOtherNotReady) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); + + EagerLoadPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(EagerLoadPolicy, DoesNotUnloadWhenOtherInError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + + EagerLoadPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(EagerLoadPolicy, LoadingBlocksUnloadEvenIfOtherReady) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + + EagerLoadPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(EagerLoadPolicy, ErrorBlocksUnloadEvenIfOtherReady) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + + EagerLoadPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/eager_unload_policy.cc b/tensorflow_serving/core/eager_unload_policy.cc index 702e7fc7fdc..93461bc5fcb 100644 --- a/tensorflow_serving/core/eager_unload_policy.cc +++ b/tensorflow_serving/core/eager_unload_policy.cc @@ -29,7 +29,20 @@ optional EagerUnloadPolicy::GetNextAction( } } - // Second and only if no action was found earlier, iterate over all + // Second, see if there are any not-aspired versions that aren't in an end + // state (kDisabled or kError). If so, do nothing for now. + const bool not_aspired_not_finished = + std::any_of(all_versions.begin(), all_versions.end(), + [](const AspiredServableStateSnapshot& version) { + return !version.is_aspired && + version.state != LoaderHarness::State::kDisabled && + version.state != LoaderHarness::State::kError; + }); + if (not_aspired_not_finished) { + return nullopt; + } + + // Third and only if no action was found earlier, iterate over all // versions and find any in kNew that are aspired. Load the first if any. for (const auto& version : all_versions) { if (version.state == LoaderHarness::State::kNew && version.is_aspired) { diff --git a/tensorflow_serving/core/eager_unload_policy_test.cc b/tensorflow_serving/core/eager_unload_policy_test.cc index 7745d4386a8..1add44988bf 100644 --- a/tensorflow_serving/core/eager_unload_policy_test.cc +++ b/tensorflow_serving/core/eager_unload_policy_test.cc @@ -69,6 +69,54 @@ TEST(EagerUnloadPolicy, ReturnsNoActionWhenNone) { EXPECT_FALSE(action); } +TEST(EagerUnloadPolicy, DoesNotLoadWhenOthersStillUnloading) { + std::vector versions; + versions.push_back( + {{"test", 1}, LoaderHarness::State::kUnloadRequested, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); + + EagerUnloadPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(EagerUnloadPolicy, LoadIfUnaspiredIsError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); + + EagerUnloadPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(2, action->id.version); +} + +TEST(EagerUnloadPolicy, ErrorAndUnloadRequestedPreventLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back( + {{"test", 2}, LoaderHarness::State::kUnloadRequested, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + EagerUnloadPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(EagerUnloadPolicy, ErrorAndDisabledAllowLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kDisabled, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + EagerUnloadPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(3, action->id.version); +} + } // namespace } // namespace serving } // namespace tensorflow From 051f0fb38620c0174babaf215faa1ecb5b158bc8 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 6 Oct 2016 15:48:01 -0800 Subject: [PATCH 0027/8554] Adds a utility method to load servables fast. - By making the manager use a higher number of threads than at serving time. - Useful at server startup time. Change: 135422357 --- tensorflow_serving/core/BUILD | 14 ++++ .../core/aspired_versions_manager.h | 11 +++ .../core/load_servables_fast.cc | 76 +++++++++++++++++++ tensorflow_serving/core/load_servables_fast.h | 56 ++++++++++++++ tensorflow_serving/model_servers/BUILD | 1 + .../model_servers/server_core.cc | 26 ++++--- .../model_servers/server_core.h | 11 ++- 7 files changed, 178 insertions(+), 17 deletions(-) create mode 100644 tensorflow_serving/core/load_servables_fast.cc create mode 100644 tensorflow_serving/core/load_servables_fast.h diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 3b22d6169f5..88db5254cee 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -643,3 +643,17 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", ], ) + +cc_library( + name = "load_servables_fast", + srcs = ["load_servables_fast.cc"], + hdrs = ["load_servables_fast.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":aspired_versions_manager", + ":servable_state_monitor", + "@org_tensorflow//tensorflow/core:lib", + ], +) diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 8d0e342ee1e..701989c0b9f 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -42,8 +42,16 @@ limitations under the License. namespace tensorflow { namespace serving { +class AspiredVersionsManager; + namespace internal { + class AspiredVersionsManagerTargetImpl; + +Status ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + const std::function& wait_until_loaded_fn, uint32 num_threads); + } // namespace internal namespace test_util { @@ -170,6 +178,9 @@ class AspiredVersionsManager : public Manager, private: friend class internal::AspiredVersionsManagerTargetImpl; friend class test_util::AspiredVersionsManagerTestAccess; + friend Status internal::ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + const std::function& wait_until_loaded_fn, uint32 num_threads); AspiredVersionsManager( int64 manage_state_interval_micros, Env* env, diff --git a/tensorflow_serving/core/load_servables_fast.cc b/tensorflow_serving/core/load_servables_fast.cc new file mode 100644 index 00000000000..6fedaf126bf --- /dev/null +++ b/tensorflow_serving/core/load_servables_fast.cc @@ -0,0 +1,76 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/load_servables_fast.h" + +#include +#include + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow_serving/core/servable_state.h" +#include "tensorflow_serving/core/source.h" +#include "tensorflow_serving/core/target.h" + +namespace tensorflow { +namespace serving { + +namespace internal { + +Status ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + const std::function& wait_until_loaded_fn, + const uint32 num_threads) { + const uint32 prev_num_load_unload_threads = + manager->num_load_unload_threads(); + manager->SetNumLoadUnloadThreads(num_threads); + ConnectSourceToTarget(source, manager); + const Status status = wait_until_loaded_fn(); + manager->SetNumLoadUnloadThreads(prev_num_load_unload_threads); + return status; +} + +} // namespace internal + +Status ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + ServableStateMonitor* servable_state_monitor, + const std::vector& servables, const uint32 num_threads) { + return internal::ConnectSourceWithFastInitialLoad( + manager, source, + [&]() { + std::map states_reached; + const bool all_servables_available = + servable_state_monitor->WaitUntilServablesReachState( + servables, ServableState::ManagerState::kAvailable, + &states_reached); + if (!all_servables_available) { + string message = "Some models did not become available: {"; + for (const auto& id_and_state : states_reached) { + if (id_and_state.second != + ServableState::ManagerState::kAvailable) { + strings::StrAppend(&message, id_and_state.first.DebugString(), + ", "); + } + } + strings::StrAppend(&message, "}"); + return errors::Unknown(message); + } + return Status::OK(); + }, + num_threads); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/load_servables_fast.h b/tensorflow_serving/core/load_servables_fast.h new file mode 100644 index 00000000000..e378d3c83d3 --- /dev/null +++ b/tensorflow_serving/core/load_servables_fast.h @@ -0,0 +1,56 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ +#define TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ + +#include +#include + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/aspired_versions_manager.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/servable_state_monitor.h" + +namespace tensorflow { +namespace serving { + +// Connects 'source' to 'manager', and speeds up loading of the servables +// matching 'initial_servables'. The speeding up is accomplished by boosting the +// number of threads used for loading until the initial servables have been +// loaded, and then resetting it to the manager's originally configured value. +Status ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + ServableStateMonitor* servable_state_monitor, + const std::vector& initial_servables, + uint32 num_threads = 1 * port::NumSchedulableCPUs()); + +//// +// Implementation detail. API readers may skip. +/// + +namespace internal { + +Status ConnectSourceWithFastInitialLoad( + AspiredVersionsManager* manager, Source>* source, + const std::function& wait_until_loaded_fn, uint32 num_threads); + +} // namespace internal + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_LOAD_SERVABLES_FAST_H_ diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index d3ec2c0760c..4bd0db28476 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -46,6 +46,7 @@ cc_library( "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:aspired_versions_manager", "//tensorflow_serving/core:eager_load_policy", + "//tensorflow_serving/core:load_servables_fast", "//tensorflow_serving/core:servable_state_monitor", "//tensorflow_serving/core:source", "//tensorflow_serving/core:source_adapter", diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 22e3ec5a6ef..2743987ff65 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -21,6 +21,7 @@ limitations under the License. #include "google/protobuf/wrappers.pb.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/load_servables_fast.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" @@ -151,15 +152,9 @@ Status ServerCore::AddModelsViaModelConfigList() { ValidateAllListedModelsAreOfSamePlatform(config_, &model_platform)); // Create the source adapter if we haven't done so. - bool is_first_config = storage_path_source_ == nullptr; - ModelServerSourceAdapter* source_adapter = nullptr; + const bool is_first_config = storage_path_source_ == nullptr; if (is_first_config) { model_platform_ = model_platform; - std::unique_ptr new_source_adapter; - TF_RETURN_IF_ERROR(CreateSourceAdapter(model_platform_, manager_.get(), - &new_source_adapter)); - source_adapter = new_source_adapter.get(); - manager_.AddDependency(std::move(new_source_adapter)); } // Determine if config transition is legal. @@ -173,10 +168,21 @@ Status ServerCore::AddModelsViaModelConfigList() { const FileSystemStoragePathSourceConfig source_config = CreateStoragePathSourceConfig(config_); if (is_first_config) { + std::unique_ptr source_adapter; + TF_RETURN_IF_ERROR(CreateSourceAdapter(model_platform_, &source_adapter)); TF_RETURN_IF_ERROR( - CreateFileSystemStoragePathSource(source_config, source_adapter)); + CreateFileSystemStoragePathSource(source_config, source_adapter.get())); + std::vector static_servables; + for (const auto& model : config_.model_config_list().config()) { + static_servables.push_back(ServableRequest::Latest(model.name())); + } + TF_RETURN_IF_ERROR(ConnectSourceWithFastInitialLoad( + manager_.get(), source_adapter.get(), servable_state_monitor_.get(), + static_servables)); + manager_.AddDependency(std::move(source_adapter)); } else { TF_RETURN_IF_ERROR(ReloadFileSystemStoragePathSourceConfig(source_config)); + TF_RETURN_IF_ERROR(WaitUntilConfiguredModelsAvailable()); } return Status::OK(); } @@ -213,7 +219,6 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { switch (config_.config_case()) { case ModelServerConfig::kModelConfigList: { TF_RETURN_IF_ERROR(AddModelsViaModelConfigList()); - TF_RETURN_IF_ERROR(WaitUntilConfiguredModelsAvailable()); break; } case ModelServerConfig::kCustomModelConfig: { @@ -231,10 +236,9 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { } Status ServerCore::CreateSourceAdapter( - const string& model_platform, Target>* target, + const string& model_platform, std::unique_ptr* adapter) { TF_RETURN_IF_ERROR(source_adapter_creator_(model_platform, adapter)); - ConnectSourceToTarget(adapter->get(), target); return Status::OK(); } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index b06972dc80e..7fa7e64f947 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -159,10 +159,9 @@ class ServerCore { Status CreateAspiredVersionsManager( std::unique_ptr* manager); - // Creates a platform-specific Loader Source and connects it to the supplied - // target. + // Creates a platform-specific Loader Source. Status CreateSourceAdapter( - const string& model_platform, Target>* target, + const string& model_platform, std::unique_ptr* adapter); // Creates a FileSystemStoragePathSourceConfig from the ModelConfigList of @@ -175,9 +174,9 @@ class ServerCore { Status WaitUntilConfiguredModelsAvailable() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); - // Creates a FileSystemStoragePathSource, connects it to the supplied - // target, stores the pointer in 'storage_path_source_' and transfers the - // ownership to 'manager_'. + // Creates a FileSystemStoragePathSource, connects it to the supplied target, + // stores the pointer in 'storage_path_source_' and transfers the ownership to + // 'manager_'. Status CreateFileSystemStoragePathSource( const FileSystemStoragePathSourceConfig& source_config, Target* target) EXCLUSIVE_LOCKS_REQUIRED(config_mu_); From 2c60105c81e4dfd3d6812d976b7c913a93249304 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Fri, 7 Oct 2016 13:37:18 -0800 Subject: [PATCH 0028/8554] Changes default num_threads for ConnectSourceWithFastInitialLoad to 4*NumCPUs. Change: 135518676 --- tensorflow_serving/core/load_servables_fast.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/core/load_servables_fast.h b/tensorflow_serving/core/load_servables_fast.h index e378d3c83d3..8a5ca0df617 100644 --- a/tensorflow_serving/core/load_servables_fast.h +++ b/tensorflow_serving/core/load_servables_fast.h @@ -36,7 +36,7 @@ Status ConnectSourceWithFastInitialLoad( AspiredVersionsManager* manager, Source>* source, ServableStateMonitor* servable_state_monitor, const std::vector& initial_servables, - uint32 num_threads = 1 * port::NumSchedulableCPUs()); + uint32 num_threads = 4 * port::NumSchedulableCPUs()); //// // Implementation detail. API readers may skip. From cc47b3f18d89e232b79db5e8d244115f87a8482e Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 11 Oct 2016 09:46:32 -0800 Subject: [PATCH 0029/8554] Add a total model RAM limit parameter to ModelServer. Change: 135813669 --- tensorflow_serving/model_servers/BUILD | 2 ++ .../model_servers/server_core.cc | 21 +++++++++++++++++++ .../model_servers/server_core.h | 6 ++++++ 3 files changed, 29 insertions(+) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 4bd0db28476..4b4848fee56 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -51,6 +51,8 @@ cc_library( "//tensorflow_serving/core:source", "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", + "//tensorflow_serving/resources:resource_tracker", + "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", "//tensorflow_serving/util:event_bus", diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 2743987ff65..cfd718329b8 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/core/load_servables_fast.h" #include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" @@ -278,6 +279,9 @@ Status ServerCore::CreateAspiredVersionsManager( std::unique_ptr* const manager) { std::unique_ptr aspired_versions_manager; AspiredVersionsManager::Options manager_options; + std::unique_ptr resource_tracker; + TF_RETURN_IF_ERROR(CreateResourceTracker(&resource_tracker)); + manager_options.resource_tracker = std::move(resource_tracker); manager_options.servable_event_bus = servable_event_bus_.get(); manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); manager_options.num_load_unload_threads = @@ -287,6 +291,23 @@ Status ServerCore::CreateAspiredVersionsManager( return AspiredVersionsManager::Create(std::move(manager_options), manager); } +Status ServerCore::CreateResourceTracker( + std::unique_ptr* resource_tracker) { + ResourceUtil::Options resource_util_options; + resource_util_options.devices[device_types::kMain] = 1; + auto resource_util = + std::unique_ptr(new ResourceUtil(resource_util_options)); + ResourceAllocation total_resources; + ResourceAllocation::Entry* main_memory_resource = + total_resources.add_resource_quantities(); + main_memory_resource->mutable_resource()->set_device(device_types::kMain); + main_memory_resource->mutable_resource()->set_kind(resource_kinds::kRamBytes); + main_memory_resource->set_quantity( + server_core_config_.total_model_memory_limit_bytes); + return ResourceTracker::Create(total_resources, std::move(resource_util), + resource_tracker); +} + // ************************************************************************ // Request Processing. // ************************************************************************ diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 7fa7e64f947..09e3e7738ce 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -54,6 +54,8 @@ namespace serving { // ServerCore tuning parameters. struct ServerCoreConfig { + // Total model size limit, in terms of main memory, in bytes. + uint64 total_model_memory_limit_bytes = ULLONG_MAX; // Time interval between file-system polls, in seconds. int32 file_system_poll_wait_seconds = 30; // The number of threads used to load and unload models. If set to 0, then @@ -159,6 +161,10 @@ class ServerCore { Status CreateAspiredVersionsManager( std::unique_ptr* manager); + // Creates a ResourceTracker. + Status CreateResourceTracker( + std::unique_ptr* resource_tracker); + // Creates a platform-specific Loader Source. Status CreateSourceAdapter( const string& model_platform, From 80b7c59828598046984bd8629bf13bdbc97e74b4 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Tue, 11 Oct 2016 11:01:00 -0800 Subject: [PATCH 0030/8554] Changes default num_threads for ConnectSourceWithFastInitialLoad. Change: 135823233 --- tensorflow_serving/model_servers/server_core.cc | 2 +- tensorflow_serving/model_servers/server_core.h | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index cfd718329b8..0293325fba0 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -179,7 +179,7 @@ Status ServerCore::AddModelsViaModelConfigList() { } TF_RETURN_IF_ERROR(ConnectSourceWithFastInitialLoad( manager_.get(), source_adapter.get(), servable_state_monitor_.get(), - static_servables)); + static_servables, server_core_config_.num_initial_load_unload_threads)); manager_.AddDependency(std::move(source_adapter)); } else { TF_RETURN_IF_ERROR(ReloadFileSystemStoragePathSourceConfig(source_config)); diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 09e3e7738ce..f42ce1d98fb 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -58,10 +58,17 @@ struct ServerCoreConfig { uint64 total_model_memory_limit_bytes = ULLONG_MAX; // Time interval between file-system polls, in seconds. int32 file_system_poll_wait_seconds = 30; + // The number of threads used to load and unload models. If set to 0, then // no thread pool is used and the loads/unloads are performed serially in // the manager thread. int32 num_load_unload_threads = 0; + + // The number of load/unload threads used to load the initial set of models at + // server startup. This is set high to load up the initial set of models fast, + // after this the server uses num_load_unload_threads. + int32 num_initial_load_unload_threads = 4.0 * port::NumSchedulableCPUs(); + // Maximum number of times we retry loading a model, after the first failure, // before we give up" int32 max_num_load_retries = 5; From a64b4edc9c0b2d57d51163696261f4fdceabb204 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 14 Oct 2016 15:41:38 -0800 Subject: [PATCH 0031/8554] Modify tensorflow command line flag parsing: - Allow help text to be specified for each flag - Add a flag "--help" to print the help text - Rename ParseFlags to Flags::Parse(); new routine Flags::Usage() returns a usage message. - Change uses to new format In some cases reorder with InitMain(), which should be called after flag parsing. Change: 136212902 --- tensorflow_serving/model_servers/main.cc | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 04be75e5342..b0185cdfe9f 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -42,6 +42,7 @@ limitations under the License. #include #include #include +#include #include "google/protobuf/wrappers.pb.h" #include "grpc++/security/server_credentials.h" @@ -182,21 +183,22 @@ int main(int argc, char** argv) { bool enable_batching = false; tensorflow::string model_name = "default"; tensorflow::string model_base_path; - const bool parse_result = tensorflow::ParseFlags( - &argc, argv, {tensorflow::Flag("port", &port), - tensorflow::Flag("enable_batching", &enable_batching), - tensorflow::Flag("model_name", &model_name), - tensorflow::Flag("model_base_path", &model_base_path)}); + std::vector flag_list = { + tensorflow::Flag("port", &port, "port to listen on"), + tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), + tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("model_base_path", &model_base_path, + "path to export (required)")}; + string usage = tensorflow::Flags::Usage(argv[0], flag_list); + const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || model_base_path.empty()) { - std::cout << "Usage: model_server" - << " [--port=8500]" - << " [--enable_batching]" - << " [--model_name=my_name]" - << " --model_base_path=/path/to/export" << std::endl; + std::cout << usage; return -1; } - tensorflow::port::InitMain(argv[0], &argc, &argv); + if (argc != 1) { + std::cout << "unknown argument: " << argv[1] << "\n" << usage; + } ModelServerConfig config = BuildSingleModelConfig(model_name, model_base_path); From a1d7253dbc0688535f63109d0d491d8a7c2ea96e Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 18 Oct 2016 09:45:13 -0800 Subject: [PATCH 0032/8554] Make AspiredVersionPolicy choice a ServerCore config parameter. Change: 136491185 --- tensorflow_serving/model_servers/BUILD | 2 +- tensorflow_serving/model_servers/main.cc | 11 ++++++-- .../model_servers/server_core.cc | 26 +++++++++++-------- .../model_servers/server_core.h | 16 ++++++++---- .../model_servers/test_util/BUILD | 1 + .../test_util/server_core_test_util.cc | 3 +++ 6 files changed, 40 insertions(+), 19 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 4b4848fee56..4540196268e 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -45,7 +45,6 @@ cc_library( "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:aspired_versions_manager", - "//tensorflow_serving/core:eager_load_policy", "//tensorflow_serving/core:load_servables_fast", "//tensorflow_serving/core:servable_state_monitor", "//tensorflow_serving/core:source", @@ -104,6 +103,7 @@ cc_binary( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "//tensorflow_serving/apis:prediction_service_proto", + "//tensorflow_serving/core:eager_load_policy", "//tensorflow_serving/core:servable_state_monitor", "@grpc//:grpc++", ] + TENSORFLOW_DEPS + SUPPORTED_TENSORFLOW_OPS, diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index b0185cdfe9f..59c3a804114 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -58,6 +58,7 @@ limitations under the License. #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #include "tensorflow_serving/apis/prediction_service.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" +#include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/server_core.h" @@ -65,7 +66,9 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" using tensorflow::serving::AspiredVersionsManager; +using tensorflow::serving::AspiredVersionPolicy; using tensorflow::serving::BatchingParameters; +using tensorflow::serving::EagerLoadPolicy; using tensorflow::serving::EventBus; using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; @@ -212,12 +215,16 @@ int main(int argc, char** argv) { "model_server_batch_threads"); } + ServerCoreConfig core_config; + core_config.aspired_version_policy = + std::unique_ptr(new EagerLoadPolicy); + std::unique_ptr core; TF_CHECK_OK(ServerCore::Create( config, std::bind(CreateSourceAdapter, source_adapter_config, std::placeholders::_1, std::placeholders::_2), - &CreateServableStateMonitor, &LoadCustomModelConfig, ServerCoreConfig(), - &core)); + &CreateServableStateMonitor, &LoadCustomModelConfig, + std::move(core_config), &core)); RunServer(port, std::move(core)); return 0; diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 0293325fba0..5cb326dc2a3 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -20,7 +20,6 @@ limitations under the License. #include "google/protobuf/any.pb.h" #include "google/protobuf/wrappers.pb.h" #include "tensorflow/core/platform/logging.h" -#include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/core/load_servables_fast.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/resources/resource_values.h" @@ -86,12 +85,15 @@ Status ServerCore::Create( const SourceAdapterCreator& source_adapter_creator, const ServableStateMonitorCreator& servable_state_monitor_creator, const CustomModelConfigLoader& custom_model_config_loader, - const ServerCoreConfig& server_core_config, + ServerCoreConfig server_core_config, std::unique_ptr* server_core) { - server_core->reset( - new ServerCore(source_adapter_creator, servable_state_monitor_creator, - custom_model_config_loader, server_core_config)); - TF_RETURN_IF_ERROR((*server_core)->Initialize()); + std::unique_ptr aspired_version_policy = + std::move(server_core_config.aspired_version_policy); + server_core->reset(new ServerCore( + source_adapter_creator, servable_state_monitor_creator, + custom_model_config_loader, std::move(server_core_config))); + TF_RETURN_IF_ERROR( + (*server_core)->Initialize(std::move(aspired_version_policy))); return (*server_core)->ReloadConfig(config); } @@ -103,21 +105,22 @@ ServerCore::ServerCore( const SourceAdapterCreator& source_adapter_creator, const ServableStateMonitorCreator& servable_state_monitor_creator, const CustomModelConfigLoader& custom_model_config_loader, - const ServerCoreConfig& server_core_config) + ServerCoreConfig server_core_config) : source_adapter_creator_(source_adapter_creator), servable_state_monitor_creator_(servable_state_monitor_creator), custom_model_config_loader_(custom_model_config_loader), - server_core_config_(server_core_config), + server_core_config_(std::move(server_core_config)), servable_event_bus_(EventBus::CreateEventBus()) {} -Status ServerCore::Initialize() { +Status ServerCore::Initialize(std::unique_ptr policy) { std::unique_ptr servable_state_monitor; TF_RETURN_IF_ERROR(servable_state_monitor_creator_(servable_event_bus_.get(), &servable_state_monitor)); servable_state_monitor_ = std::move(servable_state_monitor); std::unique_ptr aspired_versions_manager; - TF_RETURN_IF_ERROR(CreateAspiredVersionsManager(&aspired_versions_manager)); + TF_RETURN_IF_ERROR(CreateAspiredVersionsManager(std::move(policy), + &aspired_versions_manager)); manager_.SetOwned(std::move(aspired_versions_manager)); return Status::OK(); @@ -276,6 +279,7 @@ Status ServerCore::ReloadFileSystemStoragePathSourceConfig( } Status ServerCore::CreateAspiredVersionsManager( + std::unique_ptr aspired_version_policy, std::unique_ptr* const manager) { std::unique_ptr aspired_versions_manager; AspiredVersionsManager::Options manager_options; @@ -283,7 +287,7 @@ Status ServerCore::CreateAspiredVersionsManager( TF_RETURN_IF_ERROR(CreateResourceTracker(&resource_tracker)); manager_options.resource_tracker = std::move(resource_tracker); manager_options.servable_event_bus = servable_event_bus_.get(); - manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + manager_options.aspired_version_policy = std::move(aspired_version_policy); manager_options.num_load_unload_threads = server_core_config_.num_load_unload_threads; manager_options.max_num_load_retries = diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index f42ce1d98fb..0db630eb92a 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -59,6 +59,9 @@ struct ServerCoreConfig { // Time interval between file-system polls, in seconds. int32 file_system_poll_wait_seconds = 30; + // The AspiredVersionPolicy to use for the manager. Must be non-null. + std::unique_ptr aspired_version_policy; + // The number of threads used to load and unload models. If set to 0, then // no thread pool is used and the loads/unloads are performed serially in // the manager thread. @@ -113,8 +116,7 @@ class ServerCore { const SourceAdapterCreator& source_adapter_creator, const ServableStateMonitorCreator& servable_state_monitor_creator, const CustomModelConfigLoader& custom_model_config_loader, - const ServerCoreConfig& server_core_config, - std::unique_ptr* core); + ServerCoreConfig server_core_config, std::unique_ptr* core); // Updates the server core with all the models and sources per the // ModelServerConfig. Like Create(), waits for all statically configured @@ -151,7 +153,7 @@ class ServerCore { ServerCore(const SourceAdapterCreator& source_adapter_creator, const ServableStateMonitorCreator& servable_state_monitor_creator, const CustomModelConfigLoader& custom_model_config_loader, - const ServerCoreConfig& server_core_config); + ServerCoreConfig server_core_config); private: friend class test_util::ServerCoreTestAccess; @@ -162,10 +164,12 @@ class ServerCore { // Initializes server core. // Must be run once and only once per ServerCore instance. - Status Initialize(); + Status Initialize( + std::unique_ptr aspired_version_policy); - // Creates a AspiredVersionsManager with the EagerLoadPolicy. + // Creates a AspiredVersionsManager with the specified policy. Status CreateAspiredVersionsManager( + std::unique_ptr policy, std::unique_ptr* manager); // Creates a ResourceTracker. @@ -223,6 +227,8 @@ class ServerCore { SourceAdapterCreator source_adapter_creator_; ServableStateMonitorCreator servable_state_monitor_creator_; CustomModelConfigLoader custom_model_config_loader_; + + // The config passed to the ctor, minus the AspiredVersionPolicy. ServerCoreConfig server_core_config_; std::shared_ptr> servable_event_bus_; diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 0125c392035..62858aae14f 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -56,6 +56,7 @@ cc_library( "//visibility:public", ], deps = [ + "//tensorflow_serving/core:eager_load_policy", "//tensorflow_serving/core:servable_id", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/model_servers:model_platform_types", diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index cdad830bbd7..c4d69db6b3c 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" +#include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/test_util/test_util.h" @@ -40,6 +41,8 @@ ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { ServerCoreConfig ServerCoreTest::GetTestServerCoreConfig() { ServerCoreConfig config; config.file_system_poll_wait_seconds = 0; + config.aspired_version_policy = + std::unique_ptr(new EagerLoadPolicy); return config; } From 42c70e62fcdcaf42bf92f2041e3ab4ef561bab42 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 18 Oct 2016 10:11:41 -0800 Subject: [PATCH 0033/8554] Update tf.Example parsing code in inception export Change: 136495081 --- tensorflow_serving/example/inception_export.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index 551bfc011ac..252b3f60d63 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -67,11 +67,10 @@ def export(): # Input transformation. serialized_tf_example = tf.placeholder(tf.string, name='tf_example') feature_configs = { - 'jpeg_encoded': tf.FixedLenFeature( - shape=[], dtype=tf.string, default_value=''), + 'image/encoded': tf.FixedLenFeature(shape=[], dtype=tf.string), } tf_example = tf.parse_example(serialized_tf_example, feature_configs) - jpegs = tf_example['jpeg_encoded'] + jpegs = tf_example['image/encoded'] images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32) # Run inference. From c204ce3e28e78d351f6a23cbdac7e61791f83f2f Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Tue, 18 Oct 2016 13:37:40 -0800 Subject: [PATCH 0034/8554] Increase server core test size. Change: 136521870 --- tensorflow_serving/model_servers/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 4540196268e..7b6cce6c512 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -63,6 +63,7 @@ cc_library( cc_test( name = "server_core_test", + size = "medium", srcs = ["server_core_test.cc"], data = [ "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", From 9aa7e72f8e5119dcad7bfd47be7dfced5ad7a1d9 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Tue, 18 Oct 2016 17:52:50 -0700 Subject: [PATCH 0035/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 0db279f16ce..c8a45a8e236 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 0db279f16ce21568c414694ea55e2dec07ad6972 +Subproject commit c8a45a8e236776bed1d14fd71f3b6755bd63cc58 diff --git a/tf_models b/tf_models index f98c5ded31d..77080ad11f0 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit f98c5ded31d7da0c2d127c28b2c16f0307a368f0 +Subproject commit 77080ad11f016e5365d28d0681cc4c0a95a7061f From d70b45f83bafe9da0c9271face8c083fcb4b65dc Mon Sep 17 00:00:00 2001 From: wai chee yau Date: Sat, 22 Oct 2016 12:49:42 +1100 Subject: [PATCH 0036/8554] add command line option for file system poll wait (seconds) --- tensorflow_serving/model_servers/main.cc | 7 +++++++ .../model_servers/tensorflow_model_server_test.py | 7 ++++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 59c3a804114..9278c6e937b 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -185,11 +185,13 @@ int main(int argc, char** argv) { tensorflow::int32 port = 8500; bool enable_batching = false; tensorflow::string model_name = "default"; + tensorflow::int32 file_system_polling_interval = -1; tensorflow::string model_base_path; std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("file_system_polling_interval", &file_system_polling_interval, "interval in seconds between each poll of the file system for new model"), tensorflow::Flag("model_base_path", &model_base_path, "path to export (required)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); @@ -219,6 +221,11 @@ int main(int argc, char** argv) { core_config.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); + // Override default file polling interval + if (file_system_polling_interval > 0) { + core_config.file_system_poll_wait_seconds = file_system_polling_interval; + } + std::unique_ptr core; TF_CHECK_OK(ServerCore::Create( config, std::bind(CreateSourceAdapter, source_adapter_config, diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 79f4427d9e6..c6dc8219ab7 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -68,13 +68,14 @@ def TerminateProcs(self): if self.server_proc is not None: self.server_proc.terminate() - def RunServer(self, port, model_name, model_path): + def RunServer(self, port, model_name, model_path, file_system_polling_interval): """Run tensorflow_model_server using test config.""" print 'Starting test server...' command = os.path.join(self.binary_dir, 'tensorflow_model_server') command += ' --port=' + str(port) command += ' --model_name=' + model_name command += ' --model_base_path=' + model_path + command += ' --file_system_polling_interval=' + file_system_polling_interval command += ' --alsologtostderr' print command self.server_proc = subprocess.Popen(shlex.split(command)) @@ -109,7 +110,7 @@ def testPredict(self): atexit.register(self.TerminateProcs) model_server_address = self.RunServer( PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'half_plus_two')) + os.path.join(self.testdata_dir, 'half_plus_two'), 1) time.sleep(5) self.VerifyPredictRequest(model_server_address) self.VerifyPredictRequest(model_server_address, specify_output=False) @@ -119,7 +120,7 @@ def testBadModel(self): atexit.register(self.TerminateProcs) model_server_address = self.RunServer( PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'bad_half_plus_two')) + os.path.join(self.testdata_dir, 'bad_half_plus_two'), 1) time.sleep(5) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address) From 2fe1bf9bf274f510c0f88fcadaab97a1716d3035 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 19 Oct 2016 17:06:40 -0800 Subject: [PATCH 0037/8554] Update required grpc version to 0.15 or higher. See https://github.com/tensorflow/serving/issues/209 and https://github.com/grpc/grpc/issues/7133#issuecomment-231868432. Change: 136667019 --- tensorflow_serving/g3doc/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 4f4667455c5..8c9a154902c 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -30,7 +30,7 @@ following steps: ### gRPC -Our tutorials use [gRPC](http://www.grpc.io) (0.13 or higher) as our RPC +Our tutorials use [gRPC](http://www.grpc.io) (1.0.0 or higher) as our RPC framework. You can find the installation instructions [here](https://github.com/grpc/grpc/tree/master/src/python/grpcio). From aa4e79f0e000724d945af1619f06b96e596784c4 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 20 Oct 2016 13:22:39 -0800 Subject: [PATCH 0038/8554] Clarify in ModelServer documentation that base path should exclude version#. (Some users are confused about this. We do document it in the ModelServerConfig proto, but no in model-server main.cc.) Change: 136759754 --- tensorflow_serving/model_servers/main.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 59c3a804114..3716e8bdd3b 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -32,7 +32,11 @@ limitations under the License. // // To serve a single model, run with: // $path_to_binary/tensorflow_model_server \ -// --model_base_path=[/tmp/my_model | gs://gcs_address] \ +// --model_base_path=[/tmp/my_model | gs://gcs_address] +// IMPORTANT: Be sure the base path excludes the version directory. For +// example for a model at /tmp/my_model/123, where 123 is the version, the base +// path is /tmp/my_model. +// // To specify model name (default "default"): --model_name=my_name // To specify port (default 8500): --port=my_port // To enable batching (default disabled): --enable_batching From 0391c442251eced9a1a144de3da2b9c5cf3f4383 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 21 Oct 2016 09:13:40 -0800 Subject: [PATCH 0039/8554] Delete simple_server, which is superseded by model_server. Change: 136844593 --- tensorflow_serving/servables/tensorflow/BUILD | 48 --------- .../servables/tensorflow/simple_servers.cc | 99 ------------------- .../servables/tensorflow/simple_servers.h | 58 ----------- .../tensorflow/simple_servers_test.cc | 93 ----------------- 4 files changed, 298 deletions(-) delete mode 100644 tensorflow_serving/servables/tensorflow/simple_servers.cc delete mode 100644 tensorflow_serving/servables/tensorflow/simple_servers.h delete mode 100644 tensorflow_serving/servables/tensorflow/simple_servers_test.cc diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 3101cff50a1..8675dcf6205 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -147,54 +147,6 @@ serving_proto_library( ], ) -cc_library( - name = "simple_servers", - srcs = ["simple_servers.cc"], - hdrs = ["simple_servers.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - ":session_bundle_source_adapter", - ":session_bundle_source_adapter_proto", - "//tensorflow_serving/core:aspired_versions_manager_builder", - "//tensorflow_serving/core:eager_load_policy", - "//tensorflow_serving/core:loader", - "//tensorflow_serving/core:manager", - "//tensorflow_serving/core:source", - "//tensorflow_serving/core:source_adapter", - "//tensorflow_serving/core:storage_path", - "//tensorflow_serving/core:target", - "//tensorflow_serving/sources/storage_path:file_system_storage_path_source", - "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - -cc_test( - name = "simple_servers_test", - srcs = ["simple_servers_test.cc"], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], - # Link in all registered kernels. - linkstatic = 1, - deps = [ - ":simple_servers", - "//tensorflow_serving/core:servable_handle", - "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util", - "//tensorflow_serving/util:unique_ptr_with_deps", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/core:core_cpu", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", - "@org_tensorflow//tensorflow/core:test", - "@org_tensorflow//tensorflow/core:testlib", - ], -) - cc_library( name = "serving_session", srcs = ["serving_session.cc"], diff --git a/tensorflow_serving/servables/tensorflow/simple_servers.cc b/tensorflow_serving/servables/tensorflow/simple_servers.cc deleted file mode 100644 index 358b892a645..00000000000 --- a/tensorflow_serving/servables/tensorflow/simple_servers.cc +++ /dev/null @@ -1,99 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/servables/tensorflow/simple_servers.h" - -#include -#include -#include - -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow_serving/core/aspired_versions_manager_builder.h" -#include "tensorflow_serving/core/eager_load_policy.h" -#include "tensorflow_serving/core/loader.h" -#include "tensorflow_serving/core/source.h" -#include "tensorflow_serving/core/source_adapter.h" -#include "tensorflow_serving/core/storage_path.h" -#include "tensorflow_serving/core/target.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" -#include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" -#include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" - -namespace tensorflow { -namespace serving { -namespace simple_servers { - -namespace { - -// Creates a Source that monitors a filesystem's base_path for new -// directories. Upon finding these, it provides the target with the new version -// (a directory). The servable_name param simply allows this source to create -// all AspiredVersions for the target with the same servable_name. -Status CreateStoragePathSource( - const string& base_path, const string& servable_name, - std::unique_ptr>* path_source) { - FileSystemStoragePathSourceConfig config; - config.set_servable_name(servable_name); - config.set_base_path(base_path); - config.set_file_system_poll_wait_seconds(1); - - std::unique_ptr file_system_source; - TF_RETURN_IF_ERROR( - FileSystemStoragePathSource::Create(config, &file_system_source)); - - *path_source = std::move(file_system_source); - return Status::OK(); -} - -// Creates a SessionBundle Source by adapting the underlying -// FileSystemStoragePathSource. These two are connected in the -// 'CreateSingleTFModelManagerFromBasePath' method, with the -// FileSystemStoragePathSource as the Source and the SessionBundleSource as the -// Target. -Status CreateSessionBundleSource( - std::unique_ptr* source) { - SessionBundleSourceAdapterConfig config; - TF_RETURN_IF_ERROR(SessionBundleSourceAdapter::Create(config, source)); - - return Status::OK(); -} - -} // namespace - -Status CreateSingleTFModelManagerFromBasePath( - const string& base_path, std::unique_ptr* const manager) { - std::unique_ptr bundle_source; - TF_RETURN_IF_ERROR(CreateSessionBundleSource(&bundle_source)); - std::unique_ptr> path_source; - TF_RETURN_IF_ERROR( - CreateStoragePathSource(base_path, "default", &path_source)); - - AspiredVersionsManagerBuilder::Options manager_options; - manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); - std::unique_ptr builder; - TF_CHECK_OK(AspiredVersionsManagerBuilder::Create(std::move(manager_options), - &builder)); - builder->AddSourceChain(std::move(path_source), std::move(bundle_source)); - *manager = builder->Build(); - - return Status::OK(); -} - -} // namespace simple_servers -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/simple_servers.h b/tensorflow_serving/servables/tensorflow/simple_servers.h deleted file mode 100644 index e3bab8869e8..00000000000 --- a/tensorflow_serving/servables/tensorflow/simple_servers.h +++ /dev/null @@ -1,58 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Bootstrapping and configuration utilities for creating simple servers of -// TensorFlow models. Intended for basic instantiation with default configs. -// -// Note: All methods expect TensorFlow exports conforming to the export format -// specified at tensorflow_serving/servables/tensorflow/README.md. -#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ -#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ - -#include -#include - -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/core/manager.h" -#include "tensorflow_serving/core/storage_path.h" -#include "tensorflow_serving/core/target.h" - -namespace tensorflow { -namespace serving { -namespace simple_servers { - -// TODO(b/25969594): Add full test coverage. - -// Creates a Manager and associated Source for a single TensorFlow model that -// automatically loads new versions over time. All versions of the model will be -// loaded from new directories under the specified base path. Uses default -// SessionOptions. -// -// The servables loaded and served from this manager are of type -// tensorflow::serving::SessionBundle. -// -// When new versions arrive the Manager will unload the previous version before -// loading the new version. This is preferable from a resource utilization -// perspective, but has reduced availability. -Status CreateSingleTFModelManagerFromBasePath( - const string& base_path, std::unique_ptr* manager); - -} // namespace simple_servers -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ diff --git a/tensorflow_serving/servables/tensorflow/simple_servers_test.cc b/tensorflow_serving/servables/tensorflow/simple_servers_test.cc deleted file mode 100644 index 57954274129..00000000000 --- a/tensorflow_serving/servables/tensorflow/simple_servers_test.cc +++ /dev/null @@ -1,93 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/servables/tensorflow/simple_servers.h" - -#include -#include -#include -#include - -#include -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/core/framework/tensor.h" -#include "tensorflow/core/framework/tensor_testutil.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/public/session.h" -#include "tensorflow_serving/core/servable_handle.h" -#include "tensorflow_serving/test_util/test_util.h" - -namespace tensorflow { -namespace serving { -namespace { - -class SimpleServersTest : public ::testing::Test { - protected: - SimpleServersTest() - : test_data_path_(test_util::ContribTestSrcDirPath( - "session_bundle/example/half_plus_two")) {} - - // Test that a SessionBundle handles a single request for the half plus two - // model properly. The request has size=2, for batching purposes. - void TestSingleRequest(const SessionBundle& bundle) { - const Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); - // half plus two: output should be input / 2 + 2. - const Tensor expected_output = - test::AsTensor({100.0f / 2 + 2, 42.0f / 2 + 2}, {2}); - - // Note that "x" and "y" are the actual names of the nodes in the graph. - // The saved manifest binds these to "input" and "output" respectively, but - // these tests are focused on the raw underlying session without bindings. - const std::vector> inputs = {{"x", input}}; - const std::vector output_names = {"y"}; - const std::vector empty_targets; - std::vector outputs; - - TF_ASSERT_OK( - bundle.session->Run(inputs, output_names, empty_targets, &outputs)); - - ASSERT_EQ(1, outputs.size()); - const auto& single_output = outputs.at(0); - test::ExpectTensorEqual(expected_output, single_output); - } - - // Test data path, to be initialized to point at an export of half-plus-two. - const string test_data_path_; -}; - -TEST_F(SimpleServersTest, Basic) { - std::unique_ptr manager; - const Status status = simple_servers::CreateSingleTFModelManagerFromBasePath( - test_data_path_, &manager); - TF_CHECK_OK(status); - // We wait until the manager starts serving the servable. - // TODO(b/25545570): Use the waiter api when it's ready. - while (manager->ListAvailableServableIds().empty()) { - Env::Default()->SleepForMicroseconds(1000); - } - ServableHandle bundle; - const Status handle_status = - manager->GetServableHandle(ServableRequest::Latest("default"), &bundle); - TF_CHECK_OK(handle_status); - TestSingleRequest(*bundle); -} - -} // namespace -} // namespace serving -} // namespace tensorflow From 6d35bf4877e438d57cc1e931e92296170fc17351 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 21 Oct 2016 10:48:39 -0800 Subject: [PATCH 0040/8554] Automated rollback of change 136844593 Change: 136856442 --- tensorflow_serving/servables/tensorflow/BUILD | 48 +++++++++ .../servables/tensorflow/simple_servers.cc | 99 +++++++++++++++++++ .../servables/tensorflow/simple_servers.h | 58 +++++++++++ .../tensorflow/simple_servers_test.cc | 93 +++++++++++++++++ 4 files changed, 298 insertions(+) create mode 100644 tensorflow_serving/servables/tensorflow/simple_servers.cc create mode 100644 tensorflow_serving/servables/tensorflow/simple_servers.h create mode 100644 tensorflow_serving/servables/tensorflow/simple_servers_test.cc diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 8675dcf6205..3101cff50a1 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -147,6 +147,54 @@ serving_proto_library( ], ) +cc_library( + name = "simple_servers", + srcs = ["simple_servers.cc"], + hdrs = ["simple_servers.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":session_bundle_source_adapter", + ":session_bundle_source_adapter_proto", + "//tensorflow_serving/core:aspired_versions_manager_builder", + "//tensorflow_serving/core:eager_load_policy", + "//tensorflow_serving/core:loader", + "//tensorflow_serving/core:manager", + "//tensorflow_serving/core:source", + "//tensorflow_serving/core:source_adapter", + "//tensorflow_serving/core:storage_path", + "//tensorflow_serving/core:target", + "//tensorflow_serving/sources/storage_path:file_system_storage_path_source", + "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "simple_servers_test", + srcs = ["simple_servers_test.cc"], + data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + # Link in all registered kernels. + linkstatic = 1, + deps = [ + ":simple_servers", + "//tensorflow_serving/core:servable_handle", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "//tensorflow_serving/util:unique_ptr_with_deps", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@org_tensorflow//tensorflow/core:testlib", + ], +) + cc_library( name = "serving_session", srcs = ["serving_session.cc"], diff --git a/tensorflow_serving/servables/tensorflow/simple_servers.cc b/tensorflow_serving/servables/tensorflow/simple_servers.cc new file mode 100644 index 00000000000..358b892a645 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/simple_servers.cc @@ -0,0 +1,99 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/simple_servers.h" + +#include +#include +#include + +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/aspired_versions_manager_builder.h" +#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/source.h" +#include "tensorflow_serving/core/source_adapter.h" +#include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/core/target.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" +#include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" +#include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" + +namespace tensorflow { +namespace serving { +namespace simple_servers { + +namespace { + +// Creates a Source that monitors a filesystem's base_path for new +// directories. Upon finding these, it provides the target with the new version +// (a directory). The servable_name param simply allows this source to create +// all AspiredVersions for the target with the same servable_name. +Status CreateStoragePathSource( + const string& base_path, const string& servable_name, + std::unique_ptr>* path_source) { + FileSystemStoragePathSourceConfig config; + config.set_servable_name(servable_name); + config.set_base_path(base_path); + config.set_file_system_poll_wait_seconds(1); + + std::unique_ptr file_system_source; + TF_RETURN_IF_ERROR( + FileSystemStoragePathSource::Create(config, &file_system_source)); + + *path_source = std::move(file_system_source); + return Status::OK(); +} + +// Creates a SessionBundle Source by adapting the underlying +// FileSystemStoragePathSource. These two are connected in the +// 'CreateSingleTFModelManagerFromBasePath' method, with the +// FileSystemStoragePathSource as the Source and the SessionBundleSource as the +// Target. +Status CreateSessionBundleSource( + std::unique_ptr* source) { + SessionBundleSourceAdapterConfig config; + TF_RETURN_IF_ERROR(SessionBundleSourceAdapter::Create(config, source)); + + return Status::OK(); +} + +} // namespace + +Status CreateSingleTFModelManagerFromBasePath( + const string& base_path, std::unique_ptr* const manager) { + std::unique_ptr bundle_source; + TF_RETURN_IF_ERROR(CreateSessionBundleSource(&bundle_source)); + std::unique_ptr> path_source; + TF_RETURN_IF_ERROR( + CreateStoragePathSource(base_path, "default", &path_source)); + + AspiredVersionsManagerBuilder::Options manager_options; + manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + std::unique_ptr builder; + TF_CHECK_OK(AspiredVersionsManagerBuilder::Create(std::move(manager_options), + &builder)); + builder->AddSourceChain(std::move(path_source), std::move(bundle_source)); + *manager = builder->Build(); + + return Status::OK(); +} + +} // namespace simple_servers +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/simple_servers.h b/tensorflow_serving/servables/tensorflow/simple_servers.h new file mode 100644 index 00000000000..e3bab8869e8 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/simple_servers.h @@ -0,0 +1,58 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Bootstrapping and configuration utilities for creating simple servers of +// TensorFlow models. Intended for basic instantiation with default configs. +// +// Note: All methods expect TensorFlow exports conforming to the export format +// specified at tensorflow_serving/servables/tensorflow/README.md. +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ + +#include +#include + +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/core/target.h" + +namespace tensorflow { +namespace serving { +namespace simple_servers { + +// TODO(b/25969594): Add full test coverage. + +// Creates a Manager and associated Source for a single TensorFlow model that +// automatically loads new versions over time. All versions of the model will be +// loaded from new directories under the specified base path. Uses default +// SessionOptions. +// +// The servables loaded and served from this manager are of type +// tensorflow::serving::SessionBundle. +// +// When new versions arrive the Manager will unload the previous version before +// loading the new version. This is preferable from a resource utilization +// perspective, but has reduced availability. +Status CreateSingleTFModelManagerFromBasePath( + const string& base_path, std::unique_ptr* manager); + +} // namespace simple_servers +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ diff --git a/tensorflow_serving/servables/tensorflow/simple_servers_test.cc b/tensorflow_serving/servables/tensorflow/simple_servers_test.cc new file mode 100644 index 00000000000..57954274129 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/simple_servers_test.cc @@ -0,0 +1,93 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/simple_servers.h" + +#include +#include +#include +#include + +#include +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +class SimpleServersTest : public ::testing::Test { + protected: + SimpleServersTest() + : test_data_path_(test_util::ContribTestSrcDirPath( + "session_bundle/example/half_plus_two")) {} + + // Test that a SessionBundle handles a single request for the half plus two + // model properly. The request has size=2, for batching purposes. + void TestSingleRequest(const SessionBundle& bundle) { + const Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + // half plus two: output should be input / 2 + 2. + const Tensor expected_output = + test::AsTensor({100.0f / 2 + 2, 42.0f / 2 + 2}, {2}); + + // Note that "x" and "y" are the actual names of the nodes in the graph. + // The saved manifest binds these to "input" and "output" respectively, but + // these tests are focused on the raw underlying session without bindings. + const std::vector> inputs = {{"x", input}}; + const std::vector output_names = {"y"}; + const std::vector empty_targets; + std::vector outputs; + + TF_ASSERT_OK( + bundle.session->Run(inputs, output_names, empty_targets, &outputs)); + + ASSERT_EQ(1, outputs.size()); + const auto& single_output = outputs.at(0); + test::ExpectTensorEqual(expected_output, single_output); + } + + // Test data path, to be initialized to point at an export of half-plus-two. + const string test_data_path_; +}; + +TEST_F(SimpleServersTest, Basic) { + std::unique_ptr manager; + const Status status = simple_servers::CreateSingleTFModelManagerFromBasePath( + test_data_path_, &manager); + TF_CHECK_OK(status); + // We wait until the manager starts serving the servable. + // TODO(b/25545570): Use the waiter api when it's ready. + while (manager->ListAvailableServableIds().empty()) { + Env::Default()->SleepForMicroseconds(1000); + } + ServableHandle bundle; + const Status handle_status = + manager->GetServableHandle(ServableRequest::Latest("default"), &bundle); + TF_CHECK_OK(handle_status); + TestSingleRequest(*bundle); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From f86400a4a76efa1151477beb09e1e2a1acbaad80 Mon Sep 17 00:00:00 2001 From: wai chee yau Date: Tue, 25 Oct 2016 17:54:39 +1100 Subject: [PATCH 0041/8554] rename the flag to match variable name file_system_poll_wait_seconds. update test --- tensorflow_serving/model_servers/main.cc | 10 +++------- .../model_servers/tensorflow_model_server_test.py | 7 +++---- 2 files changed, 6 insertions(+), 11 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 9278c6e937b..0382595a2d9 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -185,13 +185,13 @@ int main(int argc, char** argv) { tensorflow::int32 port = 8500; bool enable_batching = false; tensorflow::string model_name = "default"; - tensorflow::int32 file_system_polling_interval = -1; + tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), tensorflow::Flag("model_name", &model_name, "name of model"), - tensorflow::Flag("file_system_polling_interval", &file_system_polling_interval, "interval in seconds between each poll of the file system for new model"), + tensorflow::Flag("file_system_poll_wait_seconds", &file_system_poll_wait_seconds, "interval in seconds between each poll of the file system for new model version"), tensorflow::Flag("model_base_path", &model_base_path, "path to export (required)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); @@ -220,11 +220,7 @@ int main(int argc, char** argv) { ServerCoreConfig core_config; core_config.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); - - // Override default file polling interval - if (file_system_polling_interval > 0) { - core_config.file_system_poll_wait_seconds = file_system_polling_interval; - } + core_config.file_system_poll_wait_seconds = file_system_poll_wait_seconds; std::unique_ptr core; TF_CHECK_OK(ServerCore::Create( diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index c6dc8219ab7..79f4427d9e6 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -68,14 +68,13 @@ def TerminateProcs(self): if self.server_proc is not None: self.server_proc.terminate() - def RunServer(self, port, model_name, model_path, file_system_polling_interval): + def RunServer(self, port, model_name, model_path): """Run tensorflow_model_server using test config.""" print 'Starting test server...' command = os.path.join(self.binary_dir, 'tensorflow_model_server') command += ' --port=' + str(port) command += ' --model_name=' + model_name command += ' --model_base_path=' + model_path - command += ' --file_system_polling_interval=' + file_system_polling_interval command += ' --alsologtostderr' print command self.server_proc = subprocess.Popen(shlex.split(command)) @@ -110,7 +109,7 @@ def testPredict(self): atexit.register(self.TerminateProcs) model_server_address = self.RunServer( PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'half_plus_two'), 1) + os.path.join(self.testdata_dir, 'half_plus_two')) time.sleep(5) self.VerifyPredictRequest(model_server_address) self.VerifyPredictRequest(model_server_address, specify_output=False) @@ -120,7 +119,7 @@ def testBadModel(self): atexit.register(self.TerminateProcs) model_server_address = self.RunServer( PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'bad_half_plus_two'), 1) + os.path.join(self.testdata_dir, 'bad_half_plus_two')) time.sleep(5) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address) From ace1b2904a3d4ec6b487f48617051b02173619bc Mon Sep 17 00:00:00 2001 From: Gilman Callsen Date: Thu, 27 Oct 2016 13:32:47 -0400 Subject: [PATCH 0042/8554] Fixes #201 - Update inception_k8s.json config to use new location for tensorflow model server. (#213) --- tensorflow_serving/example/inception_k8s.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/example/inception_k8s.json b/tensorflow_serving/example/inception_k8s.json index 7877af436d1..764e18089db 100644 --- a/tensorflow_serving/example/inception_k8s.json +++ b/tensorflow_serving/example/inception_k8s.json @@ -25,7 +25,7 @@ "-c" ], "args": [ - "/serving/bazel-bin/tensorflow_serving/example/inception_inference --port=9000 /serving/inception-export" + "/serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=inception --model_base_path=/serving/inception-export" ], "ports": [ { @@ -58,4 +58,4 @@ }, "type": "LoadBalancer" } -} \ No newline at end of file +} From baa653c0d9f9075b1ed539479936faf81c3e258c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 26 Oct 2016 04:33:44 -0800 Subject: [PATCH 0043/8554] Removed "-parse_headers" from BUILD package features. Change: 137265338 --- tensorflow_serving/apis/BUILD | 5 +---- tensorflow_serving/batching/BUILD | 5 +---- tensorflow_serving/batching/test_util/BUILD | 5 +---- tensorflow_serving/config/BUILD | 5 +---- tensorflow_serving/core/BUILD | 5 +---- tensorflow_serving/core/test_util/BUILD | 5 +---- tensorflow_serving/example/BUILD | 5 +---- tensorflow_serving/model_servers/BUILD | 5 +---- tensorflow_serving/model_servers/test_util/BUILD | 5 +---- tensorflow_serving/resources/BUILD | 5 +---- tensorflow_serving/servables/hashmap/BUILD | 5 +---- tensorflow_serving/servables/tensorflow/BUILD | 5 +---- tensorflow_serving/sources/storage_path/BUILD | 5 +---- tensorflow_serving/util/BUILD | 5 +---- 14 files changed, 14 insertions(+), 56 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 893aa10a59d..10cbf6d9454 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -2,10 +2,7 @@ package( default_visibility = ["//visibility:public"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index 1ed82ea0675..4a3f809d042 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -2,10 +2,7 @@ package( default_visibility = ["//tensorflow_serving:internal"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/batching/test_util/BUILD b/tensorflow_serving/batching/test_util/BUILD index 35802960058..86f96f15750 100644 --- a/tensorflow_serving/batching/test_util/BUILD +++ b/tensorflow_serving/batching/test_util/BUILD @@ -2,10 +2,7 @@ package( default_visibility = ["//tensorflow_serving:internal"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index fdb757d46e3..39917a4883f 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -2,10 +2,7 @@ package( default_visibility = ["//visibility:public"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 88db5254cee..de1f2562746 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -4,10 +4,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 3d501805ed3..ec94071788d 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -4,10 +4,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index d1718218dae..4b365b5acd8 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -2,10 +2,7 @@ package( default_visibility = ["//tensorflow_serving:internal"], - features = [ - "-parse_headers", - "no_layering_check", - ], + features = ["no_layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 7b6cce6c512..405062817ab 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -4,10 +4,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 62858aae14f..1b02c54bc1b 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -4,10 +4,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/resources/BUILD b/tensorflow_serving/resources/BUILD index d753acb9e92..a96336f7fc2 100644 --- a/tensorflow_serving/resources/BUILD +++ b/tensorflow_serving/resources/BUILD @@ -2,10 +2,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/servables/hashmap/BUILD b/tensorflow_serving/servables/hashmap/BUILD index bb85e297b82..e34ad29efb8 100644 --- a/tensorflow_serving/servables/hashmap/BUILD +++ b/tensorflow_serving/servables/hashmap/BUILD @@ -2,10 +2,7 @@ package( default_visibility = ["//tensorflow_serving:internal"], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 3101cff50a1..5bbe59ecc68 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -4,10 +4,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/sources/storage_path/BUILD b/tensorflow_serving/sources/storage_path/BUILD index ee64514823f..01aa40b2b13 100644 --- a/tensorflow_serving/sources/storage_path/BUILD +++ b/tensorflow_serving/sources/storage_path/BUILD @@ -4,10 +4,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 013179d3303..ac6362d15e2 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -4,10 +4,7 @@ package( default_visibility = [ "//tensorflow_serving:internal", ], - features = [ - "-layering_check", - "-parse_headers", - ], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 From ef2866ea835b9616b3adeee37310b238e5802242 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 26 Oct 2016 12:44:40 -0800 Subject: [PATCH 0044/8554] Simplify ResourceUtil by not requiring input data be normalized, and mark it as externally visible. Change: 137314340 --- tensorflow_serving/resources/BUILD | 1 + .../resources/resource_tracker.cc | 4 -- tensorflow_serving/resources/resource_util.cc | 41 +++++++++++++++---- tensorflow_serving/resources/resource_util.h | 32 ++++++++++----- 4 files changed, 57 insertions(+), 21 deletions(-) diff --git a/tensorflow_serving/resources/BUILD b/tensorflow_serving/resources/BUILD index a96336f7fc2..6c5522b8749 100644 --- a/tensorflow_serving/resources/BUILD +++ b/tensorflow_serving/resources/BUILD @@ -44,6 +44,7 @@ cc_library( name = "resource_util", srcs = ["resource_util.cc"], hdrs = ["resource_util.h"], + visibility = ["//visibility:public"], deps = [ ":resources_proto", "@org_tensorflow//tensorflow/core:lib", diff --git a/tensorflow_serving/resources/resource_tracker.cc b/tensorflow_serving/resources/resource_tracker.cc index adfc9a8550b..259b21aadbc 100644 --- a/tensorflow_serving/resources/resource_tracker.cc +++ b/tensorflow_serving/resources/resource_tracker.cc @@ -45,7 +45,6 @@ Status ResourceTracker::ReserveResources(const Loader& servable, ResourceAllocation servable_resources; TF_RETURN_IF_ERROR(servable.EstimateResources(&servable_resources)); TF_RETURN_IF_ERROR(util_->VerifyValidity(servable_resources)); - servable_resources = util_->Normalize(servable_resources); ResourceAllocation conservative_proposed_used_resources = util_->Overbind(used_resources_); @@ -54,7 +53,6 @@ Status ResourceTracker::ReserveResources(const Loader& servable, if (util_->LessThanOrEqual(conservative_proposed_used_resources, total_resources_)) { util_->Add(servable_resources, &used_resources_); - DCHECK(util_->IsNormalized(used_resources_)); *success = true; } else { LOG(INFO) << "Insufficient resources to load servable " @@ -76,10 +74,8 @@ Status ResourceTracker::RecomputeUsedResources( ResourceAllocation servable_resources; TF_RETURN_IF_ERROR(servable->EstimateResources(&servable_resources)); TF_RETURN_IF_ERROR(util_->VerifyValidity(servable_resources)); - servable_resources = util_->Normalize(servable_resources); util_->Add(servable_resources, &used_resources_); } - DCHECK(util_->IsNormalized(used_resources_)); return Status::OK(); } diff --git a/tensorflow_serving/resources/resource_util.cc b/tensorflow_serving/resources/resource_util.cc index 4aca447a358..993711db7e1 100644 --- a/tensorflow_serving/resources/resource_util.cc +++ b/tensorflow_serving/resources/resource_util.cc @@ -179,6 +179,33 @@ bool ResourceUtil::IsNormalized(const ResourceAllocation& allocation) const { } bool ResourceUtil::IsBound(const ResourceAllocation& allocation) const { + return IsBoundNormalized(Normalize(allocation)); +} + +void ResourceUtil::Add(const ResourceAllocation& to_add, + ResourceAllocation* base) const { + *base = Normalize(*base); + return AddNormalized(Normalize(to_add), base); +} + +bool ResourceUtil::Subtract(const ResourceAllocation& to_subtract, + ResourceAllocation* base) const { + *base = Normalize(*base); + return SubtractNormalized(Normalize(to_subtract), base); +} + +bool ResourceUtil::LessThanOrEqual(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const { + return LessThanOrEqualNormalized(Normalize(lhs), Normalize(rhs)); +} + +ResourceAllocation ResourceUtil::Overbind( + const ResourceAllocation& allocation) const { + return OverbindNormalized(Normalize(allocation)); +} + +bool ResourceUtil::IsBoundNormalized( + const ResourceAllocation& allocation) const { DCHECK(IsNormalized(allocation)); for (const auto& entry : allocation.resource_quantities()) { if (!entry.resource().has_device_instance()) { @@ -188,8 +215,8 @@ bool ResourceUtil::IsBound(const ResourceAllocation& allocation) const { return true; } -void ResourceUtil::Add(const ResourceAllocation& to_add, - ResourceAllocation* base) const { +void ResourceUtil::AddNormalized(const ResourceAllocation& to_add, + ResourceAllocation* base) const { DCHECK(IsNormalized(to_add)); DCHECK(IsNormalized(*base)); for (const ResourceAllocation::Entry& to_add_entry : @@ -201,8 +228,8 @@ void ResourceUtil::Add(const ResourceAllocation& to_add, DCHECK(IsNormalized(*base)); } -bool ResourceUtil::Subtract(const ResourceAllocation& to_subtract, - ResourceAllocation* base) const { +bool ResourceUtil::SubtractNormalized(const ResourceAllocation& to_subtract, + ResourceAllocation* base) const { DCHECK(IsNormalized(to_subtract)); DCHECK(IsNormalized(*base)); // We buffer the mutations to 'base' so that if we bail out due to a negative @@ -233,8 +260,8 @@ bool ResourceUtil::Subtract(const ResourceAllocation& to_subtract, return true; } -bool ResourceUtil::LessThanOrEqual(const ResourceAllocation& lhs, - const ResourceAllocation& rhs) const { +bool ResourceUtil::LessThanOrEqualNormalized( + const ResourceAllocation& lhs, const ResourceAllocation& rhs) const { const Status validity = VerifyValidity(lhs); DCHECK_EQ(Status::OK(), validity); if (!validity.ok()) { @@ -282,7 +309,7 @@ bool ResourceUtil::LessThanOrEqual(const ResourceAllocation& lhs, return true; } -ResourceAllocation ResourceUtil::Overbind( +ResourceAllocation ResourceUtil::OverbindNormalized( const ResourceAllocation& allocation) const { const Status validity = VerifyValidity(allocation); DCHECK_EQ(Status::OK(), validity); diff --git a/tensorflow_serving/resources/resource_util.h b/tensorflow_serving/resources/resource_util.h index ba8f4c4fc07..fe18114d1bd 100644 --- a/tensorflow_serving/resources/resource_util.h +++ b/tensorflow_serving/resources/resource_util.h @@ -61,8 +61,6 @@ class ResourceUtil { // Determines whether 'allocation' is bound, defined as follows: // 1. An individual entry is bound iff a device_instance is supplied. // 2. An allocation is bound iff every entry is bound. - // - // Assumes the input is normalized. bool IsBound(const ResourceAllocation& allocation) const; // Adds 'to_add' to 'base'. @@ -71,16 +69,12 @@ class ResourceUtil { // {(GPU//RAM/8)} to {(GPU/instance_0/RAM/16), // (GPU//RAM/4)} yields {(GPU/instance_0/RAM/16), // (GPU//RAM/12)}. - // - // Assumes the inputs are normalized, and produces normalized output. void Add(const ResourceAllocation& to_add, ResourceAllocation* base) const; // Attempts to subtract 'to_subtract' from 'base'. Like Add(), keeps bound and // unbound entries separate. Returns true and mutates 'base' iff the // subtraction is legal, i.e. no negative quantities (which cannot be // represented) are produced. - // - // Assumes the inputs are normalized, and produces normalized output. bool Subtract(const ResourceAllocation& to_subtract, ResourceAllocation* base) const; @@ -92,8 +86,6 @@ class ResourceUtil { // the unbound quantity in 'lhs' is <= the quantity in 'rhs' bound to I. // // IMPORTANT: Assumes 'rhs' is bound; has undefined behavior otherwise. - // - // Assumes the inputs are normalized. bool LessThanOrEqual(const ResourceAllocation& lhs, const ResourceAllocation& rhs) const; @@ -108,11 +100,31 @@ class ResourceUtil { // This operation is useful for reasoning about monotonicity and availability // of resources, not as a means to permanently bind resources to devices // (because it binds resources redundantly to all device instances). - // - // Assumes the inputs are normalized, and produces normalized output. ResourceAllocation Overbind(const ResourceAllocation& allocation) const; private: + // Like IsBound(), but assumes the input is normalized. + bool IsBoundNormalized(const ResourceAllocation& allocation) const; + + // Like Add(), but assumes the input is normalized and produces normalized + // output. + void AddNormalized(const ResourceAllocation& to_add, + ResourceAllocation* base) const; + + // Like Subtract(), but assumes the input is normalized and produces + // normalized output. + bool SubtractNormalized(const ResourceAllocation& to_subtract, + ResourceAllocation* base) const; + + // Like LessThanOrEqual(), but assumes the input is normalized. + bool LessThanOrEqualNormalized(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const; + + // Like Overbind(), but assumes the input is normalized and produces + // normalized output. + ResourceAllocation OverbindNormalized( + const ResourceAllocation& allocation) const; + const std::map devices_; TF_DISALLOW_COPY_AND_ASSIGN(ResourceUtil); From 8da2f7632efd7c54bf60a632e8b551795a82992a Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 26 Oct 2016 13:18:20 -0800 Subject: [PATCH 0045/8554] Merge changes from github. Change: 137318673 --- tensorflow_serving/model_servers/main.cc | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 3716e8bdd3b..b048a32de09 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -189,11 +189,16 @@ int main(int argc, char** argv) { tensorflow::int32 port = 8500; bool enable_batching = false; tensorflow::string model_name = "default"; + tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("file_system_poll_wait_seconds", + &file_system_poll_wait_seconds, + "interval in seconds between each poll of the file " + "system for new model version"), tensorflow::Flag("model_base_path", &model_base_path, "path to export (required)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); @@ -222,6 +227,7 @@ int main(int argc, char** argv) { ServerCoreConfig core_config; core_config.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); + core_config.file_system_poll_wait_seconds = file_system_poll_wait_seconds; std::unique_ptr core; TF_CHECK_OK(ServerCore::Create( From 5400a485d9be05da5771ef256a4a313ce8132cf6 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 27 Oct 2016 11:41:14 -0800 Subject: [PATCH 0046/8554] Replaces use of explicit HashString with implicit std::hash for BasicManager::ManagedMap. Change: 137427766 --- tensorflow_serving/core/basic_manager.h | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index d580ade21bf..a96bf16ec2d 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -360,15 +360,11 @@ class BasicManager : public Manager { uint32 num_load_unload_threads() const LOCKS_EXCLUDED(num_load_unload_threads_mu_); - struct HashString { - uint64 operator()(const string& str) const { return Hash64(str); } - }; // Keys are the servable names. // Values are the harnesses for each servable version. The values when // fetched, are unordered. using ManagedMap = - std::unordered_multimap, - HashString>; + std::unordered_multimap>; // Fetches the harness with this id from the harness_map_. Returns // harness_map_.end(), if the harness is not found. From 5a0e1a972caa1f59a5831da83314d24f193d2cc0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 28 Oct 2016 15:05:40 -0800 Subject: [PATCH 0047/8554] Public no-op. Change: 137564393 --- tensorflow_serving/g3doc/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 8c9a154902c..ee1e5c09f4f 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -7,7 +7,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel TensorFlow Serving requires Bazel 0.3.1 or higher. You can find the Bazel -installation instructions [here](http://bazel.io/docs/install.html). +installation instructions [here](http://bazel.build/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the following steps: From eacbee06c157ae2c0239e64a37cb2a8f9fad56bf Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 1 Nov 2016 11:16:48 -0800 Subject: [PATCH 0048/8554] Add logging to LoaderHarness. Change: 137854851 --- tensorflow_serving/core/loader_harness.cc | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/core/loader_harness.cc b/tensorflow_serving/core/loader_harness.cc index 0eebabb5014..37cd41c3c6b 100644 --- a/tensorflow_serving/core/loader_harness.cc +++ b/tensorflow_serving/core/loader_harness.cc @@ -29,7 +29,9 @@ LoaderHarness::LoaderHarness(const ServableId& id, : id_(id), loader_(std::move(loader)), additional_state_(nullptr), - options_(options) {} + options_(options) { + VLOG(1) << "Starting to manage servable version " << id_; +} LoaderHarness::~LoaderHarness() { mutex_lock l(mu_); @@ -54,6 +56,8 @@ Status LoaderHarness::LoadRequested() { StateDebugString(State::kNew)); } state_ = State::kLoadRequested; + VLOG(1) << "Load requested for servable version " << id_; + return Status::OK(); } From cac139e3ca40eb81f8061bdfcb3cd26c85bab39a Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 1 Nov 2016 14:44:57 -0700 Subject: [PATCH 0049/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index c8a45a8e236..d9a89a5cf63 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit c8a45a8e236776bed1d14fd71f3b6755bd63cc58 +Subproject commit d9a89a5cf63b2ecedf68d3eefdc24be3f519e503 diff --git a/tf_models b/tf_models index 77080ad11f0..0dea688a68a 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 77080ad11f016e5365d28d0681cc4c0a95a7061f +Subproject commit 0dea688a68a7179e62cb2e68390c24b2ef7b0ef0 From 0667a82b905d712691f1d2303fe86a2a90937fd3 Mon Sep 17 00:00:00 2001 From: Montana Flynn Date: Thu, 3 Nov 2016 15:10:46 -0700 Subject: [PATCH 0050/8554] Switch replication controller to deployment (#226) --- tensorflow_serving/example/inception_k8s.json | 97 +++++++------------ tensorflow_serving/g3doc/serving_inception.md | 72 +++++++------- 2 files changed, 72 insertions(+), 97 deletions(-) diff --git a/tensorflow_serving/example/inception_k8s.json b/tensorflow_serving/example/inception_k8s.json index 764e18089db..89631768dd5 100644 --- a/tensorflow_serving/example/inception_k8s.json +++ b/tensorflow_serving/example/inception_k8s.json @@ -1,61 +1,36 @@ -{ - "apiVersion": "v1", - "kind": "ReplicationController", - "metadata": { - "name": "inception-controller" - }, - "spec": { - "replicas": 3, - "selector": { - "worker": "inception-pod" - }, - "template": { - "metadata": { - "labels": { - "worker": "inception-pod" - } - }, - "spec": { - "containers": [ - { - "name": "inception-container", - "image": "gcr.io/tensorflow-serving/inception", - "command": [ - "/bin/sh", - "-c" - ], - "args": [ - "/serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=inception --model_base_path=/serving/inception-export" - ], - "ports": [ - { - "containerPort": 9000 - } - ] - } - ], - "restartPolicy": "Always" - } - } - } -} - -{ - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "inception-service" - }, - "spec": { - "ports": [ - { - "port": 9000, - "targetPort": 9000 - } - ], - "selector": { - "worker": "inception-pod" - }, - "type": "LoadBalancer" - } -} +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: inception-deployment +spec: + replicas: 3 + template: + metadata: + labels: + app: inception-server + spec: + containers: + - name: inception-container + image: gcr.io/tensorflow-serving/inception:0.2.0 + command: + - /bin/sh + - -c + args: + - /serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server + --port=9000 --model_name=inception --model_base_path=/serving/inception-export + ports: + - containerPort: 9000 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: inception-service + name: inception-service +spec: + ports: + - port: 9000 + targetPort: 9000 + selector: + run: inception-service + type: LoadBalancer diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index fe3245f3cf1..38749c73cf3 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -216,13 +216,13 @@ Next we push the image to the Registry, $ gcloud docker push gcr.io/tensorflow-serving/inception ``` -### Create Kubernetes Replication Controller and Service +### Create Kubernetes Deployment and Service -The deployment consists of multiple replicas of `inception_inference` server +The deployment consists of 3 replicas of `inception_inference` server controlled by a -[Kubernetes Replication Controller](https://cloud.google.com/container-engine/docs/replicationcontrollers/operations). +[Kubernetes Deployment](http://kubernetes.io/docs/user-guide/deployments/). The replicas are exposed externally by a -[Kubernetes Service](https://cloud.google.com/container-engine/docs/services/operations) +[Kubernetes Service](http://kubernetes.io/docs/user-guide/services/) along with an [External Load Balancer](http://kubernetes.io/docs/user-guide/load-balancer/). @@ -231,64 +231,64 @@ We create them using the example Kubernetes config ```shell $ kubectl create -f tensorflow_serving/example/inception_k8s.json -replicationcontroller "inception-controller" created +deployment "inception-deployment" created service "inception-service" created ``` -To view status of the replication controller and pods: +To view status of the deployment and pods: ```shell -$ kubectl get rc -CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE -inception-controller inception-container gcr.io/tensorflow-serving/inception worker=inception-pod 3 20s +$ kc get deployments +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +inception-deployment 3 3 3 3 5s ``` ```shell -$ kubectl get pod +$ kubectl get pods NAME READY STATUS RESTARTS AGE -inception-controller-bbcbc 1/1 Running 0 1m -inception-controller-cj6l2 1/1 Running 0 1m -inception-controller-t1uep 1/1 Running 0 1m +inception-deployment-bbcbc 1/1 Running 0 10s +inception-deployment-cj6l2 1/1 Running 0 10s +inception-deployment-t1uep 1/1 Running 0 10s ``` To view status of the service: ```shell -$ kubectl get svc -NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE -inception-service 10.15.242.244 146.148.88.232 9000/TCP worker=inception-pod 3m -kubernetes 10.15.240.1 443/TCP 1h +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +inception-service 10.239.240.227 104.155.184.157 9000/TCP 1m ``` +It can take a while for everything to be up and running. + ```shell -$ kubectl describe svc inception-service -Name: inception-service -Namespace: default -Labels: -Selector: worker=inception-pod -Type: LoadBalancer -IP: 10.15.242.244 -LoadBalancer Ingress: 146.148.88.232 -Port: 9000/TCP -NodePort: 32006/TCP -Endpoints: 10.12.2.4:9000,10.12.4.4:9000,10.12.4.5:9000 -Session Affinity: None +$ kubectl describe service inception-service +Name: inception-service +Namespace: default +Labels: run=inception-service +Selector: run=inception-service +Type: LoadBalancer +IP: 10.239.240.227 +LoadBalancer Ingress: 104.155.184.157 +Port: 9000/TCP +NodePort: 30334/TCP +Endpoints: +Session Affinity: None Events: - FirstSeen LastSeen Count From SubobjectPath Reason Message - ───────── ──────── ───── ──── ───────────── ────── ─────── - 4m 3m 2 {service-controller } CreatingLoadBalancer Creating load balancer - 3m 2m 2 {service-controller } CreatedLoadBalancer Created load balancer + FirstSeen LastSeen Count From SubobjectPath Type Reason Message + --------- -------- ----- ---- ------------- -------- ------ ------- + 1m 1m 1 {service-controller } Normal CreatingLoadBalancer Creating load balancer + 1m 1m 1 {service-controller } Normal CreatedLoadBalancer Created load balancer ``` -It can take a while for everything to be up and running. The service external -IP address is listed next to LoadBalancer Ingress. +The service external IP address is listed next to LoadBalancer Ingress. ### Query the model We can now query the service at its external address from our local host. ```shell -$ bazel-bin/tensorflow_serving/example/inception_client --server=146.148.88.232:9000 --image=/path/to/my_cat_image.jpg +$ bazel-bin/tensorflow_serving/example/inception_client --server=104.155.184.157:9000 --image=/path/to/my_cat_image.jpg outputs { key: "classes" value { From 7511cfa623a5cb804e7f1b78ae9253cfc9e2b12c Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 1 Nov 2016 14:31:13 -0800 Subject: [PATCH 0051/8554] Add VLOG(1) logging to managers and aspired-version policies. Change: 137879815 --- .../core/aspired_versions_manager.cc | 20 +++++++++++++++++++ tensorflow_serving/core/basic_manager.cc | 5 +++++ tensorflow_serving/core/eager_load_policy.cc | 3 +++ .../core/eager_unload_policy.cc | 4 ++++ 4 files changed, 32 insertions(+) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 887ff94befc..5e862008693 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -103,6 +103,16 @@ std::set GetVersionNumbers( return version_numbers; } +// Creates a debug string for a given vector of servable versions. +string ServableVersionsDebugString( + const std::vector>>& versions) { + std::vector version_strings; + for (const ServableData>& version : versions) { + version_strings.push_back(version.id().DebugString()); + } + return str_util::Join(version_strings, ", "); +} + } // namespace namespace internal { @@ -221,6 +231,8 @@ void AspiredVersionsManager::EnqueueAspiredVersionsRequest( { mutex_lock l(pending_aspired_versions_requests_mu_); + VLOG(1) << "Enqueueing aspired versions request: " + << ServableVersionsDebugString(versions); pending_aspired_versions_requests_[servable_name.ToString()] = std::move(versions); } @@ -229,6 +241,9 @@ void AspiredVersionsManager::EnqueueAspiredVersionsRequest( void AspiredVersionsManager::ProcessAspiredVersionsRequest( const StringPiece servable_name, std::vector>> versions) { + VLOG(1) << "Processing aspired versions request: " + << ServableVersionsDebugString(versions); + const std::set next_aspired_versions = GetVersionNumbers(versions); // We gather all the servables with the servable_name and @@ -319,6 +334,8 @@ AspiredVersionsManager::GetNextAction() { std::sort(actions.begin(), actions.end(), CompareActions()); const optional next_action = !actions.empty() ? actions[0] : nullopt; + VLOG(1) << "Taking action: " + << (next_action ? next_action->DebugString() : ""); return next_action; } @@ -377,6 +394,9 @@ void AspiredVersionsManager::HandlePendingAspiredVersionsRequests() { if (ContainsAnyReaspiredVersions(servable_name, versions)) { // Sit on it for now. We'll check again later. ++it; + VLOG(1) << "Postponing processing of aspired versions request due to " + "re-aspired version(s) among: " + << ServableVersionsDebugString(versions); } else { ProcessAspiredVersionsRequest(servable_name, std::move(versions)); it = pending_aspired_versions_requests_.erase(it); diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index e16b4e605a2..68f4d6bf37d 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -311,6 +311,8 @@ Status BasicManager::ManageServableInternal( std::function(const ServableId&, std::unique_ptr)> harness_creator) { + VLOG(1) << "Request to start managing servable " << servable.id(); + mutex_lock l(mu_); const auto iter = BasicManager::FindHarnessInMap(servable.id()); @@ -351,6 +353,7 @@ Status BasicManager::ManageServable( } Status BasicManager::StopManagingServable(const ServableId& id) { + VLOG(1) << "Request to stop managing servable " << id; mutex_lock l(mu_); const auto it = FindHarnessInMap(id); if (it == managed_map_.end()) { @@ -471,6 +474,7 @@ Status BasicManager::ExecuteLoad(LoaderHarness* harness) { void BasicManager::LoadServable(const ServableId& id, const DoneCallback done_callback) { + VLOG(1) << "Request to load servable " << id; LoadOrUnloadRequest request; request.kind = LoadOrUnloadRequest::Kind::kLoad; request.servable_id = id; @@ -511,6 +515,7 @@ Status BasicManager::ExecuteUnload(LoaderHarness* harness) { void BasicManager::UnloadServable(const ServableId& id, const DoneCallback done_callback) { + VLOG(1) << "Request to unload servable " << id; LoadOrUnloadRequest request; request.kind = LoadOrUnloadRequest::Kind::kUnload; request.servable_id = id; diff --git a/tensorflow_serving/core/eager_load_policy.cc b/tensorflow_serving/core/eager_load_policy.cc index 53c8f56990b..07649fafaab 100644 --- a/tensorflow_serving/core/eager_load_policy.cc +++ b/tensorflow_serving/core/eager_load_policy.cc @@ -24,6 +24,7 @@ optional EagerLoadPolicy::GetNextAction( // If there is a new aspired version, load it. for (const auto& version : all_versions) { if (version.is_aspired && version.state == LoaderHarness::State::kNew) { + VLOG(1) << "EagerLoadPolicy requesting to load servable " << version.id; return {{Action::kLoad, version.id}}; } } @@ -37,6 +38,7 @@ optional EagerLoadPolicy::GetNextAction( version.state != LoaderHarness::State::kReady; }); if (aspired_not_serving) { + VLOG(1) << "EagerLoadPolicy requesting no-op"; return nullopt; } @@ -44,6 +46,7 @@ optional EagerLoadPolicy::GetNextAction( // latter. for (const auto& version : all_versions) { if (!version.is_aspired && version.state == LoaderHarness::State::kReady) { + VLOG(1) << "EagerLoadPolicy requesting to unload servable " << version.id; return {{Action::kUnload, version.id}}; } } diff --git a/tensorflow_serving/core/eager_unload_policy.cc b/tensorflow_serving/core/eager_unload_policy.cc index 93461bc5fcb..b2b59cb7500 100644 --- a/tensorflow_serving/core/eager_unload_policy.cc +++ b/tensorflow_serving/core/eager_unload_policy.cc @@ -24,6 +24,8 @@ optional EagerUnloadPolicy::GetNextAction( // aspired. Unload the first if any. for (const auto& version : all_versions) { if (version.state == LoaderHarness::State::kReady && !version.is_aspired) { + VLOG(1) << "EagerUnloadPolicy requesting to unload servable " + << version.id; return AspiredVersionPolicy::ServableAction( {Action::kUnload, version.id}); } @@ -39,6 +41,7 @@ optional EagerUnloadPolicy::GetNextAction( version.state != LoaderHarness::State::kError; }); if (not_aspired_not_finished) { + VLOG(1) << "EagerUnloadPolicy requesting no-op"; return nullopt; } @@ -46,6 +49,7 @@ optional EagerUnloadPolicy::GetNextAction( // versions and find any in kNew that are aspired. Load the first if any. for (const auto& version : all_versions) { if (version.state == LoaderHarness::State::kNew && version.is_aspired) { + VLOG(1) << "EagerUnloadPolicy requesting to load servable " << version.id; return AspiredVersionPolicy::ServableAction({Action::kLoad, version.id}); } } From 912301606353a9432f3d7838f8a9d74cb7afc0c1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 Nov 2016 14:04:27 -0800 Subject: [PATCH 0052/8554] Add Options for ServerCore and change the factory method to use Options. Change: 137999252 --- tensorflow_serving/model_servers/main.cc | 23 ++++++++------ .../model_servers/server_core.cc | 22 ++++++------- .../model_servers/server_core.h | 31 ++++++++++++++----- .../test_util/server_core_test_util.cc | 29 +++++++++-------- 4 files changed, 63 insertions(+), 42 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index b048a32de09..d5e6fba7221 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -212,7 +212,8 @@ int main(int argc, char** argv) { std::cout << "unknown argument: " << argv[1] << "\n" << usage; } - ModelServerConfig config = + ServerCore::Options options; + options.model_server_config = BuildSingleModelConfig(model_name, model_base_path); SessionBundleSourceAdapterConfig source_adapter_config; @@ -223,18 +224,22 @@ int main(int argc, char** argv) { batching_parameters->mutable_thread_pool_name()->set_value( "model_server_batch_threads"); } + options.source_adapter_creator = [source_adapter_config]( + const string& model_platform, + std::unique_ptr* adapter) { + return CreateSourceAdapter(source_adapter_config, model_platform, adapter); + }; - ServerCoreConfig core_config; - core_config.aspired_version_policy = + options.servable_state_monitor_creator = &CreateServableStateMonitor; + options.custom_model_config_loader = &LoadCustomModelConfig; + + options.server_core_config.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); - core_config.file_system_poll_wait_seconds = file_system_poll_wait_seconds; + options.server_core_config.file_system_poll_wait_seconds = + file_system_poll_wait_seconds; std::unique_ptr core; - TF_CHECK_OK(ServerCore::Create( - config, std::bind(CreateSourceAdapter, source_adapter_config, - std::placeholders::_1, std::placeholders::_2), - &CreateServableStateMonitor, &LoadCustomModelConfig, - std::move(core_config), &core)); + TF_CHECK_OK(ServerCore::Create(std::move(options), &core)); RunServer(port, std::move(core)); return 0; diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 5cb326dc2a3..dc7d88fe154 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -80,21 +80,19 @@ Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, // Public Methods. // ************************************************************************ -Status ServerCore::Create( - const ModelServerConfig& config, - const SourceAdapterCreator& source_adapter_creator, - const ServableStateMonitorCreator& servable_state_monitor_creator, - const CustomModelConfigLoader& custom_model_config_loader, - ServerCoreConfig server_core_config, - std::unique_ptr* server_core) { +Status ServerCore::Create(Options options, + std::unique_ptr* server_core) { + // We need to move the aspired_version_policy first because we will move the + // server_core_config (which contains aspired_version_policy) below. std::unique_ptr aspired_version_policy = - std::move(server_core_config.aspired_version_policy); - server_core->reset(new ServerCore( - source_adapter_creator, servable_state_monitor_creator, - custom_model_config_loader, std::move(server_core_config))); + std::move(options.server_core_config.aspired_version_policy); + server_core->reset(new ServerCore(options.source_adapter_creator, + options.servable_state_monitor_creator, + options.custom_model_config_loader, + std::move(options.server_core_config))); TF_RETURN_IF_ERROR( (*server_core)->Initialize(std::move(aspired_version_policy))); - return (*server_core)->ReloadConfig(config); + return (*server_core)->ReloadConfig(options.model_server_config); } // ************************************************************************ diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 0db630eb92a..efb2ee2ac45 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -83,8 +83,6 @@ class ServerCoreTestAccess; class ServerCore { public: - virtual ~ServerCore() = default; - using ModelServerSourceAdapter = SourceAdapter>; @@ -105,18 +103,35 @@ class ServerCore { const ::google::protobuf::Any& any, EventBus* event_bus, UniquePtrWithDeps* manager)>; + struct Options { + // ModelServer configuration. + ModelServerConfig model_server_config; + + // ServerCore tuning parameters. + // TODO(b/32336469): move the fields in ServerCoreConfig into this struct. + ServerCoreConfig server_core_config; + + // A function for creating ModelServerSourceAdapter based on the + // 'platform_type'. + SourceAdapterCreator source_adapter_creator; + + // A function for creating ServableStateMonitor. + ServableStateMonitorCreator servable_state_monitor_creator; + + // A function for instantiating and connecting custom sources and source + // adapters to the manager. + CustomModelConfigLoader custom_model_config_loader; + }; + + virtual ~ServerCore() = default; + // Creates a ServerCore instance with all the models and sources per the // ModelServerConfig. // // For models statically configured with ModelConfigList, waits for them // to be made available (or hit an error) for serving before returning. // Returns an error status if any such model fails to load. - static Status Create( - const ModelServerConfig& config, - const SourceAdapterCreator& source_adapter_creator, - const ServableStateMonitorCreator& servable_state_monitor_creator, - const CustomModelConfigLoader& custom_model_config_loader, - ServerCoreConfig server_core_config, std::unique_ptr* core); + static Status Create(Options options, std::unique_ptr* core); // Updates the server core with all the models and sources per the // ModelServerConfig. Like Create(), waits for all statically configured diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index c4d69db6b3c..12b237ac3b3 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -56,19 +56,22 @@ Status ServerCoreTest::CreateServerCore( const ModelServerConfig& config, const ServerCore::SourceAdapterCreator& source_adapter_creator, std::unique_ptr* server_core) { - return ServerCore::Create( - config, source_adapter_creator, // ServerCore::SourceAdapterCreator - [](EventBus* event_bus, - std::unique_ptr* monitor) -> Status { - monitor->reset(new ServableStateMonitor(event_bus)); - return Status::OK(); - }, // ServerCore::ServableStateMonitor - [](const ::google::protobuf::Any& any, EventBus* event_bus, - UniquePtrWithDeps* manager) -> Status { - return Status::OK(); - }, // ServerCore::CustomModelConfigLoader - GetTestServerCoreConfig(), - server_core); + ServerCore::Options options; + options.model_server_config = config; + options.source_adapter_creator = source_adapter_creator; + options.servable_state_monitor_creator = []( + EventBus* event_bus, + std::unique_ptr* monitor) -> Status { + monitor->reset(new ServableStateMonitor(event_bus)); + return Status::OK(); + }; + options.custom_model_config_loader = []( + const ::google::protobuf::Any& any, EventBus* event_bus, + UniquePtrWithDeps* manager) -> Status { + return Status::OK(); + }; + options.server_core_config = GetTestServerCoreConfig(); + return ServerCore::Create(std::move(options), server_core); } } // namespace test_util From d67f6e60a436af46ef886e38efb650cb7589758e Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 2 Nov 2016 14:37:49 -0800 Subject: [PATCH 0053/8554] Update install instructions to include libcurl3-dev package. Change: 138003543 --- tensorflow_serving/g3doc/setup.md | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index ee1e5c09f4f..3426e0ee035 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -42,6 +42,7 @@ To install TensorFlow Serving dependencies, execute the following: sudo apt-get update && sudo apt-get install -y \ build-essential \ curl \ + libcurl3-dev \ git \ libfreetype6-dev \ libpng12-dev \ From 02f7aae66e24e2754566b51f6c9610318ff74d60 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 2 Nov 2016 14:49:10 -0800 Subject: [PATCH 0054/8554] Add signature_name to ModelSpec. Change: 138004925 --- tensorflow_serving/apis/model.proto | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tensorflow_serving/apis/model.proto b/tensorflow_serving/apis/model.proto index 889303ff45d..e74cf383895 100644 --- a/tensorflow_serving/apis/model.proto +++ b/tensorflow_serving/apis/model.proto @@ -15,4 +15,8 @@ message ModelSpec { // co-trained and/or have inter-dependencies on the versions used at inference // time. google.protobuf.Int64Value version = 2; + + // A named signature to evaluate. If unspecified, the default signature will + // be used. Note that only MultiInference will initially support this. + string signature_name = 3; } From 50129ec0b407e5778282ee8d7610a9494cfcf6b2 Mon Sep 17 00:00:00 2001 From: wai chee yau Date: Sat, 5 Nov 2016 03:03:51 +1100 Subject: [PATCH 0055/8554] Add options to serve multiple versions of model (#217) * aspire all model versions in the model base path * update description and naming --- tensorflow_serving/g3doc/serving_advanced.md | 5 +- .../file_system_storage_path_source.cc | 109 ++++++++++++++---- .../file_system_storage_path_source.proto | 15 +++ .../file_system_storage_path_source_test.cc | 32 +++++ 4 files changed, 136 insertions(+), 25 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 5173297a86d..0841c8991ab 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -142,9 +142,8 @@ With all these, `ServerCore` internally does the following: `AspiredVersionsManager` that manages all such `Loader` instances created by the `SessionBundleSourceAdapter`. -Whenever a new version is available, this `AspiredVersionsManager` always -unloads the old version and replaces it with the new one. If you want to -start customizing, you are encouraged to understand the components that it +Whenever a new version is available, this `AspiredVersionsManager` loads the new version, and, +under its default behavior, unloads the old one. If you want to start customizing, you are encouraged to understand the components that it creates internally, and how to configure them. It is worth mentioning that TensorFlow Serving is designed from scratch to be diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index a824fa1edc7..94a6f405ff7 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -76,6 +76,84 @@ std::set GetDeletedServables( return deleted_servables; } +//Add a new ServableData for the model version to the the vector of versions to aspire +void AspireVersion( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const string& version_relative_path, + const int version_number, + std::vector>* versions) { + const ServableId servable_id = {servable.servable_name(), version_number}; + const string full_path = + io::JoinPath(servable.base_path(), version_relative_path); + versions->emplace_back(ServableData(servable_id, full_path)); +} + +// Attempts to parse the version path as integer, upon success writes the result to 'version_number' and returns true. +// Otherwise returns false when fail to parse the version path. +bool ParseVersionNumber(const string& version_path, int64 *version_number) { + return strings::safe_strto64(version_path.c_str(), version_number); +} + +// Update the servable data to include +// all the model versions found in the base path as aspired versions. +// The argument 'children' represents a list of base-path children from the file system. +// Returns true if one or more valid servable version paths are found, otherwise returns false. +bool AspireAllVersions( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const std::vector& children, // represents a list of base-path children from the file system. + std::vector>* versions) { + + bool at_least_one_version_found = false; + for (const string& child : children) { + // Identify all the versions, among children that can be interpreted as + // version numbers. + int64 version_number; + if (ParseVersionNumber(child, &version_number)) { + // Emit all the aspired-versions data. + AspireVersion(servable, child, version_number, versions); + at_least_one_version_found = true; + } + } + + return at_least_one_version_found; +} + +// Update the servable data to include +// the latest version found in the base path as aspired version. +// The argument 'children' represents a list of base-path children from the file system. +// Returns true if valid servable path is found, otherwise returns false. +bool AspireLatestVersion( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const std::vector& children, + std::vector>* versions) { + + // Identify the latest version, among children that can be interpreted as + // version numbers. + int latest_version_child_index; + int64 latest_version_number; + bool at_least_one_version_found = false; + for (int i = 0; i < children.size(); ++i) { + int64 version_number; + if (!ParseVersionNumber(children[i], &version_number)) { + continue; + } + + // Check if this is the largest version number + if (!at_least_one_version_found || latest_version_number < version_number) { + latest_version_child_index = i; + latest_version_number = version_number; + } + at_least_one_version_found = true; + } + + // Emit the latest aspired version. + if (at_least_one_version_found) { + AspireVersion(servable, children[latest_version_child_index], latest_version_number, versions); + } + + return at_least_one_version_found; +} + // Like PollFileSystemForConfig(), but for a single servable. Status PollFileSystemForServable( const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, @@ -105,30 +183,17 @@ Status PollFileSystemForServable( children.clear(); children.insert(children.begin(), real_children.begin(), real_children.end()); - // Identify the latest version, among children that can be interpreted as - // version numbers. - int latest_version_child = -1; - int64 latest_version; - for (int i = 0; i < children.size(); ++i) { - const string& child = children[i]; - int64 child_version_num; - if (!strings::safe_strto64(child.c_str(), &child_version_num)) { - continue; - } - - if (latest_version_child < 0 || latest_version < child_version_num) { - latest_version_child = i; - latest_version = child_version_num; - } + bool at_least_one_version_found = false; + switch (servable.version_policy()) { + case FileSystemStoragePathSourceConfig::LATEST_VERSION: + at_least_one_version_found = AspireLatestVersion(servable, children, versions); + break; + case FileSystemStoragePathSourceConfig::ALL_VERSIONS: + at_least_one_version_found = AspireAllVersions(servable, children, versions); + break; } - // Emit the aspired-versions data. - if (latest_version_child >= 0) { - const ServableId servable_id = {servable.servable_name(), latest_version}; - const string full_path = - io::JoinPath(servable.base_path(), children[latest_version_child]); - versions->emplace_back(ServableData(servable_id, full_path)); - } else { + if (!at_least_one_version_found) { LOG(WARNING) << "No versions of servable " << servable.servable_name() << " found under base path " << servable.base_path(); } diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto index ca4d0c15cf9..388abf59988 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto @@ -4,6 +4,17 @@ package tensorflow.serving; // Config proto for FileSystemStoragePathSource. message FileSystemStoragePathSourceConfig { + + // The policy to define how many versions of the servable should be + // served at the same time. + enum VersionPolicy { + // Only serve the latest version that exists in the base path. + // This is the default behavior. + LATEST_VERSION = 0; + // Serves all the versions that exist in the base path. + ALL_VERSIONS = 1; + } + // A servable name and base path to look for versions of the servable. message ServableToMonitor { // The servable name to supply in aspired-versions callback calls. Child @@ -12,6 +23,10 @@ message FileSystemStoragePathSourceConfig { // The path to monitor, i.e. look for child paths of the form base_path/123. string base_path = 2; + + // The policy to determines the number of versions of the servable to be served at the same time. + VersionPolicy version_policy = 3; + }; // The servables to monitor for new versions, and aspire. diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index ca06f005106..b3b623ebfba 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -171,6 +171,38 @@ TEST(FileSystemStoragePathSourceTest, MultipleVersions) { .PollFileSystemAndInvokeCallback()); } +TEST(FileSystemStoragePathSourceTest, MultipleVersionsAtTheSameTime) { + const string base_path = io::JoinPath(testing::TmpDir(), "MultipleVersionsAtTheSameTime"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + TF_ASSERT_OK(Env::Default()->CreateDir( + io::JoinPath(base_path, "non_numerical_child"))); + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, "42"))); + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, "17"))); + + auto config = test_util::CreateProto( + strings::Printf("servables: { " + " version_policy: ALL_VERSIONS " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL(*target, SetAspiredVersions(Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 17}, io::JoinPath(base_path, "17")), + ServableData({"test_servable_name", 42}, io::JoinPath(base_path, "42"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + TEST(FileSystemStoragePathSourceTest, MultipleServables) { FileSystemStoragePathSourceConfig config; config.set_fail_if_zero_versions_at_startup(false); From c2c68c21a2b62bd218f747dc288cdf4903c2f366 Mon Sep 17 00:00:00 2001 From: Jan Margeta Date: Mon, 7 Nov 2016 19:40:43 +0100 Subject: [PATCH 0056/8554] Fix inception and mnist examples for Python 3 (#234) - Ran 2to3 converter to add print statement parentheses --- tensorflow_serving/example/inception_client.py | 2 +- tensorflow_serving/example/inception_export.py | 4 ++-- tensorflow_serving/example/mnist_client.py | 8 ++++---- tensorflow_serving/example/mnist_export.py | 18 +++++++++--------- tensorflow_serving/example/mnist_input_data.py | 10 +++++----- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py index e4e8a880210..18dd4af8a24 100644 --- a/tensorflow_serving/example/inception_client.py +++ b/tensorflow_serving/example/inception_client.py @@ -46,7 +46,7 @@ def main(_): request.inputs['images'].CopyFrom( tf.contrib.util.make_tensor_proto(data, shape=[1])) result = stub.Predict(request, 10.0) # 10 secs timeout - print result + print(result) if __name__ == '__main__': diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index 252b3f60d63..35669a5565c 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -108,7 +108,7 @@ def export(): print('Successfully loaded model from %s at step=%s.' % (ckpt.model_checkpoint_path, global_step)) else: - print 'No checkpoint file found at %s' % FLAGS.checkpoint_dir + print('No checkpoint file found at %s' % FLAGS.checkpoint_dir) return # Export inference model. @@ -129,7 +129,7 @@ def export(): default_graph_signature=classification_signature, named_graph_signatures=named_graph_signature) model_exporter.export(FLAGS.export_dir, tf.constant(global_step), sess) - print 'Successfully exported model to %s' % FLAGS.export_dir + print('Successfully exported model to %s' % FLAGS.export_dir) def preprocess_image(image_buffer): diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index 38899eb52c7..23c011b1254 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -105,7 +105,7 @@ def _callback(result_future): exception = result_future.exception() if exception: result_counter.inc_error() - print exception + print(exception) else: sys.stdout.write('.') sys.stdout.flush() @@ -154,14 +154,14 @@ def do_inference(hostport, work_dir, concurrency, num_tests): def main(_): if FLAGS.num_tests > 10000: - print 'num_tests should not be greater than 10k' + print('num_tests should not be greater than 10k') return if not FLAGS.server: - print 'please specify server host:port' + print('please specify server host:port') return error_rate = do_inference(FLAGS.server, FLAGS.work_dir, FLAGS.concurrency, FLAGS.num_tests) - print '\nInference error rate: %s%%' % (error_rate * 100) + print('\nInference error rate: %s%%' % (error_rate * 100)) if __name__ == '__main__': diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 08fae18e0a5..b4a724e5c7d 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -47,14 +47,14 @@ def main(_): '[--export_version=y] export_dir') sys.exit(-1) if FLAGS.training_iteration <= 0: - print 'Please specify a positive value for training iteration.' + print('Please specify a positive value for training iteration.') sys.exit(-1) if FLAGS.export_version <= 0: - print 'Please specify a positive value for version number.' + print('Please specify a positive value for version number.') sys.exit(-1) # Train model - print 'Training model...' + print('Training model...') mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) sess = tf.InteractiveSession() serialized_tf_example = tf.placeholder(tf.string, name='tf_example') @@ -73,23 +73,23 @@ def main(_): values, indices = tf.nn.top_k(y, 10) prediction_classes = tf.contrib.lookup.index_to_string( tf.to_int64(indices), - mapping=tf.constant([str(i) for i in xrange(10)])) + mapping=tf.constant([str(i) for i in range(10)])) for _ in range(FLAGS.training_iteration): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x: batch[0], y_: batch[1]}) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) - print 'training accuracy %g' % sess.run(accuracy, + print('training accuracy %g' % sess.run(accuracy, feed_dict={x: mnist.test.images, - y_: mnist.test.labels}) - print 'Done training!' + y_: mnist.test.labels})) + print('Done training!') # Export model # WARNING(break-tutorial-inline-code): The following code snippet is # in-lined in tutorials, please update tutorial documents accordingly # whenever code changes. export_path = sys.argv[-1] - print 'Exporting trained model to', export_path + print('Exporting trained model to %s' % export_path) init_op = tf.group(tf.initialize_all_tables(), name='init_op') saver = tf.train.Saver(sharded=True) model_exporter = exporter.Exporter(saver) @@ -104,7 +104,7 @@ def main(_): 'inputs': exporter.generic_signature({'images': x}), 'outputs': exporter.generic_signature({'scores': y})}) model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess) - print 'Done exporting!' + print('Done exporting!') if __name__ == '__main__': diff --git a/tensorflow_serving/example/mnist_input_data.py b/tensorflow_serving/example/mnist_input_data.py index 66693e059d2..49f5bf6eaf5 100644 --- a/tensorflow_serving/example/mnist_input_data.py +++ b/tensorflow_serving/example/mnist_input_data.py @@ -39,7 +39,7 @@ def maybe_download(filename, work_directory): if not os.path.exists(filepath): filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath) statinfo = os.stat(filepath) - print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') + print('Successfully downloaded %s %d bytes.' % (filename, statinfo.st_size)) return filepath @@ -50,7 +50,7 @@ def _read32(bytestream): def extract_images(filename): """Extract the images into a 4D uint8 numpy array [index, y, x, depth].""" - print('Extracting', filename) + print('Extracting %s' % filename) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2051: @@ -77,7 +77,7 @@ def dense_to_one_hot(labels_dense, num_classes=10): def extract_labels(filename, one_hot=False): """Extract the labels into a 1D uint8 numpy array [index].""" - print('Extracting', filename) + print('Extracting %s' % filename) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2049: @@ -144,8 +144,8 @@ def next_batch(self, batch_size, fake_data=False): fake_label = [1] + [0] * 9 else: fake_label = 0 - return [fake_image for _ in xrange(batch_size)], [ - fake_label for _ in xrange(batch_size)] + return [fake_image for _ in range(batch_size)], [ + fake_label for _ in range(batch_size)] start = self._index_in_epoch self._index_in_epoch += batch_size if self._index_in_epoch > self._num_examples: From 1c87a1409ffc5067d12c1a537ca53672d2c5d1b3 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 3 Nov 2016 15:10:28 -0800 Subject: [PATCH 0057/8554] Merge changes from github. Change: 138131621 --- tensorflow_serving/example/inception_k8s.json | 97 +++++++------------ tensorflow_serving/g3doc/serving_inception.md | 72 +++++++------- tensorflow_serving/model_servers/BUILD | 1 - 3 files changed, 72 insertions(+), 98 deletions(-) diff --git a/tensorflow_serving/example/inception_k8s.json b/tensorflow_serving/example/inception_k8s.json index 7877af436d1..89631768dd5 100644 --- a/tensorflow_serving/example/inception_k8s.json +++ b/tensorflow_serving/example/inception_k8s.json @@ -1,61 +1,36 @@ -{ - "apiVersion": "v1", - "kind": "ReplicationController", - "metadata": { - "name": "inception-controller" - }, - "spec": { - "replicas": 3, - "selector": { - "worker": "inception-pod" - }, - "template": { - "metadata": { - "labels": { - "worker": "inception-pod" - } - }, - "spec": { - "containers": [ - { - "name": "inception-container", - "image": "gcr.io/tensorflow-serving/inception", - "command": [ - "/bin/sh", - "-c" - ], - "args": [ - "/serving/bazel-bin/tensorflow_serving/example/inception_inference --port=9000 /serving/inception-export" - ], - "ports": [ - { - "containerPort": 9000 - } - ] - } - ], - "restartPolicy": "Always" - } - } - } -} - -{ - "kind": "Service", - "apiVersion": "v1", - "metadata": { - "name": "inception-service" - }, - "spec": { - "ports": [ - { - "port": 9000, - "targetPort": 9000 - } - ], - "selector": { - "worker": "inception-pod" - }, - "type": "LoadBalancer" - } -} \ No newline at end of file +apiVersion: extensions/v1beta1 +kind: Deployment +metadata: + name: inception-deployment +spec: + replicas: 3 + template: + metadata: + labels: + app: inception-server + spec: + containers: + - name: inception-container + image: gcr.io/tensorflow-serving/inception:0.2.0 + command: + - /bin/sh + - -c + args: + - /serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server + --port=9000 --model_name=inception --model_base_path=/serving/inception-export + ports: + - containerPort: 9000 +--- +apiVersion: v1 +kind: Service +metadata: + labels: + run: inception-service + name: inception-service +spec: + ports: + - port: 9000 + targetPort: 9000 + selector: + run: inception-service + type: LoadBalancer diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index fe3245f3cf1..38749c73cf3 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -216,13 +216,13 @@ Next we push the image to the Registry, $ gcloud docker push gcr.io/tensorflow-serving/inception ``` -### Create Kubernetes Replication Controller and Service +### Create Kubernetes Deployment and Service -The deployment consists of multiple replicas of `inception_inference` server +The deployment consists of 3 replicas of `inception_inference` server controlled by a -[Kubernetes Replication Controller](https://cloud.google.com/container-engine/docs/replicationcontrollers/operations). +[Kubernetes Deployment](http://kubernetes.io/docs/user-guide/deployments/). The replicas are exposed externally by a -[Kubernetes Service](https://cloud.google.com/container-engine/docs/services/operations) +[Kubernetes Service](http://kubernetes.io/docs/user-guide/services/) along with an [External Load Balancer](http://kubernetes.io/docs/user-guide/load-balancer/). @@ -231,64 +231,64 @@ We create them using the example Kubernetes config ```shell $ kubectl create -f tensorflow_serving/example/inception_k8s.json -replicationcontroller "inception-controller" created +deployment "inception-deployment" created service "inception-service" created ``` -To view status of the replication controller and pods: +To view status of the deployment and pods: ```shell -$ kubectl get rc -CONTROLLER CONTAINER(S) IMAGE(S) SELECTOR REPLICAS AGE -inception-controller inception-container gcr.io/tensorflow-serving/inception worker=inception-pod 3 20s +$ kc get deployments +NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE +inception-deployment 3 3 3 3 5s ``` ```shell -$ kubectl get pod +$ kubectl get pods NAME READY STATUS RESTARTS AGE -inception-controller-bbcbc 1/1 Running 0 1m -inception-controller-cj6l2 1/1 Running 0 1m -inception-controller-t1uep 1/1 Running 0 1m +inception-deployment-bbcbc 1/1 Running 0 10s +inception-deployment-cj6l2 1/1 Running 0 10s +inception-deployment-t1uep 1/1 Running 0 10s ``` To view status of the service: ```shell -$ kubectl get svc -NAME CLUSTER_IP EXTERNAL_IP PORT(S) SELECTOR AGE -inception-service 10.15.242.244 146.148.88.232 9000/TCP worker=inception-pod 3m -kubernetes 10.15.240.1 443/TCP 1h +$ kubectl get services +NAME CLUSTER-IP EXTERNAL-IP PORT(S) AGE +inception-service 10.239.240.227 104.155.184.157 9000/TCP 1m ``` +It can take a while for everything to be up and running. + ```shell -$ kubectl describe svc inception-service -Name: inception-service -Namespace: default -Labels: -Selector: worker=inception-pod -Type: LoadBalancer -IP: 10.15.242.244 -LoadBalancer Ingress: 146.148.88.232 -Port: 9000/TCP -NodePort: 32006/TCP -Endpoints: 10.12.2.4:9000,10.12.4.4:9000,10.12.4.5:9000 -Session Affinity: None +$ kubectl describe service inception-service +Name: inception-service +Namespace: default +Labels: run=inception-service +Selector: run=inception-service +Type: LoadBalancer +IP: 10.239.240.227 +LoadBalancer Ingress: 104.155.184.157 +Port: 9000/TCP +NodePort: 30334/TCP +Endpoints: +Session Affinity: None Events: - FirstSeen LastSeen Count From SubobjectPath Reason Message - ───────── ──────── ───── ──── ───────────── ────── ─────── - 4m 3m 2 {service-controller } CreatingLoadBalancer Creating load balancer - 3m 2m 2 {service-controller } CreatedLoadBalancer Created load balancer + FirstSeen LastSeen Count From SubobjectPath Type Reason Message + --------- -------- ----- ---- ------------- -------- ------ ------- + 1m 1m 1 {service-controller } Normal CreatingLoadBalancer Creating load balancer + 1m 1m 1 {service-controller } Normal CreatedLoadBalancer Created load balancer ``` -It can take a while for everything to be up and running. The service external -IP address is listed next to LoadBalancer Ingress. +The service external IP address is listed next to LoadBalancer Ingress. ### Query the model We can now query the service at its external address from our local host. ```shell -$ bazel-bin/tensorflow_serving/example/inception_client --server=146.148.88.232:9000 --image=/path/to/my_cat_image.jpg +$ bazel-bin/tensorflow_serving/example/inception_client --server=104.155.184.157:9000 --image=/path/to/my_cat_image.jpg outputs { key: "classes" value { diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 405062817ab..553c45bae0f 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -54,7 +54,6 @@ cc_library( "//tensorflow_serving/util:event_bus", "//tensorflow_serving/util:unique_ptr_with_deps", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:cc_wkt_protos", ], ) From 6a69f6e741f6cea9690fc34e4f74f3f30b08f96f Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 4 Nov 2016 10:42:26 -0800 Subject: [PATCH 0058/8554] Reduce noise in VLOG(1) logging. Change: 138215643 --- tensorflow_serving/core/aspired_versions_manager.cc | 5 +++-- tensorflow_serving/core/eager_load_policy.cc | 1 - tensorflow_serving/core/eager_unload_policy.cc | 1 - tensorflow_serving/servables/tensorflow/BUILD | 4 ++-- 4 files changed, 5 insertions(+), 6 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 5e862008693..6337d6db86f 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -334,8 +334,9 @@ AspiredVersionsManager::GetNextAction() { std::sort(actions.begin(), actions.end(), CompareActions()); const optional next_action = !actions.empty() ? actions[0] : nullopt; - VLOG(1) << "Taking action: " - << (next_action ? next_action->DebugString() : ""); + if (next_action) { + VLOG(1) << "Taking action: " << next_action->DebugString(); + } return next_action; } diff --git a/tensorflow_serving/core/eager_load_policy.cc b/tensorflow_serving/core/eager_load_policy.cc index 07649fafaab..5d7c6eb4a5b 100644 --- a/tensorflow_serving/core/eager_load_policy.cc +++ b/tensorflow_serving/core/eager_load_policy.cc @@ -38,7 +38,6 @@ optional EagerLoadPolicy::GetNextAction( version.state != LoaderHarness::State::kReady; }); if (aspired_not_serving) { - VLOG(1) << "EagerLoadPolicy requesting no-op"; return nullopt; } diff --git a/tensorflow_serving/core/eager_unload_policy.cc b/tensorflow_serving/core/eager_unload_policy.cc index b2b59cb7500..c1acd15cc08 100644 --- a/tensorflow_serving/core/eager_unload_policy.cc +++ b/tensorflow_serving/core/eager_unload_policy.cc @@ -41,7 +41,6 @@ optional EagerUnloadPolicy::GetNextAction( version.state != LoaderHarness::State::kError; }); if (not_aspired_not_finished) { - VLOG(1) << "EagerUnloadPolicy requesting no-op"; return nullopt; } diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 5bbe59ecc68..0da6b8dc5ac 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -45,12 +45,12 @@ cc_library( "//visibility:public", ], deps = [ + ":serving_session", ":session_bundle_config_proto", "//tensorflow_serving/batching:batching_session", "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/resources:resources_proto", - "//tensorflow_serving/servables/tensorflow:serving_session", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", @@ -88,13 +88,13 @@ cc_library( "//visibility:public", ], deps = [ + ":serving_session", ":session_bundle_factory", ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:simple_loader", "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", - "//tensorflow_serving/servables/tensorflow:serving_session", "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", From bb426fc604bf7f5976dbfa731a96d9993e28f790 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 4 Nov 2016 11:04:46 -0800 Subject: [PATCH 0059/8554] Merge changes from github. Change: 138218130 --- tensorflow_serving/g3doc/serving_advanced.md | 4 +- .../file_system_storage_path_source.cc | 116 ++++++++++++++---- .../file_system_storage_path_source.proto | 14 +++ .../file_system_storage_path_source_test.cc | 38 ++++++ 4 files changed, 148 insertions(+), 24 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 5173297a86d..7c1f2fa662a 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -142,8 +142,8 @@ With all these, `ServerCore` internally does the following: `AspiredVersionsManager` that manages all such `Loader` instances created by the `SessionBundleSourceAdapter`. -Whenever a new version is available, this `AspiredVersionsManager` always -unloads the old version and replaces it with the new one. If you want to +Whenever a new version is available, this `AspiredVersionsManager` loads the new +version, and under its default behavior unloads the old one. If you want to start customizing, you are encouraged to understand the components that it creates internally, and how to configure them. diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index a824fa1edc7..3147e6a9282 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -76,6 +76,86 @@ std::set GetDeletedServables( return deleted_servables; } +// Adds a new ServableData for the model version to the vector of versions to +// aspire. +void AspireVersion( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const string& version_relative_path, const int version_number, + std::vector>* versions) { + const ServableId servable_id = {servable.servable_name(), version_number}; + const string full_path = + io::JoinPath(servable.base_path(), version_relative_path); + versions->emplace_back(ServableData(servable_id, full_path)); +} + +// Converts the string version path to an integer. +// Returns false if the input is invalid. +bool ParseVersionNumber(const string& version_path, int64* version_number) { + return strings::safe_strto64(version_path.c_str(), version_number); +} + +// Update the servable data to include all the model versions found in the base +// path as aspired versions. +// The argument 'children' represents a list of base-path children from the file +// system. +// Returns true if one or more valid servable version paths are found, otherwise +// returns false. +bool AspireAllVersions( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const std::vector& children, + std::vector>* versions) { + bool at_least_one_version_found = false; + for (const string& child : children) { + // Identify all the versions, among children that can be interpreted as + // version numbers. + int64 version_number; + if (ParseVersionNumber(child, &version_number)) { + // Emit all the aspired-versions data. + AspireVersion(servable, child, version_number, versions); + at_least_one_version_found = true; + } + } + + return at_least_one_version_found; +} + +// Update the servable data to include the latest version found in the base path +// as aspired version. +// The argument 'children' represents a list of base-path children from the file +// system. +// Returns true if a valid servable path is found, otherwise returns false. +bool AspireLatestVersion( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const std::vector& children, + std::vector>* versions) { + // Identify the latest version among children that can be interpreted as + // version numbers. + int latest_version_child_index; + int64 latest_version_number; + bool at_least_one_version_found = false; + for (int i = 0; i < children.size(); ++i) { + int64 version_number; + if (!ParseVersionNumber(children[i], &version_number)) { + continue; + } + + // Check if this is the largest version number. + if (!at_least_one_version_found || latest_version_number < version_number) { + latest_version_child_index = i; + latest_version_number = version_number; + } + at_least_one_version_found = true; + } + + // Emit the latest aspired version. + if (at_least_one_version_found) { + AspireVersion(servable, children[latest_version_child_index], + latest_version_number, versions); + } + + return at_least_one_version_found; +} + // Like PollFileSystemForConfig(), but for a single servable. Status PollFileSystemForServable( const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, @@ -105,30 +185,22 @@ Status PollFileSystemForServable( children.clear(); children.insert(children.begin(), real_children.begin(), real_children.end()); - // Identify the latest version, among children that can be interpreted as - // version numbers. - int latest_version_child = -1; - int64 latest_version; - for (int i = 0; i < children.size(); ++i) { - const string& child = children[i]; - int64 child_version_num; - if (!strings::safe_strto64(child.c_str(), &child_version_num)) { - continue; - } - - if (latest_version_child < 0 || latest_version < child_version_num) { - latest_version_child = i; - latest_version = child_version_num; - } + bool at_least_one_version_found = false; + switch (servable.version_policy()) { + case FileSystemStoragePathSourceConfig::LATEST_VERSION: + at_least_one_version_found = + AspireLatestVersion(servable, children, versions); + break; + case FileSystemStoragePathSourceConfig::ALL_VERSIONS: + at_least_one_version_found = + AspireAllVersions(servable, children, versions); + break; + default: + return errors::Internal("Unhandled servable version_policy: ", + servable.version_policy()); } - // Emit the aspired-versions data. - if (latest_version_child >= 0) { - const ServableId servable_id = {servable.servable_name(), latest_version}; - const string full_path = - io::JoinPath(servable.base_path(), children[latest_version_child]); - versions->emplace_back(ServableData(servable_id, full_path)); - } else { + if (!at_least_one_version_found) { LOG(WARNING) << "No versions of servable " << servable.servable_name() << " found under base path " << servable.base_path(); } diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto index ca4d0c15cf9..5f067c45221 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto @@ -4,6 +4,16 @@ package tensorflow.serving; // Config proto for FileSystemStoragePathSource. message FileSystemStoragePathSourceConfig { + // The policy to define how many versions of the servable should be + // served at the same time. + enum VersionPolicy { + // Only serve the latest version that exists in the base path. + // This is the default behavior. + LATEST_VERSION = 0; + // Serves all the versions that exist in the base path. + ALL_VERSIONS = 1; + } + // A servable name and base path to look for versions of the servable. message ServableToMonitor { // The servable name to supply in aspired-versions callback calls. Child @@ -12,6 +22,10 @@ message FileSystemStoragePathSourceConfig { // The path to monitor, i.e. look for child paths of the form base_path/123. string base_path = 2; + + // The policy to determines the number of versions of the servable to be + // served at the same time. + VersionPolicy version_policy = 3; }; // The servables to monitor for new versions, and aspire. diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index ca06f005106..19e12f0948c 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -171,6 +171,44 @@ TEST(FileSystemStoragePathSourceTest, MultipleVersions) { .PollFileSystemAndInvokeCallback()); } +TEST(FileSystemStoragePathSourceTest, MultipleVersionsAtTheSameTime) { + const string base_path = + io::JoinPath(testing::TmpDir(), "MultipleVersionsAtTheSameTime"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + TF_ASSERT_OK(Env::Default()->CreateDir( + io::JoinPath(base_path, "non_numerical_child"))); + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, "42"))); + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, "17"))); + + auto config = test_util::CreateProto( + strings::Printf("servables: { " + " version_policy: ALL_VERSIONS " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 17}, + io::JoinPath(base_path, "17")), + ServableData({"test_servable_name", 42}, + io::JoinPath(base_path, "42"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + TEST(FileSystemStoragePathSourceTest, MultipleServables) { FileSystemStoragePathSourceConfig config; config.set_fail_if_zero_versions_at_startup(false); From 5db0869bb4de19842156e0b13a915ebcdbc46c5c Mon Sep 17 00:00:00 2001 From: Jonathan Hseu Date: Fri, 4 Nov 2016 11:53:50 -0800 Subject: [PATCH 0060/8554] Change FileExists to return tensorflow::Status. Also done separately by @llhe at github.com/tensorflow/tensorflow/pull/5370. We needed to do this change internally to fix all callers. Motivation: The existing FileExists interface doesn't allow callers to distinguish between file not found vs. filesystem errors. Semantics changes: - gfile.Exists in Python now throws an exception for filesystem errors. It continues to return true/false if it can accurately determine whether a file exists. - RecursivelyCreateDir now returns errors for filesystem errors when calling FileExists. Change: 138224013 --- .../servables/tensorflow/session_bundle_factory.cc | 2 +- .../sources/storage_path/file_system_storage_path_source.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc index a6bcb1e3223..2faf39d80d7 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc @@ -81,7 +81,7 @@ Status SessionBundleFactory::Create( Status SessionBundleFactory::EstimateResourceRequirement( const string& path, ResourceAllocation* estimate) const { const char kVariablesFilenameRegexp[] = "export(-[0-9]+-of-[0-9]+)?"; - if (!Env::Default()->FileExists(path)) { + if (!Env::Default()->FileExists(path).ok()) { return errors::NotFound("Nonexistent export path: ", path); } diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 3147e6a9282..2f04d31a837 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -164,7 +164,7 @@ Status PollFileSystemForServable( // we don't emit an empty aspired-versions list for a non-existent (or // transiently unavailable) base-path. (On some platforms, GetChildren() // returns an empty list instead of erring if the base path isn't found.) - if (!Env::Default()->FileExists(servable.base_path())) { + if (!Env::Default()->FileExists(servable.base_path()).ok()) { return errors::InvalidArgument("Could not find base path ", servable.base_path(), " for servable ", servable.servable_name()); From 6caa0abfe8f2dd9b52735a859ff436abd1b2bedd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 4 Nov 2016 15:03:27 -0800 Subject: [PATCH 0061/8554] Move fields inside ServerCore::ServerCoreConfig to ServerCore::Options. Change: 138245613 --- tensorflow_serving/model_servers/BUILD | 7 +- tensorflow_serving/model_servers/main.cc | 18 +---- .../model_servers/server_core.cc | 74 ++++++++++++------- .../model_servers/server_core.h | 69 ++++++++--------- .../test_util/mock_server_core.h | 32 ++++---- .../test_util/server_core_test_util.cc | 27 +++---- .../test_util/server_core_test_util.h | 8 +- 7 files changed, 122 insertions(+), 113 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 553c45bae0f..31d0bfe067e 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -47,13 +47,15 @@ cc_library( "//tensorflow_serving/core:source", "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", - "//tensorflow_serving/resources:resource_tracker", "//tensorflow_serving/resources:resource_values", + "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter", + "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", "//tensorflow_serving/util:event_bus", "//tensorflow_serving/util:unique_ptr_with_deps", "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:cc_wkt_protos", ], ) @@ -97,11 +99,12 @@ cc_binary( ":model_platform_types", ":server_core", "@protobuf//:cc_wkt_protos", + "//net/grpc", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "//tensorflow_serving/apis:prediction_service_proto", + "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:eager_load_policy", - "//tensorflow_serving/core:servable_state_monitor", "@grpc//:grpc++", ] + TENSORFLOW_DEPS + SUPPORTED_TENSORFLOW_OPS, ) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index d5e6fba7221..ab59e521390 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -63,7 +63,6 @@ limitations under the License. #include "tensorflow_serving/apis/prediction_service.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/core/eager_load_policy.h" -#include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" @@ -77,9 +76,7 @@ using tensorflow::serving::EventBus; using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; -using tensorflow::serving::ServableStateMonitor; using tensorflow::serving::ServerCore; -using tensorflow::serving::ServerCoreConfig; using tensorflow::serving::SessionBundleSourceAdapter; using tensorflow::serving::SessionBundleSourceAdapterConfig; using tensorflow::serving::Target; @@ -112,13 +109,6 @@ tensorflow::Status CreateSourceAdapter( return tensorflow::Status::OK(); } -tensorflow::Status CreateServableStateMonitor( - EventBus* event_bus, - std::unique_ptr* monitor) { - monitor->reset(new ServableStateMonitor(event_bus)); - return tensorflow::Status::OK(); -} - tensorflow::Status LoadCustomModelConfig( const ::google::protobuf::Any& any, EventBus* servable_event_bus, @@ -212,6 +202,8 @@ int main(int argc, char** argv) { std::cout << "unknown argument: " << argv[1] << "\n" << usage; } + // For ServerCore Options, we leave servable_state_monitor_creator unspecified + // so the default servable_state_monitor_creator will be used. ServerCore::Options options; options.model_server_config = BuildSingleModelConfig(model_name, model_base_path); @@ -230,13 +222,11 @@ int main(int argc, char** argv) { return CreateSourceAdapter(source_adapter_config, model_platform, adapter); }; - options.servable_state_monitor_creator = &CreateServableStateMonitor; options.custom_model_config_loader = &LoadCustomModelConfig; - options.server_core_config.aspired_version_policy = + options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); - options.server_core_config.file_system_poll_wait_seconds = - file_system_poll_wait_seconds; + options.file_system_poll_wait_seconds = file_system_poll_wait_seconds; std::unique_ptr core; TF_CHECK_OK(ServerCore::Create(std::move(options), &core)); diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index dc7d88fe154..63134622185 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -23,6 +23,8 @@ limitations under the License. #include "tensorflow_serving/core/load_servables_fast.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/resources/resource_values.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" @@ -82,14 +84,37 @@ Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, Status ServerCore::Create(Options options, std::unique_ptr* server_core) { + if (options.source_adapter_creator == nullptr) { + options.source_adapter_creator = []( + const string& model_platform, + std::unique_ptr* adapter) { + SessionBundleSourceAdapterConfig source_adapter_config; + if (model_platform != kTensorFlowModelPlatform) { + return errors::InvalidArgument( + "ModelServer supports only TensorFlow model."); + } + std::unique_ptr typed_adapter; + TF_RETURN_IF_ERROR(SessionBundleSourceAdapter::Create( + source_adapter_config, &typed_adapter)); + *adapter = std::move(typed_adapter); + return Status::OK(); + }; + } + + if (options.servable_state_monitor_creator == nullptr) { + options.servable_state_monitor_creator = []( + EventBus* event_bus, + std::unique_ptr* monitor) { + monitor->reset(new ServableStateMonitor(event_bus)); + return Status::OK(); + }; + } + // We need to move the aspired_version_policy first because we will move the // server_core_config (which contains aspired_version_policy) below. std::unique_ptr aspired_version_policy = - std::move(options.server_core_config.aspired_version_policy); - server_core->reset(new ServerCore(options.source_adapter_creator, - options.servable_state_monitor_creator, - options.custom_model_config_loader, - std::move(options.server_core_config))); + std::move(options.aspired_version_policy); + server_core->reset(new ServerCore(std::move(options))); TF_RETURN_IF_ERROR( (*server_core)->Initialize(std::move(aspired_version_policy))); return (*server_core)->ReloadConfig(options.model_server_config); @@ -99,21 +124,14 @@ Status ServerCore::Create(Options options, // Server Setup and Initialization. // ************************************************************************ -ServerCore::ServerCore( - const SourceAdapterCreator& source_adapter_creator, - const ServableStateMonitorCreator& servable_state_monitor_creator, - const CustomModelConfigLoader& custom_model_config_loader, - ServerCoreConfig server_core_config) - : source_adapter_creator_(source_adapter_creator), - servable_state_monitor_creator_(servable_state_monitor_creator), - custom_model_config_loader_(custom_model_config_loader), - server_core_config_(std::move(server_core_config)), +ServerCore::ServerCore(Options options) + : options_(std::move(options)), servable_event_bus_(EventBus::CreateEventBus()) {} Status ServerCore::Initialize(std::unique_ptr policy) { std::unique_ptr servable_state_monitor; - TF_RETURN_IF_ERROR(servable_state_monitor_creator_(servable_event_bus_.get(), - &servable_state_monitor)); + TF_RETURN_IF_ERROR(options_.servable_state_monitor_creator( + servable_event_bus_.get(), &servable_state_monitor)); servable_state_monitor_ = std::move(servable_state_monitor); std::unique_ptr aspired_versions_manager; @@ -180,7 +198,7 @@ Status ServerCore::AddModelsViaModelConfigList() { } TF_RETURN_IF_ERROR(ConnectSourceWithFastInitialLoad( manager_.get(), source_adapter.get(), servable_state_monitor_.get(), - static_servables, server_core_config_.num_initial_load_unload_threads)); + static_servables, options_.num_initial_load_unload_threads)); manager_.AddDependency(std::move(source_adapter)); } else { TF_RETURN_IF_ERROR(ReloadFileSystemStoragePathSourceConfig(source_config)); @@ -190,8 +208,13 @@ Status ServerCore::AddModelsViaModelConfigList() { } Status ServerCore::AddModelsViaCustomModelConfig() { - return custom_model_config_loader_(config_.custom_model_config(), - servable_event_bus_.get(), &manager_); + if (options_.custom_model_config_loader == nullptr) { + return errors::InvalidArgument( + "Missing custom_model_config_loader in ServerCore Options"); + } + + return options_.custom_model_config_loader( + config_.custom_model_config(), servable_event_bus_.get(), &manager_); } Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { @@ -240,7 +263,7 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { Status ServerCore::CreateSourceAdapter( const string& model_platform, std::unique_ptr* adapter) { - TF_RETURN_IF_ERROR(source_adapter_creator_(model_platform, adapter)); + TF_RETURN_IF_ERROR(options_.source_adapter_creator(model_platform, adapter)); return Status::OK(); } @@ -248,7 +271,7 @@ FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( const ModelServerConfig& config) const { FileSystemStoragePathSourceConfig source_config; source_config.set_file_system_poll_wait_seconds( - server_core_config_.file_system_poll_wait_seconds); + options_.file_system_poll_wait_seconds); for (const auto& model : config.model_config_list().config()) { LOG(INFO) << " (Re-)adding model: " << model.name(); FileSystemStoragePathSourceConfig::ServableToMonitor* servable = @@ -286,10 +309,8 @@ Status ServerCore::CreateAspiredVersionsManager( manager_options.resource_tracker = std::move(resource_tracker); manager_options.servable_event_bus = servable_event_bus_.get(); manager_options.aspired_version_policy = std::move(aspired_version_policy); - manager_options.num_load_unload_threads = - server_core_config_.num_load_unload_threads; - manager_options.max_num_load_retries = - server_core_config_.max_num_load_retries; + manager_options.num_load_unload_threads = options_.num_load_unload_threads; + manager_options.max_num_load_retries = options_.max_num_load_retries; return AspiredVersionsManager::Create(std::move(manager_options), manager); } @@ -304,8 +325,7 @@ Status ServerCore::CreateResourceTracker( total_resources.add_resource_quantities(); main_memory_resource->mutable_resource()->set_device(device_types::kMain); main_memory_resource->mutable_resource()->set_kind(resource_kinds::kRamBytes); - main_memory_resource->set_quantity( - server_core_config_.total_model_memory_limit_bytes); + main_memory_resource->set_quantity(options_.total_model_memory_limit_bytes); return ResourceTracker::Create(total_resources, std::move(resource_util), resource_tracker); } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index efb2ee2ac45..c6b9160a404 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -29,6 +29,7 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_SERVER_CORE_H_ #define TENSORFLOW_SERVING_MODEL_SERVERS_SERVER_CORE_H_ +#include #include #include #include @@ -52,31 +53,6 @@ limitations under the License. namespace tensorflow { namespace serving { -// ServerCore tuning parameters. -struct ServerCoreConfig { - // Total model size limit, in terms of main memory, in bytes. - uint64 total_model_memory_limit_bytes = ULLONG_MAX; - // Time interval between file-system polls, in seconds. - int32 file_system_poll_wait_seconds = 30; - - // The AspiredVersionPolicy to use for the manager. Must be non-null. - std::unique_ptr aspired_version_policy; - - // The number of threads used to load and unload models. If set to 0, then - // no thread pool is used and the loads/unloads are performed serially in - // the manager thread. - int32 num_load_unload_threads = 0; - - // The number of load/unload threads used to load the initial set of models at - // server startup. This is set high to load up the initial set of models fast, - // after this the server uses num_load_unload_threads. - int32 num_initial_load_unload_threads = 4.0 * port::NumSchedulableCPUs(); - - // Maximum number of times we retry loading a model, after the first failure, - // before we give up" - int32 max_num_load_retries = 5; -}; - namespace test_util { class ServerCoreTestAccess; } // namespace test_util @@ -107,15 +83,37 @@ class ServerCore { // ModelServer configuration. ModelServerConfig model_server_config; - // ServerCore tuning parameters. - // TODO(b/32336469): move the fields in ServerCoreConfig into this struct. - ServerCoreConfig server_core_config; + // The AspiredVersionPolicy to use for the manager. Must be non-null. + std::unique_ptr aspired_version_policy; + + // The number of threads used to load and unload models. If set to 0, then + // no thread pool is used and the loads/unloads are performed serially in + // the manager thread. + int32 num_load_unload_threads = 0; + + // The number of load/unload threads used to load the initial set of models + // at server startup. This is set high to load up the initial set of models + // fast, after this the server uses num_load_unload_threads. + int32 num_initial_load_unload_threads = 4.0 * port::NumSchedulableCPUs(); + + // Total model size limit, in terms of main memory, in bytes. + uint64 total_model_memory_limit_bytes = std::numeric_limits::max(); + + // Maximum number of times we retry loading a model, after the first + // failure, before we give up. + int32 max_num_load_retries = 5; + + // Time interval between file-system polls, in seconds. + int32 file_system_poll_wait_seconds = 30; // A function for creating ModelServerSourceAdapter based on the // 'platform_type'. + // If not specified, a default creator that creates + // SessionBundleSourceAdapter for TensorFlow will be used. SourceAdapterCreator source_adapter_creator; - // A function for creating ServableStateMonitor. + // A function for creating ServableStateMonitor. If not specified, a default + // creator that creates ServableStateMonitor will be used. ServableStateMonitorCreator servable_state_monitor_creator; // A function for instantiating and connecting custom sources and source @@ -165,10 +163,7 @@ class ServerCore { } protected: - ServerCore(const SourceAdapterCreator& source_adapter_creator, - const ServableStateMonitorCreator& servable_state_monitor_creator, - const CustomModelConfigLoader& custom_model_config_loader, - ServerCoreConfig server_core_config); + explicit ServerCore(Options options); private: friend class test_util::ServerCoreTestAccess; @@ -239,12 +234,8 @@ class ServerCore { // Lists available servable ids from the wrapped aspired-versions-manager. std::vector ListAvailableServableIds() const; - SourceAdapterCreator source_adapter_creator_; - ServableStateMonitorCreator servable_state_monitor_creator_; - CustomModelConfigLoader custom_model_config_loader_; - - // The config passed to the ctor, minus the AspiredVersionPolicy. - ServerCoreConfig server_core_config_; + // The options passed to the ctor, minus the AspiredVersionPolicy. + Options options_; std::shared_ptr> servable_event_bus_; std::shared_ptr servable_state_monitor_; diff --git a/tensorflow_serving/model_servers/test_util/mock_server_core.h b/tensorflow_serving/model_servers/test_util/mock_server_core.h index a1251c8320b..1c8b9deae18 100644 --- a/tensorflow_serving/model_servers/test_util/mock_server_core.h +++ b/tensorflow_serving/model_servers/test_util/mock_server_core.h @@ -32,20 +32,26 @@ namespace test_util { class MockServerCore : public ServerCore { public: + static Options GetOptions( + const SourceAdapterCreator& source_adapter_creator) { + Options options; + options.source_adapter_creator = source_adapter_creator; + options.servable_state_monitor_creator = []( + EventBus* event_bus, + std::unique_ptr* monitor) -> Status { + monitor->reset(new ServableStateMonitor(event_bus)); + return Status::OK(); + }; + options.custom_model_config_loader = []( + const ::google::protobuf::Any& any, EventBus* event_bus, + UniquePtrWithDeps* manager) -> Status { + return Status::OK(); + }; + return options; + } + explicit MockServerCore(const SourceAdapterCreator& source_adapter_creator) - : ServerCore( - source_adapter_creator, // ServerCore::SourceAdapterCreator - [](EventBus* event_bus, - std::unique_ptr* monitor) -> Status { - monitor->reset(new ServableStateMonitor(event_bus)); - return Status::OK(); - }, // ServerCore::ServableStateMonitorCreator - [](const ::google::protobuf::Any& any, - EventBus* event_bus, - UniquePtrWithDeps* manager) -> Status { - return Status::OK(); - }, // ServerCore::CustomModelConfigLoader - ServerCoreConfig()) {} + : ServerCore(GetOptions(source_adapter_creator)) {} MOCK_CONST_METHOD0(servable_state_monitor, ServableStateMonitor*()); MOCK_METHOD1(ReloadConfig, Status(const ModelServerConfig&)); diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 12b237ac3b3..ae948bfc0f4 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -38,14 +38,6 @@ ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { return config; } -ServerCoreConfig ServerCoreTest::GetTestServerCoreConfig() { - ServerCoreConfig config; - config.file_system_poll_wait_seconds = 0; - config.aspired_version_policy = - std::unique_ptr(new EagerLoadPolicy); - return config; -} - Status ServerCoreTest::CreateServerCore( const ModelServerConfig& config, std::unique_ptr* server_core) { return CreateServerCore( @@ -56,21 +48,26 @@ Status ServerCoreTest::CreateServerCore( const ModelServerConfig& config, const ServerCore::SourceAdapterCreator& source_adapter_creator, std::unique_ptr* server_core) { + // For ServerCore Options, we leave servable_state_monitor_creator unspecified + // so the default servable_state_monitor_creator will be used. ServerCore::Options options; options.model_server_config = config; options.source_adapter_creator = source_adapter_creator; - options.servable_state_monitor_creator = []( - EventBus* event_bus, - std::unique_ptr* monitor) -> Status { - monitor->reset(new ServableStateMonitor(event_bus)); - return Status::OK(); - }; options.custom_model_config_loader = []( const ::google::protobuf::Any& any, EventBus* event_bus, UniquePtrWithDeps* manager) -> Status { return Status::OK(); }; - options.server_core_config = GetTestServerCoreConfig(); + return CreateServerCore(std::move(options), server_core); +} + +Status ServerCoreTest::CreateServerCore( + ServerCore::Options options, std::unique_ptr* server_core) { + options.file_system_poll_wait_seconds = 0; + if (options.aspired_version_policy == nullptr) { + options.aspired_version_policy = + std::unique_ptr(new EagerLoadPolicy); + } return ServerCore::Create(std::move(options), server_core); } diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index 29ca8851898..15c79caa8ca 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -45,9 +45,6 @@ class ServerCoreTest : public ::testing::Test { // Returns ModelServerConfig that contains test model. ModelServerConfig GetTestModelServerConfig(); - // Returns ServerCoreConfig that uses continuous polling, to speed up testing. - ServerCoreConfig GetTestServerCoreConfig(); - // Create a ServerCore object configured to use FakeLoaderSourceAdapter. Status CreateServerCore(const ModelServerConfig& config, std::unique_ptr* server_core); @@ -57,6 +54,11 @@ class ServerCoreTest : public ::testing::Test { const ModelServerConfig& config, const ServerCore::SourceAdapterCreator& source_adapter_creator, std::unique_ptr* server_core); + + // Create a ServerCore object with the supplied options. The ServerCore uses + // continuous polling to speed up testing. + Status CreateServerCore(ServerCore::Options options, + std::unique_ptr* server_core); }; } // namespace test_util From 2bfccd3a6823a16fe48cc78dc7b9d43451bf9e09 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 4 Nov 2016 16:30:50 -0800 Subject: [PATCH 0062/8554] Adds Batch::RemoveTask() method, for transfering ownership of tasks out of the batch. Change: 138253896 --- tensorflow_serving/batching/batch_scheduler.h | 17 +++++++++++++++++ .../batching/batch_scheduler_test.cc | 5 +++++ 2 files changed, 22 insertions(+) diff --git a/tensorflow_serving/batching/batch_scheduler.h b/tensorflow_serving/batching/batch_scheduler.h index 0780c5bc4d6..76f8ad499fa 100644 --- a/tensorflow_serving/batching/batch_scheduler.h +++ b/tensorflow_serving/batching/batch_scheduler.h @@ -85,6 +85,10 @@ class Batch { // Dies if the batch is closed. void AddTask(std::unique_ptr task); + // Removes the most recently added task. Returns nullptr if the batch is + // empty. + std::unique_ptr RemoveTask(); + // Returns the number of tasks in the batch. int num_tasks() const; @@ -194,6 +198,19 @@ void Batch::AddTask(std::unique_ptr task) { } } +template +std::unique_ptr Batch::RemoveTask() { + { + mutex_lock l(mu_); + if (tasks_.empty()) { + return nullptr; + } + std::unique_ptr task = std::move(tasks_.back()); + tasks_.pop_back(); + return task; + } +} + template int Batch::num_tasks() const { { diff --git a/tensorflow_serving/batching/batch_scheduler_test.cc b/tensorflow_serving/batching/batch_scheduler_test.cc index b5188a8c6b5..6efe313653b 100644 --- a/tensorflow_serving/batching/batch_scheduler_test.cc +++ b/tensorflow_serving/batching/batch_scheduler_test.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow_serving/batching/batch_scheduler.h" +#include #include #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" @@ -72,6 +73,10 @@ TEST(BatchTest, Basic) { EXPECT_EQ(task0->size() + task1->size(), batch.size()); EXPECT_EQ(task0->size(), batch.task(0).size()); EXPECT_EQ(task1->size(), batch.task(1).size()); + + EXPECT_EQ(7, batch.RemoveTask()->size()); + EXPECT_EQ(3, batch.RemoveTask()->size()); + EXPECT_TRUE(batch.empty()); } TEST(BatchTest, WaitUntilClosed) { From 35552975fcec4a6bb2159cbb4afdb4e5f5dd0dd7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 8 Nov 2016 11:35:18 -0800 Subject: [PATCH 0063/8554] Export Manager interface in ServerCore. Change: 138543185 --- .../core/aspired_versions_manager.h | 1 + tensorflow_serving/model_servers/BUILD | 2 ++ .../model_servers/server_core.cc | 8 ------- .../model_servers/server_core.h | 23 ++++++++++++------- .../model_servers/server_core_test.cc | 19 ++++++++++----- .../test_util/server_core_test_util.cc | 4 ---- .../test_util/server_core_test_util.h | 13 ----------- 7 files changed, 31 insertions(+), 39 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 701989c0b9f..19d10949e6f 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -178,6 +178,7 @@ class AspiredVersionsManager : public Manager, private: friend class internal::AspiredVersionsManagerTargetImpl; friend class test_util::AspiredVersionsManagerTestAccess; + friend class ServerCore; friend Status internal::ConnectSourceWithFastInitialLoad( AspiredVersionsManager* manager, Source>* source, const std::function& wait_until_loaded_fn, uint32 num_threads); diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 31d0bfe067e..40caf62cb98 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -69,6 +69,8 @@ cc_test( ], deps = [ ":server_core", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/core:servable_state", "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:test_main", diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 63134622185..9c287686d65 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -348,13 +348,5 @@ Status ServerCore::ServableRequestFromModelSpec( return Status::OK(); } -// ************************************************************************ -// Test Access. -// ************************************************************************ - -std::vector ServerCore::ListAvailableServableIds() const { - return manager_->ListAvailableServableIds(); -} - } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index c6b9160a404..9b2f0b6ed69 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -57,7 +57,7 @@ namespace test_util { class ServerCoreTestAccess; } // namespace test_util -class ServerCore { +class ServerCore : public Manager { public: using ModelServerSourceAdapter = SourceAdapter>; @@ -131,6 +131,10 @@ class ServerCore { // Returns an error status if any such model fails to load. static Status Create(Options options, std::unique_ptr* core); + std::vector ListAvailableServableIds() const override { + return manager_->ListAvailableServableIds(); + } + // Updates the server core with all the models and sources per the // ModelServerConfig. Like Create(), waits for all statically configured // servables to be made available before returning, and returns an error if @@ -158,8 +162,7 @@ class ServerCore { ServableHandle* const handle) { ServableRequest servable_request; ServableRequestFromModelSpec(model_spec, &servable_request); - TF_RETURN_IF_ERROR(manager_->GetServableHandle(servable_request, handle)); - return Status::OK(); + return manager_->GetServableHandle(servable_request, handle); } protected: @@ -227,12 +230,16 @@ class ServerCore { Status ServableRequestFromModelSpec(const ModelSpec& model_spec, ServableRequest* servable_request) const; - // ************************************************************************ - // Test Access. - // ************************************************************************ + Status GetUntypedServableHandle( + const ServableRequest& request, + std::unique_ptr* untyped_handle) override { + return manager_->GetUntypedServableHandle(request, untyped_handle); + } - // Lists available servable ids from the wrapped aspired-versions-manager. - std::vector ListAvailableServableIds() const; + std::map> + GetAvailableUntypedServableHandles() const override { + return manager_->GetAvailableUntypedServableHandles(); + } // The options passed to the ctor, minus the AspiredVersionPolicy. Options options_; diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 3a4e54a597f..4c1b62f3d2a 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -16,6 +16,8 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/model.proto.h" +#include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" @@ -32,12 +34,19 @@ TEST_F(ServerCoreTest, CreateWaitsTillModelsAvailable) { TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfig(), &server_core)); const std::vector available_servables = - test_util::ServerCoreTestAccess(server_core.get()) - .ListAvailableServableIds(); + server_core->ListAvailableServableIds(); ASSERT_EQ(available_servables.size(), 1); const ServableId expected_id = {test_util::kTestModelName, test_util::kTestModelVersion}; EXPECT_EQ(available_servables.at(0), expected_id); + + ModelSpec model_spec; + model_spec.set_name(test_util::kTestModelName); + model_spec.mutable_version()->set_value(test_util::kTestModelVersion); + ServableHandle servable_handle; + TF_ASSERT_OK( + server_core->GetServableHandle(model_spec, &servable_handle)); + EXPECT_EQ(servable_handle.id(), expected_id); } TEST_F(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { @@ -49,8 +58,7 @@ TEST_F(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { TF_ASSERT_OK(server_core->ReloadConfig(GetTestModelServerConfig())); const std::vector available_servables = - test_util::ServerCoreTestAccess(server_core.get()) - .ListAvailableServableIds(); + server_core->ListAvailableServableIds(); ASSERT_EQ(available_servables.size(), 1); const ServableId expected_id = {test_util::kTestModelName, test_util::kTestModelVersion}; @@ -120,8 +128,7 @@ TEST_F(ServerCoreTest, DeprecatedModelTypeConfig) { TF_ASSERT_OK(CreateServerCore(config, &server_core)); const std::vector available_servables = - test_util::ServerCoreTestAccess(server_core.get()) - .ListAvailableServableIds(); + server_core->ListAvailableServableIds(); ASSERT_EQ(available_servables.size(), 1); const ServableId expected_id = {test_util::kTestModelName, test_util::kTestModelVersion}; diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index ae948bfc0f4..fcd848d0fbe 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -24,10 +24,6 @@ namespace tensorflow { namespace serving { namespace test_util { -std::vector ServerCoreTestAccess::ListAvailableServableIds() const { - return core_->ListAvailableServableIds(); -} - ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { ModelServerConfig config; auto model = config.mutable_model_config_list()->add_config(); diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index 15c79caa8ca..0f5d5d4e624 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -24,19 +24,6 @@ namespace tensorflow { namespace serving { namespace test_util { -// A test utility that provides access to private ServerCore members. -class ServerCoreTestAccess { - public: - explicit ServerCoreTestAccess(ServerCore* core) : core_(core) {} - - // Returns the list of available servable-ids from the manager in server - // core. - std::vector ListAvailableServableIds() const; - - private: - ServerCore* const core_; -}; - constexpr char kTestModelName[] = "test_model"; constexpr int kTestModelVersion = 123; From 22e7447417594d37b590cc87698da9ea48be0592 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 10 Nov 2016 09:51:26 -0800 Subject: [PATCH 0064/8554] Updates SessionBundle memory estimation. - Takes into account all the file-sizes in the export-directory, to account for both variables, assets and the graph too. Change: 138771327 --- .../tensorflow/session_bundle_factory.cc | 53 ++++++++++++++----- .../tensorflow/session_bundle_factory.h | 4 +- .../tensorflow/session_bundle_factory_test.cc | 4 +- 3 files changed, 44 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc index 2faf39d80d7..f4e267b6b93 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc @@ -37,6 +37,37 @@ SessionOptions GetSessionOptions(const SessionBundleConfig& config) { return options; } +// Returns all the descendants, both directories and files, recursively under +// 'dirname'. The paths returned are all prefixed with 'dirname'. +Status GetAllDescendants(const string& dirname, + std::vector* const descendants) { + Env* const env = Env::Default(); + descendants->clear(); + // Make sure that dirname exists; + TF_RETURN_IF_ERROR(env->FileExists(dirname)); + std::deque dir_q; // Queue for the BFS + std::vector dir_list; // List of all dirs discovered + dir_q.push_back(dirname); + Status ret; // Status to be returned. + // Do a BFS on the directory to discover all immediate children. + while (!dir_q.empty()) { + string dir = dir_q.front(); + dir_q.pop_front(); + std::vector children; + // GetChildren might fail if we don't have appropriate permissions. + TF_RETURN_IF_ERROR(env->GetChildren(dir, &children)); + for (const string& child : children) { + const string child_path = io::JoinPath(dir, child); + descendants->push_back(child_path); + // If the child is a directory add it to the queue. + if (env->IsDirectory(child_path).ok()) { + dir_q.push_back(child_path); + } + } + } + return Status::OK(); +} + } // namespace constexpr double SessionBundleFactory::kResourceEstimateRAMMultiplier; @@ -80,25 +111,21 @@ Status SessionBundleFactory::Create( Status SessionBundleFactory::EstimateResourceRequirement( const string& path, ResourceAllocation* estimate) const { - const char kVariablesFilenameRegexp[] = "export(-[0-9]+-of-[0-9]+)?"; if (!Env::Default()->FileExists(path).ok()) { return errors::NotFound("Nonexistent export path: ", path); } - - uint64 total_variable_file_size = 0; - std::vector files; - TF_RETURN_IF_ERROR(Env::Default()->GetChildren(path, &files)); - for (const string& file : files) { - if (!RE2::FullMatch(file, kVariablesFilenameRegexp)) { - continue; + std::vector descendants; + TF_RETURN_IF_ERROR(GetAllDescendants(path, &descendants)); + uint64 total_file_size = 0; + for (const string& descendant : descendants) { + if (!(Env::Default()->IsDirectory(descendant).ok())) { + uint64 file_size; + TF_RETURN_IF_ERROR(Env::Default()->GetFileSize(descendant, &file_size)); + total_file_size += file_size; } - const string file_path = io::JoinPath(path, file); - uint64 file_size; - TF_RETURN_IF_ERROR(Env::Default()->GetFileSize(file_path, &file_size)); - total_variable_file_size += file_size; } const uint64 ram_requirement = - total_variable_file_size * kResourceEstimateRAMMultiplier + + total_file_size * kResourceEstimateRAMMultiplier + kResourceEstimateRAMPadBytes; ResourceAllocation::Entry* ram_entry = estimate->add_resource_quantities(); diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h index b976a652f7c..e616d73d995 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h @@ -46,7 +46,7 @@ class SessionBundleFactory { public: // Constants used in the resource estimation heuristic. See the documentation // on EstimateResourceRequirements(). - static constexpr double kResourceEstimateRAMMultiplier = 1.0; + static constexpr double kResourceEstimateRAMMultiplier = 1.2; static constexpr int kResourceEstimateRAMPadBytes = 0; static Status Create(const SessionBundleConfig& config, @@ -60,7 +60,7 @@ class SessionBundleFactory { // export path. // // Uses the following crude heuristic, for now: estimated main-memory RAM = - // (combined size of variable file(s)) * kResourceEstimateRAMMultiplier + + // (combined size of all exported file(s)) * kResourceEstimateRAMMultiplier + // kResourceEstimateRAMPadBytes. // TODO(b/27694447): Improve the heuristic. At a minimum, account for GPU RAM. Status EstimateResourceRequirement(const string& path, diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index d2119ab4ac2..6f7e087cee8 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -130,9 +130,9 @@ TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithBadExport) { } TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { - const uint64 kVariableFileSize = 169; + const double kTotalFileSize = 13392.5; const uint64 expected_ram_requirement = - kVariableFileSize * SessionBundleFactory::kResourceEstimateRAMMultiplier + + kTotalFileSize * SessionBundleFactory::kResourceEstimateRAMMultiplier + SessionBundleFactory::kResourceEstimateRAMPadBytes; ResourceAllocation want; ResourceAllocation::Entry* ram_entry = want.add_resource_quantities(); From 05ea7eb2bd45e95c70c5d44038629642718678a5 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 10 Nov 2016 14:47:27 -0800 Subject: [PATCH 0065/8554] Public no-op. Change: 138808448 --- tensorflow_serving/model_servers/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 40caf62cb98..6544a33ba51 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -101,7 +101,6 @@ cc_binary( ":model_platform_types", ":server_core", "@protobuf//:cc_wkt_protos", - "//net/grpc", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "//tensorflow_serving/apis:prediction_service_proto", From 11d37a9689f3ca9337aaae68fbeee7f4f0bcb156 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 10 Nov 2016 14:59:24 -0800 Subject: [PATCH 0066/8554] LogCollector API. - An abstract interface and registration system for collecting logs. Change: 138809996 --- tensorflow_serving/core/BUILD | 21 +++++ tensorflow_serving/core/log_collector.cc | 94 +++++++++++++++++++ tensorflow_serving/core/log_collector.h | 64 +++++++++++++ tensorflow_serving/core/log_collector_test.cc | 85 +++++++++++++++++ 4 files changed, 264 insertions(+) create mode 100644 tensorflow_serving/core/log_collector.cc create mode 100644 tensorflow_serving/core/log_collector.h create mode 100644 tensorflow_serving/core/log_collector_test.cc diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index de1f2562746..9def19c7d11 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -654,3 +654,24 @@ cc_library( "@org_tensorflow//tensorflow/core:lib", ], ) + +cc_library( + name = "log_collector", + srcs = ["log_collector.cc"], + hdrs = ["log_collector.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "log_collector_test", + srcs = ["log_collector_test.cc"], + deps = [ + ":log_collector", + "//tensorflow_serving/core/test_util:test_main", + ], +) diff --git a/tensorflow_serving/core/log_collector.cc b/tensorflow_serving/core/log_collector.cc new file mode 100644 index 00000000000..a391d0f283a --- /dev/null +++ b/tensorflow_serving/core/log_collector.cc @@ -0,0 +1,94 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/log_collector.h" + +#include + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/strings/scanner.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/thread_annotations.h" + +namespace tensorflow { +namespace serving { +namespace { + +bool ParseLognamePrefix(StringPiece remaining, StringPiece* type) { + return strings::Scanner(remaining) + .OneLiteral("/") + .RestartCapture() + .ScanUntil('/') + .GetResult(&remaining, type); +} + +// This class is thread-safe. +class Registry { + public: + Status Register(const string& type, const LogCollector::Factory& factory) + LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + const auto found_it = factory_map_.find(type); + if (found_it != factory_map_.end()) { + return errors::AlreadyExists("Type ", type, " already registered."); + } + factory_map_.insert({type, factory}); + return Status::OK(); + } + + const LogCollector::Factory* Lookup(const string& type) const + LOCKS_EXCLUDED(mu_) { + mutex_lock l(mu_); + const auto found_it = factory_map_.find(type); + if (found_it == factory_map_.end()) { + return nullptr; + } + return &(found_it->second); + } + + private: + mutable mutex mu_; + std::unordered_map factory_map_ + GUARDED_BY(mu_); +}; + +auto* registry_ = new Registry(); + +} // namespace + +Status LogCollector::RegisterFactory(const string& type, + const Factory& factory) { + return registry_->Register(type, factory); +} + +Status LogCollector::Create( + const string& logname_prefix, const uint32 id, + std::unique_ptr* const log_collector) { + StringPiece remaining(logname_prefix); + StringPiece type; + if (!ParseLognamePrefix(remaining, &type)) { + return errors::InvalidArgument("Invalid logname_prefix: ", logname_prefix); + } + + auto* factory = registry_->Lookup(type.ToString()); + if (factory == nullptr) { + return errors::NotFound("Cannot find LogCollector::Factory for type: ", + type); + } + return (*factory)(remaining.ToString(), id, log_collector); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/log_collector.h b/tensorflow_serving/core/log_collector.h new file mode 100644 index 00000000000..e69502bf700 --- /dev/null +++ b/tensorflow_serving/core/log_collector.h @@ -0,0 +1,64 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ +#define TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ + +#include +#include +#include + +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace serving { + +// LogCollector defines an abstract interface to use for collecting logs. +// +// Each LogCollector implementation is registered along with a 'type', and if a +// LogCollector is created using this API, we create the LogCollector +// corresponding to the 'type' specified. +class LogCollector { + public: + virtual ~LogCollector() = default; + + // Creates a log-collector for a given 'logname_prefix' and 'id'. The + // 'logname_prefix' should be of the form '/type/', so + // that the factory registered for the type can then be used to create the + // log-collector. The 'id' argument helps in disambiguating logs from + // replicated servers (processes), so it could be a combination of + // task-id and replica-id or process-id and timestamp, etc. + static Status Create(const string& logname_prefix, const uint32 id, + std::unique_ptr* log_collector); + + using Factory = std::function; + // Registers a factory for creating log-collectors for a particular 'type'. + // Returns an error status if a factory is already registered for the + // particular 'type'. + static Status RegisterFactory(const string& type, const Factory& factory); + + // Collects the log as a protocol buffer. + virtual Status CollectMessage(const google::protobuf::Message& message) = 0; + + // Flushes buffered data so that the data can survive an application crash + // (but not an OS crash). + virtual Status Flush() = 0; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ diff --git a/tensorflow_serving/core/log_collector_test.cc b/tensorflow_serving/core/log_collector_test.cc new file mode 100644 index 00000000000..adf11f921a1 --- /dev/null +++ b/tensorflow_serving/core/log_collector_test.cc @@ -0,0 +1,85 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/log_collector.h" + +#include +#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +class FakeLogCollector : public LogCollector { + public: + Status CollectMessage(const google::protobuf::Message& message) { return Status::OK(); } + Status Flush() override { return Status::OK(); } +}; + +TEST(LogCollectorTest, NotRegistered) { + std::unique_ptr log_collector; + const auto status = + LogCollector::Create("/notregistered/logname_prefix", 0, &log_collector); + EXPECT_EQ(status.code(), error::NOT_FOUND); +} + +TEST(LogCollectorTest, Registration) { + TF_ASSERT_OK(LogCollector::RegisterFactory( + "registered", [](const string& logname_prefix, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return Status::OK(); + })); + std::unique_ptr log_collector; + TF_ASSERT_OK( + LogCollector::Create("/registered/logname_prefix", 0, &log_collector)); +} + +TEST(LogCollectorTest, DuplicateRegistration) { + TF_ASSERT_OK(LogCollector::RegisterFactory( + "duplicate", [](const string& logname_prefix, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return Status::OK(); + })); + const auto status = LogCollector::RegisterFactory( + "duplicate", [](const string& logname_prefix, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return Status::OK(); + }); + EXPECT_EQ(status.code(), error::ALREADY_EXISTS); +} + +TEST(LogCollectorTest, Creation) { + TF_ASSERT_OK(LogCollector::RegisterFactory( + "creation", [](const string& logname_prefix, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return Status::OK(); + })); + std::unique_ptr log_collector0; + TF_ASSERT_OK( + LogCollector::Create("/creation/logname_prefix", 0, &log_collector0)); + std::unique_ptr log_collector1; + TF_ASSERT_OK( + LogCollector::Create("/creation/logname_prefix", 0, &log_collector1)); + EXPECT_NE(log_collector0.get(), log_collector1.get()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From 7f8e56038413371fbc157da2b3086baf9e75463d Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Thu, 10 Nov 2016 16:36:33 -0800 Subject: [PATCH 0067/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index d9a89a5cf63..20c3d37ecc9 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit d9a89a5cf63b2ecedf68d3eefdc24be3f519e503 +Subproject commit 20c3d37ecc9bef0e106002b9d01914efd548e66b diff --git a/tf_models b/tf_models index 0dea688a68a..e5cae330302 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 0dea688a68a7179e62cb2e68390c24b2ef7b0ef0 +Subproject commit e5cae33030244bcdff78df3139f36f0af07f4d6a From 92165559f45cd04b7b5feb24d9ee261885aef0ea Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 11 Nov 2016 09:07:24 -0800 Subject: [PATCH 0068/8554] Add VLOG(1) when encountering tensorflow errors in model server core code. Change: 138882466 --- tensorflow_serving/model_servers/main.cc | 15 +++-- .../model_servers/server_core.cc | 58 +++++++++++++++---- .../model_servers/server_core.h | 8 ++- 3 files changed, 64 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index ab59e521390..e8c82949d97 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -103,8 +103,12 @@ tensorflow::Status CreateSourceAdapter( CHECK(model_platform == kTensorFlowModelPlatform) // Crash ok << "ModelServer supports only TensorFlow model."; std::unique_ptr typed_adapter; - TF_RETURN_IF_ERROR( - SessionBundleSourceAdapter::Create(config, &typed_adapter)); + const ::tensorflow::Status status = + SessionBundleSourceAdapter::Create(config, &typed_adapter); + if (!status.ok()) { + VLOG(1) << "Error creating source adapter: " << status; + return status; + } *adapter = std::move(typed_adapter); return tensorflow::Status::OK(); } @@ -135,7 +139,6 @@ grpc::Status ToGRPCStatus(const tensorflow::Status& status) { const int kErrorMessageLimit = 1024; string error_message; if (status.error_message().length() > kErrorMessageLimit) { - LOG(ERROR) << "Truncating error: " << status.error_message(); error_message = status.error_message().substr(0, kErrorMessageLimit) + "...TRUNCATED"; } else { @@ -152,8 +155,12 @@ class PredictionServiceImpl final : public PredictionService::Service { grpc::Status Predict(ServerContext* context, const PredictRequest* request, PredictResponse* response) override { - return ToGRPCStatus( + const grpc::Status status = ToGRPCStatus( TensorflowPredictImpl::Predict(core_.get(), *request, response)); + if (!status.ok()) { + VLOG(1) << "Predict failed: " << status; + } + return status; } private: diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 9c287686d65..7540d333e26 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -130,8 +130,13 @@ ServerCore::ServerCore(Options options) Status ServerCore::Initialize(std::unique_ptr policy) { std::unique_ptr servable_state_monitor; - TF_RETURN_IF_ERROR(options_.servable_state_monitor_creator( - servable_event_bus_.get(), &servable_state_monitor)); + const tensorflow::Status status = options_.servable_state_monitor_creator( + servable_event_bus_.get(), &servable_state_monitor); + if (!status.ok()) { + VLOG(1) << "Servable state monitor creation failed: " << status; + return status; + } + servable_state_monitor_ = std::move(servable_state_monitor); std::unique_ptr aspired_versions_manager; @@ -196,9 +201,14 @@ Status ServerCore::AddModelsViaModelConfigList() { for (const auto& model : config_.model_config_list().config()) { static_servables.push_back(ServableRequest::Latest(model.name())); } - TF_RETURN_IF_ERROR(ConnectSourceWithFastInitialLoad( + const tensorflow::Status status = ConnectSourceWithFastInitialLoad( manager_.get(), source_adapter.get(), servable_state_monitor_.get(), - static_servables, options_.num_initial_load_unload_threads)); + static_servables, options_.num_initial_load_unload_threads); + if (!status.ok()) { + VLOG(1) << "Unable to ConnectSourceWithFastInitialLoad due to: " + << status; + return status; + } manager_.AddDependency(std::move(source_adapter)); } else { TF_RETURN_IF_ERROR(ReloadFileSystemStoragePathSourceConfig(source_config)); @@ -263,8 +273,12 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { Status ServerCore::CreateSourceAdapter( const string& model_platform, std::unique_ptr* adapter) { - TF_RETURN_IF_ERROR(options_.source_adapter_creator(model_platform, adapter)); - return Status::OK(); + const tensorflow::Status status = + options_.source_adapter_creator(model_platform, adapter); + if (!status.ok()) { + VLOG(1) << "Source adapter creation failed: " << status; + } + return status; } FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( @@ -286,8 +300,13 @@ Status ServerCore::CreateFileSystemStoragePathSource( const FileSystemStoragePathSourceConfig& source_config, Target* target) { std::unique_ptr storage_path_source; - TF_RETURN_IF_ERROR( - FileSystemStoragePathSource::Create(source_config, &storage_path_source)); + const tensorflow::Status status = + FileSystemStoragePathSource::Create(source_config, &storage_path_source); + if (!status.ok()) { + VLOG(1) << "Unable to create FileSystemStoragePathSource due to: " + << status; + return status; + } ConnectSourceToTarget(storage_path_source.get(), target); storage_path_source_ = storage_path_source.get(); manager_.AddDependency(std::move(storage_path_source)); @@ -296,7 +315,13 @@ Status ServerCore::CreateFileSystemStoragePathSource( Status ServerCore::ReloadFileSystemStoragePathSourceConfig( const FileSystemStoragePathSourceConfig& source_config) { - return storage_path_source_->UpdateConfig(source_config); + const tensorflow::Status status = + storage_path_source_->UpdateConfig(source_config); + if (!status.ok()) { + VLOG(1) << "Unable to ReloadFileSystemStoragePathSourceConfig due to: " + << status; + } + return status; } Status ServerCore::CreateAspiredVersionsManager( @@ -311,7 +336,12 @@ Status ServerCore::CreateAspiredVersionsManager( manager_options.aspired_version_policy = std::move(aspired_version_policy); manager_options.num_load_unload_threads = options_.num_load_unload_threads; manager_options.max_num_load_retries = options_.max_num_load_retries; - return AspiredVersionsManager::Create(std::move(manager_options), manager); + const tensorflow::Status status = + AspiredVersionsManager::Create(std::move(manager_options), manager); + if (!status.ok()) { + VLOG(1) << "Unable to CreateAspiredVersionsManager due to: " << status; + } + return status; } Status ServerCore::CreateResourceTracker( @@ -326,8 +356,12 @@ Status ServerCore::CreateResourceTracker( main_memory_resource->mutable_resource()->set_device(device_types::kMain); main_memory_resource->mutable_resource()->set_kind(resource_kinds::kRamBytes); main_memory_resource->set_quantity(options_.total_model_memory_limit_bytes); - return ResourceTracker::Create(total_resources, std::move(resource_util), - resource_tracker); + const tensorflow::Status status = ResourceTracker::Create( + total_resources, std::move(resource_util), resource_tracker); + if (!status.ok()) { + VLOG(1) << "Unable to CreateResourceTracker due to: " << status; + } + return status; } // ************************************************************************ diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 9b2f0b6ed69..6ebfcb2f0ec 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -162,7 +162,13 @@ class ServerCore : public Manager { ServableHandle* const handle) { ServableRequest servable_request; ServableRequestFromModelSpec(model_spec, &servable_request); - return manager_->GetServableHandle(servable_request, handle); + const tensorflow::Status status = + manager_->GetServableHandle(servable_request, handle); + if (!status.ok()) { + VLOG(1) << "Unable to get servable handle due to: " << status; + return status; + } + return Status::OK(); } protected: From 1261e0ec496bad9f3c95775e83fde1bf08968665 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 11 Nov 2016 09:33:47 -0800 Subject: [PATCH 0069/8554] Minor import fix. Change: 138884923 --- tensorflow_serving/model_servers/server_core.cc | 2 +- tensorflow_serving/model_servers/server_core_test.cc | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 7540d333e26..fbd9d7f3b2c 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -24,7 +24,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 4c1b62f3d2a..12669869cee 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -16,7 +16,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow_serving/apis/model.proto.h" +#include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" From 12615a4349f00e1ff6b0dfcfabdf630b014cfb48 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 11 Nov 2016 10:19:55 -0800 Subject: [PATCH 0070/8554] Fix VLOG statement errors. Change: 138890195 --- tensorflow_serving/model_servers/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index e8c82949d97..ce29981de5b 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -106,7 +106,7 @@ tensorflow::Status CreateSourceAdapter( const ::tensorflow::Status status = SessionBundleSourceAdapter::Create(config, &typed_adapter); if (!status.ok()) { - VLOG(1) << "Error creating source adapter: " << status; + VLOG(1) << "Error creating source adapter: " << status.error_message(); return status; } *adapter = std::move(typed_adapter); @@ -158,7 +158,7 @@ class PredictionServiceImpl final : public PredictionService::Service { const grpc::Status status = ToGRPCStatus( TensorflowPredictImpl::Predict(core_.get(), *request, response)); if (!status.ok()) { - VLOG(1) << "Predict failed: " << status; + VLOG(1) << "Predict failed: " << status.error_message(); } return status; } From d0b23aa9c0fec44a457728d604a6408e14acf0b8 Mon Sep 17 00:00:00 2001 From: Josh Mize Date: Tue, 15 Nov 2016 00:21:22 -0600 Subject: [PATCH 0071/8554] Fix inception_k8s.yaml filename in examples and docs --- .../example/{inception_k8s.json => inception_k8s.yaml} | 0 tensorflow_serving/g3doc/serving_inception.md | 4 ++-- 2 files changed, 2 insertions(+), 2 deletions(-) rename tensorflow_serving/example/{inception_k8s.json => inception_k8s.yaml} (100%) diff --git a/tensorflow_serving/example/inception_k8s.json b/tensorflow_serving/example/inception_k8s.yaml similarity index 100% rename from tensorflow_serving/example/inception_k8s.json rename to tensorflow_serving/example/inception_k8s.yaml diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 38749c73cf3..9447c40fae7 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -227,10 +227,10 @@ along with an [External Load Balancer](http://kubernetes.io/docs/user-guide/load-balancer/). We create them using the example Kubernetes config -[inception_k8s.json](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/inception_k8s.json). +[inception_k8s.yaml](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/inception_k8s.yaml). ```shell -$ kubectl create -f tensorflow_serving/example/inception_k8s.json +$ kubectl create -f tensorflow_serving/example/inception_k8s.yaml deployment "inception-deployment" created service "inception-service" created ``` From 3b50141d4996d8f7a451f1d4c8710c00ee9fad4a Mon Sep 17 00:00:00 2001 From: wai chee yau Date: Wed, 16 Nov 2016 07:36:26 +1100 Subject: [PATCH 0072/8554] add flag for multiple version on command line (#235) --- tensorflow_serving/config/BUILD | 1 + .../config/model_server_config.proto | 5 ++++ tensorflow_serving/model_servers/main.cc | 27 ++++++++++++++++--- .../model_servers/server_core.cc | 1 + 4 files changed, 31 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 39917a4883f..d924949c0fb 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -28,5 +28,6 @@ serving_proto_library( cc_api_version = 2, deps = [ "@protobuf//:cc_wkt_protos", + "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", ], ) diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index f5bbb3b655e..9e7e6a832a0 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -4,6 +4,7 @@ package tensorflow.serving; option cc_enable_arenas = true; import "google/protobuf/any.proto"; +import "tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto"; // The type of model. // TODO(b/31336131): DEPRECATED. @@ -29,6 +30,10 @@ message ModelConfig { // Type of model (e.g. "tensorflow"). string model_platform = 4; + + // Version policy for the model indicating how many versions of the model to be served at the same time. + // The default option is to serve only the latest version of the model. + FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5; } // Static list of models to be loaded for serving. diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index ce29981de5b..9f52d2d5a19 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -73,6 +73,9 @@ using tensorflow::serving::AspiredVersionPolicy; using tensorflow::serving::BatchingParameters; using tensorflow::serving::EagerLoadPolicy; using tensorflow::serving::EventBus; +using tensorflow::serving::FileSystemStoragePathSourceConfig; +using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy; +using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy_Name; using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; @@ -122,16 +125,19 @@ tensorflow::Status LoadCustomModelConfig( } ModelServerConfig BuildSingleModelConfig(const string& model_name, - const string& model_base_path) { + const string& model_base_path, + const FileSystemStoragePathSourceConfig_VersionPolicy& model_version_policy) { ModelServerConfig config; LOG(INFO) << "Building single TensorFlow model file config: " << " model_name: " << model_name - << " model_base_path: " << model_base_path; + << " model_base_path: " << model_base_path + << " model_version_policy: " << model_version_policy; tensorflow::serving::ModelConfig* single_model = config.mutable_model_config_list()->add_config(); single_model->set_name(model_name); single_model->set_base_path(model_base_path); single_model->set_model_platform(kTensorFlowModelPlatform); + single_model->set_version_policy(model_version_policy); return config; } @@ -188,10 +194,18 @@ int main(int argc, char** argv) { tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; + tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( + FileSystemStoragePathSourceConfig::LATEST_VERSION); std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("model_version_policy", &model_version_policy, + "The version policy which determines " + "the number of model versions to be served at the same time. " + "The default value is LATEST_VERSION, which will serve only the latest version. " + "See file_system_storage_path_source.proto for the list of " + "possible VersionPolicy."), tensorflow::Flag("file_system_poll_wait_seconds", &file_system_poll_wait_seconds, "interval in seconds between each poll of the file " @@ -209,11 +223,18 @@ int main(int argc, char** argv) { std::cout << "unknown argument: " << argv[1] << "\n" << usage; } + FileSystemStoragePathSourceConfig_VersionPolicy parsed_version_policy; + bool valid_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Parse( + model_version_policy, &parsed_version_policy); + CHECK(valid_policy) << "Invalid model_version_policy input argument: " << model_version_policy + << "\n" << usage; + + // For ServerCore Options, we leave servable_state_monitor_creator unspecified // so the default servable_state_monitor_creator will be used. ServerCore::Options options; options.model_server_config = - BuildSingleModelConfig(model_name, model_base_path); + BuildSingleModelConfig(model_name, model_base_path, parsed_version_policy); SessionBundleSourceAdapterConfig source_adapter_config; // Batching config diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index fbd9d7f3b2c..831d9e3f4cf 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -292,6 +292,7 @@ FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( source_config.add_servables(); servable->set_servable_name(model.name()); servable->set_base_path(model.base_path()); + servable->set_version_policy(model.version_policy()); } return source_config; } From be95734118f26d64697b77a34b6fecea93990d8d Mon Sep 17 00:00:00 2001 From: Li Lao Date: Mon, 14 Nov 2016 15:33:25 -0800 Subject: [PATCH 0073/8554] Factor out common functions that can be used by SessionBundleFactory and SavedModelBundleFactory. Change: 139130406 --- .../core/aspired_versions_manager.cc | 3 +- tensorflow_serving/core/basic_manager.cc | 8 +- tensorflow_serving/core/basic_manager.h | 3 +- tensorflow_serving/core/basic_manager_test.cc | 4 + tensorflow_serving/servables/tensorflow/BUILD | 69 ++++++- .../tensorflow/bundle_factory_test_util.cc | 83 ++++++++ .../tensorflow/bundle_factory_test_util.h | 58 ++++++ .../tensorflow/bundle_factory_util.cc | 191 ++++++++++++++++++ .../tensorflow/bundle_factory_util.h | 69 +++++++ .../tensorflow/bundle_factory_util_test.cc | 134 ++++++++++++ .../tensorflow/session_bundle_factory.cc | 182 ++--------------- .../tensorflow/session_bundle_factory.h | 16 +- .../tensorflow/session_bundle_factory_test.cc | 98 ++------- 13 files changed, 644 insertions(+), 274 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc create mode 100644 tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h create mode 100644 tensorflow_serving/servables/tensorflow/bundle_factory_util.cc create mode 100644 tensorflow_serving/servables/tensorflow/bundle_factory_util.h create mode 100644 tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 6337d6db86f..3610103b3b5 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -369,7 +369,8 @@ void AspiredVersionsManager::FlushServables() { for (const ServableStateSnapshot& state_snapshot : basic_manager_->GetManagedServableStateSnapshots( servable_name)) { - if ((state_snapshot.state == LoaderHarness::State::kDisabled || + if ((state_snapshot.state == LoaderHarness::State::kNew || + state_snapshot.state == LoaderHarness::State::kDisabled || state_snapshot.state == LoaderHarness::State::kError) && !state_snapshot.additional_state->is_aspired) { basic_manager_->StopManagingServable(state_snapshot.id); diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 68f4d6bf37d..233b3b587b6 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -363,12 +363,14 @@ Status BasicManager::StopManagingServable(const ServableId& id) { id.DebugString()); } const auto state = it->second->state(); - if (state != LoaderHarness::State::kError && + if (state != LoaderHarness::State::kNew && + state != LoaderHarness::State::kError && state != LoaderHarness::State::kDisabled) { LOG(ERROR) << "Request to delete harness for " << id - << ", but it is not in an end state. State: " << state; + << ", but it is not in a new or end state. State: " << state; return errors::FailedPrecondition( - "This servable is not in an end state and we cannot stop managing it: ", + "This servable is not in a new or end state and we cannot stop " + "managing it: ", id.DebugString(), " ", LoaderHarness::StateDebugString(state)); } managed_map_.erase(it); diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index a96bf16ec2d..f0c42b1340b 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -171,7 +171,8 @@ class BasicManager : public Manager { std::unique_ptr additional_state); // Tells the manager to stop managing this servable. Requires that the - // servable is currently being managed and that its state is kEnd. + // servable is currently being managed and that its state is one of {kNew, + // kError, kDisabled}. Status StopManagingServable(const ServableId& id); // Returns the names of all the servables managed by this manager. The names diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 0d74cc69e94..91034cfcdc2 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -184,6 +184,10 @@ TEST_P(BasicManagerTest, StopManagingUnknownId) { TEST_P(BasicManagerTest, StopManagingActiveServable) { const ServableId id = {kServableName3, 1}; basic_manager_->ManageServable(CreateServable(id)); + basic_manager_->LoadServable( + id, [](const Status& status) { TF_EXPECT_OK(status); }); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); EXPECT_FALSE(basic_manager_->StopManagingServable(id).ok()); } diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 0da6b8dc5ac..eff6e903b36 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -37,6 +37,66 @@ serving_proto_library( ], ) +cc_library( + name = "bundle_factory_util", + srcs = ["bundle_factory_util.cc"], + hdrs = ["bundle_factory_util.h"], + deps = [ + ":serving_session", + ":session_bundle_config_proto", + "//tensorflow_serving/batching:batch_scheduler", + "//tensorflow_serving/batching:batching_session", + "//tensorflow_serving/batching:shared_batch_scheduler", + "//tensorflow_serving/resources:resource_values", + "//tensorflow_serving/resources:resources_proto", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf//:cc_wkt_protos", + ], +) + +cc_library( + name = "bundle_factory_test_util", + testonly = 1, + srcs = ["bundle_factory_test_util.cc"], + hdrs = ["bundle_factory_test_util.h"], + deps = [ + "//tensorflow_serving/resources:resource_values", + "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/test_util", + "//testing/base/public:gunit_for_library", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", + "@org_tensorflow//tensorflow/core:testlib", + ], +) + +cc_test( + name = "bundle_factory_util_test", + size = "medium", + srcs = ["bundle_factory_util_test.cc"], + data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + deps = [ + ":bundle_factory_test_util", + ":bundle_factory_util", + ":session_bundle_config_proto", + "//tensorflow_serving/batching:batching_session", + "//tensorflow_serving/batching:shared_batch_scheduler", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/resources:resources_proto", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@protobuf//:cc_wkt_protos", + ], +) + cc_library( name = "session_bundle_factory", srcs = ["session_bundle_factory.cc"], @@ -45,17 +105,16 @@ cc_library( "//visibility:public", ], deps = [ - ":serving_session", + ":bundle_factory_util", ":session_bundle_config_proto", "//tensorflow_serving/batching:batching_session", "//tensorflow_serving/batching:shared_batch_scheduler", - "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/resources:resources_proto", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:tensorflow", - "@protobuf//:cc_wkt_protos", ], ) @@ -65,17 +124,17 @@ cc_test( srcs = ["session_bundle_factory_test.cc"], data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], deps = [ + ":bundle_factory_test_util", ":session_bundle_config_proto", ":session_bundle_factory", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/resources:resource_values", + "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", - "@org_tensorflow//tensorflow/core:testlib", "@protobuf//:cc_wkt_protos", ], ) diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc new file mode 100644 index 00000000000..3d340567997 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc @@ -0,0 +1,83 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow_serving/resources/resource_values.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +void BundleFactoryTest::TestSingleRequest(Session* session) const { + Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + // half plus two: output should be input / 2 + 2. + Tensor expected_output = + test::AsTensor({100.0f / 2 + 2, 42.0f / 2 + 2}, {2}); + + // Note that "x" and "y" are the actual names of the nodes in the graph. + // The saved manifest binds these to "input" and "output" respectively, but + // these tests are focused on the raw underlying session without bindings. + const std::vector> inputs = {{"x", input}}; + const std::vector output_names = {"y"}; + const std::vector empty_targets; + std::vector outputs; + + TF_ASSERT_OK(session->Run(inputs, output_names, empty_targets, &outputs)); + + ASSERT_EQ(1, outputs.size()); + const auto& single_output = outputs.at(0); + test::ExpectTensorEqual(expected_output, single_output); +} + +void BundleFactoryTest::TestMultipleRequests(int num_requests, + Session* session) const { + std::vector> request_threads; + for (int i = 0; i < num_requests; ++i) { + request_threads.push_back( + std::unique_ptr(Env::Default()->StartThread( + ThreadOptions(), strings::StrCat("thread_", i), + [this, session] { this->TestSingleRequest(session); }))); + } +} + +ResourceAllocation BundleFactoryTest::GetExpectedResourceEstimate( + double total_file_size) const { + // kResourceEstimateRAMMultiplier and kResourceEstimateRAMPadBytes should + // match the constants defined in bundle_factory_util.cc. + const double kResourceEstimateRAMMultiplier = 1.2; + const int kResourceEstimateRAMPadBytes = 0; + const uint64 expected_ram_requirement = + total_file_size * kResourceEstimateRAMMultiplier + + kResourceEstimateRAMPadBytes; + ResourceAllocation resource_alloc; + ResourceAllocation::Entry* ram_entry = + resource_alloc.add_resource_quantities(); + Resource* ram_resource = ram_entry->mutable_resource(); + ram_resource->set_device(device_types::kMain); + ram_resource->set_kind(resource_kinds::kRamBytes); + ram_entry->set_quantity(expected_ram_requirement); + return resource_alloc; +} + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h new file mode 100644 index 00000000000..66e47b923c3 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h @@ -0,0 +1,58 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_UTIL_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_UTIL_H_ + +#include +#include "tensorflow/core/public/session.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// Base class of tests related to bundle factories. It contains functions to +// run requests for a half plus model and estimate its resource usage. +class BundleFactoryTest : public ::testing::Test { + public: + virtual ~BundleFactoryTest() = default; + + protected: + BundleFactoryTest() + : export_dir_(test_util::ContribTestSrcDirPath( + "session_bundle/example/half_plus_two/00000123")) {} + + // Test that a Session handles a single request for the half plus two + // model properly. The request has size=2, for batching purposes. + void TestSingleRequest(Session* session) const; + + // Test that a Session handles multiple concurrent requests for the half plus + // two model properly. The request has size=2, for batching purposes. + void TestMultipleRequests(int num_requests, Session* session) const; + + // Returns the expected resource estimate for the given total file size. + ResourceAllocation GetExpectedResourceEstimate(double total_file_size) const; + + // Test data path, to be initialized to point at an export of half-plus-two. + const string export_dir_; +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_UTIL_H_ diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc new file mode 100644 index 00000000000..e58908c310e --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc @@ -0,0 +1,191 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" + +#include "google/protobuf/wrappers.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/batching/batch_scheduler.h" +#include "tensorflow_serving/resources/resource_values.h" +#include "tensorflow_serving/servables/tensorflow/serving_session.h" + +namespace tensorflow { +namespace serving { + +namespace { + +using Batcher = SharedBatchScheduler; + +// Constants used in the resource estimation heuristic. See the documentation +// on EstimateResourceRequirements(). +constexpr double kResourceEstimateRAMMultiplier = 1.2; +constexpr int kResourceEstimateRAMPadBytes = 0; + +// Returns all the descendants, both directories and files, recursively under +// 'dirname'. The paths returned are all prefixed with 'dirname'. +Status GetAllDescendants(const string& dirname, + std::vector* const descendants) { + Env* const env = Env::Default(); + descendants->clear(); + // Make sure that dirname exists; + TF_RETURN_IF_ERROR(env->FileExists(dirname)); + std::deque dir_q; // Queue for the BFS + std::vector dir_list; // List of all dirs discovered + dir_q.push_back(dirname); + Status ret; // Status to be returned. + // Do a BFS on the directory to discover all immediate children. + while (!dir_q.empty()) { + string dir = dir_q.front(); + dir_q.pop_front(); + std::vector children; + // GetChildren might fail if we don't have appropriate permissions. + TF_RETURN_IF_ERROR(env->GetChildren(dir, &children)); + for (const string& child : children) { + const string child_path = io::JoinPath(dir, child); + descendants->push_back(child_path); + // If the child is a directory add it to the queue. + if (env->IsDirectory(child_path).ok()) { + dir_q.push_back(child_path); + } + } + } + return Status::OK(); +} + +} // namespace + +SessionOptions GetSessionOptions(const SessionBundleConfig& config) { + SessionOptions options; + options.target = config.session_target(); + options.config = config.session_config(); + return options; +} + +RunOptions GetRunOptions(const SessionBundleConfig& config) { + RunOptions run_options; + if (config.has_session_run_load_threadpool_index()) { + run_options.set_inter_op_thread_pool( + config.session_run_load_threadpool_index().value()); + } + return run_options; +} + +Status CreateBatchScheduler(const BatchingParameters& batching_config, + std::shared_ptr* batch_scheduler) { + if (!batching_config.allowed_batch_sizes().empty()) { + // Verify that the last allowed batch size matches the max batch size. + const int last_allowed_size = batching_config.allowed_batch_sizes( + batching_config.allowed_batch_sizes().size() - 1); + const int max_size = batching_config.has_max_batch_size() + ? batching_config.max_batch_size().value() + : Batcher::QueueOptions().max_batch_size; + if (last_allowed_size != max_size) { + return errors::InvalidArgument( + "Last entry in allowed_batch_sizes must match max_batch_size; last " + "entry was ", + last_allowed_size, "; expected ", max_size); + } + } + + Batcher::Options options; + if (batching_config.has_num_batch_threads()) { + options.num_batch_threads = batching_config.num_batch_threads().value(); + } + if (batching_config.has_thread_pool_name()) { + options.thread_pool_name = batching_config.thread_pool_name().value(); + } + return Batcher::Create(options, batch_scheduler); +} + +Status EstimateResourceFromPath(const string& path, + ResourceAllocation* estimate) { + if (!Env::Default()->FileExists(path).ok()) { + return errors::NotFound("Nonexistent export path: ", path); + } + std::vector descendants; + TF_RETURN_IF_ERROR(GetAllDescendants(path, &descendants)); + uint64 total_file_size = 0; + for (const string& descendant : descendants) { + if (!(Env::Default()->IsDirectory(descendant).ok())) { + uint64 file_size; + TF_RETURN_IF_ERROR(Env::Default()->GetFileSize(descendant, &file_size)); + total_file_size += file_size; + } + } + const uint64 ram_requirement = + total_file_size * kResourceEstimateRAMMultiplier + + kResourceEstimateRAMPadBytes; + + ResourceAllocation::Entry* ram_entry = estimate->add_resource_quantities(); + Resource* ram_resource = ram_entry->mutable_resource(); + ram_resource->set_device(device_types::kMain); + ram_resource->set_kind(resource_kinds::kRamBytes); + ram_entry->set_quantity(ram_requirement); + + return Status::OK(); +} + +Status WrapSessionForBatching(const BatchingParameters& batching_config, + std::shared_ptr batch_scheduler, + std::unique_ptr* session) { + LOG(INFO) << "Wrapping session to perform batch processing"; + + if (batch_scheduler == nullptr) { + return errors::Internal("batch_scheduler not set"); + } + if (*session == nullptr) { + return errors::Internal("session not set"); + } + + Batcher::QueueOptions queue_options; + if (batching_config.has_max_batch_size()) { + queue_options.max_batch_size = batching_config.max_batch_size().value(); + } + if (batching_config.has_batch_timeout_micros()) { + queue_options.batch_timeout_micros = + batching_config.batch_timeout_micros().value(); + } + if (batching_config.has_max_enqueued_batches()) { + queue_options.max_enqueued_batches = + batching_config.max_enqueued_batches().value(); + } + + BatchingSessionOptions batching_session_options; + for (int allowed_batch_size : batching_config.allowed_batch_sizes()) { + batching_session_options.allowed_batch_sizes.push_back(allowed_batch_size); + } + + auto create_queue = [batch_scheduler, queue_options]( + std::function>)> + process_batch_callback, + std::unique_ptr>* queue) { + TF_RETURN_IF_ERROR(batch_scheduler->AddQueue( + queue_options, process_batch_callback, queue)); + return Status::OK(); + }; + return CreateBatchingSession(batching_session_options, create_queue, + std::move(*session), session); +} + +Status WrapSession(std::unique_ptr* session) { + session->reset(new ServingSessionWrapper(std::move(*session))); + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h new file mode 100644 index 00000000000..dcf79187802 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h @@ -0,0 +1,69 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_UTIL_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_UTIL_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow/core/public/session_options.h" +#include "tensorflow_serving/batching/batching_session.h" +#include "tensorflow_serving/batching/shared_batch_scheduler.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" + +namespace tensorflow { +namespace serving { + +// Returns SessionOptions based on the SessionBundleConfig. +// TODO(b/32248363): add SavedModelBundleConfig after we switch Model Server to +// Saved Model. +SessionOptions GetSessionOptions(const SessionBundleConfig& config); + +// Returns RunOptions based on SessionBundleConfig. +// TODO(b/32248363): add SavedModelBundleConfig after we switch Model Server to +// Saved Model. +RunOptions GetRunOptions(const SessionBundleConfig& config); + +// Creates a BatchScheduler based on the batching configuration. +Status CreateBatchScheduler( + const BatchingParameters& batching_config, + std::shared_ptr>* + batch_scheduler); + +// Estimates the resources a session bundle or saved model bundle will use once +// loaded, from its export or saved model path. +// +// Uses the following crude heuristic, for now: estimated main-memory RAM = +// (combined size of all exported file(s)) * kResourceEstimateRAMMultiplier + +// kResourceEstimateRAMPadBytes. +// TODO(b/27694447): Improve the heuristic. At a minimum, account for GPU RAM. +Status EstimateResourceFromPath(const string& path, + ResourceAllocation* estimate); + +// Wraps a session in a new session that automatically batches Run() calls. +Status WrapSessionForBatching( + const BatchingParameters& batching_config, + std::shared_ptr> batch_scheduler, + std::unique_ptr* session); + +// Wraps a session in a new session that only supports Run() without batching. +Status WrapSession(std::unique_ptr* session); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_UTIL_H_ diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc new file mode 100644 index 00000000000..d31a3259269 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc @@ -0,0 +1,134 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" + +#include +#include +#include +#include + +#include "google/protobuf/wrappers.pb.h" +#include +#include +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow/core/public/session_options.h" +#include "tensorflow_serving/batching/batching_session.h" +#include "tensorflow_serving/batching/shared_batch_scheduler.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" + +namespace tensorflow { +namespace serving { +namespace { + +using test_util::EqualsProto; +using Batcher = SharedBatchScheduler; + +class BundleFactoryUtilTest : public test_util::BundleFactoryTest { + public: + virtual ~BundleFactoryUtilTest() = default; +}; + +TEST_F(BundleFactoryUtilTest, GetSessionOptions) { + SessionBundleConfig bundle_config; + + constexpr char kTarget[] = "target"; + bundle_config.set_session_target(kTarget); + ConfigProto *config_proto = bundle_config.mutable_session_config(); + config_proto->set_allow_soft_placement(true); + + SessionOptions session_options = GetSessionOptions(bundle_config); + EXPECT_EQ(session_options.target, kTarget); + EXPECT_THAT(session_options.config, EqualsProto(*config_proto)); +} + +TEST_F(BundleFactoryUtilTest, GetRunOptions) { + SessionBundleConfig bundle_config; + + // Set the threadpool index to use for session-run calls to 1. + bundle_config.mutable_session_run_load_threadpool_index()->set_value(1); + + RunOptions want; + want.set_inter_op_thread_pool(1); + EXPECT_THAT(GetRunOptions(bundle_config), EqualsProto(want)); +} + +TEST_F(BundleFactoryUtilTest, WrapSession) { + // Create a SessionBundle and wrap the session. + SessionBundle bundle; + TF_ASSERT_OK(LoadSessionBundleFromPathUsingRunOptions( + SessionOptions(), RunOptions(), export_dir_, &bundle)); + TF_ASSERT_OK(WrapSession(&bundle.session)); + TestSingleRequest(bundle.session.get()); +} + +TEST_F(BundleFactoryUtilTest, WrapSessionForBatching) { + // Create a SessionBundle. + SessionBundle bundle; + TF_ASSERT_OK(LoadSessionBundleFromPathUsingRunOptions( + SessionOptions(), RunOptions(), export_dir_, &bundle)); + + // Create BatchingParameters and batch scheduler. + BatchingParameters batching_params; + batching_params.mutable_max_batch_size()->set_value(2); + batching_params.mutable_max_enqueued_batches()->set_value(INT_MAX); + + std::shared_ptr batcher; + TF_ASSERT_OK(CreateBatchScheduler(batching_params, &batcher)); + + // Wrap the session. + TF_ASSERT_OK( + WrapSessionForBatching(batching_params, batcher, &bundle.session)); + + // Run multiple requests concurrently. They should be executed as 5 batches. + TestMultipleRequests(10, bundle.session.get()); +} + +TEST_F(BundleFactoryUtilTest, BatchingConfigError) { + BatchingParameters batching_params; + batching_params.mutable_max_batch_size()->set_value(2); + // The last entry in 'allowed_batch_sizes' is supposed to equal + // 'max_batch_size'. Let's violate that constraint and ensure we get an error. + batching_params.add_allowed_batch_sizes(1); + batching_params.add_allowed_batch_sizes(3); + std::shared_ptr batch_scheduler; + EXPECT_FALSE(CreateBatchScheduler(batching_params, &batch_scheduler).ok()); +} + +TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithBadExport) { + ResourceAllocation resource_requirement; + const Status status = + EstimateResourceFromPath("/a/bogus/export/dir", &resource_requirement); + EXPECT_FALSE(status.ok()); +} + +TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithGoodExport) { + const double kTotalFileSize = 13392.5; + ResourceAllocation expected = GetExpectedResourceEstimate(kTotalFileSize); + + ResourceAllocation actual; + TF_ASSERT_OK(EstimateResourceFromPath(export_dir_, &actual)); + EXPECT_THAT(actual, EqualsProto(expected)); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc index f4e267b6b93..e812d87a475 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc @@ -15,95 +15,21 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/session_bundle_factory.h" -#include "google/protobuf/wrappers.pb.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/regexp.h" -#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/public/session_options.h" -#include "tensorflow_serving/resources/resource_values.h" -#include "tensorflow_serving/servables/tensorflow/serving_session.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" namespace tensorflow { namespace serving { -namespace { - -SessionOptions GetSessionOptions(const SessionBundleConfig& config) { - SessionOptions options; - options.target = config.session_target(); - options.config = config.session_config(); - return options; -} - -// Returns all the descendants, both directories and files, recursively under -// 'dirname'. The paths returned are all prefixed with 'dirname'. -Status GetAllDescendants(const string& dirname, - std::vector* const descendants) { - Env* const env = Env::Default(); - descendants->clear(); - // Make sure that dirname exists; - TF_RETURN_IF_ERROR(env->FileExists(dirname)); - std::deque dir_q; // Queue for the BFS - std::vector dir_list; // List of all dirs discovered - dir_q.push_back(dirname); - Status ret; // Status to be returned. - // Do a BFS on the directory to discover all immediate children. - while (!dir_q.empty()) { - string dir = dir_q.front(); - dir_q.pop_front(); - std::vector children; - // GetChildren might fail if we don't have appropriate permissions. - TF_RETURN_IF_ERROR(env->GetChildren(dir, &children)); - for (const string& child : children) { - const string child_path = io::JoinPath(dir, child); - descendants->push_back(child_path); - // If the child is a directory add it to the queue. - if (env->IsDirectory(child_path).ok()) { - dir_q.push_back(child_path); - } - } - } - return Status::OK(); -} - -} // namespace - -constexpr double SessionBundleFactory::kResourceEstimateRAMMultiplier; -constexpr int SessionBundleFactory::kResourceEstimateRAMPadBytes; - Status SessionBundleFactory::Create( const SessionBundleConfig& config, std::unique_ptr* factory) { std::shared_ptr batcher; - // Populate 'batcher' if batching is configured. if (config.has_batching_parameters()) { - const BatchingParameters& batching_config = config.batching_parameters(); - - if (!batching_config.allowed_batch_sizes().empty()) { - // Verify that the last allowed batch size matches the max batch size. - const int last_allowed_size = batching_config.allowed_batch_sizes( - batching_config.allowed_batch_sizes().size() - 1); - const int max_size = batching_config.has_max_batch_size() - ? batching_config.max_batch_size().value() - : Batcher::QueueOptions().max_batch_size; - if (last_allowed_size != max_size) { - return errors::InvalidArgument( - "Last entry in allowed_batch_sizes must match max_batch_size; last " - "entry was ", - last_allowed_size, "; expected ", max_size); - } - } - - Batcher::Options options; - if (batching_config.has_num_batch_threads()) { - options.num_batch_threads = batching_config.num_batch_threads().value(); - } - if (batching_config.has_thread_pool_name()) { - options.thread_pool_name = batching_config.thread_pool_name().value(); - } - TF_RETURN_IF_ERROR(Batcher::Create(options, &batcher)); + TF_RETURN_IF_ERROR( + CreateBatchScheduler(config.batching_parameters(), &batcher)); } factory->reset(new SessionBundleFactory(config, batcher)); return Status::OK(); @@ -111,105 +37,29 @@ Status SessionBundleFactory::Create( Status SessionBundleFactory::EstimateResourceRequirement( const string& path, ResourceAllocation* estimate) const { - if (!Env::Default()->FileExists(path).ok()) { - return errors::NotFound("Nonexistent export path: ", path); - } - std::vector descendants; - TF_RETURN_IF_ERROR(GetAllDescendants(path, &descendants)); - uint64 total_file_size = 0; - for (const string& descendant : descendants) { - if (!(Env::Default()->IsDirectory(descendant).ok())) { - uint64 file_size; - TF_RETURN_IF_ERROR(Env::Default()->GetFileSize(descendant, &file_size)); - total_file_size += file_size; - } - } - const uint64 ram_requirement = - total_file_size * kResourceEstimateRAMMultiplier + - kResourceEstimateRAMPadBytes; - - ResourceAllocation::Entry* ram_entry = estimate->add_resource_quantities(); - Resource* ram_resource = ram_entry->mutable_resource(); - ram_resource->set_device(device_types::kMain); - ram_resource->set_kind(resource_kinds::kRamBytes); - ram_entry->set_quantity(ram_requirement); - - return Status::OK(); + return EstimateResourceFromPath(path, estimate); } Status SessionBundleFactory::CreateSessionBundle( const string& path, std::unique_ptr* bundle) { bundle->reset(new SessionBundle); + TF_RETURN_IF_ERROR(LoadSessionBundleFromPathUsingRunOptions( + GetSessionOptions(config_), GetRunOptions(config_), path, bundle->get())); - // Setup RunOptions for the session, if specified and load session-bundle. - if (this->config_.has_session_run_load_threadpool_index()) { - RunOptions run_options; - run_options.set_inter_op_thread_pool( - this->config_.session_run_load_threadpool_index().value()); - TF_RETURN_IF_ERROR(LoadSessionBundleFromPathUsingRunOptions( - GetSessionOptions(this->config_), run_options, path, bundle->get())); - } else { - TF_RETURN_IF_ERROR(LoadSessionBundleFromPath( - GetSessionOptions(this->config_), path, bundle->get())); - } - - // Initialize batching, if specified. - if (this->config_.has_batching_parameters()) { - TF_RETURN_IF_ERROR(this->WrapSessionForBatching(bundle->get())); - } else { - (*bundle)->session.reset( - new ServingSessionWrapper(std::move((*bundle)->session))); + if (config_.has_batching_parameters()) { + LOG(INFO) << "Wrapping session to perform batch processing"; + if (batch_scheduler_ == nullptr) { + return errors::Internal("batch_scheduler_ not set"); + } + return WrapSessionForBatching(config_.batching_parameters(), + batch_scheduler_, &(*bundle)->session); } - return Status::OK(); + return WrapSession(&(*bundle)->session); } SessionBundleFactory::SessionBundleFactory( const SessionBundleConfig& config, std::shared_ptr batch_scheduler) : config_(config), batch_scheduler_(batch_scheduler) {} -Status SessionBundleFactory::WrapSessionForBatching(SessionBundle* bundle) { - LOG(INFO) << "Wrapping SessionBundle session to perform batch processing"; - - if (batch_scheduler_ == nullptr) { - return errors::Internal("batch_scheduler_ not set"); - } - if (!config_.has_batching_parameters()) { - return errors::Internal("No batching parameters"); - } - const BatchingParameters& batching_config = config_.batching_parameters(); - - Batcher::QueueOptions queue_options; - if (batching_config.has_max_batch_size()) { - queue_options.max_batch_size = batching_config.max_batch_size().value(); - } - if (batching_config.has_batch_timeout_micros()) { - queue_options.batch_timeout_micros = - batching_config.batch_timeout_micros().value(); - } - if (batching_config.has_max_enqueued_batches()) { - queue_options.max_enqueued_batches = - batching_config.max_enqueued_batches().value(); - } - - BatchingSessionOptions batching_session_options; - for (int allowed_batch_size : batching_config.allowed_batch_sizes()) { - batching_session_options.allowed_batch_sizes.push_back(allowed_batch_size); - } - - auto create_queue = [this, queue_options]( - std::function>)> - process_batch_callback, - std::unique_ptr>* batch_scheduler) { - TF_RETURN_IF_ERROR(this->batch_scheduler_->AddQueue( - queue_options, process_batch_callback, batch_scheduler)); - return Status::OK(); - }; - TF_RETURN_IF_ERROR( - CreateBatchingSession(batching_session_options, create_queue, - std::move(bundle->session), &bundle->session)); - - return Status::OK(); -} - } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h index e616d73d995..ab96398d5ca 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SESSION_BUNDLE_FACTORY_H_ #include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/batching/batching_session.h" #include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" @@ -44,11 +45,6 @@ namespace serving { // This class is thread-safe. class SessionBundleFactory { public: - // Constants used in the resource estimation heuristic. See the documentation - // on EstimateResourceRequirements(). - static constexpr double kResourceEstimateRAMMultiplier = 1.2; - static constexpr int kResourceEstimateRAMPadBytes = 0; - static Status Create(const SessionBundleConfig& config, std::unique_ptr* factory); @@ -58,11 +54,6 @@ class SessionBundleFactory { // Estimates the resources a session bundle will use once loaded, from its // export path. - // - // Uses the following crude heuristic, for now: estimated main-memory RAM = - // (combined size of all exported file(s)) * kResourceEstimateRAMMultiplier + - // kResourceEstimateRAMPadBytes. - // TODO(b/27694447): Improve the heuristic. At a minimum, account for GPU RAM. Status EstimateResourceRequirement(const string& path, ResourceAllocation* estimate) const; @@ -72,12 +63,9 @@ class SessionBundleFactory { SessionBundleFactory(const SessionBundleConfig& config, std::shared_ptr batch_scheduler); - // Wraps a batching queue with retrier around 'bundle->session'. - Status WrapSessionForBatching(SessionBundle* bundle); - const SessionBundleConfig config_; - // A shared batch scheduler. One queue is used for each session this adapter + // A shared batch scheduler. One queue is used for each session this factory // emits. If batching is not configured, this remains null. std::shared_ptr batch_scheduler_; diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index 6f7e087cee8..a5a17de7252 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -24,12 +24,11 @@ limitations under the License. #include #include #include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/types.h" #include "tensorflow/core/public/session.h" -#include "tensorflow_serving/resources/resource_values.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/test_util/test_util.h" @@ -39,38 +38,9 @@ namespace { using test_util::EqualsProto; -class SessionBundleFactoryTest : public ::testing::Test { - protected: - SessionBundleFactoryTest() - : export_dir_(test_util::ContribTestSrcDirPath( - "session_bundle/example/half_plus_two/00000123")) {} - - // Test data path, to be initialized to point at an export of half-plus-two. - const string export_dir_; - - // Test that a SessionBundle handles a single request for the half plus two - // model properly. The request has size=2, for batching purposes. - void TestSingleRequest(SessionBundle* bundle) { - Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); - // half plus two: output should be input / 2 + 2. - Tensor expected_output = - test::AsTensor({100.0f / 2 + 2, 42.0f / 2 + 2}, {2}); - - // Note that "x" and "y" are the actual names of the nodes in the graph. - // The saved manifest binds these to "input" and "output" respectively, but - // these tests are focused on the raw underlying session without bindings. - const std::vector> inputs = {{"x", input}}; - const std::vector output_names = {"y"}; - const std::vector empty_targets; - std::vector outputs; - - TF_ASSERT_OK( - bundle->session->Run(inputs, output_names, empty_targets, &outputs)); - - ASSERT_EQ(1, outputs.size()); - const auto& single_output = outputs.at(0); - test::ExpectTensorEqual(expected_output, single_output); - } +class SessionBundleFactoryTest : public test_util::BundleFactoryTest { + public: + virtual ~SessionBundleFactoryTest() = default; }; TEST_F(SessionBundleFactoryTest, Basic) { @@ -79,7 +49,7 @@ TEST_F(SessionBundleFactoryTest, Basic) { TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); std::unique_ptr bundle; TF_ASSERT_OK(factory->CreateSessionBundle(export_dir_, &bundle)); - TestSingleRequest(bundle.get()); + TestSingleRequest(bundle->session.get()); } TEST_F(SessionBundleFactoryTest, Batching) { @@ -91,63 +61,23 @@ TEST_F(SessionBundleFactoryTest, Batching) { TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); std::unique_ptr bundle; TF_ASSERT_OK(factory->CreateSessionBundle(export_dir_, &bundle)); - SessionBundle* bundle_raw = bundle.get(); - - // Run multiple requests concurrently. They should be executed as - // 'kNumRequestsToTest/2' batches. - { - const int kNumRequestsToTest = 10; - std::vector> request_threads; - for (int i = 0; i < kNumRequestsToTest; ++i) { - request_threads.push_back( - std::unique_ptr(Env::Default()->StartThread( - ThreadOptions(), strings::StrCat("thread_", i), - [this, bundle_raw] { this->TestSingleRequest(bundle_raw); }))); - } - } -} - -TEST_F(SessionBundleFactoryTest, BatchingConfigError) { - SessionBundleConfig config; - BatchingParameters* batching_params = config.mutable_batching_parameters(); - batching_params->mutable_max_batch_size()->set_value(2); - // The last entry in 'allowed_batch_sizes' is supposed to equal - // 'max_batch_size'. Let's violate that constraint and ensure we get an error. - batching_params->add_allowed_batch_sizes(1); - batching_params->add_allowed_batch_sizes(3); - std::unique_ptr factory; - EXPECT_FALSE(SessionBundleFactory::Create(config, &factory).ok()); -} -TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithBadExport) { - const SessionBundleConfig config; - std::unique_ptr factory; - TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); - ResourceAllocation resource_requirement; - const Status status = factory->EstimateResourceRequirement( - "/a/bogus/export/dir", &resource_requirement); - EXPECT_FALSE(status.ok()); + // Run multiple requests concurrently. They should be executed as 5 batches, + // as request size is set to 2 in TestMultipleRequests(). + TestMultipleRequests(10, bundle->session.get()); } TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { const double kTotalFileSize = 13392.5; - const uint64 expected_ram_requirement = - kTotalFileSize * SessionBundleFactory::kResourceEstimateRAMMultiplier + - SessionBundleFactory::kResourceEstimateRAMPadBytes; - ResourceAllocation want; - ResourceAllocation::Entry* ram_entry = want.add_resource_quantities(); - Resource* ram_resource = ram_entry->mutable_resource(); - ram_resource->set_device(device_types::kMain); - ram_resource->set_kind(resource_kinds::kRamBytes); - ram_entry->set_quantity(expected_ram_requirement); + ResourceAllocation expected = GetExpectedResourceEstimate(kTotalFileSize); const SessionBundleConfig config; std::unique_ptr factory; TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); - ResourceAllocation got; - TF_ASSERT_OK(factory->EstimateResourceRequirement(export_dir_, &got)); + ResourceAllocation actual; + TF_ASSERT_OK(factory->EstimateResourceRequirement(export_dir_, &actual)); - EXPECT_THAT(got, EqualsProto(want)); + EXPECT_THAT(actual, EqualsProto(expected)); } TEST_F(SessionBundleFactoryTest, RunOptions) { @@ -171,7 +101,7 @@ TEST_F(SessionBundleFactoryTest, RunOptions) { std::unique_ptr bundle; TF_ASSERT_OK(factory->CreateSessionBundle(export_dir_, &bundle)); - TestSingleRequest(bundle.get()); + TestSingleRequest(bundle->session.get()); } TEST_F(SessionBundleFactoryTest, RunOptionsError) { From c6fa6e304a1d3a2158fc8b3210fcf75804f3dec9 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Tue, 15 Nov 2016 11:15:28 -0800 Subject: [PATCH 0074/8554] Fix Bazel version referenced in Dockerfile.devel. Change: 139223546 --- tensorflow_serving/g3doc/setup.md | 8 ++++---- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 3426e0ee035..03e44c7fb61 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -6,7 +6,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel -TensorFlow Serving requires Bazel 0.3.1 or higher. You can find the Bazel +TensorFlow Serving requires Bazel 0.3.2 or higher. You can find the Bazel installation instructions [here](http://bazel.build/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the @@ -14,13 +14,13 @@ following steps: 1. Download the relevant binary from [here](https://github.com/bazelbuild/bazel/releases). - Let's say you downloaded bazel-0.3.1-installer-linux-x86_64.sh. You would + Let's say you downloaded bazel-0.3.2-installer-linux-x86_64.sh. You would execute: ~~~shell cd ~/Downloads - chmod +x bazel-0.3.1-installer-linux-x86_64.sh - ./bazel-0.3.1-installer-linux-x86_64.sh --user + chmod +x bazel-0.3.2-installer-linux-x86_64.sh + ./bazel-0.3.2-installer-linux-x86_64.sh --user ~~~ 2. Set up your environment. Put this in your ~/.bashrc. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index f4d65c43ace..f0e42edc8ad 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -56,7 +56,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.3.1 +ENV BAZEL_VERSION 0.3.2 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ From be3570a710ec0617ef40e42132cc1d201d00400f Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 15 Nov 2016 11:39:47 -0800 Subject: [PATCH 0075/8554] Add a regression test for the recently-fixed race involving un-aspiring a servable in state kNew. Change: 139226914 --- .../core/aspired_versions_manager_test.cc | 50 +++++++++++++++++++ 1 file changed, 50 insertions(+) diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 05adee0132d..07cbddb0760 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -992,6 +992,56 @@ TEST_P(AspiredVersionsManagerTest, second_load_called.WaitForNotification(); } +TEST_P(AspiredVersionsManagerTest, UnaspireNewServableThenImmediatelyReaspire) { + // Like UnaspireThenImmediatelyReaspire, but covers the case in which the + // servable is in state kNew when it gets unaspired. + // (Regression test for b/27766674.) + + const ServableId id = {kServableName, 7}; + + std::vector>> first_aspired_versions; + test_util::MockLoader* first_loader = new NiceMock(); + EXPECT_CALL(*first_loader, Load(_)).Times(0); + first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(first_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + // (We *don't* call InvokePolicyAndExecuteAction(), thus causing the servable + // to remain in state kNew.) + + // Now, we'll un-aspire the servable, and then re-aspire it with a new loader. + // The manager should get rid of the first loader, then bring up the second + // one. + + std::vector>> empty_aspired_versions; + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(empty_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + + // Re-aspire the servable with a fresh loader. + std::vector>> second_aspired_versions; + test_util::MockLoader* second_loader = new NiceMock(); + second_aspired_versions.push_back( + {id, std::unique_ptr(second_loader)}); + Notification second_load_called; + EXPECT_CALL(*second_loader, Load(_)).WillOnce(InvokeWithoutArgs([&]() { + second_load_called.Notify(); + return Status::OK(); + })); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(second_aspired_versions)); + // The first HandlePendingAspiredVersionsRequests() call will do nothing, + // because the first loader remains in the manager (with state kNew). + HandlePendingAspiredVersionsRequests(); + // FlushServables() should remove the first loader, thus clearing the way for + // a subsequent HandlePendingAspiredVersionsRequests() call to accept the + // second loader. + FlushServables(); + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + second_load_called.WaitForNotification(); +} + class MockAspiredVersionPolicy : public AspiredVersionPolicy { public: MOCK_CONST_METHOD1(GetNextAction, From 83237d83ddc67505766214f07f23b4a056113896 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Tue, 15 Nov 2016 21:07:46 -0800 Subject: [PATCH 0076/8554] Update documentation for ServerCore. Change: 139287084 --- tensorflow_serving/g3doc/serving_advanced.md | 44 +++++++++++--------- 1 file changed, 25 insertions(+), 19 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 7c1f2fa662a..078654559e3 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -89,18 +89,25 @@ TensorFlow Serving `ServerCore`, which internally wraps an int main(int argc, char** argv) { ... + ServerCore::Options options; + options.model_server_config = model_server_config; + options.source_adaptor_creator = [source_adapter_config]( + const string& platform_type, + std::unique_ptr* adapter) { + return CreateSourceAdapter(source_adapter_config, platform_type, adapter); + }; + options.servable_state_monitor_creator = &CreateServableStateMonitor; + options.custom_model_config_loader = &LoadCustomModelConfig; std::unique_ptr core; - TF_CHECK_OK(ServerCore::Create( - config, std::bind(CreateSourceAdapter, source_adapter_config, - std::placeholders::_1, std::placeholders::_2), - &CreateServableStateMonitor, &LoadDynamicModelConfig, &core)); + TF_CHECK_OK(ServerCore::Create(options, &core)); RunServer(port, std::move(core)); return 0; } ~~~ -`ServerCore::Create()` takes four parameters: +`ServerCore::Create()` takes a ServerCore::Options parameter. Here are a few +commonly used options: * `ModelServerConfig` that specifies models to be loaded. Models are declared either through `model_config_list`, which declares a static list of models, or @@ -117,11 +124,9 @@ int main(int argc, char** argv) { creates the base `ServableStateMonitor`, which keeps track of servable states in memory. You can extend it to add state tracking capabilities (e.g. persists state change to disk, remote server, etc.) - * `DynamicModelConfigLoader` that loads models from `dynamic_model_config`. - The standard TensorFlow model server supports only `model_config_list` for - now and therefore `LoadDynamicModelConfig` CHECK-fails when called. You can - extend it to add dynamic model discovery/loading capabilities (e.g. through - RPC, external service, etc.) + * `CustomModelConfigLoader` that instantiates and connects the necessary + custom sources and source adapters to the manager based on a passed in config + (any). `SessionBundle` is a key component of TensorFlow Serving. It represents a TensorFlow model loaded from a given path and provides the same `Session::Run` @@ -140,7 +145,8 @@ With all these, `ServerCore` internally does the following: to a `Loader`. * Instantiates a specific implementation of `Manager` called `AspiredVersionsManager` that manages all such `Loader` instances created by - the `SessionBundleSourceAdapter`. + the `SessionBundleSourceAdapter`. `ServerCore` exports `Manager` interface by + forwarding the calls to `AspiredVersionsManager`. Whenever a new version is available, this `AspiredVersionsManager` loads the new version, and under its default behavior unloads the old one. If you want to @@ -149,14 +155,14 @@ creates internally, and how to configure them. It is worth mentioning that TensorFlow Serving is designed from scratch to be very flexible and extensible. You can build various plugins to customize system -behavior, while taking advantage of generic core components like -`AspiredVersionsManager`. For example, you could build a data source plugin that -monitors cloud storage instead of local storage, or you could build a version -policy plugin that does version transition in a different way -- in fact, you -could even build a custom model plugin that serves non-TensorFlow models. These -topics are out of scope for this tutorial. However, you can refer to the -[custom source](custom_source.md) and [custom servable](custom_servable.md) -tutorials for more information. +behavior, while taking advantage of generic core components like `ServerCore` +and `AspiredVersionsManager`. For example, you could build a data source plugin +that monitors cloud storage instead of local storage, or you could build a +version policy plugin that does version transition in a different way -- in +fact, you could even build a custom model plugin that serves non-TensorFlow +models. These topics are out of scope for this tutorial. However, you can refer +to the [custom source](custom_source.md) and +[custom servable](custom_servable.md) tutorials for more information. ## Batching From 476d5de191ccfaeab7b57f6475d216b4574a3b72 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Wed, 16 Nov 2016 11:21:53 -0800 Subject: [PATCH 0077/8554] Update ServerCore documentation. Change: 139355730 --- tensorflow_serving/g3doc/serving_advanced.md | 21 ++++++-------------- 1 file changed, 6 insertions(+), 15 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 078654559e3..f0685afc811 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -116,17 +116,8 @@ commonly used options: * `SourceAdapterCreator` that creates the `SourceAdapter`, which adapts `StoragePath` (the path where a model version is discovered) to model `Loader` (loads the model version from storage path and provides state - transition interfaces to the `Manager`). In this case, `CreateSourceAdapter` - creates `SessionBundleSourceAdapter`, which we will explain later. - * `ServableStateMonitorCreator` that creates `ServableStateMonitor`, which - keeps track for `Servable` (model version) state transition and provides a - query interface to the user. In this case, `CreateServableStateMonitor` - creates the base `ServableStateMonitor`, which keeps track of servable states - in memory. You can extend it to add state tracking capabilities (e.g. persists - state change to disk, remote server, etc.) - * `CustomModelConfigLoader` that instantiates and connects the necessary - custom sources and source adapters to the manager based on a passed in config - (any). + transition interfaces to the `Manager`). If not specified, a + `SessionBundleSourceAdapter` will be created, which we will explain later. `SessionBundle` is a key component of TensorFlow Serving. It represents a TensorFlow model loaded from a given path and provides the same `Session::Run` @@ -145,8 +136,8 @@ With all these, `ServerCore` internally does the following: to a `Loader`. * Instantiates a specific implementation of `Manager` called `AspiredVersionsManager` that manages all such `Loader` instances created by - the `SessionBundleSourceAdapter`. `ServerCore` exports `Manager` interface by - forwarding the calls to `AspiredVersionsManager`. + the `SessionBundleSourceAdapter`. `ServerCore` exports the `Manager` interface + by delegating the calls to `AspiredVersionsManager`. Whenever a new version is available, this `AspiredVersionsManager` loads the new version, and under its default behavior unloads the old one. If you want to @@ -161,8 +152,8 @@ that monitors cloud storage instead of local storage, or you could build a version policy plugin that does version transition in a different way -- in fact, you could even build a custom model plugin that serves non-TensorFlow models. These topics are out of scope for this tutorial. However, you can refer -to the [custom source](custom_source.md) and -[custom servable](custom_servable.md) tutorials for more information. +to the [custom source](custom_source.md) and [custom servable] +(custom_servable.md) tutorials for more information. ## Batching From a335cd8831f458ec9b0c856838529d3dc625fc96 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 17 Nov 2016 09:48:44 -0800 Subject: [PATCH 0078/8554] Adding __version__ and __git_version__ information into saved meta graphs. Change: 139471934 --- .../servables/tensorflow/bundle_factory_util_test.cc | 8 +++++++- .../servables/tensorflow/session_bundle_factory_test.cc | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc index d31a3259269..7c51df62fa7 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/public/session_options.h" +#include "tensorflow/core/public/version.h" #include "tensorflow_serving/batching/batching_session.h" #include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" @@ -121,7 +122,12 @@ TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithBadExport) { } TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithGoodExport) { - const double kTotalFileSize = 13392.5; + // The length of the file's version strings might change, so we don't + // hardcode their size. They are 4 bytes for tags & size, plus the actual + // length of the strings. + const double kVersionSize = + 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); + const double kTotalFileSize = 13392.5 + kVersionSize; ResourceAllocation expected = GetExpectedResourceEstimate(kTotalFileSize); ResourceAllocation actual; diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index a5a17de7252..3ec0f1b1150 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/public/session.h" +#include "tensorflow/core/public/version.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" @@ -68,7 +69,12 @@ TEST_F(SessionBundleFactoryTest, Batching) { } TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { - const double kTotalFileSize = 13392.5; + // The length of the file's version strings might change, so we don't + // hardcode their size. They are 4 bytes for tags & size, plus the actual + // length of the strings. + const double kVersionSize = + 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); + const double kTotalFileSize = 13392.5 + kVersionSize; ResourceAllocation expected = GetExpectedResourceEstimate(kTotalFileSize); const SessionBundleConfig config; From 88a1452aef57232186dae69d936770fd018f504b Mon Sep 17 00:00:00 2001 From: Li Lao Date: Thu, 17 Nov 2016 14:01:32 -0800 Subject: [PATCH 0079/8554] Add Saved Model Bundle factory and source adapter. Change: 139503883 --- tensorflow_serving/apis/BUILD | 5 +- .../apis/prediction_service_pb2.py | 8 +- tensorflow_serving/example/BUILD | 2 + .../example/inception_client.py | 2 +- tensorflow_serving/example/mnist_client.py | 2 +- .../tensorflow_model_server_test.py | 8 +- .../tensorflow_model_server_test_client.py | 2 +- tensorflow_serving/servables/tensorflow/BUILD | 150 +++++++++++++++--- .../tensorflow/bundle_factory_test.h | 133 ++++++++++++++++ .../tensorflow/bundle_factory_test_util.cc | 31 +++- .../tensorflow/bundle_factory_test_util.h | 35 ++-- .../tensorflow/bundle_factory_util_test.cc | 22 ++- .../tensorflow/saved_model_bundle_factory.cc | 67 ++++++++ .../tensorflow/saved_model_bundle_factory.h | 80 ++++++++++ .../saved_model_bundle_factory_test.cc | 124 +++++++++++++++ .../saved_model_bundle_source_adapter.cc | 77 +++++++++ .../saved_model_bundle_source_adapter.h | 66 ++++++++ .../saved_model_bundle_source_adapter_test.cc | 92 +++++++++++ .../tensorflow/session_bundle_factory_test.cc | 93 +++-------- .../session_bundle_source_adapter_test.cc | 39 +---- 20 files changed, 867 insertions(+), 171 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/bundle_factory_test.h create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 10cbf6d9454..951ced7082e 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -79,5 +79,8 @@ serving_proto_library( py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], - deps = [":predict_proto_py_pb2"], + deps = [ + ":predict_proto_py_pb2", + "//third_party/py/grpc/google", + ], ) diff --git a/tensorflow_serving/apis/prediction_service_pb2.py b/tensorflow_serving/apis/prediction_service_pb2.py index a2813d44fb1..a9ab71f3933 100644 --- a/tensorflow_serving/apis/prediction_service_pb2.py +++ b/tensorflow_serving/apis/prediction_service_pb2.py @@ -43,10 +43,10 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) -from grpc.beta import implementations as beta_implementations -from grpc.beta import interfaces as beta_interfaces -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities +from grpc.google.beta import implementations as beta_implementations +from grpc.google.beta import interfaces as beta_interfaces +from grpc.google.framework.common import cardinality +from grpc.google.framework.interfaces.face import utilities as face_utilities class BetaPredictionServiceServicer(object): diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 4b365b5acd8..501b2361bf9 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -48,6 +48,7 @@ py_binary( ":mnist_input_data", "//tensorflow_serving/apis:predict_proto_py_pb2", "//tensorflow_serving/apis:prediction_service_proto_py_pb2", + "//third_party/py/grpc/google", "@org_tensorflow//tensorflow:tensorflow_py", ], ) @@ -72,6 +73,7 @@ py_binary( deps = [ "//tensorflow_serving/apis:predict_proto_py_pb2", "//tensorflow_serving/apis:prediction_service_proto_py_pb2", + "//third_party/py/grpc/google", "@org_tensorflow//tensorflow:tensorflow_py", ], ) diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py index e4e8a880210..8875a4957e1 100644 --- a/tensorflow_serving/example/inception_client.py +++ b/tensorflow_serving/example/inception_client.py @@ -20,7 +20,7 @@ # This is a placeholder for a Google-internal import. -from grpc.beta import implementations +from grpc.google.beta import implementations import tensorflow as tf from tensorflow_serving.apis import predict_pb2 diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index 38899eb52c7..5cf6bf369b8 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -30,7 +30,7 @@ # This is a placeholder for a Google-internal import. -from grpc.beta import implementations +from grpc.google.beta import implementations import numpy import tensorflow as tf diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 79f4427d9e6..19fd9462cdc 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -27,10 +27,10 @@ # This is a placeholder for a Google-internal import. -from grpc import * -from grpc.beta import implementations -from grpc.beta import interfaces as beta_interfaces -from grpc.framework.interfaces.face import face +from grpc.google import * +from grpc.google.beta import implementations +from grpc.google.beta import interfaces as beta_interfaces +from grpc.google.framework.interfaces.face import face import tensorflow as tf from tensorflow.core.framework import types_pb2 diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py b/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py index 1ab70c99e04..927b55f0523 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py @@ -19,7 +19,7 @@ # This is a placeholder for a Google-internal import. -from grpc.beta import implementations +from grpc.google.beta import implementations import tensorflow as tf from tensorflow.core.framework import types_pb2 diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index eff6e903b36..297742dd75c 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -56,6 +56,31 @@ cc_library( ], ) +cc_test( + name = "bundle_factory_util_test", + size = "medium", + srcs = ["bundle_factory_util_test.cc"], + data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + deps = [ + ":bundle_factory_test_util", + ":bundle_factory_util", + ":session_bundle_config_proto", + "//tensorflow_serving/batching:batching_session", + "//tensorflow_serving/batching:shared_batch_scheduler", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@protobuf//:cc_wkt_protos", + ], +) + cc_library( name = "bundle_factory_test_util", testonly = 1, @@ -74,24 +99,18 @@ cc_library( ], ) -cc_test( - name = "bundle_factory_util_test", - size = "medium", - srcs = ["bundle_factory_util_test.cc"], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], +cc_library( + name = "bundle_factory_test", + testonly = 1, + srcs = ["bundle_factory_test.h"], deps = [ ":bundle_factory_test_util", - ":bundle_factory_util", ":session_bundle_config_proto", - "//tensorflow_serving/batching:batching_session", - "//tensorflow_serving/batching:shared_batch_scheduler", - "//tensorflow_serving/core/test_util:test_main", + "//external:gtest", "//tensorflow_serving/resources:resources_proto", - "@org_tensorflow//tensorflow/contrib/session_bundle", + "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@protobuf//:cc_wkt_protos", ], @@ -124,14 +143,59 @@ cc_test( srcs = ["session_bundle_factory_test.cc"], data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], deps = [ - ":bundle_factory_test_util", + ":bundle_factory_test", ":session_bundle_config_proto", ":session_bundle_factory", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@protobuf//:cc_wkt_protos", + ], +) + +cc_library( + name = "saved_model_bundle_factory", + srcs = ["saved_model_bundle_factory.cc"], + hdrs = ["saved_model_bundle_factory.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":bundle_factory_util", + ":session_bundle_config_proto", + "//tensorflow_serving/batching:batching_session", + "//tensorflow_serving/batching:shared_batch_scheduler", + "//tensorflow_serving/resources:resources_proto", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", + "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "saved_model_bundle_factory_test", + size = "medium", + srcs = ["saved_model_bundle_factory_test.cc"], + data = [ + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + "@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two", + ], + deps = [ + ":bundle_factory_test", + ":saved_model_bundle_factory", + ":session_bundle_config_proto", + "//tensorflow_serving/core/test_util:test_main", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", @@ -170,24 +234,70 @@ cc_test( # Link in all registered kernels. linkstatic = 1, deps = [ + ":bundle_factory_test_util", ":session_bundle_config_proto", ":session_bundle_source_adapter", ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:servable_data", - "//tensorflow_serving/core:servable_id", - "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core/test_util:source_adapter_test_util", "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", - "//tensorflow_serving/util:any_ptr", "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/core:core_cpu", - "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", - "@org_tensorflow//tensorflow/core:testlib", + ], +) + +cc_library( + name = "saved_model_bundle_source_adapter", + srcs = ["saved_model_bundle_source_adapter.cc"], + hdrs = ["saved_model_bundle_source_adapter.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":saved_model_bundle_factory", + ":session_bundle_source_adapter_proto", + "//tensorflow_serving/core:loader", + "//tensorflow_serving/core:simple_loader", + "//tensorflow_serving/core:source_adapter", + "//tensorflow_serving/core:storage_path", + "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/util:optional", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "saved_model_bundle_source_adapter_test", + size = "medium", + srcs = ["saved_model_bundle_source_adapter_test.cc"], + data = [ + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + "@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two", + ], + # Link in all registered kernels. + linkstatic = 1, + deps = [ + ":bundle_factory_test_util", + ":saved_model_bundle_source_adapter", + ":session_bundle_config_proto", + ":session_bundle_source_adapter_proto", + "//tensorflow_serving/core:loader", + "//tensorflow_serving/core:servable_data", + "//tensorflow_serving/core/test_util:source_adapter_test_util", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", ], ) diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h new file mode 100644 index 00000000000..05a0a8eb900 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h @@ -0,0 +1,133 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_H_ + +#include +#include +#include +#include + +#include "google/protobuf/wrappers.pb.h" +#include +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +using test_util::EqualsProto; + +// The base class for SessionBundleFactoryTest and SavedModelBundleFactoryTest. +class BundleFactoryTest : public ::testing::Test { + public: + explicit BundleFactoryTest(const string &export_dir) + : export_dir_(export_dir) {} + + virtual ~BundleFactoryTest() = default; + + protected: + // Test functions to be used by subclasses. + void TestBasic() const { + const SessionBundleConfig config; + std::unique_ptr session; + TF_ASSERT_OK(CreateSession(config, &session)); + TestSingleRequest(session.get()); + } + + void TestBatching() const { + SessionBundleConfig config; + BatchingParameters *batching_params = config.mutable_batching_parameters(); + batching_params->mutable_max_batch_size()->set_value(2); + batching_params->mutable_max_enqueued_batches()->set_value(INT_MAX); + std::unique_ptr session; + TF_ASSERT_OK(CreateSession(config, &session)); + + // Run multiple requests concurrently. They should be executed as 5 batches, + // as request size is set to 2 in TestMultipleRequests(). + TestMultipleRequests(10, session.get()); + } + + template + void TestEstimateResourceRequirementWithGoodExport( + double total_file_size) const { + ResourceAllocation expected = GetExpectedResourceEstimate(total_file_size); + + const SessionBundleConfig config; + std::unique_ptr factory; + TF_ASSERT_OK(FactoryType::Create(config, &factory)); + ResourceAllocation actual; + TF_ASSERT_OK(factory->EstimateResourceRequirement(export_dir_, &actual)); + + EXPECT_THAT(actual, EqualsProto(expected)); + } + + void TestRunOptions() const { + SessionBundleConfig config; + + // Configure the session-config with two threadpools. The first is setup + // with default settings. The second is explicitly setup with 1 thread. + config.mutable_session_config()->add_session_inter_op_thread_pool(); + config.mutable_session_config() + ->add_session_inter_op_thread_pool() + ->set_num_threads(1); + + // Set the threadpool index to use for session-run calls to 1. + config.mutable_session_run_load_threadpool_index()->set_value(1); + + // Since the session_run_load_threadpool_index in the config is set, the + // session-bundle should be loaded successfully from path with RunOptions. + std::unique_ptr session; + TF_ASSERT_OK(CreateSession(config, &session)); + + TestSingleRequest(session.get()); + } + + void TestRunOptionsError() const { + // Session bundle config with the default global threadpool. + SessionBundleConfig config; + + // Invalid threadpool index to use for session-run calls. + config.mutable_session_run_load_threadpool_index()->set_value(100); + + // Since RunOptions used in the session run calls refers to an invalid + // threadpool index, load session bundle from path should fail. + std::unique_ptr session; + EXPECT_FALSE(CreateSession(config, &session).ok()); + } + + // Test data path, to be initialized to point at a SessionBundle export or + // SavedModel of half-plus-two. + string export_dir_; + + private: + // Creates a Session with the given configuration and export path. + virtual Status CreateSession(const SessionBundleConfig &config, + std::unique_ptr *session) const = 0; +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_H_ diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc index 3d340567997..71af3580696 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc @@ -19,14 +19,34 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" #include "tensorflow_serving/resources/resource_values.h" +#include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { namespace serving { namespace test_util { -void BundleFactoryTest::TestSingleRequest(Session* session) const { +namespace { + +const char kTestSavedModelPath[] = + "cc/saved_model/testdata/half_plus_two_sharded"; +const char kTestSessionBundleExportPath[] = + "session_bundle/example/half_plus_two/00000123"; + +} // namespace + +string GetTestSavedModelPath() { + return io::JoinPath(testing::TensorFlowSrcRoot(), kTestSavedModelPath); +} + +string GetTestSessionBundleExportPath() { + return test_util::ContribTestSrcDirPath(kTestSessionBundleExportPath); +} + +void TestSingleRequest(Session* session) { Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); // half plus two: output should be input / 2 + 2. Tensor expected_output = @@ -47,19 +67,17 @@ void BundleFactoryTest::TestSingleRequest(Session* session) const { test::ExpectTensorEqual(expected_output, single_output); } -void BundleFactoryTest::TestMultipleRequests(int num_requests, - Session* session) const { +void TestMultipleRequests(int num_requests, Session* session) { std::vector> request_threads; for (int i = 0; i < num_requests; ++i) { request_threads.push_back( std::unique_ptr(Env::Default()->StartThread( ThreadOptions(), strings::StrCat("thread_", i), - [this, session] { this->TestSingleRequest(session); }))); + [session] { TestSingleRequest(session); }))); } } -ResourceAllocation BundleFactoryTest::GetExpectedResourceEstimate( - double total_file_size) const { +ResourceAllocation GetExpectedResourceEstimate(double total_file_size) { // kResourceEstimateRAMMultiplier and kResourceEstimateRAMPadBytes should // match the constants defined in bundle_factory_util.cc. const double kResourceEstimateRAMMultiplier = 1.2; @@ -80,4 +98,3 @@ ResourceAllocation BundleFactoryTest::GetExpectedResourceEstimate( } // namespace test_util } // namespace serving } // namespace tensorflow - diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h index 66e47b923c3..efebb216d41 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h @@ -19,37 +19,28 @@ limitations under the License. #include #include "tensorflow/core/public/session.h" #include "tensorflow_serving/resources/resources.pb.h" -#include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { namespace serving { namespace test_util { -// Base class of tests related to bundle factories. It contains functions to -// run requests for a half plus model and estimate its resource usage. -class BundleFactoryTest : public ::testing::Test { - public: - virtual ~BundleFactoryTest() = default; +// Returns the path of the Saved Model (the pb version) for the half plus two +// model. +string GetTestSavedModelPath(); - protected: - BundleFactoryTest() - : export_dir_(test_util::ContribTestSrcDirPath( - "session_bundle/example/half_plus_two/00000123")) {} +// Returns the Session Bundle export path for the half plus two model. +string GetTestSessionBundleExportPath(); - // Test that a Session handles a single request for the half plus two - // model properly. The request has size=2, for batching purposes. - void TestSingleRequest(Session* session) const; +// Test that a Session handles a single request for the half plus two +// model properly. The request has size=2, for batching purposes. +void TestSingleRequest(Session* session); - // Test that a Session handles multiple concurrent requests for the half plus - // two model properly. The request has size=2, for batching purposes. - void TestMultipleRequests(int num_requests, Session* session) const; +// Test that a Session handles multiple concurrent requests for the half plus +// two model properly. The request has size=2, for batching purposes. +void TestMultipleRequests(int num_requests, Session* session); - // Returns the expected resource estimate for the given total file size. - ResourceAllocation GetExpectedResourceEstimate(double total_file_size) const; - - // Test data path, to be initialized to point at an export of half-plus-two. - const string export_dir_; -}; +// Returns the expected resource estimate for the given total file size. +ResourceAllocation GetExpectedResourceEstimate(double total_file_size); } // namespace test_util } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc index 7c51df62fa7..bd9f6b70a00 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc @@ -35,6 +35,7 @@ limitations under the License. #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { namespace serving { @@ -43,9 +44,15 @@ namespace { using test_util::EqualsProto; using Batcher = SharedBatchScheduler; -class BundleFactoryUtilTest : public test_util::BundleFactoryTest { - public: +class BundleFactoryUtilTest : public ::testing::Test { + protected: + BundleFactoryUtilTest() + : export_dir_(test_util::GetTestSessionBundleExportPath()) {} + virtual ~BundleFactoryUtilTest() = default; + + // Test data path, to be initialized to point at an export of half-plus-two. + const string export_dir_; }; TEST_F(BundleFactoryUtilTest, GetSessionOptions) { @@ -74,15 +81,19 @@ TEST_F(BundleFactoryUtilTest, GetRunOptions) { TEST_F(BundleFactoryUtilTest, WrapSession) { // Create a SessionBundle and wrap the session. + // TODO(b/32248363): use SavedModelBundle instead of SessionBundle when we + // switch the Model Server to use Saved Model. SessionBundle bundle; TF_ASSERT_OK(LoadSessionBundleFromPathUsingRunOptions( SessionOptions(), RunOptions(), export_dir_, &bundle)); TF_ASSERT_OK(WrapSession(&bundle.session)); - TestSingleRequest(bundle.session.get()); + test_util::TestSingleRequest(bundle.session.get()); } TEST_F(BundleFactoryUtilTest, WrapSessionForBatching) { // Create a SessionBundle. + // TODO(b/32248363): use SavedModelBundle instead of SessionBundle when we + // switch the Model Server to use Saved Model. SessionBundle bundle; TF_ASSERT_OK(LoadSessionBundleFromPathUsingRunOptions( SessionOptions(), RunOptions(), export_dir_, &bundle)); @@ -100,7 +111,7 @@ TEST_F(BundleFactoryUtilTest, WrapSessionForBatching) { WrapSessionForBatching(batching_params, batcher, &bundle.session)); // Run multiple requests concurrently. They should be executed as 5 batches. - TestMultipleRequests(10, bundle.session.get()); + test_util::TestMultipleRequests(10, bundle.session.get()); } TEST_F(BundleFactoryUtilTest, BatchingConfigError) { @@ -128,7 +139,8 @@ TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithGoodExport) { const double kVersionSize = 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); const double kTotalFileSize = 13392.5 + kVersionSize; - ResourceAllocation expected = GetExpectedResourceEstimate(kTotalFileSize); + ResourceAllocation expected = + test_util::GetExpectedResourceEstimate(kTotalFileSize); ResourceAllocation actual; TF_ASSERT_OK(EstimateResourceFromPath(export_dir_, &actual)); diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc new file mode 100644 index 00000000000..f176a13a978 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -0,0 +1,67 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h" + +#include "tensorflow/cc/saved_model/tag_constants.h" +#include "tensorflow/contrib/session_bundle/bundle_shim.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/public/session_options.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" + +namespace tensorflow { +namespace serving { + +Status SavedModelBundleFactory::Create( + const SessionBundleConfig& config, + std::unique_ptr* factory) { + std::shared_ptr batcher; + if (config.has_batching_parameters()) { + TF_RETURN_IF_ERROR( + CreateBatchScheduler(config.batching_parameters(), &batcher)); + } + factory->reset(new SavedModelBundleFactory(config, batcher)); + return Status::OK(); +} + +Status SavedModelBundleFactory::EstimateResourceRequirement( + const string& path, ResourceAllocation* estimate) const { + return EstimateResourceFromPath(path, estimate); +} + +Status SavedModelBundleFactory::CreateSavedModelBundle( + const string& path, std::unique_ptr* bundle) { + bundle->reset(new SavedModelBundle); + TF_RETURN_IF_ERROR(LoadSessionBundleOrSavedModelBundle( + GetSessionOptions(config_), GetRunOptions(config_), path, + {kSavedModelTagServe}, bundle->get())); + if (config_.has_batching_parameters()) { + LOG(INFO) << "Wrapping session to perform batch processing"; + if (batch_scheduler_ == nullptr) { + return errors::Internal("batch_scheduler_ not set"); + } + return WrapSessionForBatching(config_.batching_parameters(), + batch_scheduler_, &(*bundle)->session); + } + return WrapSession(&(*bundle)->session); +} + +SavedModelBundleFactory::SavedModelBundleFactory( + const SessionBundleConfig& config, std::shared_ptr batch_scheduler) + : config_(config), batch_scheduler_(batch_scheduler) {} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h new file mode 100644 index 00000000000..2681df44cee --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h @@ -0,0 +1,80 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_FACTORY_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_FACTORY_H_ + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow_serving/batching/batching_session.h" +#include "tensorflow_serving/batching/shared_batch_scheduler.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" + +namespace tensorflow { +namespace serving { + +// A factory that creates SavedModelBundles from SavedModel or SessionBundle +// export paths. +// +// The emitted sessions only support Run(), and although not enforced it is +// expected that the client will only make non-mutating Run() calls. (If this +// restriction, which we've added as a safety measure, is problematic for your +// use-case please contact the TensorFlow Serving team to discuss disabling it.) +// +// If the config calls for batching, the emitted sessions automatically batch +// Run() calls behind the scenes, using a SharedBatchScheduler owned by the +// factory. The 'config.num_batch_threads' threads are shared across all session +// instances created by this factory. However, each session has its own +// dedicated queue of size 'config.max_enqueued_batches'. +// +// The factory can also estimate the resource (e.g. RAM) requirements of a +// SavedModelBundle based on the SavedModel (i.e. prior to loading the session). +// +// This class is thread-safe. +class SavedModelBundleFactory { + public: + static Status Create(const SessionBundleConfig& config, + std::unique_ptr* factory); + + // Instantiates a bundle from a given export or SavedModel path. + Status CreateSavedModelBundle(const string& path, + std::unique_ptr* bundle); + + // Estimates the resources a SavedModel bundle will use once loaded, from its + // export path. + Status EstimateResourceRequirement(const string& path, + ResourceAllocation* estimate) const; + + private: + using Batcher = SharedBatchScheduler; + + SavedModelBundleFactory(const SessionBundleConfig& config, + std::shared_ptr batch_scheduler); + + const SessionBundleConfig config_; + + // A shared batch scheduler. One queue is used for each session this factory + // emits. If batching is not configured, this remains null. + std::shared_ptr batch_scheduler_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelBundleFactory); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_FACTORY_H_ diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc new file mode 100644 index 00000000000..6f59d3444d1 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -0,0 +1,124 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h" + +#include +#include +#include +#include + +#include "google/protobuf/wrappers.pb.h" +#include +#include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow/core/public/version.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" + +namespace tensorflow { +namespace serving { +namespace { + +// Creates a new session based on the config and export path. +Status CreateSessionFromPath(const SessionBundleConfig& config, + const string& path, + std::unique_ptr* session) { + std::unique_ptr factory; + TF_RETURN_IF_ERROR(SavedModelBundleFactory::Create(config, &factory)); + std::unique_ptr bundle; + TF_RETURN_IF_ERROR(factory->CreateSavedModelBundle(path, &bundle)); + *session = std::move(bundle->session); + return Status::OK(); +} + +// Tests SavedModelBundleFactory with native SavedModel. +class SavedModelBundleFactoryTest : public test_util::BundleFactoryTest { + public: + SavedModelBundleFactoryTest() + : test_util::BundleFactoryTest(test_util::GetTestSavedModelPath()) {} + + virtual ~SavedModelBundleFactoryTest() = default; + + private: + Status CreateSession(const SessionBundleConfig& config, + std::unique_ptr* session) const override { + return CreateSessionFromPath(config, export_dir_, session); + } +}; + +TEST_F(SavedModelBundleFactoryTest, Basic) { TestBasic(); } + +TEST_F(SavedModelBundleFactoryTest, Batching) { TestBatching(); } + +TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { + const double kTotalFileSize = 7492; + TestEstimateResourceRequirementWithGoodExport( + kTotalFileSize); +} + +TEST_F(SavedModelBundleFactoryTest, RunOptions) { TestRunOptions(); } + +TEST_F(SavedModelBundleFactoryTest, RunOptionsError) { TestRunOptionsError(); } + +// Tests SavedModelBundleFactory with SessionBundle export. +class SavedModelBundleFactoryBackwardCompatibilityTest + : public test_util::BundleFactoryTest { + public: + SavedModelBundleFactoryBackwardCompatibilityTest() + : test_util::BundleFactoryTest( + test_util::GetTestSessionBundleExportPath()) {} + + virtual ~SavedModelBundleFactoryBackwardCompatibilityTest() = default; + + private: + Status CreateSession(const SessionBundleConfig& config, + std::unique_ptr* session) const override { + return CreateSessionFromPath(config, export_dir_, session); + } +}; + +TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, Basic) { TestBasic(); } + +TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, Batching) { + TestBatching(); +} + +TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, + EstimateResourceRequirementWithGoodExport) { + // The length of the file's version strings might change, so we don't + // hardcode their size. They are 4 bytes for tags & size, plus the actual + // length of the strings. + const double kVersionSize = + 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); + const double kTotalFileSize = 13392.5 + kVersionSize; + TestEstimateResourceRequirementWithGoodExport( + kTotalFileSize); +} + +TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, RunOptions) { + TestRunOptions(); +} + +TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, RunOptionsError) { + TestRunOptionsError(); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc new file mode 100644 index 00000000000..11fcf02d548 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc @@ -0,0 +1,77 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h" + +#include +#include + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/simple_loader.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/util/optional.h" + +namespace tensorflow { +namespace serving { + +Status SavedModelBundleSourceAdapter::Create( + const SessionBundleSourceAdapterConfig& config, + std::unique_ptr* adapter) { + std::unique_ptr bundle_factory; + TF_RETURN_IF_ERROR( + SavedModelBundleFactory::Create(config.config(), &bundle_factory)); + adapter->reset(new SavedModelBundleSourceAdapter(std::move(bundle_factory))); + return Status::OK(); +} + +SavedModelBundleSourceAdapter::~SavedModelBundleSourceAdapter() { Detach(); } + +SavedModelBundleSourceAdapter::SavedModelBundleSourceAdapter( + std::unique_ptr bundle_factory) + : bundle_factory_(std::move(bundle_factory)) {} + +Status SavedModelBundleSourceAdapter::Convert(const StoragePath& path, + std::unique_ptr* loader) { + std::shared_ptr bundle_factory = bundle_factory_; + auto servable_creator = [bundle_factory, + path](std::unique_ptr* bundle) { + return bundle_factory->CreateSavedModelBundle(path, bundle); + }; + auto resource_estimator = [bundle_factory, + path](ResourceAllocation* estimate) { + return bundle_factory->EstimateResourceRequirement(path, estimate); + }; + loader->reset( + new SimpleLoader(servable_creator, resource_estimator)); + return Status::OK(); +} + +std::function>>*)> +SavedModelBundleSourceAdapter::GetCreator( + const SessionBundleSourceAdapterConfig& config) { + return [config](std::unique_ptr>>* source) { + std::unique_ptr typed_source; + TF_RETURN_IF_ERROR( + SavedModelBundleSourceAdapter::Create(config, &typed_source)); + *source = std::move(typed_source); + return Status::OK(); + }; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h new file mode 100644 index 00000000000..9e75d43b9bb --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h @@ -0,0 +1,66 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_SOURCE_ADAPTER_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_SOURCE_ADAPTER_H_ + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/source_adapter.h" +#include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" + +namespace tensorflow { +namespace serving { + +// A SourceAdapter that creates SavedModelBundle Loaders from SavedModel paths. +// It keeps a SavedModelBundleFactory as its state, which may house a batch +// scheduler that is shared across all of the SavedModel bundles it emits. +class SavedModelBundleSourceAdapter final + : public UnarySourceAdapter> { + public: + // TODO(b/32248363): add SavedModelBundleSourceAdapterConfig after we switch + // Model Server to Saved Model. + static Status Create(const SessionBundleSourceAdapterConfig& config, + std::unique_ptr* adapter); + + ~SavedModelBundleSourceAdapter() override; + + // Returns a function to create a SavedModel bundle source adapter. + static std::function>>*)> + GetCreator(const SessionBundleSourceAdapterConfig& config); + + private: + explicit SavedModelBundleSourceAdapter( + std::unique_ptr bundle_factory); + + Status Convert(const StoragePath& path, + std::unique_ptr* loader) override; + + // We use a shared ptr to share ownership with Loaders we emit, in case they + // outlive this object. + std::shared_ptr bundle_factory_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelBundleSourceAdapter); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_SOURCE_ADAPTER_H_ diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc new file mode 100644 index 00000000000..81072b7fcc7 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc @@ -0,0 +1,92 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h" + +#include +#include +#include +#include + +#include +#include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/servable_data.h" +#include "tensorflow_serving/core/test_util/source_adapter_test_util.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using test_util::EqualsProto; + +class SavedModelBundleSourceAdapterTest : public ::testing::Test { + protected: + void TestSavedModelBundleSourceAdapter( + const SessionBundleSourceAdapterConfig& config, + const string& export_dir) const { + std::unique_ptr loader; + { + std::unique_ptr adapter; + TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(config, &adapter)); + ServableData> loader_data = + test_util::RunSourceAdapter(export_dir, adapter.get()); + TF_ASSERT_OK(loader_data.status()); + loader = loader_data.ConsumeDataOrDie(); + + // Let the adapter fall out of scope and be deleted. The loader we got + // from it should be unaffected. Regression test coverage for b/30202207. + } + + // We should get a non-empty resource estimate, and we should get the same + // value twice (via memoization). + ResourceAllocation first_resource_estimate; + TF_ASSERT_OK(loader->EstimateResources(&first_resource_estimate)); + EXPECT_FALSE(first_resource_estimate.resource_quantities().empty()); + ResourceAllocation second_resource_estimate; + TF_ASSERT_OK(loader->EstimateResources(&second_resource_estimate)); + EXPECT_THAT(second_resource_estimate, EqualsProto(first_resource_estimate)); + + TF_ASSERT_OK(loader->Load(ResourceAllocation())); + + const SavedModelBundle* bundle = loader->servable().get(); + test_util::TestSingleRequest(bundle->session.get()); + + loader->Unload(); + } +}; + +TEST_F(SavedModelBundleSourceAdapterTest, Basic) { + const SessionBundleSourceAdapterConfig config; + TestSavedModelBundleSourceAdapter(config, test_util::GetTestSavedModelPath()); +} + +TEST_F(SavedModelBundleSourceAdapterTest, BackwardCompatibility) { + const SessionBundleSourceAdapterConfig config; + TestSavedModelBundleSourceAdapter( + config, test_util::GetTestSessionBundleExportPath()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index 3ec0f1b1150..19a951b1fc3 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -28,8 +28,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/public/version.h" -#include "tensorflow_serving/resources/resources.pb.h" -#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/test_util/test_util.h" @@ -41,32 +40,27 @@ using test_util::EqualsProto; class SessionBundleFactoryTest : public test_util::BundleFactoryTest { public: + SessionBundleFactoryTest() + : test_util::BundleFactoryTest( + test_util::GetTestSessionBundleExportPath()) {} + virtual ~SessionBundleFactoryTest() = default; + + private: + Status CreateSession(const SessionBundleConfig& config, + std::unique_ptr* session) const override { + std::unique_ptr factory; + TF_RETURN_IF_ERROR(SessionBundleFactory::Create(config, &factory)); + std::unique_ptr bundle; + TF_RETURN_IF_ERROR(factory->CreateSessionBundle(export_dir_, &bundle)); + *session = std::move(bundle->session); + return Status::OK(); + } }; -TEST_F(SessionBundleFactoryTest, Basic) { - const SessionBundleConfig config; - std::unique_ptr factory; - TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); - std::unique_ptr bundle; - TF_ASSERT_OK(factory->CreateSessionBundle(export_dir_, &bundle)); - TestSingleRequest(bundle->session.get()); -} +TEST_F(SessionBundleFactoryTest, Basic) { TestBasic(); } -TEST_F(SessionBundleFactoryTest, Batching) { - SessionBundleConfig config; - BatchingParameters* batching_params = config.mutable_batching_parameters(); - batching_params->mutable_max_batch_size()->set_value(2); - batching_params->mutable_max_enqueued_batches()->set_value(INT_MAX); - std::unique_ptr factory; - TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); - std::unique_ptr bundle; - TF_ASSERT_OK(factory->CreateSessionBundle(export_dir_, &bundle)); - - // Run multiple requests concurrently. They should be executed as 5 batches, - // as request size is set to 2 in TestMultipleRequests(). - TestMultipleRequests(10, bundle->session.get()); -} +TEST_F(SessionBundleFactoryTest, Batching) { TestBatching(); } TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { // The length of the file's version strings might change, so we don't @@ -75,56 +69,13 @@ TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { const double kVersionSize = 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); const double kTotalFileSize = 13392.5 + kVersionSize; - ResourceAllocation expected = GetExpectedResourceEstimate(kTotalFileSize); - - const SessionBundleConfig config; - std::unique_ptr factory; - TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); - ResourceAllocation actual; - TF_ASSERT_OK(factory->EstimateResourceRequirement(export_dir_, &actual)); - - EXPECT_THAT(actual, EqualsProto(expected)); -} - -TEST_F(SessionBundleFactoryTest, RunOptions) { - SessionBundleConfig config; - - // Configure the session-config with two threadpools. The first is setup with - // default settings. The second is explicitly setup with 1 thread. - config.mutable_session_config()->add_session_inter_op_thread_pool(); - config.mutable_session_config() - ->add_session_inter_op_thread_pool() - ->set_num_threads(1); - - // Set the threadpool index to use for session-run calls to 1. - config.mutable_session_run_load_threadpool_index()->set_value(1); - - std::unique_ptr factory; - TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); - - // Since the session_run_load_threadpool_index in the config is set, the - // session-bundle should be loaded successfully from path with RunOptions. - std::unique_ptr bundle; - TF_ASSERT_OK(factory->CreateSessionBundle(export_dir_, &bundle)); - - TestSingleRequest(bundle->session.get()); + TestEstimateResourceRequirementWithGoodExport( + kTotalFileSize); } -TEST_F(SessionBundleFactoryTest, RunOptionsError) { - // Session bundle config with the default global threadpool. - SessionBundleConfig config; - - // Invalid threadpool index to use for session-run calls. - config.mutable_session_run_load_threadpool_index()->set_value(100); +TEST_F(SessionBundleFactoryTest, RunOptions) { TestRunOptions(); } - std::unique_ptr factory; - TF_ASSERT_OK(SessionBundleFactory::Create(config, &factory)); - - // Since RunOptions used in the session run calls refers to an invalid - // threadpool index, load session bundle from path should fail. - std::unique_ptr bundle; - EXPECT_FALSE(factory->CreateSessionBundle(export_dir_, &bundle).ok()); -} +TEST_F(SessionBundleFactoryTest, RunOptionsError) { TestRunOptionsError(); } } // namespace } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc index f257f08c46e..631877da9c9 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc @@ -23,17 +23,13 @@ limitations under the License. #include #include #include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/io/path.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow/core/public/session.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_data.h" -#include "tensorflow_serving/core/servable_id.h" -#include "tensorflow_serving/core/source_adapter.h" #include "tensorflow_serving/core/test_util/source_adapter_test_util.h" +#include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" #include "tensorflow_serving/test_util/test_util.h" @@ -47,38 +43,13 @@ using test_util::EqualsProto; class SessionBundleSourceAdapterTest : public ::testing::Test { protected: SessionBundleSourceAdapterTest() - : export_dir_(test_util::ContribTestSrcDirPath( - "session_bundle/example/half_plus_two/00000123")) {} + : export_dir_(test_util::GetTestSessionBundleExportPath()) {} // Test data path, to be initialized to point at an export of half-plus-two. const string export_dir_; - // Test that a SessionBundle handles a single request for the half plus two - // model properly. The request has size=2, for batching purposes. - void TestSingleRequest(const SessionBundle* bundle) { - Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); - // half plus two: output should be input / 2 + 2. - Tensor expected_output = - test::AsTensor({100.0f / 2 + 2, 42.0f / 2 + 2}, {2}); - - // Note that "x" and "y" are the actual names of the nodes in the graph. - // The saved manifest binds these to "input" and "output" respectively, but - // these tests are focused on the raw underlying session without bindings. - const std::vector> inputs = {{"x", input}}; - const std::vector output_names = {"y"}; - const std::vector empty_targets; - std::vector outputs; - - TF_ASSERT_OK( - bundle->session->Run(inputs, output_names, empty_targets, &outputs)); - - ASSERT_EQ(1, outputs.size()); - const auto& single_output = outputs.at(0); - test::ExpectTensorEqual(expected_output, single_output); - } - void TestSessionBundleSourceAdapter( - const SessionBundleSourceAdapterConfig& config) { + const SessionBundleSourceAdapterConfig& config) const { std::unique_ptr loader; { std::unique_ptr adapter; @@ -104,7 +75,7 @@ class SessionBundleSourceAdapterTest : public ::testing::Test { TF_ASSERT_OK(loader->Load(ResourceAllocation())); const SessionBundle* bundle = loader->servable().get(); - TestSingleRequest(bundle); + test_util::TestSingleRequest(bundle->session.get()); loader->Unload(); } From 16e3c9c5246dee4a7348547f1692b8d9a264fa1c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 17 Nov 2016 14:35:30 -0800 Subject: [PATCH 0080/8554] Remove unused import Change: 139508592 --- tensorflow_serving/apis/BUILD | 5 +---- tensorflow_serving/apis/prediction_service_pb2.py | 8 ++++---- tensorflow_serving/example/BUILD | 2 -- tensorflow_serving/example/inception_client.py | 2 +- tensorflow_serving/example/mnist_client.py | 2 +- .../model_servers/tensorflow_model_server_test.py | 7 +++---- .../model_servers/tensorflow_model_server_test_client.py | 2 +- 7 files changed, 11 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 951ced7082e..10cbf6d9454 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -79,8 +79,5 @@ serving_proto_library( py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], - deps = [ - ":predict_proto_py_pb2", - "//third_party/py/grpc/google", - ], + deps = [":predict_proto_py_pb2"], ) diff --git a/tensorflow_serving/apis/prediction_service_pb2.py b/tensorflow_serving/apis/prediction_service_pb2.py index a9ab71f3933..a2813d44fb1 100644 --- a/tensorflow_serving/apis/prediction_service_pb2.py +++ b/tensorflow_serving/apis/prediction_service_pb2.py @@ -43,10 +43,10 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) -from grpc.google.beta import implementations as beta_implementations -from grpc.google.beta import interfaces as beta_interfaces -from grpc.google.framework.common import cardinality -from grpc.google.framework.interfaces.face import utilities as face_utilities +from grpc.beta import implementations as beta_implementations +from grpc.beta import interfaces as beta_interfaces +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities class BetaPredictionServiceServicer(object): diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 501b2361bf9..4b365b5acd8 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -48,7 +48,6 @@ py_binary( ":mnist_input_data", "//tensorflow_serving/apis:predict_proto_py_pb2", "//tensorflow_serving/apis:prediction_service_proto_py_pb2", - "//third_party/py/grpc/google", "@org_tensorflow//tensorflow:tensorflow_py", ], ) @@ -73,7 +72,6 @@ py_binary( deps = [ "//tensorflow_serving/apis:predict_proto_py_pb2", "//tensorflow_serving/apis:prediction_service_proto_py_pb2", - "//third_party/py/grpc/google", "@org_tensorflow//tensorflow:tensorflow_py", ], ) diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py index 8875a4957e1..e4e8a880210 100644 --- a/tensorflow_serving/example/inception_client.py +++ b/tensorflow_serving/example/inception_client.py @@ -20,7 +20,7 @@ # This is a placeholder for a Google-internal import. -from grpc.google.beta import implementations +from grpc.beta import implementations import tensorflow as tf from tensorflow_serving.apis import predict_pb2 diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index 5cf6bf369b8..38899eb52c7 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -30,7 +30,7 @@ # This is a placeholder for a Google-internal import. -from grpc.google.beta import implementations +from grpc.beta import implementations import numpy import tensorflow as tf diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 19fd9462cdc..7105e1f58cf 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -27,10 +27,9 @@ # This is a placeholder for a Google-internal import. -from grpc.google import * -from grpc.google.beta import implementations -from grpc.google.beta import interfaces as beta_interfaces -from grpc.google.framework.interfaces.face import face +from grpc.beta import implementations +from grpc.beta import interfaces as beta_interfaces +from grpc.framework.interfaces.face import face import tensorflow as tf from tensorflow.core.framework import types_pb2 diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py b/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py index 927b55f0523..1ab70c99e04 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test_client.py @@ -19,7 +19,7 @@ # This is a placeholder for a Google-internal import. -from grpc.google.beta import implementations +from grpc.beta import implementations import tensorflow as tf from tensorflow.core.framework import types_pb2 From 685124fb016f1ba7e0eca47dc28f150b819cd6cc Mon Sep 17 00:00:00 2001 From: Li Lao Date: Thu, 17 Nov 2016 21:06:38 -0800 Subject: [PATCH 0081/8554] Fix OSS issues. Change: 139542767 --- tensorflow_serving/servables/tensorflow/BUILD | 2 -- .../servables/tensorflow/bundle_factory_test_util.cc | 4 +--- tensorflow_serving/test_util/test_util.cc | 9 +++++++-- tensorflow_serving/test_util/test_util.h | 4 ++++ 4 files changed, 12 insertions(+), 7 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 297742dd75c..df1174da53a 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -90,7 +90,6 @@ cc_library( "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", - "//testing/base/public:gunit_for_library", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", @@ -106,7 +105,6 @@ cc_library( deps = [ ":bundle_factory_test_util", ":session_bundle_config_proto", - "//external:gtest", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:core_cpu", diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc index 71af3580696..247eabbd16c 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc @@ -19,9 +19,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/test.h" #include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/test_util/test_util.h" @@ -39,7 +37,7 @@ const char kTestSessionBundleExportPath[] = } // namespace string GetTestSavedModelPath() { - return io::JoinPath(testing::TensorFlowSrcRoot(), kTestSavedModelPath); + return test_util::TensorflowTestSrcDirPath(kTestSavedModelPath); } string GetTestSessionBundleExportPath() { diff --git a/tensorflow_serving/test_util/test_util.cc b/tensorflow_serving/test_util/test_util.cc index ec8294e8226..ebf2ac80c66 100644 --- a/tensorflow_serving/test_util/test_util.cc +++ b/tensorflow_serving/test_util/test_util.cc @@ -23,10 +23,15 @@ namespace tensorflow { namespace serving { namespace test_util { -string ContribTestSrcDirPath(const string& relative_path) { +string TensorflowTestSrcDirPath(const string& relative_path) { const string base_path = tensorflow::io::JoinPath( // getenv("TEST_SRCDIR"), // - "tf_serving/external/org_tensorflow/tensorflow/contrib/"); + "tf_serving/external/org_tensorflow/tensorflow/"); + return tensorflow::io::JoinPath(base_path, relative_path); +} + +string ContribTestSrcDirPath(const string& relative_path) { + const string base_path = TensorflowTestSrcDirPath("contrib/"); return tensorflow::io::JoinPath(base_path, relative_path); } diff --git a/tensorflow_serving/test_util/test_util.h b/tensorflow_serving/test_util/test_util.h index 3d75e9c09ef..f47f0c959ae 100644 --- a/tensorflow_serving/test_util/test_util.h +++ b/tensorflow_serving/test_util/test_util.h @@ -33,6 +33,10 @@ namespace test_util { template T CreateProto(const string& textual_proto); +// Return an absolute runfiles srcdir given a path relative to +// tensorflow. +string TensorflowTestSrcDirPath(const string& relative_path); + // Return an absolute runfiles srcdir given a path relative to // tensorflow/contrib. string ContribTestSrcDirPath(const string& relative_path); From 2308dabc50c377e53a17f90a17b594a8b4402319 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 18 Nov 2016 11:57:55 -0800 Subject: [PATCH 0082/8554] Have BatchingSession::Run() call clear() on the output tensor vector. That makes it behave the same way as regular Session::Run(). Change: 139606357 --- tensorflow_serving/batching/batching_session.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index e2deee54713..9bea6ac49bb 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -128,6 +128,8 @@ Status BatchingSession::Run( "BatchingSession does not support target nodes"); } + outputs->clear(); + Notification done; Status status; auto task = std::unique_ptr(new BatchingSessionTask); From 873494f53648f498cfde12b19b683fdab8002a31 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Fri, 18 Nov 2016 13:15:39 -0800 Subject: [PATCH 0083/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 20c3d37ecc9..dca48e8b5ad 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 20c3d37ecc9bef0e106002b9d01914efd548e66b +Subproject commit dca48e8b5adbb328c09a43b7d19300b52680d7ac diff --git a/tf_models b/tf_models index e5cae330302..33c4e784f4d 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit e5cae33030244bcdff78df3139f36f0af07f4d6a +Subproject commit 33c4e784f4d0dc2cf5e90527fd44bbe161857b9e From c4f72af52d3577f6285b1e5aee1e14a00f78a2fe Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 18 Nov 2016 15:11:47 -0800 Subject: [PATCH 0084/8554] Update batch-scheduler tuning recommendation. Change: 139628681 --- tensorflow_serving/batching/README.md | 77 ++++++++++++++++++--------- 1 file changed, 53 insertions(+), 24 deletions(-) diff --git a/tensorflow_serving/batching/README.md b/tensorflow_serving/batching/README.md index 4f2a9b8d32b..eff48c18146 100644 --- a/tensorflow_serving/batching/README.md +++ b/tensorflow_serving/batching/README.md @@ -91,30 +91,59 @@ invoked on a separate thread to process the batch. A good illustration of how to use this API is found in the implementation of `BatchingSession` in `batching_session.cc`. -The parameters that govern `BasicBatchScheduler` are: - -* `max_batch_size`: The maximum size of any batch. The largest value supported -by the hardware should be used; it will likely be model-dependent. - -* `batch_timeout_micros`: A way to bound request latency. The scheduler will -avoid delaying a task too long by processing an underfull batch, if needed. -(See `basic_batch_scheduler.h` for the exact latency contract.) A value slightly -above zero, e.g. 1 millisecond, tends to smooth out batch sizes when the request -rate is low, thus keeping tail latency low while still maintaining high -throughput. The best value to use is of course a function of your workload and -system. - -* `num_batch_threads`: The number of threads used to process batches. - -* `max_enqueued_batches`: The number of batches worth of tasks the scheduler is -willing to enqueue. For online serving with bounded latency (and the option to -route request to other server instances), you may want to set this equal to -`num_batch_threads`. For bulk processing jobs and throughput-oriented -benchmarks, you may want to set it much higher. - -(If you need to limit the possible batch sizes, as in `BatchingSession`'s -`allowed_batch_sizes` parameter, you can have your callback code pad the -batches.) +## Batch Scheduling Parameters and Tuning + +The parameters that govern batch scheduling (e.g. in +`BasicBatchScheduler::Options`) are: + +* `max_batch_size`: The maximum size of any batch. This parameter governs the +throughput/latency tradeoff, and also avoids having batches that are so large +they exceed some resource constraint (e.g. GPU memory to hold a batch's data). + +* `batch_timeout_micros`: The maximum amount of time to wait before executing a +batch (even if it hasn't reached `max_batch_size`). Used to rein in tail +latency. (See `basic_batch_scheduler.h` for the exact latency contract.) + +* `num_batch_threads`: The degree of parallelism, i.e. the maximum number of +batches processed concurrently. + +* `max_enqueued_batches`: The number of batches worth of tasks that can be +enqueued to the scheduler. Used to bound queueing delay, by turning away +requests that would take a long time to get to, rather than building up a large +backlog. + +### Recommended Approach to Choosing Scheduling Parameters + +Here is one way to choose values for the aforementioned parameters: + +1. Set `num_batch_threads` to the number of CPU cores. + +2. Temporarily set `batch_timeout_micros` and `max_enqueued_batches` to infinity +while you tune `max_batch_size` to achieve the desired balance between +throughput and average latency. The best value is typically in the hundreds or +thousands, and depends on your model, system and environment, as well as your +throughput and latency goals. + +3. For online serving, tune `batch_timeout_micros` to rein in tail latency. The +idea is that batches normally get filled to `max_batch_size`, but occasionally +when there is a lapse in incoming requests, to avoid introducing a latency spike +it makes sense to process whatever's in the queue even if it represents an +underfull batch. The best value for `batch_timeout_micros` is typically a few +milliseconds, and depends on your context and goals. Zero is a value to +consider; it works well for some workloads. (For bulk processing jobs, choose a +large value, perhaps a few seconds, to ensure good throughput but not wait too +long for the final (and likely underfull) batch.) + +4. For online serving, depending on the policy used to (re-)route requests to +server instances, consider setting `max_enqueued_batches` equal to +`num_batch_threads` to minimize queueing delay at a given server while keeping +it busy. (For bulk processing jobs, set `max_enqueued_batches` to a generous +value, but low enough to avoid out-of-memory crashes.) + +5. If you need to constrain the set of possible batch sizes (e.g. just 100, 200 +or 400, rather than any value between 1 and 400): If you are using +`BatchingSession` you can set the `allowed_batch_sizes` parameter. Otherwise, +you can arrange for your callback code to pad the batches with dummy elements. ## Servers with Multiple Models, Model Versions or Subtasks From 438e3c4a20322dd88c2bd54d427e41279d25d152 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 18 Nov 2016 16:41:04 -0800 Subject: [PATCH 0085/8554] Add unit test for predict impl. Change: 139638272 --- .../model_servers/server_core.h | 10 +- tensorflow_serving/servables/tensorflow/BUILD | 22 ++ .../servables/tensorflow/predict_impl_test.cc | 258 ++++++++++++++++++ 3 files changed, 287 insertions(+), 3 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/predict_impl_test.cc diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 6ebfcb2f0ec..3aeeed4c03b 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -161,9 +161,13 @@ class ServerCore : public Manager { Status GetServableHandle(const ModelSpec& model_spec, ServableHandle* const handle) { ServableRequest servable_request; - ServableRequestFromModelSpec(model_spec, &servable_request); - const tensorflow::Status status = - manager_->GetServableHandle(servable_request, handle); + tensorflow::Status status = + ServableRequestFromModelSpec(model_spec, &servable_request); + if (!status.ok()) { + VLOG(1) << "Unable to get servable handle due to: " << status; + return status; + } + status = manager_->GetServableHandle(servable_request, handle); if (!status.ok()) { VLOG(1) << "Unable to get servable handle due to: " << status; return status; diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index df1174da53a..4b79a36ebfa 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -384,3 +384,25 @@ cc_library( "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) + +cc_test( + name = "predict_impl_test", + size = "medium", + srcs = ["predict_impl_test.cc"], + data = [ + "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export", + "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + ], + deps = [ + ":predict_impl", + "//tensorflow_serving/core:eager_load_policy", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:test", + ], +) diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc new file mode 100644 index 00000000000..aa2990e8889 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -0,0 +1,258 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/predict_impl.h" + +#include +#include +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; + +class PredictImplTest : public ::testing::Test { + public: + static void SetUpTestCase() { + TF_ASSERT_OK(CreateServerCore( + "/servables/tensorflow/testdata/half_plus_two", &server_core_)); + TF_ASSERT_OK( + CreateServerCore("/servables/tensorflow/testdata/bad_half_plus_two", + &server_core_bad_model_)); + } + + static void TearDownTestCase() { + server_core_.reset(); + server_core_bad_model_.reset(); + } + + protected: + static Status CreateServerCore(const string& model_path, + std::unique_ptr* server_core) { + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + + ModelServerConfig config; + auto model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(test_util::TestSrcDirPath(model_path)); + model_config->set_model_platform(kTensorFlowModelPlatform); + options.model_server_config = config; + + options.aspired_version_policy = + std::unique_ptr(new EagerLoadPolicy); + return ServerCore::Create(std::move(options), server_core); + } + + static std::unique_ptr server_core_; + static std::unique_ptr server_core_bad_model_; +}; + +std::unique_ptr PredictImplTest::server_core_; +std::unique_ptr PredictImplTest::server_core_bad_model_; + +TEST_F(PredictImplTest, MissingOrEmptyModelSpec) { + PredictRequest request; + PredictResponse response; + + // Empty request is invalid. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->clear_name(); + + // Model name is not specified. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); + + // Model name is wrong, not found. + model_spec->set_name("test"); + EXPECT_EQ( + tensorflow::error::NOT_FOUND, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); +} + +TEST_F(PredictImplTest, EmptyInputList) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + // The input is empty. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); +} + +TEST_F(PredictImplTest, InputTensorsDontMatchModelSpecInputs) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + TensorProto tensor_proto; + tensor_proto.add_string_val("any_key"); + tensor_proto.set_dtype(tensorflow::DT_STRING); + tensor_proto.mutable_tensor_shape()->add_dim()->set_size(1); + + auto inputs = request.mutable_inputs(); + (*inputs)["key"] = tensor_proto; + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); +} + +TEST_F(PredictImplTest, OutputFiltersDontMatchModelSpecOutputs) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + ServableHandle bundle; + TF_ASSERT_OK(server_core_->GetServableHandle(request.model_spec(), &bundle)); + Signature signature; + TF_ASSERT_OK(GetNamedSignature("inputs", bundle->meta_graph_def, &signature)); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + + for (const auto& input : signature.generic_signature().map()) { + (*request.mutable_inputs())[input.first] = tensor_proto; + } + + request.add_output_filter("output_filter"); + + // Output filter like this doesn't exist. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); + + request.clear_output_filter(); + request.add_output_filter("y"); + EXPECT_TRUE( + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .ok()); + request.add_output_filter("y"); + + // Duplicate output filter specified. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); +} + +TEST_F(PredictImplTest, InputTensorsHaveWrongType) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + ServableHandle bundle; + TF_ASSERT_OK(server_core_->GetServableHandle(request.model_spec(), &bundle)); + + TensorProto tensor_proto; + tensor_proto.add_string_val("any_key"); + tensor_proto.set_dtype(tensorflow::DT_STRING); + tensor_proto.mutable_tensor_shape()->add_dim()->set_size(1); + + Signature signature; + TF_ASSERT_OK(GetNamedSignature("inputs", bundle->meta_graph_def, &signature)); + for (const auto& input : signature.generic_signature().map()) { + (*request.mutable_inputs())[input.first] = tensor_proto; + } + request.add_output_filter("y"); + // Input tensors are all wrong. + EXPECT_EQ( + tensorflow::error::INTERNAL, + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .code()); +} + +TEST_F(PredictImplTest, ModelMissingSignatures) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + // Model is missing signatures. + EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, + TensorflowPredictImpl::Predict(server_core_bad_model_.get(), + request, &response) + .code()); +} + +TEST_F(PredictImplTest, PredictionSuccess) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + ServableHandle bundle; + TF_ASSERT_OK(server_core_->GetServableHandle(request.model_spec(), &bundle)); + Signature signature; + TF_ASSERT_OK(GetNamedSignature("inputs", bundle->meta_graph_def, &signature)); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + + for (const auto& input : signature.generic_signature().map()) { + (*request.mutable_inputs())[input.first] = tensor_proto; + } + + EXPECT_TRUE( + TensorflowPredictImpl::Predict(server_core_.get(), request, &response) + .ok()); + TensorProto output_tensor_proto; + output_tensor_proto.add_float_val(3); + output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); + output_tensor_proto.mutable_tensor_shape(); + PredictResponse test_response; + (*test_response.mutable_outputs())["y"] = output_tensor_proto; + EXPECT_THAT(test_response, test_util::EqualsProto(response)); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From 66ac9112f194bbdec4acc4ccda50059d62b04d5a Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Mon, 21 Nov 2016 11:54:53 -0800 Subject: [PATCH 0086/8554] Merge changes from github. Change: 139813468 --- tensorflow_serving/config/BUILD | 1 + .../config/model_server_config.proto | 6 ++++ .../example/inception_client.py | 4 ++- .../example/inception_export.py | 6 ++-- ...{inception_k8s.json => inception_k8s.yaml} | 0 tensorflow_serving/example/mnist_client.py | 10 +++--- tensorflow_serving/example/mnist_export.py | 24 +++++++------ .../example/mnist_input_data.py | 13 ++++--- tensorflow_serving/g3doc/serving_inception.md | 4 +-- tensorflow_serving/model_servers/main.cc | 35 ++++++++++++++++--- .../model_servers/server_core.cc | 1 + 11 files changed, 74 insertions(+), 30 deletions(-) rename tensorflow_serving/example/{inception_k8s.json => inception_k8s.yaml} (100%) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 39917a4883f..4ea3d923b26 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -27,6 +27,7 @@ serving_proto_library( srcs = ["model_server_config.proto"], cc_api_version = 2, deps = [ + "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", "@protobuf//:cc_wkt_protos", ], ) diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index f5bbb3b655e..26b45c68650 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -4,6 +4,7 @@ package tensorflow.serving; option cc_enable_arenas = true; import "google/protobuf/any.proto"; +import "tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto"; // The type of model. // TODO(b/31336131): DEPRECATED. @@ -29,6 +30,11 @@ message ModelConfig { // Type of model (e.g. "tensorflow"). string model_platform = 4; + + // Version policy for the model indicating how many versions of the model to + // be served at the same time. + // The default option is to serve only the latest version of the model. + FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5; } // Static list of models to be loaded for serving. diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py index e4e8a880210..77899ad5ea3 100644 --- a/tensorflow_serving/example/inception_client.py +++ b/tensorflow_serving/example/inception_client.py @@ -18,6 +18,8 @@ """Send JPEG image to tensorflow_model_server loaded with inception model. """ +from __future__ import print_function + # This is a placeholder for a Google-internal import. from grpc.beta import implementations @@ -46,7 +48,7 @@ def main(_): request.inputs['images'].CopyFrom( tf.contrib.util.make_tensor_proto(data, shape=[1])) result = stub.Predict(request, 10.0) # 10 secs timeout - print result + print(result) if __name__ == '__main__': diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index 252b3f60d63..03df5f78998 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -20,6 +20,8 @@ tensorflow_model_server. """ +from __future__ import print_function + import os.path # This is a placeholder for a Google-internal import. @@ -108,7 +110,7 @@ def export(): print('Successfully loaded model from %s at step=%s.' % (ckpt.model_checkpoint_path, global_step)) else: - print 'No checkpoint file found at %s' % FLAGS.checkpoint_dir + print('No checkpoint file found at %s' % FLAGS.checkpoint_dir) return # Export inference model. @@ -129,7 +131,7 @@ def export(): default_graph_signature=classification_signature, named_graph_signatures=named_graph_signature) model_exporter.export(FLAGS.export_dir, tf.constant(global_step), sess) - print 'Successfully exported model to %s' % FLAGS.export_dir + print('Successfully exported model to %s' % FLAGS.export_dir) def preprocess_image(image_buffer): diff --git a/tensorflow_serving/example/inception_k8s.json b/tensorflow_serving/example/inception_k8s.yaml similarity index 100% rename from tensorflow_serving/example/inception_k8s.json rename to tensorflow_serving/example/inception_k8s.yaml diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index 38899eb52c7..a93ffd7e374 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -25,6 +25,8 @@ mnist_client.py --num_tests=100 --server=localhost:9000 """ +from __future__ import print_function + import sys import threading @@ -105,7 +107,7 @@ def _callback(result_future): exception = result_future.exception() if exception: result_counter.inc_error() - print exception + print(exception) else: sys.stdout.write('.') sys.stdout.flush() @@ -154,14 +156,14 @@ def do_inference(hostport, work_dir, concurrency, num_tests): def main(_): if FLAGS.num_tests > 10000: - print 'num_tests should not be greater than 10k' + print('num_tests should not be greater than 10k') return if not FLAGS.server: - print 'please specify server host:port' + print('please specify server host:port') return error_rate = do_inference(FLAGS.server, FLAGS.work_dir, FLAGS.concurrency, FLAGS.num_tests) - print '\nInference error rate: %s%%' % (error_rate * 100) + print('\nInference error rate: %s%%' % (error_rate * 100)) if __name__ == '__main__': diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 08fae18e0a5..0c008b3bb42 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -25,6 +25,8 @@ Usage: mnist_export.py [--training_iteration=x] [--export_version=y] export_dir """ +from __future__ import print_function + import sys # This is a placeholder for a Google-internal import. @@ -47,14 +49,14 @@ def main(_): '[--export_version=y] export_dir') sys.exit(-1) if FLAGS.training_iteration <= 0: - print 'Please specify a positive value for training iteration.' + print('Please specify a positive value for training iteration.') sys.exit(-1) if FLAGS.export_version <= 0: - print 'Please specify a positive value for version number.' + print('Please specify a positive value for version number.') sys.exit(-1) # Train model - print 'Training model...' + print('Training model...') mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) sess = tf.InteractiveSession() serialized_tf_example = tf.placeholder(tf.string, name='tf_example') @@ -72,24 +74,24 @@ def main(_): train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) values, indices = tf.nn.top_k(y, 10) prediction_classes = tf.contrib.lookup.index_to_string( - tf.to_int64(indices), - mapping=tf.constant([str(i) for i in xrange(10)])) + tf.to_int64(indices), mapping=tf.constant([str(i) for i in range(10)])) for _ in range(FLAGS.training_iteration): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x: batch[0], y_: batch[1]}) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) - print 'training accuracy %g' % sess.run(accuracy, - feed_dict={x: mnist.test.images, - y_: mnist.test.labels}) - print 'Done training!' + print('training accuracy %g' % + sess.run(accuracy, + feed_dict={x: mnist.test.images, + y_: mnist.test.labels})) + print('Done training!') # Export model # WARNING(break-tutorial-inline-code): The following code snippet is # in-lined in tutorials, please update tutorial documents accordingly # whenever code changes. export_path = sys.argv[-1] - print 'Exporting trained model to', export_path + print('Exporting trained model to %s' % export_path) init_op = tf.group(tf.initialize_all_tables(), name='init_op') saver = tf.train.Saver(sharded=True) model_exporter = exporter.Exporter(saver) @@ -104,7 +106,7 @@ def main(_): 'inputs': exporter.generic_signature({'images': x}), 'outputs': exporter.generic_signature({'scores': y})}) model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess) - print 'Done exporting!' + print('Done exporting!') if __name__ == '__main__': diff --git a/tensorflow_serving/example/mnist_input_data.py b/tensorflow_serving/example/mnist_input_data.py index 66693e059d2..6a532cb2197 100644 --- a/tensorflow_serving/example/mnist_input_data.py +++ b/tensorflow_serving/example/mnist_input_data.py @@ -17,6 +17,8 @@ """Functions for downloading and reading MNIST data.""" +from __future__ import print_function + import gzip import os @@ -39,7 +41,7 @@ def maybe_download(filename, work_directory): if not os.path.exists(filepath): filepath, _ = urllib.request.urlretrieve(SOURCE_URL + filename, filepath) statinfo = os.stat(filepath) - print('Successfully downloaded', filename, statinfo.st_size, 'bytes.') + print('Successfully downloaded %s %d bytes.' % (filename, statinfo.st_size)) return filepath @@ -50,7 +52,7 @@ def _read32(bytestream): def extract_images(filename): """Extract the images into a 4D uint8 numpy array [index, y, x, depth].""" - print('Extracting', filename) + print('Extracting %s' % filename) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2051: @@ -77,7 +79,7 @@ def dense_to_one_hot(labels_dense, num_classes=10): def extract_labels(filename, one_hot=False): """Extract the labels into a 1D uint8 numpy array [index].""" - print('Extracting', filename) + print('Extracting %s' % filename) with gzip.open(filename) as bytestream: magic = _read32(bytestream) if magic != 2049: @@ -144,8 +146,9 @@ def next_batch(self, batch_size, fake_data=False): fake_label = [1] + [0] * 9 else: fake_label = 0 - return [fake_image for _ in xrange(batch_size)], [ - fake_label for _ in xrange(batch_size)] + return [fake_image for _ in range(batch_size)], [ + fake_label for _ in range(batch_size) + ] start = self._index_in_epoch self._index_in_epoch += batch_size if self._index_in_epoch > self._num_examples: diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 38749c73cf3..9447c40fae7 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -227,10 +227,10 @@ along with an [External Load Balancer](http://kubernetes.io/docs/user-guide/load-balancer/). We create them using the example Kubernetes config -[inception_k8s.json](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/inception_k8s.json). +[inception_k8s.yaml](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/inception_k8s.yaml). ```shell -$ kubectl create -f tensorflow_serving/example/inception_k8s.json +$ kubectl create -f tensorflow_serving/example/inception_k8s.yaml deployment "inception-deployment" created service "inception-service" created ``` diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index ce29981de5b..d21a7cf5a86 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -73,6 +73,9 @@ using tensorflow::serving::AspiredVersionPolicy; using tensorflow::serving::BatchingParameters; using tensorflow::serving::EagerLoadPolicy; using tensorflow::serving::EventBus; +using tensorflow::serving::FileSystemStoragePathSourceConfig; +using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy; +using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy_Name; using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; @@ -121,17 +124,21 @@ tensorflow::Status LoadCustomModelConfig( << "ModelServer does not yet support custom model config."; } -ModelServerConfig BuildSingleModelConfig(const string& model_name, - const string& model_base_path) { +ModelServerConfig BuildSingleModelConfig( + const string& model_name, const string& model_base_path, + const FileSystemStoragePathSourceConfig_VersionPolicy& + model_version_policy) { ModelServerConfig config; LOG(INFO) << "Building single TensorFlow model file config: " << " model_name: " << model_name - << " model_base_path: " << model_base_path; + << " model_base_path: " << model_base_path + << " model_version_policy: " << model_version_policy; tensorflow::serving::ModelConfig* single_model = config.mutable_model_config_list()->add_config(); single_model->set_name(model_name); single_model->set_base_path(model_base_path); single_model->set_model_platform(kTensorFlowModelPlatform); + single_model->set_version_policy(model_version_policy); return config; } @@ -188,10 +195,20 @@ int main(int argc, char** argv) { tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; + tensorflow::string model_version_policy = + FileSystemStoragePathSourceConfig_VersionPolicy_Name( + FileSystemStoragePathSourceConfig::LATEST_VERSION); std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag( + "model_version_policy", &model_version_policy, + "The version policy which determines the number of model versions to " + "be served at the same time. The default value is LATEST_VERSION, " + "which will serve only the latest version. See " + "file_system_storage_path_source.proto for the list of possible " + "VersionPolicy."), tensorflow::Flag("file_system_poll_wait_seconds", &file_system_poll_wait_seconds, "interval in seconds between each poll of the file " @@ -209,11 +226,19 @@ int main(int argc, char** argv) { std::cout << "unknown argument: " << argv[1] << "\n" << usage; } + FileSystemStoragePathSourceConfig_VersionPolicy parsed_version_policy; + bool valid_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Parse( + model_version_policy, &parsed_version_policy); + QCHECK(valid_policy) // Crash ok. + << "Invalid model_version_policy input argument: " << model_version_policy + << "\n" + << usage; + // For ServerCore Options, we leave servable_state_monitor_creator unspecified // so the default servable_state_monitor_creator will be used. ServerCore::Options options; - options.model_server_config = - BuildSingleModelConfig(model_name, model_base_path); + options.model_server_config = BuildSingleModelConfig( + model_name, model_base_path, parsed_version_policy); SessionBundleSourceAdapterConfig source_adapter_config; // Batching config diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index fbd9d7f3b2c..831d9e3f4cf 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -292,6 +292,7 @@ FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( source_config.add_servables(); servable->set_servable_name(model.name()); servable->set_base_path(model.base_path()); + servable->set_version_policy(model.version_policy()); } return source_config; } From 3805c635ea6dcf0823ca3e2a1966e47c7d4bf55f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 22 Nov 2016 11:44:08 -0800 Subject: [PATCH 0087/8554] Allow SourceRouter to handle non default constructable types by changing IdentitySourceAdapter to use SourceAdapter as base class and implement Adapt() method. This removes the dependence on the UnarySourceAdapter, which requires that the output type be default contructable. Change: 139935668 --- tensorflow_serving/core/source_router.h | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/tensorflow_serving/core/source_router.h b/tensorflow_serving/core/source_router.h index 51bf8f0ab17..94e6fc3d478 100644 --- a/tensorflow_serving/core/source_router.h +++ b/tensorflow_serving/core/source_router.h @@ -96,21 +96,25 @@ namespace internal { // A SourceAdapter that passes through data unchanged. Used to implement the // output ports. template -class IdentitySourceAdapter final : public UnarySourceAdapter { +class IdentitySourceAdapter final : public SourceAdapter { public: IdentitySourceAdapter() = default; ~IdentitySourceAdapter() override { TargetBase::Detach(); } - protected: - Status Convert(const T& data, T* converted_data) override { - *converted_data = data; - return Status::OK(); - } - private: + std::vector> Adapt( + const StringPiece servable_name, + std::vector> versions) final; + TF_DISALLOW_COPY_AND_ASSIGN(IdentitySourceAdapter); }; +template +std::vector> IdentitySourceAdapter::Adapt( + const StringPiece servable_name, std::vector> versions) { + return versions; +} + } // namespace internal template From 980f7cfcc9c884fcee7f2d90b3e113f5d8050d47 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Wed, 23 Nov 2016 21:40:02 -0800 Subject: [PATCH 0088/8554] Plug SavedModelBundleSourceAdapter into server core and model server. Change: 140101220 --- tensorflow_serving/model_servers/BUILD | 6 +-- tensorflow_serving/model_servers/main.cc | 44 ++++++++++++++----- .../model_servers/server_core.cc | 18 +++++--- .../model_servers/server_core.h | 14 +++++- .../model_servers/server_core_test.cc | 18 +++++--- .../model_servers/test_util/BUILD | 1 + .../test_util/server_core_test_util.cc | 15 ++++++- .../test_util/server_core_test_util.h | 22 +++++++++- 8 files changed, 107 insertions(+), 31 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 6544a33ba51..55a9719cfc0 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -48,6 +48,7 @@ cc_library( "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", "//tensorflow_serving/resources:resource_values", + "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source", @@ -63,10 +64,6 @@ cc_test( name = "server_core_test", size = "medium", srcs = ["server_core_test.cc"], - data = [ - "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", - "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", - ], deps = [ ":server_core", "//tensorflow_serving/apis:model_proto", @@ -87,6 +84,7 @@ SUPPORTED_TENSORFLOW_OPS = [ TENSORFLOW_DEPS = [ "@org_tensorflow//tensorflow/core:tensorflow", + "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:predict_impl", ] diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index d21a7cf5a86..234c9b1e94d 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -66,6 +66,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" using tensorflow::serving::AspiredVersionsManager; @@ -80,6 +81,7 @@ using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; using tensorflow::serving::ServerCore; +using tensorflow::serving::SavedModelBundleSourceAdapter; using tensorflow::serving::SessionBundleSourceAdapter; using tensorflow::serving::SessionBundleSourceAdapterConfig; using tensorflow::serving::Target; @@ -101,18 +103,29 @@ namespace { tensorflow::Status CreateSourceAdapter( const SessionBundleSourceAdapterConfig& config, - const string& model_platform, + const string& model_platform, bool use_saved_model, std::unique_ptr* adapter) { CHECK(model_platform == kTensorFlowModelPlatform) // Crash ok << "ModelServer supports only TensorFlow model."; - std::unique_ptr typed_adapter; - const ::tensorflow::Status status = - SessionBundleSourceAdapter::Create(config, &typed_adapter); - if (!status.ok()) { - VLOG(1) << "Error creating source adapter: " << status.error_message(); - return status; + if (use_saved_model) { + std::unique_ptr typed_adapter; + const ::tensorflow::Status status = + SavedModelBundleSourceAdapter::Create(config, &typed_adapter); + if (!status.ok()) { + VLOG(1) << "Error creating source adapter: " << status.error_message(); + return status; + } + *adapter = std::move(typed_adapter); + } else { + std::unique_ptr typed_adapter; + const ::tensorflow::Status status = + SessionBundleSourceAdapter::Create(config, &typed_adapter); + if (!status.ok()) { + VLOG(1) << "Error creating source adapter: " << status.error_message(); + return status; + } + *adapter = std::move(typed_adapter); } - *adapter = std::move(typed_adapter); return tensorflow::Status::OK(); } @@ -195,6 +208,7 @@ int main(int argc, char** argv) { tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; + bool use_saved_model = false; tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( FileSystemStoragePathSourceConfig::LATEST_VERSION); @@ -214,7 +228,12 @@ int main(int argc, char** argv) { "interval in seconds between each poll of the file " "system for new model version"), tensorflow::Flag("model_base_path", &model_base_path, - "path to export (required)")}; + "path to export (required)"), + tensorflow::Flag("use_saved_model", &use_saved_model, + "If true, use SavedModel in the server; otherwise, use " + "SessionBundle. It is used by tensorflow serving team " + "to control the rollout of SavedModel and is not " + "expected to be set by users directly.")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || model_base_path.empty()) { @@ -248,10 +267,13 @@ int main(int argc, char** argv) { batching_parameters->mutable_thread_pool_name()->set_value( "model_server_batch_threads"); } - options.source_adapter_creator = [source_adapter_config]( + + options.use_saved_model = use_saved_model; + options.source_adapter_creator = [source_adapter_config, use_saved_model]( const string& model_platform, std::unique_ptr* adapter) { - return CreateSourceAdapter(source_adapter_config, model_platform, adapter); + return CreateSourceAdapter(source_adapter_config, model_platform, + use_saved_model, adapter); }; options.custom_model_config_loader = &LoadCustomModelConfig; diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 831d9e3f4cf..58b3acaecb7 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow_serving/core/load_servables_fast.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/resources/resource_values.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" @@ -85,7 +86,7 @@ Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, Status ServerCore::Create(Options options, std::unique_ptr* server_core) { if (options.source_adapter_creator == nullptr) { - options.source_adapter_creator = []( + options.source_adapter_creator = [&options]( const string& model_platform, std::unique_ptr* adapter) { SessionBundleSourceAdapterConfig source_adapter_config; @@ -93,10 +94,17 @@ Status ServerCore::Create(Options options, return errors::InvalidArgument( "ModelServer supports only TensorFlow model."); } - std::unique_ptr typed_adapter; - TF_RETURN_IF_ERROR(SessionBundleSourceAdapter::Create( - source_adapter_config, &typed_adapter)); - *adapter = std::move(typed_adapter); + if (options.use_saved_model) { + std::unique_ptr typed_adapter; + TF_RETURN_IF_ERROR(SavedModelBundleSourceAdapter::Create( + source_adapter_config, &typed_adapter)); + *adapter = std::move(typed_adapter); + } else { + std::unique_ptr typed_adapter; + TF_RETURN_IF_ERROR(SessionBundleSourceAdapter::Create( + source_adapter_config, &typed_adapter)); + *adapter = std::move(typed_adapter); + } return Status::OK(); }; } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 3aeeed4c03b..5fce83ac8b9 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -109,9 +109,21 @@ class ServerCore : public Manager { // A function for creating ModelServerSourceAdapter based on the // 'platform_type'. // If not specified, a default creator that creates - // SessionBundleSourceAdapter for TensorFlow will be used. + // SessionBundleSourceAdapter or SavedModelBundleSourceAdapter for + // TensorFlow will be used, depending on use_saved_model. SourceAdapterCreator source_adapter_creator; + // Whether to use SavedModelBundle or SessionBundle. If + // source_adapter_creator is not specified, SavedModelBundleSourceAdapter + // will be created when this option is true and SessionBundleSourceAdapter + // will be created when it is false. If source_adapter_creator is specified, + // the creator will be used and this option will be ignored. + // This option is used by tensorflow serving team to control the rollout of + // SavedModelBundle and is not expected to be set by users directly. + // It should always be set to false (except for tests) until + // SavedModelBundle is supported by the service API implementation. + bool use_saved_model = false; + // A function for creating ServableStateMonitor. If not specified, a default // creator that creates ServableStateMonitor will be used. ServableStateMonitorCreator servable_state_monitor_creator; diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 12669869cee..d611188332a 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -29,7 +29,7 @@ namespace { using test_util::ServerCoreTest; -TEST_F(ServerCoreTest, CreateWaitsTillModelsAvailable) { +TEST_P(ServerCoreTest, CreateWaitsTillModelsAvailable) { std::unique_ptr server_core; TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfig(), &server_core)); @@ -49,7 +49,7 @@ TEST_F(ServerCoreTest, CreateWaitsTillModelsAvailable) { EXPECT_EQ(servable_handle.id(), expected_id); } -TEST_F(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { +TEST_P(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { // Create a server with no models, initially. std::unique_ptr server_core; TF_ASSERT_OK(CreateServerCore(ModelServerConfig(), &server_core)); @@ -65,7 +65,7 @@ TEST_F(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { EXPECT_EQ(available_servables.at(0), expected_id); } -TEST_F(ServerCoreTest, ErroringModel) { +TEST_P(ServerCoreTest, ErroringModel) { std::unique_ptr server_core; Status status = CreateServerCore( GetTestModelServerConfig(), @@ -82,7 +82,7 @@ TEST_F(ServerCoreTest, ErroringModel) { EXPECT_FALSE(status.ok()); } -TEST_F(ServerCoreTest, IllegalReconfigurationToCustomConfig) { +TEST_P(ServerCoreTest, IllegalReconfigurationToCustomConfig) { // Create a ServerCore with ModelConfigList config. std::unique_ptr server_core; TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfig(), &server_core)); @@ -95,7 +95,7 @@ TEST_F(ServerCoreTest, IllegalReconfigurationToCustomConfig) { ::testing::HasSubstr("Cannot transition to requested config")); } -TEST_F(ServerCoreTest, IllegalReconfigurationFromCustomConfig) { +TEST_P(ServerCoreTest, IllegalReconfigurationFromCustomConfig) { // Create a ServerCore with custom config. std::unique_ptr server_core; ModelServerConfig config; @@ -108,7 +108,7 @@ TEST_F(ServerCoreTest, IllegalReconfigurationFromCustomConfig) { ::testing::HasSubstr("Cannot transition to requested config")); } -TEST_F(ServerCoreTest, IllegalConfigModelTypeAndPlatformSet) { +TEST_P(ServerCoreTest, IllegalConfigModelTypeAndPlatformSet) { // Create a ServerCore with both model_type and model_platform set. std::unique_ptr server_core; ModelServerConfig config = GetTestModelServerConfig(); @@ -118,7 +118,7 @@ TEST_F(ServerCoreTest, IllegalConfigModelTypeAndPlatformSet) { ::testing::HasSubstr("Illegal setting both")); } -TEST_F(ServerCoreTest, DeprecatedModelTypeConfig) { +TEST_P(ServerCoreTest, DeprecatedModelTypeConfig) { // Create a ServerCore with deprecated config. std::unique_ptr server_core; ModelServerConfig config = GetTestModelServerConfig(); @@ -135,6 +135,10 @@ TEST_F(ServerCoreTest, DeprecatedModelTypeConfig) { EXPECT_EQ(available_servables.at(0), expected_id); } +INSTANTIATE_TEST_CASE_P( + TestType, ServerCoreTest, + ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES))); + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 1b02c54bc1b..7e59c205aef 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -48,6 +48,7 @@ cc_library( data = [ "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "@org_tensorflow//tensorflow/python/saved_model/example:versioned_saved_model_half_plus_two_data", ], visibility = [ "//visibility:public", diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index fcd848d0fbe..561d89e8c43 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" +#include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/model_servers/model_platform_types.h" @@ -28,8 +29,13 @@ ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { ModelServerConfig config; auto model = config.mutable_model_config_list()->add_config(); model->set_name(kTestModelName); - model->set_base_path(test_util::TestSrcDirPath( - "/servables/tensorflow/testdata/half_plus_two")); + if (GetTestType() == SAVED_MODEL) { + model->set_base_path(test_util::TensorflowTestSrcDirPath( + "/python/saved_model/example/saved_model_half_plus_two")); + } else { + model->set_base_path(test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/half_plus_two")); + } model->set_model_platform(kTensorFlowModelPlatform); return config; } @@ -64,6 +70,11 @@ Status ServerCoreTest::CreateServerCore( options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); } + TestType test_type = GetTestType(); + if (test_type == SAVED_MODEL || + test_type == SAVED_MODEL_BACKWARD_COMPATIBILITY) { + options.use_saved_model = true; + } return ServerCore::Create(std::move(options), server_core); } diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index 0f5d5d4e624..63d2285c943 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -27,7 +27,23 @@ namespace test_util { constexpr char kTestModelName[] = "test_model"; constexpr int kTestModelVersion = 123; -class ServerCoreTest : public ::testing::Test { +// ServerCoreTest is parameterized based on the TestType enum defined below. +// TODO(b/32248363): remove the parameter and TestType after we switch Model +// Server to Saved Model. +class ServerCoreTest : public ::testing::TestWithParam { + public: + // The parameter of this test. + enum TestType { + // SessionBundle is used on export. + SESSION_BUNDLE, + // SavedModelBundle is used on export. + SAVED_MODEL_BACKWARD_COMPATIBILITY, + // SavedModelBundle is used on native Saved Model. + SAVED_MODEL, + // This should always be the last value. + NUM_TEST_TYPES, + }; + protected: // Returns ModelServerConfig that contains test model. ModelServerConfig GetTestModelServerConfig(); @@ -46,9 +62,13 @@ class ServerCoreTest : public ::testing::Test { // continuous polling to speed up testing. Status CreateServerCore(ServerCore::Options options, std::unique_ptr* server_core); + + // Returns test type. This is the parameter of this test. + TestType GetTestType() { return static_cast(GetParam()); } }; } // namespace test_util } // namespace serving } // namespace tensorflow + #endif // TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ From 151c4c48a071ab767c551673b491a11b04a5f9b5 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 28 Nov 2016 16:06:33 -0800 Subject: [PATCH 0089/8554] Make the Predict API support SavedModel in addition to the existing SessionBundle. Change: 140414357 --- tensorflow_serving/model_servers/main.cc | 20 ++- tensorflow_serving/servables/tensorflow/BUILD | 3 + .../servables/tensorflow/predict_impl.cc | 146 +++++++++++++++- .../servables/tensorflow/predict_impl.h | 16 +- .../servables/tensorflow/predict_impl_test.cc | 164 +++++++++--------- 5 files changed, 246 insertions(+), 103 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 234c9b1e94d..af87d886066 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -85,7 +85,7 @@ using tensorflow::serving::SavedModelBundleSourceAdapter; using tensorflow::serving::SessionBundleSourceAdapter; using tensorflow::serving::SessionBundleSourceAdapterConfig; using tensorflow::serving::Target; -using tensorflow::serving::TensorflowPredictImpl; +using tensorflow::serving::TensorflowPredictor; using tensorflow::serving::UniquePtrWithDeps; using tensorflow::string; @@ -170,13 +170,15 @@ grpc::Status ToGRPCStatus(const tensorflow::Status& status) { class PredictionServiceImpl final : public PredictionService::Service { public: - explicit PredictionServiceImpl(std::unique_ptr core) - : core_(std::move(core)) {} + explicit PredictionServiceImpl(std::unique_ptr core, + bool use_saved_model) + : core_(std::move(core)), + predictor_(new TensorflowPredictor(use_saved_model)) {} grpc::Status Predict(ServerContext* context, const PredictRequest* request, PredictResponse* response) override { - const grpc::Status status = ToGRPCStatus( - TensorflowPredictImpl::Predict(core_.get(), *request, response)); + const grpc::Status status = + ToGRPCStatus(predictor_->Predict(core_.get(), *request, response)); if (!status.ok()) { VLOG(1) << "Predict failed: " << status.error_message(); } @@ -185,12 +187,14 @@ class PredictionServiceImpl final : public PredictionService::Service { private: std::unique_ptr core_; + std::unique_ptr predictor_; }; -void RunServer(int port, std::unique_ptr core) { +void RunServer(int port, std::unique_ptr core, + bool use_saved_model) { // "0.0.0.0" is the way to listen on localhost in gRPC. const string server_address = "0.0.0.0:" + std::to_string(port); - PredictionServiceImpl service(std::move(core)); + PredictionServiceImpl service(std::move(core), use_saved_model); ServerBuilder builder; std::shared_ptr creds = InsecureServerCredentials(); builder.AddListeningPort(server_address, creds); @@ -284,7 +288,7 @@ int main(int argc, char** argv) { std::unique_ptr core; TF_CHECK_OK(ServerCore::Create(std::move(options), &core)); - RunServer(port, std::move(core)); + RunServer(port, std::move(core), use_saved_model); return 0; } diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 4b79a36ebfa..66017aaf400 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -379,7 +379,10 @@ cc_library( "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/model_servers:server_core", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:signature", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", ], diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index 411f7494f18..6b4e005db48 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -21,6 +21,8 @@ limitations under the License. #include #include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/signature_constants.h" #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/contrib/session_bundle/signature.h" #include "tensorflow/core/lib/core/errors.h" @@ -29,15 +31,11 @@ limitations under the License. namespace tensorflow { namespace serving { +namespace { -Status TensorflowPredictImpl::Predict(ServerCore* core, - const PredictRequest& request, - PredictResponse* response) { - if (!request.has_model_spec()) { - return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, - "Missing ModelSpec"); - } - +// Implementation of Predict using the legacy SessionBundle GenericSignature. +Status SessionBundlePredict(ServerCore* core, const PredictRequest& request, + PredictResponse* response) { // Validate signatures. ServableHandle bundle; TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); @@ -129,5 +127,137 @@ Status TensorflowPredictImpl::Predict(ServerCore* core, return Status::OK(); } +// Validate a SignatureDef to make sure it's compatible with prediction, and +// if so, populate the input and output tensor names. +Status PreProcessPrediction(const SignatureDef& signature, + const PredictRequest& request, + std::vector>* inputs, + std::vector* output_tensor_names, + std::vector* output_tensor_aliases) { + if (signature.method_name() != kPredictMethodName) { + return errors::Internal(strings::StrCat( + "Expected prediction signature method_name to be ", kPredictMethodName, + ". Was: ", signature.method_name())); + } + if (signature.inputs().empty()) { + return errors::Internal(strings::StrCat( + "Expected at least one input Tensor in prediction signature.")); + } + if (signature.outputs().empty()) { + return errors::Internal(strings::StrCat( + "Expected at least one output Tensor in prediction signature.")); + } + + // Verify and prepare input. + if (request.inputs().size() != signature.inputs().size()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "input size does not match signature"); + } + for (auto& input : request.inputs()) { + const string& alias = input.first; + auto iter = signature.inputs().find(alias); + if (iter == signature.inputs().end()) { + return tensorflow::Status( + tensorflow::error::INVALID_ARGUMENT, + "input tensor alias not found in signature: " + alias); + } + Tensor tensor; + if (!tensor.FromProto(input.second)) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "tensor parsing error: " + alias); + } + inputs->emplace_back(std::make_pair(iter->second.name(), tensor)); + } + + // Prepare run target. + std::set seen_outputs; + std::vector output_filter(request.output_filter().begin(), + request.output_filter().end()); + for (auto& alias : output_filter) { + auto iter = signature.outputs().find(alias); + if (iter == signature.outputs().end()) { + return tensorflow::Status( + tensorflow::error::INVALID_ARGUMENT, + "output tensor alias not found in signature: " + alias); + } + if (seen_outputs.find(alias) != seen_outputs.end()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "duplicate output tensor alias: " + alias); + } + seen_outputs.insert(alias); + output_tensor_names->emplace_back(iter->second.name()); + output_tensor_aliases->emplace_back(alias); + } + // When no output is specified, fetch all output tensors specified in + // the signature. + if (output_tensor_names->empty()) { + for (auto& iter : signature.outputs()) { + output_tensor_names->emplace_back(iter.second.name()); + output_tensor_aliases->emplace_back(iter.first); + } + } + return Status::OK(); +} + +// Validate results and populate a PredictResponse. +Status PostProcessPredictionResult( + const SignatureDef& signature, + const std::vector& output_tensor_aliases, + const std::vector& output_tensors, PredictResponse* response) { + // Validate and return output. + if (output_tensors.size() != output_tensor_aliases.size()) { + return tensorflow::Status(tensorflow::error::UNKNOWN, + "Predict internal error"); + } + for (int i = 0; i < output_tensors.size(); i++) { + output_tensors[i].AsProtoField( + &((*response->mutable_outputs())[output_tensor_aliases[i]])); + } + return Status::OK(); +} + +// Implementation of Predict using the SavedModel SignatureDef format. +Status SavedModelPredict(ServerCore* core, const PredictRequest& request, + PredictResponse* response) { + // Validate signatures. + ServableHandle bundle; + TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); + auto iter = bundle->meta_graph_def.signature_def().find( + kDefaultServingSignatureDefKey); + if (iter == bundle->meta_graph_def.signature_def().end()) { + return errors::FailedPrecondition( + "Default serving signature key not found."); + } + SignatureDef signature = iter->second; + + std::vector> input_tensors; + std::vector output_tensor_names; + std::vector output_tensor_aliases; + TF_RETURN_IF_ERROR(PreProcessPrediction(signature, request, &input_tensors, + &output_tensor_names, + &output_tensor_aliases)); + std::vector outputs; + TF_RETURN_IF_ERROR( + bundle->session->Run(input_tensors, output_tensor_names, {}, &outputs)); + + return PostProcessPredictionResult(signature, output_tensor_aliases, outputs, + response); +} + +} // namespace + +Status TensorflowPredictor::Predict(ServerCore* core, + const PredictRequest& request, + PredictResponse* response) { + if (!request.has_model_spec()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Missing ModelSpec"); + } + if (use_saved_model_) { + return SavedModelPredict(core, request, response); + } + return SessionBundlePredict(core, request, response); +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.h b/tensorflow_serving/servables/tensorflow/predict_impl.h index 65442d462bf..d23718eeb35 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.h +++ b/tensorflow_serving/servables/tensorflow/predict_impl.h @@ -24,11 +24,19 @@ namespace tensorflow { namespace serving { // Utility methods for implementation of PredictionService::Predict. -// TODO(30682232): add unit test for TensorflowPredictImpl. -class TensorflowPredictImpl { +class TensorflowPredictor { public: - static Status Predict(ServerCore* core, const PredictRequest& request, - PredictResponse* response); + explicit TensorflowPredictor(bool use_saved_model) + : use_saved_model_(use_saved_model) {} + + Status Predict(ServerCore* core, const PredictRequest& request, + PredictResponse* response); + + private: + // If use_saved_model_ is true, a SavedModelBundle handle will be retrieved + // from the ServerCore and the new SavedModel SignatureDef format will be + // used. + bool use_saved_model_; }; } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index aa2990e8889..eaba1d6d34c 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -30,23 +30,36 @@ namespace { constexpr char kTestModelName[] = "test_model"; constexpr int kTestModelVersion = 123; -class PredictImplTest : public ::testing::Test { +const char kInputTensorKey[] = "x"; +const char kOutputTensorKey[] = "y"; + +// Parameter is 'bool use_saved_model'. +class PredictImplTest : public ::testing::TestWithParam { public: static void SetUpTestCase() { TF_ASSERT_OK(CreateServerCore( - "/servables/tensorflow/testdata/half_plus_two", &server_core_)); + "/servables/tensorflow/testdata/half_plus_two", false, &server_core_)); + TF_ASSERT_OK( + CreateServerCore("/servables/tensorflow/testdata/bad_half_plus_two", + false, &server_core_bad_model_)); + + TF_ASSERT_OK( + CreateServerCore("/servables/tensorflow/testdata/half_plus_two", true, + &saved_model_server_core_)); TF_ASSERT_OK( CreateServerCore("/servables/tensorflow/testdata/bad_half_plus_two", - &server_core_bad_model_)); + true, &saved_model_server_core_bad_model_)); } static void TearDownTestCase() { server_core_.reset(); server_core_bad_model_.reset(); + saved_model_server_core_.reset(); + saved_model_server_core_bad_model_.reset(); } protected: - static Status CreateServerCore(const string& model_path, + static Status CreateServerCore(const string& model_path, bool use_saved_model, std::unique_ptr* server_core) { // For ServerCore Options, we leave servable_state_monitor_creator // unspecified so the default servable_state_monitor_creator will be used. @@ -58,61 +71,75 @@ class PredictImplTest : public ::testing::Test { model_config->set_base_path(test_util::TestSrcDirPath(model_path)); model_config->set_model_platform(kTensorFlowModelPlatform); options.model_server_config = config; - + options.use_saved_model = use_saved_model; options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); return ServerCore::Create(std::move(options), server_core); } + ServerCore* GetServerCore() { + if (GetParam()) { + return saved_model_server_core_.get(); + } + return server_core_.get(); + } + + ServerCore* GetServerCoreWithBadModel() { + if (GetParam()) { + return saved_model_server_core_bad_model_.get(); + } + return server_core_bad_model_.get(); + } + + private: static std::unique_ptr server_core_; static std::unique_ptr server_core_bad_model_; + static std::unique_ptr saved_model_server_core_; + static std::unique_ptr saved_model_server_core_bad_model_; }; std::unique_ptr PredictImplTest::server_core_; std::unique_ptr PredictImplTest::server_core_bad_model_; +std::unique_ptr PredictImplTest::saved_model_server_core_; +std::unique_ptr PredictImplTest::saved_model_server_core_bad_model_; -TEST_F(PredictImplTest, MissingOrEmptyModelSpec) { +TEST_P(PredictImplTest, MissingOrEmptyModelSpec) { PredictRequest request; PredictResponse response; // Empty request is invalid. - EXPECT_EQ( - tensorflow::error::INVALID_ARGUMENT, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + TensorflowPredictor predictor(GetParam()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetServerCore(), request, &response).code()); ModelSpec* model_spec = request.mutable_model_spec(); model_spec->clear_name(); // Model name is not specified. - EXPECT_EQ( - tensorflow::error::INVALID_ARGUMENT, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetServerCore(), request, &response).code()); // Model name is wrong, not found. model_spec->set_name("test"); - EXPECT_EQ( - tensorflow::error::NOT_FOUND, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + EXPECT_EQ(tensorflow::error::NOT_FOUND, + predictor.Predict(GetServerCore(), request, &response).code()); } -TEST_F(PredictImplTest, EmptyInputList) { +TEST_P(PredictImplTest, EmptyInputList) { PredictRequest request; PredictResponse response; ModelSpec* model_spec = request.mutable_model_spec(); model_spec->set_name(kTestModelName); model_spec->mutable_version()->set_value(kTestModelVersion); + + TensorflowPredictor predictor(GetParam()); // The input is empty. - EXPECT_EQ( - tensorflow::error::INVALID_ARGUMENT, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetServerCore(), request, &response).code()); } -TEST_F(PredictImplTest, InputTensorsDontMatchModelSpecInputs) { +TEST_P(PredictImplTest, InputTensorsDontMatchModelSpecInputs) { PredictRequest request; PredictResponse response; @@ -125,15 +152,14 @@ TEST_F(PredictImplTest, InputTensorsDontMatchModelSpecInputs) { tensor_proto.set_dtype(tensorflow::DT_STRING); tensor_proto.mutable_tensor_shape()->add_dim()->set_size(1); + TensorflowPredictor predictor(GetParam()); auto inputs = request.mutable_inputs(); (*inputs)["key"] = tensor_proto; - EXPECT_EQ( - tensorflow::error::INVALID_ARGUMENT, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetServerCore(), request, &response).code()); } -TEST_F(PredictImplTest, OutputFiltersDontMatchModelSpecOutputs) { +TEST_P(PredictImplTest, OutputFiltersDontMatchModelSpecOutputs) { PredictRequest request; PredictResponse response; @@ -141,42 +167,28 @@ TEST_F(PredictImplTest, OutputFiltersDontMatchModelSpecOutputs) { model_spec->set_name(kTestModelName); model_spec->mutable_version()->set_value(kTestModelVersion); - ServableHandle bundle; - TF_ASSERT_OK(server_core_->GetServableHandle(request.model_spec(), &bundle)); - Signature signature; - TF_ASSERT_OK(GetNamedSignature("inputs", bundle->meta_graph_def, &signature)); - TensorProto tensor_proto; tensor_proto.add_float_val(2.0); tensor_proto.set_dtype(tensorflow::DT_FLOAT); - - for (const auto& input : signature.generic_signature().map()) { - (*request.mutable_inputs())[input.first] = tensor_proto; - } - + (*request.mutable_inputs())[kInputTensorKey] = tensor_proto; request.add_output_filter("output_filter"); + TensorflowPredictor predictor(GetParam()); // Output filter like this doesn't exist. - EXPECT_EQ( - tensorflow::error::INVALID_ARGUMENT, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetServerCore(), request, &response).code()); request.clear_output_filter(); - request.add_output_filter("y"); - EXPECT_TRUE( - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .ok()); - request.add_output_filter("y"); + request.add_output_filter(kOutputTensorKey); + TF_EXPECT_OK(predictor.Predict(GetServerCore(), request, &response)); + request.add_output_filter(kOutputTensorKey); // Duplicate output filter specified. - EXPECT_EQ( - tensorflow::error::INVALID_ARGUMENT, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetServerCore(), request, &response).code()); } -TEST_F(PredictImplTest, InputTensorsHaveWrongType) { +TEST_P(PredictImplTest, InputTensorsHaveWrongType) { PredictRequest request; PredictResponse response; @@ -184,28 +196,20 @@ TEST_F(PredictImplTest, InputTensorsHaveWrongType) { model_spec->set_name(kTestModelName); model_spec->mutable_version()->set_value(kTestModelVersion); - ServableHandle bundle; - TF_ASSERT_OK(server_core_->GetServableHandle(request.model_spec(), &bundle)); - TensorProto tensor_proto; tensor_proto.add_string_val("any_key"); tensor_proto.set_dtype(tensorflow::DT_STRING); tensor_proto.mutable_tensor_shape()->add_dim()->set_size(1); + (*request.mutable_inputs())[kInputTensorKey] = tensor_proto; + request.add_output_filter(kOutputTensorKey); - Signature signature; - TF_ASSERT_OK(GetNamedSignature("inputs", bundle->meta_graph_def, &signature)); - for (const auto& input : signature.generic_signature().map()) { - (*request.mutable_inputs())[input.first] = tensor_proto; - } - request.add_output_filter("y"); + TensorflowPredictor predictor(GetParam()); // Input tensors are all wrong. - EXPECT_EQ( - tensorflow::error::INTERNAL, - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .code()); + EXPECT_EQ(tensorflow::error::INTERNAL, + predictor.Predict(GetServerCore(), request, &response).code()); } -TEST_F(PredictImplTest, ModelMissingSignatures) { +TEST_P(PredictImplTest, ModelMissingSignatures) { PredictRequest request; PredictResponse response; @@ -214,13 +218,13 @@ TEST_F(PredictImplTest, ModelMissingSignatures) { model_spec->mutable_version()->set_value(kTestModelVersion); // Model is missing signatures. + TensorflowPredictor predictor(GetParam()); EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, - TensorflowPredictImpl::Predict(server_core_bad_model_.get(), - request, &response) + predictor.Predict(GetServerCoreWithBadModel(), request, &response) .code()); } -TEST_F(PredictImplTest, PredictionSuccess) { +TEST_P(PredictImplTest, PredictionSuccess) { PredictRequest request; PredictResponse response; @@ -228,31 +232,25 @@ TEST_F(PredictImplTest, PredictionSuccess) { model_spec->set_name(kTestModelName); model_spec->mutable_version()->set_value(kTestModelVersion); - ServableHandle bundle; - TF_ASSERT_OK(server_core_->GetServableHandle(request.model_spec(), &bundle)); - Signature signature; - TF_ASSERT_OK(GetNamedSignature("inputs", bundle->meta_graph_def, &signature)); - TensorProto tensor_proto; tensor_proto.add_float_val(2.0); tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())[kInputTensorKey] = tensor_proto; - for (const auto& input : signature.generic_signature().map()) { - (*request.mutable_inputs())[input.first] = tensor_proto; - } - - EXPECT_TRUE( - TensorflowPredictImpl::Predict(server_core_.get(), request, &response) - .ok()); + TensorflowPredictor predictor(GetParam()); + TF_EXPECT_OK(predictor.Predict(GetServerCore(), request, &response)); TensorProto output_tensor_proto; output_tensor_proto.add_float_val(3); output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); output_tensor_proto.mutable_tensor_shape(); PredictResponse test_response; - (*test_response.mutable_outputs())["y"] = output_tensor_proto; + (*test_response.mutable_outputs())[kOutputTensorKey] = output_tensor_proto; EXPECT_THAT(test_response, test_util::EqualsProto(response)); } +// Test all ClassifierTest test cases with both SessionBundle and SavedModel. +INSTANTIATE_TEST_CASE_P(UseSavedModel, PredictImplTest, ::testing::Bool()); + } // namespace } // namespace serving } // namespace tensorflow From c9dc76cabd799d5487c2e38e1ec761c51018bd95 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Mon, 28 Nov 2016 22:43:31 -0800 Subject: [PATCH 0090/8554] Add flags for output dir in saved_model_half_plus_two.py. Remove static Saved Model of half plus two and its dependency. Change: 140441828 --- tensorflow_serving/model_servers/test_util/BUILD | 2 +- tensorflow_serving/servables/tensorflow/BUILD | 4 ++-- .../servables/tensorflow/bundle_factory_test_util.cc | 2 +- .../servables/tensorflow/saved_model_bundle_factory_test.cc | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 7e59c205aef..8a91db49651 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -48,7 +48,7 @@ cc_library( data = [ "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", - "@org_tensorflow//tensorflow/python/saved_model/example:versioned_saved_model_half_plus_two_data", + "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", ], visibility = [ "//visibility:public", diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 66017aaf400..42c3e6480ea 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -184,8 +184,8 @@ cc_test( size = "medium", srcs = ["saved_model_bundle_factory_test.cc"], data = [ - "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", "@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two", + "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", ], deps = [ ":bundle_factory_test", @@ -276,8 +276,8 @@ cc_test( size = "medium", srcs = ["saved_model_bundle_source_adapter_test.cc"], data = [ - "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", "@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two", + "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", ], # Link in all registered kernels. linkstatic = 1, diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc index 247eabbd16c..4e77380e2b6 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc @@ -30,7 +30,7 @@ namespace test_util { namespace { const char kTestSavedModelPath[] = - "cc/saved_model/testdata/half_plus_two_sharded"; + "python/saved_model/example/saved_model_half_plus_two/00000123"; const char kTestSessionBundleExportPath[] = "session_bundle/example/half_plus_two/00000123"; diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index 6f59d3444d1..73796ea796f 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -67,7 +67,7 @@ TEST_F(SavedModelBundleFactoryTest, Basic) { TestBasic(); } TEST_F(SavedModelBundleFactoryTest, Batching) { TestBatching(); } TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { - const double kTotalFileSize = 7492; + const double kTotalFileSize = 7512; TestEstimateResourceRequirementWithGoodExport( kTotalFileSize); } From d13baace1cff432a47c7761609501ecd563b0eb0 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Wed, 30 Nov 2016 08:47:55 -0800 Subject: [PATCH 0091/8554] Update saved-model half plus two with a Predict signature def. Change: 140609389 --- .../servables/tensorflow/saved_model_bundle_factory_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index 73796ea796f..728a3779806 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -67,7 +67,7 @@ TEST_F(SavedModelBundleFactoryTest, Basic) { TestBasic(); } TEST_F(SavedModelBundleFactoryTest, Batching) { TestBatching(); } TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { - const double kTotalFileSize = 7512; + const double kTotalFileSize = 7603; TestEstimateResourceRequirementWithGoodExport( kTotalFileSize); } From ba2aca41e640953b028cc8dad1e5a4d52446df0a Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 1 Dec 2016 13:47:11 -0800 Subject: [PATCH 0092/8554] Fix export path in test. Change: 140772395 --- tensorflow_serving/model_servers/BUILD | 1 + .../tensorflow_model_server_test.py | 60 +++++++++++++++---- .../test_util/server_core_test_util.cc | 6 ++ .../servables/tensorflow/predict_impl_test.cc | 11 ++-- .../saved_model_bundle_factory_test.cc | 2 +- 5 files changed, 64 insertions(+), 16 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 55a9719cfc0..bfc01621206 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -118,6 +118,7 @@ py_test( "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export.meta", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", ], tags = [ "local", diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 7105e1f58cf..4d7ed12d88d 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -67,13 +67,14 @@ def TerminateProcs(self): if self.server_proc is not None: self.server_proc.terminate() - def RunServer(self, port, model_name, model_path): + def RunServer(self, port, model_name, model_path, use_saved_model): """Run tensorflow_model_server using test config.""" print 'Starting test server...' command = os.path.join(self.binary_dir, 'tensorflow_model_server') command += ' --port=' + str(port) command += ' --model_name=' + model_name command += ' --model_base_path=' + model_path + command += ' --use_saved_model=' + str(use_saved_model).lower() command += ' --alsologtostderr' print command self.server_proc = subprocess.Popen(shlex.split(command)) @@ -103,29 +104,66 @@ def VerifyPredictRequest(self, self.assertEquals(1, len(result.outputs['y'].float_val)) self.assertEquals(3.0, result.outputs['y'].float_val[0]) - def testPredict(self): - """Test PredictionService.Predict implementation.""" + def _GetSavedModelBundlePath(self): + """Returns a path to a model in SavedModel format.""" + return os.path.join(os.environ['TEST_SRCDIR'], + 'tf_serving/external/org_tensorflow/tensorflow/', + 'python/saved_model/example/saved_model_half_plus_two') + + def _GetSessionBundlePath(self): + """Returns a path to a model in SessionBundle format.""" + return os.path.join(self.testdata_dir, 'half_plus_two') + + def _TestPredict(self, model_path, use_saved_model): + """Helper method to test prediction. + + Args: + model_path: Path to the model on disk. + use_saved_model: Whether the model server should use SavedModel. + """ atexit.register(self.TerminateProcs) - model_server_address = self.RunServer( - PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'half_plus_two')) + model_server_address = self.RunServer(PickUnusedPort(), 'default', + model_path, use_saved_model) time.sleep(5) self.VerifyPredictRequest(model_server_address) self.VerifyPredictRequest(model_server_address, specify_output=False) - def testBadModel(self): - """Test PredictionService.Predict against a bad model export.""" + def testPredictSessionBundle(self): + """Test PredictionService.Predict implementation with SessionBundle.""" + self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False) + + def testPredictSavedModel(self): + """Test PredictionService.Predict implementation with SavedModel.""" + self._TestPredict(self._GetSavedModelBundlePath(), use_saved_model=True) + + def testPredictUpconvertedSavedModel(self): + """Test PredictionService.Predict implementation. + + Using a SessionBundle converted to a SavedModel. + """ + self._TestPredict(self._GetSessionBundlePath(), use_saved_model=True) + + def _TestBadModel(self, use_saved_model): + """Helper method to test against a bad model export.""" atexit.register(self.TerminateProcs) + # Both SessionBundle and SavedModel use the same bad model path, but in the + # case of SavedModel, the export will get up-converted to a SavedModel. model_server_address = self.RunServer( PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'bad_half_plus_two')) + os.path.join(self.testdata_dir, 'bad_half_plus_two'), use_saved_model) time.sleep(5) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address) self.assertIs(beta_interfaces.StatusCode.FAILED_PRECONDITION, error.exception.code) - self.assertTrue(error.exception.details.startswith( - 'Expected exactly one signatures proto')) + + def _TestBadModelUpconvertedSavedModel(self): + """Test Predict against a bad upconverted SavedModel model export.""" + self._TestBadModel(use_saved_model=True) + + def _TestBadModelSessionBundle(self): + """Test Predict against a bad SessionBundle model export.""" + self._TestBadModel(use_saved_model=False) if __name__ == '__main__': diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 561d89e8c43..9193d8607d1 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -55,6 +55,9 @@ Status ServerCoreTest::CreateServerCore( ServerCore::Options options; options.model_server_config = config; options.source_adapter_creator = source_adapter_creator; + // Reduce the number of initial load thread to be num_load_unload_threads to + // avoid timing out in tests. + options.num_initial_load_unload_threads = options.num_load_unload_threads; options.custom_model_config_loader = []( const ::google::protobuf::Any& any, EventBus* event_bus, UniquePtrWithDeps* manager) -> Status { @@ -66,6 +69,9 @@ Status ServerCoreTest::CreateServerCore( Status ServerCoreTest::CreateServerCore( ServerCore::Options options, std::unique_ptr* server_core) { options.file_system_poll_wait_seconds = 0; + // Reduce the number of initial load thread to be num_load_unload_threads to + // avoid timing out in tests. + options.num_initial_load_unload_threads = options.num_load_unload_threads; if (options.aspired_version_policy == nullptr) { options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index eaba1d6d34c..7f1f3c46b81 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -61,19 +61,22 @@ class PredictImplTest : public ::testing::TestWithParam { protected: static Status CreateServerCore(const string& model_path, bool use_saved_model, std::unique_ptr* server_core) { - // For ServerCore Options, we leave servable_state_monitor_creator - // unspecified so the default servable_state_monitor_creator will be used. - ServerCore::Options options; - ModelServerConfig config; auto model_config = config.mutable_model_config_list()->add_config(); model_config->set_name(kTestModelName); model_config->set_base_path(test_util::TestSrcDirPath(model_path)); model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; options.model_server_config = config; options.use_saved_model = use_saved_model; options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); + // Reduce the number of initial load thread to be num_load_unload_threads to + // avoid timing out in tests. + options.num_initial_load_unload_threads = options.num_load_unload_threads; return ServerCore::Create(std::move(options), server_core); } diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index 728a3779806..d63c8ae8731 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -67,7 +67,7 @@ TEST_F(SavedModelBundleFactoryTest, Basic) { TestBasic(); } TEST_F(SavedModelBundleFactoryTest, Batching) { TestBatching(); } TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { - const double kTotalFileSize = 7603; + const double kTotalFileSize = 7592; TestEstimateResourceRequirementWithGoodExport( kTotalFileSize); } From 4068829d228c990ab4836807a48c20749d6485c6 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Thu, 1 Dec 2016 15:35:32 -0800 Subject: [PATCH 0093/8554] Remove hardcoded file size for SavedModel and SessionBundle export test data. Change: 140786990 --- tensorflow_serving/servables/tensorflow/BUILD | 3 ++ .../tensorflow/bundle_factory_test_util.cc | 30 +++++++++++++++++++ .../tensorflow/bundle_factory_test_util.h | 11 +++++++ .../tensorflow/bundle_factory_util_test.cc | 8 ++--- .../saved_model_bundle_factory_test.cc | 12 ++++---- .../tensorflow/session_bundle_factory_test.cc | 9 ++---- 6 files changed, 54 insertions(+), 19 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 42c3e6480ea..a80d32a7c08 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -90,6 +90,7 @@ cc_library( "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:constants", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", @@ -142,6 +143,7 @@ cc_test( data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], deps = [ ":bundle_factory_test", + ":bundle_factory_test_util", ":session_bundle_config_proto", ":session_bundle_factory", "//tensorflow_serving/core/test_util:test_main", @@ -189,6 +191,7 @@ cc_test( ], deps = [ ":bundle_factory_test", + ":bundle_factory_test_util", ":saved_model_bundle_factory", ":session_bundle_config_proto", "//tensorflow_serving/core/test_util:test_main", diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc index 4e77380e2b6..2a4fe4576c8 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc @@ -15,10 +15,12 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" +#include "tensorflow/cc/saved_model/constants.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/env.h" #include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/test_util/test_util.h" @@ -44,6 +46,34 @@ string GetTestSessionBundleExportPath() { return test_util::ContribTestSrcDirPath(kTestSessionBundleExportPath); } +std::vector GetTestSavedModelFiles() { + const string dir = GetTestSavedModelPath(); + return {tensorflow::io::JoinPath(dir, kSavedModelAssetsDirectory, "foo.txt"), + tensorflow::io::JoinPath(dir, kSavedModelFilenamePb), + tensorflow::io::JoinPath(dir, kSavedModelVariablesFilename, + "variables.data-00000-of-00001"), + tensorflow::io::JoinPath(dir, kSavedModelVariablesFilename, + "variables.index")}; +} + +std::vector GetTestSessionBundleExportFiles() { + const string dir = GetTestSessionBundleExportPath(); + return {tensorflow::io::JoinPath(dir, "export.meta"), + tensorflow::io::JoinPath(dir, "export-00000-of-00001")}; +} + +uint64 GetTotalFileSize(const std::vector& files) { + uint64 total_file_size = 0; + for (const string& file : files) { + if (!(Env::Default()->IsDirectory(file).ok())) { + uint64 file_size; + TF_CHECK_OK(Env::Default()->GetFileSize(file, &file_size)); + total_file_size += file_size; + } + } + return total_file_size; +} + void TestSingleRequest(Session* session) { Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); // half plus two: output should be input / 2 + 2. diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h index efebb216d41..093e34b137a 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h @@ -31,6 +31,17 @@ string GetTestSavedModelPath(); // Returns the Session Bundle export path for the half plus two model. string GetTestSessionBundleExportPath(); +// Returns the paths of the files of the Saved Model (the pb version) for the +// half plus two model. +std::vector GetTestSavedModelFiles(); + +// Returns the paths of the files of the Session Bundle export for the half plus +// two model. +std::vector GetTestSessionBundleExportFiles(); + +// Returns the total size of the given files. Requires the files to exist. +uint64 GetTotalFileSize(const std::vector& files); + // Test that a Session handles a single request for the half plus two // model properly. The request has size=2, for batching purposes. void TestSingleRequest(Session* session); diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc index bd9f6b70a00..10580401afb 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc @@ -133,12 +133,8 @@ TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithBadExport) { } TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithGoodExport) { - // The length of the file's version strings might change, so we don't - // hardcode their size. They are 4 bytes for tags & size, plus the actual - // length of the strings. - const double kVersionSize = - 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); - const double kTotalFileSize = 13392.5 + kVersionSize; + const double kTotalFileSize = + test_util::GetTotalFileSize(test_util::GetTestSessionBundleExportFiles()); ResourceAllocation expected = test_util::GetExpectedResourceEstimate(kTotalFileSize); diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index d63c8ae8731..ac26c951fb3 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/core/public/session.h" #include "tensorflow/core/public/version.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" namespace tensorflow { @@ -67,7 +68,8 @@ TEST_F(SavedModelBundleFactoryTest, Basic) { TestBasic(); } TEST_F(SavedModelBundleFactoryTest, Batching) { TestBatching(); } TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { - const double kTotalFileSize = 7592; + const double kTotalFileSize = + test_util::GetTotalFileSize(test_util::GetTestSavedModelFiles()); TestEstimateResourceRequirementWithGoodExport( kTotalFileSize); } @@ -101,12 +103,8 @@ TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, Batching) { TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, EstimateResourceRequirementWithGoodExport) { - // The length of the file's version strings might change, so we don't - // hardcode their size. They are 4 bytes for tags & size, plus the actual - // length of the strings. - const double kVersionSize = - 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); - const double kTotalFileSize = 13392.5 + kVersionSize; + const double kTotalFileSize = + test_util::GetTotalFileSize(test_util::GetTestSessionBundleExportFiles()); TestEstimateResourceRequirementWithGoodExport( kTotalFileSize); } diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index 19a951b1fc3..a195f5b7876 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/core/public/session.h" #include "tensorflow/core/public/version.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/test_util/test_util.h" @@ -63,12 +64,8 @@ TEST_F(SessionBundleFactoryTest, Basic) { TestBasic(); } TEST_F(SessionBundleFactoryTest, Batching) { TestBatching(); } TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { - // The length of the file's version strings might change, so we don't - // hardcode their size. They are 4 bytes for tags & size, plus the actual - // length of the strings. - const double kVersionSize = - 4 + strlen(TF_VERSION_STRING) + strlen(tf_git_version()); - const double kTotalFileSize = 13392.5 + kVersionSize; + const double kTotalFileSize = + test_util::GetTotalFileSize(test_util::GetTestSessionBundleExportFiles()); TestEstimateResourceRequirementWithGoodExport( kTotalFileSize); } From 68f8fe489f4b1e0cb54b2a8fe605f5e248b5abd5 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 2 Dec 2016 10:22:03 -0800 Subject: [PATCH 0094/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index dca48e8b5ad..4172ca2cd7a 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit dca48e8b5adbb328c09a43b7d19300b52680d7ac +Subproject commit 4172ca2cd7ac34be67bda2600944d284e8907b95 diff --git a/tf_models b/tf_models index 33c4e784f4d..4bcbe8e140a 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 33c4e784f4d0dc2cf5e90527fd44bbe161857b9e +Subproject commit 4bcbe8e140a6c83c151b93567f33e60e486922bd From 245a52f95f4ea7816080e93d4da2f7d3553ec20a Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Fri, 2 Dec 2016 11:11:07 -0800 Subject: [PATCH 0095/8554] Remove redundant build dependency --- tensorflow_serving/config/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 3fbff93c787..d924949c0fb 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -27,7 +27,6 @@ serving_proto_library( srcs = ["model_server_config.proto"], cc_api_version = 2, deps = [ - "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", "@protobuf//:cc_wkt_protos", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", ], From a7d6daff24b786a7c3fa1f4ee2eb259c3a941652 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 2 Dec 2016 11:35:54 -0800 Subject: [PATCH 0096/8554] Minor formatting fix. --- .../sources/storage_path/file_system_storage_path_source.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto index df5e123c310..5f067c45221 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto @@ -4,7 +4,6 @@ package tensorflow.serving; // Config proto for FileSystemStoragePathSource. message FileSystemStoragePathSourceConfig { - // The policy to define how many versions of the servable should be // served at the same time. enum VersionPolicy { From c7527ca09750dcdf4c3a5487b91dd90d5df6ce35 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 2 Dec 2016 10:46:01 -0800 Subject: [PATCH 0097/8554] Internal change. Change: 140867651 --- tensorflow_serving/util/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index ac6362d15e2..56b1b8df872 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -1,6 +1,7 @@ # Description: Tensorflow Serving utils. package( + default_hdrs_check = "loose", default_visibility = [ "//tensorflow_serving:internal", ], From 41c01d8b4684bffc97eae172412f13a7e56e9b2d Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 2 Dec 2016 14:21:27 -0800 Subject: [PATCH 0098/8554] Separate the load & unload executors in BasicManager, to allow the # of load threads to be controlled separately from the # of unload threads. Change: 140893221 --- .../core/aspired_versions_manager.cc | 13 ++- .../core/aspired_versions_manager.h | 24 +++--- .../core/aspired_versions_manager_test.cc | 33 +++++--- tensorflow_serving/core/basic_manager.cc | 83 +++++++++++-------- tensorflow_serving/core/basic_manager.h | 55 ++++++------ tensorflow_serving/core/basic_manager_test.cc | 82 +++++++++--------- tensorflow_serving/core/caching_manager.cc | 4 +- tensorflow_serving/core/caching_manager.h | 13 +-- .../core/caching_manager_test.cc | 21 +++-- .../core/load_servables_fast.cc | 7 +- tensorflow_serving/core/static_manager.cc | 3 +- .../core/test_util/manager_test_util.cc | 19 ++--- .../core/test_util/manager_test_util.h | 8 +- .../model_servers/server_core.cc | 5 +- .../model_servers/server_core.h | 21 +++-- .../test_util/server_core_test_util.cc | 12 +-- .../servables/tensorflow/predict_impl_test.cc | 6 +- 17 files changed, 233 insertions(+), 176 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 3610103b3b5..3a6ed7c8a72 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -151,8 +151,8 @@ Status AspiredVersionsManager::Create( } BasicManager::Options basic_manager_options; basic_manager_options.resource_tracker = std::move(options.resource_tracker); - basic_manager_options.num_load_unload_threads = - options.num_load_unload_threads; + basic_manager_options.num_load_threads = options.num_load_threads; + basic_manager_options.num_unload_threads = options.num_unload_threads; basic_manager_options.max_num_load_retries = options.max_num_load_retries; basic_manager_options.load_retry_interval_micros = options.load_retry_interval_micros; @@ -419,13 +419,12 @@ void AspiredVersionsManager::InvokePolicyAndExecuteAction() { PerformAction(*next_action); } -void AspiredVersionsManager::SetNumLoadUnloadThreads( - const uint32 num_load_unload_threads) { - basic_manager_->SetNumLoadUnloadThreads(num_load_unload_threads); +void AspiredVersionsManager::SetNumLoadThreads(const uint32 num_load_threads) { + basic_manager_->SetNumLoadThreads(num_load_threads); } -uint32 AspiredVersionsManager::num_load_unload_threads() const { - return basic_manager_->num_load_unload_threads(); +uint32 AspiredVersionsManager::num_load_threads() const { + return basic_manager_->num_load_threads(); } } // namespace serving diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 19d10949e6f..db4342f18b3 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -94,12 +94,17 @@ class AspiredVersionsManager : public Manager, // The AspiredVersionPolicy to use for the manager. Must be non-null. std::unique_ptr aspired_version_policy; - // The number of threads in the thread-pool used to load and unload - // servables. + // The number of threads in the thread-pool used to load servables. // - // If set as 0, we don't use a thread-pool, and servable loads/unloads are + // If set as 0, we don't use a thread-pool, and servable loads are performed + // serially in the manager's main work loop. + uint32 num_load_threads = 0; + + // The number of threads in the thread-pool used to unload servables. + // + // If set as 0, we don't use a thread-pool, and servable unloads are // performed serially in the manager's main work loop. - uint32 num_load_unload_threads = 0; + uint32 num_unload_threads = 0; // Maximum number of times we retry loading a servable, after the first // failure, before we give up. @@ -244,13 +249,12 @@ class AspiredVersionsManager : public Manager, void InvokePolicyAndExecuteAction() LOCKS_EXCLUDED(basic_manager_read_modify_write_mu_); - // Sets the number of load/unload threads. + // Sets the number of load threads. // - // We immediately block all new load/unload requests while the current - // executor is destructed, a new one is created and then swapped with the - // current one. - void SetNumLoadUnloadThreads(uint32 num_load_unload_threads); - uint32 num_load_unload_threads() const; + // We immediately block all new load requests while the current executor is + // destructed, a new one is created and then swapped with the current one. + void SetNumLoadThreads(uint32 num_load_threads); + uint32 num_load_threads() const; std::unique_ptr aspired_version_policy_; diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 07cbddb0760..17e7b3bd3f8 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -61,16 +61,22 @@ ServableData> CreateAspiredVersion( return CreateServableData(id, std::move(loader)); } -class AspiredVersionsManagerTest : public ::testing::TestWithParam { +// We parameterize this test with the number of load & unload threads. (Zero +// means use an in-line executor instead of a thread pool.) +struct ThreadPoolSizes { + uint64 num_load_threads; + uint64 num_unload_threads; +}; +class AspiredVersionsManagerTest + : public ::testing::TestWithParam { protected: AspiredVersionsManagerTest() : servable_event_bus_(EventBus::CreateEventBus()), - servable_state_monitor_(servable_event_bus_.get()) { + servable_state_monitor_(servable_event_bus_.get()), + thread_pool_sizes_(GetParam()) { AspiredVersionsManager::Options manager_options; - // We parameterize this test to either run with or without a - // thread-pool. - num_load_unload_threads_ = GetParam(); - manager_options.num_load_unload_threads = num_load_unload_threads_; + manager_options.num_load_threads = thread_pool_sizes_.num_load_threads; + manager_options.num_unload_threads = thread_pool_sizes_.num_unload_threads; // The state manager thread won't be run automatically. manager_options.manage_state_interval_micros = -1; manager_options.env = Env::Default(); @@ -142,13 +148,18 @@ class AspiredVersionsManagerTest : public ::testing::TestWithParam { std::shared_ptr> servable_event_bus_; ServableStateMonitor servable_state_monitor_; - uint32 num_load_unload_threads_; + ThreadPoolSizes thread_pool_sizes_; uint32 max_num_load_retries_; std::unique_ptr manager_; }; -INSTANTIATE_TEST_CASE_P(WithOrWithoutThreadPool, AspiredVersionsManagerTest, - ::testing::Values(0 /* WithoutThreadPool */, 4)); +INSTANTIATE_TEST_CASE_P( + WithOrWithoutThreadPools, AspiredVersionsManagerTest, + ::testing::Values( + ThreadPoolSizes{0, 0} /* without load or unload threadpools */, + ThreadPoolSizes{2, 0} /* with just a load threadpool */, + ThreadPoolSizes{0, 2} /* with just an unload threadpool */, + ThreadPoolSizes{4, 4} /* with load and unload threadpools */)); TEST_P(AspiredVersionsManagerTest, ServableHandleNotFoundMissingLoaderName) { ServableHandle handle; @@ -575,8 +586,8 @@ TEST_P(AspiredVersionsManagerTest, DestructOnNonServingThread) { {ServableState::ManagerState::kEnd}); FlushServables(); // The servable has been deleted in this thread if there is no - // thread-pool for load/unload. - if (num_load_unload_threads_ == 0) { + // thread-pool for unload. + if (thread_pool_sizes_.num_unload_threads == 0) { EXPECT_TRUE(FakeLoader::was_deleted_in_this_thread()); } done_unload_servable.Notify(); diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 233b3b587b6..429df9e1dba 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -39,20 +39,16 @@ namespace serving { namespace { -std::unique_ptr CreateLoadUnloadExecutor( - Env* const env, const uint32 num_load_unload_threads) { - std::unique_ptr load_unload_executor; - if (num_load_unload_threads == 0) { - LOG(INFO) << "Creating InlineExecutor for BasicManager."; - load_unload_executor.reset(new InlineExecutor()); +std::unique_ptr CreateExecutor(Env* const env, + const uint32 num_threads, + const string& threadpool_name) { + std::unique_ptr executor; + if (num_threads == 0) { + executor.reset(new InlineExecutor()); } else { - LOG(INFO) << "Creating ThreadPoolExecutor for BasicManager with " - "num_load_unload_threads: " - << num_load_unload_threads; - load_unload_executor.reset(new ThreadPoolExecutor( - env, "BasicManager_LoadUnload_ThreadPool", num_load_unload_threads)); + executor.reset(new ThreadPoolExecutor(env, threadpool_name, num_threads)); } - return load_unload_executor; + return executor; } } // namespace @@ -209,34 +205,40 @@ Status BasicManager::Create(Options options, harness_options.max_num_load_retries = options.max_num_load_retries; harness_options.load_retry_interval_micros = options.load_retry_interval_micros; - manager->reset(new BasicManager(options.env, options.num_load_unload_threads, + manager->reset(new BasicManager(options.env, options.num_load_threads, + options.num_unload_threads, std::move(options.resource_tracker), options.servable_event_bus, harness_options)); return Status::OK(); } -BasicManager::BasicManager(Env* const env, const uint32 num_load_unload_threads, +BasicManager::BasicManager(Env* const env, const uint32 num_load_threads, + const uint32 num_unload_threads, std::unique_ptr resource_tracker, EventBus* servable_event_bus, const LoaderHarness::Options& harness_options) : harness_options_(harness_options), servable_event_bus_(servable_event_bus), env_(env), - num_load_unload_threads_(num_load_unload_threads) { + num_load_threads_(num_load_threads) { { - mutex_lock l(num_load_unload_threads_mu_); - load_unload_executor_ = - CreateLoadUnloadExecutor(env_, num_load_unload_threads_); + mutex_lock l(num_load_threads_mu_); + load_executor_ = + CreateExecutor(env_, num_load_threads, "BasicManager_Load_ThreadPool"); } + unload_executor_ = CreateExecutor(env_, num_unload_threads, + "BasicManager_Unload_ThreadPool"); resource_tracker_ = std::move(resource_tracker); } BasicManager::~BasicManager() { + // Reset the executors first to finish all pending loads/unloads. { - mutex_lock l(num_load_unload_threads_mu_); - // Reset the executor first to finish all pending loads/unloads. - load_unload_executor_.reset(); + mutex_lock l(num_load_threads_mu_); + load_executor_.reset(); } + unload_executor_.reset(); + UnloadAllServables(); } @@ -546,20 +548,19 @@ Status BasicManager::ExecuteLoadOrUnload(const LoadOrUnloadRequest& request, return execution_status; } -void BasicManager::SetNumLoadUnloadThreads( - const uint32 num_load_unload_threads) { - mutex_lock l(num_load_unload_threads_mu_); +void BasicManager::SetNumLoadThreads(const uint32 num_load_threads) { + mutex_lock l(num_load_threads_mu_); - load_unload_executor_.reset(); - num_load_unload_threads_ = num_load_unload_threads; - load_unload_executor_ = - CreateLoadUnloadExecutor(env_, num_load_unload_threads_); + load_executor_.reset(); + num_load_threads_ = num_load_threads; + load_executor_ = + CreateExecutor(env_, num_load_threads_, "BasicManager_Load_ThreadPool"); } -uint32 BasicManager::num_load_unload_threads() const { - mutex_lock l(num_load_unload_threads_mu_); +uint32 BasicManager::num_load_threads() const { + mutex_lock l(num_load_threads_mu_); - return num_load_unload_threads_; + return num_load_threads_; } void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, @@ -584,11 +585,21 @@ void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, done_callback(status); return; } - { - mutex_lock l(num_load_unload_threads_mu_); - load_unload_executor_->Schedule([this, request, done_callback]() { - HandleLoadOrUnloadRequest(request, done_callback); - }); + + switch (request.kind) { + case LoadOrUnloadRequest::Kind::kLoad: { + mutex_lock l(num_load_threads_mu_); + load_executor_->Schedule([this, request, done_callback]() { + HandleLoadOrUnloadRequest(request, done_callback); + }); + break; + } + case LoadOrUnloadRequest::Kind::kUnload: { + unload_executor_->Schedule([this, request, done_callback]() { + HandleLoadOrUnloadRequest(request, done_callback); + }); + break; + } } } diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index f0c42b1340b..9cac6a4de5b 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -101,7 +101,7 @@ class BasicManagerTestAccess; // ... // // TF_CHECK_OK(manager.UnloadServable(id)); -// TF_CHECK_OK(manager.UnmanagerServable(id)); +// TF_CHECK_OK(manager.StopManagingServable(id)); class BasicManager : public Manager { public: struct Options { @@ -109,12 +109,15 @@ class BasicManager : public Manager { // If left as nullptr, we do not validate servable resource usage. std::unique_ptr resource_tracker; - // The number of threads in the thread-pool used to load and unload - // servables. + // The number of threads in the thread-pool used to load servables. // - // If set as 0, we don't use a thread-pool, and the {Load,Unload}Servable() - // methods block. - uint32 num_load_unload_threads = 0; + // If set as 0, we don't use a thread-pool, and LoadServable() blocks. + uint32 num_load_threads = 0; + + // The number of threads in the thread-pool used to unload servables. + // + // If set as 0, we don't use a thread-pool, and UnloadServable() blocks. + uint32 num_unload_threads = 0; // EventBus to publish servable state changes. This is optional, if unset, // we don't publish. @@ -239,9 +242,9 @@ class BasicManager : public Manager { // kQuiescing state, schedules the unload and returns, otherwise it completes // the unload before returning. // - // REQUIRES: This manager should have been managing this servable already, for - // it to be unloaded, else calls 'done_callback' with an error status. Do not - // call this multiple times on the same servable. Only one of those will + // REQUIRES: This manager should have loaded and made this servable available, + // for it to be unloaded, else calls 'done_callback' with an error status. Do + // not call this multiple times on the same servable. Only one of those will // succeed and the rest will fail with an error status. void UnloadServable(const ServableId& id, DoneCallback done_callback); @@ -249,7 +252,7 @@ class BasicManager : public Manager { friend class AspiredVersionsManager; friend class test_util::BasicManagerTestAccess; - BasicManager(Env* env, uint32 num_load_unload_threads, + BasicManager(Env* env, uint32 num_load_threads, uint32 num_unload_threads, std::unique_ptr resource_tracker, EventBus* servable_event_bus, const LoaderHarness::Options& harness_options); @@ -350,16 +353,15 @@ class BasicManager : public Manager { // are ready to be served. void UpdateServingMap() EXCLUSIVE_LOCKS_REQUIRED(mu_); - // Sets the number of load/unload threads. + // Sets the number of load threads. // - // We block all new load/unload requests while the old thread pool is - // destructed, a new one is created and then swapped with the old one. Note - // that destructing the old thread pool blocks until all threads are done, so - // it could block for a long time. - void SetNumLoadUnloadThreads(uint32 num_load_unload_threads) - LOCKS_EXCLUDED(num_load_unload_threads_mu_); - uint32 num_load_unload_threads() const - LOCKS_EXCLUDED(num_load_unload_threads_mu_); + // We block all new load requests while the old thread pool is destructed, a + // new one is created and then swapped with the old one. Note that destructing + // the old thread pool blocks until all threads are done, so it could block + // for a long time. + void SetNumLoadThreads(uint32 num_load_threads) + LOCKS_EXCLUDED(num_load_threads_mu_); + uint32 num_load_threads() const LOCKS_EXCLUDED(num_load_threads_mu_); // Keys are the servable names. // Values are the harnesses for each servable version. The values when @@ -458,11 +460,16 @@ class BasicManager : public Manager { Env* const env_; - mutable mutex num_load_unload_threads_mu_; - uint32 num_load_unload_threads_ GUARDED_BY(num_load_unload_threads_mu_); - // The executor used for executing load and unload of servables. - std::unique_ptr load_unload_executor_ - GUARDED_BY(num_load_unload_threads_mu_); + // The number of load threads and the associated executor. They can be changed + // after instantiation of the manager via SetNumLoadThreads(). + mutable mutex num_load_threads_mu_; + uint32 num_load_threads_ GUARDED_BY(num_load_threads_mu_); + // The executor used for executing loads of servables. + std::unique_ptr load_executor_ GUARDED_BY(num_load_threads_mu_); + + // The executor used for executing unloads of servables. (Unlike for loads, + // the unload executor is fixed for the lifetime of the manager.) + std::unique_ptr unload_executor_; // Used to serialize the decision phases of the load/unload requests. mutable mutex load_unload_decision_phase_mu_; diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 91034cfcdc2..70c62c6d6d3 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -74,15 +74,21 @@ ServableData> CreateServable( return CreateServableData(id, std::move(loader)); } -// We parameterize this test to either run with or without a thread-pool. -class BasicManagerTest : public ::testing::TestWithParam { +// We parameterize this test with the number of load & unload threads. (Zero +// means use an in-line executor instead of a thread pool.) +struct ThreadPoolSizes { + uint64 num_load_threads; + uint64 num_unload_threads; +}; +class BasicManagerTest : public ::testing::TestWithParam { protected: BasicManagerTest() - : num_load_unload_threads_(GetParam()), + : thread_pool_sizes_(GetParam()), servable_event_bus_(EventBus::CreateEventBus()), servable_state_monitor_(servable_event_bus_.get()) { BasicManager::Options options; - options.num_load_unload_threads = num_load_unload_threads_; + options.num_load_threads = thread_pool_sizes_.num_load_threads; + options.num_unload_threads = thread_pool_sizes_.num_unload_threads; options.servable_event_bus = servable_event_bus_.get(); options.max_num_load_retries = 10; options.load_retry_interval_micros = 0; @@ -109,15 +115,19 @@ class BasicManagerTest : public ::testing::TestWithParam { } } - int num_load_unload_threads_; + ThreadPoolSizes thread_pool_sizes_; std::shared_ptr> servable_event_bus_; ServableStateMonitor servable_state_monitor_; std::unique_ptr basic_manager_; }; -INSTANTIATE_TEST_CASE_P(WithOrWithoutThreadPool, BasicManagerTest, - ::testing::Values(0 /* WithoutThreadPool */, - kNumThreads)); +INSTANTIATE_TEST_CASE_P( + WithOrWithoutThreadPools, BasicManagerTest, + ::testing::Values( + ThreadPoolSizes{0, 0} /* without load or unload threadpools */, + ThreadPoolSizes{2, 0} /* with just a load threadpool */, + ThreadPoolSizes{0, 2} /* with just an unload threadpool */, + ThreadPoolSizes{4, 4} /* with load and unload threadpools */)); TEST_P(BasicManagerTest, ServableHandleNotFoundMissingLoaderName) { ServableHandle handle; @@ -525,7 +535,7 @@ TEST_P(BasicManagerTest, DestructOnNonServingThread) { basic_manager_->StopManagingServable(id); // The servable has been deleted in this thread if there is no // thread-pool for load/unload. - if (num_load_unload_threads_ == 0) { + if (thread_pool_sizes_.num_load_threads == 0) { EXPECT_TRUE(FakeLoader::was_deleted_in_this_thread()); } done_unload_servable.Notify(); @@ -725,7 +735,7 @@ TEST_P(BasicManagerTest, EventBusServableLifecycle) { TEST_P(BasicManagerTest, NoEventBus) { BasicManager::Options options; // Single threaded execution. - options.num_load_unload_threads = 0; + options.num_load_threads = 0; // No event bus. options.servable_event_bus = nullptr; std::unique_ptr manager; @@ -796,11 +806,11 @@ TEST_P(BasicManagerTest, InterleavedLoadsAndUnloads) { } } -class SetNumLoadUnloadThreadsBasicManagerTest : public ::testing::Test { +class SetNumLoadThreadsBasicManagerTest : public ::testing::Test { protected: - SetNumLoadUnloadThreadsBasicManagerTest() { + SetNumLoadThreadsBasicManagerTest() { BasicManager::Options options; - options.num_load_unload_threads = 0; + options.num_load_threads = 0; options.max_num_load_retries = 10; options.load_retry_interval_micros = 0; TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager_)); @@ -809,15 +819,15 @@ class SetNumLoadUnloadThreadsBasicManagerTest : public ::testing::Test { std::unique_ptr basic_manager_; }; -TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, ThreadPoolSwapped) { +TEST_F(SetNumLoadThreadsBasicManagerTest, ThreadPoolSwapped) { test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); - manager_test_access.SetNumLoadUnloadThreads(2); - EXPECT_EQ(2, manager_test_access.num_load_unload_threads()); + manager_test_access.SetNumLoadThreads(2); + EXPECT_EQ(2, manager_test_access.num_load_threads()); const auto load_done_fn = [&](const Status& status) { TF_ASSERT_OK(status); // Tests whether the threadpools are actually swapped in - // SetNumLoadUnloadThreads(). + // SetNumLoadThreads(). static thread_local int per_thread_load_ctr = 0; ++per_thread_load_ctr; EXPECT_EQ(1, per_thread_load_ctr); @@ -827,8 +837,8 @@ TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, ThreadPoolSwapped) { basic_manager_->ManageServable(CreateServable(id0)); basic_manager_->LoadServable(id0, load_done_fn); - manager_test_access.SetNumLoadUnloadThreads(0); - EXPECT_EQ(0, manager_test_access.num_load_unload_threads()); + manager_test_access.SetNumLoadThreads(0); + EXPECT_EQ(0, manager_test_access.num_load_threads()); const ServableId id1 = {kServableName3, 1}; basic_manager_->ManageServable(CreateServable(id1)); @@ -838,11 +848,10 @@ TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, ThreadPoolSwapped) { basic_manager_.reset(); } -TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, - ThreadPoolsNotAliveSimultaneously) { +TEST_F(SetNumLoadThreadsBasicManagerTest, ThreadPoolsNotAliveSimultaneously) { test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); - manager_test_access.SetNumLoadUnloadThreads(1); - EXPECT_EQ(1, manager_test_access.num_load_unload_threads()); + manager_test_access.SetNumLoadThreads(1); + EXPECT_EQ(1, manager_test_access.num_load_threads()); std::set data_race_set; const auto data_race_fn = [&](const Status& status) { @@ -863,12 +872,12 @@ TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, }); { - ThreadPoolExecutor executor(Env::Default(), "SetNumLoadUnloadThreads", + ThreadPoolExecutor executor(Env::Default(), "SetNumLoadThreads", kNumThreads); executor.Schedule([&]() { notify_for_setting.WaitForNotification(); - manager_test_access.SetNumLoadUnloadThreads(1); - EXPECT_EQ(1, manager_test_access.num_load_unload_threads()); + manager_test_access.SetNumLoadThreads(1); + EXPECT_EQ(1, manager_test_access.num_load_threads()); }); executor.Schedule([&]() { @@ -886,12 +895,11 @@ TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, // Tests whether the fast-load scenario works. In the fast-load scenario we try // to load a bunch of servables as fast as possible using a lot of threads. -TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, FastLoad) { +TEST_F(SetNumLoadThreadsBasicManagerTest, FastLoad) { test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); - const uint32 prev_num_load_unload_threads = - manager_test_access.num_load_unload_threads(); - manager_test_access.SetNumLoadUnloadThreads(32); - EXPECT_EQ(32, manager_test_access.num_load_unload_threads()); + const uint32 prev_num_load_threads = manager_test_access.num_load_threads(); + manager_test_access.SetNumLoadThreads(32); + EXPECT_EQ(32, manager_test_access.num_load_threads()); { ThreadPoolExecutor executor(Env::Default(), "FirstThreadPoolLoads", @@ -903,16 +911,15 @@ TEST_F(SetNumLoadUnloadThreadsBasicManagerTest, FastLoad) { basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); // We don't wait for load to be done here because we want to test that - // SetNumLoadUnloadThreads() waits properly till all queued loads are + // SetNumLoadThreads() waits properly till all queued loads are // finished. If a queued load hasn't been finished the corresponding // UnloadServable() will fail. }); } } - manager_test_access.SetNumLoadUnloadThreads(prev_num_load_unload_threads); - EXPECT_EQ(prev_num_load_unload_threads, - manager_test_access.num_load_unload_threads()); + manager_test_access.SetNumLoadThreads(prev_num_load_threads); + EXPECT_EQ(prev_num_load_threads, manager_test_access.num_load_threads()); { ThreadPoolExecutor executor(Env::Default(), "Unloads", kNumThreads); @@ -1124,8 +1131,9 @@ class ResourceConstrainedBasicManagerTest : public ::testing::Test { // Seed the manager with ten resource units. options.resource_tracker = CreateSimpleResourceTracker(10); options.servable_event_bus = servable_event_bus_.get(); - // Allow up to two load/unload requests to be processed concurrently. - options.num_load_unload_threads = 2; + // Allow up to two loads and two unloads to be processed concurrently. + options.num_load_threads = 2; + options.num_unload_threads = 2; // We don't want retries. options.max_num_load_retries = 0; TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager_)); diff --git a/tensorflow_serving/core/caching_manager.cc b/tensorflow_serving/core/caching_manager.cc index b1ee51f8dba..703dc62cd30 100644 --- a/tensorflow_serving/core/caching_manager.cc +++ b/tensorflow_serving/core/caching_manager.cc @@ -35,8 +35,8 @@ Status CachingManager::Create( // Set up basic manager options from the caching manager options. BasicManager::Options basic_manager_options; basic_manager_options.resource_tracker = std::move(options.resource_tracker); - basic_manager_options.num_load_unload_threads = - options.num_load_unload_threads; + basic_manager_options.num_load_threads = options.num_load_threads; + basic_manager_options.num_unload_threads = options.num_unload_threads; basic_manager_options.max_num_load_retries = options.max_num_load_retries; basic_manager_options.load_retry_interval_micros = options.load_retry_interval_micros; diff --git a/tensorflow_serving/core/caching_manager.h b/tensorflow_serving/core/caching_manager.h index 38cea83f524..a1283c56e96 100644 --- a/tensorflow_serving/core/caching_manager.h +++ b/tensorflow_serving/core/caching_manager.h @@ -48,12 +48,15 @@ class CachingManager : public Manager { // If left as nullptr, we do not validate servable resource usage. std::unique_ptr resource_tracker; - // The number of threads in the thread-pool used to load and unload - // servables. + // The number of threads in the thread-pool used to load servables. // - // If set as 0, we don't use a thread-pool, and the {Load,Unload}Servable() - // methods block. - uint32 num_load_unload_threads = 0; + // If set as 0, we don't use a thread-pool, and LoadServable() blocks. + uint32 num_load_threads = 0; + + // The number of threads in the thread-pool used to unload servables. + // + // If set as 0, we don't use a thread-pool. + uint32 num_unload_threads = 0; // EventBus to publish servable state changes. This is optional, if unset, // we don't publish. diff --git a/tensorflow_serving/core/caching_manager_test.cc b/tensorflow_serving/core/caching_manager_test.cc index a3ac3e6121a..cdd01672f58 100644 --- a/tensorflow_serving/core/caching_manager_test.cc +++ b/tensorflow_serving/core/caching_manager_test.cc @@ -138,7 +138,13 @@ constexpr char kServableName2[] = "kServableName2"; constexpr int kNumThreads = 10; -class CachingManagerTest : public ::testing::TestWithParam { +// We parameterize this test with the number of load & unload threads. (Zero +// means use an in-line executor instead of a thread pool.) +struct ThreadPoolSizes { + uint64 num_load_threads; + uint64 num_unload_threads; +}; +class CachingManagerTest : public ::testing::TestWithParam { protected: CachingManagerTest() : servable_event_bus_(EventBus::CreateEventBus()), @@ -146,7 +152,8 @@ class CachingManagerTest : public ::testing::TestWithParam { CachingManager::Options options; options.env = Env::Default(); options.servable_event_bus = servable_event_bus_.get(); - options.num_load_unload_threads = GetParam(); + options.num_load_threads = GetParam().num_load_threads; + options.num_unload_threads = GetParam().num_unload_threads; options.max_num_load_retries = 1; options.load_retry_interval_micros = 0; @@ -165,7 +172,8 @@ class CachingManagerTest : public ::testing::TestWithParam { CachingManager::Options options; options.env = Env::Default(); options.servable_event_bus = servable_event_bus_.get(); - options.num_load_unload_threads = GetParam(); + options.num_load_threads = GetParam().num_load_threads; + options.num_unload_threads = GetParam().num_unload_threads; options.max_num_load_retries = 1; options.load_retry_interval_micros = 0; @@ -191,8 +199,11 @@ class CachingManagerTest : public ::testing::TestWithParam { StringLoaderFactory* string_loader_factory_; }; -INSTANTIATE_TEST_CASE_P(WithOrWithoutThreadPool, CachingManagerTest, - ::testing::Values(0 /* WithoutThreadPool */, 4)); +INSTANTIATE_TEST_CASE_P( + WithOrWithoutThreadPools, CachingManagerTest, + ::testing::Values( + ThreadPoolSizes{0, 0} /* without load or unload threadpools */, + ThreadPoolSizes{4, 4} /* with load and unload threadpools */)); /////////////////////////////////////////////////////////////////////////////// // Servable handles. diff --git a/tensorflow_serving/core/load_servables_fast.cc b/tensorflow_serving/core/load_servables_fast.cc index 6fedaf126bf..653d908b225 100644 --- a/tensorflow_serving/core/load_servables_fast.cc +++ b/tensorflow_serving/core/load_servables_fast.cc @@ -32,12 +32,11 @@ Status ConnectSourceWithFastInitialLoad( AspiredVersionsManager* manager, Source>* source, const std::function& wait_until_loaded_fn, const uint32 num_threads) { - const uint32 prev_num_load_unload_threads = - manager->num_load_unload_threads(); - manager->SetNumLoadUnloadThreads(num_threads); + const uint32 prev_num_load_threads = manager->num_load_threads(); + manager->SetNumLoadThreads(num_threads); ConnectSourceToTarget(source, manager); const Status status = wait_until_loaded_fn(); - manager->SetNumLoadUnloadThreads(prev_num_load_unload_threads); + manager->SetNumLoadThreads(prev_num_load_threads); return status; } diff --git a/tensorflow_serving/core/static_manager.cc b/tensorflow_serving/core/static_manager.cc index 7b838fa3d64..d1e21489ce7 100644 --- a/tensorflow_serving/core/static_manager.cc +++ b/tensorflow_serving/core/static_manager.cc @@ -21,7 +21,8 @@ namespace serving { StaticManagerBuilder::StaticManagerBuilder() { BasicManager::Options basic_manager_options; // We don't want multithreading. - basic_manager_options.num_load_unload_threads = 0; + basic_manager_options.num_load_threads = 0; + basic_manager_options.num_unload_threads = 0; const Status basic_manager_status = BasicManager::Create(std::move(basic_manager_options), &basic_manager_); if (!basic_manager_status.ok()) { diff --git a/tensorflow_serving/core/test_util/manager_test_util.cc b/tensorflow_serving/core/test_util/manager_test_util.cc index 60b2fbf48c8..db102796192 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.cc +++ b/tensorflow_serving/core/test_util/manager_test_util.cc @@ -35,25 +35,24 @@ void AspiredVersionsManagerTestAccess::InvokePolicyAndExecuteAction() { manager_->InvokePolicyAndExecuteAction(); } -void AspiredVersionsManagerTestAccess::SetNumLoadUnloadThreads( - const uint32 num_load_unload_threads) { - manager_->SetNumLoadUnloadThreads(num_load_unload_threads); +void AspiredVersionsManagerTestAccess::SetNumLoadThreads( + const uint32 num_load_threads) { + manager_->SetNumLoadThreads(num_load_threads); } -uint32 AspiredVersionsManagerTestAccess::num_load_unload_threads() const { - return manager_->num_load_unload_threads(); +uint32 AspiredVersionsManagerTestAccess::num_load_threads() const { + return manager_->num_load_threads(); } BasicManagerTestAccess::BasicManagerTestAccess(BasicManager* manager) : manager_(manager) {} -void BasicManagerTestAccess::SetNumLoadUnloadThreads( - const uint32 num_load_unload_threads) { - manager_->SetNumLoadUnloadThreads(num_load_unload_threads); +void BasicManagerTestAccess::SetNumLoadThreads(const uint32 num_load_threads) { + manager_->SetNumLoadThreads(num_load_threads); } -uint32 BasicManagerTestAccess::num_load_unload_threads() const { - return manager_->num_load_unload_threads(); +uint32 BasicManagerTestAccess::num_load_threads() const { + return manager_->num_load_threads(); } CachingManagerTestAccess::CachingManagerTestAccess(CachingManager* manager) diff --git a/tensorflow_serving/core/test_util/manager_test_util.h b/tensorflow_serving/core/test_util/manager_test_util.h index 2b50a6c05c6..3ceef79b6a9 100644 --- a/tensorflow_serving/core/test_util/manager_test_util.h +++ b/tensorflow_serving/core/test_util/manager_test_util.h @@ -38,9 +38,9 @@ class AspiredVersionsManagerTestAccess { // Invokes InvokePolicyAndExecuteAction() on the manager. void InvokePolicyAndExecuteAction(); - void SetNumLoadUnloadThreads(uint32 num_load_unload_threads); + void SetNumLoadThreads(uint32 num_load_threads); - uint32 num_load_unload_threads() const; + uint32 num_load_threads() const; private: AspiredVersionsManager* const manager_; @@ -53,9 +53,9 @@ class BasicManagerTestAccess { public: explicit BasicManagerTestAccess(BasicManager* manager); - void SetNumLoadUnloadThreads(uint32 num_load_unload_threads); + void SetNumLoadThreads(uint32 num_load_threads); - uint32 num_load_unload_threads() const; + uint32 num_load_threads() const; private: BasicManager* const manager_; diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 58b3acaecb7..c52fa4ee55a 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -211,7 +211,7 @@ Status ServerCore::AddModelsViaModelConfigList() { } const tensorflow::Status status = ConnectSourceWithFastInitialLoad( manager_.get(), source_adapter.get(), servable_state_monitor_.get(), - static_servables, options_.num_initial_load_unload_threads); + static_servables, options_.num_initial_load_threads); if (!status.ok()) { VLOG(1) << "Unable to ConnectSourceWithFastInitialLoad due to: " << status; @@ -343,7 +343,8 @@ Status ServerCore::CreateAspiredVersionsManager( manager_options.resource_tracker = std::move(resource_tracker); manager_options.servable_event_bus = servable_event_bus_.get(); manager_options.aspired_version_policy = std::move(aspired_version_policy); - manager_options.num_load_unload_threads = options_.num_load_unload_threads; + manager_options.num_load_threads = options_.num_load_threads; + manager_options.num_unload_threads = options_.num_unload_threads; manager_options.max_num_load_retries = options_.max_num_load_retries; const tensorflow::Status status = AspiredVersionsManager::Create(std::move(manager_options), manager); diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 5fce83ac8b9..55c801bd348 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -86,15 +86,18 @@ class ServerCore : public Manager { // The AspiredVersionPolicy to use for the manager. Must be non-null. std::unique_ptr aspired_version_policy; - // The number of threads used to load and unload models. If set to 0, then - // no thread pool is used and the loads/unloads are performed serially in - // the manager thread. - int32 num_load_unload_threads = 0; - - // The number of load/unload threads used to load the initial set of models - // at server startup. This is set high to load up the initial set of models - // fast, after this the server uses num_load_unload_threads. - int32 num_initial_load_unload_threads = 4.0 * port::NumSchedulableCPUs(); + // The number of threads used to load models. If set to 0, then no thread + // pool is used and loads are performed serially in the manager thread. + int32 num_load_threads = 0; + + // The number of load threads used to load the initial set of models at + // server startup. This is set high to load up the initial set of models + // fast, after this the server uses num_load_threads. + int32 num_initial_load_threads = 4.0 * port::NumSchedulableCPUs(); + + // The number of threads used to unload models. If set to 0, then no thread + // pool is used and unloads are performed serially in the manager thread. + int32 num_unload_threads = 0; // Total model size limit, in terms of main memory, in bytes. uint64 total_model_memory_limit_bytes = std::numeric_limits::max(); diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 9193d8607d1..195f1d584b1 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -55,9 +55,9 @@ Status ServerCoreTest::CreateServerCore( ServerCore::Options options; options.model_server_config = config; options.source_adapter_creator = source_adapter_creator; - // Reduce the number of initial load thread to be num_load_unload_threads to - // avoid timing out in tests. - options.num_initial_load_unload_threads = options.num_load_unload_threads; + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; options.custom_model_config_loader = []( const ::google::protobuf::Any& any, EventBus* event_bus, UniquePtrWithDeps* manager) -> Status { @@ -69,9 +69,9 @@ Status ServerCoreTest::CreateServerCore( Status ServerCoreTest::CreateServerCore( ServerCore::Options options, std::unique_ptr* server_core) { options.file_system_poll_wait_seconds = 0; - // Reduce the number of initial load thread to be num_load_unload_threads to - // avoid timing out in tests. - options.num_initial_load_unload_threads = options.num_load_unload_threads; + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; if (options.aspired_version_policy == nullptr) { options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 7f1f3c46b81..afcd713bde1 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -74,9 +74,9 @@ class PredictImplTest : public ::testing::TestWithParam { options.use_saved_model = use_saved_model; options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); - // Reduce the number of initial load thread to be num_load_unload_threads to - // avoid timing out in tests. - options.num_initial_load_unload_threads = options.num_load_unload_threads; + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; return ServerCore::Create(std::move(options), server_core); } From 65b12a89d00732481709d2682ccbfe3990c45688 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Fri, 2 Dec 2016 14:33:21 -0800 Subject: [PATCH 0099/8554] Update file system storage path source logging message to use VLOG. Change: 140894703 --- .../storage_path/file_system_storage_path_source.cc | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 2f04d31a837..3b772bb9940 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -321,10 +321,10 @@ Status FileSystemStoragePathSource::PollFileSystemAndInvokeCallback() { const std::vector>& versions = entry.second; for (const ServableData& version : versions) { if (version.status().ok()) { - LOG(INFO) << "File-system polling update: Servable:" << version.id() - << "; Servable path: " << version.DataOrDie() - << "; Polling frequency: " - << config_.file_system_poll_wait_seconds(); + VLOG(1) << "File-system polling update: Servable:" << version.id() + << "; Servable path: " << version.DataOrDie() + << "; Polling frequency: " + << config_.file_system_poll_wait_seconds(); } } aspired_versions_callback_(servable, versions); From 2a64eff82b7a8d3f57d8fa016187376cc10effac Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Fri, 2 Dec 2016 18:09:58 -0800 Subject: [PATCH 0100/8554] RequestLogger abstraction. - Logs a sample of requests hitting the server. - Adds appropriate protos for configuration and metadata. Change: 140916680 --- tensorflow_serving/config/BUILD | 8 ++ .../config/logging_config.proto | 21 ++++++ tensorflow_serving/core/BUILD | 24 ++++++ tensorflow_serving/core/logging.proto | 14 ++++ tensorflow_serving/core/request_logger.cc | 46 ++++++++++++ tensorflow_serving/core/request_logger.h | 74 +++++++++++++++++++ 6 files changed, 187 insertions(+) create mode 100644 tensorflow_serving/config/logging_config.proto create mode 100644 tensorflow_serving/core/logging.proto create mode 100644 tensorflow_serving/core/request_logger.cc create mode 100644 tensorflow_serving/core/request_logger.h diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 4ea3d923b26..a777bee760c 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -31,3 +31,11 @@ serving_proto_library( "@protobuf//:cc_wkt_protos", ], ) + +serving_proto_library( + name = "logging_config_proto", + srcs = ["logging_config.proto"], + cc_api_version = 2, + deps = [ + ], +) diff --git a/tensorflow_serving/config/logging_config.proto b/tensorflow_serving/config/logging_config.proto new file mode 100644 index 00000000000..ffa7998d879 --- /dev/null +++ b/tensorflow_serving/config/logging_config.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +message SamplingConfig { + // Requests will be logged uniformly at random with this probability. Valid + // range: [0, 1.0]. + double sampling_rate = 1; +} + +// Configuration for logging query/responses. +message LoggingConfig { + // Identifies the type of the LogCollector we will use to collect these logs. + string type = 1; + + // The prefix to use for the filenames of the logs. + string filename_prefix = 2; + + SamplingConfig sampling_config = 3; +} diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 9def19c7d11..6612fab9d26 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -675,3 +675,27 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", ], ) + +load("//tensorflow_serving:serving.bzl", "serving_proto_library") + +serving_proto_library( + name = "logging_proto", + srcs = ["logging.proto"], + cc_api_version = 2, + deps = [ + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/config:logging_config_proto", + ], +) + +cc_library( + name = "request_logger", + srcs = ["request_logger.cc"], + hdrs = ["request_logger.h"], + deps = [ + ":log_collector", + ":logging_proto", + "//tensorflow_serving/config:logging_config_proto", + "@org_tensorflow//tensorflow/core:lib", + ], +) diff --git a/tensorflow_serving/core/logging.proto b/tensorflow_serving/core/logging.proto new file mode 100644 index 00000000000..a8c756f310c --- /dev/null +++ b/tensorflow_serving/core/logging.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/model.proto"; +import "tensorflow_serving/config/logging_config.proto"; + +// Metadata logged along with the request logs. +message LogMetadata { + ModelSpec model_spec = 1; + SamplingConfig sampling_config = 2; + // TODO(b/33279154): Add more metadata as mentioned in the bug. +} diff --git a/tensorflow_serving/core/request_logger.cc b/tensorflow_serving/core/request_logger.cc new file mode 100644 index 00000000000..6ec1f99f923 --- /dev/null +++ b/tensorflow_serving/core/request_logger.cc @@ -0,0 +1,46 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/request_logger.h" + +#include + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/random/random.h" + +namespace tensorflow { +namespace serving { + +RequestLogger::RequestLogger(const LoggingConfig& logging_config, + std::unique_ptr log_collector) + : logging_config_(logging_config), + log_collector_(std::move(log_collector)), + uniform_sampler_() {} + +Status RequestLogger::Log(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata) { + const double sampling_rate = + logging_config_.sampling_config().sampling_rate(); + if (uniform_sampler_.Sample(sampling_rate)) { + std::unique_ptr log; + TF_RETURN_IF_ERROR(CreateLogMessage(request, response, log_metadata, &log)); + TF_RETURN_IF_ERROR(log_collector_->CollectMessage(*log)); + } + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/request_logger.h b/tensorflow_serving/core/request_logger.h new file mode 100644 index 00000000000..290e5cd5ceb --- /dev/null +++ b/tensorflow_serving/core/request_logger.h @@ -0,0 +1,74 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ + +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/log_collector.h" +#include "tensorflow_serving/core/logging.pb.h" + +namespace tensorflow { +namespace serving { + +// Abstraction to log requests and responses hitting a server. The log storage +// is handled by the log-collector. We sample requests based on the config. +class RequestLogger { + public: + RequestLogger(const LoggingConfig& logging_config, + std::unique_ptr log_collector); + + virtual ~RequestLogger() = default; + + // Writes the log for the particular request, respone and metadata, if we + // decide to sample it. + Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, + const LogMetadata& log_metadata); + + private: + // Creates the log message given the request, response and metadata. + // Implementations override it to create the particular message that they want + // to be logged. + virtual Status CreateLogMessage(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata, + std::unique_ptr* log) = 0; + + // A sampler which samples uniformly at random. + class UniformSampler { + public: + UniformSampler() : rd_(), gen_(rd_()), dist_(0, 1) {} + + // Returns true if the sampler decides to sample it with a probability + // 'rate'. + bool Sample(const double rate) { return dist_(gen_) < rate; } + + private: + std::random_device rd_; + std::mt19937 gen_; + std::uniform_real_distribution dist_; + }; + + const LoggingConfig logging_config_; + std::unique_ptr log_collector_; + UniformSampler uniform_sampler_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ From 543c89517cdcf944e79b3cee97aed0fcb1679a16 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 6 Dec 2016 17:34:18 -0800 Subject: [PATCH 0101/8554] Increase model_server max input message size to maxInt32 Change: 141251297 --- tensorflow_serving/model_servers/main.cc | 2 ++ tensorflow_serving/util/BUILD | 1 - 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index af87d886066..21032372ae5 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -58,6 +58,7 @@ limitations under the License. #include "grpc/grpc.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/command_line_flags.h" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #include "tensorflow_serving/apis/prediction_service.pb.h" @@ -199,6 +200,7 @@ void RunServer(int port, std::unique_ptr core, std::shared_ptr creds = InsecureServerCredentials(); builder.AddListeningPort(server_address, creds); builder.RegisterService(&service); + builder.SetMaxMessageSize(tensorflow::kint32max); std::unique_ptr server(builder.BuildAndStart()); LOG(INFO) << "Running ModelServer at " << server_address << " ..."; server->Wait(); diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 56b1b8df872..ac6362d15e2 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -1,7 +1,6 @@ # Description: Tensorflow Serving utils. package( - default_hdrs_check = "loose", default_visibility = [ "//tensorflow_serving:internal", ], From 0410f887966d42bd759773322b393e6e352c8f71 Mon Sep 17 00:00:00 2001 From: dlorenc Date: Wed, 7 Dec 2016 10:39:53 -0800 Subject: [PATCH 0102/8554] Switch the inception tag from 0.2.0 to latest. (#268) --- tensorflow_serving/example/inception_k8s.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/example/inception_k8s.yaml b/tensorflow_serving/example/inception_k8s.yaml index 89631768dd5..302e84e2299 100644 --- a/tensorflow_serving/example/inception_k8s.yaml +++ b/tensorflow_serving/example/inception_k8s.yaml @@ -11,7 +11,7 @@ spec: spec: containers: - name: inception-container - image: gcr.io/tensorflow-serving/inception:0.2.0 + image: gcr.io/tensorflow-serving/inception command: - /bin/sh - -c From 3b93f83e5efdfbb5fab205b18a3c22f5d3ae4ac7 Mon Sep 17 00:00:00 2001 From: Dominique d'Argent Date: Wed, 7 Dec 2016 22:59:53 +0100 Subject: [PATCH 0103/8554] Fix integer overflow in version number (#269) --- .../sources/storage_path/file_system_storage_path_source.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 2f04d31a837..75011244e7e 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -80,7 +80,7 @@ std::set GetDeletedServables( // aspire. void AspireVersion( const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, - const string& version_relative_path, const int version_number, + const string& version_relative_path, const int64 version_number, std::vector>* versions) { const ServableId servable_id = {servable.servable_name(), version_number}; const string full_path = From 69c3f10cd8014ecd5e4c713efed1d52ec9999662 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Wed, 7 Dec 2016 11:28:38 -0800 Subject: [PATCH 0104/8554] Minor comment update. Change: 141334124 --- tensorflow_serving/servables/tensorflow/bundle_factory_util.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc index e58908c310e..fe5dcd469c1 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc @@ -32,7 +32,7 @@ namespace { using Batcher = SharedBatchScheduler; // Constants used in the resource estimation heuristic. See the documentation -// on EstimateResourceRequirements(). +// on EstimateResourceFromPath(). constexpr double kResourceEstimateRAMMultiplier = 1.2; constexpr int kResourceEstimateRAMPadBytes = 0; From b6f6c3fed11f91dd48540dee0eaf3ba5a735ec2b Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 9 Dec 2016 14:13:00 -0800 Subject: [PATCH 0105/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 4172ca2cd7a..123a0f9a421 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 4172ca2cd7ac34be67bda2600944d284e8907b95 +Subproject commit 123a0f9a421af8a7b40d9cbf9b2905e86218bac3 diff --git a/tf_models b/tf_models index 4bcbe8e140a..e81ff571108 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 4bcbe8e140a6c83c151b93567f33e60e486922bd +Subproject commit e81ff571108dd817e4a85ebbfecd666e3906eef1 From c28f2982ef7df2a06c46157f9a5cea9979cf1022 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 9 Dec 2016 14:37:42 -0800 Subject: [PATCH 0106/8554] Enable BatchingSession to handle multiple signatures, and use one batching queue per signature. Change: 141601494 --- tensorflow_serving/batching/BUILD | 7 +- .../batching/batching_session.cc | 405 +++++++++++++----- .../batching/batching_session.h | 101 ++++- .../batching/batching_session_test.cc | 293 +++++++++---- tensorflow_serving/servables/tensorflow/BUILD | 1 + .../tensorflow/bundle_factory_test_util.cc | 15 +- .../tensorflow/bundle_factory_test_util.h | 4 + .../tensorflow/bundle_factory_util.cc | 13 +- .../tensorflow/bundle_factory_util.h | 7 +- .../tensorflow/bundle_factory_util_test.cc | 5 +- .../tensorflow/saved_model_bundle_factory.cc | 21 +- .../tensorflow/session_bundle_factory.cc | 38 +- 12 files changed, 709 insertions(+), 201 deletions(-) diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index 4a3f809d042..f31c26bef7a 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -150,6 +150,7 @@ cc_library( ":batch_scheduler", "//tensorflow_serving/servables/tensorflow:serving_session", "//tensorflow_serving/util:cleanup", + "//tensorflow_serving/util:hash", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", @@ -162,12 +163,16 @@ cc_test( srcs = [ "batching_session_test.cc", ], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + data = [ + "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", + ], deps = [ ":batching_session", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/servables/tensorflow:serving_session", "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index 9bea6ac49bb..6549d06f092 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -23,14 +23,85 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/servables/tensorflow/serving_session.h" #include "tensorflow_serving/util/cleanup.h" +#include "tensorflow_serving/util/hash.h" namespace tensorflow { namespace serving { +namespace { + +string TensorSignatureDebugString(const TensorSignature& signature) { + return strings::StrCat("{input_tensors: <", + str_util::Join(signature.input_tensors, ", "), + ">, output_tensors: <", + str_util::Join(signature.output_tensors, ", "), ">}"); +} + +struct HashTensorSignature { + uint64 operator()(const TensorSignature& signature) const { + uint64 hash = 0xDECAFCAFFE /* seed */; + for (const string& input_tensor : signature.input_tensors) { + hash = HashCombine(hash, std::hash()(input_tensor)); + } + for (const string& output_tensor : signature.output_tensors) { + hash = HashCombine(hash, std::hash()(output_tensor)); + } + return hash; + } +}; + +struct EqTensorSignature { + bool operator()(const TensorSignature& lhs, + const TensorSignature& rhs) const { + return lhs.input_tensors == rhs.input_tensors && + lhs.output_tensors == rhs.output_tensors; + } +}; + +// Constructs a TensorSignature from a Run() call's 'inputs' and +// 'output_tensor_names' arguments. +TensorSignature TensorSignatureFromRunArgs( + const std::vector>& inputs, + const std::vector& output_tensor_names) { + TensorSignature signature; + for (const auto& entry : inputs) { + const string& tensor_name = entry.first; + signature.input_tensors.insert(tensor_name); + } + for (const string& output_tensor_name : output_tensor_names) { + signature.output_tensors.insert(output_tensor_name); + } + return signature; +} + +} // namespace + +TensorSignature TensorSignatureFromSignatureDef( + const SignatureDef& signature_def) { + return TensorSignatureFromSignatureDefs({signature_def}); +} + +TensorSignature TensorSignatureFromSignatureDefs( + const std::vector& signature_defs) { + TensorSignature tensor_signature; + for (const SignatureDef& signature_def : signature_defs) { + for (const auto& entry : signature_def.inputs()) { + const TensorInfo& tensor_info = entry.second; + tensor_signature.input_tensors.insert(tensor_info.name()); + } + for (const auto& entry : signature_def.outputs()) { + const TensorInfo& tensor_info = entry.second; + tensor_signature.output_tensors.insert(tensor_info.name()); + } + } + return tensor_signature; +} + // A session that performs batching on top of a wrapped session. See the // documentation in batching_session.h for details and constraints. class BatchingSession : public ServingSession { @@ -38,14 +109,14 @@ class BatchingSession : public ServingSession { // Constructs a BatchingSession. Arguments: // - 'options' contains parameters. See batching_session.h. // - 'wrapped' is the session to wrap with batching. - // - 'batch_scheduler_creator' constructs a batch scheduler given a process- - // batch callback. See batching_session.h for example usage. + // - 'signatures_with_scheduler_creators' specifies the set of supported + // signatures, and for each one supplies a lambda to construct a batch + // scheduler given a process-batch callback. See batching_session.h for + // example usage. static Status Create( const BatchingSessionOptions& options, std::unique_ptr wrapped, - std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, std::unique_ptr* result); ~BatchingSession() override = default; @@ -71,49 +142,61 @@ class BatchingSession : public ServingSession { int RoundToLowestAllowedBatchSize(int batch_size) const; // Merges the input tensors in a batch, via concatenation of correspondingly- - // named tensors, and extracts the output tensor names. Assumes 'batch' is - // non-empty. Returns an error if there are any mismatches among the tasks in - // the batch that violate the constraints for batchability. + // named tensors. Puts the merged inputs in the order they are in in the + // signature. Assumes 'batch' is non-empty. Returns an error if there are any + // mismatches among the tasks in the batch that violate the constraints for + // batchability. Status MergeInputTensors( - const Batch& batch, - std::vector>* merged_inputs, - std::vector* output_tensor_names); + const TensorSignature& signature, const Batch& batch, + std::vector>* merged_inputs); // Splits the output of a batched call to 'wrapped_->Run()' into individual - // task outputs. - Status SplitOutputTensors(const std::vector& combined_outputs, + // task outputs. Assumes the output tensor order matches the signature. + Status SplitOutputTensors(const TensorSignature& signature, + const std::vector& combined_outputs, Batch* batch); - // Processes one batch. Called by 'batch_scheduler_' in a batch thread. - void ProcessBatch(std::unique_ptr> batch); + // Processes one batch of Run() calls with 'signature'. Called by + // 'batch_scheduler_' in a batch thread. + void ProcessBatch(const TensorSignature& signature, + std::unique_ptr> batch); const BatchingSessionOptions options_; std::unique_ptr wrapped_; - std::unique_ptr> batch_scheduler_; + std::unordered_map>, + HashTensorSignature, EqTensorSignature> + batch_schedulers_; TF_DISALLOW_COPY_AND_ASSIGN(BatchingSession); }; Status BatchingSession::Create( const BatchingSessionOptions& options, std::unique_ptr wrapped, - std::function< - Status(std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, std::unique_ptr* result) { auto batching_session = std::unique_ptr(new BatchingSession(options)); BatchingSession* raw_batching_session = batching_session.get(); batching_session->wrapped_ = std::move(wrapped); - std::unique_ptr> batch_scheduler; - TF_RETURN_IF_ERROR(batch_scheduler_creator( - [raw_batching_session]( - std::unique_ptr> batch) { - raw_batching_session->ProcessBatch(std::move(batch)); - }, - &batch_scheduler)); - batching_session->batch_scheduler_ = std::move(batch_scheduler); + + for (const auto& entry : signatures_with_scheduler_creators) { + const TensorSignature& signature = entry.signature; + const BatchingSessionSchedulerCreator& scheduler_creator = + entry.scheduler_creator; + + std::unique_ptr> batch_scheduler; + TF_RETURN_IF_ERROR(scheduler_creator( + [signature, raw_batching_session]( + std::unique_ptr> batch) { + raw_batching_session->ProcessBatch(signature, std::move(batch)); + }, + &batch_scheduler)); + batching_session->batch_schedulers_[signature] = std::move(batch_scheduler); + } + *result = std::move(batching_session); return Status::OK(); } @@ -128,6 +211,18 @@ Status BatchingSession::Run( "BatchingSession does not support target nodes"); } + const TensorSignature signature = + TensorSignatureFromRunArgs(inputs, output_tensor_names); + auto batch_scheduler_it = batch_schedulers_.find(signature); + if (batch_scheduler_it == batch_schedulers_.end()) { + // We have a Run() call that doesn't match one of our batching signatures. + // Run it in-line. + return wrapped_->Run(inputs, output_tensor_names, target_node_names, + outputs); + } + BatchScheduler* batch_scheduler = + batch_scheduler_it->second.get(); + outputs->clear(); Notification done; @@ -140,7 +235,7 @@ Status BatchingSession::Run( task->status = &status; task->outputs = outputs; - TF_RETURN_IF_ERROR(batch_scheduler_->Schedule(&task)); + TF_RETURN_IF_ERROR(batch_scheduler->Schedule(&task)); done.WaitForNotification(); return status; } @@ -195,73 +290,69 @@ int BatchingSession::RoundToLowestAllowedBatchSize(int batch_size) const { } Status BatchingSession::MergeInputTensors( - const Batch& batch, - std::vector>* merged_inputs, - std::vector* output_tensor_names) { + const TensorSignature& signature, const Batch& batch, + std::vector>* merged_inputs) { DCHECK_GE(batch.num_tasks(), 1); if (batch.num_tasks() < 1) { return errors::Internal("Batch size expected to be positive; was ", batch.num_tasks()); } - *output_tensor_names = *batch.task(0).output_tensor_names; - std::vector input_tensor_names; - for (const auto& input : *batch.task(0).inputs) { - const string& tensor_name = input.first; - input_tensor_names.push_back(tensor_name); - } - - // Fast-path for a singleton batch with no padding. - if (batch.num_tasks() == 1 && options_.allowed_batch_sizes.empty()) { - *merged_inputs = *batch.task(0).inputs; - return Status::OK(); - } const int padding_size = RoundToLowestAllowedBatchSize(batch.size()) - batch.size(); - for (int input_tensor_idx = 0; input_tensor_idx < input_tensor_names.size(); - ++input_tensor_idx) { - const string& input_tensor_name = input_tensor_names[input_tensor_idx]; - - std::vector tensors_to_merge; - for (int task_idx = 0; task_idx < batch.num_tasks(); ++task_idx) { - const std::vector>& task_inputs = - *batch.task(task_idx).inputs; - if (task_inputs.size() != input_tensor_names.size() || - task_inputs[input_tensor_idx].first != input_tensor_name) { - return errors::InvalidArgument( - "Batching session Run() calls must supply the same input tensors"); - } - if (input_tensor_idx == 0) { - if (*batch.task(task_idx).output_tensor_names != *output_tensor_names) { - return errors::InvalidArgument( - "Batching session Run() calls must supply the same output " - "tensors"); + // For each input tensor name, a vector of tensors from the individual tasks. + std::map> tensors_to_merge; + + // Populate 'tensors_to_merge'. + for (int i = 0; i < batch.num_tasks(); ++i) { + const std::vector>& task_inputs = + *batch.task(i).inputs; + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; + + std::vector& tensor_vec = tensors_to_merge[tensor_name]; + tensor_vec.push_back(tensor); + + if (i == batch.num_tasks() - 1 && padding_size > 0) { + // This is the last task. Insert padding. + // + // Use the first row of this task's tensor as the padding data. (We know + // it represents a valid input tensor row, so it should always be safe + // to use for padding.) + // + // Slice() operates on the 0th dimension, which is the batch dimension. + // It avoids a deep copy, which is a nice efficiency bonus. + const Tensor padding_tensor = tensor.Slice(0, 1); + for (int i = 0; i < padding_size; ++i) { + tensor_vec.push_back(padding_tensor); } } - tensors_to_merge.push_back(task_inputs[input_tensor_idx].second); } - if (padding_size > 0) { - // Use the first row of the first task's input tensor for padding. - // (We know it exists, and represents a valid input tensor row, so it - // should always be safe to use for padding.) - const Tensor& first_task_tensor = - (*batch.task(0).inputs)[input_tensor_idx].second; - // Slice() operates on the 0th dimension, which is the batch dimension. It - // avoids a deep copy, which is a nice efficiency bonus. - const Tensor padding_tensor = first_task_tensor.Slice(0, 1); - for (int i = 0; i < padding_size; ++i) { - tensors_to_merge.push_back(padding_tensor); - } + } + + // Merge the tensors. + DCHECK_EQ(signature.input_tensors.size(), tensors_to_merge.size()); + if (tensors_to_merge.size() != signature.input_tensors.size()) { + return errors::Internal( + "One or more tasks does not conform to batch signature"); + } + for (const string& tensor_name : signature.input_tensors) { + auto tensors = tensors_to_merge.find(tensor_name); + DCHECK(tensors != tensors_to_merge.end()); + if (tensors == tensors_to_merge.end()) { + return errors::Internal( + "One or more tasks does not conform to batch signature"); } - merged_inputs->push_back( - {input_tensor_name, tensor::Concat(tensors_to_merge)}); + merged_inputs->push_back({tensor_name, tensor::Concat(tensors->second)}); } return Status::OK(); } Status BatchingSession::SplitOutputTensors( + const TensorSignature& signature, const std::vector& combined_outputs, Batch* batch) { DCHECK_GE(batch->num_tasks(), 1); @@ -270,12 +361,6 @@ Status BatchingSession::SplitOutputTensors( batch->num_tasks()); } - // Fast-path for a singleton batch with no padding. - if (batch->num_tasks() == 1 && options_.allowed_batch_sizes.empty()) { - *batch->mutable_task(0)->outputs = combined_outputs; - return Status::OK(); - } - std::vector task_sizes_plus_optional_padding; for (int i = 0; i < batch->num_tasks(); ++i) { task_sizes_plus_optional_padding.push_back(batch->task(i).zeroth_dim_size); @@ -286,7 +371,20 @@ Status BatchingSession::SplitOutputTensors( task_sizes_plus_optional_padding.push_back(padding_size); } - for (const Tensor& tensor : combined_outputs) { + // For each output tensor name, a divided-up tensor with one entry per task. + std::map> split_tensors; + + // Populate 'split_tensors'. + DCHECK_EQ(signature.output_tensors.size(), combined_outputs.size()); + if (combined_outputs.size() != signature.output_tensors.size()) { + return errors::Internal("Wrong number of batched output tensors"); + } + const std::vector output_tensors(signature.output_tensors.begin(), + signature.output_tensors.end()); + for (int i = 0; i < output_tensors.size(); ++i) { + const string& tensor_name = output_tensors[i]; + const Tensor& tensor = combined_outputs[i]; + if (tensor.shape().dims() == 0) { return errors::FailedPrecondition( "Batched output tensor has 0 dimensions"); @@ -306,18 +404,27 @@ Status BatchingSession::SplitOutputTensors( split_tensor.size(), " splits; expected ", task_sizes_plus_optional_padding.size()); } + split_tensors[tensor_name] = std::move(split_tensor); + } - for (int i = 0; i < batch->num_tasks(); ++i) { - BatchingSessionTask* task = batch->mutable_task(i); - task->outputs->push_back(split_tensor[i]); + for (int i = 0; i < batch->num_tasks(); ++i) { + BatchingSessionTask* task = batch->mutable_task(i); + for (const string& tensor_name : *task->output_tensor_names) { + auto split_tensor = split_tensors.find(tensor_name); + DCHECK(split_tensor != split_tensors.end()); + if (split_tensor == split_tensors.end()) { + return errors::Internal("Task does not conform to batch signature"); + } + task->outputs->push_back(split_tensor->second[i]); } - // (Ignore a possible final split_tensor entry containing the padding.) } + // (Ignore a possible final split_tensors entry containing the padding.) return Status::OK(); } void BatchingSession::ProcessBatch( + const TensorSignature& signature, std::unique_ptr> batch) { // As a possible performance optimization, consider overlapping the tensor // concatenation with waiting for the batch to close (i.e. do the @@ -341,12 +448,13 @@ void BatchingSession::ProcessBatch( }); std::vector> merged_inputs; - std::vector output_tensor_names; - status = MergeInputTensors(*batch, &merged_inputs, &output_tensor_names); + status = MergeInputTensors(signature, *batch, &merged_inputs); if (!status.ok()) { return; } + const std::vector output_tensor_names( + signature.output_tensors.begin(), signature.output_tensors.end()); std::vector combined_outputs; status = wrapped_->Run(merged_inputs, output_tensor_names, {} /* target node names */, &combined_outputs); @@ -354,25 +462,128 @@ void BatchingSession::ProcessBatch( return; } - status = SplitOutputTensors(combined_outputs, batch.get()); + status = SplitOutputTensors(signature, combined_outputs, batch.get()); } Status CreateBatchingSession( const BatchingSessionOptions& options, - std::function< - Status(std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, std::unique_ptr session, std::unique_ptr* batching_session) { std::unique_ptr internal_batching_session; TF_RETURN_IF_ERROR(BatchingSession::Create(options, std::move(session), - batch_scheduler_creator, + signatures_with_scheduler_creators, &internal_batching_session)); *batching_session = std::move(internal_batching_session); return Status::OK(); } +Status CreateBasicBatchingSession( + const BasicBatchScheduler::Options& schedule_options, + const BatchingSessionOptions& batching_session_options, + const TensorSignature& signature, std::unique_ptr session, + std::unique_ptr* batching_session) { + if (!batching_session_options.allowed_batch_sizes.empty()) { + if (batching_session_options.allowed_batch_sizes.back() != + schedule_options.max_batch_size) { + return errors::InvalidArgument( + "Last entry in allowed_batch_sizes must match max_batch_size; last " + "entry was ", + batching_session_options.allowed_batch_sizes.back(), "; expected ", + schedule_options.max_batch_size); + } + } + + auto scheduler_creator = [schedule_options]( + std::function>)> + process_batch_callback, + std::unique_ptr>* batch_scheduler) { + std::unique_ptr> + basic_batch_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + schedule_options, process_batch_callback, &basic_batch_scheduler)); + *batch_scheduler = std::move(basic_batch_scheduler); + return Status::OK(); + }; + return CreateBatchingSession(batching_session_options, + {{signature, scheduler_creator}}, + std::move(session), batching_session); +} + +////////// +// Backward-compatibility layer for old APIs. +// TODO(b/33476643): Remove. + +// A ServingSession implementation that wraps a BatchingSession that it +// constructs upon receiving its first Run() call (and deducing the signature +// to use for batching). +class SignatureInferringBatchingSession : public ServingSession { + public: + SignatureInferringBatchingSession( + const BatchingSessionOptions& options, + std::unique_ptr raw_session, + BatchingSessionSchedulerCreator batch_scheduler_creator); + + ~SignatureInferringBatchingSession() override = default; + + Status Run(const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs) override; + + private: + const BatchingSessionOptions options_; + const BatchingSessionSchedulerCreator scheduler_creator_; + + // Exactly one of these is populated at a given time. Initially, + // 'raw_session_' is populated. Upon the first Run() call, 'batching_session_' + // is created from 'raw_session_' (and 'raw_session_' is cleared). + std::unique_ptr raw_session_; + std::unique_ptr batching_session_; + + mutable mutex batching_session_creation_mu_; + + TF_DISALLOW_COPY_AND_ASSIGN(SignatureInferringBatchingSession); +}; + +SignatureInferringBatchingSession::SignatureInferringBatchingSession( + const BatchingSessionOptions& options, std::unique_ptr raw_session, + BatchingSessionSchedulerCreator batch_scheduler_creator) + : options_(options), + scheduler_creator_(batch_scheduler_creator), + raw_session_(std::move(raw_session)) {} + +Status SignatureInferringBatchingSession::Run( + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs) { + { + mutex_lock l(batching_session_creation_mu_); + if (batching_session_ == nullptr) { + // This is the first Run() call. Set up 'batching_session_'. + const TensorSignature signature = + TensorSignatureFromRunArgs(inputs, output_tensor_names); + TF_RETURN_IF_ERROR( + CreateBatchingSession(options_, {{signature, scheduler_creator_}}, + std::move(raw_session_), &batching_session_)); + } + } + return batching_session_->Run(inputs, output_tensor_names, target_node_names, + outputs); +} + +Status CreateBatchingSession( + const BatchingSessionOptions& options, + BatchingSessionSchedulerCreator batch_scheduler_creator, + std::unique_ptr session, + std::unique_ptr* batching_session) { + batching_session->reset(new SignatureInferringBatchingSession( + options, std::move(session), batch_scheduler_creator)); + return Status::OK(); +} + Status CreateBasicBatchingSession( const BasicBatchScheduler::Options& schedule_options, const BatchingSessionOptions& batching_session_options, diff --git a/tensorflow_serving/batching/batching_session.h b/tensorflow_serving/batching/batching_session.h index 576e664675c..a31165d87e4 100644 --- a/tensorflow_serving/batching/batching_session.h +++ b/tensorflow_serving/batching/batching_session.h @@ -26,6 +26,7 @@ limitations under the License. #include #include +#include "tensorflow/core/protobuf/meta_graph.pb.h" #include "tensorflow/core/public/session.h" #include "tensorflow_serving/batching/basic_batch_scheduler.h" #include "tensorflow_serving/batching/batch_scheduler.h" @@ -37,6 +38,44 @@ namespace serving { // scheduler template parameters, e.g. BasicBatchScheduler. struct BatchingSessionTask; +// A function to construct a batch scheduler for BatchingSessionTasks from a +// process-batch callback. +using BatchingSessionSchedulerCreator = std::function>)>, + std::unique_ptr>*)>; + +// The signature associated with a Session::Run() call, in terms of input and +// output tensor names (with the order in which the tensors are listed factored +// out). (Note that 'target_node_names' are not supported in batching sessions.) +struct TensorSignature { + std::set input_tensors; + std::set output_tensors; +}; + +// Constructs a TensorSignature for a given SignatureDef. +TensorSignature TensorSignatureFromSignatureDef( + const SignatureDef& signature_def); + +// Constructs a TensorSignature for a given set of SignatureDefs. The resulting +// TensorSignature represents the Session::Run() arguments that would be used +// when issuing a single Run() call that exercises the signature defs jointly. +// +// For example, say there's a graph that takes 'input' and transforms it into +// 'predicted_label' and 'confidence_score'. Suppose SignatureDef 1 requests +// only 'predicted_label' as output, and SignatureDef 2 requests only +// 'confidence_score'. A joint TensorSignature would feed 'input' and receive +// both 'predicted_label' and 'confidence_score' as output, in a single Run() +// invocation. +TensorSignature TensorSignatureFromSignatureDefs( + const std::vector& signature_defs); + +// A signature paired with a lambda to create a batch scheduler for Run() calls +// matching the signature. +struct SignatureWithBatchingSessionSchedulerCreator { + TensorSignature signature; + BatchingSessionSchedulerCreator scheduler_creator; +}; + // Options for batching tensorflow Sessions; see the Create*() functions below. struct BatchingSessionOptions { // If set, restricts the allowed tensor batch sizes. @@ -63,26 +102,28 @@ struct BatchingSessionOptions { }; // Wraps a session in a new session that automatically batches Run() calls. -// In addition to a session to wrap, takes a function that constructs a batch -// scheduler given a process-batch callback. +// Uses one batcher for each distinct Run() signature supported. In addition to +// a session to wrap, takes a list of signature/BatchingSessionSchedulerCreator +// pairs. (The number of supported signatures is typically small, and often just +// a single one.) // -// The wrapped session only supports Run(), with the following restrictions: -// - 'target_node_names' must be empty -// - 'inputs' must not be empty -// - all calls supply the same input tensor names in 'inputs' -// - all calls supply the same output tensor names in 'output_tensor_names' +// The wrapped session only batches Run() calls that conform to one of the +// specified signatures and leave 'target_node_names' empty. Other Run() calls +// are executed in-line without batching, and may harm performance. (Extra- +// signature Run() support is intended primarily for debugging and diagnostics.) // -// It is assumed that the outermost (0th) dimension of each input and output -// tensor is the batch-size dimension. All input tensors must have the same 0th- -// dimension size B; the produced output tensors are also assumed to have 0th- -// dimension size B. +// For batched calls, it is assumed that the outermost (0th) dimension of each +// input and output tensor is the batch-size dimension. All input tensors must +// have the same 0th-dimension size B; the produced output tensors are also +// assumed to have 0th-dimension size B. // // IMPORTANT: Each call to Session::Run() is synchronous, and blocks waiting for -// other Run() calls to merge with to form a large batch. Consequently, to -// achieve good throughput we recommend setting the number of client threads -// that call Session::Run() equal to about twice the maximum batch size. +// other Run() calls with the same signature to merge with to form a large +// batch. Consequently, to achieve good throughput we recommend setting the +// number of client threads that call Session::Run() equal to about twice the +// sum over all signatures of the maximum batch size. // -// Example usage: +// Example usage, for the common case of a single signature: // // BatchingSessionOptions options = ...; // auto scheduler_creator = [schedule_options, retry_options]( @@ -99,26 +140,44 @@ struct BatchingSessionOptions { // return Status::OK(); // }; // std::unique_ptr batching_session; -// TF_CHECK_OK(CreateBatchingSession(options, scheduler_creator, +// TF_CHECK_OK(CreateBatchingSession(options, {{signature, scheduler_creator}}, // std::move(session), &batching_session)); // Status CreateBatchingSession( const BatchingSessionOptions& options, - std::function< - Status(std::function>)>, - std::unique_ptr>*)> - batch_scheduler_creator, + const std::vector& + signatures_with_scheduler_creators, std::unique_ptr session, std::unique_ptr* batching_session); // A convenience for using CreateBatchingSession() to create a -// BasicBatchScheduler. +// BasicBatchScheduler for a single signature. +Status CreateBasicBatchingSession( + const typename BasicBatchScheduler::Options& + schedule_options, + const BatchingSessionOptions& batching_session_options, + const TensorSignature& signature, std::unique_ptr session, + std::unique_ptr* batching_session); + +////////// +// DEPRECATED. New code should not use the following. +// +// Backward-compatibility layer for old API. This layer interprets the signature +// of the first Run() call as the one and only batching signature. +// TODO(b/33476643): Remove after migrating all clients. +Status CreateBatchingSession( + const BatchingSessionOptions& options, + BatchingSessionSchedulerCreator batch_scheduler_creator, + std::unique_ptr session, + std::unique_ptr* batching_session); Status CreateBasicBatchingSession( const typename BasicBatchScheduler::Options& schedule_options, const BatchingSessionOptions& batching_session_options, std::unique_ptr session, std::unique_ptr* batching_session); +// +////////// ////////// // Implementation details follow. API users need not read. diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index 08d79f170bf..a326010ea08 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -16,6 +16,8 @@ limitations under the License. #include "tensorflow_serving/batching/batching_session.h" #include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/tag_constants.h" #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_shape.h" @@ -34,6 +36,8 @@ namespace tensorflow { namespace serving { namespace { +using ::testing::UnorderedElementsAre; + // A wrapper around a Session that captures the batch size. class BatchSizeCapturingSession : public ServingSession { public: @@ -64,12 +68,13 @@ class BatchSizeCapturingSession : public ServingSession { // Creates a (non-batching) session with the half-plus-two model loaded. std::unique_ptr CreateHalfPlusTwoSession() { tensorflow::SessionOptions session_options; - const string export_dir = test_util::ContribTestSrcDirPath( - "session_bundle/example/half_plus_two/00000123"); - SessionBundle session_bundle; - TF_CHECK_OK( - LoadSessionBundleFromPath(session_options, export_dir, &session_bundle)); - return std::move(session_bundle.session); + tensorflow::RunOptions run_options; + const string export_dir = test_util::TensorflowTestSrcDirPath( + "python/saved_model/example/saved_model_half_plus_two/00000123"); + SavedModelBundle bundle; + TF_CHECK_OK(LoadSavedModel(session_options, run_options, export_dir, + {kSavedModelTagServe}, &bundle)); + return std::move(bundle.session); } // Test that a session handles a single request for the half-plus-two model @@ -99,6 +104,45 @@ void ExpectError(const string& error_message, EXPECT_EQ(error_message, status.error_message()); } +// Creates a SignatureDef from a TensorSignature. +SignatureDef CreateSignatureDef(const TensorSignature& tensor_signature) { + SignatureDef signature_def; + for (const string& input_tensor : tensor_signature.input_tensors) { + TensorInfo input; + input.set_name(input_tensor); + (*signature_def.mutable_inputs())[input_tensor] = input; + } + for (const string& output_tensor : tensor_signature.output_tensors) { + TensorInfo output; + output.set_name(output_tensor); + (*signature_def.mutable_outputs())[output_tensor] = output; + } + return signature_def; +} + +TEST(BatchingSessionTest, TensorSignatureFromSignatureDef) { + const SignatureDef signature_def = + CreateSignatureDef({{"x0", "x1"}, {"y0", "y1"}}); + const TensorSignature tensor_signature = + TensorSignatureFromSignatureDef(signature_def); + EXPECT_THAT(tensor_signature.input_tensors, UnorderedElementsAre("x0", "x1")); + EXPECT_THAT(tensor_signature.output_tensors, + UnorderedElementsAre("y0", "y1")); +} + +TEST(BatchingSessionTest, TensorSignatureFromSignatureDefs) { + const SignatureDef signature_def_0 = + CreateSignatureDef({{"x0", "x1"}, {"y0", "y1"}}); + const SignatureDef signature_def_1 = + CreateSignatureDef({{"x1", "x2"}, {"y1", "y2"}}); + const TensorSignature tensor_signature = + TensorSignatureFromSignatureDefs({signature_def_0, signature_def_1}); + EXPECT_THAT(tensor_signature.input_tensors, + UnorderedElementsAre("x0", "x1", "x2")); + EXPECT_THAT(tensor_signature.output_tensors, + UnorderedElementsAre("y0", "y1", "y2")); +} + TEST(BatchingSessionTest, Basic) { BasicBatchScheduler::Options schedule_options; schedule_options.max_batch_size = 4; // fits two 2-unit tasks @@ -107,8 +151,8 @@ TEST(BatchingSessionTest, Basic) { std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); // Asynchronously send two requests whose total size is 4. The two requests in // conjunction should trigger a batch to be processed. @@ -130,80 +174,44 @@ TEST(BatchingSessionTest, SingletonBatch) { std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session)); TestSingleRequest(100.0f, 42.0f, batching_session.get()); } -TEST(BatchingSessionTest, RequestWithIncompatibleInputTensorSizes) { +TEST(BatchingSessionTest, RequestThatDoesntMatchSignatureGetsRunAnyway) { BasicBatchScheduler::Options schedule_options; + // Set the batching parameters s.t. if the request is batched the test will + // timeout. + schedule_options.max_batch_size = 100; + schedule_options.batch_timeout_micros = INT_MAX; std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); - - ExpectError( - "Batching session Run() input tensors must have equal 0th-dimension size", - {{"input_0", test::AsTensor({3}, {1})}, - {"input_1", test::AsTensor({5, 7}, {2})}}, - {"output"}, batching_session.get()); + schedule_options, batching_session_options, {{"x2"}, {"y2"}}, + CreateHalfPlusTwoSession(), &batching_session)); + // Issue a request using x/y, which doesn't match the x2/y2 signature. + TestSingleRequest(100.0f, 42.0f, batching_session.get()); } -TEST(BatchingSessionTest, RequestsWithDifferentInputTensors) { +TEST(BatchingSessionTest, RequestWithIncompatibleInputTensorSizes) { BasicBatchScheduler::Options schedule_options; - schedule_options.max_batch_size = 2; // fits two 1-unit tasks - schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; - TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); - std::unique_ptr thread_1(Env::Default()->StartThread( - ThreadOptions(), "thread_1", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same input tensors", - {{"input", test::AsTensor({3}, {1})}}, {"output"}, - batching_session.get()); - })); - std::unique_ptr thread_2(Env::Default()->StartThread( - ThreadOptions(), "thread_2", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same input tensors", - {{"input", test::AsTensor({3}, {1})}, - {"another_input", test::AsTensor({3}, {1})}}, - {"output"}, batching_session.get()); - })); -} - -TEST(BatchingSessionTest, RequestsWithDifferentOutputTensors) { - BasicBatchScheduler::Options schedule_options; - schedule_options.max_batch_size = 2; // fits two 1-unit tasks - schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger - std::unique_ptr batching_session; - BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), + schedule_options, batching_session_options, + {{"input_0", "input_1"}, {"output"}}, CreateHalfPlusTwoSession(), &batching_session)); - std::unique_ptr thread_1(Env::Default()->StartThread( - ThreadOptions(), "thread_1", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same output tensors", - {{"input", test::AsTensor({3}, {1})}}, {"output"}, - batching_session.get()); - })); - std::unique_ptr thread_2(Env::Default()->StartThread( - ThreadOptions(), "thread_2", [&batching_session] { - ExpectError( - "Batching session Run() calls must supply the same output tensors", - {{"input", test::AsTensor({3}, {1})}}, - {"output", "another_output"}, batching_session.get()); - })); + ExpectError( + "Batching session Run() input tensors must have equal 0th-dimension size", + {{"input_0", test::AsTensor({3}, {1})}, + {"input_1", test::AsTensor({5, 7}, {2})}}, + {"output"}, batching_session.get()); } -TEST(BatchingSessionTest, AllowedBatchSizes_NoPaddingNeeded) { +TEST(BatchingSessionTest, AllowedBatchSizesNoPaddingNeeded) { // Arrange to capture the batch size. std::unique_ptr batch_size_capturing_session( new BatchSizeCapturingSession(CreateHalfPlusTwoSession())); @@ -217,7 +225,7 @@ TEST(BatchingSessionTest, AllowedBatchSizes_NoPaddingNeeded) { batching_session_options.allowed_batch_sizes = {2, 4}; std::unique_ptr batching_session; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, + schedule_options, batching_session_options, {{"x"}, {"y"}}, std::move(batch_size_capturing_session), &batching_session)); TestSingleRequest(100.0f, 42.0f, batching_session.get()); @@ -239,7 +247,7 @@ TEST(BatchingSessionTest, AllowedBatchSizesRequirePadding) { batching_session_options.allowed_batch_sizes = {1, 3, 4}; std::unique_ptr batching_session; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, + schedule_options, batching_session_options, {{"x"}, {"y"}}, std::move(batch_size_capturing_session), &batching_session)); TestSingleRequest(100.0f, 42.0f, batching_session.get()); @@ -253,10 +261,10 @@ TEST(BatchingSessionTest, UnsortedAllowedBatchSizesRejected) { BatchingSessionOptions batching_session_options; batching_session_options.allowed_batch_sizes = {4, 2}; // Not sorted. std::unique_ptr batching_session; - EXPECT_FALSE( - CreateBasicBatchingSession(schedule_options, batching_session_options, - CreateHalfPlusTwoSession(), &batching_session) - .ok()); + EXPECT_FALSE(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session) + .ok()); } TEST(BatchingSessionTest, @@ -266,10 +274,147 @@ TEST(BatchingSessionTest, BatchingSessionOptions batching_session_options; batching_session_options.allowed_batch_sizes = {2, 8}; // Final entry != 4. std::unique_ptr batching_session; - EXPECT_FALSE( - CreateBasicBatchingSession(schedule_options, batching_session_options, - CreateHalfPlusTwoSession(), &batching_session) - .ok()); + EXPECT_FALSE(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateHalfPlusTwoSession(), &batching_session) + .ok()); +} + +TEST(BatchingSessionTest, DifferentOrderForInputAndOutputTensors) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 6; // fits three 2-unit tasks + schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + schedule_options.num_batch_threads = 1; + BatchingSessionOptions batching_session_options; + std::unique_ptr batching_session; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x", "x2"}, {"y", "y2"}}, + CreateHalfPlusTwoSession(), &batching_session)); + + const Tensor input0 = test::AsTensor({8.0f, 6.0f}, {2}); + const Tensor expected_output0 = test::AsTensor({6.0f, 5.0f}, {2}); + const Tensor input1 = test::AsTensor({100.0f, 42.0f}, {2}); + const Tensor expected_output1 = test::AsTensor({53.0f, 24.0f}, {2}); + + std::unique_ptr first_request_thread( + Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x", input0}, {"x2", input1}}, + {"y", "y2"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(2, outputs.size()); + test::ExpectTensorEqual(expected_output0, outputs[0]); + test::ExpectTensorEqual(expected_output1, outputs[1]); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, + {"y2", "y"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(2, outputs.size()); + test::ExpectTensorEqual(expected_output1, outputs[0]); + test::ExpectTensorEqual(expected_output0, outputs[1]); + })); + std::unique_ptr third_request_thread( + Env::Default()->StartThread(ThreadOptions(), "third_request_thread", [&] { + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, + {"y", "y2"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(2, outputs.size()); + test::ExpectTensorEqual(expected_output0, outputs[0]); + test::ExpectTensorEqual(expected_output1, outputs[1]); + })); +} + +TEST(BatchingSessionTest, MultipleSignatures) { + std::vector*> schedulers; + auto create_scheduler = [&schedulers]( + std::function>)> + process_batch_callback, + std::unique_ptr>* scheduler) { + BasicBatchScheduler::Options options; + options.max_batch_size = 4; // fits two 2-unit tasks + options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + options.num_batch_threads = 1; + std::unique_ptr> basic_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + options, process_batch_callback, &basic_scheduler)); + schedulers.push_back(basic_scheduler.get()); + *scheduler = std::move(basic_scheduler); + return Status::OK(); + }; + BatchingSessionOptions batching_session_options; + std::unique_ptr batching_session; + TF_CHECK_OK(CreateBatchingSession( + batching_session_options, {{{{"x"}, {"y"}}, create_scheduler}, + {{{"x2"}, {"y2"}}, create_scheduler}}, + CreateHalfPlusTwoSession(), &batching_session)); + ASSERT_EQ(2, schedulers.size()); + + // Create lambdas for 2-unit inference requests to each signature. + auto run_signature0_request = [&batching_session] { + Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + Tensor expected_output = test::AsTensor({52.0f, 23.0f}, {2}); + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x", input}}, {"y"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(1, outputs.size()); + test::ExpectTensorEqual(expected_output, outputs[0]); + }; + auto run_signature1_request = [&batching_session] { + Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + Tensor expected_output = test::AsTensor({53.0f, 24.0f}, {2}); + std::vector outputs; + TF_ASSERT_OK(batching_session->Run({{"x2", input}}, {"y2"} /* outputs */, + {} /* target nodes */, &outputs)); + ASSERT_EQ(1, outputs.size()); + test::ExpectTensorEqual(expected_output, outputs[0]); + }; + + // Enqueue one request for each signature. Both should block because neither + // batching queue will be full yet. + std::unique_ptr signature0_thread(Env::Default()->StartThread( + ThreadOptions(), "signature0_thread", [&] { run_signature0_request(); })); + std::unique_ptr signature1_thread(Env::Default()->StartThread( + ThreadOptions(), "signature1_thread", [&] { run_signature1_request(); })); + while (schedulers[0]->NumEnqueuedTasks() != 1 && + schedulers[1]->NumEnqueuedTasks() != 1) { + Env::Default()->SleepForMicroseconds(100); + } + + // Enqueue a second request for each signature. This should fill both queues + // and unblock all the processing. + run_signature0_request(); + EXPECT_EQ(0, schedulers[0]->NumEnqueuedTasks()); + run_signature1_request(); + EXPECT_EQ(0, schedulers[1]->NumEnqueuedTasks()); +} + +// Test the backward compatibility layer. +// TODO(b/33476643): Remove. +TEST(BatchingSessionTest, BackwardCompatibilityLayer) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 4; // fits two 2-unit tasks + schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + schedule_options.num_batch_threads = 1; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, CreateHalfPlusTwoSession(), + &batching_session)); + + // Asynchronously send two requests whose total size is 4. The two requests in + // conjunction should trigger a batch to be processed. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request_thread", [&batching_session] { + TestSingleRequest(100.0f, 42.0f, batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request_thread", [&batching_session] { + TestSingleRequest(71.5f, 18.3f, batching_session.get()); + })); } } // namespace diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index a80d32a7c08..5c8350a885f 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -129,6 +129,7 @@ cc_library( "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resources_proto", "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc index 2a4fe4576c8..09ab693bc3b 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc @@ -74,6 +74,17 @@ uint64 GetTotalFileSize(const std::vector& files) { return total_file_size; } +SignatureDef GetTestSessionSignature() { + SignatureDef signature; + TensorInfo input; + input.set_name("x:0"); + (*signature.mutable_inputs())["x"] = input; + TensorInfo output; + output.set_name("y:0"); + (*signature.mutable_outputs())["y"] = output; + return signature; +} + void TestSingleRequest(Session* session) { Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); // half plus two: output should be input / 2 + 2. @@ -83,8 +94,8 @@ void TestSingleRequest(Session* session) { // Note that "x" and "y" are the actual names of the nodes in the graph. // The saved manifest binds these to "input" and "output" respectively, but // these tests are focused on the raw underlying session without bindings. - const std::vector> inputs = {{"x", input}}; - const std::vector output_names = {"y"}; + const std::vector> inputs = {{"x:0", input}}; + const std::vector output_names = {"y:0"}; const std::vector empty_targets; std::vector outputs; diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h index 093e34b137a..946b2103f28 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_TEST_UTIL_H_ #include +#include "tensorflow/core/protobuf/meta_graph.pb.h" #include "tensorflow/core/public/session.h" #include "tensorflow_serving/resources/resources.pb.h" @@ -42,6 +43,9 @@ std::vector GetTestSessionBundleExportFiles(); // Returns the total size of the given files. Requires the files to exist. uint64 GetTotalFileSize(const std::vector& files); +// Returns a signature for the half plus two model. +SignatureDef GetTestSessionSignature(); + // Test that a Session handles a single request for the half plus two // model properly. The request has size=2, for batching purposes. void TestSingleRequest(Session* session); diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc index fe5dcd469c1..9136d2f0a49 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc @@ -142,6 +142,7 @@ Status EstimateResourceFromPath(const string& path, Status WrapSessionForBatching(const BatchingParameters& batching_config, std::shared_ptr batch_scheduler, + const std::vector& signatures, std::unique_ptr* session) { LOG(INFO) << "Wrapping session to perform batch processing"; @@ -178,7 +179,17 @@ Status WrapSessionForBatching(const BatchingParameters& batching_config, queue_options, process_batch_callback, queue)); return Status::OK(); }; - return CreateBatchingSession(batching_session_options, create_queue, + std::vector + signatures_with_scheduler_creators; + for (const SignatureDef& signature : signatures) { + const TensorSignature tensor_signature = + TensorSignatureFromSignatureDef(signature); + signatures_with_scheduler_creators.push_back( + {tensor_signature, create_queue}); + } + + return CreateBatchingSession(batching_session_options, + signatures_with_scheduler_creators, std::move(*session), session); } diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h index dcf79187802..3d53d2d0cb2 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h @@ -54,10 +54,15 @@ Status CreateBatchScheduler( Status EstimateResourceFromPath(const string& path, ResourceAllocation* estimate); -// Wraps a session in a new session that automatically batches Run() calls. +// Wraps a session in a new session that automatically batches Run() calls, for +// the given signatures. +// TODO(b/33233998): Support batching for Run() calls that use a combination of +// signatures -- i.e. sometimes construct a single TensorSignature for a set of +// SignatureDefs (usually just two of them) -- based on some config. Status WrapSessionForBatching( const BatchingParameters& batching_config, std::shared_ptr> batch_scheduler, + const std::vector& signatures, std::unique_ptr* session); // Wraps a session in a new session that only supports Run() without batching. diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc index 10580401afb..02274ceb795 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc @@ -107,8 +107,9 @@ TEST_F(BundleFactoryUtilTest, WrapSessionForBatching) { TF_ASSERT_OK(CreateBatchScheduler(batching_params, &batcher)); // Wrap the session. - TF_ASSERT_OK( - WrapSessionForBatching(batching_params, batcher, &bundle.session)); + TF_ASSERT_OK(WrapSessionForBatching(batching_params, batcher, + {test_util::GetTestSessionSignature()}, + &bundle.session)); // Run multiple requests concurrently. They should be executed as 5 batches. test_util::TestMultipleRequests(10, bundle.session.get()); diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc index f176a13a978..f96e3c1d762 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -25,6 +25,20 @@ limitations under the License. namespace tensorflow { namespace serving { +namespace { + +// Extracts the signatures from 'bundle'. +std::vector GetSignatureDefs(const SavedModelBundle& bundle) { + std::vector signature_defs; + for (const auto& entry : bundle.meta_graph_def.signature_def()) { + const SignatureDef& signature_def = entry.second; + signature_defs.push_back(signature_def); + } + return signature_defs; +} + +} // namespace + Status SavedModelBundleFactory::Create( const SessionBundleConfig& config, std::unique_ptr* factory) { @@ -53,8 +67,13 @@ Status SavedModelBundleFactory::CreateSavedModelBundle( if (batch_scheduler_ == nullptr) { return errors::Internal("batch_scheduler_ not set"); } + // Enable batching of requests to any one signature_def in the SavedModel. + // Note that in the future, the plan is to enable explicit configuration of + // the one or many SignatureDefs to enable. + const std::vector signatures = GetSignatureDefs(**bundle); return WrapSessionForBatching(config_.batching_parameters(), - batch_scheduler_, &(*bundle)->session); + batch_scheduler_, signatures, + &(*bundle)->session); } return WrapSession(&(*bundle)->session); } diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc index e812d87a475..b899f41a68b 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc @@ -15,6 +15,7 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/session_bundle_factory.h" +#include "tensorflow/contrib/session_bundle/bundle_shim.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/public/session_options.h" @@ -23,6 +24,38 @@ limitations under the License. namespace tensorflow { namespace serving { +namespace { + +// Extracts the signatures from 'bundle'. +Status GetSignatureDefs(const SessionBundle& bundle, + std::vector* signature_defs) { + MetaGraphDef meta_graph_def = bundle.meta_graph_def; + const Status conversion_status = + internal::ConvertSignaturesToSignatureDefs(&meta_graph_def); + if (!conversion_status.ok()) { + if (meta_graph_def.signature_def().empty()) { + return conversion_status; + } else { + // ConvertSignaturesToSignatureDefs() is best-effort, and sometimes isn't + // able to up-convert all the signatures. Proceeding with the subset of + // signatures that *can* be converted seems to work for existing + // SessionBundle use-cases, so we'll do that. Future use-cases will be + // strongly compelled to use SavedModelBundle which won't have this issue. + LOG(WARNING) << "Couldn't fully convert the signatures [" + << conversion_status + << "]. Proceeding with a subset of signatures for use as " + "the batching queues"; + } + } + for (const auto& entry : meta_graph_def.signature_def()) { + const SignatureDef& signature_def = entry.second; + signature_defs->push_back(signature_def); + } + return Status::OK(); +} + +} // namespace + Status SessionBundleFactory::Create( const SessionBundleConfig& config, std::unique_ptr* factory) { @@ -51,8 +84,11 @@ Status SessionBundleFactory::CreateSessionBundle( if (batch_scheduler_ == nullptr) { return errors::Internal("batch_scheduler_ not set"); } + std::vector signatures; + TF_RETURN_IF_ERROR(GetSignatureDefs(**bundle, &signatures)); return WrapSessionForBatching(config_.batching_parameters(), - batch_scheduler_, &(*bundle)->session); + batch_scheduler_, signatures, + &(*bundle)->session); } return WrapSession(&(*bundle)->session); } From 0a425605a208db8406327580744fc8a2b96aed51 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Fri, 9 Dec 2016 15:20:25 -0800 Subject: [PATCH 0107/8554] Add FileProbingEnv interface in BundleFactoryUtil for resource estimation. Change: 141606540 --- tensorflow_serving/servables/tensorflow/BUILD | 1 + .../tensorflow/bundle_factory_util.cc | 20 +++-- .../tensorflow/bundle_factory_util.h | 9 ++- tensorflow_serving/util/BUILD | 9 +++ tensorflow_serving/util/file_probing_env.cc | 40 ++++++++++ tensorflow_serving/util/file_probing_env.h | 73 +++++++++++++++++++ 6 files changed, 144 insertions(+), 8 deletions(-) create mode 100644 tensorflow_serving/util/file_probing_env.cc create mode 100644 tensorflow_serving/util/file_probing_env.h diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 5c8350a885f..db4c04475fe 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -49,6 +49,7 @@ cc_library( "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/util:file_probing_env", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc index 9136d2f0a49..d14120efe12 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc @@ -38,9 +38,8 @@ constexpr int kResourceEstimateRAMPadBytes = 0; // Returns all the descendants, both directories and files, recursively under // 'dirname'. The paths returned are all prefixed with 'dirname'. -Status GetAllDescendants(const string& dirname, +Status GetAllDescendants(const string& dirname, FileProbingEnv* env, std::vector* const descendants) { - Env* const env = Env::Default(); descendants->clear(); // Make sure that dirname exists; TF_RETURN_IF_ERROR(env->FileExists(dirname)); @@ -114,16 +113,23 @@ Status CreateBatchScheduler(const BatchingParameters& batching_config, Status EstimateResourceFromPath(const string& path, ResourceAllocation* estimate) { - if (!Env::Default()->FileExists(path).ok()) { - return errors::NotFound("Nonexistent export path: ", path); + TensorflowFileProbingEnv env(Env::Default()); + return EstimateResourceFromPath(path, &env, estimate); +} + +Status EstimateResourceFromPath(const string& path, FileProbingEnv* env, + ResourceAllocation* estimate) { + if (env == nullptr) { + return errors::Internal("FileProbingEnv not set"); } + std::vector descendants; - TF_RETURN_IF_ERROR(GetAllDescendants(path, &descendants)); + TF_RETURN_IF_ERROR(GetAllDescendants(path, env, &descendants)); uint64 total_file_size = 0; for (const string& descendant : descendants) { - if (!(Env::Default()->IsDirectory(descendant).ok())) { + if (!(env->IsDirectory(descendant).ok())) { uint64 file_size; - TF_RETURN_IF_ERROR(Env::Default()->GetFileSize(descendant, &file_size)); + TF_RETURN_IF_ERROR(env->GetFileSize(descendant, &file_size)); total_file_size += file_size; } } diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h index 3d53d2d0cb2..ad19f301a19 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/util/file_probing_env.h" namespace tensorflow { namespace serving { @@ -45,7 +46,8 @@ Status CreateBatchScheduler( batch_scheduler); // Estimates the resources a session bundle or saved model bundle will use once -// loaded, from its export or saved model path. +// loaded, from its export or saved model path. tensorflow::Env::Default() will +// be used to access the file system. // // Uses the following crude heuristic, for now: estimated main-memory RAM = // (combined size of all exported file(s)) * kResourceEstimateRAMMultiplier + @@ -54,6 +56,11 @@ Status CreateBatchScheduler( Status EstimateResourceFromPath(const string& path, ResourceAllocation* estimate); +// Similar to the above function, but also supplies a FileProbingEnv to use in +// lieu of tensorflow::Env::Default(). +Status EstimateResourceFromPath(const string& path, FileProbingEnv* env, + ResourceAllocation* estimate); + // Wraps a session in a new session that automatically batches Run() calls, for // the given signatures. // TODO(b/33233998): Support batching for Run() calls that use a combination of diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index ac6362d15e2..668915e4904 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -254,3 +254,12 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", ], ) + +cc_library( + name = "file_probing_env", + srcs = ["file_probing_env.cc"], + hdrs = ["file_probing_env.h"], + deps = [ + "@org_tensorflow//tensorflow/core:lib", + ], +) diff --git a/tensorflow_serving/util/file_probing_env.cc b/tensorflow_serving/util/file_probing_env.cc new file mode 100644 index 00000000000..b3515d197b1 --- /dev/null +++ b/tensorflow_serving/util/file_probing_env.cc @@ -0,0 +1,40 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/util/file_probing_env.h" + +namespace tensorflow { +namespace serving { + +Status TensorflowFileProbingEnv::FileExists(const string& fname) { + return env_->FileExists(fname); +} + +Status TensorflowFileProbingEnv::GetChildren(const string& dir, + std::vector* children) { + return env_->GetChildren(dir, children); +} + +Status TensorflowFileProbingEnv::IsDirectory(const string& fname) { + return env_->IsDirectory(fname); +} + +Status TensorflowFileProbingEnv::GetFileSize(const string& fname, + uint64* file_size) { + return env_->GetFileSize(fname, file_size); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/util/file_probing_env.h b/tensorflow_serving/util/file_probing_env.h new file mode 100644 index 00000000000..8f5a94cb50b --- /dev/null +++ b/tensorflow_serving/util/file_probing_env.h @@ -0,0 +1,73 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_UTIL_FILE_PROBING_ENV_H_ +#define TENSORFLOW_SERVING_UTIL_FILE_PROBING_ENV_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/env.h" + +namespace tensorflow { +namespace serving { + +// An interface used to probe the contents of a file system. This allows file +// systems other than those supported by tensorflow::Env to be used. +class FileProbingEnv { + public: + virtual ~FileProbingEnv() = default; + + // Returns OK if the named path exists and NOT_FOUND otherwise. + virtual Status FileExists(const string& fname) = 0; + + // Stores in *children the names of the children of the specified + // directory. The names are relative to "dir". + // Original contents of *children are dropped. + virtual Status GetChildren(const string& dir, + std::vector* children) = 0; + + // Returns whether the given path is a directory or not. + // See tensorflow::Env::IsDirectory() for possible status code. + virtual Status IsDirectory(const string& fname) = 0; + + // Stores the size of `fname` in `*file_size`. + virtual Status GetFileSize(const string& fname, uint64* file_size) = 0; +}; + +// An implementation of FileProbingEnv which delegates the calls to +// tensorflow::Env. +class TensorflowFileProbingEnv : public FileProbingEnv { + public: + // 'env' is owned by the caller. + TensorflowFileProbingEnv(tensorflow::Env* env) : env_(env) {} + + ~TensorflowFileProbingEnv() override = default; + + Status FileExists(const string& fname) override; + + Status GetChildren(const string& dir, std::vector* children) override; + + Status IsDirectory(const string& fname) override; + + Status GetFileSize(const string& fname, uint64* file_size) override; + + private: + // Not owned. + tensorflow::Env* env_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_UTIL_FILE_PROBING_ENV_H_ From 5ce3f913354d443e5c05742c2fc7959038e2897b Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 9 Dec 2016 15:44:47 -0800 Subject: [PATCH 0108/8554] Fix build error in request_logger. Change: 141609172 --- tensorflow_serving/core/request_logger.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_serving/core/request_logger.h b/tensorflow_serving/core/request_logger.h index 290e5cd5ceb..8224f8d4f99 100644 --- a/tensorflow_serving/core/request_logger.h +++ b/tensorflow_serving/core/request_logger.h @@ -16,6 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ #define TENSORFLOW_SERVING_CORE_REQUEST_LOGGER_H_ +#include + #include "google/protobuf/message.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/config/logging_config.pb.h" From f83efeee8d20b705826103f8d69ef4cc996ad4e8 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 13 Dec 2016 14:23:32 -0800 Subject: [PATCH 0109/8554] Creates a class registration utility. Change: 141939417 --- tensorflow_serving/batching/BUILD | 2 +- .../batching/batching_session_test.cc | 2 +- tensorflow_serving/config/BUILD | 8 + .../config/log_collector_config.proto | 12 + .../config/logging_config.proto | 11 +- tensorflow_serving/core/BUILD | 1 + tensorflow_serving/core/log_collector.cc | 29 +- tensorflow_serving/core/log_collector.h | 37 +- tensorflow_serving/core/log_collector_test.cc | 55 +-- tensorflow_serving/model_servers/BUILD | 5 +- .../tensorflow_model_server_test.py | 5 +- .../model_servers/test_util/BUILD | 5 +- .../test_util/server_core_test_util.cc | 2 +- tensorflow_serving/servables/tensorflow/BUILD | 19 +- .../tensorflow/bundle_factory_test_util.cc | 4 +- .../servables/tensorflow/predict_impl.cc | 42 +- .../servables/tensorflow/predict_impl_test.cc | 41 +- .../tensorflow/simple_servers_test.cc | 2 +- .../servables/tensorflow/testdata/BUILD | 3 +- .../testdata/export_half_plus_two.py | 3 +- .../testdata/half_plus_two/00000123/export | Bin 209 -> 0 bytes .../00000123/export.data-00000-of-00001 | Bin 0 -> 8 bytes .../half_plus_two/00000123/export.index | Bin 0 -> 142 bytes .../half_plus_two/00000123/export.meta | Bin 4273 -> 4266 bytes tensorflow_serving/util/BUILD | 52 +++ tensorflow_serving/util/class_registration.h | 364 ++++++++++++++++++ .../util/class_registration_test.cc | 273 +++++++++++++ .../util/class_registration_test.proto | 19 + .../util/class_registration_util.cc | 33 ++ .../util/class_registration_util.h | 34 ++ 30 files changed, 969 insertions(+), 94 deletions(-) create mode 100644 tensorflow_serving/config/log_collector_config.proto delete mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.index create mode 100644 tensorflow_serving/util/class_registration.h create mode 100644 tensorflow_serving/util/class_registration_test.cc create mode 100644 tensorflow_serving/util/class_registration_test.proto create mode 100644 tensorflow_serving/util/class_registration_util.cc create mode 100644 tensorflow_serving/util/class_registration_util.h diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index f31c26bef7a..17d30c0d7b4 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -164,7 +164,7 @@ cc_test( "batching_session_test.cc", ], data = [ - "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], deps = [ ":batching_session", diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index a326010ea08..8bdca26e241 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -70,7 +70,7 @@ std::unique_ptr CreateHalfPlusTwoSession() { tensorflow::SessionOptions session_options; tensorflow::RunOptions run_options; const string export_dir = test_util::TensorflowTestSrcDirPath( - "python/saved_model/example/saved_model_half_plus_two/00000123"); + "cc/saved_model/testdata/half_plus_two/00000123"); SavedModelBundle bundle; TF_CHECK_OK(LoadSavedModel(session_options, run_options, export_dir, {kSavedModelTagServe}, &bundle)); diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index a777bee760c..6786ea7b0ac 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -32,6 +32,14 @@ serving_proto_library( ], ) +serving_proto_library( + name = "log_collector_config_proto", + srcs = ["log_collector_config.proto"], + cc_api_version = 2, + deps = [ + ], +) + serving_proto_library( name = "logging_config_proto", srcs = ["logging_config.proto"], diff --git a/tensorflow_serving/config/log_collector_config.proto b/tensorflow_serving/config/log_collector_config.proto new file mode 100644 index 00000000000..4ce01d34a29 --- /dev/null +++ b/tensorflow_serving/config/log_collector_config.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +message LogCollectorConfig { + // Identifies the type of the LogCollector we will use to collect these logs. + string type = 1; + + // The prefix to use for the filenames of the logs. + string filename_prefix = 2; +} diff --git a/tensorflow_serving/config/logging_config.proto b/tensorflow_serving/config/logging_config.proto index ffa7998d879..8ae4d4c9b30 100644 --- a/tensorflow_serving/config/logging_config.proto +++ b/tensorflow_serving/config/logging_config.proto @@ -3,6 +3,8 @@ syntax = "proto3"; package tensorflow.serving; option cc_enable_arenas = true; +import "tensorflow_serving/config/log_collector_config.proto"; + message SamplingConfig { // Requests will be logged uniformly at random with this probability. Valid // range: [0, 1.0]. @@ -11,11 +13,6 @@ message SamplingConfig { // Configuration for logging query/responses. message LoggingConfig { - // Identifies the type of the LogCollector we will use to collect these logs. - string type = 1; - - // The prefix to use for the filenames of the logs. - string filename_prefix = 2; - - SamplingConfig sampling_config = 3; + LogCollectorConfig log_collector_config = 1; + SamplingConfig sampling_config = 2; } diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 6612fab9d26..694e6d16437 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -663,6 +663,7 @@ cc_library( "//visibility:public", ], deps = [ + "//tensorflow_serving/config:log_collector_config_proto", "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/core/log_collector.cc b/tensorflow_serving/core/log_collector.cc index a391d0f283a..0dc18aacebb 100644 --- a/tensorflow_serving/core/log_collector.cc +++ b/tensorflow_serving/core/log_collector.cc @@ -26,14 +26,6 @@ namespace tensorflow { namespace serving { namespace { -bool ParseLognamePrefix(StringPiece remaining, StringPiece* type) { - return strings::Scanner(remaining) - .OneLiteral("/") - .RestartCapture() - .ScanUntil('/') - .GetResult(&remaining, type); -} - // This class is thread-safe. class Registry { public: @@ -64,30 +56,27 @@ class Registry { GUARDED_BY(mu_); }; -auto* registry_ = new Registry(); +Registry* GetRegistry() { + static auto* registry = new Registry(); + return registry; +} } // namespace Status LogCollector::RegisterFactory(const string& type, const Factory& factory) { - return registry_->Register(type, factory); + return GetRegistry()->Register(type, factory); } Status LogCollector::Create( - const string& logname_prefix, const uint32 id, + const LogCollectorConfig& config, const uint32 id, std::unique_ptr* const log_collector) { - StringPiece remaining(logname_prefix); - StringPiece type; - if (!ParseLognamePrefix(remaining, &type)) { - return errors::InvalidArgument("Invalid logname_prefix: ", logname_prefix); - } - - auto* factory = registry_->Lookup(type.ToString()); + auto* factory = GetRegistry()->Lookup(config.type()); if (factory == nullptr) { return errors::NotFound("Cannot find LogCollector::Factory for type: ", - type); + config.type()); } - return (*factory)(remaining.ToString(), id, log_collector); + return (*factory)(config, id, log_collector); } } // namespace serving diff --git a/tensorflow_serving/core/log_collector.h b/tensorflow_serving/core/log_collector.h index e69502bf700..3fef09bdbef 100644 --- a/tensorflow_serving/core/log_collector.h +++ b/tensorflow_serving/core/log_collector.h @@ -22,6 +22,7 @@ limitations under the License. #include "google/protobuf/message.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/config/log_collector_config.pb.h" namespace tensorflow { namespace serving { @@ -35,13 +36,13 @@ class LogCollector { public: virtual ~LogCollector() = default; - // Creates a log-collector for a given 'logname_prefix' and 'id'. The - // 'logname_prefix' should be of the form '/type/', so - // that the factory registered for the type can then be used to create the - // log-collector. The 'id' argument helps in disambiguating logs from - // replicated servers (processes), so it could be a combination of + // Creates a log-collector for a given 'log_collector_config' and 'id'. The + // factory registered for the type, mentioned in the config, can then be used + // to create the log-collector. The 'id' argument helps in disambiguating logs + // from replicated servers (processes), so it could be a combination of // task-id and replica-id or process-id and timestamp, etc. - static Status Create(const string& logname_prefix, const uint32 id, + static Status Create(const LogCollectorConfig& log_collector_config, + const uint32 id, std::unique_ptr* log_collector); using Factory = std::function; @@ -58,7 +59,31 @@ class LogCollector { virtual Status Flush() = 0; }; +namespace register_log_collector { + +struct RegisterFactory { + RegisterFactory(const string& type, const LogCollector::Factory& factory) { + // This check happens during global object construction time, even before + // control reaches main(), so we are ok with the crash. + TF_CHECK_OK(LogCollector::RegisterFactory(type, factory)); // Crash ok. + } +}; + +} // namespace register_log_collector + } // namespace serving } // namespace tensorflow +#define REGISTER_LOG_COLLECTOR_UNIQ_HELPER(ctr, type, factory) \ + REGISTER_LOG_COLLECTOR_UNIQ(ctr, type, factory) +#define REGISTER_LOG_COLLECTOR_UNIQ(ctr, type, factory) \ + static ::tensorflow::serving::register_log_collector::RegisterFactory \ + register_lgc##ctr TF_ATTRIBUTE_UNUSED = \ + ::tensorflow::serving::register_log_collector::RegisterFactory( \ + type, factory) + +// Registers a LogCollector factory implementation for a type. +#define REGISTER_LOG_COLLECTOR(type, factory) \ + REGISTER_LOG_COLLECTOR_UNIQ_HELPER(__COUNTER__, type, factory) + #endif // TENSORFLOW_SERVING_CORE_LOG_COLLECTOR_H_ diff --git a/tensorflow_serving/core/log_collector_test.cc b/tensorflow_serving/core/log_collector_test.cc index adf11f921a1..ff68babd756 100644 --- a/tensorflow_serving/core/log_collector_test.cc +++ b/tensorflow_serving/core/log_collector_test.cc @@ -18,6 +18,7 @@ limitations under the License. #include #include "tensorflow/core/lib/core/error_codes.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/config/log_collector_config.pb.h" namespace tensorflow { namespace serving { @@ -29,34 +30,43 @@ class FakeLogCollector : public LogCollector { Status Flush() override { return Status::OK(); } }; +LogCollectorConfig CreateConfig(const string& type, + const string& filename_prefix) { + LogCollectorConfig config; + config.set_type(type); + config.set_filename_prefix(filename_prefix); + return config; +} + TEST(LogCollectorTest, NotRegistered) { std::unique_ptr log_collector; - const auto status = - LogCollector::Create("/notregistered/logname_prefix", 0, &log_collector); + const auto status = LogCollector::Create( + CreateConfig("notregistered", "filename_prefix"), 0, &log_collector); EXPECT_EQ(status.code(), error::NOT_FOUND); } TEST(LogCollectorTest, Registration) { TF_ASSERT_OK(LogCollector::RegisterFactory( - "registered", [](const string& logname_prefix, const uint32 id, + "registered", [](const LogCollectorConfig& config, const uint32 id, std::unique_ptr* log_collector) { *log_collector = std::unique_ptr(new FakeLogCollector()); return Status::OK(); })); std::unique_ptr log_collector; - TF_ASSERT_OK( - LogCollector::Create("/registered/logname_prefix", 0, &log_collector)); + TF_ASSERT_OK(LogCollector::Create( + CreateConfig("registered", "filename_prefix"), 0, &log_collector)); } +auto duplicate_factory = [](const LogCollectorConfig& config, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return Status::OK(); +}; +REGISTER_LOG_COLLECTOR("duplicate", duplicate_factory); + TEST(LogCollectorTest, DuplicateRegistration) { - TF_ASSERT_OK(LogCollector::RegisterFactory( - "duplicate", [](const string& logname_prefix, const uint32 id, - std::unique_ptr* log_collector) { - *log_collector = std::unique_ptr(new FakeLogCollector()); - return Status::OK(); - })); const auto status = LogCollector::RegisterFactory( - "duplicate", [](const string& logname_prefix, const uint32 id, + "duplicate", [](const LogCollectorConfig& config, const uint32 id, std::unique_ptr* log_collector) { *log_collector = std::unique_ptr(new FakeLogCollector()); return Status::OK(); @@ -64,19 +74,20 @@ TEST(LogCollectorTest, DuplicateRegistration) { EXPECT_EQ(status.code(), error::ALREADY_EXISTS); } +auto creation_factory = [](const LogCollectorConfig& config, const uint32 id, + std::unique_ptr* log_collector) { + *log_collector = std::unique_ptr(new FakeLogCollector()); + return Status::OK(); +}; +REGISTER_LOG_COLLECTOR("creation", creation_factory); + TEST(LogCollectorTest, Creation) { - TF_ASSERT_OK(LogCollector::RegisterFactory( - "creation", [](const string& logname_prefix, const uint32 id, - std::unique_ptr* log_collector) { - *log_collector = std::unique_ptr(new FakeLogCollector()); - return Status::OK(); - })); std::unique_ptr log_collector0; - TF_ASSERT_OK( - LogCollector::Create("/creation/logname_prefix", 0, &log_collector0)); + TF_ASSERT_OK(LogCollector::Create(CreateConfig("creation", "filename_prefix"), + 0, &log_collector0)); std::unique_ptr log_collector1; - TF_ASSERT_OK( - LogCollector::Create("/creation/logname_prefix", 0, &log_collector1)); + TF_ASSERT_OK(LogCollector::Create(CreateConfig("creation", "filename_prefix"), + 0, &log_collector1)); EXPECT_NE(log_collector0.get(), log_collector1.get()); } diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index bfc01621206..3e4fc310bf7 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -116,9 +116,10 @@ py_test( ":tensorflow_model_server", "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export", "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export.meta", - "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", - "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], tags = [ "local", diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 4d7ed12d88d..247f255e448 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -106,9 +106,8 @@ def VerifyPredictRequest(self, def _GetSavedModelBundlePath(self): """Returns a path to a model in SavedModel format.""" - return os.path.join(os.environ['TEST_SRCDIR'], - 'tf_serving/external/org_tensorflow/tensorflow/', - 'python/saved_model/example/saved_model_half_plus_two') + return os.path.join(os.environ['TEST_SRCDIR'], 'tf_serving/external/org_tensorflow/tensorflow/', + 'cc/saved_model/testdata/half_plus_two') def _GetSessionBundlePath(self): """Returns a path to a model in SessionBundle format.""" diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 8a91db49651..887625c03e2 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -46,9 +46,10 @@ cc_library( srcs = ["server_core_test_util.cc"], hdrs = ["server_core_test_util.h"], data = [ - "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", - "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], visibility = [ "//visibility:public", diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 195f1d584b1..fdc06dfc52d 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -31,7 +31,7 @@ ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { model->set_name(kTestModelName); if (GetTestType() == SAVED_MODEL) { model->set_base_path(test_util::TensorflowTestSrcDirPath( - "/python/saved_model/example/saved_model_half_plus_two")); + "/cc/saved_model/testdata/half_plus_two")); } else { model->set_base_path(test_util::TestSrcDirPath( "/servables/tensorflow/testdata/half_plus_two")); diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index db4c04475fe..3cfc51ab714 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -61,7 +61,7 @@ cc_test( name = "bundle_factory_util_test", size = "medium", srcs = ["bundle_factory_util_test.cc"], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + data = ["@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_half_plus_two"], deps = [ ":bundle_factory_test_util", ":bundle_factory_util", @@ -142,7 +142,7 @@ cc_test( name = "session_bundle_factory_test", size = "medium", srcs = ["session_bundle_factory_test.cc"], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + data = ["@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_half_plus_two"], deps = [ ":bundle_factory_test", ":bundle_factory_test_util", @@ -188,8 +188,8 @@ cc_test( size = "medium", srcs = ["saved_model_bundle_factory_test.cc"], data = [ - "@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two", - "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + "@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_half_plus_two", ], deps = [ ":bundle_factory_test", @@ -233,7 +233,7 @@ cc_test( name = "session_bundle_source_adapter_test", size = "medium", srcs = ["session_bundle_source_adapter_test.cc"], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + data = ["@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_half_plus_two"], # Link in all registered kernels. linkstatic = 1, deps = [ @@ -281,8 +281,8 @@ cc_test( size = "medium", srcs = ["saved_model_bundle_source_adapter_test.cc"], data = [ - "@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two", - "@org_tensorflow//tensorflow/python/saved_model/example:saved_model_half_plus_two_data", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + "@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_half_plus_two", ], # Link in all registered kernels. linkstatic = 1, @@ -345,7 +345,7 @@ cc_library( cc_test( name = "simple_servers_test", srcs = ["simple_servers_test.cc"], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle/example:half_plus_two"], + data = ["@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_half_plus_two"], # Link in all registered kernels. linkstatic = 1, deps = [ @@ -400,7 +400,8 @@ cc_test( data = [ "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export", "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export.meta", - "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", ], deps = [ diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc index 09ab693bc3b..fc8f327c8f6 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test_util.cc @@ -32,9 +32,9 @@ namespace test_util { namespace { const char kTestSavedModelPath[] = - "python/saved_model/example/saved_model_half_plus_two/00000123"; + "cc/saved_model/testdata/half_plus_two/00000123"; const char kTestSessionBundleExportPath[] = - "session_bundle/example/half_plus_two/00000123"; + "session_bundle/testdata/half_plus_two/00000123"; } // namespace diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index 6b4e005db48..42bd0971ad3 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -134,10 +134,13 @@ Status PreProcessPrediction(const SignatureDef& signature, std::vector>* inputs, std::vector* output_tensor_names, std::vector* output_tensor_aliases) { - if (signature.method_name() != kPredictMethodName) { + if (signature.method_name() != kPredictMethodName && + signature.method_name() != kClassifyMethodName && + signature.method_name() != kRegressMethodName) { return errors::Internal(strings::StrCat( - "Expected prediction signature method_name to be ", kPredictMethodName, - ". Was: ", signature.method_name())); + "Expected prediction signature method_name to be one of {", + kPredictMethodName, ", ", kClassifyMethodName, ", ", kRegressMethodName, + "}. Was: ", signature.method_name())); } if (signature.inputs().empty()) { return errors::Internal(strings::StrCat( @@ -155,18 +158,24 @@ Status PreProcessPrediction(const SignatureDef& signature, } for (auto& input : request.inputs()) { const string& alias = input.first; - auto iter = signature.inputs().find(alias); - if (iter == signature.inputs().end()) { - return tensorflow::Status( - tensorflow::error::INVALID_ARGUMENT, - "input tensor alias not found in signature: " + alias); + // When using a Prediction signature, tensors are aliased and the name is + // retrieved from the Tensor value, otherwise the alias (key) is the name. + string tensor_name = alias; + if (signature.method_name() == kPredictMethodName) { + auto iter = signature.inputs().find(alias); + if (iter == signature.inputs().end()) { + return tensorflow::Status( + tensorflow::error::INVALID_ARGUMENT, + "input tensor alias not found in signature: " + alias); + } + tensor_name = iter->second.name(); } Tensor tensor; if (!tensor.FromProto(input.second)) { return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, "tensor parsing error: " + alias); } - inputs->emplace_back(std::make_pair(iter->second.name(), tensor)); + inputs->emplace_back(std::make_pair(tensor_name, tensor)); } // Prepare run target. @@ -193,7 +202,13 @@ Status PreProcessPrediction(const SignatureDef& signature, if (output_tensor_names->empty()) { for (auto& iter : signature.outputs()) { output_tensor_names->emplace_back(iter.second.name()); - output_tensor_aliases->emplace_back(iter.first); + // When using a Prediction signature, the tensor output alias is the key + // in the map, otherwise we don't use aliases and just go by actual tensor + // names. + const string alias = signature.method_name() == kPredictMethodName + ? iter.first + : iter.second.name(); + output_tensor_aliases->emplace_back(alias); } } return Status::OK(); @@ -222,8 +237,11 @@ Status SavedModelPredict(ServerCore* core, const PredictRequest& request, // Validate signatures. ServableHandle bundle; TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); - auto iter = bundle->meta_graph_def.signature_def().find( - kDefaultServingSignatureDefKey); + + const string signature_name = request.model_spec().signature_name().empty() + ? kDefaultServingSignatureDefKey + : request.model_spec().signature_name(); + auto iter = bundle->meta_graph_def.signature_def().find(signature_name); if (iter == bundle->meta_graph_def.signature_def().end()) { return errors::FailedPrecondition( "Default serving signature key not found."); diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index afcd713bde1..9cd41563cdc 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -246,9 +246,44 @@ TEST_P(PredictImplTest, PredictionSuccess) { output_tensor_proto.add_float_val(3); output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); output_tensor_proto.mutable_tensor_shape(); - PredictResponse test_response; - (*test_response.mutable_outputs())[kOutputTensorKey] = output_tensor_proto; - EXPECT_THAT(test_response, test_util::EqualsProto(response)); + PredictResponse expected_response; + (*expected_response.mutable_outputs())[kOutputTensorKey] = + output_tensor_proto; + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +// Test querying a model with a named regression signature (not default). This +// will work with SavedModel but not supported in the legacy SessionBundle. +TEST_P(PredictImplTest, PredictionWithNamedRegressionSignature) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + model_spec->set_signature_name("regress"); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())["Placeholder:0"] = tensor_proto; + + TensorflowPredictor predictor(GetParam()); + // This request is expected to work with SavedModel, but not SessionBundle. + if (GetParam()) { + TF_ASSERT_OK(predictor.Predict(GetServerCore(), request, &response)); + } else { + ASSERT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetServerCore(), request, &response).code()); + return; + } + TensorProto output_tensor_proto; + output_tensor_proto.add_float_val(3); + output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); + output_tensor_proto.mutable_tensor_shape(); + PredictResponse expected_response; + (*expected_response.mutable_outputs())["Add:0"] = output_tensor_proto; + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } // Test all ClassifierTest test cases with both SessionBundle and SavedModel. diff --git a/tensorflow_serving/servables/tensorflow/simple_servers_test.cc b/tensorflow_serving/servables/tensorflow/simple_servers_test.cc index 57954274129..fe5ce7a0c4e 100644 --- a/tensorflow_serving/servables/tensorflow/simple_servers_test.cc +++ b/tensorflow_serving/servables/tensorflow/simple_servers_test.cc @@ -41,7 +41,7 @@ class SimpleServersTest : public ::testing::Test { protected: SimpleServersTest() : test_data_path_(test_util::ContribTestSrcDirPath( - "session_bundle/example/half_plus_two")) {} + "session_bundle/testdata/half_plus_two")) {} // Test that a SessionBundle handles a single request for the half plus two // model properly. The request has size=2, for batching purposes. diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 208b10f1dc7..64629175c6f 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -49,7 +49,8 @@ py_binary( # Note: re-generate these files with :export_half_plus_two whenever model # changes. exports_files([ - "half_plus_two/00000123/export", + "half_plus_two/00000123/export.data-00000-of-00001", + "half_plus_two/00000123/export.index", "half_plus_two/00000123/export.meta", "bad_half_plus_two/00000123/export", "bad_half_plus_two/00000123/export.meta", diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py index 0fbdd27d598..2222f3b194a 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py +++ b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py @@ -45,7 +45,8 @@ def Export(): export = exporter.Exporter(tf.train.Saver()) export.init(named_graph_signatures={ "inputs": exporter.generic_signature({"x": x}), - "outputs": exporter.generic_signature({"y": y}) + "outputs": exporter.generic_signature({"y": y}), + "regress": exporter.regression_signature(x, y) }) export.export(export_path, tf.constant(123), sess) diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export deleted file mode 100644 index 1e3c29c52b096bae1dbf6673683ab526fcfd0519..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 209 zcmZQz(BabJ65!$pODxJvOv*_WVvt}|V&D?u;)0398$!f{m^c_2c*GcB;tY%oLL$f- zrP#Ds7#JArIawv*4MDPEsD?o09e`?pvOsY4xOy3!bKuj$4GfG-j0`n!p)_NKx<*Eh TK1M!}5PlH;yP;d9)crO9rhOq4 diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..20bc7d454dd8450489984bd17d92c45c3a1a96a6 GIT binary patch literal 8 PcmZQzV6bOkU~m8c0fPX5 literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.index b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two/00000123/export.index new file mode 100644 index 0000000000000000000000000000000000000000..35e2ef7527905b228ebfd7f755bb3d06f6cc00e0 GIT binary patch literal 142 zcmZQzVB=tvV&Y(A;NT8REXqtw%1Py56k^a|F=ew*m*-&OjW-1G6HV%7>7Bh{>Ymqajt0;xry3Tf))>nWAUX@BoYu?)e1@m1}sw_jf1f*pHpB! z2!R;F0y?Q_Fy)^y&+izvnJpv{Z}(VK*ZUO;L(1`zy!+_xXIb#Atn2AS4PpBScV8Pu`22+fL0 z3dE4O)C;ICSm6_&(8Vkqe_?QJ8ZIAB&n_bkfepefb7ustk4582Xx8FhWYnluEbGNO zF)dwex({6Mk&E{Zl93#rXhN2@k18FV$1X~CdK+?mqf(~~JCJT~JVy}g`-6`|D$p5T z^IRG#{j}0iW5YEc5~_@W)}o6ua?gSQQ#ywbJY}z(6liMgKhijW; zFU9Pr!#(ynemXT0q;0XvE+jso8TK=AMLd7%YVO7*j-5&VMqk*1oRK~t^*w@jh+US? osSUgYVnDV7LMhIH0!fH(IOZ0qdyYkd6%C8PS1fBqW4>JZ18f9T&j0`b delta 1191 zcmb7?&2JJx7{-|ehUHzpIzR(kQemw<-~l9hsTu_nZ)#%kWaDN71Ko%V*&R^Jp$B73 z^kOqNO+0w?Vj2^@n3(zCP0ehRpl&c(%1tK9#QPAm0M@mUXTl;e5FHneAg>D9)hsNylGcHyWjl#NsI4)r zhMG#+z9N4pJQ2v(52;`w9J652vVj$6;YG;uTVgZafu-rN;t}~L@y^XLM5l8yV?+jb ze64R-Og`XMW&*L~dE`b^Nk~rS7(z7xAsF+WXjOuHQxU!=<@p<&;eVo;wJt2X8>aUV zUZ%4eB{7MTHAH+Y;w2xWX$A}H=Vgjxmo3s59 z0htGpLmfF3$`Q`xW7sO^Affh$Y%pZ}J5HSy-SP-kR9>F1j?!|1a!d===g)~xLxLbI|3AavR#TRn&ct0b^_@Vw2@R9!IW0`Y9=HKw* z3F3XrU&PO(eGKz2@t+9z`^4o)F?=C#wQ!yPO8hRobjm}`k#2NVOYa&Q8`_m7TjfVE N;!^57s&hS6{|iP-h{^x} diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 668915e4904..4c369babf76 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -18,6 +18,7 @@ filegroup( exclude = [ "**/METADATA", "**/OWNERS", + "**/google_*", ], ), ) @@ -41,6 +42,21 @@ cc_library( visibility = ["//visibility:public"], ) +cc_library( + name = "class_registration", + hdrs = [ + "class_registration.h", + ], + visibility = ["//visibility:public"], + deps = [ + ":class_registration_util", + "@org_tensorflow//tensorflow/core", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:tensorflow", + "@protobuf//:cc_wkt_protos", + ], +) + ############################################################################### # Internal targets ############################################################################### @@ -263,3 +279,39 @@ cc_library( "@org_tensorflow//tensorflow/core:lib", ], ) + +cc_library( + name = "class_registration_util", + srcs = ["class_registration_util.cc"], + hdrs = ["class_registration_util.h"], + deps = [ + "@org_tensorflow//tensorflow/core", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:tensorflow", + "@protobuf//:cc_wkt_protos", + ], +) + +cc_test( + name = "class_registration_test", + srcs = [ + "class_registration_test.cc", + ], + deps = [ + ":class_registration", + ":class_registration_test_proto", + "//tensorflow_serving/core/test_util:test_main", + ], +) + +load("//tensorflow_serving:serving.bzl", "serving_proto_library") + +serving_proto_library( + name = "class_registration_test_proto", + testonly = 1, + srcs = ["class_registration_test.proto"], + cc_api_version = 2, + deps = [ + "@protobuf//:cc_wkt_protos", + ], +) diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h new file mode 100644 index 00000000000..083cc71f1c1 --- /dev/null +++ b/tensorflow_serving/util/class_registration.h @@ -0,0 +1,364 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// A way to register subclasses of an abstract base class, and associate a +// config proto message type. Instances can be instantiated from an Any proto +// field that contains a config proto, based on the type and content of the +// config proto. +// +// IMPORTANT: Config protos used in registries must be compiled into the binary. +// +// Each registry has a name. Registry names in each namespace must be distinct. +// A registry is tied to a specific base class and factory signature. It is fine +// to have multiple registries for a given base class, whether having the same +// factory signature or multiple distinct signatures. +// +// Usage: +// +// // Define a base class. +// class MyBaseClass { +// ... +// }; +// +// // Define a registry that maps from proto message types to subclasses of +// // MyBaseClass. +// DEFINE_CLASS_REGISTRY(MyBaseClassRegistry, MyBaseClass); +// +// // Define a macro used create a specific entry in MyBaseClassRegistry that +// // maps from ConfigProto to ClassCreator::Create(). +// #define REGISTER_MY_BASE_CLASS(ClassCreator, ConfigProto) +// REGISTER_CLASS(MyBaseClassRegistry, MyBaseClass, ClassCreator, +// ConfigProto); +// +// // Declare a subclass of MyBaseClass to be created when a OneConfigProto +// // is passed to the Create*() factory methods. +// class OneClass : public MyBaseClass { +// public: +// static Status Create(const OneConfigProto& config, +// std::unique_ptr* result) { +// OneClass* raw_result = new OneClass(); +// raw_result->config_ = config; +// Status status = raw_result->Init(); +// if (status.ok()) { +// result->reset(raw_result); +// } +// return status; +// } +// +// private: +// Status Init() { +// ... initialize the object based on 'config_' +// } +// +// OneConfigProto config_; +// }; +// REGISTER_MY_BASE_CLASS(OneClass, OneConfigProto); +// +// // Create an object of type OneClass using the registry to switch on +// // the type OneConfigProto. +// OneConfigProto config = ... +// std::unique_ptr loaded_subclass; +// CHECK_OK(MyBaseClassRegistry::Create(config, &loaded_subclass)); +// +// // Same, but starting from an Any message that wraps a OneConfigProto. +// protobuf::Any any_config = ... // wraps a OneConfigProto +// std::unique_ptr loaded_subclass; +// CHECK_OK(MyBaseClassRegistry::CreateFromAny(any_config, &loaded_subclass)); +// +// Note that the subclass creator need not be the subclass itself. For example: +// +// class AnotherClass : public MyBaseClass { +// public: +// AnotherClass(int a, int b); +// ... +// }; +// +// class CreatorForAnotherClass { +// public: +// static Status Create(const OneConfigProto& config, +// std::unique_ptr* result) { +// result->reset(new AnotherClass(config.a(), config.b())); +// return Status::OK; +// } +// }; +// +// REGISTER_MY_BASE_CLASS(CreatorForAnotherClass, OneConfigProto); +// +// +// This mechanism also allows additional parameter passing into the Create() +// factory method. Consider the following example in which Create() takes an +// int and a string, in addition to the config proto: +// +// class MyParameterizedBaseClass { +// ... +// }; +// DEFINE_CLASS_REGISTRY(MyParameterizedBaseClassRegistry, +// MyParameterizedBaseClass, int, const string& +// const std::map&); +// #define REGISTER_MY_BASE_CLASS(ClassCreator, ConfigProto) +// REGISTER_CLASS(MyBaseClassRegistry, MyBaseClass, ClassCreator, +// ConfigProto, int, const string&, +// const std::map&); +// +// class OneClass : public MyParameterizedBaseClass { +// public: +// static Status Create(const OneConfigProto& config, +// int param1, const string& param2, +// const std::map& param3, +// std::unique_ptr* result) { +// ... +// } +// ... +// }; +// +// OneConfigProto config = ... +// int int_param = ... +// string string_param = ... +// std::map map_param = ... +// std::unique_ptr loaded_subclass; +// CHECK_OK(MyParameterizedBaseClassRegistry::Create(config, +// int_param, +// string_param, +// map_param, +// &loaded_subclass)); +// +// The registry name can be anything you choose, and it's fine to have multiple +// registries for the same base class, potentially with different factory +// signatures. +// +// Note that types containing a comma, e.g. std::map must use COMMA +// in place of ','. +// TODO(b/24472377): Eliminate the requirement to use COMMA. + +#ifndef TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_H_ +#define TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_H_ + +#include +#include +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/descriptor.h" +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/protobuf.h" +#include "tensorflow/core/platform/thread_annotations.h" +#include "tensorflow/core/util/port.h" +#include "tensorflow_serving/util/class_registration_util.h" + +namespace tensorflow { +namespace serving { +namespace internal { + +// The interface for a factory method that takes a protobuf::Message as +// input to construct an object of type BaseClass. +template +class AbstractClassRegistrationFactory { + public: + virtual ~AbstractClassRegistrationFactory() = default; + + // Creates an object using this factory. Fails if 'config' is not of the + // expected type. + virtual Status Create(const protobuf::Message& config, + AdditionalFactoryArgs... args, + std::unique_ptr* result) const = 0; +}; + +// The interface for a factory method that takes a protobuf::Message as +// input to construct an object of type BaseClass. +template +class ClassRegistrationFactory + : public AbstractClassRegistrationFactory { + public: + ClassRegistrationFactory() + : config_descriptor_(Config::default_instance().GetDescriptor()) {} + + // Creates an object using this factory. Fails if 'config' is not of the + // expected type. + Status Create(const protobuf::Message& config, AdditionalFactoryArgs... args, + std::unique_ptr* result) const override { + if (config.GetDescriptor() != config_descriptor_) { + return errors::InvalidArgument( + "Supplied config proto of type ", config.GetDescriptor()->full_name(), + " does not match expected type ", config_descriptor_->full_name()); + } + return Class::Create(static_cast(config), + std::forward(args)..., result); + } + + private: + const protobuf::Descriptor* const config_descriptor_; + + TF_DISALLOW_COPY_AND_ASSIGN(ClassRegistrationFactory); +}; + +constexpr char kTypeGoogleApisComPrefix[] = "type.googleapis.com/"; + +// A static map whose keys are proto message names, and values are +// ClassRegistrationFactories. Includes a Create() factory method that +// performs a lookup in the map and calls Create() on the +// ClassRegistrationFactory it finds. +template +class ClassRegistry { + public: + using FactoryType = + AbstractClassRegistrationFactory; + + // Creates an instance of BaseClass based on a config proto. + static Status Create(const protobuf::Message& config, + AdditionalFactoryArgs... args, + std::unique_ptr* result) { + const string& config_proto_message_type = + config.GetDescriptor()->full_name(); + auto* factory = LookupFromMap(config_proto_message_type); + if (factory == nullptr) { + return errors::InvalidArgument( + "Couldn't find factory for config proto message type ", + config_proto_message_type, "\nconfig=", config.DebugString()); + } + return factory->Create(config, std::forward(args)..., + result); + } + + // Creates an instance of BaseClass based on a config proto embedded in an Any + // message. + // + // Requires that the config proto in the Any has a compiled-in descriptor. + static Status CreateFromAny(const google::protobuf::Any& any_config, + AdditionalFactoryArgs... args, + std::unique_ptr* result) { + // Copy the config to a proto message of the indicated type. + string full_type_name; + Status parse_status = + ParseUrlForAnyType(any_config.type_url(), &full_type_name); + if (!parse_status.ok()) { + return parse_status; + } + const protobuf::Descriptor* descriptor = + protobuf::DescriptorPool::generated_pool()->FindMessageTypeByName( + full_type_name); + if (descriptor == nullptr) { + return errors::Internal( + "Unable to find compiled-in proto descriptor of type ", + full_type_name); + } + std::unique_ptr config( + protobuf::MessageFactory::generated_factory() + ->GetPrototype(descriptor) + ->New()); + any_config.UnpackTo(config.get()); + return Create(*config, std::forward(args)..., + result); + } + + // Nested class whose instantiation inserts a key/value pair into the factory + // map. + class MapInserter { + public: + MapInserter(const string& config_proto_message_type, FactoryType* factory) { + InsertIntoMap(config_proto_message_type, factory); + } + }; + + private: + // Inserts a key/value pair into the factory map. + static void InsertIntoMap(const string& config_proto_message_type, + FactoryType* factory) { + LockableFactoryMap* global_map = GlobalFactoryMap(); + { + mutex_lock lock(global_map->mu); + global_map->factory_map.insert({config_proto_message_type, factory}); + } + } + + // Retrieves a value from the factory map, or returns nullptr if no value was + // found. + static FactoryType* LookupFromMap(const string& config_proto_message_type) { + LockableFactoryMap* global_map = GlobalFactoryMap(); + { + mutex_lock lock(global_map->mu); + auto it = global_map->factory_map.find(config_proto_message_type); + if (it == global_map->factory_map.end()) { + return nullptr; + } + return it->second; + } + } + + // A map from proto descriptor names to factories, with a lock. + struct LockableFactoryMap { + mutex mu; + std::unordered_map factory_map GUARDED_BY(mu); + }; + + // Returns a pointer to the factory map. There is one factory map per set of + // template parameters of this class. + static LockableFactoryMap* GlobalFactoryMap() { + static auto* global_map = []() -> LockableFactoryMap* { + return new LockableFactoryMap; + }(); + return global_map; + } + + TF_DISALLOW_COPY_AND_ASSIGN(ClassRegistry); +}; + +} // namespace internal + +// Used to enable the following macros to work with types containing commas +// (e.g. map). +// TODO(b/24472377): Eliminate the requirement to use COMMA, via some fancy +// macro gymnastics. +#define COMMA , + +// Given a base class BaseClass, creates a registry named RegistryName. +#define DEFINE_CLASS_REGISTRY(RegistryName, BaseClass, ...) \ + class RegistryName : public ::tensorflow::serving::internal::ClassRegistry< \ + RegistryName, BaseClass, ##__VA_ARGS__> {}; + +// Registers a factory that creates subclasses of BaseClass by calling +// ClassCreator::Create(). +#define REGISTER_CLASS(RegistryName, BaseClass, ClassCreator, config_proto, \ + ...) \ + REGISTER_CLASS_UNIQ_HELPER(__COUNTER__, RegistryName, BaseClass, \ + ClassCreator, config_proto, ##__VA_ARGS__) + +#define REGISTER_CLASS_UNIQ_HELPER(cnt, RegistryName, BaseClass, ClassCreator, \ + config_proto, ...) \ + REGISTER_CLASS_UNIQ(cnt, RegistryName, BaseClass, ClassCreator, \ + config_proto, ##__VA_ARGS__) + +#define REGISTER_CLASS_UNIQ(cnt, RegistryName, BaseClass, ClassCreator, \ + config_proto, ...) \ + static ::tensorflow::serving::internal::ClassRegistry< \ + RegistryName, BaseClass, ##__VA_ARGS__>::MapInserter \ + register_class_##cnt( \ + (config_proto::default_instance().GetDescriptor()->full_name()), \ + (new ::tensorflow::serving::internal::ClassRegistrationFactory< \ + BaseClass, ClassCreator, config_proto, ##__VA_ARGS__>)); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_H_ diff --git a/tensorflow_serving/util/class_registration_test.cc b/tensorflow_serving/util/class_registration_test.cc new file mode 100644 index 00000000000..ad779f7f703 --- /dev/null +++ b/tensorflow_serving/util/class_registration_test.cc @@ -0,0 +1,273 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/util/class_registration.h" + +#include + +#include +#include +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/util/class_registration_test.pb.h" + +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +namespace tensorflow { +namespace serving { +namespace { + +// A base class and associated registry. +class MyBaseClass { + public: + virtual ~MyBaseClass() = default; + virtual string class_name() const = 0; + virtual string config_data() const = 0; +}; +DEFINE_CLASS_REGISTRY(MyBaseClassRegistry, MyBaseClass); +#define REGISTER_MY_BASE_CLASS(SubClassCreator, ConfigProto) \ + REGISTER_CLASS(MyBaseClassRegistry, MyBaseClass, SubClassCreator, \ + ConfigProto); + +// A subclass of MyBaseClass that should be instantiated via Config1. +class SubClass1 : public MyBaseClass { + public: + static Status Create(const Config1& config, + std::unique_ptr* result) { + if (config.GetDescriptor()->full_name() != "tensorflow.serving.Config1") { + return errors::InvalidArgument("Wrong type of config proto: ", + config.GetDescriptor()->full_name()); + } + auto* raw_result = new SubClass1(); + result->reset(raw_result); + raw_result->config_ = config; + return Status::OK(); + } + + string class_name() const override { return "SubClass1"; } + + string config_data() const override { return config_.string_field(); } + + private: + Config1 config_; +}; +REGISTER_MY_BASE_CLASS(SubClass1, Config1); + +// A subclass of MyBaseClass that should be instantiated via Config2. +class SubClass2 : public MyBaseClass { + public: + static Status Create(const Config2& config, + std::unique_ptr* result) { + if (config.GetDescriptor()->full_name() != "tensorflow.serving.Config2") { + return errors::InvalidArgument("Wrong type of config proto: ", + config.GetDescriptor()->full_name()); + } + auto* raw_result = new SubClass2(); + result->reset(raw_result); + raw_result->config_ = config; + return Status::OK(); + } + + string class_name() const override { return "SubClass2"; } + + string config_data() const override { return config_.string_field(); } + + private: + Config2 config_; +}; + +// A creator of SubClass2 objects. +class SubClass2Creator { + public: + static Status Create(const Config2& config, + std::unique_ptr* result) { + return SubClass2::Create(config, result); + } +}; +REGISTER_MY_BASE_CLASS(SubClass2Creator, Config2); + +TEST(ClassRegistrationTest, InstantiateFromRawConfig) { + std::unique_ptr loaded_subclass; + + Config1 config1; + config1.set_string_field("foo"); + TF_ASSERT_OK(MyBaseClassRegistry::Create(config1, &loaded_subclass)); + EXPECT_EQ("SubClass1", loaded_subclass->class_name()); + EXPECT_EQ("foo", loaded_subclass->config_data()); + + Config2 config2; + config2.set_string_field("bar"); + TF_ASSERT_OK(MyBaseClassRegistry::Create(config2, &loaded_subclass)); + EXPECT_EQ("SubClass2", loaded_subclass->class_name()); + EXPECT_EQ("bar", loaded_subclass->config_data()); +} + +TEST(ClassRegistrationTest, InstantiateFromAny) { + std::unique_ptr loaded_subclass; + + Config1 config1; + config1.set_string_field("foo"); + google::protobuf::Any any_config1; + any_config1.PackFrom(config1); + TF_ASSERT_OK( + MyBaseClassRegistry::CreateFromAny(any_config1, &loaded_subclass)); + EXPECT_EQ("SubClass1", loaded_subclass->class_name()); + EXPECT_EQ("foo", loaded_subclass->config_data()); + + Config2 config2; + config2.set_string_field("bar"); + google::protobuf::Any any_config2; + any_config2.PackFrom(config2); + TF_ASSERT_OK( + MyBaseClassRegistry::CreateFromAny(any_config2, &loaded_subclass)); + EXPECT_EQ("SubClass2", loaded_subclass->class_name()); + EXPECT_EQ("bar", loaded_subclass->config_data()); +} + +// A second registry for MyBaseClass, with a different name. +DEFINE_CLASS_REGISTRY(AlternateMyBaseClassRegistry, MyBaseClass); +#define REGISTER_MY_BASE_CLASS_USING_ALTERNATE_REGISTRY(SubClassCreator, \ + ConfigProto) \ + REGISTER_CLASS(AlternateMyBaseClassRegistry, MyBaseClass, SubClassCreator, \ + ConfigProto); + +// A subclass of MyBaseClass that should be instantiated via Config1, and is +// registered in the alternate registry. +class AlternateSubClass : public MyBaseClass { + public: + static Status Create(const Config1& config, + std::unique_ptr* result) { + if (config.GetDescriptor()->full_name() != "tensorflow.serving.Config1") { + return errors::InvalidArgument("Wrong type of config proto: ", + config.GetDescriptor()->full_name()); + } + auto* raw_result = new AlternateSubClass(); + result->reset(raw_result); + raw_result->config_ = config; + return Status::OK(); + } + + string class_name() const override { return "AlternateSubClass"; } + + string config_data() const override { return config_.string_field(); } + + private: + Config1 config_; +}; +REGISTER_MY_BASE_CLASS_USING_ALTERNATE_REGISTRY(AlternateSubClass, Config1); + +TEST(ClassRegistrationTest, MultipleRegistriesForSameBaseClass) { + std::unique_ptr loaded_subclass; + + Config1 config; + TF_ASSERT_OK(MyBaseClassRegistry::Create(config, &loaded_subclass)); + EXPECT_EQ("SubClass1", loaded_subclass->class_name()); + + TF_ASSERT_OK(AlternateMyBaseClassRegistry::Create(config, &loaded_subclass)); + EXPECT_EQ("AlternateSubClass", loaded_subclass->class_name()); +} + +// A base class whose subclasses' Create() methods take additional parameters, +// and associated registry. +class MyParameterizedBaseClass { + public: + virtual ~MyParameterizedBaseClass() = default; + virtual string class_name() const = 0; + virtual string config_data() const = 0; + virtual int param1_data() const = 0; + virtual string param2_data() const = 0; + virtual const std::map& param3_data() const = 0; +}; +DEFINE_CLASS_REGISTRY(MyParameterizedBaseClassRegistry, + MyParameterizedBaseClass, int, const string&, + const std::map&); +#define REGISTER_MY_PARAMETERIZED_BASE_CLASS(SubClassCreator, ConfigProto) \ + REGISTER_CLASS(MyParameterizedBaseClassRegistry, MyParameterizedBaseClass, \ + SubClassCreator, ConfigProto, int, const string&, \ + const std::map&); + +// A subclass of MyParameterizedBaseClass that should be instantiated via +// Config1. +class ParameterizedSubClass1 : public MyParameterizedBaseClass { + public: + static Status Create(const Config1& config, int param1, const string& param2, + const std::map& param3, + std::unique_ptr* result) { + if (config.GetDescriptor()->full_name() != "tensorflow.serving.Config1") { + return errors::InvalidArgument("Wrong type of config proto: ", + config.GetDescriptor()->full_name()); + } + auto* raw_result = new ParameterizedSubClass1(); + result->reset(raw_result); + raw_result->config_ = config; + raw_result->param1_ = param1; + raw_result->param2_ = param2; + raw_result->param3_ = param3; + return Status::OK(); + } + + string class_name() const override { return "ParameterizedSubClass1"; } + + string config_data() const override { return config_.string_field(); } + + int param1_data() const override { return param1_; } + + string param2_data() const override { return param2_; } + + const std::map& param3_data() const override { return param3_; } + + private: + Config1 config_; + int param1_; + string param2_; + std::map param3_; +}; +REGISTER_MY_PARAMETERIZED_BASE_CLASS(ParameterizedSubClass1, Config1); + +TEST(ClassRegistrationTest, InstantiateParameterizedFromRawConfig) { + std::unique_ptr loaded_subclass; + + Config1 config1; + config1.set_string_field("foo"); + TF_ASSERT_OK(MyParameterizedBaseClassRegistry::Create( + config1, 42, "bar", {{"floop", 1}, {"mrop", 2}}, &loaded_subclass)); + EXPECT_EQ("ParameterizedSubClass1", loaded_subclass->class_name()); + EXPECT_EQ("foo", loaded_subclass->config_data()); + EXPECT_EQ(42, loaded_subclass->param1_data()); + EXPECT_EQ("bar", loaded_subclass->param2_data()); + EXPECT_THAT(loaded_subclass->param3_data(), + UnorderedElementsAre(Pair("floop", 1), Pair("mrop", 2))); +} + +TEST(ClassRegistrationTest, InstantiateParameterizedFromAny) { + std::unique_ptr loaded_subclass; + + Config1 config1; + config1.set_string_field("foo"); + google::protobuf::Any any_config1; + any_config1.PackFrom(config1); + TF_ASSERT_OK(MyParameterizedBaseClassRegistry::CreateFromAny( + any_config1, 42, "bar", {{"floop", 1}, {"mrop", 2}}, &loaded_subclass)); + EXPECT_EQ("ParameterizedSubClass1", loaded_subclass->class_name()); + EXPECT_EQ("foo", loaded_subclass->config_data()); + EXPECT_EQ(42, loaded_subclass->param1_data()); + EXPECT_EQ("bar", loaded_subclass->param2_data()); + EXPECT_THAT(loaded_subclass->param3_data(), + UnorderedElementsAre(Pair("floop", 1), Pair("mrop", 2))); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/util/class_registration_test.proto b/tensorflow_serving/util/class_registration_test.proto new file mode 100644 index 00000000000..ff542109a79 --- /dev/null +++ b/tensorflow_serving/util/class_registration_test.proto @@ -0,0 +1,19 @@ +// Proto messages used by class_registration_test.cc. + +syntax = "proto3"; + +import "google/protobuf/any.proto"; + +package tensorflow.serving; + +message Config1 { + string string_field = 1; +} + +message Config2 { + string string_field = 1; +} + +message MessageWithAny { + google.protobuf.Any any_field = 1; +} diff --git a/tensorflow_serving/util/class_registration_util.cc b/tensorflow_serving/util/class_registration_util.cc new file mode 100644 index 00000000000..bb2fae354b8 --- /dev/null +++ b/tensorflow_serving/util/class_registration_util.cc @@ -0,0 +1,33 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/util/class_registration_util.h" + +namespace tensorflow { +namespace serving { + +Status ParseUrlForAnyType(const string& type_url, + string* const full_type_name) { + std::vector splits = str_util::Split(type_url, '/'); + if (splits.size() < 2 || splits[splits.size() - 1].empty()) { + return errors::InvalidArgument( + "Supplied config's type_url could not be parsed: ", type_url); + } + *full_type_name = splits[splits.size() - 1]; + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/util/class_registration_util.h b/tensorflow_serving/util/class_registration_util.h new file mode 100644 index 00000000000..dd102eacf1e --- /dev/null +++ b/tensorflow_serving/util/class_registration_util.h @@ -0,0 +1,34 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_UTIL_H_ +#define TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_UTIL_H_ + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/strings/str_util.h" + +namespace tensorflow { +namespace serving { + +// Parses a url whose final '/' is followed by a proto type name, e.g. +// "type.googleapis.com/some_namespace.some_proto_type_name". +// Returns Status::OK() iff parsing succeeded. +Status ParseUrlForAnyType(const string& type_url, string* const full_type_name); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_UTIL_H_ From a3ef60b1dfd867f463cf628fe1134c3e9aaf687f Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 14 Dec 2016 09:27:03 -0800 Subject: [PATCH 0110/8554] Remove BatchingSession backward-compatibility layer. Change: 142025947 --- .../batching/batching_session.cc | 104 ------------------ .../batching/batching_session.h | 20 ---- .../batching/batching_session_test.cc | 25 ----- 3 files changed, 149 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index 6549d06f092..d3bb6eb3e72 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -511,109 +511,5 @@ Status CreateBasicBatchingSession( std::move(session), batching_session); } -////////// -// Backward-compatibility layer for old APIs. -// TODO(b/33476643): Remove. - -// A ServingSession implementation that wraps a BatchingSession that it -// constructs upon receiving its first Run() call (and deducing the signature -// to use for batching). -class SignatureInferringBatchingSession : public ServingSession { - public: - SignatureInferringBatchingSession( - const BatchingSessionOptions& options, - std::unique_ptr raw_session, - BatchingSessionSchedulerCreator batch_scheduler_creator); - - ~SignatureInferringBatchingSession() override = default; - - Status Run(const std::vector>& inputs, - const std::vector& output_tensor_names, - const std::vector& target_node_names, - std::vector* outputs) override; - - private: - const BatchingSessionOptions options_; - const BatchingSessionSchedulerCreator scheduler_creator_; - - // Exactly one of these is populated at a given time. Initially, - // 'raw_session_' is populated. Upon the first Run() call, 'batching_session_' - // is created from 'raw_session_' (and 'raw_session_' is cleared). - std::unique_ptr raw_session_; - std::unique_ptr batching_session_; - - mutable mutex batching_session_creation_mu_; - - TF_DISALLOW_COPY_AND_ASSIGN(SignatureInferringBatchingSession); -}; - -SignatureInferringBatchingSession::SignatureInferringBatchingSession( - const BatchingSessionOptions& options, std::unique_ptr raw_session, - BatchingSessionSchedulerCreator batch_scheduler_creator) - : options_(options), - scheduler_creator_(batch_scheduler_creator), - raw_session_(std::move(raw_session)) {} - -Status SignatureInferringBatchingSession::Run( - const std::vector>& inputs, - const std::vector& output_tensor_names, - const std::vector& target_node_names, - std::vector* outputs) { - { - mutex_lock l(batching_session_creation_mu_); - if (batching_session_ == nullptr) { - // This is the first Run() call. Set up 'batching_session_'. - const TensorSignature signature = - TensorSignatureFromRunArgs(inputs, output_tensor_names); - TF_RETURN_IF_ERROR( - CreateBatchingSession(options_, {{signature, scheduler_creator_}}, - std::move(raw_session_), &batching_session_)); - } - } - return batching_session_->Run(inputs, output_tensor_names, target_node_names, - outputs); -} - -Status CreateBatchingSession( - const BatchingSessionOptions& options, - BatchingSessionSchedulerCreator batch_scheduler_creator, - std::unique_ptr session, - std::unique_ptr* batching_session) { - batching_session->reset(new SignatureInferringBatchingSession( - options, std::move(session), batch_scheduler_creator)); - return Status::OK(); -} - -Status CreateBasicBatchingSession( - const BasicBatchScheduler::Options& schedule_options, - const BatchingSessionOptions& batching_session_options, - std::unique_ptr session, - std::unique_ptr* batching_session) { - if (!batching_session_options.allowed_batch_sizes.empty()) { - if (batching_session_options.allowed_batch_sizes.back() != - schedule_options.max_batch_size) { - return errors::InvalidArgument( - "Last entry in allowed_batch_sizes must match max_batch_size; last " - "entry was ", - batching_session_options.allowed_batch_sizes.back(), "; expected ", - schedule_options.max_batch_size); - } - } - - auto scheduler_creator = [schedule_options]( - std::function>)> - process_batch_callback, - std::unique_ptr>* batch_scheduler) { - std::unique_ptr> - basic_batch_scheduler; - TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( - schedule_options, process_batch_callback, &basic_batch_scheduler)); - *batch_scheduler = std::move(basic_batch_scheduler); - return Status::OK(); - }; - return CreateBatchingSession(batching_session_options, scheduler_creator, - std::move(session), batching_session); -} - } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/batching/batching_session.h b/tensorflow_serving/batching/batching_session.h index a31165d87e4..7093907819e 100644 --- a/tensorflow_serving/batching/batching_session.h +++ b/tensorflow_serving/batching/batching_session.h @@ -159,26 +159,6 @@ Status CreateBasicBatchingSession( const TensorSignature& signature, std::unique_ptr session, std::unique_ptr* batching_session); -////////// -// DEPRECATED. New code should not use the following. -// -// Backward-compatibility layer for old API. This layer interprets the signature -// of the first Run() call as the one and only batching signature. -// TODO(b/33476643): Remove after migrating all clients. -Status CreateBatchingSession( - const BatchingSessionOptions& options, - BatchingSessionSchedulerCreator batch_scheduler_creator, - std::unique_ptr session, - std::unique_ptr* batching_session); -Status CreateBasicBatchingSession( - const typename BasicBatchScheduler::Options& - schedule_options, - const BatchingSessionOptions& batching_session_options, - std::unique_ptr session, - std::unique_ptr* batching_session); -// -////////// - ////////// // Implementation details follow. API users need not read. diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index 8bdca26e241..384065544d3 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -392,31 +392,6 @@ TEST(BatchingSessionTest, MultipleSignatures) { EXPECT_EQ(0, schedulers[1]->NumEnqueuedTasks()); } -// Test the backward compatibility layer. -// TODO(b/33476643): Remove. -TEST(BatchingSessionTest, BackwardCompatibilityLayer) { - BasicBatchScheduler::Options schedule_options; - schedule_options.max_batch_size = 4; // fits two 2-unit tasks - schedule_options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger - schedule_options.num_batch_threads = 1; - std::unique_ptr batching_session; - BatchingSessionOptions batching_session_options; - TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, CreateHalfPlusTwoSession(), - &batching_session)); - - // Asynchronously send two requests whose total size is 4. The two requests in - // conjunction should trigger a batch to be processed. - std::unique_ptr first_request_thread(Env::Default()->StartThread( - ThreadOptions(), "first_request_thread", [&batching_session] { - TestSingleRequest(100.0f, 42.0f, batching_session.get()); - })); - std::unique_ptr second_request_thread(Env::Default()->StartThread( - ThreadOptions(), "second_request_thread", [&batching_session] { - TestSingleRequest(71.5f, 18.3f, batching_session.get()); - })); -} - } // namespace } // namespace serving } // namespace tensorflow From 7b4f6902bcbabd1092ffbd82a460aaf2948d68c4 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 14 Dec 2016 11:57:58 -0800 Subject: [PATCH 0111/8554] Set minimum bazel requirements to 0.4.2 since it's a new TensorFlow requirement. Change: 142045403 --- WORKSPACE | 2 +- tensorflow_serving/g3doc/setup.md | 8 ++++---- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 39f4bcd6e71..eae521aa96a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,4 +11,4 @@ tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") -check_version("0.3.1") +check_version("0.4.2") diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 03e44c7fb61..58028e35292 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -6,7 +6,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel -TensorFlow Serving requires Bazel 0.3.2 or higher. You can find the Bazel +TensorFlow Serving requires Bazel 0.4.2 or higher. You can find the Bazel installation instructions [here](http://bazel.build/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the @@ -14,13 +14,13 @@ following steps: 1. Download the relevant binary from [here](https://github.com/bazelbuild/bazel/releases). - Let's say you downloaded bazel-0.3.2-installer-linux-x86_64.sh. You would + Let's say you downloaded bazel-0.4.2-installer-linux-x86_64.sh. You would execute: ~~~shell cd ~/Downloads - chmod +x bazel-0.3.2-installer-linux-x86_64.sh - ./bazel-0.3.2-installer-linux-x86_64.sh --user + chmod +x bazel-0.4.2-installer-linux-x86_64.sh + ./bazel-0.4.2-installer-linux-x86_64.sh --user ~~~ 2. Set up your environment. Put this in your ~/.bashrc. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index f0e42edc8ad..ad0cc711410 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -56,7 +56,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.3.2 +ENV BAZEL_VERSION 0.4.2 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ From 6ed30042696b881737c6f42b4fc98614b45d5d3b Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 14 Dec 2016 12:03:47 -0800 Subject: [PATCH 0112/8554] Fix build breakage in logging config. Change: 142046167 --- tensorflow_serving/config/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 6786ea7b0ac..fa6a6c5ab8a 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -45,5 +45,6 @@ serving_proto_library( srcs = ["logging_config.proto"], cc_api_version = 2, deps = [ + ":log_collector_config_proto", ], ) From c42f4f2162d8e803fa4079ad5b00d6c43265dff8 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 15 Dec 2016 10:18:16 -0800 Subject: [PATCH 0113/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 123a0f9a421..dbe5e17e2ed 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 123a0f9a421af8a7b40d9cbf9b2905e86218bac3 +Subproject commit dbe5e17e2ed307e86e1a6e79e558ec3e335d46fc diff --git a/tf_models b/tf_models index e81ff571108..12f279d6f4c 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit e81ff571108dd817e4a85ebbfecd666e3906eef1 +Subproject commit 12f279d6f4cb33574bc20109b41eb8a59f40cfd1 From ebb9c87bcaa062c64607b442a4ee038f7e80257d Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 15 Dec 2016 17:16:59 -0800 Subject: [PATCH 0114/8554] Internal change. Change: 142208699 --- tensorflow_serving/apis/prediction_service.proto | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/apis/prediction_service.proto b/tensorflow_serving/apis/prediction_service.proto index 28bdd528a7a..58df05af119 100644 --- a/tensorflow_serving/apis/prediction_service.proto +++ b/tensorflow_serving/apis/prediction_service.proto @@ -5,9 +5,10 @@ option cc_enable_arenas = true; import "tensorflow_serving/apis/predict.proto"; +// open source marker; do not remove // PredictionService provides access to machine-learned models loaded by // model_servers. service PredictionService { // Predict -- provides access to loaded TensorFlow model. rpc Predict(PredictRequest) returns (PredictResponse); -} \ No newline at end of file +} From 0195f4c28d812c231d1e836d3d95e178a69d50e3 Mon Sep 17 00:00:00 2001 From: Andrew Selle Date: Thu, 15 Dec 2016 17:35:42 -0800 Subject: [PATCH 0115/8554] Change remaining tf.mul -> tf.multiply, tf.neg -> tf.negative, and tf.sub -> tf.subtract tf.negative, tf.multiply, tf.subtract are the new names Change: 142210419 --- tensorflow_serving/example/inception_export.py | 4 ++-- .../servables/tensorflow/testdata/export_bad_half_plus_two.py | 2 +- .../servables/tensorflow/testdata/export_half_plus_two.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index 03df5f78998..1c5fbdf8157 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -156,8 +156,8 @@ def preprocess_image(image_buffer): align_corners=False) image = tf.squeeze(image, [0]) # Finally, rescale to [-1,1] instead of [0, 1) - image = tf.sub(image, 0.5) - image = tf.mul(image, 2.0) + image = tf.subtract(image, 0.5) + image = tf.multiply(image, 2.0) return image diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py b/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py index 3fe4e9a10f3..c44c55f357d 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py +++ b/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py @@ -36,7 +36,7 @@ def Export(): # Calculate, y = a*x + b # here we use a placeholder 'x' which is fed at inference time. x = tf.placeholder(tf.float32) - y = tf.add(tf.mul(a, x), b) + y = tf.add(tf.multiply(a, x), b) # Export the model without signatures. # Note that the model is intentionally exported without using exporter, diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py index 2222f3b194a..b90453d6404 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py +++ b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py @@ -38,7 +38,7 @@ def Export(): # Calculate, y = a*x + b # here we use a placeholder 'x' which is fed at inference time. x = tf.placeholder(tf.float32) - y = tf.add(tf.mul(a, x), b) + y = tf.add(tf.multiply(a, x), b) # Run an export. tf.initialize_all_variables().run() From 73f27c4695bafa19aef8a49457c6aa7a0f5f610c Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 16 Dec 2016 06:26:13 -0800 Subject: [PATCH 0116/8554] Restructure ServerCore to create the SourceAdapter via a registry, based on the "platform". (But for now it still only supports a single platform at a time.) Also clean up server_core_test_util. Change: 142254007 --- tensorflow_serving/config/BUILD | 9 ++ .../config/platform_config.proto | 19 ++++ tensorflow_serving/core/BUILD | 3 + tensorflow_serving/core/source_adapter.h | 13 +++ tensorflow_serving/core/test_util/BUILD | 11 +++ .../test_util/fake_loader_source_adapter.cc | 23 +++-- .../test_util/fake_loader_source_adapter.h | 7 +- .../fake_loader_source_adapter.proto | 8 ++ tensorflow_serving/model_servers/BUILD | 20 ++++ tensorflow_serving/model_servers/main.cc | 92 +++++++++---------- .../model_servers/model_platform_types.h | 7 +- .../model_servers/platform_config_util.cc | 48 ++++++++++ .../model_servers/platform_config_util.h | 35 +++++++ .../model_servers/server_core.cc | 58 +++++------- .../model_servers/server_core.h | 29 +----- .../model_servers/server_core_test.cc | 45 +++++---- .../model_servers/test_util/BUILD | 32 +++++++ .../test_util/mock_server_core.h | 20 +++- .../test_util/server_core_test_util.cc | 67 ++++++++------ .../test_util/server_core_test_util.h | 26 +++--- ...age_path_error_injecting_source_adapter.cc | 43 +++++++++ ...rage_path_error_injecting_source_adapter.h | 35 +++++++ ..._path_error_injecting_source_adapter.proto | 9 ++ tensorflow_serving/servables/tensorflow/BUILD | 16 ++++ .../servables/tensorflow/predict_impl_test.cc | 7 +- .../saved_model_bundle_source_adapter.cc | 18 ++++ .../saved_model_bundle_source_adapter.h | 8 +- .../saved_model_bundle_source_adapter.proto | 13 +++ .../session_bundle_source_adapter.cc | 17 ++++ .../session_bundle_source_adapter.h | 2 + tensorflow_serving/util/BUILD | 8 +- 31 files changed, 547 insertions(+), 201 deletions(-) create mode 100644 tensorflow_serving/config/platform_config.proto create mode 100644 tensorflow_serving/core/test_util/fake_loader_source_adapter.proto create mode 100644 tensorflow_serving/model_servers/platform_config_util.cc create mode 100644 tensorflow_serving/model_servers/platform_config_util.h create mode 100644 tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc create mode 100644 tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h create mode 100644 tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index fa6a6c5ab8a..2e9849b5761 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -32,6 +32,15 @@ serving_proto_library( ], ) +serving_proto_library( + name = "platform_config_proto", + srcs = ["platform_config.proto"], + cc_api_version = 2, + deps = [ + "@protobuf//:cc_wkt_protos", + ], +) + serving_proto_library( name = "log_collector_config_proto", srcs = ["log_collector_config.proto"], diff --git a/tensorflow_serving/config/platform_config.proto b/tensorflow_serving/config/platform_config.proto new file mode 100644 index 00000000000..4e506b38833 --- /dev/null +++ b/tensorflow_serving/config/platform_config.proto @@ -0,0 +1,19 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +import "google/protobuf/any.proto"; + +// Configuration for a servable platform e.g. tensorflow or other ML systems. +message PlatformConfig { + // The config proto for a SourceAdapter in the StoragePathSourceAdapter + // registry. + google.protobuf.Any source_adapter_config = 1; +}; + +message PlatformConfigMap { + // A map from a platform name to a platform config. The platform name is used + // in ModelConfig.model_platform. + map platform_configs = 1; +}; diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 694e6d16437..bbb01f2ebcb 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -145,9 +145,12 @@ cc_library( "//visibility:public", ], deps = [ + ":loader", ":servable_data", ":source", + ":storage_path", ":target", + "//tensorflow_serving/util:class_registration", "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/core/source_adapter.h b/tensorflow_serving/core/source_adapter.h index a6909e7096d..c7d9999f92a 100644 --- a/tensorflow_serving/core/source_adapter.h +++ b/tensorflow_serving/core/source_adapter.h @@ -25,9 +25,12 @@ limitations under the License. #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/source.h" +#include "tensorflow_serving/core/storage_path.h" #include "tensorflow_serving/core/target.h" +#include "tensorflow_serving/util/class_registration.h" namespace tensorflow { namespace serving { @@ -82,6 +85,16 @@ class SourceAdapter : public TargetBase, public Source { Notification outgoing_callback_set_; }; +// Define a SourceAdapter registry for the common case of adapting from a +// storage path to a loader. +using StoragePathSourceAdapter = + SourceAdapter>; +DEFINE_CLASS_REGISTRY(StoragePathSourceAdapterRegistry, + StoragePathSourceAdapter); +#define REGISTER_STORAGE_PATH_SOURCE_ADAPTER(ClassCreator, ConfigProto) \ + REGISTER_CLASS(StoragePathSourceAdapterRegistry, StoragePathSourceAdapter, \ + ClassCreator, ConfigProto); + // A source adapter that converts InputType instances to OutputType instances // one at a time (i.e. there is no interaction among members of a given aspired- // version list). Most source adapters can subclass UnarySourceAdapter, and do diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index ec94071788d..606ab9a8f9f 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -65,11 +65,22 @@ cc_library( hdrs = ["fake_loader_source_adapter.h"], visibility = ["//visibility:public"], deps = [ + ":fake_loader_source_adapter_proto", "//tensorflow_serving/core:simple_loader", "//tensorflow_serving/core:storage_path", ], ) +load("//tensorflow_serving:serving.bzl", "serving_proto_library") + +serving_proto_library( + name = "fake_loader_source_adapter_proto", + testonly = 1, + srcs = ["fake_loader_source_adapter.proto"], + cc_api_version = 2, + visibility = ["//visibility:public"], +) + cc_library( name = "fake_storage_path_source_adapter", testonly = 1, diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc index 42c0937b84e..42b051ebe53 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc @@ -42,17 +42,20 @@ FakeLoaderSourceAdapter::~FakeLoaderSourceAdapter() { } } -std::function>>*)> -FakeLoaderSourceAdapter::GetCreator() { - return [](const string& model_platform, - std::unique_ptr>>* source) { - source->reset(new FakeLoaderSourceAdapter); +// Register the source adapter. +class FakeLoaderSourceAdapterCreator { + public: + static Status Create( + const FakeLoaderSourceAdapterConfig& config, + std::unique_ptr>>* + adapter) { + adapter->reset(new FakeLoaderSourceAdapter); return Status::OK(); - }; -} + } +}; +REGISTER_STORAGE_PATH_SOURCE_ADAPTER(FakeLoaderSourceAdapterCreator, + FakeLoaderSourceAdapterConfig); + } // namespace test_util } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h index 1097569e70e..8f7bcfc08be 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/core/simple_loader.h" #include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.pb.h" namespace tensorflow { namespace serving { @@ -43,12 +44,6 @@ class FakeLoaderSourceAdapter final ~FakeLoaderSourceAdapter() override; - // Returns a function to create a fake source adapter. - static std::function>>*)> - GetCreator(); - private: const string suffix_; std::function call_on_destruct_; diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto b/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto new file mode 100644 index 00000000000..d170e32befe --- /dev/null +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto @@ -0,0 +1,8 @@ +syntax = "proto3"; + +package tensorflow.serving.test_util; + +// Config proto for FakeLoaderSourceAdapter. (Even though the adapter doesn't +// have any config parameters, the proto is needed for the adapter registry.) +message FakeLoaderSourceAdapterConfig { +} diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 3e4fc310bf7..734496dee75 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -30,6 +30,22 @@ cc_library( ], ) +cc_library( + name = "platform_config_util", + srcs = ["platform_config_util.cc"], + hdrs = ["platform_config_util.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//tensorflow_serving/config:platform_config_proto", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter_proto", + "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", + "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", + ], +) + cc_library( name = "server_core", srcs = ["server_core.cc"], @@ -41,6 +57,7 @@ cc_library( ":model_platform_types", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:model_server_config_proto", + "//tensorflow_serving/config:platform_config_proto", "//tensorflow_serving/core:aspired_versions_manager", "//tensorflow_serving/core:load_servables_fast", "//tensorflow_serving/core:servable_state_monitor", @@ -72,6 +89,8 @@ cc_test( "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/model_servers/test_util:server_core_test_util", + "//tensorflow_serving/model_servers/test_util:storage_path_error_injecting_source_adapter", + "//tensorflow_serving/model_servers/test_util:storage_path_error_injecting_source_adapter_proto", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:test", ], @@ -97,6 +116,7 @@ cc_binary( visibility = ["//tensorflow_serving:internal"], deps = [ ":model_platform_types", + ":platform_config_util", ":server_core", "@protobuf//:cc_wkt_protos", "@org_tensorflow//tensorflow/core:lib", diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 21032372ae5..780e032c116 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -57,7 +57,9 @@ limitations under the License. #include "grpc++/support/status_code_enum.h" #include "grpc/grpc.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/init_main.h" +#include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/command_line_flags.h" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" @@ -65,10 +67,9 @@ limitations under the License. #include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" -#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" using tensorflow::serving::AspiredVersionsManager; using tensorflow::serving::AspiredVersionPolicy; @@ -82,9 +83,7 @@ using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; using tensorflow::serving::ServerCore; -using tensorflow::serving::SavedModelBundleSourceAdapter; -using tensorflow::serving::SessionBundleSourceAdapter; -using tensorflow::serving::SessionBundleSourceAdapterConfig; +using tensorflow::serving::SessionBundleConfig; using tensorflow::serving::Target; using tensorflow::serving::TensorflowPredictor; using tensorflow::serving::UniquePtrWithDeps; @@ -102,34 +101,6 @@ using tensorflow::serving::PredictionService; namespace { -tensorflow::Status CreateSourceAdapter( - const SessionBundleSourceAdapterConfig& config, - const string& model_platform, bool use_saved_model, - std::unique_ptr* adapter) { - CHECK(model_platform == kTensorFlowModelPlatform) // Crash ok - << "ModelServer supports only TensorFlow model."; - if (use_saved_model) { - std::unique_ptr typed_adapter; - const ::tensorflow::Status status = - SavedModelBundleSourceAdapter::Create(config, &typed_adapter); - if (!status.ok()) { - VLOG(1) << "Error creating source adapter: " << status.error_message(); - return status; - } - *adapter = std::move(typed_adapter); - } else { - std::unique_ptr typed_adapter; - const ::tensorflow::Status status = - SessionBundleSourceAdapter::Create(config, &typed_adapter); - if (!status.ok()) { - VLOG(1) << "Error creating source adapter: " << status.error_message(); - return status; - } - *adapter = std::move(typed_adapter); - } - return tensorflow::Status::OK(); -} - tensorflow::Status LoadCustomModelConfig( const ::google::protobuf::Any& any, EventBus* servable_event_bus, @@ -151,7 +122,8 @@ ModelServerConfig BuildSingleModelConfig( config.mutable_model_config_list()->add_config(); single_model->set_name(model_name); single_model->set_base_path(model_base_path); - single_model->set_model_platform(kTensorFlowModelPlatform); + single_model->set_model_platform( + tensorflow::serving::kTensorFlowModelPlatform); single_model->set_version_policy(model_version_policy); return config; } @@ -206,6 +178,21 @@ void RunServer(int port, std::unique_ptr core, server->Wait(); } +// Parses an ascii PlatformConfigMap protobuf from 'file'. +tensorflow::serving::PlatformConfigMap ParsePlatformConfigMap( + const string& file) { + std::unique_ptr file_data; + TF_CHECK_OK( // Crash ok + tensorflow::Env::Default()->NewReadOnlyMemoryRegionFromFile(file, + &file_data)); + string file_data_str(static_cast(file_data->data()), + file_data->length()); + tensorflow::serving::PlatformConfigMap platform_config_map; + QCHECK(tensorflow::protobuf::TextFormat::ParseFromString( // Crash ok + file_data_str, &platform_config_map)); + return platform_config_map; +} + } // namespace int main(int argc, char** argv) { @@ -215,6 +202,7 @@ int main(int argc, char** argv) { tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; bool use_saved_model = false; + string platform_config_file = ""; tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( FileSystemStoragePathSourceConfig::LATEST_VERSION); @@ -239,7 +227,13 @@ int main(int argc, char** argv) { "If true, use SavedModel in the server; otherwise, use " "SessionBundle. It is used by tensorflow serving team " "to control the rollout of SavedModel and is not " - "expected to be set by users directly.")}; + "expected to be set by users directly."), + tensorflow::Flag("platform_config_file", &platform_config_file, + "If non-empty, read an ascii PlatformConfigMap protobuf " + "from the supplied file name, and use that platform " + "config instead of the Tensorflow platform. (If used, " + "--enable_batching and --use_saved_model are " + "ignored.)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || model_base_path.empty()) { @@ -265,23 +259,21 @@ int main(int argc, char** argv) { options.model_server_config = BuildSingleModelConfig( model_name, model_base_path, parsed_version_policy); - SessionBundleSourceAdapterConfig source_adapter_config; - // Batching config - if (enable_batching) { - BatchingParameters* batching_parameters = - source_adapter_config.mutable_config()->mutable_batching_parameters(); - batching_parameters->mutable_thread_pool_name()->set_value( - "model_server_batch_threads"); + if (platform_config_file.empty()) { + SessionBundleConfig session_bundle_config; + // Batching config + if (enable_batching) { + BatchingParameters* batching_parameters = + session_bundle_config.mutable_batching_parameters(); + batching_parameters->mutable_thread_pool_name()->set_value( + "model_server_batch_threads"); + } + options.platform_config_map = CreateTensorFlowPlatformConfigMap( + session_bundle_config, use_saved_model); + } else { + options.platform_config_map = ParsePlatformConfigMap(platform_config_file); } - options.use_saved_model = use_saved_model; - options.source_adapter_creator = [source_adapter_config, use_saved_model]( - const string& model_platform, - std::unique_ptr* adapter) { - return CreateSourceAdapter(source_adapter_config, model_platform, - use_saved_model, adapter); - }; - options.custom_model_config_loader = &LoadCustomModelConfig; options.aspired_version_policy = diff --git a/tensorflow_serving/model_servers/model_platform_types.h b/tensorflow_serving/model_servers/model_platform_types.h index 457ebbfe779..9701b192c1a 100644 --- a/tensorflow_serving/model_servers/model_platform_types.h +++ b/tensorflow_serving/model_servers/model_platform_types.h @@ -16,7 +16,12 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ #define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ +namespace tensorflow { +namespace serving { + constexpr char kTensorFlowModelPlatform[] = "tensorflow"; -constexpr char kOtherModelPlatform[] = "other"; + +} // namespace serving +} // namespace tensorflow #endif // TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ diff --git a/tensorflow_serving/model_servers/platform_config_util.cc b/tensorflow_serving/model_servers/platform_config_util.cc new file mode 100644 index 00000000000..fdd71bbab35 --- /dev/null +++ b/tensorflow_serving/model_servers/platform_config_util.cc @@ -0,0 +1,48 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/model_servers/platform_config_util.h" + +#include "google/protobuf/any.pb.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" + +namespace tensorflow { +namespace serving { + +PlatformConfigMap CreateTensorFlowPlatformConfigMap( + const SessionBundleConfig& session_bundle_config, bool use_saved_model) { + PlatformConfigMap platform_config_map; + ::google::protobuf::Any source_adapter_config; + if (use_saved_model) { + SavedModelBundleSourceAdapterConfig + saved_model_bundle_source_adapter_config; + *saved_model_bundle_source_adapter_config.mutable_legacy_config() = + session_bundle_config; + source_adapter_config.PackFrom(saved_model_bundle_source_adapter_config); + } else { + SessionBundleSourceAdapterConfig session_bundle_source_adapter_config; + *session_bundle_source_adapter_config.mutable_config() = + session_bundle_config; + source_adapter_config.PackFrom(session_bundle_source_adapter_config); + } + (*(*platform_config_map.mutable_platform_configs())[kTensorFlowModelPlatform] + .mutable_source_adapter_config()) = source_adapter_config; + return platform_config_map; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/model_servers/platform_config_util.h b/tensorflow_serving/model_servers/platform_config_util.h new file mode 100644 index 00000000000..170ceb84448 --- /dev/null +++ b/tensorflow_serving/model_servers/platform_config_util.h @@ -0,0 +1,35 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CONFIG_UTIL_H_ +#define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CONFIG_UTIL_H_ + +#include "tensorflow_serving/config/platform_config.proto.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.proto.h" + +namespace tensorflow { +namespace serving { + +// Creates a PlatformConfigMap containing a single entry with the key as +// kTensorFlowModelPlatform and the value as a SourceAdapter config proto for +// one of {SessionBundleSourceAdapter, SavedModelBundleSourceAdapter} (based +// on 'use_saved_model' using 'session_bundle_config'. +PlatformConfigMap CreateTensorFlowPlatformConfigMap( + const SessionBundleConfig& session_bundle_config, bool use_saved_model); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CONFIG_UTIL_H_ diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index c52fa4ee55a..7cbf4c71623 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -39,21 +39,28 @@ namespace serving { namespace { // Returns an error if it is not the case that all ModelConfigList models have -// the same model type, otherwise returns OK and sets 'model_type' to the type. +// the same model platform, otherwise returns OK and sets 'model_platform' to +// the platform. Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, string* model_platform) { for (const auto& model : config.model_config_list().config()) { - // Get platform (with backward compatibility) + // Get platform (with backward compatibility support of the deprecated + // model_type field). string platform; if (model.model_type() != ModelType::MODEL_TYPE_UNSPECIFIED) { + LOG(WARNING) << "Deprecated ModelServerConfig::model_type field used. " + "Prefer ModelServerConfig::model_platform."; if (!model.model_platform().empty()) { return errors::InvalidArgument( - "Illegal setting both model_type(deprecated) and model_platform."); + "Illegal setting both ModelServerConfig::model_type (deprecated) " + "and ModelServerConfig::model_platform."); } if (model.model_type() == ModelType::TENSORFLOW) { platform = kTensorFlowModelPlatform; } else { - platform = kOtherModelPlatform; + return errors::InvalidArgument( + strings::StrCat("ModelServerConfig::model_type choice ", + model.model_type(), " not supported.")); } } else { platform = model.model_platform(); @@ -61,7 +68,8 @@ Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, if (platform.empty()) { return errors::InvalidArgument( - "Illegal setting neither model_type(deprecated) nor model_platform."); + "Illegal setting neither ModelServerConfig::model_type (deprecated) " + "nor ModelServerConfig::model_platform."); } // Check if matches found_platform (so far) @@ -71,7 +79,7 @@ Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, // Error if not, continue if true if (platform != *model_platform) { return errors::InvalidArgument( - "Expect all models to have the same type."); + "Multiple model platforms not (yet) supported."); } } return Status::OK(); @@ -85,30 +93,6 @@ Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, Status ServerCore::Create(Options options, std::unique_ptr* server_core) { - if (options.source_adapter_creator == nullptr) { - options.source_adapter_creator = [&options]( - const string& model_platform, - std::unique_ptr* adapter) { - SessionBundleSourceAdapterConfig source_adapter_config; - if (model_platform != kTensorFlowModelPlatform) { - return errors::InvalidArgument( - "ModelServer supports only TensorFlow model."); - } - if (options.use_saved_model) { - std::unique_ptr typed_adapter; - TF_RETURN_IF_ERROR(SavedModelBundleSourceAdapter::Create( - source_adapter_config, &typed_adapter)); - *adapter = std::move(typed_adapter); - } else { - std::unique_ptr typed_adapter; - TF_RETURN_IF_ERROR(SessionBundleSourceAdapter::Create( - source_adapter_config, &typed_adapter)); - *adapter = std::move(typed_adapter); - } - return Status::OK(); - }; - } - if (options.servable_state_monitor_creator == nullptr) { options.servable_state_monitor_creator = []( EventBus* event_bus, @@ -201,7 +185,7 @@ Status ServerCore::AddModelsViaModelConfigList() { const FileSystemStoragePathSourceConfig source_config = CreateStoragePathSourceConfig(config_); if (is_first_config) { - std::unique_ptr source_adapter; + std::unique_ptr source_adapter; TF_RETURN_IF_ERROR(CreateSourceAdapter(model_platform_, &source_adapter)); TF_RETURN_IF_ERROR( CreateFileSystemStoragePathSource(source_config, source_adapter.get())); @@ -280,9 +264,17 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { Status ServerCore::CreateSourceAdapter( const string& model_platform, - std::unique_ptr* adapter) { + std::unique_ptr* adapter) { + auto config_it = + options_.platform_config_map.platform_configs().find(model_platform); + if (config_it == options_.platform_config_map.platform_configs().end()) { + return errors::FailedPrecondition(strings::StrCat( + "PlatformConfigMap has no entry for platform ", model_platform)); + } + const ::google::protobuf::Any& adapter_config = + config_it->second.source_adapter_config(); const tensorflow::Status status = - options_.source_adapter_creator(model_platform, adapter); + StoragePathSourceAdapterRegistry::CreateFromAny(adapter_config, adapter); if (!status.ok()) { VLOG(1) << "Source adapter creation failed: " << status; } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 55c801bd348..e2c401baf54 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -41,6 +41,7 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" +#include "tensorflow_serving/config/platform_config.proto.h" #include "tensorflow_serving/core/aspired_versions_manager.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/source.h" @@ -59,13 +60,6 @@ class ServerCoreTestAccess; class ServerCore : public Manager { public: - using ModelServerSourceAdapter = - SourceAdapter>; - - using SourceAdapterCreator = - std::function* adapter)>; - using ServableStateMonitorCreator = std::function* event_bus, std::unique_ptr* monitor)>; @@ -109,23 +103,8 @@ class ServerCore : public Manager { // Time interval between file-system polls, in seconds. int32 file_system_poll_wait_seconds = 30; - // A function for creating ModelServerSourceAdapter based on the - // 'platform_type'. - // If not specified, a default creator that creates - // SessionBundleSourceAdapter or SavedModelBundleSourceAdapter for - // TensorFlow will be used, depending on use_saved_model. - SourceAdapterCreator source_adapter_creator; - - // Whether to use SavedModelBundle or SessionBundle. If - // source_adapter_creator is not specified, SavedModelBundleSourceAdapter - // will be created when this option is true and SessionBundleSourceAdapter - // will be created when it is false. If source_adapter_creator is specified, - // the creator will be used and this option will be ignored. - // This option is used by tensorflow serving team to control the rollout of - // SavedModelBundle and is not expected to be set by users directly. - // It should always be set to false (except for tests) until - // SavedModelBundle is supported by the service API implementation. - bool use_saved_model = false; + // Configuration for the supported platforms. + PlatformConfigMap platform_config_map; // A function for creating ServableStateMonitor. If not specified, a default // creator that creates ServableStateMonitor will be used. @@ -217,7 +196,7 @@ class ServerCore : public Manager { // Creates a platform-specific Loader Source. Status CreateSourceAdapter( const string& model_platform, - std::unique_ptr* adapter); + std::unique_ptr* adapter); // Creates a FileSystemStoragePathSourceConfig from the ModelConfigList of // 'config'. diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index d611188332a..ece8f731d0d 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -20,7 +20,10 @@ limitations under the License. #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" +#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h" +#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { @@ -31,7 +34,8 @@ using test_util::ServerCoreTest; TEST_P(ServerCoreTest, CreateWaitsTillModelsAvailable) { std::unique_ptr server_core; - TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfig(), &server_core)); + TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfigForFakePlatform(), + &server_core)); const std::vector available_servables = server_core->ListAvailableServableIds(); @@ -55,7 +59,8 @@ TEST_P(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { TF_ASSERT_OK(CreateServerCore(ModelServerConfig(), &server_core)); // Reconfigure it to load our test model. - TF_ASSERT_OK(server_core->ReloadConfig(GetTestModelServerConfig())); + TF_ASSERT_OK( + server_core->ReloadConfig(GetTestModelServerConfigForFakePlatform())); const std::vector available_servables = server_core->ListAvailableServableIds(); @@ -66,26 +71,26 @@ TEST_P(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { } TEST_P(ServerCoreTest, ErroringModel) { + ServerCore::Options options = GetDefaultOptions(); + test_util::StoragePathErrorInjectingSourceAdapterConfig source_adapter_config; + source_adapter_config.set_error_message("injected error"); + ::google::protobuf::Any source_adapter_config_any; + source_adapter_config_any.PackFrom(source_adapter_config); + (*(*options.platform_config_map.mutable_platform_configs())[kFakePlatform] + .mutable_source_adapter_config()) = source_adapter_config_any; + options.model_server_config = GetTestModelServerConfigForFakePlatform(); std::unique_ptr server_core; - Status status = CreateServerCore( - GetTestModelServerConfig(), - [](const string& model_platform, - std::unique_ptr* source_adapter) - -> Status { - source_adapter->reset( - new ErrorInjectingSourceAdapter>( - Status(error::CANCELLED, ""))); - return Status::OK(); - }, - &server_core); + Status status = ServerCore::Create(std::move(options), &server_core); EXPECT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Some models did not become available")); } TEST_P(ServerCoreTest, IllegalReconfigurationToCustomConfig) { // Create a ServerCore with ModelConfigList config. std::unique_ptr server_core; - TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfig(), &server_core)); + TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfigForFakePlatform(), + &server_core)); // Reload with a custom config. This is not allowed since the server was // first configured with TensorFlow model platform. @@ -104,14 +109,16 @@ TEST_P(ServerCoreTest, IllegalReconfigurationFromCustomConfig) { // Reload with a ModelConfigList config. This is not allowed, since the // server was first configured with a custom config. - EXPECT_THAT(server_core->ReloadConfig(GetTestModelServerConfig()).ToString(), - ::testing::HasSubstr("Cannot transition to requested config")); + EXPECT_THAT( + server_core->ReloadConfig(GetTestModelServerConfigForFakePlatform()) + .ToString(), + ::testing::HasSubstr("Cannot transition to requested config")); } TEST_P(ServerCoreTest, IllegalConfigModelTypeAndPlatformSet) { // Create a ServerCore with both model_type and model_platform set. std::unique_ptr server_core; - ModelServerConfig config = GetTestModelServerConfig(); + ModelServerConfig config = GetTestModelServerConfigForFakePlatform(); config.mutable_model_config_list()->mutable_config(0)->set_model_type( ModelType::TENSORFLOW); EXPECT_THAT(CreateServerCore(config, &server_core).ToString(), @@ -121,7 +128,7 @@ TEST_P(ServerCoreTest, IllegalConfigModelTypeAndPlatformSet) { TEST_P(ServerCoreTest, DeprecatedModelTypeConfig) { // Create a ServerCore with deprecated config. std::unique_ptr server_core; - ModelServerConfig config = GetTestModelServerConfig(); + ModelServerConfig config = GetTestModelServerConfigForTensorflowPlatform(); config.mutable_model_config_list()->mutable_config(0)->set_model_platform(""); config.mutable_model_config_list()->mutable_config(0)->set_model_type( ModelType::TENSORFLOW); diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 887625c03e2..17a7df58832 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -34,6 +34,8 @@ cc_library( "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/core:servable_state", + "//tensorflow_serving/core/test_util:fake_loader_source_adapter", + "//tensorflow_serving/core/test_util:fake_loader_source_adapter_proto", "//tensorflow_serving/model_servers:server_core", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", @@ -59,8 +61,38 @@ cc_library( "//tensorflow_serving/core:servable_id", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter_proto", + "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", + "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:test", ], ) + +cc_library( + name = "storage_path_error_injecting_source_adapter", + testonly = 1, + srcs = ["storage_path_error_injecting_source_adapter.cc"], + hdrs = ["storage_path_error_injecting_source_adapter.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":storage_path_error_injecting_source_adapter_proto", + "//tensorflow_serving/core:source_adapter", + ], +) + +load("//tensorflow_serving:serving.bzl", "serving_proto_library") + +serving_proto_library( + name = "storage_path_error_injecting_source_adapter_proto", + testonly = 1, + srcs = ["storage_path_error_injecting_source_adapter.proto"], + cc_api_version = 2, + visibility = [ + "//visibility:public", + ], +) diff --git a/tensorflow_serving/model_servers/test_util/mock_server_core.h b/tensorflow_serving/model_servers/test_util/mock_server_core.h index 1c8b9deae18..96c37d0421d 100644 --- a/tensorflow_serving/model_servers/test_util/mock_server_core.h +++ b/tensorflow_serving/model_servers/test_util/mock_server_core.h @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow_serving/config/model_server_config.proto.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.proto.h" #include "tensorflow_serving/model_servers/server_core.h" namespace tensorflow { @@ -32,10 +33,19 @@ namespace test_util { class MockServerCore : public ServerCore { public: - static Options GetOptions( - const SourceAdapterCreator& source_adapter_creator) { + // Creates a PlatformConfigMap with a FakeLoaderSourceAdapterConfig. + static PlatformConfigMap CreateFakeLoaderPlatformConfigMap() { + ::google::protobuf::Any source_adapter_config; + source_adapter_config.PackFrom(test_util::FakeLoaderSourceAdapterConfig()); + PlatformConfigMap platform_config_map; + (*(*platform_config_map.mutable_platform_configs())["fake_servable"] + .mutable_source_adapter_config()) = source_adapter_config; + return platform_config_map; + } + + static Options GetOptions(const PlatformConfigMap& platform_config_map) { Options options; - options.source_adapter_creator = source_adapter_creator; + options.platform_config_map = platform_config_map; options.servable_state_monitor_creator = []( EventBus* event_bus, std::unique_ptr* monitor) -> Status { @@ -50,8 +60,8 @@ class MockServerCore : public ServerCore { return options; } - explicit MockServerCore(const SourceAdapterCreator& source_adapter_creator) - : ServerCore(GetOptions(source_adapter_creator)) {} + explicit MockServerCore(const PlatformConfigMap& platform_config_map) + : ServerCore(GetOptions(platform_config_map)) {} MOCK_CONST_METHOD0(servable_state_monitor, ServableStateMonitor*()); MOCK_METHOD1(ReloadConfig, Status(const ModelServerConfig&)); diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index fdc06dfc52d..f0f71ed8127 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -19,13 +19,28 @@ limitations under the License. #include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.proto.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { namespace serving { namespace test_util { -ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { +constexpr char ServerCoreTest::kFakePlatform[]; + +ModelServerConfig ServerCoreTest::GetTestModelServerConfigForFakePlatform() { + ModelServerConfig config = GetTestModelServerConfigForTensorflowPlatform(); + ModelConfig* model_config = + config.mutable_model_config_list()->mutable_config(0); + model_config->set_model_platform(kFakePlatform); + return config; +} + +ModelServerConfig +ServerCoreTest::GetTestModelServerConfigForTensorflowPlatform() { ModelServerConfig config; auto model = config.mutable_model_config_list()->add_config(); model->set_name(kTestModelName); @@ -40,47 +55,39 @@ ModelServerConfig ServerCoreTest::GetTestModelServerConfig() { return config; } -Status ServerCoreTest::CreateServerCore( - const ModelServerConfig& config, std::unique_ptr* server_core) { - return CreateServerCore( - config, test_util::FakeLoaderSourceAdapter::GetCreator(), server_core); -} - -Status ServerCoreTest::CreateServerCore( - const ModelServerConfig& config, - const ServerCore::SourceAdapterCreator& source_adapter_creator, - std::unique_ptr* server_core) { - // For ServerCore Options, we leave servable_state_monitor_creator unspecified - // so the default servable_state_monitor_creator will be used. +ServerCore::Options ServerCoreTest::GetDefaultOptions() { ServerCore::Options options; - options.model_server_config = config; - options.source_adapter_creator = source_adapter_creator; + options.file_system_poll_wait_seconds = 0; // Reduce the number of initial load threads to be num_load_threads to avoid // timing out in tests. options.num_initial_load_threads = options.num_load_threads; + options.aspired_version_policy = + std::unique_ptr(new EagerLoadPolicy); options.custom_model_config_loader = []( const ::google::protobuf::Any& any, EventBus* event_bus, UniquePtrWithDeps* manager) -> Status { return Status::OK(); }; - return CreateServerCore(std::move(options), server_core); + + // Model platforms. + const TestType test_type = GetTestType(); + const bool use_saved_model = test_type == SAVED_MODEL || + test_type == SAVED_MODEL_BACKWARD_COMPATIBILITY; + options.platform_config_map = + CreateTensorFlowPlatformConfigMap(SessionBundleConfig(), use_saved_model); + ::google::protobuf::Any fake_source_adapter_config; + fake_source_adapter_config.PackFrom( + test_util::FakeLoaderSourceAdapterConfig()); + (*(*options.platform_config_map.mutable_platform_configs())[kFakePlatform] + .mutable_source_adapter_config()) = fake_source_adapter_config; + + return options; } Status ServerCoreTest::CreateServerCore( - ServerCore::Options options, std::unique_ptr* server_core) { - options.file_system_poll_wait_seconds = 0; - // Reduce the number of initial load threads to be num_load_threads to avoid - // timing out in tests. - options.num_initial_load_threads = options.num_load_threads; - if (options.aspired_version_policy == nullptr) { - options.aspired_version_policy = - std::unique_ptr(new EagerLoadPolicy); - } - TestType test_type = GetTestType(); - if (test_type == SAVED_MODEL || - test_type == SAVED_MODEL_BACKWARD_COMPATIBILITY) { - options.use_saved_model = true; - } + const ModelServerConfig& config, std::unique_ptr* server_core) { + ServerCore::Options options = GetDefaultOptions(); + options.model_server_config = config; return ServerCore::Create(std::move(options), server_core); } diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index 63d2285c943..3ec419da221 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -45,22 +45,22 @@ class ServerCoreTest : public ::testing::TestWithParam { }; protected: - // Returns ModelServerConfig that contains test model. - ModelServerConfig GetTestModelServerConfig(); + // The name of the platform associated with FakeLoaderSourceAdapter. + static constexpr char kFakePlatform[] = "fake_servable"; - // Create a ServerCore object configured to use FakeLoaderSourceAdapter. - Status CreateServerCore(const ModelServerConfig& config, - std::unique_ptr* server_core); + // Returns ModelServerConfig that contains test model for the fake platform. + ModelServerConfig GetTestModelServerConfigForFakePlatform(); - // Create a ServerCore object with the supplied SourceAdapterCreator. - Status CreateServerCore( - const ModelServerConfig& config, - const ServerCore::SourceAdapterCreator& source_adapter_creator, - std::unique_ptr* server_core); + // Returns ModelServerConfig that contains test model for the tensorflow + // platform. + ModelServerConfig GetTestModelServerConfigForTensorflowPlatform(); - // Create a ServerCore object with the supplied options. The ServerCore uses - // continuous polling to speed up testing. - Status CreateServerCore(ServerCore::Options options, + // Creates some reasonable default ServerCore options for tests. + ServerCore::Options GetDefaultOptions(); + + // Creates a ServerCore object configured with both a fake platform and the + // tensorflow platform, using GetDefaultOptions(). + Status CreateServerCore(const ModelServerConfig& config, std::unique_ptr* server_core); // Returns test type. This is the parameter of this test. diff --git a/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc new file mode 100644 index 00000000000..d9a7c93c7b2 --- /dev/null +++ b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc @@ -0,0 +1,43 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h" +#include "tensorflow_serving/core/source_adapter.h" +#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// Register the source adapter. +class StoragePathErrorInjectingSourceAdapterCreator { + public: + static Status Create( + const StoragePathErrorInjectingSourceAdapterConfig& config, + std::unique_ptr>>* + adapter) { + adapter->reset( + new ErrorInjectingSourceAdapter>( + Status(error::CANCELLED, config.error_message()))); + return Status::OK(); + } +}; +REGISTER_STORAGE_PATH_SOURCE_ADAPTER( + StoragePathErrorInjectingSourceAdapterCreator, + StoragePathErrorInjectingSourceAdapterConfig); + +} // namespace test_util +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h new file mode 100644 index 00000000000..e06f7a790c4 --- /dev/null +++ b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h @@ -0,0 +1,35 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_STORAGE_PATH_ERROR_INJECTING_SOURCE_ADAPTER_H_ +#define TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_STORAGE_PATH_ERROR_INJECTING_SOURCE_ADAPTER_H_ + +#include "tensorflow_serving/core/source_adapter.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// An ErrorInjectingSourceAdapter> (see +// source_adapter.h) registered in StoragePathSourceAdapterRegistry and keyed on +// StoragePathErrorInjectingSourceAdapterConfig. +using StoragePathErrorInjectingSourceAdapter = + ErrorInjectingSourceAdapter>; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_STORAGE_PATH_ERROR_INJECTING_SOURCE_ADAPTER_H_ diff --git a/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto new file mode 100644 index 00000000000..39fd84f5bed --- /dev/null +++ b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto @@ -0,0 +1,9 @@ +syntax = "proto3"; + +package tensorflow.serving.test_util; + +// Config proto for StoragePathErrorInjectingSourceAdapter. +message StoragePathErrorInjectingSourceAdapterConfig { + // The error message the adapter emits. + string error_message = 1; +} diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 3cfc51ab714..f02811ae46e 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -227,6 +227,7 @@ cc_library( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", ], + alwayslink = 1, ) cc_test( @@ -263,6 +264,7 @@ cc_library( ], deps = [ ":saved_model_bundle_factory", + ":saved_model_bundle_source_adapter_proto", ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:simple_loader", @@ -274,6 +276,7 @@ cc_library( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", ], + alwayslink = 1, ) cc_test( @@ -316,6 +319,18 @@ serving_proto_library( ], ) +serving_proto_library( + name = "saved_model_bundle_source_adapter_proto", + srcs = ["saved_model_bundle_source_adapter.proto"], + cc_api_version = 2, + visibility = [ + "//visibility:public", + ], + deps = [ + ":session_bundle_config_proto", + ], +) + cc_library( name = "simple_servers", srcs = ["simple_servers.cc"], @@ -409,6 +424,7 @@ cc_test( "//tensorflow_serving/core:eager_load_policy", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", "//tensorflow_serving/model_servers:server_core", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 9cd41563cdc..4dff5cfa9fa 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -21,6 +21,10 @@ limitations under the License. #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow_serving/core/eager_load_policy.h" #include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { @@ -71,7 +75,8 @@ class PredictImplTest : public ::testing::TestWithParam { // unspecified so the default servable_state_monitor_creator will be used. ServerCore::Options options; options.model_server_config = config; - options.use_saved_model = use_saved_model; + options.platform_config_map = CreateTensorFlowPlatformConfigMap( + SessionBundleConfig(), use_saved_model); options.aspired_version_policy = std::unique_ptr(new EagerLoadPolicy); // Reduce the number of initial load threads to be num_load_threads to avoid diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc index 11fcf02d548..8b4bbb2fff9 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc @@ -73,5 +73,23 @@ SavedModelBundleSourceAdapter::GetCreator( }; } +// Register the source adapter. +class SavedModelBundleSourceAdapterCreator { + public: + static Status Create( + const SavedModelBundleSourceAdapterConfig& config, + std::unique_ptr>>* + adapter) { + std::unique_ptr bundle_factory; + TF_RETURN_IF_ERROR(SavedModelBundleFactory::Create(config.legacy_config(), + &bundle_factory)); + adapter->reset( + new SavedModelBundleSourceAdapter(std::move(bundle_factory))); + return Status::OK(); + } +}; +REGISTER_STORAGE_PATH_SOURCE_ADAPTER(SavedModelBundleSourceAdapterCreator, + SavedModelBundleSourceAdapterConfig); + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h index 9e75d43b9bb..bfe9ad0f49e 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow_serving/core/source_adapter.h" #include "tensorflow_serving/core/storage_path.h" #include "tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" namespace tensorflow { @@ -34,8 +35,9 @@ namespace serving { class SavedModelBundleSourceAdapter final : public UnarySourceAdapter> { public: - // TODO(b/32248363): add SavedModelBundleSourceAdapterConfig after we switch - // Model Server to Saved Model. + // TODO(b/32248363): Switch to SavedModelBundleSourceAdapterConfig after we + // switch Model Server to Saved Model and populate the "real" fields of + // SavedModelBundleSourceAdapterConfig. static Status Create(const SessionBundleSourceAdapterConfig& config, std::unique_ptr* adapter); @@ -47,6 +49,8 @@ class SavedModelBundleSourceAdapter final GetCreator(const SessionBundleSourceAdapterConfig& config); private: + friend class SavedModelBundleSourceAdapterCreator; + explicit SavedModelBundleSourceAdapter( std::unique_ptr bundle_factory); diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto new file mode 100644 index 00000000000..56cbe7ccb91 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto @@ -0,0 +1,13 @@ +syntax = "proto3"; + +import "tensorflow_serving/servables/tensorflow/session_bundle_config.proto"; + +package tensorflow.serving; + +// Config proto for SavedModelBundleSourceAdapter. +message SavedModelBundleSourceAdapterConfig { + // A SessionBundleConfig. + // FOR INTERNAL USE ONLY DURING TRANSITION TO SAVED_MODEL. WILL BE DEPRECATED. + // TODO(b/32248363): Replace this field with the "real" field(s). + SessionBundleConfig legacy_config = 1000; +} diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.cc b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.cc index 07e89fcda08..f24d90f69ef 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.cc @@ -71,5 +71,22 @@ SessionBundleSourceAdapter::GetCreator( }; } +// Register the source adapter. +class SessionBundleSourceAdapterCreator { + public: + static Status Create( + const SessionBundleSourceAdapterConfig& config, + std::unique_ptr>>* + adapter) { + std::unique_ptr bundle_factory; + TF_RETURN_IF_ERROR( + SessionBundleFactory::Create(config.config(), &bundle_factory)); + adapter->reset(new SessionBundleSourceAdapter(std::move(bundle_factory))); + return Status::OK(); + } +}; +REGISTER_STORAGE_PATH_SOURCE_ADAPTER(SessionBundleSourceAdapterCreator, + SessionBundleSourceAdapterConfig); + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h index 6dd8bb367fd..938a515d60b 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h +++ b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h @@ -44,6 +44,8 @@ class SessionBundleSourceAdapter final GetCreator(const SessionBundleSourceAdapterConfig& config); private: + friend class SessionBundleSourceAdapterCreator; + explicit SessionBundleSourceAdapter( std::unique_ptr bundle_factory); diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 4c369babf76..28358a985e6 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -50,9 +50,7 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":class_registration_util", - "@org_tensorflow//tensorflow/core", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:lib", "@protobuf//:cc_wkt_protos", ], ) @@ -285,9 +283,7 @@ cc_library( srcs = ["class_registration_util.cc"], hdrs = ["class_registration_util.h"], deps = [ - "@org_tensorflow//tensorflow/core", - "@org_tensorflow//tensorflow/core:framework", - "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:lib", "@protobuf//:cc_wkt_protos", ], ) From cb2bb699ff65950e0e853470224c5aa0f0e0a7cd Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 16 Dec 2016 15:12:55 -0800 Subject: [PATCH 0117/8554] Introduce a reconfigurable SourceRouter, which we plan to use to allow ServerCore to handle multiple platforms (SourceAdapters) simultaneously. Change: 142303550 --- tensorflow_serving/core/BUILD | 29 ++++ .../core/dynamic_source_router.h | 137 +++++++++++++++++ .../core/dynamic_source_router_test.cc | 144 ++++++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 tensorflow_serving/core/dynamic_source_router.h create mode 100644 tensorflow_serving/core/dynamic_source_router_test.cc diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index bbb01f2ebcb..bfc7102b984 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -226,6 +226,35 @@ cc_test( ], ) +cc_library( + name = "dynamic_source_router", + hdrs = ["dynamic_source_router.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":source_router", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "dynamic_source_router_test", + srcs = ["dynamic_source_router_test.cc"], + deps = [ + ":dynamic_source_router", + ":servable_data", + ":source", + ":storage_path", + ":target", + "//tensorflow_serving/core/test_util:mock_storage_path_target", + "//tensorflow_serving/core/test_util:test_main", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", + ], +) + cc_library( name = "storage_path", hdrs = ["storage_path.h"], diff --git a/tensorflow_serving/core/dynamic_source_router.h b/tensorflow_serving/core/dynamic_source_router.h new file mode 100644 index 00000000000..3addc980828 --- /dev/null +++ b/tensorflow_serving/core/dynamic_source_router.h @@ -0,0 +1,137 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_DYNAMIC_SOURCE_ROUTER_H_ +#define TENSORFLOW_SERVING_CORE_DYNAMIC_SOURCE_ROUTER_H_ + +#include +#include +#include + +#include "tensorflow/core/platform/macros.h" +#include "tensorflow_serving/core/source_router.h" + +namespace tensorflow { +namespace serving { + +// A SourceRouter with a fixed set of N output ports, and a dynamically +// reconfigurable map from servable name to port. The route map can only +// reference the first N-1 ports; the Nth port is reserved for servables not +// found in the route map. +template +class DynamicSourceRouter final : public SourceRouter { + public: + // Creates a DynamicSourceRouter with 'num_output_ports' output ports and an + // (initial) route map given by 'routes'. + static Status Create(int num_output_ports, + const std::map& routes, + std::unique_ptr>* result); + ~DynamicSourceRouter() override; + + // Sets the route map to 'routes'. + Status UpdateRoutes(const std::map& routes); + + protected: + int num_output_ports() const override { return num_output_ports_; } + + int Route(const StringPiece servable_name, + const std::vector>& versions) override; + + private: + DynamicSourceRouter(int num_output_ports, + const std::map& routes); + + // Returns an error if 'routes' is invalid, given 'num_output_ports'. + static Status ValidateRoutes(int num_output_ports, + const std::map& routes); + + const int num_output_ports_; + + mutable mutex routes_mu_; + std::map routes_ GUARDED_BY(routes_mu_); + + TF_DISALLOW_COPY_AND_ASSIGN(DynamicSourceRouter); +}; + +////////// +// Implementation details follow. API users need not read. + +template +Status DynamicSourceRouter::Create( + int num_output_ports, const std::map& routes, + std::unique_ptr>* result) { + TF_RETURN_IF_ERROR(ValidateRoutes(num_output_ports, routes)); + result->reset(new DynamicSourceRouter(num_output_ports, routes)); + return Status::OK(); +} + +template +DynamicSourceRouter::~DynamicSourceRouter() { + TargetBase::Detach(); +} + +template +Status DynamicSourceRouter::UpdateRoutes( + const std::map& routes) { + TF_RETURN_IF_ERROR(ValidateRoutes(num_output_ports_, routes)); + { + mutex_lock l(routes_mu_); + routes_ = routes; + } + return Status::OK(); +} + +template +int DynamicSourceRouter::Route( + const StringPiece servable_name, + const std::vector>& versions) { + mutex_lock l(routes_mu_); + auto it = routes_.find(servable_name.ToString()); + if (it == routes_.end()) { + LOG(INFO) << "Routing servable(s) from stream " << servable_name + << " to default output port " << num_output_ports_ - 1; + return num_output_ports_ - 1; + } else { + return it->second; + } +} + +template +DynamicSourceRouter::DynamicSourceRouter(int num_output_ports, + const std::map& routes) + : num_output_ports_(num_output_ports), routes_(routes) {} + +template +Status DynamicSourceRouter::ValidateRoutes( + int num_output_ports, const std::map& routes) { + for (const auto& entry : routes) { + const int port = entry.second; + if (port < 0 || port >= num_output_ports) { + return errors::InvalidArgument( + strings::StrCat("Port number out of range: ", port)); + } + if (port == num_output_ports - 1) { + return errors::InvalidArgument( + "Last port cannot be used in route map, since it's reserved for the " + "default route"); + } + } + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_DYNAMIC_SOURCE_ROUTER_H_ diff --git a/tensorflow_serving/core/dynamic_source_router_test.cc b/tensorflow_serving/core/dynamic_source_router_test.cc new file mode 100644 index 00000000000..df9cdb5d272 --- /dev/null +++ b/tensorflow_serving/core/dynamic_source_router_test.cc @@ -0,0 +1,144 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/dynamic_source_router.h" + +#include + +#include +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/servable_data.h" +#include "tensorflow_serving/core/source.h" +#include "tensorflow_serving/core/storage_path.h" +#include "tensorflow_serving/core/target.h" +#include "tensorflow_serving/core/test_util/mock_storage_path_target.h" + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::StrictMock; + +namespace tensorflow { +namespace serving { +namespace { + +TEST(DynamicSourceRouterTest, InvalidRouteMap) { + std::unique_ptr> router; + // Negative output port. + EXPECT_FALSE( + DynamicSourceRouter::Create(2, {{"foo", -1}}, &router).ok()); + // Out of range output port. + EXPECT_FALSE( + DynamicSourceRouter::Create(2, {{"foo", 2}}, &router).ok()); + // Using reserved (last) output port. + EXPECT_FALSE( + DynamicSourceRouter::Create(2, {{"foo", 1}}, &router).ok()); +} + +TEST(DynamicSourceRouterTest, ReconfigureToInvalidRouteMap) { + std::unique_ptr> router; + TF_ASSERT_OK(DynamicSourceRouter::Create(2, {{"foo", 0}}, &router)); + // Negative output port. + EXPECT_FALSE(router->UpdateRoutes({{"foo", -1}}).ok()); + // Out of range output port. + EXPECT_FALSE(router->UpdateRoutes({{"foo", 2}}).ok()); + // Using reserved (last) output port. + EXPECT_FALSE(router->UpdateRoutes({{"foo", 1}}).ok()); +} + +TEST(DynamicSourceRouterTest, Basic) { + std::unique_ptr> router; + TF_ASSERT_OK(DynamicSourceRouter::Create(4, {{"foo", 0}, {"bar", 1}}, + &router)); + std::vector*> output_ports = router->GetOutputPorts(); + ASSERT_EQ(4, output_ports.size()); + std::vector> targets; + for (int i = 0; i < output_ports.size(); ++i) { + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(output_ports[i], target.get()); + targets.push_back(std::move(target)); + } + + // "foo" goes to port 0. + EXPECT_CALL(*targets[0], SetAspiredVersions( + Eq("foo"), ElementsAre(ServableData( + {"foo", 7}, "data")))); + router->SetAspiredVersions("foo", + {ServableData({"foo", 7}, "data")}); + + // "bar" goes to port 1. + EXPECT_CALL(*targets[1], SetAspiredVersions( + Eq("bar"), ElementsAre(ServableData( + {"bar", 7}, "data")))); + router->SetAspiredVersions("bar", + {ServableData({"bar", 7}, "data")}); + + // Servable whose name doesn't match any route goes to the last port (port 3). + EXPECT_CALL(*targets[3], + SetAspiredVersions(Eq("not_foo_or_bar"), + ElementsAre(ServableData( + {"not_foo_or_bar", 7}, "data")))); + router->SetAspiredVersions( + "not_foo_or_bar", + {ServableData({"not_foo_or_bar", 7}, "data")}); +} + +TEST(DynamicSourceRouterTest, Reconfigure) { + std::unique_ptr> router; + TF_ASSERT_OK(DynamicSourceRouter::Create(2, {{"foo", 0}}, &router)); + std::vector*> output_ports = router->GetOutputPorts(); + ASSERT_EQ(2, output_ports.size()); + std::vector> targets; + for (int i = 0; i < output_ports.size(); ++i) { + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(output_ports[i], target.get()); + targets.push_back(std::move(target)); + } + + // Initially, "foo" goes to port 0 and "bar" goes to the default (port 1). + EXPECT_CALL(*targets[0], SetAspiredVersions( + Eq("foo"), ElementsAre(ServableData( + {"foo", 7}, "data")))); + router->SetAspiredVersions("foo", + {ServableData({"foo", 7}, "data")}); + EXPECT_CALL(*targets[1], SetAspiredVersions( + Eq("bar"), ElementsAre(ServableData( + {"bar", 7}, "data")))); + router->SetAspiredVersions("bar", + {ServableData({"bar", 7}, "data")}); + + router->UpdateRoutes({{"bar", 0}}); + + // Now, the routes of "foo" and "bar" should be swapped. + EXPECT_CALL(*targets[1], SetAspiredVersions( + Eq("foo"), ElementsAre(ServableData( + {"foo", 7}, "data")))); + router->SetAspiredVersions("foo", + {ServableData({"foo", 7}, "data")}); + EXPECT_CALL(*targets[0], SetAspiredVersions( + Eq("bar"), ElementsAre(ServableData( + {"bar", 7}, "data")))); + router->SetAspiredVersions("bar", + {ServableData({"bar", 7}, "data")}); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From 10636f98750deb6dd572bfa280c272443d8310e3 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Mon, 19 Dec 2016 10:22:24 -0800 Subject: [PATCH 0118/8554] Plug FileProbingEnvironment into SessionBundleFactory and SavedModelBundleFactory. Change: 142460036 --- tensorflow_serving/servables/tensorflow/BUILD | 4 ++ .../tensorflow/bundle_factory_test.h | 40 ++++++++++++++++++- .../tensorflow/saved_model_bundle_factory.cc | 6 +++ .../tensorflow/saved_model_bundle_factory.h | 8 ++++ .../saved_model_bundle_factory_test.cc | 10 +++++ .../tensorflow/session_bundle_factory.cc | 6 +++ .../tensorflow/session_bundle_factory.h | 8 ++++ .../tensorflow/session_bundle_factory_test.cc | 5 +++ tensorflow_serving/util/BUILD | 1 + tensorflow_serving/util/test_util/BUILD | 23 +++++++++++ .../util/test_util/mock_file_probing_env.h | 40 +++++++++++++++++++ 11 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 tensorflow_serving/util/test_util/BUILD create mode 100644 tensorflow_serving/util/test_util/mock_file_probing_env.h diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index f02811ae46e..688dd531abd 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -109,6 +109,8 @@ cc_library( ":session_bundle_config_proto", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", + "//tensorflow_serving/util:file_probing_env", + "//tensorflow_serving/util/test_util:mock_file_probing_env", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", @@ -129,6 +131,7 @@ cc_library( "//tensorflow_serving/batching:batching_session", "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/util:file_probing_env", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", "@org_tensorflow//tensorflow/core:core_cpu", @@ -173,6 +176,7 @@ cc_library( "//tensorflow_serving/batching:batching_session", "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resources_proto", + "//tensorflow_serving/util:file_probing_env", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h index 05a0a8eb900..b2bef5e64b4 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h @@ -26,16 +26,23 @@ limitations under the License. #include #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/public/session.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/test_util/test_util.h" +#include "tensorflow_serving/util/file_probing_env.h" +#include "tensorflow_serving/util/test_util/mock_file_probing_env.h" namespace tensorflow { namespace serving { namespace test_util { +using ::testing::DoAll; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::_; using test_util::EqualsProto; // The base class for SessionBundleFactoryTest and SavedModelBundleFactoryTest. @@ -71,14 +78,45 @@ class BundleFactoryTest : public ::testing::Test { template void TestEstimateResourceRequirementWithGoodExport( double total_file_size) const { + const SessionBundleConfig config; + std::unique_ptr factory; + TF_ASSERT_OK(FactoryType::Create(config, &factory)); + ResourceAllocation actual; + TF_ASSERT_OK(factory->EstimateResourceRequirement(export_dir_, &actual)); + ResourceAllocation expected = GetExpectedResourceEstimate(total_file_size); + EXPECT_THAT(actual, EqualsProto(expected)); + } + + template + void TestEstimateResourceRequirementWithFileProbingEnv() const { + const string export_dir = "/foo/bar"; + const string child = "child"; + const string child_path = io::JoinPath(export_dir, child); + const double file_size = 100; + + // Set up the expectation that the directory contains exactly one child with + // the given file size. + MockFileProbingEnv env; + EXPECT_CALL(env, FileExists(export_dir)) + .WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(env, GetChildren(export_dir, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(std::vector({child})), + Return(Status::OK()))); + EXPECT_CALL(env, IsDirectory(child_path)) + .WillRepeatedly(Return(errors::FailedPrecondition(""))); + EXPECT_CALL(env, GetFileSize(child_path, _)) + .WillRepeatedly( + DoAll(SetArgPointee<1>(file_size), Return(Status::OK()))); const SessionBundleConfig config; std::unique_ptr factory; TF_ASSERT_OK(FactoryType::Create(config, &factory)); ResourceAllocation actual; - TF_ASSERT_OK(factory->EstimateResourceRequirement(export_dir_, &actual)); + TF_ASSERT_OK( + factory->EstimateResourceRequirement(export_dir, &env, &actual)); + ResourceAllocation expected = GetExpectedResourceEstimate(file_size); EXPECT_THAT(actual, EqualsProto(expected)); } diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc index f96e3c1d762..996cf77cc98 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -56,6 +56,12 @@ Status SavedModelBundleFactory::EstimateResourceRequirement( return EstimateResourceFromPath(path, estimate); } +Status SavedModelBundleFactory::EstimateResourceRequirement( + const string& path, FileProbingEnv* env, + ResourceAllocation* estimate) const { + return EstimateResourceFromPath(path, env, estimate); +} + Status SavedModelBundleFactory::CreateSavedModelBundle( const string& path, std::unique_ptr* bundle) { bundle->reset(new SavedModelBundle); diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h index 2681df44cee..edda0b3ab34 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/util/file_probing_env.h" namespace tensorflow { namespace serving { @@ -56,9 +57,16 @@ class SavedModelBundleFactory { // Estimates the resources a SavedModel bundle will use once loaded, from its // export path. + // TODO(b/33078719): remove this method after we switch all the callers to + // the following one. Status EstimateResourceRequirement(const string& path, ResourceAllocation* estimate) const; + // Similar to the above method, but also supplies a FileProbingEnv to use in + // lieu of tensorflow::Env::Default(). + Status EstimateResourceRequirement(const string& path, FileProbingEnv* env, + ResourceAllocation* estimate) const; + private: using Batcher = SharedBatchScheduler; diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index ac26c951fb3..bf03f887370 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -74,6 +74,11 @@ TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { kTotalFileSize); } +TEST_F(SavedModelBundleFactoryTest, + EstimateResourceRequirementWithFileProbingEnv) { + TestEstimateResourceRequirementWithFileProbingEnv(); +} + TEST_F(SavedModelBundleFactoryTest, RunOptions) { TestRunOptions(); } TEST_F(SavedModelBundleFactoryTest, RunOptionsError) { TestRunOptionsError(); } @@ -109,6 +114,11 @@ TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, kTotalFileSize); } +TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, + EstimateResourceRequirementWithFileProbingEnv) { + TestEstimateResourceRequirementWithFileProbingEnv(); +} + TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, RunOptions) { TestRunOptions(); } diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc index b899f41a68b..4db48cc0db6 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc @@ -73,6 +73,12 @@ Status SessionBundleFactory::EstimateResourceRequirement( return EstimateResourceFromPath(path, estimate); } +Status SessionBundleFactory::EstimateResourceRequirement( + const string& path, FileProbingEnv* env, + ResourceAllocation* estimate) const { + return EstimateResourceFromPath(path, env, estimate); +} + Status SessionBundleFactory::CreateSessionBundle( const string& path, std::unique_ptr* bundle) { bundle->reset(new SessionBundle); diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h index ab96398d5ca..5250f99c1b1 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/util/file_probing_env.h" namespace tensorflow { namespace serving { @@ -54,9 +55,16 @@ class SessionBundleFactory { // Estimates the resources a session bundle will use once loaded, from its // export path. + // TODO(b/33078719): remove this method after we switch all the callers to + // the following one. Status EstimateResourceRequirement(const string& path, ResourceAllocation* estimate) const; + // Similar to the above method, but also supplies a FileProbingEnv to use in + // lieu of tensorflow::Env::Default(). + Status EstimateResourceRequirement(const string& path, FileProbingEnv* env, + ResourceAllocation* estimate) const; + private: using Batcher = SharedBatchScheduler; diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index a195f5b7876..b3857332e1b 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -70,6 +70,11 @@ TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { kTotalFileSize); } +TEST_F(SessionBundleFactoryTest, + EstimateResourceRequirementWithFileProbingEnv) { + TestEstimateResourceRequirementWithFileProbingEnv(); +} + TEST_F(SessionBundleFactoryTest, RunOptions) { TestRunOptions(); } TEST_F(SessionBundleFactoryTest, RunOptionsError) { TestRunOptionsError(); } diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 28358a985e6..ec570163fe8 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -273,6 +273,7 @@ cc_library( name = "file_probing_env", srcs = ["file_probing_env.cc"], hdrs = ["file_probing_env.h"], + visibility = ["//visibility:public"], deps = [ "@org_tensorflow//tensorflow/core:lib", ], diff --git a/tensorflow_serving/util/test_util/BUILD b/tensorflow_serving/util/test_util/BUILD new file mode 100644 index 00000000000..9196d1aa1f1 --- /dev/null +++ b/tensorflow_serving/util/test_util/BUILD @@ -0,0 +1,23 @@ +# Description: testing utils for Tensorflow Serving utils. + +package( + default_visibility = [ + "//tensorflow_serving:internal", + ], + features = ["-layering_check"], +) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +cc_library( + name = "mock_file_probing_env", + testonly = 1, + hdrs = ["mock_file_probing_env.h"], + deps = [ + "//external:gtest", + "//tensorflow_serving/util:file_probing_env", + "@org_tensorflow//tensorflow/core:lib", + ], +) diff --git a/tensorflow_serving/util/test_util/mock_file_probing_env.h b/tensorflow_serving/util/test_util/mock_file_probing_env.h new file mode 100644 index 00000000000..fd5d73e7c7e --- /dev/null +++ b/tensorflow_serving/util/test_util/mock_file_probing_env.h @@ -0,0 +1,40 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_UTIL_TEST_UTIL_MOCK_FILE_PROBING_ENV_H_ +#define TENSORFLOW_SERVING_UTIL_TEST_UTIL_MOCK_FILE_PROBING_ENV_H_ + +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/util/file_probing_env.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +class MockFileProbingEnv : public FileProbingEnv { + public: + MOCK_METHOD1(FileExists, Status(const string& fname)); + MOCK_METHOD2(GetChildren, + Status(const string& fname, std::vector* children)); + MOCK_METHOD1(IsDirectory, Status(const string& fname)); + MOCK_METHOD2(GetFileSize, Status(const string& fname, uint64* file_size)); +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_UTIL_TEST_UTIL_MOCK_FILE_PROBING_ENV_H_ From eb6e62942ff3d6c88477ae81907705b454995095 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Tue, 27 Dec 2016 12:04:54 -0800 Subject: [PATCH 0119/8554] Update bundle-factory-util visibility. Change: 143047443 --- tensorflow_serving/servables/tensorflow/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 688dd531abd..f389336fb15 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -41,6 +41,9 @@ cc_library( name = "bundle_factory_util", srcs = ["bundle_factory_util.cc"], hdrs = ["bundle_factory_util.h"], + visibility = [ + "//visibility:public", + ], deps = [ ":serving_session", ":session_bundle_config_proto", From aca4e35167ac5b1fd59ce484f18f2356e3af6f67 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 29 Dec 2016 12:59:03 -0800 Subject: [PATCH 0120/8554] Fix bug in degenerate zero-servables case of waiting until servables reach a specific state. Change: 143202146 --- .../core/servable_state_monitor.cc | 1 + .../core/servable_state_monitor_test.cc | 20 +++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/tensorflow_serving/core/servable_state_monitor.cc b/tensorflow_serving/core/servable_state_monitor.cc index 0a69e81bf51..88a778f0985 100644 --- a/tensorflow_serving/core/servable_state_monitor.cc +++ b/tensorflow_serving/core/servable_state_monitor.cc @@ -180,6 +180,7 @@ void ServableStateMonitor::NotifyWhenServablesReachState( mutex_lock l(mu_); servable_state_notification_requests_.push_back( {servables, goal_state, notifier_fn}); + MaybeSendNotifications(); } bool ServableStateMonitor::WaitUntilServablesReachState( diff --git a/tensorflow_serving/core/servable_state_monitor_test.cc b/tensorflow_serving/core/servable_state_monitor_test.cc index 6a9ace4ed70..cb48eed42b9 100644 --- a/tensorflow_serving/core/servable_state_monitor_test.cc +++ b/tensorflow_serving/core/servable_state_monitor_test.cc @@ -26,6 +26,7 @@ namespace serving { namespace { using ::testing::ElementsAre; +using ::testing::IsEmpty; using ::testing::Pair; using ::testing::UnorderedElementsAre; using ServableStateAndTime = ServableStateMonitor::ServableStateAndTime; @@ -270,6 +271,25 @@ TEST(ServableStateMonitorTest, VersionMapDescendingOrder) { Pair(7, state_1_and_time))))); } +TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateZeroServables) { + auto bus = EventBus::CreateEventBus({}); + ServableStateMonitor monitor(bus.get()); + const std::vector servables = {}; + + using ManagerState = ServableState::ManagerState; + + Notification notified; + monitor.NotifyWhenServablesReachState( + servables, ManagerState::kAvailable, + [&](const bool reached, + std::map states_reached) { + EXPECT_TRUE(reached); + EXPECT_THAT(states_reached, IsEmpty()); + notified.Notify(); + }); + notified.WaitForNotification(); +} + TEST(ServableStateMonitorTest, NotifyWhenServablesReachStateSpecificAvailable) { auto bus = EventBus::CreateEventBus({}); ServableStateMonitor monitor(bus.get()); From c0395a45dcdfa4ba4e8c5761b683dd5fc8c31acb Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 29 Dec 2016 14:31:03 -0800 Subject: [PATCH 0121/8554] Adds ServerRequestLogger. - ServerRequestLogger handles the mapping from model-names to request-loggers and uses the right one for a particular request. Change: 143208086 --- tensorflow_serving/core/BUILD | 19 ++++++ tensorflow_serving/core/request_logger.h | 2 + .../core/server_request_logger.cc | 67 ++++++++++++++++++ .../core/server_request_logger.h | 68 +++++++++++++++++++ 4 files changed, 156 insertions(+) create mode 100644 tensorflow_serving/core/server_request_logger.cc create mode 100644 tensorflow_serving/core/server_request_logger.h diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index bfc7102b984..483bd051773 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -715,6 +715,9 @@ serving_proto_library( name = "logging_proto", srcs = ["logging.proto"], cc_api_version = 2, + visibility = [ + "//visibility:public", + ], deps = [ "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:logging_config_proto", @@ -725,6 +728,9 @@ cc_library( name = "request_logger", srcs = ["request_logger.cc"], hdrs = ["request_logger.h"], + visibility = [ + "//visibility:public", + ], deps = [ ":log_collector", ":logging_proto", @@ -732,3 +738,16 @@ cc_library( "@org_tensorflow//tensorflow/core:lib", ], ) + +cc_library( + name = "server_request_logger", + srcs = ["server_request_logger.cc"], + hdrs = ["server_request_logger.h"], + deps = [ + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/config:logging_config_proto", + "//tensorflow_serving/core:logging_proto", + "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:protobuf", + ], +) diff --git a/tensorflow_serving/core/request_logger.h b/tensorflow_serving/core/request_logger.h index 8224f8d4f99..de9754d844f 100644 --- a/tensorflow_serving/core/request_logger.h +++ b/tensorflow_serving/core/request_logger.h @@ -41,6 +41,8 @@ class RequestLogger { Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, const LogMetadata& log_metadata); + const LoggingConfig& logging_config() const { return logging_config_; } + private: // Creates the log message given the request, response and metadata. // Implementations override it to create the particular message that they want diff --git a/tensorflow_serving/core/server_request_logger.cc b/tensorflow_serving/core/server_request_logger.cc new file mode 100644 index 00000000000..a4bc9c5e867 --- /dev/null +++ b/tensorflow_serving/core/server_request_logger.cc @@ -0,0 +1,67 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/server_request_logger.h" + +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/core/logging.pb.h" + +namespace tensorflow { +namespace serving { + +// static +Status ServerRequestLogger::Create( + const std::map& logging_config_map, + const std::function*)>& + request_logger_creator, + std::unique_ptr* server_request_logger) { + std::unordered_map> request_logger_map; + for (const auto& model_and_logging_config : logging_config_map) { + auto& request_logger = request_logger_map[model_and_logging_config.first]; + TF_RETURN_IF_ERROR(request_logger_creator( + model_and_logging_config.second.log_collector_config(), + &request_logger)); + } + server_request_logger->reset( + new ServerRequestLogger(std::move(request_logger_map))); + return Status::OK(); +} + +ServerRequestLogger::ServerRequestLogger( + std::unordered_map> + request_logger_map) + : request_logger_map_(std::move(request_logger_map)) {} + +Status ServerRequestLogger::Log(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata) { + const string& model_name = log_metadata.model_spec().name(); + auto found_it = request_logger_map_.find(model_name); + if (found_it == request_logger_map_.end()) { + const string error = + strings::StrCat("Cannot find request-logger for model: ", model_name); + // This shouldn't happen at all, so dchecking for capturing in tests. + DCHECK(false) << error; // Crash ok. + return errors::NotFound(error); + } + auto& request_logger = found_it->second; + return request_logger->Log(request, response, log_metadata); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/server_request_logger.h b/tensorflow_serving/core/server_request_logger.h new file mode 100644 index 00000000000..51b1a5e416d --- /dev/null +++ b/tensorflow_serving/core/server_request_logger.h @@ -0,0 +1,68 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_SERVER_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_SERVER_REQUEST_LOGGER_H_ + +#include +#include +#include +#include +#include + +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/logging.pb.h" +#include "tensorflow_serving/core/request_logger.h" + +namespace tensorflow { +namespace serving { + +// Logs a sample of requests hitting all the models in the server. +// +// Constructed based on the logging config for the server, which contains the +// sampling config. +class ServerRequestLogger { + public: + // Creates the ServerRequestLogger based on the map of logging-configs keyed + // on model-names, and a custom request-logger creator method. + static Status Create( + const std::map& logging_config_map, + const std::function*)>& + request_logger_creator, + std::unique_ptr* server_request_logger); + + ~ServerRequestLogger() = default; + + // Similar to RequestLogger::Log(). + Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, + const LogMetadata& log_metadata); + + private: + explicit ServerRequestLogger( + std::unordered_map> + request_logger_map); + + // A map from model-name to its corresponding request-logger. + std::unordered_map> + request_logger_map_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_SERVER_REQUEST_LOGGER_H_ From b34aa51e1b59c9b9799f4e1566512e4e711d904b Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 4 Jan 2017 05:12:47 -0800 Subject: [PATCH 0122/8554] Have BasicManager respond to resource-estimation glitch by erroring servable and logging to event bus. (This is a quick patch. We intend to follow this up with an overhaul of how these sorts of errors are handled, to make things more systematic and robust.) Change: 143543763 --- tensorflow_serving/core/basic_manager.cc | 16 +++++++++++++-- tensorflow_serving/core/basic_manager_test.cc | 20 +++++++++++++++++++ 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 429df9e1dba..567f1f6540f 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -657,8 +657,20 @@ Status BasicManager::ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) { resource_tracker_->RecomputeUsedResources( GetLoadersCurrentlyUsingResources()); bool resources_reserved; - TF_RETURN_IF_ERROR(resource_tracker_->ReserveResources( - *harness->loader(), &resources_reserved)); + const Status reserve_resources_status = + resource_tracker_->ReserveResources(*harness->loader(), + &resources_reserved); + if (!reserve_resources_status.ok()) { + const Status error = errors::Internal(strings::StrCat( + "Error while attempting to reserve resources to load servable ", + harness->id().DebugString(), ": ", + reserve_resources_status.error_message())); + LOG(WARNING) << error; + harness->Error(error); + PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, + harness->status()}); + return error; + } if (resources_reserved) { // Woohoo! We got our resources. LOG(INFO) << "Successfully reserved resources to load servable " diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 70c62c6d6d3..e75298596d3 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -1462,6 +1462,26 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { EqualsServableState(expected_error_state)); } +TEST_F(ResourceConstrainedBasicManagerTest, EventBusErrorOnEstimateResources) { + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock; + EXPECT_CALL(*loader, EstimateResources(_)) + .WillOnce(Return(errors::Internal("Error on estimate resources."))); + basic_manager_->ManageServable( + CreateServableData(id, std::unique_ptr(loader))); + basic_manager_->LoadServable( + id, [](const Status& status) { EXPECT_FALSE(status.ok()); }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, + {ServableState::ManagerState::kEnd}); + const ServableState error_state = { + id, ServableState::ManagerState::kEnd, + errors::Internal(strings::StrCat( + "Error while attempting to reserve resources to load servable ", + id.DebugString(), ": Error on estimate resources."))}; + EXPECT_THAT(*servable_state_monitor_.GetState(id), + EqualsServableState(error_state)); +} + } // namespace } // namespace serving } // namespace tensorflow From 61dac51ec2479a935976a2d8483c3207b46dd128 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 4 Jan 2017 09:34:27 -0800 Subject: [PATCH 0123/8554] Add multi-platform support to ServerCore. Change: 143562528 --- .../core/dynamic_source_router.h | 37 +- .../core/dynamic_source_router_test.cc | 5 +- .../test_util/fake_loader_source_adapter.cc | 2 +- .../fake_loader_source_adapter.proto | 5 +- tensorflow_serving/model_servers/BUILD | 2 + .../model_servers/server_core.cc | 341 +++++++++++++----- .../model_servers/server_core.h | 85 ++++- .../model_servers/server_core_test.cc | 201 +++++++++++ 8 files changed, 555 insertions(+), 123 deletions(-) diff --git a/tensorflow_serving/core/dynamic_source_router.h b/tensorflow_serving/core/dynamic_source_router.h index 3addc980828..c0b779c182c 100644 --- a/tensorflow_serving/core/dynamic_source_router.h +++ b/tensorflow_serving/core/dynamic_source_router.h @@ -33,15 +33,20 @@ namespace serving { template class DynamicSourceRouter final : public SourceRouter { public: + // A servable name -> output port map. + using Routes = std::map; + // Creates a DynamicSourceRouter with 'num_output_ports' output ports and an // (initial) route map given by 'routes'. - static Status Create(int num_output_ports, - const std::map& routes, + static Status Create(int num_output_ports, const Routes& routes, std::unique_ptr>* result); ~DynamicSourceRouter() override; + // Gets the current route map. + Routes GetRoutes() const; + // Sets the route map to 'routes'. - Status UpdateRoutes(const std::map& routes); + Status UpdateRoutes(const Routes& routes); protected: int num_output_ports() const override { return num_output_ports_; } @@ -50,17 +55,15 @@ class DynamicSourceRouter final : public SourceRouter { const std::vector>& versions) override; private: - DynamicSourceRouter(int num_output_ports, - const std::map& routes); + DynamicSourceRouter(int num_output_ports, const Routes& routes); // Returns an error if 'routes' is invalid, given 'num_output_ports'. - static Status ValidateRoutes(int num_output_ports, - const std::map& routes); + static Status ValidateRoutes(int num_output_ports, const Routes& routes); const int num_output_ports_; mutable mutex routes_mu_; - std::map routes_ GUARDED_BY(routes_mu_); + Routes routes_ GUARDED_BY(routes_mu_); TF_DISALLOW_COPY_AND_ASSIGN(DynamicSourceRouter); }; @@ -70,7 +73,7 @@ class DynamicSourceRouter final : public SourceRouter { template Status DynamicSourceRouter::Create( - int num_output_ports, const std::map& routes, + int num_output_ports, const Routes& routes, std::unique_ptr>* result) { TF_RETURN_IF_ERROR(ValidateRoutes(num_output_ports, routes)); result->reset(new DynamicSourceRouter(num_output_ports, routes)); @@ -83,8 +86,14 @@ DynamicSourceRouter::~DynamicSourceRouter() { } template -Status DynamicSourceRouter::UpdateRoutes( - const std::map& routes) { +typename DynamicSourceRouter::Routes DynamicSourceRouter::GetRoutes() + const { + mutex_lock l(routes_mu_); + return routes_; +} + +template +Status DynamicSourceRouter::UpdateRoutes(const Routes& routes) { TF_RETURN_IF_ERROR(ValidateRoutes(num_output_ports_, routes)); { mutex_lock l(routes_mu_); @@ -110,12 +119,12 @@ int DynamicSourceRouter::Route( template DynamicSourceRouter::DynamicSourceRouter(int num_output_ports, - const std::map& routes) + const Routes& routes) : num_output_ports_(num_output_ports), routes_(routes) {} template -Status DynamicSourceRouter::ValidateRoutes( - int num_output_ports, const std::map& routes) { +Status DynamicSourceRouter::ValidateRoutes(int num_output_ports, + const Routes& routes) { for (const auto& entry : routes) { const int port = entry.second; if (port < 0 || port >= num_output_ports) { diff --git a/tensorflow_serving/core/dynamic_source_router_test.cc b/tensorflow_serving/core/dynamic_source_router_test.cc index df9cdb5d272..832c599ce5f 100644 --- a/tensorflow_serving/core/dynamic_source_router_test.cc +++ b/tensorflow_serving/core/dynamic_source_router_test.cc @@ -63,8 +63,9 @@ TEST(DynamicSourceRouterTest, ReconfigureToInvalidRouteMap) { TEST(DynamicSourceRouterTest, Basic) { std::unique_ptr> router; - TF_ASSERT_OK(DynamicSourceRouter::Create(4, {{"foo", 0}, {"bar", 1}}, - &router)); + DynamicSourceRouter::Routes routes = {{"foo", 0}, {"bar", 1}}; + TF_ASSERT_OK(DynamicSourceRouter::Create(4, routes, &router)); + EXPECT_EQ(routes, router->GetRoutes()); std::vector*> output_ports = router->GetOutputPorts(); ASSERT_EQ(4, output_ports.size()); std::vector> targets; diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc index 42b051ebe53..ef2d31e586b 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc @@ -49,7 +49,7 @@ class FakeLoaderSourceAdapterCreator { const FakeLoaderSourceAdapterConfig& config, std::unique_ptr>>* adapter) { - adapter->reset(new FakeLoaderSourceAdapter); + adapter->reset(new FakeLoaderSourceAdapter(config.suffix())); return Status::OK(); } }; diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto b/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto index d170e32befe..d3d4adb1e90 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.proto @@ -2,7 +2,8 @@ syntax = "proto3"; package tensorflow.serving.test_util; -// Config proto for FakeLoaderSourceAdapter. (Even though the adapter doesn't -// have any config parameters, the proto is needed for the adapter registry.) +// Config proto for FakeLoaderSourceAdapter. message FakeLoaderSourceAdapterConfig { + // FakeLoaderSourceAdapter's 'suffix' ctor parameter. + string suffix = 1; } diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 734496dee75..1aa890221eb 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -59,6 +59,7 @@ cc_library( "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/config:platform_config_proto", "//tensorflow_serving/core:aspired_versions_manager", + "//tensorflow_serving/core:dynamic_source_router", "//tensorflow_serving/core:load_servables_fast", "//tensorflow_serving/core:servable_state_monitor", "//tensorflow_serving/core:source", @@ -87,6 +88,7 @@ cc_test( "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/core:servable_state", "//tensorflow_serving/core/test_util:availability_test_util", + "//tensorflow_serving/core/test_util:fake_loader_source_adapter_proto", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/model_servers/test_util:server_core_test_util", "//tensorflow_serving/model_servers/test_util:storage_path_error_injecting_source_adapter", diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 7cbf4c71623..c093cf3a8ad 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -38,48 +38,93 @@ namespace serving { namespace { -// Returns an error if it is not the case that all ModelConfigList models have -// the same model platform, otherwise returns OK and sets 'model_platform' to -// the platform. -Status ValidateAllListedModelsAreOfSamePlatform(const ModelServerConfig& config, - string* model_platform) { - for (const auto& model : config.model_config_list().config()) { - // Get platform (with backward compatibility support of the deprecated - // model_type field). - string platform; - if (model.model_type() != ModelType::MODEL_TYPE_UNSPECIFIED) { - LOG(WARNING) << "Deprecated ModelServerConfig::model_type field used. " - "Prefer ModelServerConfig::model_platform."; - if (!model.model_platform().empty()) { - return errors::InvalidArgument( - "Illegal setting both ModelServerConfig::model_type (deprecated) " - "and ModelServerConfig::model_platform."); - } - if (model.model_type() == ModelType::TENSORFLOW) { - platform = kTensorFlowModelPlatform; - } else { - return errors::InvalidArgument( - strings::StrCat("ModelServerConfig::model_type choice ", - model.model_type(), " not supported.")); - } +// Gets the platform associated with a model. +Status GetPlatform(const ModelConfig& model_config, string* platform) { + if (model_config.model_type() != ModelType::MODEL_TYPE_UNSPECIFIED) { + LOG(WARNING) << "Deprecated ModelServerConfig::model_type field used. " + "Prefer ModelServerConfig::model_platform."; + if (!model_config.model_platform().empty()) { + return errors::InvalidArgument( + "Illegal setting both ModelServerConfig::model_type (deprecated) " + "and ModelServerConfig::model_platform."); + } + if (model_config.model_type() == ModelType::TENSORFLOW) { + *platform = kTensorFlowModelPlatform; } else { - platform = model.model_platform(); + return errors::InvalidArgument( + strings::StrCat("ModelServerConfig::model_type choice ", + model_config.model_type(), " not supported.")); } + } else { + *platform = model_config.model_platform(); + } + + if (platform->empty()) { + return errors::InvalidArgument( + "Illegal setting neither ModelServerConfig::model_type (deprecated) " + "nor ModelServerConfig::model_platform."); + } + return Status::OK(); +} - if (platform.empty()) { +// Returns an error if 'config_list' is invalid in some way, e.g. a model name +// appearing multiple times. +Status ValidateModelConfigList(const ModelConfigList& config_list) { + std::set model_names; + for (const ModelConfig& config : config_list.config()) { + if (model_names.find(config.name()) != model_names.end()) { return errors::InvalidArgument( - "Illegal setting neither ModelServerConfig::model_type (deprecated) " - "nor ModelServerConfig::model_platform."); + strings::StrCat("Illegal to list model ", config.name(), + " multiple times in config list")); } + model_names.insert(config.name()); + } + return Status::OK(); +} - // Check if matches found_platform (so far) - if (model_platform->empty()) { - *model_platform = platform; +// Returns an error if a model exists in both configs, but with different +// platforms. +Status ValidateNoModelsChangePlatforms(const ModelConfigList& old_config_list, + const ModelConfigList& new_config_list) { + std::map old_model_platforms; + for (const ModelConfig& old_config : old_config_list.config()) { + string platform; + TF_RETURN_IF_ERROR(GetPlatform(old_config, &platform)); + old_model_platforms[old_config.name()] = platform; + } + for (const ModelConfig& new_config : new_config_list.config()) { + auto it = old_model_platforms.find(new_config.name()); + if (it == old_model_platforms.end()) { + continue; } - // Error if not, continue if true - if (platform != *model_platform) { + const string& old_platform = it->second; + string new_platform; + TF_RETURN_IF_ERROR(GetPlatform(new_config, &new_platform)); + if (new_platform != old_platform) { return errors::InvalidArgument( - "Multiple model platforms not (yet) supported."); + strings::StrCat("Illegal to change a model's platform. For model ", + new_config.name(), " platform was ", old_platform, + " and new platform requested is ", new_platform)); + } + } + return Status::OK(); +} + +// Unions two route maps. Gives an error if there is a key that is present in +// both 'a' and 'b' but with different values. +Status UnionRoutes(const DynamicSourceRouter::Routes& a, + const DynamicSourceRouter::Routes& b, + DynamicSourceRouter::Routes* result) { + *result = a; + for (const auto& b_entry : b) { + auto a_it = a.find(b_entry.first); + if (a_it == a.end()) { + (*result)[b_entry.first] = b_entry.second; + } else { + if (a_it->second != b_entry.second) { + return errors::InvalidArgument( + "Conflict while unioning two route maps."); + } } } return Status::OK(); @@ -118,7 +163,15 @@ Status ServerCore::Create(Options options, ServerCore::ServerCore(Options options) : options_(std::move(options)), - servable_event_bus_(EventBus::CreateEventBus()) {} + servable_event_bus_(EventBus::CreateEventBus()) { + // Number the platforms. (The proto map iteration order is nondeterministic, + // but we don't care since the numbering is arbitrary.) + int port_num = 0; + for (const auto& entry : options_.platform_config_map.platform_configs()) { + const string& platform = entry.first; + platform_to_router_port_[platform] = port_num++; + } +} Status ServerCore::Initialize(std::unique_ptr policy) { std::unique_ptr servable_state_monitor; @@ -163,47 +216,60 @@ Status ServerCore::WaitUntilConfiguredModelsAvailable() { } Status ServerCore::AddModelsViaModelConfigList() { - // Config validation. - string model_platform; - TF_RETURN_IF_ERROR( - ValidateAllListedModelsAreOfSamePlatform(config_, &model_platform)); - - // Create the source adapter if we haven't done so. - const bool is_first_config = storage_path_source_ == nullptr; - if (is_first_config) { - model_platform_ = model_platform; - } + const bool is_first_config = storage_path_source_and_router_ == nullopt; - // Determine if config transition is legal. - if (!is_first_config && model_platform_ != model_platform) { - return errors::FailedPrecondition( - "Cannot transition to requested model platform. It is only legal to " - "transition to the same model platform."); - } - - // Create/reload file system storage path source. + // Create/reload the source, source router and source adapters. const FileSystemStoragePathSourceConfig source_config = CreateStoragePathSourceConfig(config_); + DynamicSourceRouter::Routes routes; + TF_RETURN_IF_ERROR(CreateStoragePathRoutes(config_, &routes)); if (is_first_config) { - std::unique_ptr source_adapter; - TF_RETURN_IF_ERROR(CreateSourceAdapter(model_platform_, &source_adapter)); + // Construct the following source topology: + // Source -> Router -> Adapter_0 (for models using platform 0) + // -> Adapter_1 (for models using platform 1) + // -> ... + // -> ErrorAdapter (for unrecognized models) + SourceAdapters adapters; + TF_RETURN_IF_ERROR(CreateAdapters(&adapters)); + std::unique_ptr> router; + TF_RETURN_IF_ERROR(CreateRouter(routes, &adapters, &router)); + std::unique_ptr source; TF_RETURN_IF_ERROR( - CreateFileSystemStoragePathSource(source_config, source_adapter.get())); - std::vector static_servables; - for (const auto& model : config_.model_config_list().config()) { - static_servables.push_back(ServableRequest::Latest(model.name())); + CreateStoragePathSource(source_config, router.get(), &source)); + + // Connect the adapters to the manager, and wait for the models to load. + TF_RETURN_IF_ERROR(ConnectAdaptersToManagerAndAwaitModelLoads(&adapters)); + + // Stow the source components. + storage_path_source_and_router_ = {source.get(), router.get()}; + manager_.AddDependency(std::move(source)); + manager_.AddDependency(std::move(router)); + for (auto& entry : adapters.platform_adapters) { + auto& adapter = entry.second; + manager_.AddDependency(std::move(adapter)); } - const tensorflow::Status status = ConnectSourceWithFastInitialLoad( - manager_.get(), source_adapter.get(), servable_state_monitor_.get(), - static_servables, options_.num_initial_load_threads); - if (!status.ok()) { - VLOG(1) << "Unable to ConnectSourceWithFastInitialLoad due to: " - << status; - return status; - } - manager_.AddDependency(std::move(source_adapter)); + manager_.AddDependency(std::move(adapters.error_adapter)); } else { - TF_RETURN_IF_ERROR(ReloadFileSystemStoragePathSourceConfig(source_config)); + // First, add the new routes without removing the old ones. + DynamicSourceRouter::Routes old_and_new_routes; + const Status union_status = + UnionRoutes(storage_path_source_and_router_->router->GetRoutes(), + routes, &old_and_new_routes); + if (!union_status.ok()) { + // ValidateNoModelsChangePlatforms() should have detected any conflict. + DCHECK(false); + return errors::Internal("Old and new routes conflict."); + } + TF_RETURN_IF_ERROR(ReloadRoutes(old_and_new_routes)); + + // Change the source config. Among other things this will cause it to emit + // tear-downs of any models that aren't present in the new config. + TF_RETURN_IF_ERROR(ReloadStoragePathSourceConfig(source_config)); + + // Now that any old models are out of the picture, remove the old routes. + TF_RETURN_IF_ERROR(ReloadRoutes(routes)); + + // Wait for any new models to get loaded and become available. TF_RETURN_IF_ERROR(WaitUntilConfiguredModelsAvailable()); } return Status::OK(); @@ -240,6 +306,14 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { LOG(INFO) << "Taking no action for empty config."; return Status::OK(); } + if (new_config.config_case() == ModelServerConfig::kModelConfigList) { + TF_RETURN_IF_ERROR(ValidateModelConfigList(new_config.model_config_list())); + } + if (new_config.config_case() == ModelServerConfig::kModelConfigList && + config_.config_case() == ModelServerConfig::kModelConfigList) { + TF_RETURN_IF_ERROR(ValidateNoModelsChangePlatforms( + config_.model_config_list(), new_config.model_config_list())); + } config_ = new_config; LOG(INFO) << "Adding/updating models."; @@ -262,9 +336,9 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { return Status::OK(); } -Status ServerCore::CreateSourceAdapter( +Status ServerCore::CreateAdapter( const string& model_platform, - std::unique_ptr* adapter) { + std::unique_ptr* adapter) const { auto config_it = options_.platform_config_map.platform_configs().find(model_platform); if (config_it == options_.platform_config_map.platform_configs().end()) { @@ -297,30 +371,127 @@ FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( return source_config; } -Status ServerCore::CreateFileSystemStoragePathSource( - const FileSystemStoragePathSourceConfig& source_config, - Target* target) { - std::unique_ptr storage_path_source; - const tensorflow::Status status = - FileSystemStoragePathSource::Create(source_config, &storage_path_source); +Status ServerCore::CreateStoragePathRoutes( + const ModelServerConfig& config, + DynamicSourceRouter::Routes* routes) const { + for (const ModelConfig& model_config : config.model_config_list().config()) { + const string& model_name = model_config.name(); + string platform; + TF_RETURN_IF_ERROR(GetPlatform(model_config, &platform)); + auto it = platform_to_router_port_.find(platform); + if (it == platform_to_router_port_.end()) { + return errors::InvalidArgument(strings::StrCat( + "Model ", model_name, " requests unsupported platform ", platform)); + } + const int port = it->second; + (*routes)[model_name] = port; + } + return Status::OK(); +} + +Status ServerCore::CreateStoragePathSource( + const FileSystemStoragePathSourceConfig& config, + Target* target, + std::unique_ptr* source) const { + const Status status = FileSystemStoragePathSource::Create(config, source); if (!status.ok()) { VLOG(1) << "Unable to create FileSystemStoragePathSource due to: " << status; return status; } - ConnectSourceToTarget(storage_path_source.get(), target); - storage_path_source_ = storage_path_source.get(); - manager_.AddDependency(std::move(storage_path_source)); + ConnectSourceToTarget(source->get(), target); + return Status::OK(); +} + +Status ServerCore::CreateRouter( + const DynamicSourceRouter::Routes& routes, + SourceAdapters* targets, + std::unique_ptr>* router) const { + const int num_output_ports = targets->platform_adapters.size() + 1; + const Status status = DynamicSourceRouter::Create( + num_output_ports, routes, router); + if (!status.ok()) { + VLOG(1) << "Unable to create DynamicSourceRouter due to: " << status; + return status; + } + + std::vector*> output_ports = (*router)->GetOutputPorts(); + for (auto& entry : targets->platform_adapters) { + const string& platform = entry.first; + StoragePathSourceAdapter* adapter = entry.second.get(); + + auto it = platform_to_router_port_.find(platform); + if (it == platform_to_router_port_.end()) { + DCHECK(false); + return errors::Internal("Router port for platform not found."); + } + const int port_num = it->second; + + ConnectSourceToTarget(output_ports[port_num], adapter); + } + ConnectSourceToTarget(output_ports[output_ports.size() - 1], + targets->error_adapter.get()); + + return Status::OK(); +} + +Status ServerCore::CreateAdapters(SourceAdapters* adapters) const { + for (const auto& entry : platform_to_router_port_) { + const string& platform = entry.first; + std::unique_ptr adapter; + TF_RETURN_IF_ERROR(CreateAdapter(platform, &adapter)); + adapters->platform_adapters[platform] = std::move(adapter); + } + adapters->error_adapter.reset( + new ErrorInjectingSourceAdapter>( + errors::Internal("No platform found for model"))); return Status::OK(); } -Status ServerCore::ReloadFileSystemStoragePathSourceConfig( +Status ServerCore::ConnectAdaptersToManagerAndAwaitModelLoads( + SourceAdapters* adapters) { + std::map> models_by_platform; + for (const ModelConfig& model_config : config_.model_config_list().config()) { + string platform; + TF_RETURN_IF_ERROR(GetPlatform(model_config, &platform)); + models_by_platform[platform].push_back( + ServableRequest::Latest(model_config.name())); + } + + for (auto& entry : adapters->platform_adapters) { + const string& platform = entry.first; + StoragePathSourceAdapter* adapter = entry.second.get(); + + const Status status = ConnectSourceWithFastInitialLoad( + manager_.get(), adapter, servable_state_monitor_.get(), + models_by_platform[platform], options_.num_initial_load_threads); + if (!status.ok()) { + VLOG(1) << "Unable to ConnectSourceWithFastInitialLoad due to: " + << status; + return status; + } + } + ConnectSourceToTarget(adapters->error_adapter.get(), manager_.get()); + + return Status::OK(); +} + +Status ServerCore::ReloadStoragePathSourceConfig( const FileSystemStoragePathSourceConfig& source_config) { - const tensorflow::Status status = - storage_path_source_->UpdateConfig(source_config); + const Status status = + storage_path_source_and_router_->source->UpdateConfig(source_config); if (!status.ok()) { - VLOG(1) << "Unable to ReloadFileSystemStoragePathSourceConfig due to: " - << status; + VLOG(1) << "Unable to ReloadStoragePathSourceConfig due to: " << status; + } + return status; +} + +Status ServerCore::ReloadRoutes( + const DynamicSourceRouter::Routes& routes) { + const Status status = + storage_path_source_and_router_->router->UpdateRoutes(routes); + if (!status.ok()) { + VLOG(1) << "Unable to ReloadRoutes due to: " << status; } return status; } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index e2c401baf54..da18c1f25bd 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -43,12 +43,14 @@ limitations under the License. #include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/config/platform_config.proto.h" #include "tensorflow_serving/core/aspired_versions_manager.h" +#include "tensorflow_serving/core/dynamic_source_router.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/source.h" #include "tensorflow_serving/core/source_adapter.h" #include "tensorflow_serving/core/storage_path.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/util/event_bus.h" +#include "tensorflow_serving/util/optional.h" #include "tensorflow_serving/util/unique_ptr_with_deps.h" namespace tensorflow { @@ -193,33 +195,72 @@ class ServerCore : public Manager { Status CreateResourceTracker( std::unique_ptr* resource_tracker); - // Creates a platform-specific Loader Source. - Status CreateSourceAdapter( + // Creates a platform-specific source adapter. + Status CreateAdapter( const string& model_platform, - std::unique_ptr* adapter); + std::unique_ptr* adapter) const; // Creates a FileSystemStoragePathSourceConfig from the ModelConfigList of // 'config'. FileSystemStoragePathSourceConfig CreateStoragePathSourceConfig( const ModelServerConfig& config) const; + // Creates routes for a DynamicSourceRouter from the ModelConfigList of + // 'config'. + Status CreateStoragePathRoutes( + const ModelServerConfig& config, + DynamicSourceRouter::Routes* routes) const; + // Waits for all models from the ModelConfigList in 'config_' to be loaded. // Returns an error if any configured model fails to load. Status WaitUntilConfiguredModelsAvailable() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); - // Creates a FileSystemStoragePathSource, connects it to the supplied target, - // stores the pointer in 'storage_path_source_' and transfers the ownership to - // 'manager_'. - Status CreateFileSystemStoragePathSource( - const FileSystemStoragePathSourceConfig& source_config, - Target* target) EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + // Creates a FileSystemStoragePathSource and connects it to the supplied + // target. + Status CreateStoragePathSource( + const FileSystemStoragePathSourceConfig& config, + Target* target, + std::unique_ptr* source) const + EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + + // The source adapters to deploy, to handle the configured platforms as well + // as models whose platform is unknown (errors). + // + // Importantly, we deploy one source adapter per platform, not one per model, + // to handle cross-model optimizations that some platforms/adapters may employ + // e.g. cross-model batch scheduling. + struct SourceAdapters { + // One adapter for each platform. + map> platform_adapters; + + // An extra adapter to report errors for models with no configured platform. + std::unique_ptr error_adapter; + }; + + // Creates a source router and connects it to the supplied adapter targets. + Status CreateRouter( + const DynamicSourceRouter::Routes& routes, + SourceAdapters* targets, + std::unique_ptr>* router) const; - // Updates the 'storage_path_source_' config. - Status ReloadFileSystemStoragePathSourceConfig( + // Creates a set of source adapters based on options_.platform_config_map. + Status CreateAdapters(SourceAdapters* adapters) const; + + // Connects the source adapters to the manager and waits it to load all + // configured models. + Status ConnectAdaptersToManagerAndAwaitModelLoads(SourceAdapters* adapters) + EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + + // Updates the config of 'storage_path_source_and_router_->source'. + Status ReloadStoragePathSourceConfig( const FileSystemStoragePathSourceConfig& source_config) EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + // Updates the configured routes of 'storage_path_source_and_router_->router'. + Status ReloadRoutes(const DynamicSourceRouter::Routes& routes) + EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + // Adds/reloads models through ModelConfigList of 'config_'. Status AddModelsViaModelConfigList() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); @@ -248,6 +289,11 @@ class ServerCore : public Manager { // The options passed to the ctor, minus the AspiredVersionPolicy. Options options_; + // All of the supported platforms (i.e. the ones given in + // 'options_.platform_config_map'), and a router output port number for each. + // Used to deterministically associate a platform with a source adapter. + std::map platform_to_router_port_; + std::shared_ptr> servable_event_bus_; std::shared_ptr servable_state_monitor_; UniquePtrWithDeps manager_; @@ -255,15 +301,16 @@ class ServerCore : public Manager { // The most recent config supplied to ReloadConfig(). ModelServerConfig config_ GUARDED_BY(config_mu_); - // Model platform of the source adapter created for ModelConfigList. - // Empty if the source adapter is not yet created. - string model_platform_ GUARDED_BY(config_mu_); + struct StoragePathSourceAndRouter { + FileSystemStoragePathSource* source; + DynamicSourceRouter* router; + }; - // If the configuration uses a file-system source, this is populated with a - // pointer to the source (to enable reconfiguration later). The source is - // owned by 'manager_'. - FileSystemStoragePathSource* storage_path_source_ GUARDED_BY(config_mu_) = - nullptr; + // If the configuration uses a file-system source, this is populated with + // pointers to the source and router (to enable reconfiguration later). Both + // are owned by 'manager_'. + optional storage_path_source_and_router_ + GUARDED_BY(config_mu_); // A mutex for reconfiguration, used by ReloadConfig(). mutex config_mu_; diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index ece8f731d0d..9e785056c5d 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.proto.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" #include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h" @@ -142,6 +143,206 @@ TEST_P(ServerCoreTest, DeprecatedModelTypeConfig) { EXPECT_EQ(available_servables.at(0), expected_id); } +TEST_P(ServerCoreTest, DuplicateModelNameInConfig) { + std::unique_ptr server_core; + ModelServerConfig config = GetTestModelServerConfigForTensorflowPlatform(); + *config.mutable_model_config_list()->add_config() = + config.model_config_list().config(0); + EXPECT_FALSE(CreateServerCore(config, &server_core).ok()); +} + +TEST_P(ServerCoreTest, UnknownModelPlatform) { + std::unique_ptr server_core; + ModelServerConfig config = GetTestModelServerConfigForTensorflowPlatform(); + config.mutable_model_config_list()->mutable_config(0)->set_model_platform( + "not_a_known_platform"); + EXPECT_FALSE(CreateServerCore(config, &server_core).ok()); +} + +// Creates a model name that incorporates 'platform'. Useful for tests that have +// one model for a given platform. +string ModelNameForPlatform(const string& platform) { + return strings::StrCat("model_for_", platform); +} + +// Builds a ModelSpec with a model named 'ModelNameForPlatform(platform)' and +// version 0. +ModelSpec ModelSpecForPlatform(const string& platform) { + ModelSpec spec; + spec.set_name(ModelNameForPlatform(platform)); + spec.mutable_version()->set_value(0); + return spec; +} + +// Builds a ModelConfig with a model named 'ModelNameForPlatform(platform)', +// base path '/' and platform 'platform'. +ModelConfig ModelConfigForPlatform(const string& root_path, + const string& platform) { + const string model_name = ModelNameForPlatform(platform); + ModelConfig config; + config.set_name(model_name); + config.set_base_path(io::JoinPath(root_path, model_name)); + config.set_model_platform(platform); + return config; +} + +// Creates a directory for the given version of the model. +void CreateModelDir(const ModelConfig& model_config, int version) { + TF_CHECK_OK(Env::Default()->CreateDir(model_config.base_path())); + const string version_str = strings::StrCat(version); + TF_CHECK_OK(Env::Default()->CreateDir( + io::JoinPath(model_config.base_path(), version_str))); +} + +// Adds 'platform' to 'platform_config_map' with a fake source adapter that +// uses suffix 'suffix_for_'. +void CreateFakePlatform(const string& platform, + PlatformConfigMap* platform_config_map) { + test_util::FakeLoaderSourceAdapterConfig source_adapter_config; + source_adapter_config.set_suffix(strings::StrCat("suffix_for_", platform)); + ::google::protobuf::Any source_adapter_config_any; + source_adapter_config_any.PackFrom(source_adapter_config); + (*(*platform_config_map->mutable_platform_configs())[platform] + .mutable_source_adapter_config()) = source_adapter_config_any; +} + +// Constructs the servable data that a platform's fake source adapter will emit. +string ServableDataForPlatform(const string& root_path, const string& platform, + int version) { + const string version_str = strings::StrCat(version); + return io::JoinPath(root_path, ModelNameForPlatform(platform), version_str, + strings::StrCat("suffix_for_", platform)); +} + +TEST_P(ServerCoreTest, MultiplePlatforms) { + const string root_path = io::JoinPath( + testing::TmpDir(), strings::StrCat("MultiplePlatforms_", GetTestType())); + TF_ASSERT_OK(Env::Default()->CreateDir(root_path)); + + // Create a ServerCore with two platforms, and one model for each platform. + ServerCore::Options options = GetDefaultOptions(); + options.platform_config_map.Clear(); + const std::vector platforms = {"platform_0", "platform_1"}; + for (const string& platform : platforms) { + CreateFakePlatform(platform, &options.platform_config_map); + const ModelConfig model_config = + ModelConfigForPlatform(root_path, platform); + *options.model_server_config.mutable_model_config_list()->add_config() = + model_config; + CreateModelDir(model_config, 0 /* version */); + } + std::unique_ptr server_core; + TF_ASSERT_OK(ServerCore::Create(std::move(options), &server_core)); + + // Verify the models got loaded via the platform-specific source adapters. + for (const string& platform : platforms) { + ServableHandle servable_handle; + TF_ASSERT_OK(server_core->GetServableHandle( + ModelSpecForPlatform(platform), &servable_handle)); + const string model_name = ModelNameForPlatform(platform); + const auto expected_servable_id = ServableId{model_name, 0}; + EXPECT_EQ(servable_handle.id(), expected_servable_id); + EXPECT_EQ(ServableDataForPlatform(root_path, platform, 0 /* version */), + *servable_handle); + } +} + +TEST_P(ServerCoreTest, MultiplePlatformsWithConfigChange) { + const string root_path = io::JoinPath( + testing::TmpDir(), + strings::StrCat("MultiplePlatformsWithConfigChange_", GetTestType())); + TF_ASSERT_OK(Env::Default()->CreateDir(root_path)); + + // Create config for three platforms, and one model per platform. + ServerCore::Options options = GetDefaultOptions(); + options.platform_config_map.Clear(); + const std::vector platforms = {"platform_0", "platform_1", + "platform_2"}; + std::vector models; + for (const string& platform : platforms) { + CreateFakePlatform(platform, &options.platform_config_map); + const ModelConfig model_config = + ModelConfigForPlatform(root_path, platform); + models.push_back(model_config); + CreateModelDir(model_config, 0 /* version */); + } + + auto verify_model_loaded = [&root_path](ServerCore* server_core, + const string& platform) { + ServableHandle servable_handle; + TF_ASSERT_OK(server_core->GetServableHandle( + ModelSpecForPlatform(platform), &servable_handle)); + const string model_name = ModelNameForPlatform(platform); + const auto expected_servable_id = ServableId{model_name, 0}; + EXPECT_EQ(servable_handle.id(), expected_servable_id); + EXPECT_EQ(ServableDataForPlatform(root_path, platform, 0 /* version */), + *servable_handle); + }; + auto verify_model_not_loaded = [&root_path](ServerCore* server_core, + const string& platform) { + ServableHandle servable_handle; + EXPECT_FALSE(server_core + ->GetServableHandle(ModelSpecForPlatform(platform), + &servable_handle) + .ok()); + }; + + // Initially configure the ServerCore to have models 0 and 1. + ModelServerConfig* initial_model_config = &options.model_server_config; + (*initial_model_config->mutable_model_config_list()->add_config()) = + models[0]; + (*initial_model_config->mutable_model_config_list()->add_config()) = + models[1]; + std::unique_ptr server_core; + TF_ASSERT_OK(ServerCore::Create(std::move(options), &server_core)); + verify_model_loaded(server_core.get(), platforms[0]); + verify_model_loaded(server_core.get(), platforms[1]); + verify_model_not_loaded(server_core.get(), platforms[2]); + + // Reload with models 1 and 2. + ModelServerConfig new_model_config; + (*new_model_config.mutable_model_config_list()->add_config()) = models[1]; + (*new_model_config.mutable_model_config_list()->add_config()) = models[2]; + TF_ASSERT_OK(server_core->ReloadConfig(new_model_config)); + verify_model_not_loaded(server_core.get(), platforms[0]); + verify_model_loaded(server_core.get(), platforms[1]); + verify_model_loaded(server_core.get(), platforms[2]); +} + +TEST_P(ServerCoreTest, IllegalToChangeModelPlatform) { + const string root_path = io::JoinPath( + testing::TmpDir(), + strings::StrCat("IllegalToChangeModelPlatform_", GetTestType())); + TF_ASSERT_OK(Env::Default()->CreateDir(root_path)); + + ServerCore::Options options = GetDefaultOptions(); + options.platform_config_map.Clear(); + const std::vector platforms = {"platform_0", "platform_1"}; + for (const string& platform : platforms) { + CreateFakePlatform(platform, &options.platform_config_map); + } + + // Configure a model for platform 0. + ModelServerConfig initial_config; + const ModelConfig model_config = + ModelConfigForPlatform(root_path, platforms[0]); + *initial_config.mutable_model_config_list()->add_config() = model_config; + CreateModelDir(model_config, 0 /* version */); + + options.model_server_config = initial_config; + std::unique_ptr server_core; + TF_ASSERT_OK(ServerCore::Create(std::move(options), &server_core)); + + // Attempt to switch the existing model to platform 1. + ModelServerConfig new_config = initial_config; + new_config.mutable_model_config_list()->mutable_config(0)->set_model_platform( + platforms[1]); + const Status reconfigure_status = server_core->ReloadConfig(new_config); + EXPECT_FALSE(reconfigure_status.ok()); + EXPECT_THAT(reconfigure_status.ToString(), + ::testing::HasSubstr("Illegal to change a model's platform")); +} + INSTANTIATE_TEST_CASE_P( TestType, ServerCoreTest, ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES))); From b556d2f81b3c63973207bc512c4ee75a527aecf7 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Wed, 4 Jan 2017 09:48:03 -0800 Subject: [PATCH 0124/8554] Enable SavedModel in model-server. Change: 143563826 --- tensorflow_serving/model_servers/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 780e032c116..83bb941ff69 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -201,7 +201,7 @@ int main(int argc, char** argv) { tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; - bool use_saved_model = false; + bool use_saved_model = true; string platform_config_file = ""; tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( From 7a95d06c3764bf8d782528ea8626d66dd3f675d7 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Wed, 4 Jan 2017 16:46:11 -0800 Subject: [PATCH 0125/8554] Remove a few ununsed functions. Change: 143612122 --- tensorflow_serving/servables/tensorflow/BUILD | 5 +-- .../tensorflow/bundle_factory_test.h | 35 ------------------- .../tensorflow/bundle_factory_util_test.cc | 31 ++++++++++++++++ .../tensorflow/saved_model_bundle_factory.cc | 6 ---- .../tensorflow/saved_model_bundle_factory.h | 6 ---- .../saved_model_bundle_factory_test.cc | 10 ------ .../tensorflow/session_bundle_factory.cc | 6 ---- .../tensorflow/session_bundle_factory.h | 6 ---- .../tensorflow/session_bundle_factory_test.cc | 5 --- 9 files changed, 32 insertions(+), 78 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index f389336fb15..b611a9450dc 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -74,6 +74,7 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", + "//tensorflow_serving/util/test_util:mock_file_probing_env", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", @@ -112,8 +113,6 @@ cc_library( ":session_bundle_config_proto", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", - "//tensorflow_serving/util:file_probing_env", - "//tensorflow_serving/util/test_util:mock_file_probing_env", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", @@ -134,7 +133,6 @@ cc_library( "//tensorflow_serving/batching:batching_session", "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resources_proto", - "//tensorflow_serving/util:file_probing_env", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", "@org_tensorflow//tensorflow/core:core_cpu", @@ -179,7 +177,6 @@ cc_library( "//tensorflow_serving/batching:batching_session", "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resources_proto", - "//tensorflow_serving/util:file_probing_env", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h index b2bef5e64b4..173df46ef29 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h @@ -26,14 +26,11 @@ limitations under the License. #include #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/public/session.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/test_util/test_util.h" -#include "tensorflow_serving/util/file_probing_env.h" -#include "tensorflow_serving/util/test_util/mock_file_probing_env.h" namespace tensorflow { namespace serving { @@ -88,38 +85,6 @@ class BundleFactoryTest : public ::testing::Test { EXPECT_THAT(actual, EqualsProto(expected)); } - template - void TestEstimateResourceRequirementWithFileProbingEnv() const { - const string export_dir = "/foo/bar"; - const string child = "child"; - const string child_path = io::JoinPath(export_dir, child); - const double file_size = 100; - - // Set up the expectation that the directory contains exactly one child with - // the given file size. - MockFileProbingEnv env; - EXPECT_CALL(env, FileExists(export_dir)) - .WillRepeatedly(Return(Status::OK())); - EXPECT_CALL(env, GetChildren(export_dir, _)) - .WillRepeatedly(DoAll(SetArgPointee<1>(std::vector({child})), - Return(Status::OK()))); - EXPECT_CALL(env, IsDirectory(child_path)) - .WillRepeatedly(Return(errors::FailedPrecondition(""))); - EXPECT_CALL(env, GetFileSize(child_path, _)) - .WillRepeatedly( - DoAll(SetArgPointee<1>(file_size), Return(Status::OK()))); - - const SessionBundleConfig config; - std::unique_ptr factory; - TF_ASSERT_OK(FactoryType::Create(config, &factory)); - ResourceAllocation actual; - TF_ASSERT_OK( - factory->EstimateResourceRequirement(export_dir, &env, &actual)); - - ResourceAllocation expected = GetExpectedResourceEstimate(file_size); - EXPECT_THAT(actual, EqualsProto(expected)); - } - void TestRunOptions() const { SessionBundleConfig config; diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc index 02274ceb795..7331768f44b 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/public/session_options.h" @@ -36,11 +37,15 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/test_util/test_util.h" +#include "tensorflow_serving/util/test_util/mock_file_probing_env.h" namespace tensorflow { namespace serving { namespace { +using ::testing::_; +using ::testing::Return; +using ::testing::SetArgPointee; using test_util::EqualsProto; using Batcher = SharedBatchScheduler; @@ -144,6 +149,32 @@ TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithGoodExport) { EXPECT_THAT(actual, EqualsProto(expected)); } +TEST_F(BundleFactoryUtilTest, EstimateResourceFromPathWithFileProbingEnv) { + const string export_dir = "/foo/bar"; + const string child = "child"; + const string child_path = io::JoinPath(export_dir, child); + const double file_size = 100; + + // Set up the expectation that the directory contains exactly one child with + // the given file size. + test_util::MockFileProbingEnv env; + EXPECT_CALL(env, FileExists(export_dir)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(env, GetChildren(export_dir, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(std::vector({child})), + Return(Status::OK()))); + EXPECT_CALL(env, IsDirectory(child_path)) + .WillRepeatedly(Return(errors::FailedPrecondition(""))); + EXPECT_CALL(env, GetFileSize(child_path, _)) + .WillRepeatedly(DoAll(SetArgPointee<1>(file_size), Return(Status::OK()))); + + ResourceAllocation actual; + TF_ASSERT_OK(EstimateResourceFromPath(export_dir, &env, &actual)); + + ResourceAllocation expected = + test_util::GetExpectedResourceEstimate(file_size); + EXPECT_THAT(actual, EqualsProto(expected)); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc index 996cf77cc98..f96e3c1d762 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -56,12 +56,6 @@ Status SavedModelBundleFactory::EstimateResourceRequirement( return EstimateResourceFromPath(path, estimate); } -Status SavedModelBundleFactory::EstimateResourceRequirement( - const string& path, FileProbingEnv* env, - ResourceAllocation* estimate) const { - return EstimateResourceFromPath(path, env, estimate); -} - Status SavedModelBundleFactory::CreateSavedModelBundle( const string& path, std::unique_ptr* bundle) { bundle->reset(new SavedModelBundle); diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h index edda0b3ab34..65f25aebc69 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h @@ -23,7 +23,6 @@ limitations under the License. #include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" -#include "tensorflow_serving/util/file_probing_env.h" namespace tensorflow { namespace serving { @@ -62,11 +61,6 @@ class SavedModelBundleFactory { Status EstimateResourceRequirement(const string& path, ResourceAllocation* estimate) const; - // Similar to the above method, but also supplies a FileProbingEnv to use in - // lieu of tensorflow::Env::Default(). - Status EstimateResourceRequirement(const string& path, FileProbingEnv* env, - ResourceAllocation* estimate) const; - private: using Batcher = SharedBatchScheduler; diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index bf03f887370..ac26c951fb3 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -74,11 +74,6 @@ TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { kTotalFileSize); } -TEST_F(SavedModelBundleFactoryTest, - EstimateResourceRequirementWithFileProbingEnv) { - TestEstimateResourceRequirementWithFileProbingEnv(); -} - TEST_F(SavedModelBundleFactoryTest, RunOptions) { TestRunOptions(); } TEST_F(SavedModelBundleFactoryTest, RunOptionsError) { TestRunOptionsError(); } @@ -114,11 +109,6 @@ TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, kTotalFileSize); } -TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, - EstimateResourceRequirementWithFileProbingEnv) { - TestEstimateResourceRequirementWithFileProbingEnv(); -} - TEST_F(SavedModelBundleFactoryBackwardCompatibilityTest, RunOptions) { TestRunOptions(); } diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc index 4db48cc0db6..b899f41a68b 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.cc @@ -73,12 +73,6 @@ Status SessionBundleFactory::EstimateResourceRequirement( return EstimateResourceFromPath(path, estimate); } -Status SessionBundleFactory::EstimateResourceRequirement( - const string& path, FileProbingEnv* env, - ResourceAllocation* estimate) const { - return EstimateResourceFromPath(path, env, estimate); -} - Status SessionBundleFactory::CreateSessionBundle( const string& path, std::unique_ptr* bundle) { bundle->reset(new SessionBundle); diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h index 5250f99c1b1..3037e8f73c6 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h @@ -22,7 +22,6 @@ limitations under the License. #include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" -#include "tensorflow_serving/util/file_probing_env.h" namespace tensorflow { namespace serving { @@ -60,11 +59,6 @@ class SessionBundleFactory { Status EstimateResourceRequirement(const string& path, ResourceAllocation* estimate) const; - // Similar to the above method, but also supplies a FileProbingEnv to use in - // lieu of tensorflow::Env::Default(). - Status EstimateResourceRequirement(const string& path, FileProbingEnv* env, - ResourceAllocation* estimate) const; - private: using Batcher = SharedBatchScheduler; diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index b3857332e1b..a195f5b7876 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -70,11 +70,6 @@ TEST_F(SessionBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { kTotalFileSize); } -TEST_F(SessionBundleFactoryTest, - EstimateResourceRequirementWithFileProbingEnv) { - TestEstimateResourceRequirementWithFileProbingEnv(); -} - TEST_F(SessionBundleFactoryTest, RunOptions) { TestRunOptions(); } TEST_F(SessionBundleFactoryTest, RunOptionsError) { TestRunOptionsError(); } From 4403ee166b24c75a37e4b8896a1f8df85b67f1c5 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 5 Jan 2017 14:15:08 -0800 Subject: [PATCH 0126/8554] Refactor BasicManager::ApproveLoad() by splitting out a separate ReserveResources() method and consolidating the error state transition into a single location. Change: 143707593 --- tensorflow_serving/core/basic_manager.cc | 100 +++++++++++------------ tensorflow_serving/core/basic_manager.h | 10 +++ 2 files changed, 60 insertions(+), 50 deletions(-) diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 567f1f6540f..0c8f225cd4d 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -631,71 +631,33 @@ Status BasicManager::ApproveLoadOrUnload(const LoadOrUnloadRequest& request, TF_RETURN_IF_ERROR(GetHealthyHarness(request.servable_id, harness)); - Status approval_status; switch (request.kind) { case LoadOrUnloadRequest::Kind::kLoad: { - approval_status = ApproveLoad(*harness, &l); + TF_RETURN_IF_ERROR(ApproveLoad(*harness, &l)); break; } case LoadOrUnloadRequest::Kind::kUnload: { - approval_status = ApproveUnload(*harness); + TF_RETURN_IF_ERROR(ApproveUnload(*harness)); break; } } - if (approval_status.ok()) { - ++num_ongoing_load_unload_executions_; - } + ++num_ongoing_load_unload_executions_; - return approval_status; + return Status::OK(); } Status BasicManager::ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) { if (resource_tracker_ != nullptr) { // Attempt to reserve resources for the load. - while (true) { - resource_tracker_->RecomputeUsedResources( - GetLoadersCurrentlyUsingResources()); - bool resources_reserved; - const Status reserve_resources_status = - resource_tracker_->ReserveResources(*harness->loader(), - &resources_reserved); - if (!reserve_resources_status.ok()) { - const Status error = errors::Internal(strings::StrCat( - "Error while attempting to reserve resources to load servable ", - harness->id().DebugString(), ": ", - reserve_resources_status.error_message())); - LOG(WARNING) << error; - harness->Error(error); - PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, - harness->status()}); - return error; - } - if (resources_reserved) { - // Woohoo! We got our resources. - LOG(INFO) << "Successfully reserved resources to load servable " - << harness->id().DebugString(); - break; - } - - // We weren't able to reserve the resources. See if there are any ongoing - // load/unload executions that may be temporarily tying up resources. - if (num_ongoing_load_unload_executions_ == 0) { - // There are no ongoing load/unloads, so we really are out of resources - // for this servable. - LOG(WARNING) << "Unable to reserve resources to load servable " - << harness->id().DebugString(); - const Status error = errors::ResourceExhausted( - "Insufficient resources to load servable ", - harness->id().DebugString()); - harness->Error(error); - PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, - harness->status()}); - return error; - } else { - // Wait until at least one load/unload request finishes, then retry. - num_ongoing_load_unload_executions_cv_.wait(*mu_lock); - } + const Status resource_reservation_status = + ReserveResources(harness, mu_lock); + if (!resource_reservation_status.ok()) { + LOG(WARNING) << resource_reservation_status; + harness->Error(resource_reservation_status); + PublishOnEventBus({harness->id(), ServableState::ManagerState::kEnd, + resource_reservation_status}); + return resource_reservation_status; } } @@ -718,6 +680,44 @@ Status BasicManager::ApproveUnload(LoaderHarness* harness) { return Status::OK(); } +Status BasicManager::ReserveResources(LoaderHarness* harness, + mutex_lock* mu_lock) { + while (true) { + resource_tracker_->RecomputeUsedResources( + GetLoadersCurrentlyUsingResources()); + bool resources_reserved; + const Status reserve_resources_status = resource_tracker_->ReserveResources( + *harness->loader(), &resources_reserved); + if (!reserve_resources_status.ok()) { + return errors::Internal(strings::StrCat( + "Error while attempting to reserve resources to load servable ", + harness->id().DebugString(), ": ", + reserve_resources_status.error_message())); + } + if (resources_reserved) { + // Woohoo! We got our resources. + LOG(INFO) << "Successfully reserved resources to load servable " + << harness->id().DebugString(); + return Status::OK(); + } + + // We weren't able to reserve the resources. See if there are any + // ongoing load/unload executions that may be temporarily tying up + // resources. + if (num_ongoing_load_unload_executions_ == 0) { + // There are no ongoing load/unloads, so we really are out of + // resources for this servable. + return errors::ResourceExhausted( + "Insufficient resources to load servable ", + harness->id().DebugString()); + } else { + // Wait until at least one load/unload request finishes, then retry. + VLOG(1) << "Waiting for another load/unload request to finish"; + num_ongoing_load_unload_executions_cv_.wait(*mu_lock); + } + } +} + void BasicManager::PublishOnEventBus(const ServableState& state) { if (servable_event_bus_ != nullptr) { servable_event_bus_->Publish(state); diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index 9cac6a4de5b..ef6aa40d956 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -332,6 +332,16 @@ class BasicManager : public Manager { // prevents a subsequent unload request from proceeding concurrently. Status ApproveUnload(LoaderHarness* harness) EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Attempts to reserve the resources required to load the servable in + // 'harness'. Does not make any state transitions on 'harness' -- merely + // reserves the resources in 'resource_tracker_' (upon success) or returns an + // error. + // + // Argument 'mu_lock' is a lock held on 'mu_'. It is released temporarily via + // 'num_ongoing_load_unload_executions_cv_'. + Status ReserveResources(LoaderHarness* harness, mutex_lock* mu_lock) + EXCLUSIVE_LOCKS_REQUIRED(mu_); + // The execution phase of loading/unloading a servable. Delegates to either // ExecuteLoad() or ExecuteUnload(). // From 50b2833416c97778a376e13a833d713f14e10d1d Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 5 Jan 2017 14:26:46 -0800 Subject: [PATCH 0127/8554] Add some vlog statements. Change: 143709118 --- tensorflow_serving/core/aspired_versions_manager.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 3a6ed7c8a72..ea458d6e6fb 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -261,6 +261,7 @@ void AspiredVersionsManager::ProcessAspiredVersionsRequest( // If this version is not part of the aspired versions. if (std::find(next_aspired_versions.begin(), next_aspired_versions.end(), state_snapshot.id.version) == next_aspired_versions.end()) { + VLOG(1) << "Setting is_aspired=false for " << state_snapshot.id; basic_manager_->GetAdditionalServableState(state_snapshot.id) ->is_aspired = false; basic_manager_->CancelLoadServableRetry(state_snapshot.id); @@ -282,6 +283,7 @@ void AspiredVersionsManager::ProcessAspiredVersionsRequest( // if this aspired version is not already present in the map. if (std::find(additions.begin(), additions.end(), version.id().version) != additions.end()) { + VLOG(1) << "Adding " << version.id() << "to BasicManager"; const Status manage_status = basic_manager_->ManageServableWithAdditionalState( std::move(version), std::unique_ptr(new Aspired{true})); @@ -373,6 +375,7 @@ void AspiredVersionsManager::FlushServables() { state_snapshot.state == LoaderHarness::State::kDisabled || state_snapshot.state == LoaderHarness::State::kError) && !state_snapshot.additional_state->is_aspired) { + VLOG(1) << "Removing " << state_snapshot.id << "from BasicManager"; basic_manager_->StopManagingServable(state_snapshot.id); } } From 189818f15a0915fb512bb34e314a8cf43ebcb279 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Fri, 6 Jan 2017 12:44:32 -0800 Subject: [PATCH 0128/8554] Adds a Retry abstraction for retrying function calls. - And uses it in LoaderHarness and BasicManager for retrying loads and estimating resources, respectively. Change: 143803207 --- .../basic_batch_scheduler_benchmark.cc | 2 +- tensorflow_serving/core/BUILD | 1 + .../aspired_versions_manager_benchmark.cc | 2 +- tensorflow_serving/core/basic_manager.cc | 15 +++- tensorflow_serving/core/basic_manager_test.cc | 71 ++++++++++++++++++ tensorflow_serving/core/loader_harness.cc | 31 ++------ tensorflow_serving/util/BUILD | 19 +++++ .../util/fast_read_dynamic_ptr_benchmark.cc | 2 +- tensorflow_serving/util/retrier.cc | 56 ++++++++++++++ tensorflow_serving/util/retrier.h | 41 +++++++++++ tensorflow_serving/util/retrier_test.cc | 73 +++++++++++++++++++ 11 files changed, 284 insertions(+), 29 deletions(-) create mode 100644 tensorflow_serving/util/retrier.cc create mode 100644 tensorflow_serving/util/retrier.h create mode 100644 tensorflow_serving/util/retrier_test.cc diff --git a/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc b/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc index 9ee939fd670..40d6315eba9 100644 --- a/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc +++ b/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc @@ -19,7 +19,7 @@ limitations under the License. // Run with: // bazel run -c opt --dynamic_mode=off \ // tensorflow_serving/batching:basic_batch_scheduler_benchmark -- -// --benchmarks=. --benchmark_use_picoseconds +// --benchmarks=. // For a longer run time and more consistent results for the throughput // benchmark, consider a min time e.g.: --benchmark_min_time=60.0 diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 483bd051773..81fd4478417 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -416,6 +416,7 @@ cc_library( ":loader", ":servable_id", "//tensorflow_serving/util:optional", + "//tensorflow_serving/util:retrier", "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc index 85ffbe37cb1..f078f4d3a9f 100644 --- a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc +++ b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc @@ -16,7 +16,7 @@ limitations under the License. // Run with: // bazel run -c opt --dynamic_mode=off \ // tensorflow_serving/core:aspired_versions_manager_benchmark -- -// --benchmarks=. --benchmark_use_picoseconds +// --benchmarks=. // For a longer run time and more consistent results, consider a min time // e.g.: --benchmark_min_time=60.0 diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 0c8f225cd4d..9d1ed22ec00 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -32,6 +32,7 @@ limitations under the License. #include "tensorflow_serving/util/cleanup.h" #include "tensorflow_serving/util/hash.h" #include "tensorflow_serving/util/inline_executor.h" +#include "tensorflow_serving/util/retrier.h" #include "tensorflow_serving/util/threadpool_executor.h" namespace tensorflow { @@ -686,8 +687,18 @@ Status BasicManager::ReserveResources(LoaderHarness* harness, resource_tracker_->RecomputeUsedResources( GetLoadersCurrentlyUsingResources()); bool resources_reserved; - const Status reserve_resources_status = resource_tracker_->ReserveResources( - *harness->loader(), &resources_reserved); + // We retry reserving resources because it may involve transiently failing + // operations like file-reads. + const Status reserve_resources_status = + Retry(strings::StrCat("Reserving resources for servable: ", + harness->id().DebugString()), + harness_options_.max_num_load_retries, + harness_options_.load_retry_interval_micros, + [&]() EXCLUSIVE_LOCKS_REQUIRED(mu_) { + return resource_tracker_->ReserveResources(*harness->loader(), + &resources_reserved); + }, + [&]() { return harness->cancel_load_retry(); }); if (!reserve_resources_status.ok()) { return errors::Internal(strings::StrCat( "Error while attempting to reserve resources to load servable ", diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index e75298596d3..fc78960c18d 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -1482,6 +1482,77 @@ TEST_F(ResourceConstrainedBasicManagerTest, EventBusErrorOnEstimateResources) { EqualsServableState(error_state)); } +TEST(EstimateResourcesRetriedTest, Succeeds) { + std::shared_ptr> servable_event_bus = + EventBus::CreateEventBus(); + ServableStateMonitor servable_state_monitor(servable_event_bus.get()); + + BasicManager::Options options; + // Seed the manager with ten resource units. + options.resource_tracker = CreateSimpleResourceTracker(10); + options.servable_event_bus = servable_event_bus.get(); + options.num_load_threads = 0; + options.num_unload_threads = 0; + + options.max_num_load_retries = 1; + options.load_retry_interval_micros = 0; + + std::unique_ptr basic_manager; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager)); + + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock; + EXPECT_CALL(*loader, EstimateResources(_)) + .WillOnce(Return(errors::Internal("Error on estimate resources."))) + .WillOnce(Return(Status::OK())); + basic_manager->ManageServable( + CreateServableData(id, std::unique_ptr(loader))); + basic_manager->LoadServable( + id, [](const Status& status) { EXPECT_TRUE(status.ok()); }); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor, id, {ServableState::ManagerState::kAvailable}); + const ServableState available_state = { + id, ServableState::ManagerState::kAvailable, Status::OK()}; + EXPECT_THAT(*servable_state_monitor.GetState(id), + EqualsServableState(available_state)); +} + +TEST(EstimateResourcesRetriedTest, Fails) { + std::shared_ptr> servable_event_bus = + EventBus::CreateEventBus(); + ServableStateMonitor servable_state_monitor(servable_event_bus.get()); + + BasicManager::Options options; + // Seed the manager with ten resource units. + options.resource_tracker = CreateSimpleResourceTracker(10); + options.servable_event_bus = servable_event_bus.get(); + options.num_load_threads = 0; + options.num_unload_threads = 0; + + options.max_num_load_retries = 1; + options.load_retry_interval_micros = 0; + + std::unique_ptr basic_manager; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager)); + + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock; + EXPECT_CALL(*loader, EstimateResources(_)) + .WillOnce(Return(errors::Internal("Error on estimate resources."))) + .WillOnce(Return(errors::Internal("Error on estimate resources."))) + .WillRepeatedly(Return(Status::OK())); + basic_manager->ManageServable( + CreateServableData(id, std::unique_ptr(loader))); + basic_manager->LoadServable( + id, [](const Status& status) { EXPECT_FALSE(status.ok()); }); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor, id, + {ServableState::ManagerState::kEnd}); + const ServableState available_state = { + id, ServableState::ManagerState::kEnd, + errors::Internal("Error on estimate resources.")}; + EXPECT_FALSE(servable_state_monitor.GetState(id)->health.ok()); +} + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/loader_harness.cc b/tensorflow_serving/core/loader_harness.cc index 37cd41c3c6b..bd2601d397c 100644 --- a/tensorflow_serving/core/loader_harness.cc +++ b/tensorflow_serving/core/loader_harness.cc @@ -18,7 +18,9 @@ limitations under the License. #include #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/env.h" +#include "tensorflow_serving/util/retrier.h" namespace tensorflow { namespace serving { @@ -89,30 +91,11 @@ Status LoaderHarness::Load(const ResourceAllocation& available_resources) { LOG(INFO) << "Loading servable version " << id_; } - const Status status = [&]() { - Status load_status; - int num_tries = 0; - do { - if (num_tries > 0) { - if (cancel_load_retry()) { - LOG(INFO) << "Load retry cancelled for servable: " << id_; - break; - } - Env::Default()->SleepForMicroseconds( - options_.load_retry_interval_micros); - LOG(INFO) << "Retrying load on servable version: " << id_ - << " retry: " << num_tries; - } - load_status = loader_->Load(available_resources); - if (!load_status.ok()) { - LOG(ERROR) << "Servable: " << id_ << " load failure: " << load_status; - } - ++num_tries; - } while (!cancel_load_retry() && !load_status.ok() && - (num_tries - 1) < options_.max_num_load_retries); - - return load_status; - }(); + const Status status = + Retry(strings::StrCat("Loading servable: ", id_.DebugString()), + options_.max_num_load_retries, options_.load_retry_interval_micros, + [&]() { return loader_->Load(available_resources); }, + [&]() { return cancel_load_retry(); }); { mutex_lock l(mu_); diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index ec570163fe8..1637afb8d05 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -301,6 +301,25 @@ cc_test( ], ) +cc_library( + name = "retrier", + srcs = ["retrier.cc"], + hdrs = ["retrier.h"], + deps = [ + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "retrier_test", + srcs = ["retrier_test.cc"], + deps = [ + ":retrier", + "//tensorflow_serving/core/test_util:test_main", + "@org_tensorflow//tensorflow/core:lib", + ], +) + load("//tensorflow_serving:serving.bzl", "serving_proto_library") serving_proto_library( diff --git a/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc b/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc index 08f8cf07af2..cb468a95484 100644 --- a/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc +++ b/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc @@ -28,7 +28,7 @@ limitations under the License. // Run with: // bazel run -c opt \ // tensorflow_serving/util:fast_read_dynamic_ptr_benchmark -- -// --benchmarks=. --benchmark_use_picoseconds +// --benchmarks=. // For a longer run time and more consistent results, consider a min time // e.g.: --benchmark_min_time=60.0 diff --git a/tensorflow_serving/util/retrier.cc b/tensorflow_serving/util/retrier.cc new file mode 100644 index 00000000000..9f03a8d4490 --- /dev/null +++ b/tensorflow_serving/util/retrier.cc @@ -0,0 +1,56 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/util/retrier.h" + +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/logging.h" + +namespace tensorflow { +namespace serving { + +Status Retry(const string& description, const uint32 max_num_retries, + const int64 retry_interval_micros, + const std::function& retried_fn, + const std::function& is_cancelled) { + return [&]() { + Status status; + int num_tries = 0; + do { + if (num_tries > 0) { + Env::Default()->SleepForMicroseconds(retry_interval_micros); + LOG(INFO) << "Retrying of " << description << " retry: " << num_tries; + } + status = retried_fn(); + if (!status.ok()) { + LOG(ERROR) << description << " failed: " << status; + } + ++num_tries; + } while (!is_cancelled() && !status.ok() && + num_tries < max_num_retries + 1); + + if (is_cancelled()) { + LOG(INFO) << "Retrying of " << description << " was cancelled."; + } + if (num_tries == max_num_retries + 1) { + LOG(INFO) << "Retrying of " << description + << " exhausted max_num_retries: " << max_num_retries; + } + return status; + }(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/util/retrier.h b/tensorflow_serving/util/retrier.h new file mode 100644 index 00000000000..6fe26f77617 --- /dev/null +++ b/tensorflow_serving/util/retrier.h @@ -0,0 +1,41 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_UTIL_RETRIER_H_ +#define TENSORFLOW_SERVING_UTIL_RETRIER_H_ + +#include +#include + +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace serving { + +// Tries running 'retried_fn' once, and if it doesn't succeed, retries running +// the 'retried_fn' till it returns an ok status or max_num_retries are +// exhausted or cancelled() returns true. Each retry is attempted after an +// interval of 'retry_interval_micros'. The 'description' is useful for logging. +// +// Returns the status returned by the last call to 'retried_fn'. +Status Retry(const string& description, uint32 max_num_retries, + int64 retry_interval_micros, + const std::function& retried_fn, + const std::function& is_cancelled = [] { return false; }); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_UTIL_RETRIER_H_ diff --git a/tensorflow_serving/util/retrier_test.cc b/tensorflow_serving/util/retrier_test.cc new file mode 100644 index 00000000000..082a4a3da64 --- /dev/null +++ b/tensorflow_serving/util/retrier_test.cc @@ -0,0 +1,73 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/util/retrier.h" + +#include +#include +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::HasSubstr; + +TEST(RetrierTest, RetryFinallySucceeds) { + auto retried_fn = []() { + static int count = 0; + ++count; + if (count == 1) { + return errors::Unknown("Error"); + } + return Status::OK(); + }; + + TF_EXPECT_OK(Retry("RetryFinallySucceeds", 1 /* max_num_retries */, + 1 /* retry_interval_micros */, retried_fn)); +} + +TEST(RetrierTest, RetryFinallyFails) { + auto retried_fn = []() { + static int count = 0; + if (++count <= 2) { + return errors::Unknown("Error"); + } + return Status::OK(); + }; + + const auto status = Retry("RetryFinallyFails", 1 /* max_num_retries */, + 0 /* retry_interval_micros */, retried_fn); + EXPECT_THAT(status.error_message(), HasSubstr("Error")); +} + +TEST(RetrierTest, RetryCancelled) { + int call_count = 0; + auto retried_fn = [&]() { + ++call_count; + return errors::Unknown("Error"); + }; + const auto status = Retry("RetryCancelled", 10 /* max_num_retries */, + 0 /* retry_interval_micros */, retried_fn, + []() { return true; } /* cancelled */); + EXPECT_THAT(status.error_message(), HasSubstr("Error")); + EXPECT_EQ(1, call_count); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From 4fdb26c5d40a9a6ff0d1ab7d2bf8bc8c997827c4 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 10 Jan 2017 12:23:34 -0800 Subject: [PATCH 0129/8554] Remove "available_resources" argument from Loader::Load(). We had not been passing a real value for it, and with the (newer) ability to run multiple loads concurrently there's no way to get the semantics correct with "available_resources", so we're dropping it. It will be up to loaders to coordinate with device resource managers (e.g. GPUs) to reason about available resources on specific device instances (e.g. memory on GPU1 vs GPU2). Change: 144113576 --- .../core/aspired_versions_manager_test.cc | 25 +++--- tensorflow_serving/core/basic_manager.cc | 2 +- tensorflow_serving/core/basic_manager_test.cc | 44 +++++----- tensorflow_serving/core/loader.h | 23 ++--- tensorflow_serving/core/loader_harness.cc | 11 ++- tensorflow_serving/core/loader_harness.h | 3 +- .../core/loader_harness_test.cc | 84 ++++++++----------- tensorflow_serving/core/simple_loader.h | 5 +- tensorflow_serving/core/simple_loader_test.cc | 8 +- .../core/test_util/mock_loader.h | 2 +- .../resources/resource_tracker_test.cc | 2 +- .../hashmap/hashmap_source_adapter_test.cc | 2 +- .../saved_model_bundle_source_adapter_test.cc | 2 +- .../session_bundle_source_adapter_test.cc | 2 +- 14 files changed, 90 insertions(+), 125 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 17e7b3bd3f8..98d02f7f82e 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -664,12 +664,11 @@ TEST_P(AspiredVersionsManagerTest, EventBusServableLifecycle) { Notification load_called; Notification load_continue; - EXPECT_CALL(*loader, Load(_)) - .WillOnce(InvokeWithoutArgs([&]() { - load_called.Notify(); - load_continue.WaitForNotification(); - return Status::OK(); - })); + EXPECT_CALL(*loader, Load()).WillOnce(InvokeWithoutArgs([&]() { + load_called.Notify(); + load_continue.WaitForNotification(); + return Status::OK(); + })); std::unique_ptr load_thread( Env::Default()->StartThread(ThreadOptions(), "LoadThread", @@ -751,7 +750,7 @@ TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallySucceeds) { test_util::MockLoader* loader = new NiceMock; // We succeed on the last load, before the manager gives up. - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(Return(errors::Internal("Error on load."))) .WillOnce(Return(Status::OK())); @@ -871,7 +870,7 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { std::vector>> first_aspired_versions; test_util::MockLoader* first_loader = new NiceMock(); first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); - EXPECT_CALL(*first_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*first_loader, Load()).WillOnce(Return(Status::OK())); manager_->GetAspiredVersionsCallback()(kServableName, std::move(first_aspired_versions)); HandlePendingAspiredVersionsRequests(); @@ -914,7 +913,7 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { second_aspired_versions.push_back( {id, std::unique_ptr(second_loader)}); Notification second_load_called; - EXPECT_CALL(*second_loader, Load(_)).WillOnce(InvokeWithoutArgs([&]() { + EXPECT_CALL(*second_loader, Load()).WillOnce(InvokeWithoutArgs([&]() { second_load_called.Notify(); return Status::OK(); })); @@ -953,7 +952,7 @@ TEST_P(AspiredVersionsManagerTest, std::vector>> first_aspired_versions; test_util::MockLoader* first_loader = new NiceMock(); first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); - EXPECT_CALL(*first_loader, Load(_)) + EXPECT_CALL(*first_loader, Load()) .WillRepeatedly(Return(Status(error::UNKNOWN, "first load failing"))); manager_->GetAspiredVersionsCallback()(kServableName, std::move(first_aspired_versions)); @@ -977,7 +976,7 @@ TEST_P(AspiredVersionsManagerTest, second_aspired_versions.push_back( {id, std::unique_ptr(second_loader)}); Notification second_load_called; - EXPECT_CALL(*second_loader, Load(_)).WillOnce(InvokeWithoutArgs([&]() { + EXPECT_CALL(*second_loader, Load()).WillOnce(InvokeWithoutArgs([&]() { second_load_called.Notify(); return Status::OK(); })); @@ -1012,7 +1011,7 @@ TEST_P(AspiredVersionsManagerTest, UnaspireNewServableThenImmediatelyReaspire) { std::vector>> first_aspired_versions; test_util::MockLoader* first_loader = new NiceMock(); - EXPECT_CALL(*first_loader, Load(_)).Times(0); + EXPECT_CALL(*first_loader, Load()).Times(0); first_aspired_versions.push_back({id, std::unique_ptr(first_loader)}); manager_->GetAspiredVersionsCallback()(kServableName, std::move(first_aspired_versions)); @@ -1035,7 +1034,7 @@ TEST_P(AspiredVersionsManagerTest, UnaspireNewServableThenImmediatelyReaspire) { second_aspired_versions.push_back( {id, std::unique_ptr(second_loader)}); Notification second_load_called; - EXPECT_CALL(*second_loader, Load(_)).WillOnce(InvokeWithoutArgs([&]() { + EXPECT_CALL(*second_loader, Load()).WillOnce(InvokeWithoutArgs([&]() { second_load_called.Notify(); return Status::OK(); })); diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 9d1ed22ec00..d09b01ad892 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -461,7 +461,7 @@ Status BasicManager::ExecuteLoad(LoaderHarness* harness) { // deleted by another thread that called StopManagingServable(). We don't hold // the lock while calling Load() as the latter may block. const ServableId id = harness->id(); - const Status load_status = harness->Load(ResourceAllocation()); + const Status load_status = harness->Load(); if (!load_status.ok()) { PublishOnEventBus({id, ServableState::ManagerState::kEnd, load_status}); diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index fc78960c18d..8ce498f4e4c 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -274,7 +274,7 @@ TEST_P(BasicManagerTest, UpdateServingMapServableHandleLatest) { .WillByDefault(Return(AnyPtr(&servable))); ON_CALL(*notify_to_unload, EstimateResources(_)) .WillByDefault(Return(Status::OK())); - ON_CALL(*notify_to_unload, Load(_)).WillByDefault(Return(Status::OK())); + ON_CALL(*notify_to_unload, Load()).WillByDefault(Return(Status::OK())); const ServableId id1 = {kServableName3, 1}; basic_manager_->ManageServable( {id1, std::unique_ptr(notify_to_unload)}); @@ -673,12 +673,11 @@ TEST_P(BasicManagerTest, EventBusServableLifecycle) { Notification load_called; Notification load_continue; - EXPECT_CALL(*loader, Load(_)) - .WillOnce(InvokeWithoutArgs([&]() { - load_called.Notify(); - load_continue.WaitForNotification(); - return Status::OK(); - })); + EXPECT_CALL(*loader, Load()).WillOnce(InvokeWithoutArgs([&]() { + load_called.Notify(); + load_continue.WaitForNotification(); + return Status::OK(); + })); std::unique_ptr load_thread( Env::Default()->StartThread(ThreadOptions(), "LoadThread", [&]() { @@ -1024,7 +1023,7 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorFinallySucceeds) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); basic_manager_->ManageServable({id, std::unique_ptr(loader)}); - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(Return(errors::Internal("Load error."))) .WillRepeatedly(Return(Status::OK())); basic_manager_->LoadServable( @@ -1035,7 +1034,7 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorFinallyFails) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); basic_manager_->ManageServable({id, std::unique_ptr(loader)}); - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillRepeatedly(Return(errors::Internal("Load error."))); basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_EQ(errors::Internal("Load error."), status); @@ -1050,7 +1049,7 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorCancelledLoad) { Notification load_called; Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { load_called.Notify(); load_should_return.WaitForNotification(); @@ -1077,7 +1076,7 @@ TEST_P(BasicManagerTest, LoadAfterCancelledLoad) { Notification load_called; Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { load_called.Notify(); load_should_return.WaitForNotification(); @@ -1156,7 +1155,7 @@ class BarrierLoader : public Loader { return Status::OK(); } - Status Load(const ResourceAllocation& available_resources) override { + Status Load() override { counter_->DecrementCount(); counter_->Wait(); return Status::OK(); @@ -1199,7 +1198,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, InsufficientResources) { *estimate = CreateResourceQuantity(10 /* = total system resources */); return Status::OK(); })); - EXPECT_CALL(*hogging_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*hogging_loader, Load()).WillOnce(Return(Status::OK())); basic_manager_->ManageServable( CreateServableData(hogging_id, std::unique_ptr(hogging_loader))); Notification hogging_loaded; @@ -1251,7 +1250,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { *estimate = CreateResourceQuantity(10); return Status::OK(); })); - EXPECT_CALL(*failing_loader, Load(_)) + EXPECT_CALL(*failing_loader, Load()) .WillOnce(Return(errors::Unknown("Load failure"))); basic_manager_->ManageServable( CreateServableData(failing_id, std::unique_ptr(failing_loader))); @@ -1274,7 +1273,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { *estimate = CreateResourceQuantity(10); return Status::OK(); })); - EXPECT_CALL(*succeeding_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*succeeding_loader, Load()).WillOnce(Return(Status::OK())); basic_manager_->ManageServable(CreateServableData( succeeding_id, std::unique_ptr(succeeding_loader))); basic_manager_->LoadServable( @@ -1295,7 +1294,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, return Status::OK(); })) .RetiresOnSaturation(); - EXPECT_CALL(*overestimating_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*overestimating_loader, Load()).WillOnce(Return(Status::OK())); EXPECT_CALL(*overestimating_loader, EstimateResources(_)) .WillOnce(Invoke([](ResourceAllocation* estimate) { *estimate = CreateResourceQuantity(5 /* lower estimate after load */); @@ -1324,7 +1323,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, *estimate = CreateResourceQuantity(5); return Status::OK(); })); - EXPECT_CALL(*succeeding_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*succeeding_loader, Load()).WillOnce(Return(Status::OK())); basic_manager_->ManageServable(CreateServableData( succeeding_id, std::unique_ptr(succeeding_loader))); basic_manager_->LoadServable( @@ -1340,7 +1339,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedAfterUnload) { return Status::OK(); })); Notification load_done; - EXPECT_CALL(*unloading_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*unloading_loader, Load()).WillOnce(Return(Status::OK())); basic_manager_->ManageServable(CreateServableData( unloading_id, std::unique_ptr(unloading_loader))); basic_manager_->LoadServable(unloading_id, @@ -1376,7 +1375,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedAfterUnload) { *estimate = CreateResourceQuantity(10); return Status::OK(); })); - EXPECT_CALL(*succeeding_loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*succeeding_loader, Load()).WillOnce(Return(Status::OK())); basic_manager_->ManageServable(CreateServableData( succeeding_id, std::unique_ptr(succeeding_loader))); basic_manager_->LoadServable( @@ -1401,7 +1400,7 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { return Status::OK(); })); // Load won't be called because resources are not enough to load it. - EXPECT_CALL(*denied_loader, Load(_)).Times(0); + EXPECT_CALL(*denied_loader, Load()).Times(0); basic_manager_->ManageServable( CreateServableData(denied_id, std::unique_ptr(denied_loader))); @@ -1428,9 +1427,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { denied_estimate_started.WaitForNotification(); // The second servable's Load() call shouldn't occur until after the first // servable's load request exits its decision phase. - EXPECT_CALL(*succeeding_loader, Load(_)) - .WillOnce(Invoke([&finish_denied_estimate]( - const ResourceAllocation& available_resources) { + EXPECT_CALL(*succeeding_loader, Load()) + .WillOnce(Invoke([&finish_denied_estimate]() { // Ensure that the first servable's load request has been given // permission to exit its decision phase. EXPECT_TRUE(finish_denied_estimate.HasBeenNotified()); diff --git a/tensorflow_serving/core/loader.h b/tensorflow_serving/core/loader.h index 4de200af228..1b5b4347480 100644 --- a/tensorflow_serving/core/loader.h +++ b/tensorflow_serving/core/loader.h @@ -18,6 +18,7 @@ limitations under the License. #include +#include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/core/source.h" #include "tensorflow_serving/resources/resources.pb.h" @@ -76,11 +77,8 @@ class Loader { // Fetches any data that needs to be loaded before using the servable returned // by servable(). May use no more resources than the estimate reported by - // EstimateResources(). If that estimate included unbound resources (e.g. 2GB - // of GPU RAM, but not specifying which of two GPU devices to use), then the - // binding of resources to specific device instances must take into account - // the availability on each instance, given by 'available_resources'. - virtual Status Load(const ResourceAllocation& available_resources) = 0; + // EstimateResources(). + virtual Status Load() = 0; // Frees any resources allocated during Load() (except perhaps for resources // shared across servables that are still needed for other active ones). @@ -119,10 +117,9 @@ class Loader { virtual AnyPtr servable() = 0; }; -// A Loader that is oblivious to resources. Its Load() method does not take -// an 'available_resources' argument. Its EstimateResources() method returns 0, -// thus effectively disabling resource-based safety checks in the serving -// system. +// A Loader that is oblivious to resources. Its EstimateResources() method +// returns 0, thus effectively disabling resource-based safety checks in the +// serving system. // // Loaders that are experimental, or run in environments that do not need the // resource safety checks, can subclass ResourceUnsafeLoader instead of Loader. @@ -132,14 +129,6 @@ class ResourceUnsafeLoader : public Loader { estimate->Clear(); return Status::OK(); } - - Status Load(const ResourceAllocation& available_resources) final { - return Load(); - } - - private: - // Subclasses implement this overload, which lacks 'available_resources'. - virtual Status Load() = 0; }; // A source that emits Loader unique pointers. diff --git a/tensorflow_serving/core/loader_harness.cc b/tensorflow_serving/core/loader_harness.cc index bd2601d397c..9b0208bdedd 100644 --- a/tensorflow_serving/core/loader_harness.cc +++ b/tensorflow_serving/core/loader_harness.cc @@ -78,7 +78,7 @@ Status LoaderHarness::LoadApproved() { return Status::OK(); } -Status LoaderHarness::Load(const ResourceAllocation& available_resources) { +Status LoaderHarness::Load() { { mutex_lock l(mu_); if (state_ != State::kLoadApproved) { @@ -91,11 +91,10 @@ Status LoaderHarness::Load(const ResourceAllocation& available_resources) { LOG(INFO) << "Loading servable version " << id_; } - const Status status = - Retry(strings::StrCat("Loading servable: ", id_.DebugString()), - options_.max_num_load_retries, options_.load_retry_interval_micros, - [&]() { return loader_->Load(available_resources); }, - [&]() { return cancel_load_retry(); }); + const Status status = Retry( + strings::StrCat("Loading servable: ", id_.DebugString()), + options_.max_num_load_retries, options_.load_retry_interval_micros, + [&]() { return loader_->Load(); }, [&]() { return cancel_load_retry(); }); { mutex_lock l(mu_); diff --git a/tensorflow_serving/core/loader_harness.h b/tensorflow_serving/core/loader_harness.h index 07209d79213..69c393284bb 100644 --- a/tensorflow_serving/core/loader_harness.h +++ b/tensorflow_serving/core/loader_harness.h @@ -154,8 +154,7 @@ class LoaderHarness final { // // Legal to call iff current state is kApprovedForLoading. Returns an error // status if violated. - Status Load(const ResourceAllocation& available_resources) - LOCKS_EXCLUDED(mu_); + Status Load() LOCKS_EXCLUDED(mu_); // Transitions the state of the harness to kUnloadRequested. Returns ok if the // state was transitioned successfully, else returns an error status. diff --git a/tensorflow_serving/core/loader_harness_test.cc b/tensorflow_serving/core/loader_harness_test.cc index e02274d314f..9afdcba7961 100644 --- a/tensorflow_serving/core/loader_harness_test.cc +++ b/tensorflow_serving/core/loader_harness_test.cc @@ -73,12 +73,12 @@ TEST(LoaderHarnessTest, LoadRequested) { TEST(LoaderHarnessTest, Quiesce) { test_util::MockLoader* loader = new StrictMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); EXPECT_CALL(*loader, Unload()).WillOnce(Return()); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); TF_ASSERT_OK(harness.UnloadRequested()); TF_ASSERT_OK(harness.StartQuiescing()); @@ -96,32 +96,15 @@ TEST(LoaderHarnessTest, Load) { LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); EXPECT_CALL(*loader, Unload()).WillOnce(Return()); - const auto available_resources = test_util::CreateProto( - "resource_quantities { " - " resource { " - " device: 'main' " - " kind: 'processing' " - " } " - " quantity: 100 " - "} " - "resource_quantities { " - " resource { " - " device: 'gpu' " - " kind: 'ram' " - " } " - " quantity: 4 " - "} "); - Notification load_called; Notification load_should_return; - EXPECT_CALL(*loader, Load(EqualsProto(available_resources))) - .WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); { - std::unique_ptr test_thread(Env::Default()->StartThread( - ThreadOptions(), "test", [&available_resources, &harness]() { + std::unique_ptr test_thread( + Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - EXPECT_TRUE(harness.Load(available_resources).ok()); + EXPECT_TRUE(harness.Load().ok()); })); // Deleting the thread here forces join and ensures that // LoaderHarness::Load() returns. @@ -134,10 +117,10 @@ TEST(LoaderHarnessTest, Load) { TEST(LoaderHarnessTest, Unload) { test_util::MockLoader* loader = new StrictMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); Notification unload_called; Notification unload_should_return; @@ -154,10 +137,10 @@ TEST(LoaderHarnessTest, Unload) { TEST(LoaderHarnessTest, UnloadRequested) { test_util::MockLoader* loader = new NiceMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); TF_ASSERT_OK(harness.UnloadRequested()); EXPECT_EQ(LoaderHarness::State::kUnloadRequested, harness.state()); @@ -183,14 +166,14 @@ TEST(LoaderHarnessTest, LoadError) { Notification load_called; Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(Return(errors::Unknown("test load error"))); { std::unique_ptr test_thread( Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - Status status = harness.Load(ResourceAllocation()); + Status status = harness.Load(); EXPECT_THAT(status.error_message(), HasSubstr("test load error")); })); } @@ -228,14 +211,14 @@ TEST(LoaderHarnessTest, NonApprovedLoadFails) { test_util::MockLoader* loader = new NiceMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_FALSE(harness.Load(ResourceAllocation()).ok()); + EXPECT_FALSE(harness.Load().ok()); } TEST(LoaderHarnessTest, MultipleLoadApprovedOnlyFirstOneSucceeds) { test_util::MockLoader* loader = new NiceMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); const Status second_approve_for_loading_status = harness.LoadApproved(); @@ -245,7 +228,7 @@ TEST(LoaderHarnessTest, MultipleLoadApprovedOnlyFirstOneSucceeds) { EXPECT_THAT(second_approve_for_loading_status.error_message(), HasSubstr("cannot be approved for loading")); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); QuiesceAndUnload(&harness); } @@ -253,12 +236,12 @@ TEST(LoaderHarnessTest, MultipleLoadsOnlyFirstOneSucceeds) { test_util::MockLoader* loader = new NiceMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load(_)).WillRepeatedly(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillRepeatedly(Return(Status::OK())); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); - const Status second_load_status = harness.Load(ResourceAllocation()); + TF_ASSERT_OK(harness.Load()); + const Status second_load_status = harness.Load(); EXPECT_FALSE(second_load_status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, second_load_status.code()); EXPECT_THAT(second_load_status.error_message(), @@ -272,9 +255,9 @@ TEST(LoaderHarnessTest, MultipleUnloadRequestedOnlyFirstOneSucceeds) { LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); TF_ASSERT_OK(harness.LoadRequested()); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); TF_ASSERT_OK(harness.UnloadRequested()); const Status second_status = harness.UnloadRequested(); @@ -291,9 +274,9 @@ TEST(LoaderHarnessTest, MultipleStartQuiescingOnlyFirstOneSucceeds) { LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); TF_ASSERT_OK(harness.LoadRequested()); - EXPECT_CALL(*loader, Load(_)).WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load(ResourceAllocation())); + TF_ASSERT_OK(harness.Load()); TF_ASSERT_OK(harness.UnloadRequested()); TF_ASSERT_OK(harness.StartQuiescing()); @@ -313,13 +296,13 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorFinallySucceeds) { LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader), options); - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(InvokeWithoutArgs( []() { return errors::Unknown("test load error"); })) .WillOnce(InvokeWithoutArgs([]() { return Status::OK(); })); TF_EXPECT_OK(harness.LoadRequested()); TF_EXPECT_OK(harness.LoadApproved()); - TF_EXPECT_OK(harness.Load(ResourceAllocation())); + TF_EXPECT_OK(harness.Load()); QuiesceAndUnload(&harness); } @@ -332,13 +315,12 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorFinallyFails) { LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader), options); - EXPECT_CALL(*loader, Load(_)) - .Times(2) - .WillRepeatedly(InvokeWithoutArgs( - []() { return errors::Unknown("test load error"); })); + EXPECT_CALL(*loader, Load()).Times(2).WillRepeatedly(InvokeWithoutArgs([]() { + return errors::Unknown("test load error"); + })); TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - const Status status = harness.Load(ResourceAllocation()); + const Status status = harness.Load(); EXPECT_THAT(status.error_message(), HasSubstr("test load error")); } @@ -353,7 +335,7 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorCancelledLoad) { Notification load_called; Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { return errors::Unknown("test load error"); })) @@ -364,7 +346,7 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorCancelledLoad) { TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); harness.set_cancel_load_retry(true); - const Status status = harness.Load(ResourceAllocation()); + const Status status = harness.Load(); EXPECT_THAT(status.error_message(), HasSubstr("test load error")); })); } @@ -379,7 +361,7 @@ TEST(LoaderHarnessTest, LoadAfterCancelledLoad) { Notification load_called; Notification load_should_return; - EXPECT_CALL(*loader, Load(_)) + EXPECT_CALL(*loader, Load()) .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { load_called.Notify(); load_should_return.WaitForNotification(); @@ -391,7 +373,7 @@ TEST(LoaderHarnessTest, LoadAfterCancelledLoad) { Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { TF_ASSERT_OK(harness.LoadRequested()); TF_ASSERT_OK(harness.LoadApproved()); - const Status status = harness.Load(ResourceAllocation()); + const Status status = harness.Load(); EXPECT_THAT(status.error_message(), HasSubstr("test load error")); })); load_called.WaitForNotification(); @@ -399,7 +381,7 @@ TEST(LoaderHarnessTest, LoadAfterCancelledLoad) { load_should_return.Notify(); } - const Status second_load_status = harness.Load(ResourceAllocation()); + const Status second_load_status = harness.Load(); ASSERT_FALSE(second_load_status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, second_load_status.code()); } diff --git a/tensorflow_serving/core/simple_loader.h b/tensorflow_serving/core/simple_loader.h index a38a1db0cea..62452a36043 100644 --- a/tensorflow_serving/core/simple_loader.h +++ b/tensorflow_serving/core/simple_loader.h @@ -85,7 +85,7 @@ class SimpleLoader : public Loader { Status EstimateResources(ResourceAllocation* estimate) const override; - Status Load(const ResourceAllocation& available_resources) override; + Status Load() override; void Unload() override; @@ -197,8 +197,7 @@ Status SimpleLoader::EstimateResources( } template -Status SimpleLoader::Load( - const ResourceAllocation& available_resources) { +Status SimpleLoader::Load() { const Status status = creator_(&servable_); return status; } diff --git a/tensorflow_serving/core/simple_loader_test.cc b/tensorflow_serving/core/simple_loader_test.cc index b9fbbb7827e..186b98cf922 100644 --- a/tensorflow_serving/core/simple_loader_test.cc +++ b/tensorflow_serving/core/simple_loader_test.cc @@ -64,7 +64,7 @@ TEST(SimpleLoaderTest, VerifyServableStates) { }, SimpleLoader::EstimateNoResources())); EXPECT_EQ(State::kNone, state); - const Status status = loader->Load(ResourceAllocation()); + const Status status = loader->Load(); TF_EXPECT_OK(status); EXPECT_EQ(State::kCtor, state); AnyPtr servable = loader->servable(); @@ -111,7 +111,7 @@ TEST(SimpleLoaderTest, LoadError) { return errors::InvalidArgument("No way!"); }, SimpleLoader::EstimateNoResources())); - const Status status = loader->Load(ResourceAllocation()); + const Status status = loader->Load(); EXPECT_EQ(error::INVALID_ARGUMENT, status.code()); EXPECT_EQ("No way!", status.error_message()); } @@ -165,7 +165,7 @@ TEST(SimpleLoaderSourceAdapterTest, Basic) { " } " " quantity: 42 " "} "))); - TF_ASSERT_OK(loader->Load(ResourceAllocation())); + TF_ASSERT_OK(loader->Load()); AnyPtr servable = loader->servable(); ASSERT_TRUE(servable.get() != nullptr); EXPECT_EQ("test_data_was_here", *servable.get()); @@ -208,7 +208,7 @@ TEST(SimpleLoaderSourceAdapterTest, OkayToDeleteAdapter) { // callbacks, despite the fact that 'adapter' has been deleted. ResourceAllocation estimate_given; TF_ASSERT_OK(loader->EstimateResources(&estimate_given)); - TF_ASSERT_OK(loader->Load(ResourceAllocation())); + TF_ASSERT_OK(loader->Load()); } } // namespace diff --git a/tensorflow_serving/core/test_util/mock_loader.h b/tensorflow_serving/core/test_util/mock_loader.h index c82f5fe86f6..65befa9c510 100644 --- a/tensorflow_serving/core/test_util/mock_loader.h +++ b/tensorflow_serving/core/test_util/mock_loader.h @@ -28,7 +28,7 @@ namespace test_util { class MockLoader : public Loader { public: MOCK_CONST_METHOD1(EstimateResources, Status(ResourceAllocation* estimate)); - MOCK_METHOD1(Load, Status(const ResourceAllocation& available_resources)); + MOCK_METHOD0(Load, Status()); MOCK_METHOD0(Unload, void()); MOCK_METHOD0(servable, AnyPtr()); }; diff --git a/tensorflow_serving/resources/resource_tracker_test.cc b/tensorflow_serving/resources/resource_tracker_test.cc index d84f842aaf8..0e6c5336a66 100644 --- a/tensorflow_serving/resources/resource_tracker_test.cc +++ b/tensorflow_serving/resources/resource_tracker_test.cc @@ -163,7 +163,7 @@ class ResourceTrackerTest : public ::testing::Test { // Disallow calls to Load()/Unload(). for (auto* loader : {loader_0_.get(), loader_1_.get(), loader_2_.get(), loader_3_.get(), invalid_resources_loader_.get()}) { - EXPECT_CALL(*loader, Load(_)).Times(0); + EXPECT_CALL(*loader, Load()).Times(0); EXPECT_CALL(*loader, Unload()).Times(0); } } diff --git a/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc b/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc index 42397961022..997359ef432 100644 --- a/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc +++ b/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc @@ -82,7 +82,7 @@ TEST(HashmapSourceAdapter, Basic) { TF_ASSERT_OK(loader_data.status()); std::unique_ptr loader = loader_data.ConsumeDataOrDie(); - TF_ASSERT_OK(loader->Load(ResourceAllocation())); + TF_ASSERT_OK(loader->Load()); const Hashmap* hashmap = loader->servable().get(); EXPECT_THAT(*hashmap, diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc index 81072b7fcc7..a6a0d44e3ec 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc @@ -67,7 +67,7 @@ class SavedModelBundleSourceAdapterTest : public ::testing::Test { TF_ASSERT_OK(loader->EstimateResources(&second_resource_estimate)); EXPECT_THAT(second_resource_estimate, EqualsProto(first_resource_estimate)); - TF_ASSERT_OK(loader->Load(ResourceAllocation())); + TF_ASSERT_OK(loader->Load()); const SavedModelBundle* bundle = loader->servable().get(); test_util::TestSingleRequest(bundle->session.get()); diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc index 631877da9c9..5b1336a0be7 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc @@ -72,7 +72,7 @@ class SessionBundleSourceAdapterTest : public ::testing::Test { TF_ASSERT_OK(loader->EstimateResources(&second_resource_estimate)); EXPECT_THAT(second_resource_estimate, EqualsProto(first_resource_estimate)); - TF_ASSERT_OK(loader->Load(ResourceAllocation())); + TF_ASSERT_OK(loader->Load()); const SessionBundle* bundle = loader->servable().get(); test_util::TestSingleRequest(bundle->session.get()); From 48dfb5581a621c7a6e038df3415e0eff02edb043 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 11 Jan 2017 11:50:06 -0800 Subject: [PATCH 0130/8554] Consolidate BasicManager's servable state transition error handling logic into a single routine inside LoaderHarness. Change: 144232244 --- tensorflow_serving/core/basic_manager.cc | 70 ++++---- tensorflow_serving/core/basic_manager.h | 6 +- tensorflow_serving/core/basic_manager_test.cc | 13 +- tensorflow_serving/core/loader_harness.cc | 75 ++++----- tensorflow_serving/core/loader_harness.h | 55 ++++--- .../core/loader_harness_test.cc | 155 ++++++------------ tensorflow_serving/core/servable_state.h | 4 +- 7 files changed, 158 insertions(+), 220 deletions(-) diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index d09b01ad892..c344e950d2a 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -202,26 +202,29 @@ void BasicManager::ServingMap::Update(const ManagedMap& managed_map) { Status BasicManager::Create(Options options, std::unique_ptr* manager) { - LoaderHarness::Options harness_options; - harness_options.max_num_load_retries = options.max_num_load_retries; - harness_options.load_retry_interval_micros = - options.load_retry_interval_micros; - manager->reset(new BasicManager(options.env, options.num_load_threads, - options.num_unload_threads, - std::move(options.resource_tracker), - options.servable_event_bus, harness_options)); + manager->reset(new BasicManager( + options.env, options.num_load_threads, options.num_unload_threads, + options.max_num_load_retries, options.load_retry_interval_micros, + std::move(options.resource_tracker), options.servable_event_bus)); return Status::OK(); } BasicManager::BasicManager(Env* const env, const uint32 num_load_threads, const uint32 num_unload_threads, + uint32 max_num_load_retries, + int64 load_retry_interval_micros, std::unique_ptr resource_tracker, - EventBus* servable_event_bus, - const LoaderHarness::Options& harness_options) - : harness_options_(harness_options), - servable_event_bus_(servable_event_bus), + EventBus* servable_event_bus) + : servable_event_bus_(servable_event_bus), env_(env), num_load_threads_(num_load_threads) { + harness_options_.max_num_load_retries = max_num_load_retries; + harness_options_.load_retry_interval_micros = load_retry_interval_micros; + harness_options_.error_callback = [this](const ServableId& id, + const Status& error) { + PublishOnEventBus({id, ServableState::ManagerState::kEnd, error}); + }; + { mutex_lock l(num_load_threads_mu_); load_executor_ = @@ -334,8 +337,6 @@ Status BasicManager::ManageServableInternal( harness_creator(servable.id(), std::move(loader)); if (!servable.status().ok()) { harness->Error(servable.status()); - PublishOnEventBus( - {harness->id(), ServableState::ManagerState::kEnd, harness->status()}); } else { PublishOnEventBus({harness->id(), ServableState::ManagerState::kStart, harness->status()}); @@ -456,24 +457,21 @@ std::vector BasicManager::GetManagedServableNames() const { Status BasicManager::ExecuteLoad(LoaderHarness* harness) { PublishOnEventBus({harness->id(), ServableState::ManagerState::kLoading, harness->status()}); - // We save the id and the status of the harness so that we can publish them - // after Load(). We can't query harness again after Load() as it may be - // deleted by another thread that called StopManagingServable(). We don't hold - // the lock while calling Load() as the latter may block. + // We save the id of the harness so that we can publish it after Load(). (We + // can't query harness again after Load() as it may be deleted by another + // thread that called StopManagingServable().) const ServableId id = harness->id(); - const Status load_status = harness->Load(); - if (!load_status.ok()) { - PublishOnEventBus({id, ServableState::ManagerState::kEnd, load_status}); - return load_status; - } + // We don't hold the lock while calling Load() as it may block. + TF_RETURN_IF_ERROR(harness->Load()); { mutex_lock l(mu_); UpdateServingMap(); } - PublishOnEventBus({id, ServableState::ManagerState::kAvailable, load_status}); + PublishOnEventBus( + {id, ServableState::ManagerState::kAvailable, Status::OK()}); return Status::OK(); } @@ -497,24 +495,23 @@ void BasicManager::CancelLoadServableRetry(const ServableId& id) { } Status BasicManager::ExecuteUnload(LoaderHarness* harness) { - // We save the id and the status of the harness so that we can publish them - // after Unload(). Unload() always succeeds, and hence doesn't affect - // harness->status(). We can't query harness again after Unload() as it may be - // deleted by another thread that called StopManagingServable(). We don't hold - // the lock while calling Unload() as the latter may block. + // We save the id of the harness so that we can publish it after Unload(). (We + // can't query harness again after Unload() as it may be deleted by another + // thread that called StopManagingServable().) const ServableId id = harness->id(); - const Status pre_unload_status = [&] { + + { // StartQuiescing() would have been already called. mutex_lock l(mu_); PublishOnEventBus( {id, ServableState::ManagerState::kUnloading, harness->status()}); UpdateServingMap(); - harness->DoneQuiescing(); - return harness->status(); - }(); + TF_RETURN_IF_ERROR(harness->DoneQuiescing()); + } - harness->Unload(); - PublishOnEventBus({id, ServableState::ManagerState::kEnd, pre_unload_status}); + // We don't hold the lock while calling Unload() as it may block. + TF_RETURN_IF_ERROR(harness->Unload()); + PublishOnEventBus({id, ServableState::ManagerState::kEnd, Status::OK()}); return Status::OK(); } @@ -673,9 +670,6 @@ Status BasicManager::ApproveLoad(LoaderHarness* harness, mutex_lock* mu_lock) { Status BasicManager::ApproveUnload(LoaderHarness* harness) { // Transition to state kQuiescing inside the decision phase, to prevent any // concurrent unload requests from executing. - // - // StartQuiescing() returns an error status if the harness is not in a state - // to be quiesced. TF_RETURN_IF_ERROR(harness->StartQuiescing()); return Status::OK(); diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index ef6aa40d956..296355bd2e4 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -253,9 +253,9 @@ class BasicManager : public Manager { friend class test_util::BasicManagerTestAccess; BasicManager(Env* env, uint32 num_load_threads, uint32 num_unload_threads, + uint32 max_num_load_retries, int64 load_retry_interval_micros, std::unique_ptr resource_tracker, - EventBus* servable_event_bus, - const LoaderHarness::Options& harness_options); + EventBus* servable_event_bus); // Starts managing the servable. // @@ -394,7 +394,7 @@ class BasicManager : public Manager { // options, if not we ignore it. void PublishOnEventBus(const ServableState& state); - const LoaderHarness::Options harness_options_; + LoaderHarness::Options harness_options_; // The event bus to which to publish servable state change events, or nullptr // if no bus has been configured. diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 8ce498f4e4c..a14b7e2f5db 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -586,8 +586,7 @@ TEST_P(BasicManagerTest, MultipleLoadServables) { basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); - EXPECT_THAT(status.error_message(), - HasSubstr("cannot be transitioned to load-requested")); + EXPECT_THAT(status.error_message(), HasSubstr("Duplicate load request")); }); } @@ -605,7 +604,8 @@ TEST_P(BasicManagerTest, MultipleUnloadServables) { basic_manager_->UnloadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); - EXPECT_THAT(status.error_message(), HasSubstr("cannot be transitioned")); + EXPECT_THAT(status.error_message(), + HasSubstr("unload already requested/ongoing")); }); } @@ -624,8 +624,7 @@ TEST_P(BasicManagerTest, UnloadWithoutLoad) { basic_manager_->UnloadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); - EXPECT_THAT(status.error_message(), - HasSubstr("cannot be transitioned to unload-requested")); + EXPECT_THAT(status.error_message(), HasSubstr("Servable not loaded")); }); } @@ -961,7 +960,7 @@ TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { if (!statuses[i].ok()) { EXPECT_EQ(error::FAILED_PRECONDITION, statuses[i].code()); EXPECT_THAT(statuses[i].error_message(), - HasSubstr("cannot be transitioned to load-requested")); + HasSubstr("Duplicate load request")); } else { ++num_status_ok; } @@ -1010,7 +1009,7 @@ TEST_P(BasicManagerTest, ConcurrentUnloadsOnlyOneSucceeds) { HasSubstr("not being managed")); } else { EXPECT_THAT(statuses[i].error_message(), - HasSubstr("cannot be transitioned to unload-requested")); + HasSubstr("unload already requested/ongoing")); } } else { ++num_status_ok; diff --git a/tensorflow_serving/core/loader_harness.cc b/tensorflow_serving/core/loader_harness.cc index 9b0208bdedd..56136f10559 100644 --- a/tensorflow_serving/core/loader_harness.cc +++ b/tensorflow_serving/core/loader_harness.cc @@ -51,11 +51,7 @@ Status LoaderHarness::LoadRequested() { mutex_lock l(mu_); if (state_ != State::kNew) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), - " cannot be transitioned to load-requested. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kNew)); + return errors::FailedPrecondition("Duplicate load request"); } state_ = State::kLoadRequested; VLOG(1) << "Load requested for servable version " << id_; @@ -65,29 +61,16 @@ Status LoaderHarness::LoadRequested() { Status LoaderHarness::LoadApproved() { mutex_lock l(mu_); - - if (state_ != State::kLoadRequested) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), - " cannot be approved for loading. In state: ", StateDebugString(state_), - " instead of state: ", StateDebugString(State::kLoadRequested)); - } - state_ = State::kLoadApproved; + TF_RETURN_IF_ERROR( + TransitionState(State::kLoadRequested, State::kLoadApproved)); LOG(INFO) << "Approving load for servable version " << id_; - return Status::OK(); } Status LoaderHarness::Load() { { mutex_lock l(mu_); - if (state_ != State::kLoadApproved) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), " cannot be loaded. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kLoadApproved)); - } - state_ = State::kLoading; + TF_RETURN_IF_ERROR(TransitionState(State::kLoadApproved, State::kLoading)); LOG(INFO) << "Loading servable version " << id_; } @@ -98,9 +81,8 @@ Status LoaderHarness::Load() { { mutex_lock l(mu_); - DCHECK_EQ(State::kLoading, state_); if (status.ok()) { - state_ = State::kReady; + TF_RETURN_IF_ERROR(TransitionState(State::kLoading, State::kReady)); LOG(INFO) << "Successfully loaded servable version " << id_; } else { ErrorInternal(status); @@ -112,13 +94,9 @@ Status LoaderHarness::Load() { Status LoaderHarness::UnloadRequested() { mutex_lock l(mu_); - if (state_ != State::kReady) { return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), - " cannot be transitioned to unload-requested. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kReady)); + "Servable not loaded, or unload already requested/ongoing"); } state_ = State::kUnloadRequested; return Status::OK(); @@ -134,11 +112,10 @@ bool LoaderHarness::cancel_load_retry() { return cancel_load_retry_; } -void LoaderHarness::Unload() { +Status LoaderHarness::Unload() { { mutex_lock l(mu_); - DCHECK_EQ(state_, State::kQuiesced); - state_ = State::kUnloading; + TF_RETURN_IF_ERROR(TransitionState(State::kQuiesced, State::kUnloading)); LOG(INFO) << "Unloading servable version " << id_; } @@ -146,35 +123,34 @@ void LoaderHarness::Unload() { { mutex_lock l(mu_); - DCHECK_EQ(state_, State::kUnloading); - state_ = State::kDisabled; + TF_RETURN_IF_ERROR(TransitionState(State::kUnloading, State::kDisabled)); LOG(INFO) << "Done unloading servable version " << id_; } + + return Status::OK(); } Status LoaderHarness::StartQuiescing() { mutex_lock l(mu_); - if (state_ != State::kUnloadRequested) { - return errors::FailedPrecondition( - "Servable: ", id_.DebugString(), " cannot be quiesced. In state: ", - StateDebugString(state_), " instead of state: ", - StateDebugString(State::kUnloadRequested)); - } - state_ = State::kQuiescing; + TF_RETURN_IF_ERROR( + TransitionState(State::kUnloadRequested, State::kQuiescing)); LOG(INFO) << "Quiescing servable version " << id_; return Status::OK(); } -void LoaderHarness::DoneQuiescing() { +Status LoaderHarness::DoneQuiescing() { mutex_lock l(mu_); - DCHECK_EQ(state_, State::kQuiescing); - state_ = State::kQuiesced; + TF_RETURN_IF_ERROR(TransitionState(State::kQuiescing, State::kQuiesced)); LOG(INFO) << "Done quiescing servable version " << id_; + return Status::OK(); } void LoaderHarness::ErrorInternal(const Status status) { state_ = State::kError; status_ = status; + if (options_.error_callback) { + options_.error_callback(id(), status); + } LOG(INFO) << "Encountered an error for servable version " << id_ << ": " << status_; } @@ -184,6 +160,19 @@ void LoaderHarness::Error(const Status status) { ErrorInternal(status); } +Status LoaderHarness::TransitionState(State from, State to) { + if (state_ != from) { + const Status error = errors::Internal( + "Illegal request to transition from state ", StateDebugString(state_), + " to ", StateDebugString(to)); + DCHECK(false) << error; + ErrorInternal(error); + return error; + } + state_ = to; + return Status::OK(); +} + Status LoaderHarness::status() const { mutex_lock l(mu_); return status_; diff --git a/tensorflow_serving/core/loader_harness.h b/tensorflow_serving/core/loader_harness.h index 69c393284bb..5febfca1bba 100644 --- a/tensorflow_serving/core/loader_harness.h +++ b/tensorflow_serving/core/loader_harness.h @@ -98,6 +98,10 @@ class LoaderHarness final { // The interval, in microseconds, between each servable load retry. uint64 load_retry_interval_micros = 0; + + // An (optional) function to call upon transitioning to state kError. + std::function + error_callback; }; LoaderHarness(const ServableId& id, std::unique_ptr loader, @@ -132,34 +136,32 @@ class LoaderHarness final { template ServableStateSnapshot loader_state_snapshot() const LOCKS_EXCLUDED(mu_); - // Transitions the state of the harness to kLoadRequested. Returns ok if the - // state was transitioned successfully, else returns an error status. - // - // REQUIRES: That the harness is in state kNew when this method is called. + // Transitions the state of the harness to kLoadRequested iff its current + // state is kNew. The test-and-change is done transactionally, so this method + // can be used to ensure that at most one Load() request can proceed. Status LoadRequested() LOCKS_EXCLUDED(mu_); // Transitions to kLoadApproved. // - // Legal to call iff current state is kLoadRequested. Returns an error status - // if violated. + // REQUIRES: State is kLoadRequested when called. Otherwise DCHECK-fails, + // transitions to state kError and invokes 'options_.error_callback'. Status LoadApproved() LOCKS_EXCLUDED(mu_); // Transitions to kLoading, delegates to Servable::Load(), then transitions - // either to kReady if Load() succeeds, or to kError if Load() fails. This - // call may take a long time. + // either to kReady if Load() succeeds, or to kError (and invokes 'options_. + // error_callback') if Load() fails. This call may take a long time. // // We retry the Servable::Load() according to the options set during // construction of this class. We stop retrying and give up if 1. we have // reached max_num_load_retries or, 2. if cancel_load() is set to true. // - // Legal to call iff current state is kApprovedForLoading. Returns an error - // status if violated. + // REQUIRES: State is kLoadApproved when called. Otherwise DCHECK-fails, + // transitions to state kError and invokes 'options_.error_callback'. Status Load() LOCKS_EXCLUDED(mu_); - // Transitions the state of the harness to kUnloadRequested. Returns ok if the - // state was transitioned successfully, else returns an error status. - // - // REQUIRES: That the harness is in state kReady when this method is called. + // Transitions the state of the harness to kUnloadRequested iff its current + // state is kReady. The test-and-change is done transactionally, so this + // method can be used to ensure that at most one Load() request can proceed. Status UnloadRequested() LOCKS_EXCLUDED(mu_); // Cancels retrying the load of the servable. This is best-effort, and does @@ -173,20 +175,27 @@ class LoaderHarness final { // Transitions to kUnloading, delegates to Servable::Unload(), then // transitions to kDisabled when Unload() is done. - void Unload() LOCKS_EXCLUDED(mu_); + // + // REQUIRES: State is kQuiesced when called. Otherwise DCHECK-fails, + // transitions to state kError and invokes 'options_.error_callback'. + Status Unload() LOCKS_EXCLUDED(mu_); // Transitions the state to kQuiescing, which means that we would like to not // give out any more handles to this servable. // - // REQUIRES: State be kReady when called, else returns an error status. + // REQUIRES: State is kUnloadRequested when called. Otherwise DCHECK-fails, + // transitions to state kError and invokes 'options_.error_callback'. Status StartQuiescing() LOCKS_EXCLUDED(mu_); // Transitions the state to kQuiesced, which means that there are no more live // handles to this servable available in the frontend. At this point, we can // unload this object. - void DoneQuiescing() LOCKS_EXCLUDED(mu_); + // + // REQUIRES: State is kQuiescing when called. Otherwise DCHECK-fails, + // transitions to state kError and invokes 'options_.error_callback'. + Status DoneQuiescing() LOCKS_EXCLUDED(mu_); - // Transitions the state to kError. + // Transitions the state to kError and invokes 'options_.error_callback'. void Error(Status status) LOCKS_EXCLUDED(mu_); // Whether anything has gone wrong with this servable. If state is kError, @@ -206,10 +215,16 @@ class LoaderHarness final { static string StateDebugString(State state); private: - // Transitions the state to kError. Private method to be used when we want to - // set an error from another method in this class, where mu_ is already held. + // Transitions the state to kError and invokes 'options_.error_callback'. + // Private method to be used when we want to set an error from another method + // in this class, where mu_ is already held. void ErrorInternal(Status status) EXCLUSIVE_LOCKS_REQUIRED(mu_); + // Expects 'state_' to equal 'from', and if so transitions it to 'to'. If not, + // DCHECK-fails, calls ErrorInternal() with a suitable error and returns the + // same error. + Status TransitionState(State from, State to) EXCLUSIVE_LOCKS_REQUIRED(mu_); + const ServableId id_; const std::unique_ptr loader_; // Additional state that the manager uses. diff --git a/tensorflow_serving/core/loader_harness_test.cc b/tensorflow_serving/core/loader_harness_test.cc index 9afdcba7961..7e4508b5be7 100644 --- a/tensorflow_serving/core/loader_harness_test.cc +++ b/tensorflow_serving/core/loader_harness_test.cc @@ -43,11 +43,17 @@ using ::testing::ReturnRef; using ::testing::StrictMock; using test_util::EqualsProto; +// Walks 'harness' through a sequence of transitions from kReady to kDisabled. void QuiesceAndUnload(LoaderHarness* const harness) { - harness->UnloadRequested(); - harness->StartQuiescing(); - harness->DoneQuiescing(); - harness->Unload(); + TF_ASSERT_OK(harness->UnloadRequested()); + TF_ASSERT_OK(harness->StartQuiescing()); + TF_ASSERT_OK(harness->DoneQuiescing()); + TF_ASSERT_OK(harness->Unload()); +} + +// Makes it s.t. it's legal to destruct 'harness'. +void EnableDestruction(LoaderHarness* const harness) { + harness->Error(errors::Unknown("Erroring servable on purpose for shutdown")); } TEST(LoaderHarnessTest, Init) { @@ -84,17 +90,16 @@ TEST(LoaderHarnessTest, Quiesce) { TF_ASSERT_OK(harness.StartQuiescing()); EXPECT_EQ(LoaderHarness::State::kQuiescing, harness.state()); - harness.DoneQuiescing(); + TF_ASSERT_OK(harness.DoneQuiescing()); EXPECT_EQ(LoaderHarness::State::kQuiesced, harness.state()); // Otherwise we break the dtor invariant and check-fail. - harness.Unload(); + TF_ASSERT_OK(harness.Unload()); } TEST(LoaderHarnessTest, Load) { test_util::MockLoader* loader = new StrictMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Unload()).WillOnce(Return()); Notification load_called; Notification load_should_return; @@ -111,7 +116,7 @@ TEST(LoaderHarnessTest, Load) { } EXPECT_EQ(LoaderHarness::State::kReady, harness.state()); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, Unload) { @@ -145,7 +150,7 @@ TEST(LoaderHarnessTest, UnloadRequested) { TF_ASSERT_OK(harness.UnloadRequested()); EXPECT_EQ(LoaderHarness::State::kUnloadRequested, harness.state()); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, LoadApproved) { @@ -181,7 +186,7 @@ TEST(LoaderHarnessTest, LoadError) { } TEST(LoaderHarnessTest, ExternallySignalledError) { - LoaderHarness harness(ServableId{"test", 0}, nullptr); + LoaderHarness harness(ServableId{"test", 0}, nullptr /* no loader */); EXPECT_EQ(LoaderHarness::State::kNew, harness.state()); const Status status = Status(error::UNKNOWN, "Some unknown error"); harness.Error(status); @@ -189,6 +194,22 @@ TEST(LoaderHarnessTest, ExternallySignalledError) { EXPECT_EQ(status, harness.status()); } +TEST(LoaderHarnessTest, ExternallySignalledErrorWithCallback) { + const ServableId id = {"test_servable", 42}; + const Status error = Status(error::UNKNOWN, "Some unknown error"); + Notification callback_called; + LoaderHarness::Options options; + options.error_callback = [&](const ServableId& callback_id, + const Status& callback_error) { + EXPECT_EQ(id, callback_id); + EXPECT_EQ(callback_error, error); + callback_called.Notify(); + }; + LoaderHarness harness(id, nullptr /* no loader */, options); + harness.Error(error); + callback_called.WaitForNotification(); +} + TEST(LoaderHarnessTest, AdditionalState) { std::unique_ptr object(new int(10)); LoaderHarness harness({"test", 42}, nullptr, std::move(object)); @@ -207,50 +228,21 @@ TEST(LoaderHarnessTest, NoAdditionalState) { EXPECT_EQ(nullptr, harness.additional_state()); } -TEST(LoaderHarnessTest, NonApprovedLoadFails) { - test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - - EXPECT_FALSE(harness.Load().ok()); -} - -TEST(LoaderHarnessTest, MultipleLoadApprovedOnlyFirstOneSucceeds) { - test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - - EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); - TF_ASSERT_OK(harness.LoadRequested()); - TF_ASSERT_OK(harness.LoadApproved()); - const Status second_approve_for_loading_status = harness.LoadApproved(); - EXPECT_FALSE(second_approve_for_loading_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, - second_approve_for_loading_status.code()); - EXPECT_THAT(second_approve_for_loading_status.error_message(), - HasSubstr("cannot be approved for loading")); - - TF_ASSERT_OK(harness.Load()); - QuiesceAndUnload(&harness); -} - -TEST(LoaderHarnessTest, MultipleLoadsOnlyFirstOneSucceeds) { +TEST(LoaderHarnessTest, MultipleLoadRequestsOnlyFirstOneSucceeds) { test_util::MockLoader* loader = new NiceMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - EXPECT_CALL(*loader, Load()).WillRepeatedly(Return(Status::OK())); - TF_ASSERT_OK(harness.LoadRequested()); - TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load()); - const Status second_load_status = harness.Load(); - EXPECT_FALSE(second_load_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, second_load_status.code()); - EXPECT_THAT(second_load_status.error_message(), - HasSubstr("cannot be loaded")); + const Status second_request_status = harness.LoadRequested(); + EXPECT_FALSE(second_request_status.ok()); + EXPECT_EQ(error::FAILED_PRECONDITION, second_request_status.code()); + EXPECT_THAT(second_request_status.error_message(), + HasSubstr("Duplicate load request")); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } -TEST(LoaderHarnessTest, MultipleUnloadRequestedOnlyFirstOneSucceeds) { +TEST(LoaderHarnessTest, MultipleUnloadRequestsOnlyFirstOneSucceeds) { test_util::MockLoader* loader = new NiceMock; LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); @@ -263,29 +255,11 @@ TEST(LoaderHarnessTest, MultipleUnloadRequestedOnlyFirstOneSucceeds) { const Status second_status = harness.UnloadRequested(); EXPECT_FALSE(second_status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, second_status.code()); - EXPECT_THAT(second_status.error_message(), - HasSubstr("cannot be transitioned to unload-requested")); + EXPECT_THAT( + second_status.error_message(), + HasSubstr("Servable not loaded, or unload already requested/ongoing")); - QuiesceAndUnload(&harness); -} - -TEST(LoaderHarnessTest, MultipleStartQuiescingOnlyFirstOneSucceeds) { - test_util::MockLoader* loader = new NiceMock; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader)); - - TF_ASSERT_OK(harness.LoadRequested()); - EXPECT_CALL(*loader, Load()).WillOnce(Return(Status::OK())); - TF_ASSERT_OK(harness.LoadApproved()); - TF_ASSERT_OK(harness.Load()); - - TF_ASSERT_OK(harness.UnloadRequested()); - TF_ASSERT_OK(harness.StartQuiescing()); - const Status second_status = harness.StartQuiescing(); - EXPECT_FALSE(second_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, second_status.code()); - EXPECT_THAT(second_status.error_message(), HasSubstr("cannot be quiesced")); - - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, RetryOnLoadErrorFinallySucceeds) { @@ -300,11 +274,11 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorFinallySucceeds) { .WillOnce(InvokeWithoutArgs( []() { return errors::Unknown("test load error"); })) .WillOnce(InvokeWithoutArgs([]() { return Status::OK(); })); - TF_EXPECT_OK(harness.LoadRequested()); - TF_EXPECT_OK(harness.LoadApproved()); - TF_EXPECT_OK(harness.Load()); + TF_ASSERT_OK(harness.LoadRequested()); + TF_ASSERT_OK(harness.LoadApproved()); + TF_ASSERT_OK(harness.Load()); - QuiesceAndUnload(&harness); + EnableDestruction(&harness); } TEST(LoaderHarnessTest, RetryOnLoadErrorFinallyFails) { @@ -351,41 +325,6 @@ TEST(LoaderHarnessTest, RetryOnLoadErrorCancelledLoad) { })); } -TEST(LoaderHarnessTest, LoadAfterCancelledLoad) { - test_util::MockLoader* loader = new NiceMock; - LoaderHarness::Options options; - options.max_num_load_retries = 10; - options.load_retry_interval_micros = 0; - LoaderHarness harness(ServableId{"test", 0}, std::unique_ptr(loader), - options); - - Notification load_called; - Notification load_should_return; - EXPECT_CALL(*loader, Load()) - .WillOnce(InvokeWithoutArgs([&load_called, &load_should_return]() { - load_called.Notify(); - load_should_return.WaitForNotification(); - return errors::Unknown("test load error"); - })) - .WillRepeatedly(InvokeWithoutArgs([]() { return Status::OK(); })); - { - std::unique_ptr test_thread( - Env::Default()->StartThread(ThreadOptions(), "test", [&harness]() { - TF_ASSERT_OK(harness.LoadRequested()); - TF_ASSERT_OK(harness.LoadApproved()); - const Status status = harness.Load(); - EXPECT_THAT(status.error_message(), HasSubstr("test load error")); - })); - load_called.WaitForNotification(); - harness.set_cancel_load_retry(true); - load_should_return.Notify(); - } - - const Status second_load_status = harness.Load(); - ASSERT_FALSE(second_load_status.ok()); - EXPECT_EQ(error::FAILED_PRECONDITION, second_load_status.code()); -} - } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/servable_state.h b/tensorflow_serving/core/servable_state.h index 3fef423b19d..bfe06135194 100644 --- a/tensorflow_serving/core/servable_state.h +++ b/tensorflow_serving/core/servable_state.h @@ -57,7 +57,9 @@ struct ServableState { // unavailable. kUnloading, - // The manager is done tracking this servable, and is about to delete it. + // This servable has reached the end of its journey in the manager. Either + // it loaded and ultimately unloaded successfully, or it hit an error at + // some point in its lifecycle. kEnd, }; ManagerState manager_state; From 0799674618642ccde55b0465d181bdbc5c12e333 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 11 Jan 2017 14:48:05 -0800 Subject: [PATCH 0131/8554] Delete unused BasicManager::DeleteHarness() method. Change: 144252763 --- tensorflow_serving/core/basic_manager.cc | 11 ----------- tensorflow_serving/core/basic_manager.h | 6 ------ 2 files changed, 17 deletions(-) diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index c344e950d2a..320a64ef2b2 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -301,17 +301,6 @@ BasicManager::ManagedMap::iterator BasicManager::FindHarnessInMap( return managed_map_.end(); } -void BasicManager::DeleteHarness(const ServableId& id) { - const auto it = FindHarnessInMap(id); - DCHECK(it != managed_map_.end()); - if (it == managed_map_.end()) { - LOG(ERROR) << "Request to delete harness for " << id - << ", but no such harness found in managed_map_"; - return; - } - managed_map_.erase(it); -} - Status BasicManager::ManageServableInternal( ServableData> servable, std::function(const ServableId&, diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index 296355bd2e4..2ae0ade27c1 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -384,12 +384,6 @@ class BasicManager : public Manager { ManagedMap::iterator FindHarnessInMap(const ServableId& id) EXCLUSIVE_LOCKS_REQUIRED(mu_); - // Removes the harness associated with 'id' from 'managed_map_' and deletes - // the harness. - // - // If no matching harness is found, DCHECK-fails and logs an error. - void DeleteHarness(const ServableId& id) EXCLUSIVE_LOCKS_REQUIRED(mu_); - // Publishes the state on the event bus, if an event bus was part of the // options, if not we ignore it. void PublishOnEventBus(const ServableState& state); From 9d01226cd80b79ed4b6df450bec14d25d4c867e3 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 11 Jan 2017 16:50:04 -0800 Subject: [PATCH 0132/8554] Fix open source build Change: 144267373 --- tensorflow_serving/core/BUILD | 1 + tensorflow_serving/core/basic_manager_test.cc | 1 + tensorflow_serving/core/test_util/BUILD | 3 +++ .../core/test_util/fake_loader_source_adapter.cc | 7 +++++++ .../core/test_util/fake_loader_source_adapter.h | 4 ++++ tensorflow_serving/model_servers/platform_config_util.cc | 4 ++-- tensorflow_serving/model_servers/platform_config_util.h | 4 ++-- tensorflow_serving/model_servers/server_core.h | 6 ++++-- tensorflow_serving/model_servers/server_core_test.cc | 4 ++-- .../model_servers/test_util/mock_server_core.h | 6 +++--- .../model_servers/test_util/server_core_test_util.cc | 6 +++--- .../storage_path_error_injecting_source_adapter.cc | 2 +- tensorflow_serving/util/class_registration.h | 1 - 13 files changed, 33 insertions(+), 16 deletions(-) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 81fd4478417..19445a09c7c 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -745,6 +745,7 @@ cc_library( srcs = ["server_request_logger.cc"], hdrs = ["server_request_logger.h"], deps = [ + ":request_logger", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:logging_config_proto", "//tensorflow_serving/core:logging_proto", diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index a14b7e2f5db..15eb23bd09d 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -1502,6 +1502,7 @@ TEST(EstimateResourcesRetriedTest, Succeeds) { EXPECT_CALL(*loader, EstimateResources(_)) .WillOnce(Return(errors::Internal("Error on estimate resources."))) .WillOnce(Return(Status::OK())); + EXPECT_CALL(*loader, Load()).WillRepeatedly(Return(Status::OK())); basic_manager->ManageServable( CreateServableData(id, std::unique_ptr(loader))); basic_manager->LoadServable( diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 606ab9a8f9f..34b845ea240 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -66,8 +66,11 @@ cc_library( visibility = ["//visibility:public"], deps = [ ":fake_loader_source_adapter_proto", + "//tensorflow_serving/core:loader", "//tensorflow_serving/core:simple_loader", + "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", + "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc index ef2d31e586b..9abf2c021de 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.cc @@ -15,6 +15,13 @@ limitations under the License. #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" +#include + +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/core/loader.h" +#include "tensorflow_serving/core/source_adapter.h" + namespace tensorflow { namespace serving { namespace test_util { diff --git a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h index 8f7bcfc08be..c024d9f1c45 100644 --- a/tensorflow_serving/core/test_util/fake_loader_source_adapter.h +++ b/tensorflow_serving/core/test_util/fake_loader_source_adapter.h @@ -16,7 +16,11 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_SOURCE_ADAPTER_H_ #define TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOADER_SOURCE_ADAPTER_H_ +#include + #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/macros.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/simple_loader.h" #include "tensorflow_serving/core/storage_path.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.pb.h" diff --git a/tensorflow_serving/model_servers/platform_config_util.cc b/tensorflow_serving/model_servers/platform_config_util.cc index fdd71bbab35..ca555c37374 100644 --- a/tensorflow_serving/model_servers/platform_config_util.cc +++ b/tensorflow_serving/model_servers/platform_config_util.cc @@ -17,8 +17,8 @@ limitations under the License. #include "google/protobuf/any.pb.h" #include "tensorflow_serving/model_servers/model_platform_types.h" -#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/model_servers/platform_config_util.h b/tensorflow_serving/model_servers/platform_config_util.h index 170ceb84448..7f9de4299ff 100644 --- a/tensorflow_serving/model_servers/platform_config_util.h +++ b/tensorflow_serving/model_servers/platform_config_util.h @@ -16,8 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CONFIG_UTIL_H_ #define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_CONFIG_UTIL_H_ -#include "tensorflow_serving/config/platform_config.proto.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_config.proto.h" +#include "tensorflow_serving/config/platform_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index da18c1f25bd..11ae0aa0b2e 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -30,6 +30,7 @@ limitations under the License. #define TENSORFLOW_SERVING_MODEL_SERVERS_SERVER_CORE_H_ #include +#include #include #include #include @@ -41,7 +42,7 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" -#include "tensorflow_serving/config/platform_config.proto.h" +#include "tensorflow_serving/config/platform_config.pb.h" #include "tensorflow_serving/core/aspired_versions_manager.h" #include "tensorflow_serving/core/dynamic_source_router.h" #include "tensorflow_serving/core/servable_state_monitor.h" @@ -232,7 +233,8 @@ class ServerCore : public Manager { // e.g. cross-model batch scheduling. struct SourceAdapters { // One adapter for each platform. - map> platform_adapters; + std::map> + platform_adapters; // An extra adapter to report errors for models with no configured platform. std::unique_ptr error_adapter; diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 9e785056c5d..d20fef97ec0 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -20,11 +20,11 @@ limitations under the License. #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" -#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.proto.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.pb.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" #include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h" -#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto.h" +#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.pb.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { diff --git a/tensorflow_serving/model_servers/test_util/mock_server_core.h b/tensorflow_serving/model_servers/test_util/mock_server_core.h index 96c37d0421d..53985381393 100644 --- a/tensorflow_serving/model_servers/test_util/mock_server_core.h +++ b/tensorflow_serving/model_servers/test_util/mock_server_core.h @@ -20,11 +20,11 @@ limitations under the License. #include #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" -#include "tensorflow_serving/apis/model.proto.h" -#include "tensorflow_serving/config/model_server_config.proto.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" -#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.proto.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.pb.h" #include "tensorflow_serving/model_servers/server_core.h" namespace tensorflow { diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index f0f71ed8127..6f8dea33985 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -20,9 +20,9 @@ limitations under the License. #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" -#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_config.proto.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { diff --git a/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc index d9a7c93c7b2..d4c815bf795 100644 --- a/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc +++ b/tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.cc @@ -15,7 +15,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h" #include "tensorflow_serving/core/source_adapter.h" -#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.proto.h" +#include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.pb.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h index 083cc71f1c1..8ddd78c6926 100644 --- a/tensorflow_serving/util/class_registration.h +++ b/tensorflow_serving/util/class_registration.h @@ -161,7 +161,6 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/util/port.h" #include "tensorflow_serving/util/class_registration_util.h" namespace tensorflow { From 2882a83b260dbf5a7b5a9801d9e084b57a95be38 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 12 Jan 2017 09:29:46 -0800 Subject: [PATCH 0133/8554] Makes Status param a reference in LoaderHarness::Error(...) - To prevent unwanted copying. Change: 144334308 --- tensorflow_serving/core/loader_harness.cc | 6 +++--- tensorflow_serving/core/loader_harness.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/core/loader_harness.cc b/tensorflow_serving/core/loader_harness.cc index 56136f10559..393412ab6eb 100644 --- a/tensorflow_serving/core/loader_harness.cc +++ b/tensorflow_serving/core/loader_harness.cc @@ -145,7 +145,7 @@ Status LoaderHarness::DoneQuiescing() { return Status::OK(); } -void LoaderHarness::ErrorInternal(const Status status) { +void LoaderHarness::ErrorInternal(const Status& status) { state_ = State::kError; status_ = status; if (options_.error_callback) { @@ -155,12 +155,12 @@ void LoaderHarness::ErrorInternal(const Status status) { << status_; } -void LoaderHarness::Error(const Status status) { +void LoaderHarness::Error(const Status& status) { mutex_lock l(mu_); ErrorInternal(status); } -Status LoaderHarness::TransitionState(State from, State to) { +Status LoaderHarness::TransitionState(const State from, const State to) { if (state_ != from) { const Status error = errors::Internal( "Illegal request to transition from state ", StateDebugString(state_), diff --git a/tensorflow_serving/core/loader_harness.h b/tensorflow_serving/core/loader_harness.h index 5febfca1bba..b65b391574c 100644 --- a/tensorflow_serving/core/loader_harness.h +++ b/tensorflow_serving/core/loader_harness.h @@ -196,7 +196,7 @@ class LoaderHarness final { Status DoneQuiescing() LOCKS_EXCLUDED(mu_); // Transitions the state to kError and invokes 'options_.error_callback'. - void Error(Status status) LOCKS_EXCLUDED(mu_); + void Error(const Status& status) LOCKS_EXCLUDED(mu_); // Whether anything has gone wrong with this servable. If state is kError, // this will be non-OK. If not OK, the error could be something that occurred @@ -218,7 +218,7 @@ class LoaderHarness final { // Transitions the state to kError and invokes 'options_.error_callback'. // Private method to be used when we want to set an error from another method // in this class, where mu_ is already held. - void ErrorInternal(Status status) EXCLUSIVE_LOCKS_REQUIRED(mu_); + void ErrorInternal(const Status& status) EXCLUSIVE_LOCKS_REQUIRED(mu_); // Expects 'state_' to equal 'from', and if so transitions it to 'to'. If not, // DCHECK-fails, calls ErrorInternal() with a suitable error and returns the From d29a87d095b25ced7b8c462f4270b2cb279d6e66 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 12 Jan 2017 10:58:55 -0800 Subject: [PATCH 0134/8554] Fixed bug that caused FileSystemStoragePathSource to convert servable versions from int64 to int32, making longer numeric strings (e.g. timestamps) to overflow and become negative. Change: 144344752 --- .../file_system_storage_path_source.cc | 2 +- .../file_system_storage_path_source_test.cc | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 3b772bb9940..2437dd7fb15 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -80,7 +80,7 @@ std::set GetDeletedServables( // aspire. void AspireVersion( const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, - const string& version_relative_path, const int version_number, + const string& version_relative_path, const int64 version_number, std::vector>* versions) { const ServableId servable_id = {servable.servable_name(), version_number}; const string full_path = diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index 19e12f0948c..55c08508fa5 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -343,6 +343,39 @@ TEST(FileSystemStoragePathSourceTest, AttemptToChangePollingPeriod) { EXPECT_FALSE(source->UpdateConfig(new_config).ok()); } +TEST(FileSystemStoragePathSourceTest, ParseTimestampedVersion) { + static_assert(static_cast(20170111173521LL) == 944751505, + "Version overflows if cast to int32."); + const string base_path = + io::JoinPath(testing::TmpDir(), "ParseTimestampedVersion"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + TF_ASSERT_OK( + Env::Default()->CreateDir(io::JoinPath(base_path, "20170111173521"))); + auto config = test_util::CreateProto( + strings::Printf("servables: { " + " version_policy: ALL_VERSIONS " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL(*target, SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre(ServableData( + {"test_servable_name", 20170111173521LL}, + io::JoinPath(base_path, "20170111173521"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + } // namespace } // namespace serving } // namespace tensorflow From c61ce90a3000c6acd9f471c97fc5c41864a077e5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jan 2017 11:15:16 -0800 Subject: [PATCH 0135/8554] Allow configuring number of tensorflow intra and inter threads via a flag. Change: 144463109 --- tensorflow_serving/model_servers/main.cc | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 83bb941ff69..5b36f72eba8 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -61,6 +61,7 @@ limitations under the License. #include "tensorflow/core/platform/init_main.h" #include "tensorflow/core/platform/protobuf.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/util/command_line_flags.h" #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #include "tensorflow_serving/apis/prediction_service.pb.h" @@ -202,6 +203,9 @@ int main(int argc, char** argv) { tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; bool use_saved_model = true; + // Tensorflow session parallelism of zero means that both inter and intra op + // thread pools will be auto configured. + int64 tensorflow_session_parallelism = 0; string platform_config_file = ""; tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( @@ -228,6 +232,12 @@ int main(int argc, char** argv) { "SessionBundle. It is used by tensorflow serving team " "to control the rollout of SavedModel and is not " "expected to be set by users directly."), + tensorflow::Flag("tensorflow_session_parallelism", + &tensorflow_session_parallelism, + "Number of threads to use for running a " + "Tensorflow session. Auto-configured by default." + "Note that this option is ignored if " + "--platform_config_file is non-empty."), tensorflow::Flag("platform_config_file", &platform_config_file, "If non-empty, read an ascii PlatformConfigMap protobuf " "from the supplied file name, and use that platform " @@ -268,6 +278,11 @@ int main(int argc, char** argv) { batching_parameters->mutable_thread_pool_name()->set_value( "model_server_batch_threads"); } + + session_bundle_config.mutable_session_config() + ->set_intra_op_parallelism_threads(tensorflow_session_parallelism); + session_bundle_config.mutable_session_config() + ->set_inter_op_parallelism_threads(tensorflow_session_parallelism); options.platform_config_map = CreateTensorFlowPlatformConfigMap( session_bundle_config, use_saved_model); } else { From 99d21f198dcb313635d5afb6774efb00bc7ce9af Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 13 Jan 2017 12:14:17 -0800 Subject: [PATCH 0136/8554] Google-internal documentation change. Change: 144470234 --- tensorflow_serving/g3doc/METADATA | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/METADATA b/tensorflow_serving/g3doc/METADATA index 8fb0d57fc32..230bbb59486 100644 --- a/tensorflow_serving/g3doc/METADATA +++ b/tensorflow_serving/g3doc/METADATA @@ -1,5 +1,6 @@ name: "TensorFlow Serving" g3doc: { - sitemap_file: "_includes/nav.md" + include: "/learning/serving/g3doc/METADATA" + sitemap_file: "/learning/serving/g3doc/users/sitemap.md" } From e0e87670050f6f394232efbc2cbeb14f23db3475 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Fri, 13 Jan 2017 12:40:13 -0800 Subject: [PATCH 0137/8554] Allows logging requests through the ServerCore. - Adds config option to allow requests hitting certain models to be logged. - Adds tests. Change: 144472989 --- tensorflow_serving/config/BUILD | 1 + .../config/model_server_config.proto | 4 + tensorflow_serving/core/BUILD | 47 ++++++ tensorflow_serving/core/log_collector.h | 3 + tensorflow_serving/core/request_logger.cc | 6 +- .../core/request_logger_test.cc | 127 ++++++++++++++++ .../core/server_request_logger.cc | 54 ++++--- .../core/server_request_logger.h | 34 +++-- .../core/server_request_logger_test.cc | 143 ++++++++++++++++++ tensorflow_serving/core/test_util/BUILD | 37 +++++ .../core/test_util/fake_log_collector.h | 46 ++++++ .../core/test_util/mock_log_collector.h | 37 +++++ .../core/test_util/mock_request_logger.h | 48 ++++++ tensorflow_serving/model_servers/BUILD | 5 + .../model_servers/server_core.cc | 19 +++ .../model_servers/server_core.h | 18 ++- .../model_servers/server_core_test.cc | 73 +++++++++ 17 files changed, 670 insertions(+), 32 deletions(-) create mode 100644 tensorflow_serving/core/request_logger_test.cc create mode 100644 tensorflow_serving/core/server_request_logger_test.cc create mode 100644 tensorflow_serving/core/test_util/fake_log_collector.h create mode 100644 tensorflow_serving/core/test_util/mock_log_collector.h create mode 100644 tensorflow_serving/core/test_util/mock_request_logger.h diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 2e9849b5761..78ada7438ed 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -27,6 +27,7 @@ serving_proto_library( srcs = ["model_server_config.proto"], cc_api_version = 2, deps = [ + ":logging_config_proto", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", "@protobuf//:cc_wkt_protos", ], diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index 26b45c68650..5a057248853 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -4,6 +4,7 @@ package tensorflow.serving; option cc_enable_arenas = true; import "google/protobuf/any.proto"; +import "tensorflow_serving/config/logging_config.proto"; import "tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto"; // The type of model. @@ -35,6 +36,9 @@ message ModelConfig { // be served at the same time. // The default option is to serve only the latest version of the model. FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5; + + // Configures logging requests and responses, to the model. + LoggingConfig logging_config = 6; } // Static list of models to be loaded for serving. diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 19445a09c7c..e261a025652 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -737,6 +737,27 @@ cc_library( ":logging_proto", "//tensorflow_serving/config:logging_config_proto", "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:protobuf", + ], +) + +cc_test( + name = "request_logger_test", + size = "small", + srcs = ["request_logger_test.cc"], + deps = [ + ":log_collector", + ":logging_proto", + ":request_logger", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/apis:predict_proto", + "//tensorflow_serving/config:logging_config_proto", + "//tensorflow_serving/core/test_util:mock_log_collector", + "//tensorflow_serving/core/test_util:mock_request_logger", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:protobuf", ], ) @@ -744,11 +765,37 @@ cc_library( name = "server_request_logger", srcs = ["server_request_logger.cc"], hdrs = ["server_request_logger.h"], + visibility = [ + "//visibility:public", + ], deps = [ ":request_logger", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:logging_config_proto", "//tensorflow_serving/core:logging_proto", + "//tensorflow_serving/util:fast_read_dynamic_ptr", + "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:protobuf", + ], +) + +cc_test( + name = "server_request_logger_test", + size = "small", + srcs = ["server_request_logger_test.cc"], + deps = [ + ":log_collector", + ":logging_proto", + ":request_logger", + ":server_request_logger", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/apis:predict_proto", + "//tensorflow_serving/config:logging_config_proto", + "//tensorflow_serving/core/test_util:fake_log_collector", + "//tensorflow_serving/core/test_util:mock_log_collector", + "//tensorflow_serving/core/test_util:mock_request_logger", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:lib", "@protobuf//:protobuf", ], diff --git a/tensorflow_serving/core/log_collector.h b/tensorflow_serving/core/log_collector.h index 3fef09bdbef..8e842124f99 100644 --- a/tensorflow_serving/core/log_collector.h +++ b/tensorflow_serving/core/log_collector.h @@ -57,6 +57,9 @@ class LogCollector { // Flushes buffered data so that the data can survive an application crash // (but not an OS crash). virtual Status Flush() = 0; + + protected: + LogCollector() = default; }; namespace register_log_collector { diff --git a/tensorflow_serving/core/request_logger.cc b/tensorflow_serving/core/request_logger.cc index 6ec1f99f923..e114eb8d4a7 100644 --- a/tensorflow_serving/core/request_logger.cc +++ b/tensorflow_serving/core/request_logger.cc @@ -34,9 +34,13 @@ Status RequestLogger::Log(const google::protobuf::Message& request, const LogMetadata& log_metadata) { const double sampling_rate = logging_config_.sampling_config().sampling_rate(); + LogMetadata log_metadata_with_config = log_metadata; + *log_metadata_with_config.mutable_sampling_config() = + logging_config_.sampling_config(); if (uniform_sampler_.Sample(sampling_rate)) { std::unique_ptr log; - TF_RETURN_IF_ERROR(CreateLogMessage(request, response, log_metadata, &log)); + TF_RETURN_IF_ERROR( + CreateLogMessage(request, response, log_metadata_with_config, &log)); TF_RETURN_IF_ERROR(log_collector_->CollectMessage(*log)); } return Status::OK(); diff --git a/tensorflow_serving/core/request_logger_test.cc b/tensorflow_serving/core/request_logger_test.cc new file mode 100644 index 00000000000..1ece4ef79d5 --- /dev/null +++ b/tensorflow_serving/core/request_logger_test.cc @@ -0,0 +1,127 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/request_logger.h" + +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/wrappers.pb.h" +#include "google/protobuf/message.h" +#include +#include +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/log_collector.h" +#include "tensorflow_serving/core/logging.pb.h" +#include "tensorflow_serving/core/test_util/mock_log_collector.h" +#include "tensorflow_serving/core/test_util/mock_request_logger.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::_; +using ::testing::HasSubstr; +using ::testing::Invoke; +using ::testing::NiceMock; +using ::testing::Return; + +class RequestLoggerTest : public ::testing::Test { + protected: + RequestLoggerTest() { + LoggingConfig logging_config; + logging_config.mutable_sampling_config()->set_sampling_rate(1.0); + log_collector_ = new NiceMock(); + request_logger_ = std::unique_ptr>( + new NiceMock(logging_config, log_collector_)); + } + + NiceMock* log_collector_; + std::unique_ptr> request_logger_; +}; + +TEST_F(RequestLoggerTest, Simple) { + ModelSpec model_spec; + model_spec.set_name("model"); + model_spec.mutable_version()->set_value(10); + + PredictRequest request; + *request.mutable_model_spec() = model_spec; + + PredictResponse response; + response.mutable_outputs()->insert({"tensor", TensorProto()}); + LogMetadata log_metadata; + *log_metadata.mutable_model_spec() = model_spec; + + EXPECT_CALL(*request_logger_, CreateLogMessage(_, _, _, _)) + .WillOnce(Invoke([&](const google::protobuf::Message& actual_request, + const google::protobuf::Message& actual_response, + const LogMetadata& actual_log_metadata, + std::unique_ptr* log) { + EXPECT_THAT(static_cast(actual_request), + test_util::EqualsProto(request)); + EXPECT_THAT(static_cast(actual_response), + test_util::EqualsProto(PredictResponse())); + LogMetadata expected_log_metadata = log_metadata; + expected_log_metadata.mutable_sampling_config()->set_sampling_rate(1.0); + EXPECT_THAT(actual_log_metadata, + test_util::EqualsProto(expected_log_metadata)); + *log = + std::unique_ptr(new google::protobuf::Any()); + return Status::OK(); + })); + EXPECT_CALL(*log_collector_, CollectMessage(_)) + .WillOnce(Return(Status::OK())); + TF_ASSERT_OK(request_logger_->Log(request, PredictResponse(), log_metadata)); +} + +TEST_F(RequestLoggerTest, ErroringCreateLogMessage) { + EXPECT_CALL(*request_logger_, CreateLogMessage(_, _, _, _)) + .WillRepeatedly(Return(errors::Internal("Error"))); + EXPECT_CALL(*log_collector_, CollectMessage(_)).Times(0); + const auto error_status = + request_logger_->Log(PredictRequest(), PredictResponse(), LogMetadata()); + ASSERT_FALSE(error_status.ok()); + EXPECT_THAT(error_status.error_message(), HasSubstr("Error")); +} + +TEST_F(RequestLoggerTest, ErroringCollectMessage) { + EXPECT_CALL(*request_logger_, CreateLogMessage(_, _, _, _)) + .WillRepeatedly(Invoke([&](const google::protobuf::Message& actual_request, + const google::protobuf::Message& actual_response, + const LogMetadata& actual_log_metadata, + std::unique_ptr* log) { + *log = + std::unique_ptr(new google::protobuf::Any()); + return Status::OK(); + })); + EXPECT_CALL(*log_collector_, CollectMessage(_)) + .WillRepeatedly(Return(errors::Internal("Error"))); + const auto error_status = + request_logger_->Log(PredictRequest(), PredictResponse(), LogMetadata()); + ASSERT_FALSE(error_status.ok()); + EXPECT_THAT(error_status.error_message(), HasSubstr("Error")); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/server_request_logger.cc b/tensorflow_serving/core/server_request_logger.cc index a4bc9c5e867..a292c9e8d80 100644 --- a/tensorflow_serving/core/server_request_logger.cc +++ b/tensorflow_serving/core/server_request_logger.cc @@ -25,39 +25,51 @@ namespace serving { // static Status ServerRequestLogger::Create( - const std::map& logging_config_map, - const std::function*)>& request_logger_creator, std::unique_ptr* server_request_logger) { - std::unordered_map> request_logger_map; - for (const auto& model_and_logging_config : logging_config_map) { - auto& request_logger = request_logger_map[model_and_logging_config.first]; - TF_RETURN_IF_ERROR(request_logger_creator( - model_and_logging_config.second.log_collector_config(), - &request_logger)); - } - server_request_logger->reset( - new ServerRequestLogger(std::move(request_logger_map))); + server_request_logger->reset(new ServerRequestLogger(request_logger_creator)); return Status::OK(); } ServerRequestLogger::ServerRequestLogger( - std::unordered_map> - request_logger_map) - : request_logger_map_(std::move(request_logger_map)) {} + const std::function*)>& + request_logger_creator) + : request_logger_map_( + std::unique_ptr(new RequestLoggerMap())), + request_logger_creator_(request_logger_creator) {} + +Status ServerRequestLogger::Update( + const std::map& logging_config_map) { + if (!logging_config_map.empty() && !request_logger_creator_) { + return errors::InvalidArgument("No request-logger-creator provided."); + } + std::unique_ptr request_logger_map(new RequestLoggerMap()); + for (const auto& model_and_logging_config : logging_config_map) { + auto& request_logger = + (*request_logger_map)[model_and_logging_config.first]; + TF_RETURN_IF_ERROR(request_logger_creator_(model_and_logging_config.second, + &request_logger)); + } + request_logger_map_.Update(std::move(request_logger_map)); + return Status::OK(); +} Status ServerRequestLogger::Log(const google::protobuf::Message& request, const google::protobuf::Message& response, const LogMetadata& log_metadata) { const string& model_name = log_metadata.model_spec().name(); - auto found_it = request_logger_map_.find(model_name); - if (found_it == request_logger_map_.end()) { - const string error = - strings::StrCat("Cannot find request-logger for model: ", model_name); - // This shouldn't happen at all, so dchecking for capturing in tests. - DCHECK(false) << error; // Crash ok. - return errors::NotFound(error); + auto request_logger_map = request_logger_map_.get(); + if (request_logger_map->empty()) { + VLOG(2) << "Request logger map is empty."; + return Status::OK(); + } + auto found_it = request_logger_map->find(model_name); + if (found_it == request_logger_map->end()) { + VLOG(2) << "Cannot find request-logger for model: " << model_name; + return Status::OK(); } auto& request_logger = found_it->second; return request_logger->Log(request, response, log_metadata); diff --git a/tensorflow_serving/core/server_request_logger.h b/tensorflow_serving/core/server_request_logger.h index 51b1a5e416d..b730643fcec 100644 --- a/tensorflow_serving/core/server_request_logger.h +++ b/tensorflow_serving/core/server_request_logger.h @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow_serving/config/logging_config.pb.h" #include "tensorflow_serving/core/logging.pb.h" #include "tensorflow_serving/core/request_logger.h" +#include "tensorflow_serving/util/fast_read_dynamic_ptr.h" namespace tensorflow { namespace serving { @@ -37,29 +38,44 @@ namespace serving { // sampling config. class ServerRequestLogger { public: - // Creates the ServerRequestLogger based on the map of logging-configs keyed - // on model-names, and a custom request-logger creator method. + // Creates the ServerRequestLogger based on a custom request_logger_creator + // method. + // + // You can create an empty ServerRequestLogger with an empty + // request_logger_creator. static Status Create( - const std::map& logging_config_map, - const std::function*)>& request_logger_creator, std::unique_ptr* server_request_logger); ~ServerRequestLogger() = default; + // Updates the logger with the new 'logging_config_map'. + // + // If the ServerRequestLogger was created using an empty + // request_logger_creator, this will return an error if a non-empty + // logging_config_map is passed in. + Status Update(const std::map& logging_config_map); + // Similar to RequestLogger::Log(). Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, const LogMetadata& log_metadata); private: explicit ServerRequestLogger( - std::unordered_map> - request_logger_map); + const std::function*)>& + request_logger_creator); + + // A map from model_name to its corresponding RequestLogger. + using RequestLoggerMap = + std::unordered_map>; + FastReadDynamicPtr request_logger_map_; - // A map from model-name to its corresponding request-logger. - std::unordered_map> - request_logger_map_; + std::function*)> + request_logger_creator_; }; } // namespace serving diff --git a/tensorflow_serving/core/server_request_logger_test.cc b/tensorflow_serving/core/server_request_logger_test.cc new file mode 100644 index 00000000000..b9bd83d0ad7 --- /dev/null +++ b/tensorflow_serving/core/server_request_logger_test.cc @@ -0,0 +1,143 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/server_request_logger.h" + +#include +#include + +#include "google/protobuf/any.pb.h" +#include "google/protobuf/message.h" +#include +#include +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" +#include "tensorflow_serving/config/log_collector_config.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/log_collector.h" +#include "tensorflow_serving/core/logging.pb.h" +#include "tensorflow_serving/core/test_util/fake_log_collector.h" +#include "tensorflow_serving/core/test_util/mock_request_logger.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::_; +using ::testing::Invoke; +using ::testing::NiceMock; + +LogCollectorConfig CreateLogCollectorConfig(const string& type, + const string& filename_prefix) { + LogCollectorConfig config; + config.set_type(type); + config.set_filename_prefix(filename_prefix); + return config; +} + +std::pair CreateLoggingConfigForModel( + const string& model_name) { + LoggingConfig logging_config; + *logging_config.mutable_log_collector_config() = + CreateLogCollectorConfig("", strings::StrCat("/file/", model_name)); + logging_config.mutable_sampling_config()->set_sampling_rate(1.0); + return {model_name, logging_config}; +} + +class ServerRequestLoggerTest : public ::testing::Test { + protected: + ServerRequestLoggerTest() { + TF_CHECK_OK(ServerRequestLogger::Create( + [&](const LoggingConfig& logging_config, + std::unique_ptr* const request_logger) { + const string& filename_prefix = + logging_config.log_collector_config().filename_prefix(); + log_collector_map_[filename_prefix] = new FakeLogCollector(); + auto mock_request_logger = + std::unique_ptr>( + new NiceMock( + logging_config, log_collector_map_[filename_prefix])); + ON_CALL(*mock_request_logger, CreateLogMessage(_, _, _, _)) + .WillByDefault(Invoke([&](const google::protobuf::Message& actual_request, + const google::protobuf::Message& actual_response, + const LogMetadata& actual_log_metadata, + std::unique_ptr* log) { + *log = std::unique_ptr( + new google::protobuf::Any()); + return Status::OK(); + })); + *request_logger = std::move(mock_request_logger); + return Status::OK(); + }, + &server_request_logger_)); + } + + std::unordered_map log_collector_map_; + std::unique_ptr server_request_logger_; +}; + +TEST_F(ServerRequestLoggerTest, Empty) { + TF_ASSERT_OK(server_request_logger_->Update({})); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + LogMetadata())); + // No log-collectors should have been made. + EXPECT_TRUE(log_collector_map_.empty()); +} + +TEST_F(ServerRequestLoggerTest, AbsentModel) { + std::map model_logging_configs; + model_logging_configs.insert(CreateLoggingConfigForModel("model0")); + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + LogMetadata log_metadata; + auto* const model_spec = log_metadata.mutable_model_spec(); + model_spec->set_name("absent_model"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata)); + ASSERT_EQ(1, log_collector_map_.size()); + EXPECT_EQ(0, log_collector_map_["/file/model0"]->collect_count()); +} + +TEST_F(ServerRequestLoggerTest, MultipleModels) { + std::map model_logging_configs; + model_logging_configs.insert(CreateLoggingConfigForModel("model0")); + model_logging_configs.insert(CreateLoggingConfigForModel("model1")); + TF_ASSERT_OK(server_request_logger_->Update(model_logging_configs)); + + LogMetadata log_metadata0; + auto* const model_spec0 = log_metadata0.mutable_model_spec(); + model_spec0->set_name("model0"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata0)); + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0"]->collect_count()); + EXPECT_EQ(0, log_collector_map_["/file/model1"]->collect_count()); + + LogMetadata log_metadata1; + auto* const model_spec = log_metadata1.mutable_model_spec(); + model_spec->set_name("model1"); + TF_ASSERT_OK(server_request_logger_->Log(PredictRequest(), PredictResponse(), + log_metadata1)); + ASSERT_EQ(2, log_collector_map_.size()); + EXPECT_EQ(1, log_collector_map_["/file/model0"]->collect_count()); + EXPECT_EQ(1, log_collector_map_["/file/model1"]->collect_count()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 34b845ea240..b1d86bbf8b6 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -145,3 +145,40 @@ cc_library( "//tensorflow_serving/core:caching_manager", ], ) + +cc_library( + name = "mock_log_collector", + testonly = 1, + hdrs = ["mock_log_collector.h"], + deps = [ + "//external:gtest", + "//tensorflow_serving/core:log_collector", + "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:protobuf", + ], +) + +cc_library( + name = "fake_log_collector", + testonly = 1, + hdrs = ["fake_log_collector.h"], + deps = [ + "//external:gtest", + "//tensorflow_serving/core:log_collector", + "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:protobuf", + ], +) + +cc_library( + name = "mock_request_logger", + testonly = 1, + hdrs = ["mock_request_logger.h"], + deps = [ + "//external:gtest", + "//tensorflow_serving/core:logging_proto", + "//tensorflow_serving/core:request_logger", + "@org_tensorflow//tensorflow/core:lib", + "@protobuf//:protobuf", + ], +) diff --git a/tensorflow_serving/core/test_util/fake_log_collector.h b/tensorflow_serving/core/test_util/fake_log_collector.h new file mode 100644 index 00000000000..34a68c65f28 --- /dev/null +++ b/tensorflow_serving/core/test_util/fake_log_collector.h @@ -0,0 +1,46 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOG_COLLECTOR_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOG_COLLECTOR_H_ + +#include "google/protobuf/message.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/log_collector.h" + +namespace tensorflow { +namespace serving { + +// FakeLogCollector which does nothing except count the number of times +// CollectMessage has been called on it. +class FakeLogCollector : public LogCollector { + public: + Status CollectMessage(const google::protobuf::Message& message) override { + ++collect_count_; + return Status::OK(); + } + + Status Flush() override { return Status::OK(); } + + int collect_count() const { return collect_count_; } + + private: + int collect_count_ = 0; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_FAKE_LOG_COLLECTOR_H_ diff --git a/tensorflow_serving/core/test_util/mock_log_collector.h b/tensorflow_serving/core/test_util/mock_log_collector.h new file mode 100644 index 00000000000..13b11e65354 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_log_collector.h @@ -0,0 +1,37 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOG_COLLECTOR_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOG_COLLECTOR_H_ + +#include "google/protobuf/message.h" +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/log_collector.h" + +namespace tensorflow { +namespace serving { + +class MockLogCollector : public LogCollector { + public: + MockLogCollector() = default; + MOCK_METHOD1(CollectMessage, Status(const google::protobuf::Message& message)); + MOCK_METHOD0(Flush, Status()); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_LOG_COLLECTOR_H_ diff --git a/tensorflow_serving/core/test_util/mock_request_logger.h b/tensorflow_serving/core/test_util/mock_request_logger.h new file mode 100644 index 00000000000..be7590cb0bd --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_request_logger.h @@ -0,0 +1,48 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_REQUEST_LOGGER_H_ + +#include "google/protobuf/message.h" +#include +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/config/logging_config.pb.h" +#include "tensorflow_serving/core/log_collector.h" +#include "tensorflow_serving/core/logging.pb.h" +#include "tensorflow_serving/core/request_logger.h" + +namespace tensorflow { +namespace serving { + +class MockRequestLogger : public RequestLogger { + public: + // Unfortunately NiceMock doesn't support ctors with move-only types, so we + // have to do this workaround. + MockRequestLogger(const LoggingConfig& logging_config, + LogCollector* log_collector) + : RequestLogger(logging_config, + std::unique_ptr(log_collector)) {} + + MOCK_METHOD4(CreateLogMessage, Status(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata, + std::unique_ptr* log)); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_REQUEST_LOGGER_H_ diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 1aa890221eb..dccb14c6101 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -62,6 +62,7 @@ cc_library( "//tensorflow_serving/core:dynamic_source_router", "//tensorflow_serving/core:load_servables_fast", "//tensorflow_serving/core:servable_state_monitor", + "//tensorflow_serving/core:server_request_logger", "//tensorflow_serving/core:source", "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", @@ -85,16 +86,20 @@ cc_test( deps = [ ":server_core", "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/core:servable_state", "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:fake_loader_source_adapter_proto", + "//tensorflow_serving/core/test_util:fake_log_collector", + "//tensorflow_serving/core/test_util:mock_request_logger", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/model_servers/test_util:server_core_test_util", "//tensorflow_serving/model_servers/test_util:storage_path_error_injecting_source_adapter", "//tensorflow_serving/model_servers/test_util:storage_path_error_injecting_source_adapter_proto", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:test", + "@protobuf//:cc_wkt_protos", ], ) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index c093cf3a8ad..799a84eb703 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -147,6 +147,11 @@ Status ServerCore::Create(Options options, }; } + if (options.server_request_logger == nullptr) { + TF_RETURN_IF_ERROR( + ServerRequestLogger::Create(nullptr, &options.server_request_logger)); + } + // We need to move the aspired_version_policy first because we will move the // server_core_config (which contains aspired_version_policy) below. std::unique_ptr aspired_version_policy = @@ -285,6 +290,19 @@ Status ServerCore::AddModelsViaCustomModelConfig() { config_.custom_model_config(), servable_event_bus_.get(), &manager_); } +Status ServerCore::MaybeUpdateServerRequestLogger() { + std::map logging_config_map; + for (const auto& model_config : config_.model_config_list().config()) { + if (model_config.has_logging_config()) { + logging_config_map.insert( + {model_config.name(), model_config.logging_config()}); + } + } + TF_RETURN_IF_ERROR( + options_.server_request_logger->Update(logging_config_map)); + return Status::OK(); +} + Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { mutex_lock l(config_mu_); @@ -320,6 +338,7 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { switch (config_.config_case()) { case ModelServerConfig::kModelConfigList: { TF_RETURN_IF_ERROR(AddModelsViaModelConfigList()); + TF_RETURN_IF_ERROR(MaybeUpdateServerRequestLogger()); break; } case ModelServerConfig::kCustomModelConfig: { diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 11ae0aa0b2e..d3c536e8819 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -41,11 +41,13 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/config/logging_config.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/config/platform_config.pb.h" #include "tensorflow_serving/core/aspired_versions_manager.h" #include "tensorflow_serving/core/dynamic_source_router.h" #include "tensorflow_serving/core/servable_state_monitor.h" +#include "tensorflow_serving/core/server_request_logger.h" #include "tensorflow_serving/core/source.h" #include "tensorflow_serving/core/source_adapter.h" #include "tensorflow_serving/core/storage_path.h" @@ -116,6 +118,9 @@ class ServerCore : public Manager { // A function for instantiating and connecting custom sources and source // adapters to the manager. CustomModelConfigLoader custom_model_config_loader; + + // Logger used for logging requests hitting the server. + std::unique_ptr server_request_logger; }; virtual ~ServerCore() = default; @@ -172,8 +177,16 @@ class ServerCore : public Manager { return Status::OK(); } + // Writes the log for the particular request, response and metadata, if we + // decide to sample it and if request-logging was configured for the + // particular model. + Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, + const LogMetadata& log_metadata) { + return options_.server_request_logger->Log(request, response, log_metadata); + } + protected: - explicit ServerCore(Options options); + ServerCore(Options options); private: friend class test_util::ServerCoreTestAccess; @@ -269,6 +282,9 @@ class ServerCore : public Manager { // Adds/reloads models through custom model config of 'config_'. Status AddModelsViaCustomModelConfig() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + // Updates the ServerRequestLogger based on the ModelConfigList. + Status MaybeUpdateServerRequestLogger() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + // ************************************************************************ // Request Processing. // ************************************************************************ diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index d20fef97ec0..b25876c47f3 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -15,12 +15,17 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" +#include "google/protobuf/any.pb.h" +#include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.pb.h" +#include "tensorflow_serving/core/test_util/fake_log_collector.h" +#include "tensorflow_serving/core/test_util/mock_request_logger.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" #include "tensorflow_serving/model_servers/test_util/storage_path_error_injecting_source_adapter.h" @@ -31,6 +36,9 @@ namespace tensorflow { namespace serving { namespace { +using ::testing::_; +using ::testing::Invoke; +using ::testing::NiceMock; using test_util::ServerCoreTest; TEST_P(ServerCoreTest, CreateWaitsTillModelsAvailable) { @@ -343,6 +351,71 @@ TEST_P(ServerCoreTest, IllegalToChangeModelPlatform) { ::testing::HasSubstr("Illegal to change a model's platform")); } +TEST_P(ServerCoreTest, RequestLoggingOff) { + // Create a ServerCore with deprecated config. + std::unique_ptr server_core; + const ModelServerConfig config = + GetTestModelServerConfigForTensorflowPlatform(); + TF_ASSERT_OK(CreateServerCore(config, &server_core)); + + TF_ASSERT_OK( + server_core->Log(PredictRequest(), PredictResponse(), LogMetadata())); +} + +TEST_P(ServerCoreTest, RequestLoggingOn) { + std::unordered_map log_collector_map; + ServerCore::Options options = GetDefaultOptions(); + TF_CHECK_OK(ServerRequestLogger::Create( + [&](const LoggingConfig& logging_config, + std::unique_ptr* const request_logger) { + const string& filename_prefix = + logging_config.log_collector_config().filename_prefix(); + log_collector_map[filename_prefix] = new FakeLogCollector(); + auto mock_request_logger = std::unique_ptr>( + new NiceMock( + logging_config, log_collector_map[filename_prefix])); + ON_CALL(*mock_request_logger, CreateLogMessage(_, _, _, _)) + .WillByDefault(Invoke([&](const google::protobuf::Message& actual_request, + const google::protobuf::Message& actual_response, + const LogMetadata& actual_log_metadata, + std::unique_ptr* log) { + *log = std::unique_ptr( + new google::protobuf::Any()); + return Status::OK(); + })); + *request_logger = std::move(mock_request_logger); + return Status::OK(); + }, + &options.server_request_logger)); + + // We now setup a model-server-config with a model which switches on request + // logging. + LogCollectorConfig log_collector_config; + log_collector_config.set_type(""); + log_collector_config.set_filename_prefix(test_util::kTestModelName); + LoggingConfig logging_config; + *logging_config.mutable_log_collector_config() = log_collector_config; + logging_config.mutable_sampling_config()->set_sampling_rate(1.0); + + ModelServerConfig model_server_config = + GetTestModelServerConfigForTensorflowPlatform(); + *model_server_config.mutable_model_config_list() + ->mutable_config(0) + ->mutable_logging_config() = logging_config; + options.model_server_config = model_server_config; + + std::unique_ptr server_core; + TF_ASSERT_OK(ServerCore::Create(std::move(options), &server_core)); + + LogMetadata log_metadata0; + auto* const model_spec0 = log_metadata0.mutable_model_spec(); + model_spec0->set_name(test_util::kTestModelName); + TF_ASSERT_OK( + server_core->Log(PredictRequest(), PredictResponse(), log_metadata0)); + ASSERT_EQ(1, log_collector_map.size()); + EXPECT_EQ(1, log_collector_map[test_util::kTestModelName]->collect_count()); +} + INSTANTIATE_TEST_CASE_P( TestType, ServerCoreTest, ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES))); From 5be51781d33cc73f18a92b69e737724f38d5c71c Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Sat, 14 Jan 2017 17:48:28 -0800 Subject: [PATCH 0138/8554] Log when bypassing batcher. Change: 144548935 --- tensorflow_serving/batching/batching_session.cc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index d3bb6eb3e72..f6b71df3aa7 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -217,6 +217,9 @@ Status BatchingSession::Run( if (batch_scheduler_it == batch_schedulers_.end()) { // We have a Run() call that doesn't match one of our batching signatures. // Run it in-line. + LOG(WARNING) << "Request doesn't match any declared signature. Bypassing " + "batcher. Request signature is: " + << TensorSignatureDebugString(signature); return wrapped_->Run(inputs, output_tensor_names, target_node_names, outputs); } From 18852e8a36d691c4bc9fd7666309645bec94ff76 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 17 Jan 2017 13:57:16 -0800 Subject: [PATCH 0139/8554] Minor namespace fix. Change: 144751889 --- tensorflow_serving/model_servers/main.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 5b36f72eba8..2a6002ed456 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -205,7 +205,7 @@ int main(int argc, char** argv) { bool use_saved_model = true; // Tensorflow session parallelism of zero means that both inter and intra op // thread pools will be auto configured. - int64 tensorflow_session_parallelism = 0; + tensorflow::int64 tensorflow_session_parallelism = 0; string platform_config_file = ""; tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( From fd6cbd2ccfb302ca2bc8495f82069cc134609217 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Wed, 18 Jan 2017 14:40:10 -0800 Subject: [PATCH 0140/8554] Add AvailabilityPreservePolicy based on EagerLoadPolicy with reduced memory usage: it unloads un-aspired versions more aggressively and loads aspired versions in the order of descending version number. Add and ResourcePreservePolicy based on EagerUnloadPolicy but loads aspired versions in the order of descending version number. Change: 144883970 --- tensorflow_serving/core/BUILD | 63 ++++++- .../core/aspired_version_policy.cc | 36 ++++ .../core/aspired_version_policy.h | 9 + .../core/aspired_version_policy_test.cc | 64 +++++++ .../core/aspired_versions_manager_test.cc | 116 +++++++++---- .../core/availability_preserving_policy.cc | 86 ++++++++++ .../core/availability_preserving_policy.h | 50 ++++++ .../availability_preserving_policy_test.cc | 162 ++++++++++++++++++ tensorflow_serving/core/eager_load_policy.h | 2 + tensorflow_serving/core/eager_unload_policy.h | 2 + .../core/resource_preserving_policy.cc | 61 +++++++ .../core/resource_preserving_policy.h | 52 ++++++ .../core/resource_preserving_policy_test.cc | 123 +++++++++++++ 13 files changed, 790 insertions(+), 36 deletions(-) create mode 100644 tensorflow_serving/core/aspired_version_policy.cc create mode 100644 tensorflow_serving/core/aspired_version_policy_test.cc create mode 100644 tensorflow_serving/core/availability_preserving_policy.cc create mode 100644 tensorflow_serving/core/availability_preserving_policy.h create mode 100644 tensorflow_serving/core/availability_preserving_policy_test.cc create mode 100644 tensorflow_serving/core/resource_preserving_policy.cc create mode 100644 tensorflow_serving/core/resource_preserving_policy.h create mode 100644 tensorflow_serving/core/resource_preserving_policy_test.cc diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index e261a025652..b8857ab5960 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -513,7 +513,7 @@ cc_test( srcs = ["aspired_versions_manager_test.cc"], deps = [ ":aspired_versions_manager", - ":eager_load_policy", + ":availability_preserving_policy", ":servable_state_monitor", "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:fake_loader", @@ -614,6 +614,7 @@ cc_test( cc_library( name = "aspired_version_policy", + srcs = ["aspired_version_policy.cc"], hdrs = ["aspired_version_policy.h"], visibility = [ "//visibility:public", @@ -626,6 +627,18 @@ cc_library( ], ) +cc_test( + name = "aspired_version_policy_test", + srcs = ["aspired_version_policy_test.cc"], + deps = [ + ":aspired_version_policy", + ":loader_harness", + ":servable_id", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/util:optional", + ], +) + cc_library( name = "eager_load_policy", srcs = ["eager_load_policy.cc"], @@ -674,6 +687,54 @@ cc_test( ], ) +cc_library( + name = "availability_preserving_policy", + srcs = ["availability_preserving_policy.cc"], + hdrs = ["availability_preserving_policy.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":aspired_version_policy", + ":loader_harness", + "//tensorflow_serving/util:optional", + ], +) + +cc_test( + name = "availability_preserving_policy_test", + srcs = ["availability_preserving_policy_test.cc"], + deps = [ + ":availability_preserving_policy", + ":servable_id", + "//tensorflow_serving/core/test_util:test_main", + ], +) + +cc_library( + name = "resource_preserving_policy", + srcs = ["resource_preserving_policy.cc"], + hdrs = ["resource_preserving_policy.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":aspired_version_policy", + ":loader_harness", + "//tensorflow_serving/util:optional", + ], +) + +cc_test( + name = "resource_preserving_policy_test", + srcs = ["resource_preserving_policy_test.cc"], + deps = [ + ":resource_preserving_policy", + ":servable_id", + "//tensorflow_serving/core/test_util:test_main", + ], +) + cc_library( name = "load_servables_fast", srcs = ["load_servables_fast.cc"], diff --git a/tensorflow_serving/core/aspired_version_policy.cc b/tensorflow_serving/core/aspired_version_policy.cc new file mode 100644 index 00000000000..6a022fc979f --- /dev/null +++ b/tensorflow_serving/core/aspired_version_policy.cc @@ -0,0 +1,36 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/aspired_version_policy.h" + +namespace tensorflow { +namespace serving { + +optional AspiredVersionPolicy::GetHighestAspiredNewServableId( + const std::vector& all_versions) { + optional highest_version_id; + for (const auto& version : all_versions) { + if (version.is_aspired && version.state == LoaderHarness::State::kNew) { + if (!highest_version_id || + version.id.version > highest_version_id.value().version) { + highest_version_id = version.id; + } + } + } + return highest_version_id; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/aspired_version_policy.h b/tensorflow_serving/core/aspired_version_policy.h index 83013fe31be..b7f6447aa42 100644 --- a/tensorflow_serving/core/aspired_version_policy.h +++ b/tensorflow_serving/core/aspired_version_policy.h @@ -77,6 +77,15 @@ class AspiredVersionPolicy { // that the servable stream is up to date. virtual optional GetNextAction( const std::vector& all_versions) const = 0; + + protected: + // Returns the aspired ServableId with the highest version that matches + // kNew state, if any exists. + static optional GetHighestAspiredNewServableId( + const std::vector& all_versions); + + private: + friend class AspiredVersionPolicyTest; }; inline bool operator==(const AspiredVersionPolicy::ServableAction& lhs, diff --git a/tensorflow_serving/core/aspired_version_policy_test.cc b/tensorflow_serving/core/aspired_version_policy_test.cc new file mode 100644 index 00000000000..3b5c07d06b4 --- /dev/null +++ b/tensorflow_serving/core/aspired_version_policy_test.cc @@ -0,0 +1,64 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +#include "tensorflow_serving/core/aspired_version_policy.h" + +#include + +#include +#include "tensorflow_serving/core/loader_harness.h" +#include "tensorflow_serving/core/servable_id.h" +#include "tensorflow_serving/util/optional.h" + +namespace tensorflow { +namespace serving { + +class AspiredVersionPolicyTest : public ::testing::Test { + public: + // Expose the protected AspiredVersionPolicy::GetHighestAspiredNewServableId() + // method. + optional GetHighestAspiredNewServableId( + const std::vector& all_versions) { + return AspiredVersionPolicy::GetHighestAspiredNewServableId(all_versions); + } +}; + +TEST_F(AspiredVersionPolicyTest, NoHighestAspiredNewServableId) { + // No new aspired versions. + std::vector versions; + versions.push_back({{"test", 10}, LoaderHarness::State::kNew, false}); + versions.push_back({{"test", 9}, LoaderHarness::State::kUnloading, true}); + optional highest_aspired_new = + GetHighestAspiredNewServableId(versions); + ASSERT_FALSE(highest_aspired_new); +} + +TEST_F(AspiredVersionPolicyTest, HighestAspiredNewServableId) { + // Three new aspired versions and two other versions. + std::vector versions; + versions.push_back({{"test", 10}, LoaderHarness::State::kNew, false}); + versions.push_back({{"test", 9}, LoaderHarness::State::kUnloading, true}); + versions.push_back({{"test", 1}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + optional highest_aspired_new = + GetHighestAspiredNewServableId(versions); + ASSERT_TRUE(highest_aspired_new); + EXPECT_EQ("test", highest_aspired_new.value().name); + EXPECT_EQ(5, highest_aspired_new.value().version); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 98d02f7f82e..b658d010a4a 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -25,7 +25,7 @@ limitations under the License. #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader.h" @@ -80,7 +80,8 @@ class AspiredVersionsManagerTest // The state manager thread won't be run automatically. manager_options.manage_state_interval_micros = -1; manager_options.env = Env::Default(); - manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + manager_options.aspired_version_policy.reset( + new AvailabilityPreservingPolicy()); manager_options.servable_event_bus = servable_event_bus_.get(); max_num_load_retries_ = 1; manager_options.max_num_load_retries = max_num_load_retries_; @@ -197,7 +198,11 @@ TEST_P(AspiredVersionsManagerTest, ServableHandleLatest) { manager_->GetAspiredVersionsCallback()(kServableName, std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); @@ -434,7 +439,11 @@ TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateLoad) { ASSERT_FALSE(not_ready_status.ok()) << not_ready_status; EXPECT_EQ(error::NOT_FOUND, not_ready_status.code()); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); @@ -638,7 +647,11 @@ TEST_P(AspiredVersionsManagerTest, EventBusErrorOnLoad) { EXPECT_THAT(*servable_state_monitor_.GetState(id), EqualsServableState(start_state)); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); @@ -670,9 +683,15 @@ TEST_P(AspiredVersionsManagerTest, EventBusServableLifecycle) { return Status::OK(); })); - std::unique_ptr load_thread( - Env::Default()->StartThread(ThreadOptions(), "LoadThread", - [&]() { InvokePolicyAndExecuteAction(); })); + std::unique_ptr load_unload_thread( + Env::Default()->StartThread(ThreadOptions(), "LoadUnloadThread", [&]() { + // Unload version 0 and load the new aspired version. Version 1 may or + // may not be unloaded (depending on whether load/unload thread pools + // are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } + })); load_called.WaitForNotification(); @@ -695,17 +714,17 @@ TEST_P(AspiredVersionsManagerTest, EventBusServableLifecycle) { Notification unload_called; Notification unload_continue; - EXPECT_CALL(*loader, Unload()) - .WillOnce(Invoke([&]() { - unload_called.Notify(); - unload_continue.WaitForNotification(); - })); + EXPECT_CALL(*loader, Unload()).WillOnce(Invoke([&]() { + unload_called.Notify(); + unload_continue.WaitForNotification(); + })); std::unique_ptr unload_thread( Env::Default()->StartThread(ThreadOptions(), "UnloadThread", [&]() { - for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { - InvokePolicyAndExecuteAction(); - } + // Call InvokePolicyAndExecuteAction() twice to unload version 1 and the + // new version, in case version 1 has not been unloaded previously. + InvokePolicyAndExecuteAction(); + InvokePolicyAndExecuteAction(); })); unload_called.WaitForNotification(); @@ -731,7 +750,7 @@ TEST_P(AspiredVersionsManagerTest, NoEventBus) { // The state manager thread won't be run automatically. options.manage_state_interval_micros = -1; options.env = Env::Default(); - options.aspired_version_policy.reset(new EagerLoadPolicy()); + options.aspired_version_policy.reset(new AvailabilityPreservingPolicy()); std::unique_ptr aspired_versions_manager; TF_ASSERT_OK(AspiredVersionsManager::Create(std::move(options), &aspired_versions_manager)); @@ -761,7 +780,11 @@ TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallySucceeds) { std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); @@ -784,7 +807,11 @@ TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallyFails) { std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); @@ -794,10 +821,11 @@ TEST_P(AspiredVersionsManagerTest, RetryOnLoadErrorFinallyFails) { EqualsServableState(error_state)); } -// Tests the interaction between AspiredVersionsManager and the EagerLoadPolicy. -// Specifically, we want to make sure that the manager will not try to unload a -// serving version that is no longer aspired if the new aspired version was not -// able to start serving. +// Tests the interaction between AspiredVersionsManager and the +// AvailabilityPreservingPolicy. +// Specifically, we want to make sure that the manager will not try to unload +// all serving versions that are no longer aspired if the new aspired version +// was not able to start serving. TEST_P(AspiredVersionsManagerTest, AspireErrorDontUnload) { const std::vector expected_before = {{kServableName, 0}, {kServableName, 1}, @@ -819,16 +847,24 @@ TEST_P(AspiredVersionsManagerTest, AspireErrorDontUnload) { std::move(aspired_versions)); HandlePendingAspiredVersionsRequests(); + // Will unload version 0. + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + // Will try to load version 7 and fail. InvokePolicyAndExecuteAction(); WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); } - // The same servables as before as we can't unload the current servables after - // the failure to load the new one. + // For kServableName, version 0 has been unloaded. For kServableName2, both + // versions should still be loaded. + const std::vector expected_after_first_load = { + {kServableName, 1}, {kServableName2, 0}, {kServableName2, 1}}; EXPECT_THAT(manager_->ListAvailableServableIds(), - UnorderedElementsAreArray(expected_before)); + UnorderedElementsAreArray(expected_after_first_load)); // Now successfully loading a new version should allow the older versions to // be unloaded. @@ -846,12 +882,8 @@ TEST_P(AspiredVersionsManagerTest, AspireErrorDontUnload) { WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); - // Will unload version 0 and 1. - InvokePolicyAndExecuteAction(); + // Will unload version 1. InvokePolicyAndExecuteAction(); - WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, - {kServableName, 0}, - {ServableState::ManagerState::kEnd}); WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, {kServableName, 1}, {ServableState::ManagerState::kEnd}); @@ -874,6 +906,9 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { manager_->GetAspiredVersionsCallback()(kServableName, std::move(first_aspired_versions)); HandlePendingAspiredVersionsRequests(); + + // Version 0 is unloaded and the new aspired version is loaded. + InvokePolicyAndExecuteAction(); InvokePolicyAndExecuteAction(); // Pin 'first_loader' in the manager by holding a handle to its servable. @@ -904,8 +939,11 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { // The following thread will block trying to unload the first loader, while we // hold the handle. std::unique_ptr unload_thread( - Env::Default()->StartThread(ThreadOptions(), "UnloadThread", - [&]() { InvokePolicyAndExecuteAction(); })); + Env::Default()->StartThread(ThreadOptions(), "UnloadThread", [&]() { + // Unload version 1 and the newly un-aspired version. + InvokePolicyAndExecuteAction(); + InvokePolicyAndExecuteAction(); + })); // Re-aspire the servable with a fresh loader. std::vector>> second_aspired_versions; @@ -957,7 +995,11 @@ TEST_P(AspiredVersionsManagerTest, manager_->GetAspiredVersionsCallback()(kServableName, std::move(first_aspired_versions)); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); @@ -1048,7 +1090,11 @@ TEST_P(AspiredVersionsManagerTest, UnaspireNewServableThenImmediatelyReaspire) { // second loader. FlushServables(); HandlePendingAspiredVersionsRequests(); - InvokePolicyAndExecuteAction(); + // Unload version 0 and load the new aspired version. Version 1 may or may not + // be unloaded (depending on whether load/unload thread pools are used). + for (int i = 0; i < kNumVersionsPerServable + 1; ++i) { + InvokePolicyAndExecuteAction(); + } second_load_called.WaitForNotification(); } diff --git a/tensorflow_serving/core/availability_preserving_policy.cc b/tensorflow_serving/core/availability_preserving_policy.cc new file mode 100644 index 00000000000..73287542a16 --- /dev/null +++ b/tensorflow_serving/core/availability_preserving_policy.cc @@ -0,0 +1,86 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/core/loader_harness.h" + +namespace tensorflow { +namespace serving { + +namespace { + +// Returns the ServableId with the lowest version, if any exists. +optional GetLowestServableId( + const std::vector& all_versions) { + const auto& iterator = + std::min_element(all_versions.begin(), all_versions.end(), + [](const AspiredServableStateSnapshot& a, + const AspiredServableStateSnapshot& b) { + return a.id.version < b.id.version; + }); + if (iterator == all_versions.end()) { + return nullopt; + } else { + return iterator->id; + } +} + +} // namespace + +optional +AvailabilityPreservingPolicy::GetNextAction( + const std::vector& all_versions) const { + // We first try to unload non-aspired versions (if any). + bool has_aspired = false; + bool has_aspired_serving = false; + std::vector unaspired_serving_versions; + for (const auto& version : all_versions) { + if (version.is_aspired) { + has_aspired = true; + if (version.state == LoaderHarness::State::kReady) { + has_aspired_serving = true; + } + } else if (version.state == LoaderHarness::State::kReady) { + unaspired_serving_versions.push_back(version); + } + } + + // If there is no aspired version, there is at least one aspired version + // that is ready, or there are more than one un-aspired versions that are + // ready, unload the lowest non-aspired version. + if (!has_aspired || has_aspired_serving || + unaspired_serving_versions.size() > 1) { + optional version_to_unload = + GetLowestServableId(unaspired_serving_versions); + if (version_to_unload) { + return {{Action::kUnload, version_to_unload.value()}}; + } + } + + // If there is at least one new aspired version, load the one with the + // highest version number. + optional highest_new_aspired_version_id = + GetHighestAspiredNewServableId(all_versions); + if (highest_new_aspired_version_id) { + VLOG(1) << "AvailabilityPreservingPolicy requesting to load servable " + << highest_new_aspired_version_id.value(); + return {{Action::kLoad, highest_new_aspired_version_id.value()}}; + } + + return nullopt; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/availability_preserving_policy.h b/tensorflow_serving/core/availability_preserving_policy.h new file mode 100644 index 00000000000..df68bae5850 --- /dev/null +++ b/tensorflow_serving/core/availability_preserving_policy.h @@ -0,0 +1,50 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_AVAILABILITY_PRESERVING_POLICY_H_ +#define TENSORFLOW_SERVING_CORE_AVAILABILITY_PRESERVING_POLICY_H_ + +#include + +#include "tensorflow_serving/core/aspired_version_policy.h" +#include "tensorflow_serving/core/loader_harness.h" +#include "tensorflow_serving/util/optional.h" + +namespace tensorflow { +namespace serving { + +// AspiredVersionPolicy that provides servable availability with the trade-off +// of temporary increased resource consumption while newly-aspired versions load +// followed by newly-un-aspired versions unloading. At the same time, it tries +// to minimize the resource usage caused by loading more versions than needed to +// maintain availability. +// +// Here is a detailed description of how this policy works: +// First, if there are any unaspired loaded versions, we unload the smallest +// such version, *unless* that is the only loaded version (to avoid compromising +// availability). +// Second, if there are no non-aspired versions we are permitted to unload, we +// load the aspired new version with the highest version number. +class AvailabilityPreservingPolicy final : public AspiredVersionPolicy { + public: + optional GetNextAction( + const std::vector& all_versions) + const override; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_AVAILABILITY_PRESERVING_POLICY_H_ diff --git a/tensorflow_serving/core/availability_preserving_policy_test.cc b/tensorflow_serving/core/availability_preserving_policy_test.cc new file mode 100644 index 00000000000..f632e448542 --- /dev/null +++ b/tensorflow_serving/core/availability_preserving_policy_test.cc @@ -0,0 +1,162 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/availability_preserving_policy.h" + +#include +#include "tensorflow_serving/core/servable_id.h" + +namespace tensorflow { +namespace serving { +namespace { + +// None unloadable (ready+unaspired); multiple loadable (new+aspired). Loads the +// highest loadable version. +TEST(AvailabilityPreservingPolicyTest, LoadsNewAspired) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); + + AvailabilityPreservingPolicy policy; + auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(5, action->id.version); +} + +// Both unloadable and loadable versions present. Unloading doesn't compromise +// availability. Opts to unload. +TEST(AvailabilityPreservingPolicyTest, UnloadsNonAspiredFirst) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 4}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); + + AvailabilityPreservingPolicy policy; + auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(3, action->id.version); +} + +// One unloadable. Nothing aspired so the goal is to unload all versions and +// lose availability. Unloads. +TEST(AvailabilityPreservingPolicyTest, UnloadsFirstNonAspiredWhenNoAspired) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kDisabled, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kError, false}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(2, action->id.version); +} + +// None unloadable or loadable. Takes no action. +TEST(AvailabilityPreservingPolicyTest, ReturnsNoActionWhenNone) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kLoading, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 5}, LoaderHarness::State::kDisabled, false}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +// One unloadable; none loadable. Unloading would compromise availability. Takes +// no action. +TEST(AvailabilityPreservingPolicyTest, DoesNotUnloadWhenOtherNotReady) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +// One unloadable; none loadable. Unloading would compromise availability. Takes +// no action. +TEST(AvailabilityPreservingPolicyTest, DoesNotUnloadWhenOtherInError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +// One unloadable; none loadable. Unloading doesn't compromise availability. +// Unloads. +TEST(AvailabilityPreservingPolicyTest, UnloadIfOtherReadyEvenIfLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(1, action->id.version); +} + +// One unloadable; none loadable. Unloading doesn't compromise availability. +// Unloads. +TEST(AvailabilityPreservingPolicyTest, UnloadIfOtherReadyEvenIfError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(1, action->id.version); +} + +// Multiple unloadable; none loadable. Availability is achieved despite no +// aspired versions being loaded, because one or more non-aspired versions are +// loaded. Unloading one non-aspired version doesn't compromise availability. +// Unloads the lowest such version. +TEST(AvailabilityPreservingPolicyTest, UnloadIfNoAspiredVersionsReady) { + std::vector versions; + versions.push_back({{"test", 2}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kLoading, true}); + + AvailabilityPreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(1, action->id.version); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/eager_load_policy.h b/tensorflow_serving/core/eager_load_policy.h index 40986cd5d3e..2a16c5c9d64 100644 --- a/tensorflow_serving/core/eager_load_policy.h +++ b/tensorflow_serving/core/eager_load_policy.h @@ -25,6 +25,8 @@ limitations under the License. namespace tensorflow { namespace serving { +// Deprecated. Use AvailabilityPreservePolicy instead. +// // AspiredVersionPolicy that loads any aspired versions of a servable before // unloading any no-longer-aspired versions. // diff --git a/tensorflow_serving/core/eager_unload_policy.h b/tensorflow_serving/core/eager_unload_policy.h index 761cfd2bfb3..c57d98f35fa 100644 --- a/tensorflow_serving/core/eager_unload_policy.h +++ b/tensorflow_serving/core/eager_unload_policy.h @@ -25,6 +25,8 @@ limitations under the License. namespace tensorflow { namespace serving { +// Deprecated. Please use ResourcePreservePolicy instead. +// // ServablePolicy that eagerly unloads any no-longer-aspired versions of a // servable stream and only after done unloading, loads newly aspired versions. // diff --git a/tensorflow_serving/core/resource_preserving_policy.cc b/tensorflow_serving/core/resource_preserving_policy.cc new file mode 100644 index 00000000000..51bb74c486a --- /dev/null +++ b/tensorflow_serving/core/resource_preserving_policy.cc @@ -0,0 +1,61 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/resource_preserving_policy.h" + +namespace tensorflow { +namespace serving { + +optional +ResourcePreservingPolicy::GetNextAction( + const std::vector& all_versions) const { + // First iterate over all_versions and find any in kReady that are no longer + // aspired. Unload the first if any. + for (const auto& version : all_versions) { + if (version.state == LoaderHarness::State::kReady && !version.is_aspired) { + VLOG(1) << "ResourcePreservingPolicy requesting to unload servable " + << version.id; + return {{Action::kUnload, version.id}}; + } + } + + // Second, see if there are any not-aspired versions that aren't in an end + // state (kDisabled or kError). If so, do nothing for now. + const bool not_aspired_not_finished = + std::any_of(all_versions.begin(), all_versions.end(), + [](const AspiredServableStateSnapshot& version) { + return !version.is_aspired && + version.state != LoaderHarness::State::kDisabled && + version.state != LoaderHarness::State::kError; + }); + if (not_aspired_not_finished) { + return nullopt; + } + + // Third and only if no action was found earlier, load the aspired new + // servable with the highest version if any. + optional highest_aspired_new_version_id = + GetHighestAspiredNewServableId(all_versions); + if (highest_aspired_new_version_id) { + VLOG(1) << "ResourcePreservingPolicy requesting to load servable " + << highest_aspired_new_version_id; + return {{Action::kLoad, highest_aspired_new_version_id.value()}}; + } + + return nullopt; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/core/resource_preserving_policy.h b/tensorflow_serving/core/resource_preserving_policy.h new file mode 100644 index 00000000000..1f4f0364dc4 --- /dev/null +++ b/tensorflow_serving/core/resource_preserving_policy.h @@ -0,0 +1,52 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_RESOURCE_PRESERVING_POLICY_H_ +#define TENSORFLOW_SERVING_CORE_RESOURCE_PRESERVING_POLICY_H_ + +#include + +#include "tensorflow_serving/core/aspired_version_policy.h" +#include "tensorflow_serving/core/loader_harness.h" +#include "tensorflow_serving/util/optional.h" + +namespace tensorflow { +namespace serving { + +// ServablePolicy that eagerly unloads any no-longer-aspired versions of a +// servable stream and only after done unloading, loads newly aspired versions +// in the order of descending version number. +// +// This policy minimizes resource consumption with the trade-off of temporary +// servable unavailability while all old versions unload followed by the new +// versions loading. +// +// Servables with a single version consuming the majority of their host's +// resources must use this policy to prevent deadlock. Other typical use-cases +// will be for multi-servable environments where clients can tolerate brief +// interruptions to a single servable's availability on a replica. +// +// NB: This policy does not in any way solve cross-replica availability. +class ResourcePreservingPolicy final : public AspiredVersionPolicy { + public: + optional GetNextAction( + const std::vector& all_versions) + const override; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_RESOURCE_PRESERVING_POLICY_H_ diff --git a/tensorflow_serving/core/resource_preserving_policy_test.cc b/tensorflow_serving/core/resource_preserving_policy_test.cc new file mode 100644 index 00000000000..19467ff4d3a --- /dev/null +++ b/tensorflow_serving/core/resource_preserving_policy_test.cc @@ -0,0 +1,123 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/core/resource_preserving_policy.h" + +#include +#include "tensorflow_serving/core/servable_id.h" + +namespace tensorflow { +namespace serving { +namespace { + +// Test that the first ready and non-aspired version is unloaded first. +TEST(ResourcePreservingPolicyTest, UnloadsFirstNonAspired) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); + versions.push_back({{"test", 4}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 5}, LoaderHarness::State::kReady, false}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); + EXPECT_EQ(3, action->id.version); +} + +// Test that the aspired new version with the highest version is loaded when +// there are none to unload. +TEST(ResourcePreservingPolicyTest, LoadsFirstAspiredWhenNoneToUnload) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kDisabled, false}); + versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(5, action->id.version); +} + +// Test that no action is returned (empty optional) when there are no versions +// needing loading or unloading. +TEST(ResourcePreservingPolicyTest, ReturnsNoActionWhenNone) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); + versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); + versions.push_back({{"test", 3}, LoaderHarness::State::kLoading, true}); + versions.push_back({{"test", 4}, LoaderHarness::State::kUnloading, false}); + versions.push_back({{"test", 5}, LoaderHarness::State::kDisabled, false}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(ResourcePreservingPolicyTest, DoesNotLoadWhenOthersStillUnloading) { + std::vector versions; + versions.push_back( + {{"test", 1}, LoaderHarness::State::kUnloadRequested, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(ResourcePreservingPolicyTest, LoadIfUnaspiredIsError) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(2, action->id.version); +} + +TEST(ResourcePreservingPolicyTest, ErrorAndUnloadRequestedPreventLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back( + {{"test", 2}, LoaderHarness::State::kUnloadRequested, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + EXPECT_FALSE(action); +} + +TEST(ResourcePreservingPolicyTest, ErrorAndDisabledAllowLoading) { + std::vector versions; + versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); + versions.push_back({{"test", 2}, LoaderHarness::State::kDisabled, false}); + versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); + + ResourcePreservingPolicy policy; + const auto action = policy.GetNextAction(versions); + ASSERT_TRUE(action); + EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); + EXPECT_EQ(3, action->id.version); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From b3140b5b4030fdf44290ab1d3aec3d631313a32d Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 19 Jan 2017 12:42:12 -0800 Subject: [PATCH 0141/8554] Regenerate prediction_service_pb2.py auto-generated python proto file to fix an issue with optimized version of protobuf 3.1.0. Change: 144996116 --- .../apis/prediction_service_pb2.py | 185 ++++++++++-------- 1 file changed, 104 insertions(+), 81 deletions(-) diff --git a/tensorflow_serving/apis/prediction_service_pb2.py b/tensorflow_serving/apis/prediction_service_pb2.py index a2813d44fb1..7af1bb1cf92 100644 --- a/tensorflow_serving/apis/prediction_service_pb2.py +++ b/tensorflow_serving/apis/prediction_service_pb2.py @@ -16,7 +16,7 @@ # source: tensorflow_serving/apis/prediction_service.proto import sys -_b = sys.version_info[0] < 3 and (lambda x: x) or (lambda x: x.encode('latin1')) +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) from google.protobuf import descriptor as _descriptor from google.protobuf import message as _message from google.protobuf import reflection as _reflection @@ -26,118 +26,141 @@ _sym_db = _symbol_database.Default() -from tensorflow_serving.apis import predict_pb2 as third__party_dot_tensorflow__serving_dot_apis_dot_predict__pb2 + +from tensorflow_serving.apis import predict_pb2 as tensorflow__serving_dot_apis_dot_predict__pb2 + DESCRIPTOR = _descriptor.FileDescriptor( - name='tensorflow_serving/apis/prediction_service.proto', - package='tensorflow.serving', - syntax='proto3', - serialized_pb=_b( - '\n=0.15.0.""" + """PredictionService provides access to machine-learned models loaded by + model_servers. + """ + def Predict(self, request, context): + """Predict -- provides access to loaded TensorFlow model. """ context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) class BetaPredictionServiceStub(object): - """PredictionService provides basic machine learning methods. + """The Beta API is deprecated for 0.15.0 and later. - TODO(b/28599843): Decide whether to keep the separate services in addition to - this combined service. + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This class was generated + only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" + """PredictionService provides access to machine-learned models loaded by + model_servers. """ - - def Predict(self, - request, - timeout, - metadata=None, - with_call=False, - protocol_options=None): - """Predict. + def Predict(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """Predict -- provides access to loaded TensorFlow model. """ raise NotImplementedError() - Predict.future = None -def beta_create_PredictionService_server(servicer, - pool=None, - pool_size=None, - default_timeout=None, - maximum_timeout=None): +def beta_create_PredictionService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This function was + generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" request_deserializers = { - ('tensorflow.serving.PredictionService', 'Predict'): - third__party_dot_tensorflow__serving_dot_apis_dot_predict__pb2. - PredictRequest.FromString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, } response_serializers = { - ('tensorflow.serving.PredictionService', 'Predict'): - third__party_dot_tensorflow__serving_dot_apis_dot_predict__pb2. - PredictResponse.SerializeToString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, } method_implementations = { - ('tensorflow.serving.PredictionService', 'Predict'): - face_utilities.unary_unary_inline(servicer.Predict), + ('tensorflow.serving.PredictionService', 'Predict'): face_utilities.unary_unary_inline(servicer.Predict), } - server_options = beta_implementations.server_options( - request_deserializers=request_deserializers, - response_serializers=response_serializers, - thread_pool=pool, - thread_pool_size=pool_size, - default_timeout=default_timeout, - maximum_timeout=maximum_timeout) - return beta_implementations.server( - method_implementations, options=server_options) - - -def beta_create_PredictionService_stub(channel, - host=None, - metadata_transformer=None, - pool=None, - pool_size=None): + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + + +def beta_create_PredictionService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This function was + generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" request_serializers = { - ('tensorflow.serving.PredictionService', 'Predict'): - third__party_dot_tensorflow__serving_dot_apis_dot_predict__pb2. - PredictRequest.SerializeToString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, } response_deserializers = { - ('tensorflow.serving.PredictionService', 'Predict'): - third__party_dot_tensorflow__serving_dot_apis_dot_predict__pb2. - PredictResponse.FromString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, + } + cardinalities = { + 'Predict': cardinality.Cardinality.UNARY_UNARY, } - cardinalities = {'Predict': cardinality.Cardinality.UNARY_UNARY,} - stub_options = beta_implementations.stub_options( - host=host, - metadata_transformer=metadata_transformer, - request_serializers=request_serializers, - response_deserializers=response_deserializers, - thread_pool=pool, - thread_pool_size=pool_size) - return beta_implementations.dynamic_stub( - channel, - 'tensorflow.serving.PredictionService', - cardinalities, - options=stub_options) -# @@protoc_insertion_point(module_scope) + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.PredictionService', cardinalities, options=stub_options) From ba28bf6cfe7bf10c78abb50aaf1a80677b94cc5c Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 19 Jan 2017 12:54:05 -0800 Subject: [PATCH 0142/8554] Update workspace files to match the new location of TensorFlow's check_version function. Change: 144997521 --- WORKSPACE | 2 +- tensorflow_serving/workspace.bzl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index eae521aa96a..56cc78339c8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -10,5 +10,5 @@ load('//tensorflow_serving:workspace.bzl', 'tf_serving_workspace') tf_serving_workspace() # Specify the minimum required bazel version. -load("@org_tensorflow//tensorflow:tensorflow.bzl", "check_version") +load("@org_tensorflow//tensorflow:workspace.bzl", "check_version") check_version("0.4.2") diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index 58b276f1f5b..93d9a79fee2 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -12,7 +12,7 @@ def tf_serving_workspace(): path = "tf_models/inception", ) - tf_workspace() + tf_workspace(path_prefix = "", tf_repo_name = "org_tensorflow") # ===== gRPC dependencies ===== native.bind( From e3479d062fe203ed404f2ccf636a39a56fa065c4 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 19 Jan 2017 13:40:13 -0800 Subject: [PATCH 0143/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index dbe5e17e2ed..b00fc538638 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit dbe5e17e2ed307e86e1a6e79e558ec3e335d46fc +Subproject commit b00fc538638f87ac45be9105057b9865f0f9418b diff --git a/tf_models b/tf_models index 12f279d6f4c..a689e124a83 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 12f279d6f4cb33574bc20109b41eb8a59f40cfd1 +Subproject commit a689e124a831be313fe9a219ca367477490ad89e From 798da1158453d1221325325727a267a799344d8c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 25 Jan 2017 17:25:38 -0800 Subject: [PATCH 0144/8554] Add GetModelMetadata API to model servers. Change: 145619100 --- tensorflow_serving/apis/BUILD | 28 ++- .../apis/get_model_metadata.proto | 29 +++ .../apis/prediction_service.proto | 5 + .../apis/prediction_service_pb2.py | 50 +++- tensorflow_serving/core/BUILD | 4 +- .../aspired_versions_manager_benchmark.cc | 6 +- .../aspired_versions_manager_builder_test.cc | 5 +- .../core/servable_state_monitor.cc | 6 + .../core/servable_state_monitor.h | 2 +- .../core/server_request_logger.cc | 10 + .../core/server_request_logger_test.cc | 12 + tensorflow_serving/core/test_util/BUILD | 1 + tensorflow_serving/model_servers/BUILD | 3 +- tensorflow_serving/model_servers/main.cc | 40 +++- .../model_servers/test_util/BUILD | 2 +- .../test_util/server_core_test_util.cc | 4 +- tensorflow_serving/servables/tensorflow/BUILD | 61 ++++- .../tensorflow/get_model_metadata_impl.cc | 92 ++++++++ .../tensorflow/get_model_metadata_impl.h | 36 +++ .../get_model_metadata_impl_test.cc | 218 ++++++++++++++++++ .../servables/tensorflow/predict_impl_test.cc | 4 +- .../servables/tensorflow/simple_servers.cc | 5 +- 22 files changed, 592 insertions(+), 31 deletions(-) create mode 100644 tensorflow_serving/apis/get_model_metadata.proto create mode 100644 tensorflow_serving/servables/tensorflow/get_model_metadata_impl.cc create mode 100644 tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h create mode 100644 tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 10cbf6d9454..58a20eb3359 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -23,6 +23,28 @@ filegroup( load("//tensorflow_serving:serving.bzl", "serving_proto_library") load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") +serving_proto_library( + name = "get_model_metadata_proto", + srcs = ["get_model_metadata.proto"], + cc_api_version = 2, + go_api_version = 2, + java_api_version = 2, + deps = [ + ":model_proto", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf//:cc_wkt_protos", + ], +) + +serving_proto_library_py( + name = "get_model_metadata_proto_py_pb2", + srcs = ["get_model_metadata.proto"], + proto_library = "get_model_metadata_proto", + deps = [ + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + serving_proto_library( name = "model_proto", srcs = ["model.proto"], @@ -72,6 +94,7 @@ serving_proto_library( go_api_version = 2, java_api_version = 2, deps = [ + ":get_model_metadata_proto", ":predict_proto", ], ) @@ -79,5 +102,8 @@ serving_proto_library( py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], - deps = [":predict_proto_py_pb2"], + deps = [ + ":get_model_metadata_proto_py_pb2", + ":predict_proto_py_pb2", + ], ) diff --git a/tensorflow_serving/apis/get_model_metadata.proto b/tensorflow_serving/apis/get_model_metadata.proto new file mode 100644 index 00000000000..d9773e9ae3a --- /dev/null +++ b/tensorflow_serving/apis/get_model_metadata.proto @@ -0,0 +1,29 @@ +syntax = "proto3"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +import "google/protobuf/any.proto"; +import "tensorflow/core/protobuf/meta_graph.proto"; +import "tensorflow_serving/apis/model.proto"; + +// Message returned for "signature_def" field. +message SignatureDefMap { + map signature_def = 1; +}; + +message GetModelMetadataRequest { + // Model Specification indicating which model we are querying for metadata. + ModelSpec model_spec = 1; + // Metadata fields to get. Currently supported: "signature_def". + repeated string metadata_field = 2; +} + +message GetModelMetadataResponse { + // Model Specification indicating which model this metadata belongs to. + ModelSpec model_spec = 1; + // Map of metadata field name to metadata field. The options for metadata + // field name are listed in GetModelMetadataRequest. Currently supported: + // "signature_def". + map metadata = 2; +} diff --git a/tensorflow_serving/apis/prediction_service.proto b/tensorflow_serving/apis/prediction_service.proto index 58df05af119..3f978b2c540 100644 --- a/tensorflow_serving/apis/prediction_service.proto +++ b/tensorflow_serving/apis/prediction_service.proto @@ -3,6 +3,7 @@ syntax = "proto3"; package tensorflow.serving; option cc_enable_arenas = true; +import "tensorflow_serving/apis/get_model_metadata.proto"; import "tensorflow_serving/apis/predict.proto"; // open source marker; do not remove @@ -11,4 +12,8 @@ import "tensorflow_serving/apis/predict.proto"; service PredictionService { // Predict -- provides access to loaded TensorFlow model. rpc Predict(PredictRequest) returns (PredictResponse); + + // GetModelMetadata - provides access to metadata for loaded models. + rpc GetModelMetadata(GetModelMetadataRequest) + returns (GetModelMetadataResponse); } diff --git a/tensorflow_serving/apis/prediction_service_pb2.py b/tensorflow_serving/apis/prediction_service_pb2.py index 7af1bb1cf92..3130f0a4f38 100644 --- a/tensorflow_serving/apis/prediction_service_pb2.py +++ b/tensorflow_serving/apis/prediction_service_pb2.py @@ -27,6 +27,7 @@ _sym_db = _symbol_database.Default() +from tensorflow_serving.apis import get_model_metadata_pb2 as tensorflow__serving_dot_apis_dot_get__model__metadata__pb2 from tensorflow_serving.apis import predict_pb2 as tensorflow__serving_dot_apis_dot_predict__pb2 @@ -34,9 +35,9 @@ name='tensorflow_serving/apis/prediction_service.proto', package='tensorflow.serving', syntax='proto3', - serialized_pb=_b('\n0tensorflow_serving/apis/prediction_service.proto\x12\x12tensorflow.serving\x1a%tensorflow_serving/apis/predict.proto2g\n\x11PredictionService\x12R\n\x07Predict\x12\".tensorflow.serving.PredictRequest\x1a#.tensorflow.serving.PredictResponseB\x03\xf8\x01\x01\x62\x06proto3') + serialized_pb=_b('\n0tensorflow_serving/apis/prediction_service.proto\x12\x12tensorflow.serving\x1a\x30tensorflow_serving/apis/get_model_metadata.proto\x1a%tensorflow_serving/apis/predict.proto2\xd6\x01\n\x11PredictionService\x12R\n\x07Predict\x12\".tensorflow.serving.PredictRequest\x1a#.tensorflow.serving.PredictResponse\x12m\n\x10GetModelMetadata\x12+.tensorflow.serving.GetModelMetadataRequest\x1a,.tensorflow.serving.GetModelMetadataResponseB\x03\xf8\x01\x01\x62\x06proto3') , - dependencies=[tensorflow__serving_dot_apis_dot_predict__pb2.DESCRIPTOR,]) + dependencies=[tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_predict__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -54,7 +55,8 @@ class PredictionServiceStub(object): - """PredictionService provides access to machine-learned models loaded by + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by model_servers. """ @@ -69,10 +71,16 @@ def __init__(self, channel): request_serializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, response_deserializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, ) + self.GetModelMetadata = channel.unary_unary( + '/tensorflow.serving.PredictionService/GetModelMetadata', + request_serializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.FromString, + ) class PredictionServiceServicer(object): - """PredictionService provides access to machine-learned models loaded by + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by model_servers. """ @@ -83,6 +91,13 @@ def Predict(self, request, context): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def GetModelMetadata(self, request, context): + """GetModelMetadata - provides access to metadata for loaded models. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_PredictionServiceServicer_to_server(servicer, server): rpc_method_handlers = { @@ -91,6 +106,11 @@ def add_PredictionServiceServicer_to_server(servicer, server): request_deserializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, response_serializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, ), + 'GetModelMetadata': grpc.unary_unary_rpc_method_handler( + servicer.GetModelMetadata, + request_deserializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'tensorflow.serving.PredictionService', rpc_method_handlers) @@ -103,13 +123,18 @@ class BetaPredictionServiceServicer(object): It is recommended to use the GA API (classes and functions in this file not marked beta) for all further purposes. This class was generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """PredictionService provides access to machine-learned models loaded by + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by model_servers. """ def Predict(self, request, context): """Predict -- provides access to loaded TensorFlow model. """ context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + def GetModelMetadata(self, request, context): + """GetModelMetadata - provides access to metadata for loaded models. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) class BetaPredictionServiceStub(object): @@ -118,7 +143,8 @@ class BetaPredictionServiceStub(object): It is recommended to use the GA API (classes and functions in this file not marked beta) for all further purposes. This class was generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """PredictionService provides access to machine-learned models loaded by + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by model_servers. """ def Predict(self, request, timeout, metadata=None, with_call=False, protocol_options=None): @@ -126,6 +152,11 @@ def Predict(self, request, timeout, metadata=None, with_call=False, protocol_opt """ raise NotImplementedError() Predict.future = None + def GetModelMetadata(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """GetModelMetadata - provides access to metadata for loaded models. + """ + raise NotImplementedError() + GetModelMetadata.future = None def beta_create_PredictionService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): @@ -135,12 +166,15 @@ def beta_create_PredictionService_server(servicer, pool=None, pool_size=None, de file not marked beta) for all further purposes. This function was generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" request_deserializers = { + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.FromString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, } response_serializers = { + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.SerializeToString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, } method_implementations = { + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): face_utilities.unary_unary_inline(servicer.GetModelMetadata), ('tensorflow.serving.PredictionService', 'Predict'): face_utilities.unary_unary_inline(servicer.Predict), } server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) @@ -154,13 +188,17 @@ def beta_create_PredictionService_stub(channel, host=None, metadata_transformer= file not marked beta) for all further purposes. This function was generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" request_serializers = { + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.SerializeToString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, } response_deserializers = { + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.FromString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, } cardinalities = { + 'GetModelMetadata': cardinality.Cardinality.UNARY_UNARY, 'Predict': cardinality.Cardinality.UNARY_UNARY, } stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.PredictionService', cardinalities, options=stub_options) +# @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index b8857ab5960..0a71d1d20f3 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -365,7 +365,7 @@ cc_test( srcs = ["aspired_versions_manager_builder_test.cc"], deps = [ ":aspired_versions_manager_builder", - ":eager_load_policy", + ":availability_preserving_policy", ":servable_data", ":servable_handle", ":servable_state_monitor", @@ -534,7 +534,7 @@ cc_test( deps = [ ":aspired_version_policy", ":aspired_versions_manager", - ":eager_load_policy", + ":availability_preserving_policy", ":loader", ":manager", ":servable_data", diff --git a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc index f078f4d3a9f..6a73ec9edee 100644 --- a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc +++ b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc @@ -41,7 +41,7 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/aspired_version_policy.h" #include "tensorflow_serving/core/aspired_versions_manager.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/manager.h" #include "tensorflow_serving/core/servable_data.h" @@ -74,7 +74,7 @@ class BenchmarkState { AspiredVersionsManager::Options options; // Do policy thread won't be run automatically. options.manage_state_interval_micros = -1; - options.aspired_version_policy.reset(new EagerLoadPolicy()); + options.aspired_version_policy.reset(new AvailabilityPreservingPolicy()); TF_CHECK_OK(AspiredVersionsManager::Create(std::move(options), &manager_)); } @@ -304,7 +304,7 @@ static void BM_GetServableHandle(const int iters) { AspiredVersionsManager::Options options; // Do policy thread won't be run automatically. options.manage_state_interval_micros = -1; - options.aspired_version_policy.reset(new EagerLoadPolicy()); + options.aspired_version_policy.reset(new AvailabilityPreservingPolicy()); std::unique_ptr manager; TF_CHECK_OK(AspiredVersionsManager::Create(std::move(options), &manager)); auto aspired_versions_callback = manager->GetAspiredVersionsCallback(); diff --git a/tensorflow_serving/core/aspired_versions_manager_builder_test.cc b/tensorflow_serving/core/aspired_versions_manager_builder_test.cc index c9da9dd712c..4fe48b9a9f5 100644 --- a/tensorflow_serving/core/aspired_versions_manager_builder_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_builder_test.cc @@ -18,7 +18,7 @@ limitations under the License. #include #include #include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state_monitor.h" @@ -46,7 +46,8 @@ class AspiredVersionsManagerBuilderTest : public ::testing::Test { servable_state_monitor_(servable_event_bus_.get()) { AspiredVersionsManagerBuilder::Options manager_options; manager_options.servable_event_bus = servable_event_bus_.get(); - manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + manager_options.aspired_version_policy.reset( + new AvailabilityPreservingPolicy()); TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( std::move(manager_options), &builder_)); } diff --git a/tensorflow_serving/core/servable_state_monitor.cc b/tensorflow_serving/core/servable_state_monitor.cc index 88a778f0985..93f295b5a9d 100644 --- a/tensorflow_serving/core/servable_state_monitor.cc +++ b/tensorflow_serving/core/servable_state_monitor.cc @@ -115,6 +115,12 @@ ServableStateMonitor::ServableStateMonitor(EventBus* bus, this->HandleEvent(state_and_time); })) {} +ServableStateMonitor::~ServableStateMonitor() { + // Halt event handling first, before tearing down state that event handling + // may access such as 'servable_state_notification_requests_'. + bus_subscription_ = nullptr; +} + optional ServableStateMonitor::GetStateAndTimeInternal( const ServableId& servable_id) const { diff --git a/tensorflow_serving/core/servable_state_monitor.h b/tensorflow_serving/core/servable_state_monitor.h index e4c4f08c1ba..f496d2c19bf 100644 --- a/tensorflow_serving/core/servable_state_monitor.h +++ b/tensorflow_serving/core/servable_state_monitor.h @@ -77,7 +77,7 @@ class ServableStateMonitor { explicit ServableStateMonitor(EventBus* bus, const Options& options = Options()); - virtual ~ServableStateMonitor() = default; + virtual ~ServableStateMonitor(); // Returns the current state of one servable, or nullopt if that servable is // not being tracked. diff --git a/tensorflow_serving/core/server_request_logger.cc b/tensorflow_serving/core/server_request_logger.cc index a292c9e8d80..9e5267e3239 100644 --- a/tensorflow_serving/core/server_request_logger.cc +++ b/tensorflow_serving/core/server_request_logger.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow_serving/core/server_request_logger.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/gtl/map_util.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/core/logging.pb.h" @@ -46,10 +47,19 @@ Status ServerRequestLogger::Update( if (!logging_config_map.empty() && !request_logger_creator_) { return errors::InvalidArgument("No request-logger-creator provided."); } + std::set filename_prefixes; std::unique_ptr request_logger_map(new RequestLoggerMap()); for (const auto& model_and_logging_config : logging_config_map) { auto& request_logger = (*request_logger_map)[model_and_logging_config.first]; + const string& filename_prefix = + model_and_logging_config.second.log_collector_config() + .filename_prefix(); + if (!gtl::InsertIfNotPresent(&filename_prefixes, filename_prefix)) { + // Logs for each model is supposed to be separated from each other. + return errors::InvalidArgument( + "Duplicate LogCollectorConfig::filename_prefix(): ", filename_prefix); + } TF_RETURN_IF_ERROR(request_logger_creator_(model_and_logging_config.second, &request_logger)); } diff --git a/tensorflow_serving/core/server_request_logger_test.cc b/tensorflow_serving/core/server_request_logger_test.cc index b9bd83d0ad7..6cec506207d 100644 --- a/tensorflow_serving/core/server_request_logger_test.cc +++ b/tensorflow_serving/core/server_request_logger_test.cc @@ -40,6 +40,7 @@ namespace serving { namespace { using ::testing::_; +using ::testing::HasSubstr; using ::testing::Invoke; using ::testing::NiceMock; @@ -113,6 +114,17 @@ TEST_F(ServerRequestLoggerTest, AbsentModel) { EXPECT_EQ(0, log_collector_map_["/file/model0"]->collect_count()); } +TEST_F(ServerRequestLoggerTest, DuplicateFilenamePrefix) { + std::map model_logging_configs; + model_logging_configs.insert(CreateLoggingConfigForModel("model0")); + const std::pair model_and_logging_config = + CreateLoggingConfigForModel("model0"); + model_logging_configs.insert({"model1", model_and_logging_config.second}); + const auto status = server_request_logger_->Update(model_logging_configs); + EXPECT_THAT(status.error_message(), + HasSubstr("Duplicate LogCollectorConfig::filename_prefix()")); +} + TEST_F(ServerRequestLoggerTest, MultipleModels) { std::map model_logging_configs; model_logging_configs.insert(CreateLoggingConfigForModel("model0")); diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index b1d86bbf8b6..0b7c3a4a186 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -150,6 +150,7 @@ cc_library( name = "mock_log_collector", testonly = 1, hdrs = ["mock_log_collector.h"], + visibility = ["//visibility:public"], deps = [ "//external:gtest", "//tensorflow_serving/core:log_collector", diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index dccb14c6101..a5c16c2baf7 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -110,6 +110,7 @@ SUPPORTED_TENSORFLOW_OPS = [ TENSORFLOW_DEPS = [ "@org_tensorflow//tensorflow/core:tensorflow", + "//tensorflow_serving/servables/tensorflow:get_model_metadata_impl", "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:predict_impl", @@ -130,7 +131,7 @@ cc_binary( "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", - "//tensorflow_serving/core:eager_load_policy", + "//tensorflow_serving/core:availability_preserving_policy", "@grpc//:grpc++", ] + TENSORFLOW_DEPS + SUPPORTED_TENSORFLOW_OPS, ) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 2a6002ed456..e626d1c4127 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -21,8 +21,9 @@ limitations under the License. // // ModelServer prioritizes easy invocation over flexibility, // and thus serves a statically configured set of models. New versions of these -// models will be loaded and managed over time using the EagerLoadPolicy at: -// tensorflow_serving/core/eager_load_policy.h. +// models will be loaded and managed over time using the +// AvailabilityPreservingPolicy at: +// tensorflow_serving/core/availability_preserving_policy.h. // by AspiredVersionsManager at: // tensorflow_serving/core/aspired_versions_manager.h // @@ -66,20 +67,26 @@ limitations under the License. #include "tensorflow_serving/apis/prediction_service.grpc.pb.h" #include "tensorflow_serving/apis/prediction_service.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" #include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" +namespace grpc { +class ServerCompletionQueue; +} // namespace grpc + using tensorflow::serving::AspiredVersionsManager; using tensorflow::serving::AspiredVersionPolicy; +using tensorflow::serving::AvailabilityPreservingPolicy; using tensorflow::serving::BatchingParameters; -using tensorflow::serving::EagerLoadPolicy; using tensorflow::serving::EventBus; using tensorflow::serving::FileSystemStoragePathSourceConfig; using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy; using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy_Name; +using tensorflow::serving::GetModelMetadataImpl; using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; @@ -96,6 +103,8 @@ using grpc::ServerAsyncResponseWriter; using grpc::ServerBuilder; using grpc::ServerContext; using grpc::ServerCompletionQueue; +using tensorflow::serving::GetModelMetadataRequest; +using tensorflow::serving::GetModelMetadataResponse; using tensorflow::serving::PredictRequest; using tensorflow::serving::PredictResponse; using tensorflow::serving::PredictionService; @@ -147,7 +156,8 @@ class PredictionServiceImpl final : public PredictionService::Service { explicit PredictionServiceImpl(std::unique_ptr core, bool use_saved_model) : core_(std::move(core)), - predictor_(new TensorflowPredictor(use_saved_model)) {} + predictor_(new TensorflowPredictor(use_saved_model)), + use_saved_model_(use_saved_model) {} grpc::Status Predict(ServerContext* context, const PredictRequest* request, PredictResponse* response) override { @@ -159,9 +169,27 @@ class PredictionServiceImpl final : public PredictionService::Service { return status; } + grpc::Status GetModelMetadata(ServerContext* context, + const GetModelMetadataRequest* request, + GetModelMetadataResponse* response) override { + if (!use_saved_model_) { + return ToGRPCStatus(tensorflow::errors::InvalidArgument( + "GetModelMetadata API is only available when use_saved_model is " + "set to true")); + } + const grpc::Status status = + ToGRPCStatus(GetModelMetadataImpl::GetModelMetadata( + core_.get(), *request, response)); + if (!status.ok()) { + VLOG(1) << "GetModelMetadata failed: " << status.error_message(); + } + return status; + } + private: std::unique_ptr core_; std::unique_ptr predictor_; + bool use_saved_model_; }; void RunServer(int port, std::unique_ptr core, @@ -292,7 +320,7 @@ int main(int argc, char** argv) { options.custom_model_config_loader = &LoadCustomModelConfig; options.aspired_version_policy = - std::unique_ptr(new EagerLoadPolicy); + std::unique_ptr(new AvailabilityPreservingPolicy); options.file_system_poll_wait_seconds = file_system_poll_wait_seconds; std::unique_ptr core; diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 17a7df58832..64114270c99 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -57,7 +57,7 @@ cc_library( "//visibility:public", ], deps = [ - "//tensorflow_serving/core:eager_load_policy", + "//tensorflow_serving/core:availability_preserving_policy", "//tensorflow_serving/core:servable_id", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/model_servers:model_platform_types", diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 6f8dea33985..f6dc472d520 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -16,7 +16,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" @@ -62,7 +62,7 @@ ServerCore::Options ServerCoreTest::GetDefaultOptions() { // timing out in tests. options.num_initial_load_threads = options.num_load_threads; options.aspired_version_policy = - std::unique_ptr(new EagerLoadPolicy); + std::unique_ptr(new AvailabilityPreservingPolicy); options.custom_model_config_loader = []( const ::google::protobuf::Any& any, EventBus* event_bus, UniquePtrWithDeps* manager) -> Status { diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index b611a9450dc..49a6973d446 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -202,6 +202,7 @@ cc_test( ":session_bundle_config_proto", "//tensorflow_serving/core/test_util:test_main", "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", @@ -346,7 +347,7 @@ cc_library( ":session_bundle_source_adapter", ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:aspired_versions_manager_builder", - "//tensorflow_serving/core:eager_load_policy", + "//tensorflow_serving/core:availability_preserving_policy", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:manager", "//tensorflow_serving/core:source", @@ -400,6 +401,8 @@ cc_library( srcs = ["predict_impl.cc"], hdrs = ["predict_impl.h"], deps = [ + ":get_model_metadata_impl", + "//tensorflow_serving/apis:get_model_metadata_proto", "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/model_servers:server_core", @@ -412,6 +415,60 @@ cc_library( ], ) +cc_library( + name = "get_model_metadata_impl", + srcs = ["get_model_metadata_impl.cc"], + hdrs = ["get_model_metadata_impl.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//tensorflow_serving/apis:get_model_metadata_proto", + "//tensorflow_serving/core:servable_handle", + "//tensorflow_serving/model_servers:server_core", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "get_model_metadata_impl_test", + size = "medium", + srcs = ["get_model_metadata_impl_test.cc"], + data = [ + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + ], + deps = [ + ":get_model_metadata_impl", + ":saved_model_bundle_source_adapter_proto", + ":session_bundle_config_proto", + ":session_bundle_source_adapter_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/config:model_server_config_proto", + "//tensorflow_serving/config:platform_config_proto", + "//tensorflow_serving/core:aspired_version_policy", + "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/core:servable_handle", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:test", + "@protobuf//:cc_wkt_protos", + "@protobuf//:protobuf", + ], +) + cc_test( name = "predict_impl_test", size = "medium", @@ -425,7 +482,7 @@ cc_test( ], deps = [ ":predict_impl", - "//tensorflow_serving/core:eager_load_policy", + "//tensorflow_serving/core:availability_preserving_policy", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/model_servers:model_platform_types", "//tensorflow_serving/model_servers:platform_config_util", diff --git a/tensorflow_serving/servables/tensorflow/get_model_metadata_impl.cc b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl.cc new file mode 100644 index 00000000000..508a71f2106 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl.cc @@ -0,0 +1,92 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h" + +#include +#include +#include +#include +#include + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/contrib/session_bundle/bundle_shim.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow_serving/core/servable_handle.h" + +namespace tensorflow { +namespace serving { + +namespace { + +const string kSignatureDef = "signature_def"; + +Status ValidateGetModelMetadataRequest(const GetModelMetadataRequest& request) { + if (!request.has_model_spec()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Missing ModelSpec"); + } + if (request.metadata_field_size() == 0) { + return tensorflow::Status( + tensorflow::error::INVALID_ARGUMENT, + "GetModelMetadataRequest must specify at least one metadata_field"); + } + for (const auto& metadata_field : request.metadata_field()) { + if (metadata_field != kSignatureDef) { + return tensorflow::errors::InvalidArgument( + "Metadata field %s is not supported", metadata_field); + } + } + return tensorflow::Status::OK(); +} + +Status SavedModelGetSignatureDef(ServerCore* core, + const GetModelMetadataRequest& request, + GetModelMetadataResponse* response) { + ServableHandle bundle; + TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); + SignatureDefMap signature_def_map; + for (const auto& signature : bundle->meta_graph_def.signature_def()) { + (*signature_def_map.mutable_signature_def())[signature.first] = + signature.second; + } + auto model_spec = response->mutable_model_spec(); + model_spec->set_name(bundle.id().name); + model_spec->mutable_version()->set_value(bundle.id().version); + + (*response->mutable_metadata())[kSignatureDef].PackFrom(signature_def_map); + return tensorflow::Status::OK(); +} + +} // namespace + +Status GetModelMetadataImpl::GetModelMetadata( + ServerCore* core, const GetModelMetadataRequest& request, + GetModelMetadataResponse* response) { + TF_RETURN_IF_ERROR(ValidateGetModelMetadataRequest(request)); + for (const auto& metadata_field : request.metadata_field()) { + if (metadata_field == kSignatureDef) { + TF_RETURN_IF_ERROR(SavedModelGetSignatureDef(core, request, response)); + } else { + return tensorflow::errors::InvalidArgument( + "MetadataField %s is not supported", metadata_field); + } + } + return tensorflow::Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h new file mode 100644 index 00000000000..c5cd7d0cae1 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h @@ -0,0 +1,36 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_GET_MODEL_METADATA_IMPL_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_GET_MODEL_METADATA_IMPL_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/get_model_metadata.pb.h" +#include "tensorflow_serving/model_servers/server_core.h" + +namespace tensorflow { +namespace serving { + +class GetModelMetadataImpl { + public: + static Status GetModelMetadata(ServerCore* core, + const GetModelMetadataRequest& request, + GetModelMetadataResponse* response); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_GET_MODEL_METADATA_IMPL_H_ diff --git a/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc new file mode 100644 index 00000000000..91c3060af83 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc @@ -0,0 +1,218 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h" + +#include +#include +#include + +#include "google/protobuf/wrappers.pb.h" +#include "google/protobuf/map.h" +#include +#include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/contrib/session_bundle/bundle_shim.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/error_codes.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/protobuf/meta_graph.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/config/model_server_config.pb.h" +#include "tensorflow_serving/config/platform_config.pb.h" +#include "tensorflow_serving/core/aspired_version_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; +const string kSignatureDef = "signature_def"; +constexpr char kSavedModelBundlePath[] = + "cc/saved_model/testdata/half_plus_two"; + +class GetModelMetadataImplTest : public ::testing::TestWithParam { + public: + static void SetUpTestCase() { + const string session_bundle_path = test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/half_plus_two"); + TF_ASSERT_OK(CreateServerCore(session_bundle_path, false, &server_core_)); + + const string saved_model_path = + io::JoinPath(testing::TensorFlowSrcRoot(), kSavedModelBundlePath); + TF_ASSERT_OK( + CreateServerCore(saved_model_path, true, &saved_model_server_core_)); + } + + static void TearDownTestCase() { + server_core_.reset(); + saved_model_server_core_.reset(); + } + + protected: + static Status CreateServerCore(const string& model_path, + bool saved_model_on_disk, + std::unique_ptr* server_core) { + ModelServerConfig config; + auto model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(model_path); + model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + options.model_server_config = config; + options.platform_config_map = + CreateTensorFlowPlatformConfigMap(SessionBundleConfig(), true); + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + return ServerCore::Create(std::move(options), server_core); + } + + ServerCore* GetServerCore() { + if (GetParam()) { + return saved_model_server_core_.get(); + } + return server_core_.get(); + } + + private: + static std::unique_ptr server_core_; + static std::unique_ptr saved_model_server_core_; +}; + +std::unique_ptr GetModelMetadataImplTest::server_core_; +std::unique_ptr GetModelMetadataImplTest::saved_model_server_core_; + +SignatureDefMap GetSignatureDefMap(ServerCore* server_core, + const ModelSpec& model_spec) { + SignatureDefMap signature_def_map; + ServableHandle bundle; + TF_EXPECT_OK(server_core->GetServableHandle(model_spec, &bundle)); + for (const auto& signature : bundle->meta_graph_def.signature_def()) { + (*signature_def_map.mutable_signature_def())[signature.first] = + signature.second; + } + return signature_def_map; +} + +TEST_P(GetModelMetadataImplTest, EmptyOrInvalidMetadataFieldList) { + GetModelMetadataRequest request; + GetModelMetadataResponse response; + + // Empty metadata field list is invalid. + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + GetModelMetadataImpl::GetModelMetadata(GetServerCore(), request, + &response) + .code()); + request.add_metadata_field("some_stuff"); + + // Field enum is outside of valid range. + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + GetModelMetadataImpl::GetModelMetadata(GetServerCore(), request, + &response) + .code()); +} + +TEST_P(GetModelMetadataImplTest, MissingOrEmptyModelSpec) { + GetModelMetadataRequest request; + GetModelMetadataResponse response; + + request.add_metadata_field(kSignatureDef); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + GetModelMetadataImpl::GetModelMetadata(GetServerCore(), request, + &response) + .code()); + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->clear_name(); + + // Model name is not specified. + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + GetModelMetadataImpl::GetModelMetadata(GetServerCore(), request, + &response) + .code()); + + // Model name is wrong, not found. + model_spec->set_name("test"); + EXPECT_EQ(tensorflow::error::NOT_FOUND, + GetModelMetadataImpl::GetModelMetadata(GetServerCore(), request, + &response) + .code()); +} + +TEST_P(GetModelMetadataImplTest, ReturnsSignaturesForValidModel) { + GetModelMetadataRequest request; + GetModelMetadataResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + request.add_metadata_field(kSignatureDef); + + TF_EXPECT_OK(GetModelMetadataImpl::GetModelMetadata(GetServerCore(), request, + &response)); + EXPECT_THAT(response.model_spec(), + test_util::EqualsProto(request.model_spec())); + EXPECT_EQ(response.metadata_size(), 1); + SignatureDefMap received_signature_def_map; + response.metadata().at(kSignatureDef).UnpackTo(&received_signature_def_map); + + SignatureDefMap expected_signature_def_map = + GetSignatureDefMap(GetServerCore(), request.model_spec()); + EXPECT_THAT(response.model_spec(), + test_util::EqualsProto(request.model_spec())); + + EXPECT_EQ(expected_signature_def_map.signature_def().size(), + received_signature_def_map.signature_def().size()); + if (GetParam()) { + EXPECT_THAT( + expected_signature_def_map.signature_def().at(kRegressMethodName), + test_util::EqualsProto( + received_signature_def_map.signature_def().at(kRegressMethodName))); + } else { + EXPECT_THAT(expected_signature_def_map.signature_def().at("regress"), + test_util::EqualsProto( + received_signature_def_map.signature_def().at("regress"))); + } + EXPECT_THAT( + expected_signature_def_map.signature_def().at( + kDefaultServingSignatureDefKey), + test_util::EqualsProto(received_signature_def_map.signature_def().at( + kDefaultServingSignatureDefKey))); +} + +// Test all ClassifierTest test cases with both SessionBundle and SavedModel. +INSTANTIATE_TEST_CASE_P(UseSavedModel, GetModelMetadataImplTest, + ::testing::Bool()); + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 4dff5cfa9fa..1822627799d 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -19,7 +19,7 @@ limitations under the License. #include #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" #include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" @@ -78,7 +78,7 @@ class PredictImplTest : public ::testing::TestWithParam { options.platform_config_map = CreateTensorFlowPlatformConfigMap( SessionBundleConfig(), use_saved_model); options.aspired_version_policy = - std::unique_ptr(new EagerLoadPolicy); + std::unique_ptr(new AvailabilityPreservingPolicy); // Reduce the number of initial load threads to be num_load_threads to avoid // timing out in tests. options.num_initial_load_threads = options.num_load_threads; diff --git a/tensorflow_serving/servables/tensorflow/simple_servers.cc b/tensorflow_serving/servables/tensorflow/simple_servers.cc index 358b892a645..50e38990fda 100644 --- a/tensorflow_serving/servables/tensorflow/simple_servers.cc +++ b/tensorflow_serving/servables/tensorflow/simple_servers.cc @@ -23,7 +23,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/core/aspired_versions_manager_builder.h" -#include "tensorflow_serving/core/eager_load_policy.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/source.h" #include "tensorflow_serving/core/source_adapter.h" @@ -84,7 +84,8 @@ Status CreateSingleTFModelManagerFromBasePath( CreateStoragePathSource(base_path, "default", &path_source)); AspiredVersionsManagerBuilder::Options manager_options; - manager_options.aspired_version_policy.reset(new EagerLoadPolicy()); + manager_options.aspired_version_policy.reset( + new AvailabilityPreservingPolicy); std::unique_ptr builder; TF_CHECK_OK(AspiredVersionsManagerBuilder::Create(std::move(manager_options), &builder)); From 49d990c580e91e2a2d8afa2945e000630369acfe Mon Sep 17 00:00:00 2001 From: Li Lao Date: Thu, 26 Jan 2017 14:25:26 -0800 Subject: [PATCH 0145/8554] Fix flaky AspiredVersionsManagerTest. The flakiness was caused by asychronous model version loading and unloading. Change: 145721173 --- .../core/aspired_versions_manager_test.cc | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index b658d010a4a..814e82fd9e8 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -907,13 +907,19 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { std::move(first_aspired_versions)); HandlePendingAspiredVersionsRequests(); - // Version 0 is unloaded and the new aspired version is loaded. + // Wait for verion 0 to be unloaded and the new aspired version to be loaded. + // If we don't wait, the first_loader_handle below may be obtained before + // the loading or unloading finishes, which may block the loading or + // unloading. InvokePolicyAndExecuteAction(); InvokePolicyAndExecuteAction(); - - // Pin 'first_loader' in the manager by holding a handle to its servable. WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kAvailable}); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + + // Pin 'first_loader' in the manager by holding a handle to its servable. int servable = 42; EXPECT_CALL(*first_loader, servable()).WillOnce(InvokeWithoutArgs([&]() { return AnyPtr{&servable}; @@ -936,6 +942,7 @@ TEST_P(AspiredVersionsManagerTest, UnaspireThenImmediatelyReaspire) { manager_->GetAspiredVersionsCallback()(kServableName, std::move(empty_aspired_versions)); HandlePendingAspiredVersionsRequests(); + // The following thread will block trying to unload the first loader, while we // hold the handle. std::unique_ptr unload_thread( From 1fa59adbbd38509c9ffaa6ba89804ea4064ee3e4 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 26 Jan 2017 17:28:40 -0800 Subject: [PATCH 0146/8554] Adds monitoring for request logging. Change: 145743562 --- tensorflow_serving/core/request_logger.cc | 26 +++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/core/request_logger.cc b/tensorflow_serving/core/request_logger.cc index e114eb8d4a7..8474957e44f 100644 --- a/tensorflow_serving/core/request_logger.cc +++ b/tensorflow_serving/core/request_logger.cc @@ -17,11 +17,22 @@ limitations under the License. #include +#include "tensorflow/core/lib/core/error_codes.pb.h" #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/monitoring/counter.h" #include "tensorflow/core/lib/random/random.h" +#include "tensorflow_serving/apis/model.pb.h" namespace tensorflow { namespace serving { +namespace { + +auto* request_log_count = monitoring::Counter<2>::New( + "/tensorflow/serving/request_log_count", + "The total number of requests logged from the model server sliced " + "down by model_name and status code.", + "model_name", "status_code"); +} RequestLogger::RequestLogger(const LoggingConfig& logging_config, std::unique_ptr log_collector) @@ -38,10 +49,17 @@ Status RequestLogger::Log(const google::protobuf::Message& request, *log_metadata_with_config.mutable_sampling_config() = logging_config_.sampling_config(); if (uniform_sampler_.Sample(sampling_rate)) { - std::unique_ptr log; - TF_RETURN_IF_ERROR( - CreateLogMessage(request, response, log_metadata_with_config, &log)); - TF_RETURN_IF_ERROR(log_collector_->CollectMessage(*log)); + const auto status = [&]() { + std::unique_ptr log; + TF_RETURN_IF_ERROR( + CreateLogMessage(request, response, log_metadata_with_config, &log)); + return log_collector_->CollectMessage(*log); + }(); + request_log_count + ->GetCell(log_metadata.model_spec().name(), + error::Code_Name(status.code())) + ->IncrementBy(1); + return status; } return Status::OK(); } From 8b8cb7b2b3fdb05a7b506941384b463631f46e36 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 27 Jan 2017 09:06:29 -0800 Subject: [PATCH 0147/8554] Add additional named signatures to test half_plus_two model. Change: 145801253 --- .../batching/batching_session_test.cc | 20 +++++++++---------- .../get_model_metadata_impl_test.cc | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index 384065544d3..1a50646dfdd 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -134,13 +134,13 @@ TEST(BatchingSessionTest, TensorSignatureFromSignatureDefs) { const SignatureDef signature_def_0 = CreateSignatureDef({{"x0", "x1"}, {"y0", "y1"}}); const SignatureDef signature_def_1 = - CreateSignatureDef({{"x1", "x2"}, {"y1", "y2"}}); + CreateSignatureDef({{"x1", "x2"}, {"y1", "y3"}}); const TensorSignature tensor_signature = TensorSignatureFromSignatureDefs({signature_def_0, signature_def_1}); EXPECT_THAT(tensor_signature.input_tensors, UnorderedElementsAre("x0", "x1", "x2")); EXPECT_THAT(tensor_signature.output_tensors, - UnorderedElementsAre("y0", "y1", "y2")); + UnorderedElementsAre("y0", "y1", "y3")); } TEST(BatchingSessionTest, Basic) { @@ -188,9 +188,9 @@ TEST(BatchingSessionTest, RequestThatDoesntMatchSignatureGetsRunAnyway) { std::unique_ptr batching_session; BatchingSessionOptions batching_session_options; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, {{"x2"}, {"y2"}}, + schedule_options, batching_session_options, {{"x2"}, {"y3"}}, CreateHalfPlusTwoSession(), &batching_session)); - // Issue a request using x/y, which doesn't match the x2/y2 signature. + // Issue a request using x/y, which doesn't match the x2/y3 signature. TestSingleRequest(100.0f, 42.0f, batching_session.get()); } @@ -288,7 +288,7 @@ TEST(BatchingSessionTest, DifferentOrderForInputAndOutputTensors) { BatchingSessionOptions batching_session_options; std::unique_ptr batching_session; TF_ASSERT_OK(CreateBasicBatchingSession( - schedule_options, batching_session_options, {{"x", "x2"}, {"y", "y2"}}, + schedule_options, batching_session_options, {{"x", "x2"}, {"y", "y3"}}, CreateHalfPlusTwoSession(), &batching_session)); const Tensor input0 = test::AsTensor({8.0f, 6.0f}, {2}); @@ -300,7 +300,7 @@ TEST(BatchingSessionTest, DifferentOrderForInputAndOutputTensors) { Env::Default()->StartThread(ThreadOptions(), "first_request_thread", [&] { std::vector outputs; TF_ASSERT_OK(batching_session->Run({{"x", input0}, {"x2", input1}}, - {"y", "y2"} /* outputs */, + {"y", "y3"} /* outputs */, {} /* target nodes */, &outputs)); ASSERT_EQ(2, outputs.size()); test::ExpectTensorEqual(expected_output0, outputs[0]); @@ -310,7 +310,7 @@ TEST(BatchingSessionTest, DifferentOrderForInputAndOutputTensors) { ThreadOptions(), "second_request_thread", [&] { std::vector outputs; TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, - {"y2", "y"} /* outputs */, + {"y3", "y"} /* outputs */, {} /* target nodes */, &outputs)); ASSERT_EQ(2, outputs.size()); test::ExpectTensorEqual(expected_output1, outputs[0]); @@ -320,7 +320,7 @@ TEST(BatchingSessionTest, DifferentOrderForInputAndOutputTensors) { Env::Default()->StartThread(ThreadOptions(), "third_request_thread", [&] { std::vector outputs; TF_ASSERT_OK(batching_session->Run({{"x2", input1}, {"x", input0}}, - {"y", "y2"} /* outputs */, + {"y", "y3"} /* outputs */, {} /* target nodes */, &outputs)); ASSERT_EQ(2, outputs.size()); test::ExpectTensorEqual(expected_output0, outputs[0]); @@ -349,7 +349,7 @@ TEST(BatchingSessionTest, MultipleSignatures) { std::unique_ptr batching_session; TF_CHECK_OK(CreateBatchingSession( batching_session_options, {{{{"x"}, {"y"}}, create_scheduler}, - {{{"x2"}, {"y2"}}, create_scheduler}}, + {{{"x2"}, {"y3"}}, create_scheduler}}, CreateHalfPlusTwoSession(), &batching_session)); ASSERT_EQ(2, schedulers.size()); @@ -367,7 +367,7 @@ TEST(BatchingSessionTest, MultipleSignatures) { Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); Tensor expected_output = test::AsTensor({53.0f, 24.0f}, {2}); std::vector outputs; - TF_ASSERT_OK(batching_session->Run({{"x2", input}}, {"y2"} /* outputs */, + TF_ASSERT_OK(batching_session->Run({{"x2", input}}, {"y3"} /* outputs */, {} /* target nodes */, &outputs)); ASSERT_EQ(1, outputs.size()); test::ExpectTensorEqual(expected_output, outputs[0]); diff --git a/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc index 91c3060af83..c17664b12d3 100644 --- a/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc @@ -194,9 +194,9 @@ TEST_P(GetModelMetadataImplTest, ReturnsSignaturesForValidModel) { received_signature_def_map.signature_def().size()); if (GetParam()) { EXPECT_THAT( - expected_signature_def_map.signature_def().at(kRegressMethodName), + expected_signature_def_map.signature_def().at("regress_x_to_y"), test_util::EqualsProto( - received_signature_def_map.signature_def().at(kRegressMethodName))); + received_signature_def_map.signature_def().at("regress_x_to_y"))); } else { EXPECT_THAT(expected_signature_def_map.signature_def().at("regress"), test_util::EqualsProto( From 65f50621a192004ab5ae68e75818e94930a6778b Mon Sep 17 00:00:00 2001 From: Li Lao Date: Fri, 27 Jan 2017 22:13:30 -0800 Subject: [PATCH 0148/8554] Update user documentation and simple_servers to use SavedModelBundle. Change: 145867752 --- tensorflow_serving/example/BUILD | 40 ++++ .../example/inception_client.py | 1 + .../example/inception_saved_model.py | 201 ++++++++++++++++++ tensorflow_serving/example/mnist_client.py | 1 + .../example/mnist_saved_model.py | 140 ++++++++++++ .../g3doc/architecture_overview.md | 2 +- tensorflow_serving/g3doc/custom_servable.md | 6 +- tensorflow_serving/g3doc/custom_source.md | 26 +-- tensorflow_serving/g3doc/serving_advanced.md | 112 +++++----- tensorflow_serving/g3doc/serving_basic.md | 122 +++++------ tensorflow_serving/g3doc/serving_inception.md | 6 +- tensorflow_serving/servables/tensorflow/BUILD | 8 +- .../servables/tensorflow/simple_servers.cc | 19 +- .../servables/tensorflow/simple_servers.h | 8 +- .../tensorflow/simple_servers_test.cc | 12 +- 15 files changed, 545 insertions(+), 159 deletions(-) create mode 100644 tensorflow_serving/example/inception_saved_model.py create mode 100644 tensorflow_serving/example/mnist_saved_model.py diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 4b365b5acd8..ff7172d2795 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -27,6 +27,8 @@ py_library( srcs = ["mnist_input_data.py"], ) +# TODO(b/32628014): remove mnist_export after we no longer support +# SessionBundle. py_binary( name = "mnist_export", srcs = [ @@ -39,6 +41,24 @@ py_binary( ], ) +py_binary( + name = "mnist_saved_model", + srcs = [ + "mnist_saved_model.py", + ], + deps = [ + ":mnist_input_data", + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/python/saved_model:builder", + "@org_tensorflow//tensorflow/python/saved_model:constants", + "@org_tensorflow//tensorflow/python/saved_model:loader", + "@org_tensorflow//tensorflow/python/saved_model:signature_constants", + "@org_tensorflow//tensorflow/python/saved_model:signature_def_utils", + "@org_tensorflow//tensorflow/python/saved_model:tag_constants", + "@org_tensorflow//tensorflow/python/saved_model:utils", + ], +) + py_binary( name = "mnist_client", srcs = [ @@ -52,6 +72,8 @@ py_binary( ], ) +# TODO(b/32628014): remove inception_export after we no longer support +# SessionBundle. py_binary( name = "inception_export", srcs = [ @@ -64,6 +86,24 @@ py_binary( ], ) +py_binary( + name = "inception_saved_model", + srcs = [ + "inception_saved_model.py", + ], + deps = [ + "@inception_model//inception", + "@org_tensorflow//tensorflow:tensorflow_py", + "@org_tensorflow//tensorflow/python/saved_model:builder", + "@org_tensorflow//tensorflow/python/saved_model:constants", + "@org_tensorflow//tensorflow/python/saved_model:loader", + "@org_tensorflow//tensorflow/python/saved_model:signature_constants", + "@org_tensorflow//tensorflow/python/saved_model:signature_def_utils", + "@org_tensorflow//tensorflow/python/saved_model:tag_constants", + "@org_tensorflow//tensorflow/python/saved_model:utils", + ], +) + py_binary( name = "inception_client", srcs = [ diff --git a/tensorflow_serving/example/inception_client.py b/tensorflow_serving/example/inception_client.py index 77899ad5ea3..84cc89e95a5 100644 --- a/tensorflow_serving/example/inception_client.py +++ b/tensorflow_serving/example/inception_client.py @@ -45,6 +45,7 @@ def main(_): data = f.read() request = predict_pb2.PredictRequest() request.model_spec.name = 'inception' + request.model_spec.signature_name = 'predict_images' request.inputs['images'].CopyFrom( tf.contrib.util.make_tensor_proto(data, shape=[1])) result = stub.Predict(request, 10.0) # 10 secs timeout diff --git a/tensorflow_serving/example/inception_saved_model.py b/tensorflow_serving/example/inception_saved_model.py new file mode 100644 index 00000000000..febf2bd5d19 --- /dev/null +++ b/tensorflow_serving/example/inception_saved_model.py @@ -0,0 +1,201 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +#!/usr/bin/env python2.7 +"""Export inception model given existing training checkpoints. + +The model is exported as SavedModel with proper signatures that can be loaded by +standard tensorflow_model_server. +""" + +import os.path + +# This is a placeholder for a Google-internal import. + +import tensorflow as tf + +from tensorflow.python.saved_model import builder as saved_model_builder +from tensorflow.python.saved_model import signature_constants +from tensorflow.python.saved_model import signature_def_utils +from tensorflow.python.saved_model import tag_constants +from tensorflow.python.saved_model import utils +from tensorflow.python.util import compat +from inception import inception_model + +tf.app.flags.DEFINE_string('checkpoint_dir', '/tmp/inception_train', + """Directory where to read training checkpoints.""") +tf.app.flags.DEFINE_string('output_dir', '/tmp/inception_output', + """Directory where to export inference model.""") +tf.app.flags.DEFINE_integer('model_version', 1, + """Version number of the model.""") +tf.app.flags.DEFINE_integer('image_size', 299, + """Needs to provide same value as in training.""") +FLAGS = tf.app.flags.FLAGS + +NUM_CLASSES = 1000 +NUM_TOP_CLASSES = 5 + +WORKING_DIR = os.path.dirname(os.path.realpath(__file__)) +SYNSET_FILE = os.path.join(WORKING_DIR, 'imagenet_lsvrc_2015_synsets.txt') +METADATA_FILE = os.path.join(WORKING_DIR, 'imagenet_metadata.txt') + + +def export(): + # Create index->synset mapping + synsets = [] + with open(SYNSET_FILE) as f: + synsets = f.read().splitlines() + # Create synset->metadata mapping + texts = {} + with open(METADATA_FILE) as f: + for line in f.read().splitlines(): + parts = line.split('\t') + assert len(parts) == 2 + texts[parts[0]] = parts[1] + + with tf.Graph().as_default(): + # Build inference model. + # Please refer to Tensorflow inception model for details. + + # Input transformation. + serialized_tf_example = tf.placeholder(tf.string, name='tf_example') + feature_configs = { + 'image/encoded': tf.FixedLenFeature( + shape=[], dtype=tf.string), + } + tf_example = tf.parse_example(serialized_tf_example, feature_configs) + jpegs = tf_example['image/encoded'] + images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32) + + # Run inference. + logits, _ = inception_model.inference(images, NUM_CLASSES + 1) + + # Transform output to topK result. + values, indices = tf.nn.top_k(logits, NUM_TOP_CLASSES) + + # Create a constant string Tensor where the i'th element is + # the human readable class description for the i'th index. + # Note that the 0th index is an unused background class + # (see inception model definition code). + class_descriptions = ['unused background'] + for s in synsets: + class_descriptions.append(texts[s]) + class_tensor = tf.constant(class_descriptions) + + classes = tf.contrib.lookup.index_to_string( + tf.to_int64(indices), mapping=class_tensor) + + # Restore variables from training checkpoint. + variable_averages = tf.train.ExponentialMovingAverage( + inception_model.MOVING_AVERAGE_DECAY) + variables_to_restore = variable_averages.variables_to_restore() + saver = tf.train.Saver(variables_to_restore) + with tf.Session() as sess: + # Restore variables from training checkpoints. + ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) + if ckpt and ckpt.model_checkpoint_path: + saver.restore(sess, ckpt.model_checkpoint_path) + # Assuming model_checkpoint_path looks something like: + # /my-favorite-path/imagenet_train/model.ckpt-0, + # extract global_step from it. + global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] + print 'Successfully loaded model from %s at step=%s.' % ( + ckpt.model_checkpoint_path, global_step) + else: + print 'No checkpoint file found at %s' % FLAGS.checkpoint_dir + return + + # Export inference model. + output_path = os.path.join( + compat.as_bytes(FLAGS.output_dir), + compat.as_bytes(str(FLAGS.model_version))) + print 'Exporting trained model to', output_path + builder = saved_model_builder.SavedModelBuilder(output_path) + + # Build the signature_def_map. + classify_inputs_tensor_info = utils.build_tensor_info( + serialized_tf_example) + classes_output_tensor_info = utils.build_tensor_info(classes) + scores_output_tensor_info = utils.build_tensor_info(values) + + classification_signature = signature_def_utils.build_signature_def( + inputs={ + signature_constants.CLASSIFY_INPUTS: classify_inputs_tensor_info + }, + outputs={ + signature_constants.CLASSIFY_OUTPUT_CLASSES: + classes_output_tensor_info, + signature_constants.CLASSIFY_OUTPUT_SCORES: + scores_output_tensor_info + }, + method_name=signature_constants.CLASSIFY_METHOD_NAME) + + predict_inputs_tensor_info = utils.build_tensor_info(jpegs) + prediction_signature = signature_def_utils.build_signature_def( + inputs={'images': predict_inputs_tensor_info}, + outputs={ + 'classes': classes_output_tensor_info, + 'scores': scores_output_tensor_info + }, + method_name=signature_constants.PREDICT_METHOD_NAME) + + legacy_init_op = tf.group( + tf.initialize_all_tables(), name='legacy_init_op') + builder.add_meta_graph_and_variables( + sess, [tag_constants.SERVING], + signature_def_map={ + 'predict_images': + prediction_signature, + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + classification_signature, + }, + legacy_init_op=legacy_init_op) + + builder.save() + print 'Successfully exported model to %s' % FLAGS.output_dir + + +def preprocess_image(image_buffer): + """Preprocess JPEG encoded bytes to 3D float Tensor.""" + + # Decode the string as an RGB JPEG. + # Note that the resulting image contains an unknown height and width + # that is set dynamically by decode_jpeg. In other words, the height + # and width of image is unknown at compile-time. + image = tf.image.decode_jpeg(image_buffer, channels=3) + # After this point, all image pixels reside in [0,1) + # until the very end, when they're rescaled to (-1, 1). The various + # adjust_* ops all require this range for dtype float. + image = tf.image.convert_image_dtype(image, dtype=tf.float32) + # Crop the central region of the image with an area containing 87.5% of + # the original image. + image = tf.image.central_crop(image, central_fraction=0.875) + # Resize the image to the original height and width. + image = tf.expand_dims(image, 0) + image = tf.image.resize_bilinear( + image, [FLAGS.image_size, FLAGS.image_size], align_corners=False) + image = tf.squeeze(image, [0]) + # Finally, rescale to [-1,1] instead of [0, 1) + image = tf.subtract(image, 0.5) + image = tf.multiply(image, 2.0) + return image + + +def main(unused_argv=None): + export() + + +if __name__ == '__main__': + tf.app.run() diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index a93ffd7e374..843696ac33b 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -144,6 +144,7 @@ def do_inference(hostport, work_dir, concurrency, num_tests): for _ in range(num_tests): request = predict_pb2.PredictRequest() request.model_spec.name = 'mnist' + request.model_spec.signature_name = 'predict_images' image, label = test_data_set.next_batch(1) request.inputs['images'].CopyFrom( tf.contrib.util.make_tensor_proto(image[0], shape=[1, image[0].size])) diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py new file mode 100644 index 00000000000..35bb2d244d2 --- /dev/null +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -0,0 +1,140 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +#!/usr/bin/env python2.7 +"""Train and export a simple Softmax Regression TensorFlow model. + +The model is from the TensorFlow "MNIST For ML Beginner" tutorial. This program +simply follows all its training instructions, and uses TensorFlow SavedModel to +export the trained model with proper signatures that can be loaded by standard +tensorflow_model_server. + +Usage: mnist_export.py [--training_iteration=x] [--model_version=y] export_dir +""" + +import os +import sys + +# This is a placeholder for a Google-internal import. + +import tensorflow as tf + +from tensorflow.python.saved_model import builder as saved_model_builder +from tensorflow.python.saved_model import signature_constants +from tensorflow.python.saved_model import signature_def_utils +from tensorflow.python.saved_model import tag_constants +from tensorflow.python.saved_model import utils +from tensorflow.python.util import compat +from tensorflow_serving.example import mnist_input_data + +tf.app.flags.DEFINE_integer('training_iteration', 1000, + 'number of training iterations.') +tf.app.flags.DEFINE_integer('model_version', 1, 'version number of the model.') +tf.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory.') +FLAGS = tf.app.flags.FLAGS + + +def main(_): + if len(sys.argv) < 2 or sys.argv[-1].startswith('-'): + print('Usage: mnist_export.py [--training_iteration=x] ' + '[--model_version=y] export_dir') + sys.exit(-1) + if FLAGS.training_iteration <= 0: + print 'Please specify a positive value for training iteration.' + sys.exit(-1) + if FLAGS.model_version <= 0: + print 'Please specify a positive value for version number.' + sys.exit(-1) + + # Train model + print 'Training model...' + mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) + sess = tf.InteractiveSession() + serialized_tf_example = tf.placeholder(tf.string, name='tf_example') + feature_configs = {'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32),} + tf_example = tf.parse_example(serialized_tf_example, feature_configs) + x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name + y_ = tf.placeholder('float', shape=[None, 10]) + w = tf.Variable(tf.zeros([784, 10])) + b = tf.Variable(tf.zeros([10])) + sess.run(tf.initialize_all_variables()) + y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') + cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) + train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) + values, indices = tf.nn.top_k(y, 10) + prediction_classes = tf.contrib.lookup.index_to_string( + tf.to_int64(indices), mapping=tf.constant([str(i) for i in xrange(10)])) + for _ in range(FLAGS.training_iteration): + batch = mnist.train.next_batch(50) + train_step.run(feed_dict={x: batch[0], y_: batch[1]}) + correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) + accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) + print 'training accuracy %g' % sess.run( + accuracy, feed_dict={x: mnist.test.images, + y_: mnist.test.labels}) + print 'Done training!' + + # Export model + # WARNING(break-tutorial-inline-code): The following code snippet is + # in-lined in tutorials, please update tutorial documents accordingly + # whenever code changes. + export_path_base = sys.argv[-1] + export_path = os.path.join( + compat.as_bytes(export_path_base), + compat.as_bytes(str(FLAGS.model_version))) + print 'Exporting trained model to', export_path + builder = saved_model_builder.SavedModelBuilder(export_path) + + # Build the signature_def_map. + classification_inputs = utils.build_tensor_info(serialized_tf_example) + classification_outputs_classes = utils.build_tensor_info(prediction_classes) + classification_outputs_scores = utils.build_tensor_info(values) + + classification_signature = signature_def_utils.build_signature_def( + inputs={signature_constants.CLASSIFY_INPUTS: classification_inputs}, + outputs={ + signature_constants.CLASSIFY_OUTPUT_CLASSES: + classification_outputs_classes, + signature_constants.CLASSIFY_OUTPUT_SCORES: + classification_outputs_scores + }, + method_name=signature_constants.CLASSIFY_METHOD_NAME) + + tensor_info_x = utils.build_tensor_info(x) + tensor_info_y = utils.build_tensor_info(y) + + prediction_signature = signature_def_utils.build_signature_def( + inputs={'images': tensor_info_x}, + outputs={'scores': tensor_info_y}, + method_name=signature_constants.PREDICT_METHOD_NAME) + + legacy_init_op = tf.group(tf.initialize_all_tables(), name='legacy_init_op') + builder.add_meta_graph_and_variables( + sess, [tag_constants.SERVING], + signature_def_map={ + 'predict_images': + prediction_signature, + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + classification_signature, + }, + legacy_init_op=legacy_init_op) + + builder.save() + + print 'Done exporting!' + + +if __name__ == '__main__': + tf.app.run() diff --git a/tensorflow_serving/g3doc/architecture_overview.md b/tensorflow_serving/g3doc/architecture_overview.md index 983d65fa5b8..b0bf98dd33c 100644 --- a/tensorflow_serving/g3doc/architecture_overview.md +++ b/tensorflow_serving/g3doc/architecture_overview.md @@ -32,7 +32,7 @@ Servables do not manage their own lifecycle. Typical servables include the following: - * a TensorFlow SessionBundle (`tensorflow::Session`) + * a TensorFlow SavedModelBundle (`tensorflow::Session`) * a lookup table for embedding or vocabulary lookups #### Servable Versions diff --git a/tensorflow_serving/g3doc/custom_servable.md b/tensorflow_serving/g3doc/custom_servable.md index 80c173e29ac..658ede6bae8 100644 --- a/tensorflow_serving/g3doc/custom_servable.md +++ b/tensorflow_serving/g3doc/custom_servable.md @@ -1,11 +1,11 @@ # Creating a new kind of servable This document explains how to extend TensorFlow Serving with a new kind of -servable. The most prominent servable type is `SessionBundle`, but it can be +servable. The most prominent servable type is `SavedModelBundle`, but it can be useful to define other kinds of servables, to serve data that goes along with your model. Examples include: a vocabulary lookup table, feature transformation -logic. Any C++ class can be a servable, e.g. `int`, `std::map` -or any class defined in your binary -- let us call it `YourServable`. +logic. Any C++ class can be a servable, e.g. `int`, `std::map` or +any class defined in your binary -- let us call it `YourServable`. ## Defining a `Loader` and `SourceAdapter` for `YourServable` diff --git a/tensorflow_serving/g3doc/custom_source.md b/tensorflow_serving/g3doc/custom_source.md index cd3eb108136..1162875be19 100644 --- a/tensorflow_serving/g3doc/custom_source.md +++ b/tensorflow_serving/g3doc/custom_source.md @@ -80,15 +80,15 @@ requests. ## Using your Source to load TensorFlow sessions You will likely want to use your new source module in conjunction with -`SessionBundleSourceAdapter` -(`servables/tensorflow/session_bundle_source_adapter*`), which will interpret -each path your source emits as a TensorFlow export, and convert each path to a -loader for a TensorFlow `SessionBundle` servable. You will likely plug the -`SessionBundle` adapter into a `AspiredVersionsManager`, which takes care of -actually loading and serving the servables. A good illustration of chaining -these three kinds of modules together to get a working server library is found -in `servables/tensorflow/simple_servers.cc`. Here is a walk-through of the main -code flow (with bad error handling; real code should be more careful): +`SavedModelBundleSourceAdapter` +(`servables/tensorflow/saved_model_bundle_source_adapter*`), which will +interpret each path your source emits as a TensorFlow export, and convert each +path to a loader for a TensorFlow `SavedModelBundle` servable. You will likely +plug the `SavedModelBundle` adapter into a `AspiredVersionsManager`, which takes +care of actually loading and serving the servables. A good illustration of +chaining these three kinds of modules together to get a working server library +is found in `servables/tensorflow/simple_servers.cc`. Here is a walk-through of +the main code flow (with bad error handling; real code should be more careful): First, create a manager: @@ -96,17 +96,17 @@ First, create a manager: std::unique_ptr manager = ...; ~~~ -Then, create a `SessionBundle` source adapter and plug it into the manager: +Then, create a `SavedModelBundle` source adapter and plug it into the manager: ~~~c++ -std::unique_ptr bundle_adapter; +std::unique_ptr bundle_adapter; SessionBundleSourceAdapterConfig config; // ... populate 'config' with TensorFlow options. -TF_CHECK_OK(SessionBundleSourceAdapter::Create(config, &bundle_adapter)); +TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(config, &bundle_adapter)); ConnectSourceToTarget(bundle_adapter.get(), manager.get()); ~~~ -Lastly, create your path source and plug it into the `SessionBundle` adapter: +Lastly, create your path source and plug it into the `SavedModelBundle` adapter: ~~~c++ auto your_source = new YourPathSource(...); diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index f0685afc811..5e372f99a1a 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -15,7 +15,7 @@ tutorial. The code for this tutorial consists of two parts: * A Python file - [mnist_export.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_export.py) + [mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_saved_model.py) that trains and exports multiple versions of the model. * A C++ file @@ -44,18 +44,18 @@ $>rm -rf /tmp/mnist_model Train (with 100 iterations) and export the first version of model: ~~~shell -$>bazel build //tensorflow_serving/example:mnist_export -$>bazel-bin/tensorflow_serving/example/mnist_export --training_iteration=100 --export_version=1 /tmp/mnist_model +$>bazel build //tensorflow_serving/example:mnist_saved_model +$>bazel-bin/tensorflow_serving/example/mnist_saved_model --training_iteration=100 --model_version=1 /tmp/mnist_model ~~~ Train (with 2000 iterations) and export the second version of model: ~~~shell -$>bazel-bin/tensorflow_serving/example/mnist_export --training_iteration=2000 --export_version=2 /tmp/mnist_model +$>bazel-bin/tensorflow_serving/example/mnist_saved_model --training_iteration=2000 --model_version=2 /tmp/mnist_model ~~~ -As you can see in `mnist_export.py`, the training and exporting is done the same -way it is in the [TensorFlow Serving basic tutorial](serving_basic.md). For +As you can see in `mnist_saved_model.py`, the training and exporting is done the +same way it is in the [TensorFlow Serving basic tutorial](serving_basic.md). For demonstration purposes, you're intentionally dialing down the training iterations for the first run and exporting it as v1, while training it normally for the second run and exporting it as v2 to the same parent directory -- as we @@ -65,7 +65,7 @@ intensive training. You should see training data for each training run in your ~~~shell $>ls /tmp/mnist_model -00000001 00000002 +1 2 ~~~ ## ServerCore @@ -91,13 +91,17 @@ int main(int argc, char** argv) { ServerCore::Options options; options.model_server_config = model_server_config; - options.source_adaptor_creator = [source_adapter_config]( - const string& platform_type, - std::unique_ptr* adapter) { - return CreateSourceAdapter(source_adapter_config, platform_type, adapter); - }; options.servable_state_monitor_creator = &CreateServableStateMonitor; options.custom_model_config_loader = &LoadCustomModelConfig; + + ::google::protobuf::Any source_adapter_config; + SavedModelBundleSourceAdapterConfig + saved_model_bundle_source_adapter_config; + source_adapter_config.PackFrom(saved_model_bundle_source_adapter_config); + (*(*options.platform_config_map.mutable_platform_configs()) + [kTensorFlowModelPlatform].mutable_source_adapter_config()) = + source_adapter_config; + std::unique_ptr core; TF_CHECK_OK(ServerCore::Create(options, &core)); RunServer(port, std::move(core)); @@ -113,31 +117,35 @@ commonly used options: either through `model_config_list`, which declares a static list of models, or through `dynamic_model_config`, which declares a dynamic list of models that may get updated at runtime. - * `SourceAdapterCreator` that creates the `SourceAdapter`, which adapts - `StoragePath` (the path where a model version is discovered) to model - `Loader` (loads the model version from storage path and provides state - transition interfaces to the `Manager`). If not specified, a - `SessionBundleSourceAdapter` will be created, which we will explain later. - -`SessionBundle` is a key component of TensorFlow Serving. It represents a + * `PlatformConfigMap` that maps from the name of the platform (such as + `tensorflow`) to the `PlatformConfig`, which is used to create the + `SourceAdapter`. `SourceAdapter` adapts `StoragePath` (the path where a model + version is discovered) to model `Loader` (loads the model version from + storage path and provides state transition interfaces to the `Manager`). If + `PlatformConfig` contains `SavedModelBundleSourceAdapterConfig`, a + `SavedModelBundleSourceAdapter` will be created, which we will explain later. + +`SavedModelBundle` is a key component of TensorFlow Serving. It represents a TensorFlow model loaded from a given path and provides the same `Session::Run` -interface as TensorFlow to run inference. -`SessionBundleSourceAdapter` adapts storage path to `Loader` -so that model lifetime can be managed by `Manager`. +interface as TensorFlow to run inference. `SavedModelBundleSourceAdapter` adapts +storage path to `Loader` so that model lifetime can be managed +by `Manager`. Please note that `SavedModelBundle` is the successor of deprecated +`SessionBundle`. Users are encouraged to use `SavedModelBundle` as the support +for `SessionBundle` will soon be removed. With all these, `ServerCore` internally does the following: * Instantiates a `FileSystemStoragePathSource` that monitors model export paths declared in `model_config_list`. - * Instantiates a `SourceAdapter` using the `SourceAdapterCreator` with the - model type declared in `model_config_list` and connects the + * Instantiates a `SourceAdapter` using the `PlatformConfigMap` with the + model platform declared in `model_config_list` and connects the `FileSystemStoragePathSource` to it. This way, whenever a new model version is - discovered under the export path, the `SessionBundleSourceAdapter` adapts it - to a `Loader`. + discovered under the export path, the `SavedModelBundleSourceAdapter` + adapts it to a `Loader`. * Instantiates a specific implementation of `Manager` called `AspiredVersionsManager` that manages all such `Loader` instances created by - the `SessionBundleSourceAdapter`. `ServerCore` exports the `Manager` interface - by delegating the calls to `AspiredVersionsManager`. + the `SavedModelBundleSourceAdapter`. `ServerCore` exports the `Manager` + interface by delegating the calls to `AspiredVersionsManager`. Whenever a new version is available, this `AspiredVersionsManager` loads the new version, and under its default behavior unloads the old one. If you want to @@ -162,21 +170,23 @@ Modern hardware accelerators (GPUs, etc.) used to do machine learning inference usually achieve best computation efficiency when inference requests are run in large batches. -Batching can be turned on by providing proper `SessionBundleSourceAdapterConfig` -when creating the `SessionBundleSourceAdapter`. In this case we set the +Batching can be turned on by providing proper `SessionBundleConfig` when +creating the `SavedModelBundleSourceAdapter`. In this case we set the `BatchingParameters` with pretty much default values. Batching can be fine-tuned by setting custom timeout, batch_size, etc. values. For details, please refer to `BatchingParameters`. ~~~c++ -SessionBundleSourceAdapterConfig source_adapter_config; +SessionBundleConfig session_bundle_config; // Batching config if (enable_batching) { BatchingParameters* batching_parameters = - source_adapter_config.mutable_config()->mutable_batching_parameters(); + session_bundle_config.mutable_batching_parameters(); batching_parameters->mutable_thread_pool_name()->set_value( "model_server_batch_threads"); } +*saved_model_bundle_source_adapter_config.mutable_legacy_config() = + session_bundle_config; ~~~ Upon reaching full batch, inference requests are merged internally into a @@ -211,37 +221,37 @@ around the following key concepts: * **Model**: A machine-learned model is represented by one or more servables. Examples of servables are: - * TensorFlow session or wrappers around them, such as `SessionBundle`. + * TensorFlow session or wrappers around them, such as `SavedModelBundle`. * Other kinds of machine-learned models. * Vocabulary lookup tables. * Embedding lookup tables. - A composite model could be represented as multiple independent servables, or - as a single composite servable. A servable may also correspond to a fraction - of a Model, for example with a large lookup table sharded across many - `Manager` instances. + A composite model could be represented as multiple independent servables, or + as a single composite servable. A servable may also correspond to a fraction + of a Model, for example with a large lookup table sharded across many + `Manager` instances. To put all these into the context of this tutorial: * TensorFlow models are represented by one kind of servable -- - `SessionBundle`. `SessionBundle` internally consists of a `tensorflow:Session` - paired with some metadata about what graph is loaded into the session and how - to run it for inference. + `SavedModelBundle`. `SavedModelBundle` internally consists of a + `tensorflow:Session` paired with some metadata about what graph is loaded + into the session and how to run it for inference. * There is a file-system directory containing a stream of TensorFlow exports, - each in its own subdirectory whose name is a version number. The outer - directory can be thought of as the serialized representation of the servable - stream for the TensorFlow model being served. Each export corresponds to a - servables that can be loaded. + each in its own subdirectory whose name is a version number. The outer + directory can be thought of as the serialized representation of the servable + stream for the TensorFlow model being served. Each export corresponds to a + servables that can be loaded. * `AspiredVersionsManager` monitors the export stream, and manages lifecycle - of all SessionBundle` servables dynamically. + of all SavedModelBundle` servables dynamically. `TensorflowPredictImpl::Predict` then just: - * Requests `SessionBundle` from the manager (through ServerCore). - * Uses the `generic signatures` to map logical tensor names in `PredictRequest` - to real tensor names and bind values to tensors. + * Requests `SavedModelBundle` from the manager (through ServerCore). + * Uses the `generic signatures` to map logical tensor names in + `PredictRequest` to real tensor names and bind values to tensors. * Runs inference. ## Test and Run The Server @@ -251,9 +261,9 @@ server. ~~~shell $>mkdir /tmp/monitored -$>cp -r /tmp/mnist_model/00000001 /tmp/monitored +$>cp -r /tmp/mnist_model/1 /tmp/monitored $>bazel build //tensorflow_serving/model_servers:tensorflow_model_server -$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored +$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored --logtostderr ~~~ The server will emit log messages every one second that say @@ -274,7 +284,7 @@ Then we copy the second version of the export to the monitored folder and re-run the test: ~~~shell -$>cp -r /tmp/mnist_model/00000002 /tmp/monitored +$>cp -r /tmp/mnist_model/2 /tmp/monitored $>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 --concurrency=10 ... Inference error rate: 9.5% diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 516703da251..23ed719c50e 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -15,7 +15,7 @@ tutorial. The code for this tutorial consists of two parts: * A Python file -([mnist_export.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_export.py)) +([mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_saved_model.py)) that trains and exports the model. * A C++ file @@ -27,76 +27,72 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisit ## Train And Export TensorFlow Model -As you can see in `mnist_export.py`, the training is done the same way it is in -the MNIST For ML Beginners tutorial. The TensorFlow graph is launched in +As you can see in `mnist_saved_model.py`, the training is done the same way it +is in the MNIST For ML Beginners tutorial. The TensorFlow graph is launched in TensorFlow session `sess`, with the input tensor (image) as `x` and output tensor (Softmax score) as `y`. -Then we use TensorFlow Serving `Exporter` module to export the model. -`Exporter` saves a "snapshot" of the trained model to reliable storage so that -it can be loaded later for inference. +Then we use TensorFlow Serving `SavedModelBuilder` module to export the model. +`SavedModelBuilder` saves a "snapshot" of the trained model to reliable storage +so that it can be loaded later for inference. ~~~python -from tensorflow.contrib.session_bundle import exporter +from tensorflow.python.saved_model import builder as saved_model_builder ... -export_path = sys.argv[-1] +export_path_base = sys.argv[-1] print 'Exporting trained model to', export_path -saver = tf.train.Saver(sharded=True) -model_exporter = exporter.Exporter(saver) -model_exporter.init( - sess.graph.as_graph_def(), - named_graph_signatures={ - 'inputs': exporter.generic_signature({'images': x}), - 'outputs': exporter.generic_signature({'scores': y})}) -model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess) +builder = saved_model_builder.SavedModelBuilder(export_path) +builder.add_meta_graph_and_variables( + sess, [tag_constants.SERVING], + signature_def_map={ + 'predict': + prediction_signature, + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + classification_signature, + }, + legacy_init_op=legacy_init_op) +builder.save() ~~~ -`Exporter.__init__` takes a `tensorflow.train.Saver`, with the only requirement -being that `Saver` should have `sharded=True`. `saver` is used to serialize -graph variable values to the model export so that they can be properly restored -later. Note that since no `variable_list` is specified for the `Saver`, it will -export all variables of the graph. For more complex graphs, you can choose to -export only the variables that will be used for inference. - -`Exporter.init()` takes the following arguments: +`SavedModelBuilder.__init__` takes the following argument: +* `export_path` is the path of the export directory. `SavedModelBuilder` will +create the directory if it does not exist. In the example, we concatenate +the command line argument and `FLAGS.model_version` to obtain the export +directory. `FLAGS.model_version` specifies the **version** of the model. You +should specify a larger integer value when exporting a newer version of the same +model. Each version will be exported to a different sub-directory under the +given path. + +You can add meta graph and variables to the builder using +`SavedModelBuilder.add_meta_graph_and_variables()` with the following arguments: + * `sess` is the TensorFlow session that holds the trained model you are + exporting. - * `sess.graph.as_graph_def()` is the - [protobuf](https://developers.google.com/protocol-buffers/) of the graph. - `export` will serialize the protobuf to the model export so that the - TensorFlow graph can be properly restored later. + * `tags` is the set of tags with which to save the meta graph. - * `named_graph_signatures=...` specifies a model export **signature**. + * `signature_def_map` specifies the map of user-supplied key for a + **signature** to a tensorflow::SignatureDef to add to the meta graph. Signature specifies what type of model is being exported, and the input/output - tensors to bind to when running inference. In this case, you use - `inputs` and `outputs` as keys for `exporter.generic_signature` as such a - signature is supported by the standard `tensorflow_model_server`: + tensors to bind to when running inference. The special signature key + `serving_default` specifies the default serving signature. + `signature_def_utils.build_signature_def()` accepts the following arguments: - * `{'images': x}` specifies the input tensor binding. + * `inputs={'images': tensor_info_x}` specifies the input tensor info. - * `{'scores': y}` specifies the scores tensor binding. + * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. * `images` and `scores` are tensor alias names. They can be whatever unique strings you want, and they will become the logical names of tensor `x` and `y` that you refer to for tensor binding when sending prediction requests later. For instance, if `x` refers to the tensor with name 'long_tensor_name_foo' and `y` refers to the tensor with name - 'generated_tensor_name_bar', `exporter.generic_signature` will store - tensor logical name to real name mapping ('images' -> 'long_tensor_name_foo' - and 'scores' -> 'generated_tensor_name_bar') and allow user to refer to - these tensors with their logical names when running inference. - -`Exporter.export()` takes the following arguments: + 'generated_tensor_name_bar', `builder` will store tensor logical name to + real name mapping ('images' -> 'long_tensor_name_foo' and 'scores' -> + 'generated_tensor_name_bar') and allow user to refer to these tensors with + their logical names when running inference. - * `export_path` is the path of the export directory. `export` will create the - directory if it does not exist. - - * `tf.constant(FLAGS.export_version)` is a tensor that specifies the - **version** of the model. You should specify a larger integer value when - exporting a newer version of the same model. Each version will be exported to - a different sub-directory under the given path. - - * `sess` is the TensorFlow session that holds the trained model you are - exporting. + * `method_name` is the method used for the inference. For Prediction + requests, it should be set to `tensorflow/serving/predict`. Let's run it! @@ -107,8 +103,8 @@ $>rm -rf /tmp/mnist_model ~~~ ~~~shell -$>bazel build //tensorflow_serving/example:mnist_export -$>bazel-bin/tensorflow_serving/example/mnist_export /tmp/mnist_model +$>bazel build //tensorflow_serving/example:mnist_saved_model +$>bazel-bin/tensorflow_serving/example/mnist_saved_model /tmp/mnist_model Training model... ... @@ -122,27 +118,25 @@ Now let's take a look at the export directory. ~~~shell $>ls /tmp/mnist_model -00000001 +1 ~~~ As mentioned above, a sub-directory will be created for exporting each version -of the model. You specified `tf.constant(FLAGS.export_version)` as the model -version above, and `FLAGS.export_version` has the default value of 1, therefore -the corresponding sub-directory `00000001` is created. +of the model. `FLAGS.model_version` has the default value of 1, therefore +the corresponding sub-directory `1` is created. ~~~shell -$>ls /tmp/mnist_model/00000001 -checkpoint export-00000-of-00001 export.meta +$>ls /tmp/mnist_model/1 +saved_model.pb variables ~~~ Each version sub-directory contains the following files: - * `export.meta` is the serialized tensorflow::MetaGraphDef of the model. It - includes the graph definition of the model, as well as metadata of the model - such as signatures. + * `saved_model.pb` is the serialized tensorflow::SavedModel. It includes the + the one or more graph definitions of the model, as well as metadata of the + model such as signatures. - * `export-?????-of-?????` are files that hold the serialized variables of - the graph. + * `variables` are files that hold the serialized variables of the graphs. With that, your TensorFlow model is exported and ready to be loaded! @@ -150,7 +144,7 @@ With that, your TensorFlow model is exported and ready to be loaded! ~~~shell $>bazel build //tensorflow_serving/model_servers:tensorflow_model_server -$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ +$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ --logtostderr ~~~ ## Test The Server diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 9447c40fae7..514495fd5a8 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -54,7 +54,7 @@ Usage: model_server [--port=8500] [--enable_batching] [--model_name=my_name] --m ### Export Inception model in container In the running container, we run -[inception_export.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/inception_export.py) +[inception_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/inception_saved_model.py) to export the inception model using the released [Inception model training checkpoint](http://download.tensorflow.org/models/image/imagenet/inception-v3-2016-03-01.tar.gz). Instead of training from scratch, we use the readily available checkpoints @@ -66,11 +66,11 @@ root@c97d8e820ced:/serving# curl -O http://download.tensorflow.org/models/image/ root@c97d8e820ced:/serving# tar xzf inception-v3-2016-03-01.tar.gz root@c97d8e820ced:/serving# ls inception-v3 README.txt checkpoint model.ckpt-157585 -root@c97d8e820ced:/serving# bazel-bin/tensorflow_serving/example/inception_export --checkpoint_dir=inception-v3 --export_dir=inception-export +root@c97d8e820ced:/serving# bazel-bin/tensorflow_serving/example/inception_saved_model --checkpoint_dir=inception-v3 --output_dir=inception-export Successfully loaded model from inception-v3/model.ckpt-157585 at step=157585. Successfully exported model to inception-export root@c97d8e820ced:/serving# ls inception-export -00157585 +1 root@c97d8e820ced:/serving# [Ctrl-p] + [Ctrl-q] ``` diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 49a6973d446..e5e1a9fcccb 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -344,7 +344,7 @@ cc_library( "//visibility:public", ], deps = [ - ":session_bundle_source_adapter", + ":saved_model_bundle_source_adapter", ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:aspired_versions_manager_builder", "//tensorflow_serving/core:availability_preserving_policy", @@ -356,7 +356,7 @@ cc_library( "//tensorflow_serving/core:target", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", - "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", ], @@ -365,7 +365,7 @@ cc_library( cc_test( name = "simple_servers_test", srcs = ["simple_servers_test.cc"], - data = ["@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_half_plus_two"], + data = ["@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two"], # Link in all registered kernels. linkstatic = 1, deps = [ @@ -374,7 +374,7 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", "//tensorflow_serving/util:unique_ptr_with_deps", - "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", diff --git a/tensorflow_serving/servables/tensorflow/simple_servers.cc b/tensorflow_serving/servables/tensorflow/simple_servers.cc index 50e38990fda..1cb8904858c 100644 --- a/tensorflow_serving/servables/tensorflow/simple_servers.cc +++ b/tensorflow_serving/servables/tensorflow/simple_servers.cc @@ -19,7 +19,6 @@ limitations under the License. #include #include -#include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/core/aspired_versions_manager_builder.h" @@ -29,7 +28,7 @@ limitations under the License. #include "tensorflow_serving/core/source_adapter.h" #include "tensorflow_serving/core/storage_path.h" #include "tensorflow_serving/core/target.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" @@ -60,15 +59,15 @@ Status CreateStoragePathSource( return Status::OK(); } -// Creates a SessionBundle Source by adapting the underlying +// Creates a SavedModelBundle Source by adapting the underlying // FileSystemStoragePathSource. These two are connected in the // 'CreateSingleTFModelManagerFromBasePath' method, with the -// FileSystemStoragePathSource as the Source and the SessionBundleSource as the -// Target. -Status CreateSessionBundleSource( - std::unique_ptr* source) { +// FileSystemStoragePathSource as the Source and the SavedModelBundleSource as +// the Target. +Status CreateSavedModelBundleSource( + std::unique_ptr* source) { SessionBundleSourceAdapterConfig config; - TF_RETURN_IF_ERROR(SessionBundleSourceAdapter::Create(config, source)); + TF_RETURN_IF_ERROR(SavedModelBundleSourceAdapter::Create(config, source)); return Status::OK(); } @@ -77,8 +76,8 @@ Status CreateSessionBundleSource( Status CreateSingleTFModelManagerFromBasePath( const string& base_path, std::unique_ptr* const manager) { - std::unique_ptr bundle_source; - TF_RETURN_IF_ERROR(CreateSessionBundleSource(&bundle_source)); + std::unique_ptr bundle_source; + TF_RETURN_IF_ERROR(CreateSavedModelBundleSource(&bundle_source)); std::unique_ptr> path_source; TF_RETURN_IF_ERROR( CreateStoragePathSource(base_path, "default", &path_source)); diff --git a/tensorflow_serving/servables/tensorflow/simple_servers.h b/tensorflow_serving/servables/tensorflow/simple_servers.h index e3bab8869e8..93a773f08bd 100644 --- a/tensorflow_serving/servables/tensorflow/simple_servers.h +++ b/tensorflow_serving/servables/tensorflow/simple_servers.h @@ -16,15 +16,15 @@ limitations under the License. // Bootstrapping and configuration utilities for creating simple servers of // TensorFlow models. Intended for basic instantiation with default configs. // -// Note: All methods expect TensorFlow exports conforming to the export format -// specified at tensorflow_serving/servables/tensorflow/README.md. +// Note: All methods expect TensorFlow SavedModel conforming to the format +// specified at third_party/tensorflow/python/saved_model/README.md. #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SIMPLE_SERVERS_H_ #include #include -#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/cc/saved_model/loader.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/manager.h" @@ -43,7 +43,7 @@ namespace simple_servers { // SessionOptions. // // The servables loaded and served from this manager are of type -// tensorflow::serving::SessionBundle. +// tensorflow::SavedModelBundle. // // When new versions arrive the Manager will unload the previous version before // loading the new version. This is preferable from a resource utilization diff --git a/tensorflow_serving/servables/tensorflow/simple_servers_test.cc b/tensorflow_serving/servables/tensorflow/simple_servers_test.cc index fe5ce7a0c4e..ec31925921b 100644 --- a/tensorflow_serving/servables/tensorflow/simple_servers_test.cc +++ b/tensorflow_serving/servables/tensorflow/simple_servers_test.cc @@ -21,7 +21,7 @@ limitations under the License. #include #include -#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/cc/saved_model/loader.h" #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" @@ -40,12 +40,12 @@ namespace { class SimpleServersTest : public ::testing::Test { protected: SimpleServersTest() - : test_data_path_(test_util::ContribTestSrcDirPath( - "session_bundle/testdata/half_plus_two")) {} + : test_data_path_(test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two")) {} - // Test that a SessionBundle handles a single request for the half plus two + // Test that a SavedModelBundle handles a single request for the half plus two // model properly. The request has size=2, for batching purposes. - void TestSingleRequest(const SessionBundle& bundle) { + void TestSingleRequest(const SavedModelBundle& bundle) { const Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); // half plus two: output should be input / 2 + 2. const Tensor expected_output = @@ -81,7 +81,7 @@ TEST_F(SimpleServersTest, Basic) { while (manager->ListAvailableServableIds().empty()) { Env::Default()->SleepForMicroseconds(1000); } - ServableHandle bundle; + ServableHandle bundle; const Status handle_status = manager->GetServableHandle(ServableRequest::Latest("default"), &bundle); TF_CHECK_OK(handle_status); From 84a49fab91e94221cd8c0dafc61ed35c83121978 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 31 Jan 2017 14:40:30 -0800 Subject: [PATCH 0149/8554] Add a resource dimension called "model slots" that bounds the number of models that can share a given model server. Change: 146167969 --- tensorflow_serving/resources/resource_values.cc | 1 + tensorflow_serving/resources/resource_values.h | 7 ++++++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/resources/resource_values.cc b/tensorflow_serving/resources/resource_values.cc index bf34876311b..6f3e05e531f 100644 --- a/tensorflow_serving/resources/resource_values.cc +++ b/tensorflow_serving/resources/resource_values.cc @@ -24,6 +24,7 @@ const char* const kGpu = "gpu"; } // namespace device_types namespace resource_kinds { +const char* const kNumModelSlots = "num_model_slots"; const char* const kRamBytes = "ram_in_bytes"; const char* const kProcessingMillis = "processing_in_millicores"; } // namespace resource_kinds diff --git a/tensorflow_serving/resources/resource_values.h b/tensorflow_serving/resources/resource_values.h index 2bb30dc59e4..81616087043 100644 --- a/tensorflow_serving/resources/resource_values.h +++ b/tensorflow_serving/resources/resource_values.h @@ -26,7 +26,8 @@ namespace serving { // Standard device types. namespace device_types { -// CPU(s) and main memory. +// The primary devices such as CPU(s) and main memory, as well as aspects of the +// server as a whole. extern const char* const kMain; // Graphics processing unit(s). @@ -37,6 +38,10 @@ extern const char* const kGpu; // Standard resource kinds. namespace resource_kinds { +// If a server can accommodate at most N models, depicted as the server having N +// "model slots", this is the number of slots needed or allocated. +extern const char* const kNumModelSlots; + // RAM in bytes. extern const char* const kRamBytes; From 7098d0b7e4c38d78f45662a3e9427c50602a877c Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 31 Jan 2017 16:37:28 -0800 Subject: [PATCH 0150/8554] Minor fix to half_plus_two test path in get_model_metadata_impl_test. Change: 146182478 --- .../servables/tensorflow/get_model_metadata_impl_test.cc | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc index c17664b12d3..e0f993ccb74 100644 --- a/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/get_model_metadata_impl_test.cc @@ -51,8 +51,6 @@ namespace { constexpr char kTestModelName[] = "test_model"; constexpr int kTestModelVersion = 123; const string kSignatureDef = "signature_def"; -constexpr char kSavedModelBundlePath[] = - "cc/saved_model/testdata/half_plus_two"; class GetModelMetadataImplTest : public ::testing::TestWithParam { public: @@ -61,8 +59,8 @@ class GetModelMetadataImplTest : public ::testing::TestWithParam { "/servables/tensorflow/testdata/half_plus_two"); TF_ASSERT_OK(CreateServerCore(session_bundle_path, false, &server_core_)); - const string saved_model_path = - io::JoinPath(testing::TensorFlowSrcRoot(), kSavedModelBundlePath); + const string saved_model_path = test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two"); TF_ASSERT_OK( CreateServerCore(saved_model_path, true, &saved_model_server_core_)); } From 496b230ce668a1233fda99510959331d8427fada Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 1 Feb 2017 09:06:09 -0800 Subject: [PATCH 0151/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index b00fc538638..63b3fea438b 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit b00fc538638f87ac45be9105057b9865f0f9418b +Subproject commit 63b3fea438b1fc700db38b77ca691e083b63bb5f diff --git a/tf_models b/tf_models index a689e124a83..2fd3dcf3f31 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit a689e124a831be313fe9a219ca367477490ad89e +Subproject commit 2fd3dcf3f31707820126a4d9ce595e6a1547385d From 5058f4258871f9205745903c8509e3b994ce117f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 1 Feb 2017 11:47:31 -0800 Subject: [PATCH 0152/8554] Internal change. Change: 146270527 --- tensorflow_serving/apis/BUILD | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 58a20eb3359..b41caaa790c 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -31,8 +31,8 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf//:cc_wkt_protos", + "@protobuf//:any", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -41,7 +41,7 @@ serving_proto_library_py( srcs = ["get_model_metadata.proto"], proto_library = "get_model_metadata_proto", deps = [ - "@org_tensorflow//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow/core:protos_all_py_pb2", ], ) @@ -52,7 +52,7 @@ serving_proto_library( go_api_version = 2, java_api_version = 2, deps = [ - "@protobuf//:cc_wkt_protos", + "@protobuf//:wrappers", ], ) @@ -71,7 +71,7 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -81,7 +81,7 @@ serving_proto_library_py( proto_library = "predict_proto", deps = [ ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow/core:protos_all_py_pb2", ], ) @@ -105,5 +105,6 @@ py_library( deps = [ ":get_model_metadata_proto_py_pb2", ":predict_proto_py_pb2", + "//net/grpc/python:grpc", ], ) From d75aab53ae9821faceeb122dd3366e0f8a8aebc9 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 2 Feb 2017 14:52:34 -0800 Subject: [PATCH 0153/8554] Add an override in ServingSessionWrapper for Run(with_options). Change: 146410397 --- .../servables/tensorflow/serving_session.h | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/servables/tensorflow/serving_session.h b/tensorflow_serving/servables/tensorflow/serving_session.h index 8ea414cabf7..0106524ef2a 100644 --- a/tensorflow_serving/servables/tensorflow/serving_session.h +++ b/tensorflow_serving/servables/tensorflow/serving_session.h @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "tensorflow/core/platform/logging.h" #include "tensorflow/core/public/session.h" namespace tensorflow { @@ -47,7 +48,10 @@ class ServingSession : public Session { class ServingSessionWrapper : public ServingSession { public: explicit ServingSessionWrapper(std::unique_ptr wrapped) - : wrapped_(std::move(wrapped)) {} + : wrapped_(std::move(wrapped)) { + VLOG(2) << "Created the ServingSessionWrapper around the Session."; + } + ~ServingSessionWrapper() override = default; Status Run(const std::vector>& inputs, @@ -58,6 +62,15 @@ class ServingSessionWrapper : public ServingSession { outputs); } + Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, RunMetadata* run_metadata) override { + return wrapped_->Run(run_options, inputs, output_tensor_names, + target_node_names, outputs, run_metadata); + } + private: std::unique_ptr wrapped_; From e0ac7320d441fa78d431d0f48b29d910364781b8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 2 Feb 2017 15:04:41 -0800 Subject: [PATCH 0154/8554] Pass grpc deadline to session Run via RunOptions. Change: 146411764 --- tensorflow_serving/model_servers/main.cc | 14 +++- .../servables/tensorflow/predict_impl.cc | 24 ++++--- .../servables/tensorflow/predict_impl.h | 5 +- .../servables/tensorflow/predict_impl_test.cc | 69 +++++++++++++------ 4 files changed, 77 insertions(+), 35 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index e626d1c4127..41bd41e0cb3 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -138,6 +138,12 @@ ModelServerConfig BuildSingleModelConfig( return config; } +int DeadlineToTimeoutMillis(const gpr_timespec deadline) { + return gpr_time_to_millis( + gpr_time_sub(gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), + gpr_now(GPR_CLOCK_MONOTONIC))); +} + grpc::Status ToGRPCStatus(const tensorflow::Status& status) { const int kErrorMessageLimit = 1024; string error_message; @@ -161,8 +167,12 @@ class PredictionServiceImpl final : public PredictionService::Service { grpc::Status Predict(ServerContext* context, const PredictRequest* request, PredictResponse* response) override { - const grpc::Status status = - ToGRPCStatus(predictor_->Predict(core_.get(), *request, response)); + tensorflow::RunOptions run_options = tensorflow::RunOptions(); + // By default, this is infinite which is the same default as RunOptions. + run_options.set_timeout_in_ms( + DeadlineToTimeoutMillis(context->raw_deadline())); + const grpc::Status status = ToGRPCStatus( + predictor_->Predict(run_options, core_.get(), *request, response)); if (!status.ok()) { VLOG(1) << "Predict failed: " << status.error_message(); } diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index 42bd0971ad3..b0f3b250e9a 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -34,7 +34,8 @@ namespace serving { namespace { // Implementation of Predict using the legacy SessionBundle GenericSignature. -Status SessionBundlePredict(ServerCore* core, const PredictRequest& request, +Status SessionBundlePredict(const RunOptions& run_options, ServerCore* core, + const PredictRequest& request, PredictResponse* response) { // Validate signatures. ServableHandle bundle; @@ -111,8 +112,9 @@ Status SessionBundlePredict(ServerCore* core, const PredictRequest& request, // Run session. std::vector outputs; - TF_RETURN_IF_ERROR( - bundle->session->Run(inputs, output_tensor_names, {}, &outputs)); + RunMetadata run_metadata; + TF_RETURN_IF_ERROR(bundle->session->Run( + run_options, inputs, output_tensor_names, {}, &outputs, &run_metadata)); // Validate and return output. if (outputs.size() != output_tensor_names.size()) { @@ -232,7 +234,8 @@ Status PostProcessPredictionResult( } // Implementation of Predict using the SavedModel SignatureDef format. -Status SavedModelPredict(ServerCore* core, const PredictRequest& request, +Status SavedModelPredict(const RunOptions& run_options, ServerCore* core, + const PredictRequest& request, PredictResponse* response) { // Validate signatures. ServableHandle bundle; @@ -255,8 +258,10 @@ Status SavedModelPredict(ServerCore* core, const PredictRequest& request, &output_tensor_names, &output_tensor_aliases)); std::vector outputs; - TF_RETURN_IF_ERROR( - bundle->session->Run(input_tensors, output_tensor_names, {}, &outputs)); + RunMetadata run_metadata; + TF_RETURN_IF_ERROR(bundle->session->Run(run_options, input_tensors, + output_tensor_names, {}, &outputs, + &run_metadata)); return PostProcessPredictionResult(signature, output_tensor_aliases, outputs, response); @@ -264,7 +269,8 @@ Status SavedModelPredict(ServerCore* core, const PredictRequest& request, } // namespace -Status TensorflowPredictor::Predict(ServerCore* core, +Status TensorflowPredictor::Predict(const RunOptions& run_options, + ServerCore* core, const PredictRequest& request, PredictResponse* response) { if (!request.has_model_spec()) { @@ -272,9 +278,9 @@ Status TensorflowPredictor::Predict(ServerCore* core, "Missing ModelSpec"); } if (use_saved_model_) { - return SavedModelPredict(core, request, response); + return SavedModelPredict(run_options, core, request, response); } - return SessionBundlePredict(core, request, response); + return SessionBundlePredict(run_options, core, request, response); } } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.h b/tensorflow_serving/servables/tensorflow/predict_impl.h index d23718eeb35..38bf7398156 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.h +++ b/tensorflow_serving/servables/tensorflow/predict_impl.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_PREDICT_IMPL_H_ #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow_serving/apis/predict.pb.h" #include "tensorflow_serving/model_servers/server_core.h" @@ -29,8 +30,8 @@ class TensorflowPredictor { explicit TensorflowPredictor(bool use_saved_model) : use_saved_model_(use_saved_model) {} - Status Predict(ServerCore* core, const PredictRequest& request, - PredictResponse* response); + Status Predict(const RunOptions& run_options, ServerCore* core, + const PredictRequest& request, PredictResponse* response); private: // If use_saved_model_ is true, a SavedModelBundle handle will be retrieved diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 1822627799d..2dd06ccf822 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -99,6 +99,8 @@ class PredictImplTest : public ::testing::TestWithParam { return server_core_bad_model_.get(); } + RunOptions GetRunOptions() { return RunOptions(); } + private: static std::unique_ptr server_core_; static std::unique_ptr server_core_bad_model_; @@ -117,20 +119,26 @@ TEST_P(PredictImplTest, MissingOrEmptyModelSpec) { // Empty request is invalid. TensorflowPredictor predictor(GetParam()); - EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); ModelSpec* model_spec = request.mutable_model_spec(); model_spec->clear_name(); // Model name is not specified. - EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); // Model name is wrong, not found. model_spec->set_name("test"); - EXPECT_EQ(tensorflow::error::NOT_FOUND, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::NOT_FOUND, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); } TEST_P(PredictImplTest, EmptyInputList) { @@ -143,8 +151,10 @@ TEST_P(PredictImplTest, EmptyInputList) { TensorflowPredictor predictor(GetParam()); // The input is empty. - EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); } TEST_P(PredictImplTest, InputTensorsDontMatchModelSpecInputs) { @@ -163,8 +173,10 @@ TEST_P(PredictImplTest, InputTensorsDontMatchModelSpecInputs) { TensorflowPredictor predictor(GetParam()); auto inputs = request.mutable_inputs(); (*inputs)["key"] = tensor_proto; - EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); } TEST_P(PredictImplTest, OutputFiltersDontMatchModelSpecOutputs) { @@ -183,17 +195,22 @@ TEST_P(PredictImplTest, OutputFiltersDontMatchModelSpecOutputs) { TensorflowPredictor predictor(GetParam()); // Output filter like this doesn't exist. - EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); request.clear_output_filter(); request.add_output_filter(kOutputTensorKey); - TF_EXPECT_OK(predictor.Predict(GetServerCore(), request, &response)); + TF_EXPECT_OK( + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response)); request.add_output_filter(kOutputTensorKey); // Duplicate output filter specified. - EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); } TEST_P(PredictImplTest, InputTensorsHaveWrongType) { @@ -213,8 +230,10 @@ TEST_P(PredictImplTest, InputTensorsHaveWrongType) { TensorflowPredictor predictor(GetParam()); // Input tensors are all wrong. - EXPECT_EQ(tensorflow::error::INTERNAL, - predictor.Predict(GetServerCore(), request, &response).code()); + EXPECT_EQ( + tensorflow::error::INTERNAL, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); } TEST_P(PredictImplTest, ModelMissingSignatures) { @@ -228,7 +247,9 @@ TEST_P(PredictImplTest, ModelMissingSignatures) { // Model is missing signatures. TensorflowPredictor predictor(GetParam()); EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, - predictor.Predict(GetServerCoreWithBadModel(), request, &response) + predictor + .Predict(GetRunOptions(), GetServerCoreWithBadModel(), request, + &response) .code()); } @@ -246,7 +267,8 @@ TEST_P(PredictImplTest, PredictionSuccess) { (*request.mutable_inputs())[kInputTensorKey] = tensor_proto; TensorflowPredictor predictor(GetParam()); - TF_EXPECT_OK(predictor.Predict(GetServerCore(), request, &response)); + TF_EXPECT_OK( + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response)); TensorProto output_tensor_proto; output_tensor_proto.add_float_val(3); output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); @@ -276,10 +298,13 @@ TEST_P(PredictImplTest, PredictionWithNamedRegressionSignature) { TensorflowPredictor predictor(GetParam()); // This request is expected to work with SavedModel, but not SessionBundle. if (GetParam()) { - TF_ASSERT_OK(predictor.Predict(GetServerCore(), request, &response)); + TF_ASSERT_OK(predictor.Predict(GetRunOptions(), GetServerCore(), request, + &response)); } else { - ASSERT_EQ(tensorflow::error::INVALID_ARGUMENT, - predictor.Predict(GetServerCore(), request, &response).code()); + ASSERT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); return; } TensorProto output_tensor_proto; From 553c004a0e3cadd1a7b0e3d4861e0681853b74ff Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 2 Feb 2017 18:39:41 -0800 Subject: [PATCH 0155/8554] Move classification and regression interfaces to open source. Note that implementation of these APIs is yet to be moved, so currently model server will throw an unimplemented error. Change: 146432082 --- tensorflow_serving/apis/BUILD | 90 +++++++++++++++++-- tensorflow_serving/apis/classification.proto | 46 ++++++++++ tensorflow_serving/apis/classifier.h | 38 ++++++++ tensorflow_serving/apis/input.proto | 76 ++++++++++++++++ .../apis/prediction_service.proto | 8 ++ tensorflow_serving/apis/regression.proto | 35 ++++++++ tensorflow_serving/apis/regressor.h | 39 ++++++++ tensorflow_serving/model_servers/main.cc | 18 ++++ 8 files changed, 343 insertions(+), 7 deletions(-) create mode 100644 tensorflow_serving/apis/classification.proto create mode 100644 tensorflow_serving/apis/classifier.h create mode 100644 tensorflow_serving/apis/input.proto create mode 100644 tensorflow_serving/apis/regression.proto create mode 100644 tensorflow_serving/apis/regressor.h diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index b41caaa790c..485e1e244ba 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -31,8 +31,8 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@protobuf//:any", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf//:cc_wkt_protos", ], ) @@ -41,10 +41,29 @@ serving_proto_library_py( srcs = ["get_model_metadata.proto"], proto_library = "get_model_metadata_proto", deps = [ - "@org_tensorflow//tensorflow/core:protos_all_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", ], ) +serving_proto_library( + name = "input_proto", + srcs = ["input.proto"], + cc_api_version = 2, + go_api_version = 2, + java_api_version = 2, + deps = [ + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf//:cc_wkt_protos", + ], +) + +serving_proto_library_py( + name = "input_proto_py_pb2", + srcs = ["input.proto"], + proto_library = "input_proto", + deps = [], +) + serving_proto_library( name = "model_proto", srcs = ["model.proto"], @@ -52,7 +71,7 @@ serving_proto_library( go_api_version = 2, java_api_version = 2, deps = [ - "@protobuf//:wrappers", + "@protobuf//:cc_wkt_protos", ], ) @@ -71,7 +90,7 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -81,7 +100,7 @@ serving_proto_library_py( proto_library = "predict_proto", deps = [ ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", ], ) @@ -94,8 +113,10 @@ serving_proto_library( go_api_version = 2, java_api_version = 2, deps = [ + ":classification_proto", ":get_model_metadata_proto", ":predict_proto", + ":regression_proto", ], ) @@ -105,6 +126,61 @@ py_library( deps = [ ":get_model_metadata_proto_py_pb2", ":predict_proto_py_pb2", - "//net/grpc/python:grpc", + ], +) + +serving_proto_library( + name = "classification_proto", + srcs = ["classification.proto"], + cc_api_version = 2, + go_api_version = 2, + java_api_version = 2, + deps = [ + ":input_proto", + ":model_proto", + ], +) + +serving_proto_library_py( + name = "classification_proto_py_pb2", + srcs = ["classification.proto"], + proto_library = "classification_proto", + deps = [], +) + +serving_proto_library( + name = "regression_proto", + srcs = ["regression.proto"], + cc_api_version = 2, + go_api_version = 2, + java_api_version = 2, + deps = [ + ":input_proto", + ":model_proto", + ], +) + +serving_proto_library_py( + name = "regression_proto_py_pb2", + srcs = ["regression.proto"], + proto_library = "regression_proto", + deps = [], +) + +cc_library( + name = "classifier", + hdrs = ["classifier.h"], + deps = [ + ":classification_proto", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "regressor", + hdrs = ["regressor.h"], + deps = [ + ":regression_proto", + "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/apis/classification.proto b/tensorflow_serving/apis/classification.proto new file mode 100644 index 00000000000..a91c80f2d62 --- /dev/null +++ b/tensorflow_serving/apis/classification.proto @@ -0,0 +1,46 @@ +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/input.proto"; +import "tensorflow_serving/apis/model.proto"; + +package tensorflow.serving; + +// A single class. +message Class { + // Label or name of the class. + string label = 1; + // Score for this class (e.g., the probability the item belongs to this + // class). + float score = 2; +} + +// List of classes for a single item +// (tensorflow.Example or tensorflow.InferenceExample.features). +message Classifications { + // List of Classifications for an item sorted by decreasing score. + repeated Class classes = 1; +} + +// For tensorflow.Example this will contain one result. +// For tensorflow.InferenceExample this will contain one result for each +// InferenceExample::features and in the same order as the features. +message ClassificationResult { + repeated Classifications classifications = 1; +} + +// RPC Interfaces + +message ClassificationRequest { + // Model Specification. + ModelSpec model_spec = 1; + + // Input data. + tensorflow.serving.Input input = 2; +} + +message ClassificationResponse { + // Result of the classification. + ClassificationResult result = 1; +} diff --git a/tensorflow_serving/apis/classifier.h b/tensorflow_serving/apis/classifier.h new file mode 100644 index 00000000000..50fb1466006 --- /dev/null +++ b/tensorflow_serving/apis/classifier.h @@ -0,0 +1,38 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// +// Interface for performing classification using classification messages. + +#ifndef TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ +#define TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/classification.pb.h" + +namespace tensorflow { +namespace serving { + +class ClassifierInterface { + public: + virtual Status Classify(const ClassificationRequest& request, + ClassificationResult* result) = 0; + + virtual ~ClassifierInterface() = default; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ diff --git a/tensorflow_serving/apis/input.proto b/tensorflow_serving/apis/input.proto new file mode 100644 index 00000000000..d79af2e5fe1 --- /dev/null +++ b/tensorflow_serving/apis/input.proto @@ -0,0 +1,76 @@ +// Input used in serving APIs. Based on the tensorflow.Example family of +// feature representations. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow/core/example/example.proto"; + +package tensorflow.serving; + +// Specifies one or more fully independent input Examples. +// See examples at: +// https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/example/example.proto +message ExampleList { + repeated tensorflow.Example examples = 1; +} + +// Specifies one or more independent input Examples, with a common context +// Example. +// +// The common use case for context is to cleanly and optimally specify some +// features that are common across multiple examples. +// +// See example below with a search query as the context and multiple restaurants +// to perform some inference on. +// +// context: { +// feature: { +// key : "query" +// value: { +// bytes_list: { +// value: [ "pizza" ] +// } +// } +// } +// } +// examples: { +// feature: { +// key : "cuisine" +// value: { +// bytes_list: { +// value: [ "Pizzeria" ] +// } +// } +// } +// } +// examples: { +// feature: { +// key : "cuisine" +// value: { +// bytes_list: { +// value: [ "Taqueria" ] +// } +// } +// } +// } +// +// Implementations of ExampleListWithContext merge the context Example into each +// of the Examples. Note that feature keys must not be duplicated between the +// Examples and context Example, or the behavior is undefined. +// +// See also: +// tensorflow/core/example/example.proto +// https://developers.google.com/protocol-buffers/docs/proto3#maps +message ExampleListWithContext { + repeated tensorflow.Example examples = 1; + tensorflow.Example context = 2; +} + +message Input { + oneof kind { + ExampleList example_list = 1; + ExampleListWithContext example_list_with_context = 2; + } +} diff --git a/tensorflow_serving/apis/prediction_service.proto b/tensorflow_serving/apis/prediction_service.proto index 3f978b2c540..9ed0a0096dd 100644 --- a/tensorflow_serving/apis/prediction_service.proto +++ b/tensorflow_serving/apis/prediction_service.proto @@ -3,13 +3,21 @@ syntax = "proto3"; package tensorflow.serving; option cc_enable_arenas = true; +import "tensorflow_serving/apis/classification.proto"; import "tensorflow_serving/apis/get_model_metadata.proto"; import "tensorflow_serving/apis/predict.proto"; +import "tensorflow_serving/apis/regression.proto"; // open source marker; do not remove // PredictionService provides access to machine-learned models loaded by // model_servers. service PredictionService { + // Classify. + rpc Classify(ClassificationRequest) returns (ClassificationResponse); + + // Regress. + rpc Regress(RegressionRequest) returns (RegressionResponse); + // Predict -- provides access to loaded TensorFlow model. rpc Predict(PredictRequest) returns (PredictResponse); diff --git a/tensorflow_serving/apis/regression.proto b/tensorflow_serving/apis/regression.proto new file mode 100644 index 00000000000..48d8513cfe9 --- /dev/null +++ b/tensorflow_serving/apis/regression.proto @@ -0,0 +1,35 @@ +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/input.proto"; +import "tensorflow_serving/apis/model.proto"; + +package tensorflow.serving; + +// Regression result for a single item +// (tensorflow.Example or tensorflow.InferenceExample.features). +message Regression { + float value = 1; +} + +// For tensorflow.Example this will contain one result. +// For tensorflow.InferenceExample this will contain one result for each +// InferenceExample::features. +message RegressionResult { + repeated Regression regressions = 1; +} + +// RPC interfaces. + +message RegressionRequest { + // Model Specification. + ModelSpec model_spec = 1; + + // Input data. + tensorflow.serving.Input input = 2; +} + +message RegressionResponse { + RegressionResult result = 1; +} diff --git a/tensorflow_serving/apis/regressor.h b/tensorflow_serving/apis/regressor.h new file mode 100644 index 00000000000..95aadc873eb --- /dev/null +++ b/tensorflow_serving/apis/regressor.h @@ -0,0 +1,39 @@ + +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ +// +// Interface for performing regression using regression messages. + +#ifndef TENSORFLOW_SERVING_APIS_REGRESSOR_H +#define TENSORFLOW_SERVING_APIS_REGRESSOR_H + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/regression.pb.h" + +namespace tensorflow { +namespace serving { + +class RegressorInterface { + public: + virtual Status Regress(const RegressionRequest& request, + RegressionResult* result) = 0; + + virtual ~RegressorInterface() = default; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_APIS_REGRESSOR_H_ diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 41bd41e0cb3..c30931d54c9 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -103,10 +103,14 @@ using grpc::ServerAsyncResponseWriter; using grpc::ServerBuilder; using grpc::ServerContext; using grpc::ServerCompletionQueue; +using tensorflow::serving::ClassificationRequest; +using tensorflow::serving::ClassificationResponse; using tensorflow::serving::GetModelMetadataRequest; using tensorflow::serving::GetModelMetadataResponse; using tensorflow::serving::PredictRequest; using tensorflow::serving::PredictResponse; +using tensorflow::serving::RegressionRequest; +using tensorflow::serving::RegressionResponse; using tensorflow::serving::PredictionService; namespace { @@ -196,6 +200,20 @@ class PredictionServiceImpl final : public PredictionService::Service { return status; } + grpc::Status Classify(ServerContext* context, + const ClassificationRequest* request, + ClassificationResponse* response) override { + return ToGRPCStatus(tensorflow::errors::Unimplemented( + "Classify API is not implemented")); + } + + grpc::Status Regress(ServerContext* context, + const RegressionRequest* request, + RegressionResponse* response) override { + return ToGRPCStatus(tensorflow::errors::Unimplemented( + "Regress API is not implemented")); + } + private: std::unique_ptr core_; std::unique_ptr predictor_; From d3eca0a23d84222745d19d80cbb620363bbf24cc Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 3 Feb 2017 09:01:52 -0800 Subject: [PATCH 0156/8554] Update batch scheduling parameter defaults and guidelines, based on more recent empirical experience. Change: 146479908 --- tensorflow_serving/batching/README.md | 55 +++++++++++++------ .../batching/basic_batch_scheduler.h | 13 +++-- .../batching/shared_batch_scheduler.h | 14 +++-- .../batching/streaming_batch_scheduler.h | 5 +- 4 files changed, 55 insertions(+), 32 deletions(-) diff --git a/tensorflow_serving/batching/README.md b/tensorflow_serving/batching/README.md index eff48c18146..1c3916a0639 100644 --- a/tensorflow_serving/batching/README.md +++ b/tensorflow_serving/batching/README.md @@ -112,17 +112,47 @@ enqueued to the scheduler. Used to bound queueing delay, by turning away requests that would take a long time to get to, rather than building up a large backlog. -### Recommended Approach to Choosing Scheduling Parameters +### Performance Tuning -Here is one way to choose values for the aforementioned parameters: +The best values to use for the batch scheduling parameters depend on your model, +system and environment, as well as your throughput and latency goals. Choosing +good values is best done via experiments. Here are some guidelines that may be +helpful in selecting values to experiment with. + +#### Overall Guidelines + +First of all, while experimenting you should temporarily set +`max_enqueued_batches` to infinity. Later, for your production setup, set it as +follows: If you are performing online serving, depending on the policy used to +(re-)route requests to server instances, consider setting `max_enqueued_batches` +equal to `num_batch_threads` to minimize queueing delay at a given server while +keeping it busy. For bulk processing jobs, set `max_enqueued_batches` to a +generous value, but low enough to avoid out-of-memory crashes. + +Second, if for system architecture reasons you need to constrain the set of +possible batch sizes (e.g. just 100, 200 or 400, rather than any value between 1 +and 400): If you are using `BatchingSession` you can set the +`allowed_batch_sizes` parameter. Otherwise, you can arrange for your callback +code to pad the batches with dummy elements. + +#### CPU-only: One Approach + +If your system is CPU-only (no GPU), then consider starting with the following +values: `num_batch_threads` equal to the number of CPU cores; `max_batch_size` +to infinity; `batch_timeout_micros` to 0. Then experiment with +`batch_timeout_micros` values in the 1-10 millisecond (1000-10000 microsecond) +range, while keeping in mind that 0 may be the optimal value. + +#### GPU: One Approach + +If your model uses a GPU device for part or all of your its inference work, +consider the following approach: 1. Set `num_batch_threads` to the number of CPU cores. -2. Temporarily set `batch_timeout_micros` and `max_enqueued_batches` to infinity -while you tune `max_batch_size` to achieve the desired balance between -throughput and average latency. The best value is typically in the hundreds or -thousands, and depends on your model, system and environment, as well as your -throughput and latency goals. +2. Temporarily set `batch_timeout_micros` to infinity while you tune +`max_batch_size` to achieve the desired balance between throughput and average +latency. Consider values in the hundreds or thousands. 3. For online serving, tune `batch_timeout_micros` to rein in tail latency. The idea is that batches normally get filled to `max_batch_size`, but occasionally @@ -134,17 +164,6 @@ consider; it works well for some workloads. (For bulk processing jobs, choose a large value, perhaps a few seconds, to ensure good throughput but not wait too long for the final (and likely underfull) batch.) -4. For online serving, depending on the policy used to (re-)route requests to -server instances, consider setting `max_enqueued_batches` equal to -`num_batch_threads` to minimize queueing delay at a given server while keeping -it busy. (For bulk processing jobs, set `max_enqueued_batches` to a generous -value, but low enough to avoid out-of-memory crashes.) - -5. If you need to constrain the set of possible batch sizes (e.g. just 100, 200 -or 400, rather than any value between 1 and 400): If you are using -`BatchingSession` you can set the `allowed_batch_sizes` parameter. Otherwise, -you can arrange for your callback code to pad the batches with dummy elements. - ## Servers with Multiple Models, Model Versions or Subtasks Some server instances service multiple request types (e.g. multiple models, or diff --git a/tensorflow_serving/batching/basic_batch_scheduler.h b/tensorflow_serving/batching/basic_batch_scheduler.h index 2029502dc4e..1124a49b7e8 100644 --- a/tensorflow_serving/batching/basic_batch_scheduler.h +++ b/tensorflow_serving/batching/basic_batch_scheduler.h @@ -130,6 +130,9 @@ namespace serving { // individual per-request units. // 6. Perform any post-processing in the batch thread and/or request thread. // +// +// PERFORMANCE TUNING: See README.md. +// template class BasicBatchScheduler : public BatchScheduler { public: @@ -156,24 +159,22 @@ class BasicBatchScheduler : public BatchScheduler { // above.) // // The goal is to smooth out batch sizes under low request rates, and thus - // avoid latency spikes. The default value of 1 millisecond was determined - // via benchmarking. You may need to adjust it to suit your workload and - // environment. - int64 batch_timeout_micros = 1 * 1000 /* 1 millisecond */; + // avoid latency spikes. + int64 batch_timeout_micros = 0; // The name to use for the pool of batch threads. string thread_pool_name = {"batch_threads"}; // The number of threads to use to process batches. // Must be >= 1, and should be tuned carefully. - int num_batch_threads = 1; + int num_batch_threads = port::NumSchedulableCPUs(); // The maximum allowable number of enqueued (accepted by Schedule() but // not yet being processed on a batch thread) tasks in terms of batches. // If this limit is reached, Schedule() will return an UNAVAILABLE error. // See the class documentation above for guidelines on how to tune this // parameter. - int max_enqueued_batches = 1; + int max_enqueued_batches = 10; // The following options are typically only overridden by test code. diff --git a/tensorflow_serving/batching/shared_batch_scheduler.h b/tensorflow_serving/batching/shared_batch_scheduler.h index 4be8aa12a96..32351dc0ff4 100644 --- a/tensorflow_serving/batching/shared_batch_scheduler.h +++ b/tensorflow_serving/batching/shared_batch_scheduler.h @@ -28,6 +28,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/platform/types.h" @@ -93,6 +94,9 @@ namespace serving { // E.g. let each queue specify a "share" (an int >= 1), so e.g. with queues A // and B having shares 1 and 2 respectively, the servicing pattern is ABBABB... // +// +// PERFORMANCE TUNING: See README.md. +// template class SharedBatchScheduler : public std::enable_shared_from_this> { @@ -104,7 +108,7 @@ class SharedBatchScheduler // The number of threads to use to process batches. // Must be >= 1, and should be tuned carefully. - int num_batch_threads = 1; + int num_batch_threads = port::NumSchedulableCPUs(); // The environment to use. // (Typically only overridden by test code.) @@ -144,17 +148,15 @@ class SharedBatchScheduler // above.) // // The goal is to smooth out batch sizes under low request rates, and thus - // avoid latency spikes. The default value of 1 millisecond was determined - // via benchmarking. You may need to adjust it to suit your workload and - // environment. - int64 batch_timeout_micros = 1 * 1000 /* 1 millisecond */; + // avoid latency spikes. + int64 batch_timeout_micros = 0; // The maximum allowable number of enqueued (accepted by Schedule() but // not yet being processed on a batch thread) tasks in terms of batches. // If this limit is reached, Schedule() will return an UNAVAILABLE error. // See the class documentation above for guidelines on how to tune this // parameter. - int max_enqueued_batches = 1; + int max_enqueued_batches = 10; }; Status AddQueue(const QueueOptions& options, std::function>)> diff --git a/tensorflow_serving/batching/streaming_batch_scheduler.h b/tensorflow_serving/batching/streaming_batch_scheduler.h index f9c58e7c443..b883678d800 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler.h +++ b/tensorflow_serving/batching/streaming_batch_scheduler.h @@ -27,6 +27,7 @@ limitations under the License. #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/threadpool.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow/core/platform/macros.h" @@ -135,14 +136,14 @@ class StreamingBatchScheduler : public BatchScheduler { // // A negative value means that no timeout will be enforced. This setting is // useful in some test code. - int64 batch_timeout_micros = 10 * 1000 /* 10 milliseconds */; + int64 batch_timeout_micros = 0; // The name to use for the pool of batch threads. string thread_pool_name = "batch_threads"; // The number of threads to use to process batches. // Must be >= 1, and should be tuned carefully. - int num_batch_threads = 1; + int num_batch_threads = port::NumSchedulableCPUs(); // The following options are typically only overridden by test code. From 2c2ddb0167c060f267031cdf74d0c4612f723750 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 3 Feb 2017 12:09:44 -0800 Subject: [PATCH 0157/8554] Implement dummy Run with RunOptions for batching session. Add a high level test to ensure that batching session option works. Change: 146501600 --- .../batching/batching_session.cc | 19 ++++++++++++ .../tensorflow_model_server_test.py | 30 ++++++++++++++----- 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index f6b71df3aa7..4188c7c42b3 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -126,6 +126,14 @@ class BatchingSession : public ServingSession { const std::vector& target_node_names, std::vector* outputs) override; + // TODO(b/34971139): at the moment this method ignores run_options and + // run_metadata and behaves exactly like Run. + Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, RunMetadata* run_metadata) override; + private: explicit BatchingSession(const BatchingSessionOptions& options); @@ -201,6 +209,17 @@ Status BatchingSession::Create( return Status::OK(); } +Status BatchingSession::Run( + const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, std::vector* outputs, + RunMetadata* run_metadata) { + LOG(WARNING) << "Currently both run_options and run_metadata are ignored, " + << "see b/34971139"; + return Run(inputs, output_tensor_names, target_node_names, outputs); +} + Status BatchingSession::Run( const std::vector>& inputs, const std::vector& output_tensor_names, diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 247f255e448..23b04adc635 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -67,7 +67,8 @@ def TerminateProcs(self): if self.server_proc is not None: self.server_proc.terminate() - def RunServer(self, port, model_name, model_path, use_saved_model): + def RunServer(self, port, model_name, model_path, use_saved_model, + enable_batching): """Run tensorflow_model_server using test config.""" print 'Starting test server...' command = os.path.join(self.binary_dir, 'tensorflow_model_server') @@ -75,6 +76,7 @@ def RunServer(self, port, model_name, model_path, use_saved_model): command += ' --model_name=' + model_name command += ' --model_base_path=' + model_path command += ' --use_saved_model=' + str(use_saved_model).lower() + command += ' --enable_batching=' + str(enable_batching).lower() command += ' --alsologtostderr' print command self.server_proc = subprocess.Popen(shlex.split(command)) @@ -91,6 +93,9 @@ def VerifyPredictRequest(self, request.model_spec.name = 'default' request.inputs['x'].dtype = types_pb2.DT_FLOAT request.inputs['x'].float_val.append(2.0) + dim = request.inputs['x'].tensor_shape.dim.add() + dim.size = 1 + if specify_output: request.output_filter.append('y') # Send request @@ -113,34 +118,44 @@ def _GetSessionBundlePath(self): """Returns a path to a model in SessionBundle format.""" return os.path.join(self.testdata_dir, 'half_plus_two') - def _TestPredict(self, model_path, use_saved_model): + def _TestPredict(self, model_path, use_saved_model, enable_batching): """Helper method to test prediction. Args: model_path: Path to the model on disk. use_saved_model: Whether the model server should use SavedModel. + enable_batching: Whether model server should use BatchingSession """ atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', - model_path, use_saved_model) + model_path, use_saved_model, + enable_batching) time.sleep(5) self.VerifyPredictRequest(model_server_address) self.VerifyPredictRequest(model_server_address, specify_output=False) def testPredictSessionBundle(self): """Test PredictionService.Predict implementation with SessionBundle.""" - self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False) + self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False, + enable_batching=False) + + def testPredictBatchingSessionBundle(self): + """Test PredictionService.Predict implementation with SessionBundle.""" + self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False, + enable_batching=True) def testPredictSavedModel(self): """Test PredictionService.Predict implementation with SavedModel.""" - self._TestPredict(self._GetSavedModelBundlePath(), use_saved_model=True) + self._TestPredict(self._GetSavedModelBundlePath(), use_saved_model=True, + enable_batching=False) def testPredictUpconvertedSavedModel(self): """Test PredictionService.Predict implementation. Using a SessionBundle converted to a SavedModel. """ - self._TestPredict(self._GetSessionBundlePath(), use_saved_model=True) + self._TestPredict(self._GetSessionBundlePath(), use_saved_model=True, + enable_batching=False) def _TestBadModel(self, use_saved_model): """Helper method to test against a bad model export.""" @@ -149,7 +164,8 @@ def _TestBadModel(self, use_saved_model): # case of SavedModel, the export will get up-converted to a SavedModel. model_server_address = self.RunServer( PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'bad_half_plus_two'), use_saved_model) + os.path.join(self.testdata_dir, 'bad_half_plus_two'), use_saved_model, + enable_batching=False) time.sleep(5) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address) From 21d5a918f0254b19b7ca6a1ecb3d7bfafe87dfce Mon Sep 17 00:00:00 2001 From: Li Lao Date: Fri, 3 Feb 2017 14:44:51 -0800 Subject: [PATCH 0158/8554] Remove EagerLoadPolicy and EagerUnloadPolicy in favor of AvailabilityPreservingPolicy and ResourcePreservingPolicy. Update user doc on the policies. Change: 146519328 --- tensorflow_serving/core/BUILD | 48 ------- tensorflow_serving/core/eager_load_policy.cc | 56 -------- tensorflow_serving/core/eager_load_policy.h | 46 ------- .../core/eager_load_policy_test.cc | 116 ----------------- .../core/eager_unload_policy.cc | 60 --------- tensorflow_serving/core/eager_unload_policy.h | 53 -------- .../core/eager_unload_policy_test.cc | 122 ------------------ .../g3doc/architecture_overview.md | 18 +-- 8 files changed, 10 insertions(+), 509 deletions(-) delete mode 100644 tensorflow_serving/core/eager_load_policy.cc delete mode 100644 tensorflow_serving/core/eager_load_policy.h delete mode 100644 tensorflow_serving/core/eager_load_policy_test.cc delete mode 100644 tensorflow_serving/core/eager_unload_policy.cc delete mode 100644 tensorflow_serving/core/eager_unload_policy.h delete mode 100644 tensorflow_serving/core/eager_unload_policy_test.cc diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 0a71d1d20f3..066fef86c80 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -639,54 +639,6 @@ cc_test( ], ) -cc_library( - name = "eager_load_policy", - srcs = ["eager_load_policy.cc"], - hdrs = ["eager_load_policy.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - ":aspired_version_policy", - ":loader_harness", - "//tensorflow_serving/util:optional", - ], -) - -cc_test( - name = "eager_load_policy_test", - srcs = ["eager_load_policy_test.cc"], - deps = [ - ":eager_load_policy", - ":servable_id", - "//tensorflow_serving/core/test_util:test_main", - ], -) - -cc_library( - name = "eager_unload_policy", - srcs = ["eager_unload_policy.cc"], - hdrs = ["eager_unload_policy.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - ":aspired_version_policy", - ":loader_harness", - "//tensorflow_serving/util:optional", - ], -) - -cc_test( - name = "eager_unload_policy_test", - srcs = ["eager_unload_policy_test.cc"], - deps = [ - ":eager_unload_policy", - ":servable_id", - "//tensorflow_serving/core/test_util:test_main", - ], -) - cc_library( name = "availability_preserving_policy", srcs = ["availability_preserving_policy.cc"], diff --git a/tensorflow_serving/core/eager_load_policy.cc b/tensorflow_serving/core/eager_load_policy.cc deleted file mode 100644 index 5d7c6eb4a5b..00000000000 --- a/tensorflow_serving/core/eager_load_policy.cc +++ /dev/null @@ -1,56 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/core/eager_load_policy.h" -#include "tensorflow_serving/core/loader_harness.h" - -namespace tensorflow { -namespace serving { - -optional EagerLoadPolicy::GetNextAction( - const std::vector& all_versions) const { - // If there is a new aspired version, load it. - for (const auto& version : all_versions) { - if (version.is_aspired && version.state == LoaderHarness::State::kNew) { - VLOG(1) << "EagerLoadPolicy requesting to load servable " << version.id; - return {{Action::kLoad, version.id}}; - } - } - - // Second, check if there are any aspired versions that are not ready. In that - // case we can't unload any versions. - const bool aspired_not_serving = - std::any_of(all_versions.begin(), all_versions.end(), - [](const AspiredServableStateSnapshot& version) { - return version.is_aspired && - version.state != LoaderHarness::State::kReady; - }); - if (aspired_not_serving) { - return nullopt; - } - - // If there is no new aspired version, but a not-aspired version, unload the - // latter. - for (const auto& version : all_versions) { - if (!version.is_aspired && version.state == LoaderHarness::State::kReady) { - VLOG(1) << "EagerLoadPolicy requesting to unload servable " << version.id; - return {{Action::kUnload, version.id}}; - } - } - return nullopt; -} - -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/core/eager_load_policy.h b/tensorflow_serving/core/eager_load_policy.h deleted file mode 100644 index 2a16c5c9d64..00000000000 --- a/tensorflow_serving/core/eager_load_policy.h +++ /dev/null @@ -1,46 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ -#define TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ - -#include - -#include "tensorflow_serving/core/aspired_version_policy.h" -#include "tensorflow_serving/core/loader_harness.h" -#include "tensorflow_serving/util/optional.h" - -namespace tensorflow { -namespace serving { - -// Deprecated. Use AvailabilityPreservePolicy instead. -// -// AspiredVersionPolicy that loads any aspired versions of a servable before -// unloading any no-longer-aspired versions. -// -// This policy provides servable availability with the trade-off of temporary -// increased resource consumption while the new version loads followed by the -// old versions unloading. -class EagerLoadPolicy final : public AspiredVersionPolicy { - public: - optional GetNextAction( - const std::vector& all_versions) - const override; -}; - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_CORE_EAGER_LOAD_POLICY_H_ diff --git a/tensorflow_serving/core/eager_load_policy_test.cc b/tensorflow_serving/core/eager_load_policy_test.cc deleted file mode 100644 index 3b069e38846..00000000000 --- a/tensorflow_serving/core/eager_load_policy_test.cc +++ /dev/null @@ -1,116 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/core/eager_load_policy.h" - -#include -#include "tensorflow_serving/core/servable_id.h" - -namespace tensorflow { -namespace serving { -namespace { - -// Test that the first new and aspired version is loaded first. -TEST(EagerLoadPolicy, LoadsFirstAspired) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 4}, LoaderHarness::State::kNew, true}); - versions.push_back({{"test", 5}, LoaderHarness::State::kReady, false}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); - EXPECT_EQ(4, action->id.version); -} - -// Test that the first non-aspired version is unloaded when there are none to -// load. -TEST(EagerLoadPolicy, UnLoadsFirstNonAspiredWhenNoneLoading) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 2}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 3}, LoaderHarness::State::kDisabled, false}); - versions.push_back({{"test", 4}, LoaderHarness::State::kReady, false}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); - EXPECT_EQ(2, action->id.version); -} - -// Test that no action is returned (empty optional) when there are no versions -// needing loading or unloading. -TEST(EagerLoadPolicy, ReturnsNoActionWhenNone) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kLoading, true}); - versions.push_back({{"test", 4}, LoaderHarness::State::kUnloading, false}); - versions.push_back({{"test", 5}, LoaderHarness::State::kDisabled, false}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -TEST(EagerLoadPolicy, DoesNotUnloadWhenOtherNotReady) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -TEST(EagerLoadPolicy, DoesNotUnloadWhenOtherInError) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -TEST(EagerLoadPolicy, LoadingBlocksUnloadEvenIfOtherReady) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -TEST(EagerLoadPolicy, ErrorBlocksUnloadEvenIfOtherReady) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kReady, true}); - - EagerLoadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/core/eager_unload_policy.cc b/tensorflow_serving/core/eager_unload_policy.cc deleted file mode 100644 index c1acd15cc08..00000000000 --- a/tensorflow_serving/core/eager_unload_policy.cc +++ /dev/null @@ -1,60 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/core/eager_unload_policy.h" - -namespace tensorflow { -namespace serving { - -optional EagerUnloadPolicy::GetNextAction( - const std::vector& all_versions) const { - // First iterate over all_versions and find any in kReady that are no longer - // aspired. Unload the first if any. - for (const auto& version : all_versions) { - if (version.state == LoaderHarness::State::kReady && !version.is_aspired) { - VLOG(1) << "EagerUnloadPolicy requesting to unload servable " - << version.id; - return AspiredVersionPolicy::ServableAction( - {Action::kUnload, version.id}); - } - } - - // Second, see if there are any not-aspired versions that aren't in an end - // state (kDisabled or kError). If so, do nothing for now. - const bool not_aspired_not_finished = - std::any_of(all_versions.begin(), all_versions.end(), - [](const AspiredServableStateSnapshot& version) { - return !version.is_aspired && - version.state != LoaderHarness::State::kDisabled && - version.state != LoaderHarness::State::kError; - }); - if (not_aspired_not_finished) { - return nullopt; - } - - // Third and only if no action was found earlier, iterate over all - // versions and find any in kNew that are aspired. Load the first if any. - for (const auto& version : all_versions) { - if (version.state == LoaderHarness::State::kNew && version.is_aspired) { - VLOG(1) << "EagerUnloadPolicy requesting to load servable " << version.id; - return AspiredVersionPolicy::ServableAction({Action::kLoad, version.id}); - } - } - - return nullopt; -} - -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/core/eager_unload_policy.h b/tensorflow_serving/core/eager_unload_policy.h deleted file mode 100644 index c57d98f35fa..00000000000 --- a/tensorflow_serving/core/eager_unload_policy.h +++ /dev/null @@ -1,53 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ -#define TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ - -#include - -#include "tensorflow_serving/core/aspired_version_policy.h" -#include "tensorflow_serving/core/loader_harness.h" -#include "tensorflow_serving/util/optional.h" - -namespace tensorflow { -namespace serving { - -// Deprecated. Please use ResourcePreservePolicy instead. -// -// ServablePolicy that eagerly unloads any no-longer-aspired versions of a -// servable stream and only after done unloading, loads newly aspired versions. -// -// This policy minimizes resource consumption with the trade-off of temporary -// servable unavailability while all old versions unload followed by the new -// versions loading. -// -// Servables with a single version consuming the majority of their host's -// resources must use this policy to prevent deadlock. Other typical use-cases -// will be for multi-servable environments where clients can tolerate brief -// interruptions to a single servable's availability on a replica. -// -// NB: This policy does not in any way solve cross-replica availability. -class EagerUnloadPolicy final : public AspiredVersionPolicy { - public: - optional GetNextAction( - const std::vector& all_versions) - const override; -}; - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_CORE_EAGER_UNLOAD_POLICY_H_ diff --git a/tensorflow_serving/core/eager_unload_policy_test.cc b/tensorflow_serving/core/eager_unload_policy_test.cc deleted file mode 100644 index 1add44988bf..00000000000 --- a/tensorflow_serving/core/eager_unload_policy_test.cc +++ /dev/null @@ -1,122 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/core/eager_unload_policy.h" - -#include -#include "tensorflow_serving/core/servable_id.h" - -namespace tensorflow { -namespace serving { -namespace { - -// Test that the first ready and non-aspired version is unloaded first. -TEST(EagerUnloadPolicy, UnloadsFirstNonAspired) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kUnloading, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kReady, false}); - versions.push_back({{"test", 4}, LoaderHarness::State::kNew, true}); - versions.push_back({{"test", 5}, LoaderHarness::State::kReady, false}); - - EagerUnloadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kUnload, action->action); - EXPECT_EQ(3, action->id.version); -} - -// Test that the first aspired version is loaded when there are none to unload. -TEST(EagerUnloadPolicy, LoadsFirstAspiredWhenNoneToUnload) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 2}, LoaderHarness::State::kLoading, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); - versions.push_back({{"test", 4}, LoaderHarness::State::kDisabled, false}); - versions.push_back({{"test", 5}, LoaderHarness::State::kNew, true}); - - EagerUnloadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); - EXPECT_EQ(3, action->id.version); -} - -// Test that no action is returned (empty optional) when there are no versions -// needing loading or unloading. -TEST(EagerUnloadPolicy, ReturnsNoActionWhenNone) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kReady, true}); - versions.push_back({{"test", 2}, LoaderHarness::State::kError, true}); - versions.push_back({{"test", 3}, LoaderHarness::State::kLoading, true}); - versions.push_back({{"test", 4}, LoaderHarness::State::kUnloading, false}); - versions.push_back({{"test", 5}, LoaderHarness::State::kDisabled, false}); - - EagerUnloadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -TEST(EagerUnloadPolicy, DoesNotLoadWhenOthersStillUnloading) { - std::vector versions; - versions.push_back( - {{"test", 1}, LoaderHarness::State::kUnloadRequested, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); - - EagerUnloadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -TEST(EagerUnloadPolicy, LoadIfUnaspiredIsError) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kNew, true}); - - EagerUnloadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); - EXPECT_EQ(2, action->id.version); -} - -TEST(EagerUnloadPolicy, ErrorAndUnloadRequestedPreventLoading) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); - versions.push_back( - {{"test", 2}, LoaderHarness::State::kUnloadRequested, false}); - versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); - - EagerUnloadPolicy policy; - const auto action = policy.GetNextAction(versions); - EXPECT_FALSE(action); -} - -TEST(EagerUnloadPolicy, ErrorAndDisabledAllowLoading) { - std::vector versions; - versions.push_back({{"test", 1}, LoaderHarness::State::kError, false}); - versions.push_back({{"test", 2}, LoaderHarness::State::kDisabled, false}); - versions.push_back({{"test", 3}, LoaderHarness::State::kNew, true}); - - EagerUnloadPolicy policy; - const auto action = policy.GetNextAction(versions); - ASSERT_TRUE(action); - EXPECT_EQ(AspiredVersionPolicy::Action::kLoad, action->action); - EXPECT_EQ(3, action->id.version); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/g3doc/architecture_overview.md b/tensorflow_serving/g3doc/architecture_overview.md index b0bf98dd33c..6da0df270bd 100644 --- a/tensorflow_serving/g3doc/architecture_overview.md +++ b/tensorflow_serving/g3doc/architecture_overview.md @@ -179,14 +179,16 @@ Version Policies specify the sequence of version loading and unloading within a single servable stream. TensorFlow Serving includes two policies that accommodate most known use- -cases. These are the Eager Load Policy (always load new versions first), and -the Eager Unload Policy (always unload old versions first). For simple usage -of TensorFlow Serving where the serving availability of a model is important -and the resource costs low, the Eager Load Policy will ensure that the new -version is loaded and ready before unloading the old one. For sophisticated -usage of TensorFlow Serving, for example managing versions across multiple -server instances, the Eager Unload Policy requires the least resources (no -extra buffer for loading new versions). +cases. These are the Availability Preserving Policy (avoid leaving zero versions +loaded; typically load a new version before unloading an old one), and the +Resource Preserving Policy (avoid having two versions loaded simultaneously, +thus requiring double the resources; unload an old version before loading a new +one). For simple usage of TensorFlow Serving where the serving availability of a +model is important and the resource costs low, the Availability Preserving +Policy will ensure that the new version is loaded and ready before unloading the +old one. For sophisticated usage of TensorFlow Serving, for example managing +versions across multiple server instances, the Resource Preserving Policy +requires the least resources (no extra buffer for loading new versions). ### Source From 342eaef7df203a4314ddb1c84d8969b8326fb9d0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 8 Feb 2017 11:05:47 -0800 Subject: [PATCH 0159/8554] Fix an include and BUILD files to be OSS compatible. Change: 146931707 --- tensorflow_serving/apis/BUILD | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 485e1e244ba..ec666746bad 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -61,7 +61,9 @@ serving_proto_library_py( name = "input_proto_py_pb2", srcs = ["input.proto"], proto_library = "input_proto", - deps = [], + deps = [ + "@org_tensorflow//tensorflow/core:protos_all_py", + ], ) serving_proto_library( @@ -124,8 +126,10 @@ py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], deps = [ + ":classification_proto_py_pb2", ":get_model_metadata_proto_py_pb2", ":predict_proto_py_pb2", + ":regression_proto_py_pb2", ], ) @@ -145,7 +149,11 @@ serving_proto_library_py( name = "classification_proto_py_pb2", srcs = ["classification.proto"], proto_library = "classification_proto", - deps = [], + deps = [ + ":input_proto_py_pb2", + ":model_proto_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", + ], ) serving_proto_library( @@ -164,7 +172,11 @@ serving_proto_library_py( name = "regression_proto_py_pb2", srcs = ["regression.proto"], proto_library = "regression_proto", - deps = [], + deps = [ + ":input_proto_py_pb2", + ":model_proto_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", + ], ) cc_library( From c24ac4c613dd515dc026c0df75856e2bc076c948 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 8 Feb 2017 13:32:32 -0800 Subject: [PATCH 0160/8554] Introduce a "io_bazel_rules_closure" http_archive in our WORKSPACE file to match TensorFlow's. Change: 146949894 --- WORKSPACE | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/WORKSPACE b/WORKSPACE index 56cc78339c8..c5337768213 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -5,6 +5,18 @@ local_repository( path = "tensorflow", ) +# TensorFlow depends on "io_bazel_rules_closure" so we need this here. +# Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. +http_archive( + name = "io_bazel_rules_closure", + sha256 = "60fc6977908f999b23ca65698c2bb70213403824a84f7904310b6000d78be9ce", + strip_prefix = "rules_closure-5ca1dab6df9ad02050f7ba4e816407f88690cf7d", + urls = [ + "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/5ca1dab6df9ad02050f7ba4e816407f88690cf7d.tar.gz", # 2017-02-03 + "https://github.com/bazelbuild/rules_closure/archive/5ca1dab6df9ad02050f7ba4e816407f88690cf7d.tar.gz", + ], +) + # Please add all new TensorFlow Serving dependencies in workspace.bzl. load('//tensorflow_serving:workspace.bzl', 'tf_serving_workspace') tf_serving_workspace() From 8b63f1155a405857a56eff352d9f26142eb62500 Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Wed, 8 Feb 2017 14:51:43 -0800 Subject: [PATCH 0161/8554] Sync submodules. --- tensorflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow b/tensorflow index 63b3fea438b..e946a6b6397 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 63b3fea438b1fc700db38b77ca691e083b63bb5f +Subproject commit e946a6b63979a63f9e5a1d1603f6cc21d8aad1cf From 6058fd6a0009882ea21e71bff6e9a8b5d7a752a8 Mon Sep 17 00:00:00 2001 From: Vu Pham Date: Fri, 10 Feb 2017 23:35:55 +0100 Subject: [PATCH 0162/8554] Deploying multiple models (#294) * Removes command line config and adds file config * Adds model name parameter to mnist_client * Removes command line config and adds file config * Adds model name parameter to mnist_client * remove config_file due to bad merge * restore mnist_client * added model_base_path back * Adds checks against non entry of model_config_file or model_base_path options * Adds checks against non entry of model_config_file or model_base_path options * Refactors protobuf text parsing plus minor fixes * Removes unneeded protobuf includes * Moves QCHECK out of ParseProtoTextFile and updates command line help text * Refactors ParseProtoTextFile to return Status * Adds tests for model configuration from file * Adds half plus three savedmodel model * Updates model file configuration test to check against two different models * Removes references to removed half_plus_two2 model from build files * Refactors server test method for better clarity --- tensorflow_serving/model_servers/BUILD | 6 ++ tensorflow_serving/model_servers/main.cc | 74 ++++++++++---- .../tensorflow_model_server_test.py | 96 ++++++++++++++++-- .../servables/tensorflow/testdata/BUILD | 6 ++ .../tensorflow/testdata/bad_model_config.txt | 1 + .../tensorflow/testdata/good_model_config.txt | 14 +++ .../00000123/export.data-00000-of-00001 | Bin 0 -> 8 bytes .../half_plus_two2/00000123/export.index | Bin 0 -> 142 bytes .../half_plus_two2/00000123/export.meta | Bin 0 -> 4266 bytes .../00000123/assets/foo.txt | 1 + .../00000123/saved_model.pb | Bin 0 -> 8658 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 12 bytes .../00000123/variables/variables.index | Bin 0 -> 151 bytes 13 files changed, 171 insertions(+), 27 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.index create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.meta create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/saved_model.pb create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.index diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index a5c16c2baf7..5a7dc681645 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -147,6 +147,12 @@ py_test( "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/saved_model.pb", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/assets/foo.txt", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/variables/variables.index", + "//tensorflow_serving/servables/tensorflow/testdata:good_model_config.txt", + "//tensorflow_serving/servables/tensorflow/testdata:bad_model_config.txt", "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], tags = [ diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index c30931d54c9..1e6a3b66ed0 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -115,6 +115,20 @@ using tensorflow::serving::PredictionService; namespace { +tensorflow::Status ParseProtoTextFile(const string& file, google::protobuf::Message* message) { + std::unique_ptr file_data; + TF_RETURN_IF_ERROR( + tensorflow::Env::Default()->NewReadOnlyMemoryRegionFromFile(file, + &file_data)); + string file_data_str(static_cast(file_data->data()), + file_data->length()); + if(tensorflow::protobuf::TextFormat::ParseFromString(file_data_str, message)) { + return tensorflow::Status::OK(); + } else { + return tensorflow::errors::InvalidArgument("Invalid protobuf file: '", file, "'"); + } +} + tensorflow::Status LoadCustomModelConfig( const ::google::protobuf::Any& any, EventBus* servable_event_bus, @@ -142,6 +156,15 @@ ModelServerConfig BuildSingleModelConfig( return config; } +ModelServerConfig BuildModelConfigFromFile( + const string& file) { + LOG(INFO) << "Building from config file: " + << file; + + ModelServerConfig model_config; + TF_CHECK_OK(ParseProtoTextFile(file, &model_config)); + return model_config; +} int DeadlineToTimeoutMillis(const gpr_timespec deadline) { return gpr_time_to_millis( gpr_time_sub(gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), @@ -238,15 +261,8 @@ void RunServer(int port, std::unique_ptr core, // Parses an ascii PlatformConfigMap protobuf from 'file'. tensorflow::serving::PlatformConfigMap ParsePlatformConfigMap( const string& file) { - std::unique_ptr file_data; - TF_CHECK_OK( // Crash ok - tensorflow::Env::Default()->NewReadOnlyMemoryRegionFromFile(file, - &file_data)); - string file_data_str(static_cast(file_data->data()), - file_data->length()); tensorflow::serving::PlatformConfigMap platform_config_map; - QCHECK(tensorflow::protobuf::TextFormat::ParseFromString( // Crash ok - file_data_str, &platform_config_map)); + TF_CHECK_OK(ParseProtoTextFile(file, &platform_config_map)); return platform_config_map; } @@ -263,26 +279,36 @@ int main(int argc, char** argv) { // thread pools will be auto configured. tensorflow::int64 tensorflow_session_parallelism = 0; string platform_config_file = ""; + string model_config_file; tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( FileSystemStoragePathSourceConfig::LATEST_VERSION); std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), - tensorflow::Flag("model_name", &model_name, "name of model"), - tensorflow::Flag( - "model_version_policy", &model_version_policy, - "The version policy which determines the number of model versions to " - "be served at the same time. The default value is LATEST_VERSION, " - "which will serve only the latest version. See " - "file_system_storage_path_source.proto for the list of possible " - "VersionPolicy."), + tensorflow::Flag("model_config_file", &model_config_file, + "If non-empty, read an ascii ModelServerConfig " + "protobuf from the supplied file name, and serve the " + "models in that file. (If used, --model_name, " + "--model_base_path and --model_version_policy " + "are ignored.)"), + tensorflow::Flag("model_name", &model_name, "name of model (ignored " + "if --model_config_file flag is set"), + tensorflow::Flag("model_base_path", &model_base_path, + "path to export (ignored if --model_config_file flag " + "is set, otherwise required)"), + tensorflow::Flag("model_version_policy", &model_version_policy, + "The version policy which determines the number of model " + "versions to be served at the same time. The default " + "value is LATEST_VERSION, which will serve only the " + "latest version. " + "See file_system_storage_path_source.proto for " + "the list of possible VersionPolicy. (Ignored if " + "--model_config_file flag is set)"), tensorflow::Flag("file_system_poll_wait_seconds", &file_system_poll_wait_seconds, "interval in seconds between each poll of the file " "system for new model version"), - tensorflow::Flag("model_base_path", &model_base_path, - "path to export (required)"), tensorflow::Flag("use_saved_model", &use_saved_model, "If true, use SavedModel in the server; otherwise, use " "SessionBundle. It is used by tensorflow serving team " @@ -302,7 +328,7 @@ int main(int argc, char** argv) { "ignored.)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); - if (!parse_result || model_base_path.empty()) { + if (!parse_result || (model_base_path.empty() && model_config_file.empty())) { std::cout << usage; return -1; } @@ -322,8 +348,14 @@ int main(int argc, char** argv) { // For ServerCore Options, we leave servable_state_monitor_creator unspecified // so the default servable_state_monitor_creator will be used. ServerCore::Options options; - options.model_server_config = BuildSingleModelConfig( - model_name, model_base_path, parsed_version_policy); + + // model server config + if (model_config_file.empty()) { + options.model_server_config = BuildSingleModelConfig( + model_name, model_base_path, parsed_version_policy); + } else { + options.model_server_config = BuildModelConfigFromFile(model_config_file); + } if (platform_config_file.empty()) { SessionBundleConfig session_bundle_config; diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 23b04adc635..2e469f39b6e 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -51,15 +51,31 @@ def PickUnusedPort(): class TensorflowModelServerTest(tf.test.TestCase): """This class defines integration test cases for tensorflow_model_server.""" - def __TestSrcDirPath(self, relative_path): + def __TestSrcDirPath(self, relative_path=''): return os.path.join(os.environ['TEST_SRCDIR'], 'tf_serving/tensorflow_serving', relative_path) + def __BuildModelConfigFile(self): + """Substitutes placeholder for test directory with test directory path + in the configuration template file and writes it out to another file + used by the test""" + + with open(self._GetGoodModelConfigTemplate(), 'r') as template_file : + config = template_file.read().replace('${TEST_SRCDIR}', os.environ['TEST_SRCDIR']) + + with open(self._GetGoodModelConfigFile(), 'w') as config_file: + config_file.write(config) + def setUp(self): """Sets up integration test parameters.""" self.binary_dir = self.__TestSrcDirPath('model_servers') self.testdata_dir = self.__TestSrcDirPath('servables/tensorflow/testdata') self.server_proc = None + self.__BuildModelConfigFile() + + def tearDown(self): + """Deletes created configuration file""" + os.remove(self._GetGoodModelConfigFile()) def TerminateProcs(self): """Terminate all processes.""" @@ -83,14 +99,29 @@ def RunServer(self, port, model_name, model_path, use_saved_model, print 'Server started' return 'localhost:' + str(port) + def RunServerWithModelConfigFile(self, port, model_config_file, use_saved_model, pipe=None): + """Run tensorflow_model_server using test config.""" + print 'Starting test server...' + command = os.path.join(self.binary_dir, 'tensorflow_model_server') + command += ' --port=' + str(port) + command += ' --model_config_file=' + model_config_file + command += ' --use_saved_model=' + str(use_saved_model).lower() + + print command + self.server_proc = subprocess.Popen(shlex.split(command), stderr=pipe) + print 'Server started' + return 'localhost:' + str(port) + def VerifyPredictRequest(self, model_server_address, + expected_output, + model_name='default', specify_output=True): """Send PredictionService.Predict request and verify output.""" print 'Sending Predict request...' # Prepare request request = predict_pb2.PredictRequest() - request.model_spec.name = 'default' + request.model_spec.name = model_name request.inputs['x'].dtype = types_pb2.DT_FLOAT request.inputs['x'].float_val.append(2.0) dim = request.inputs['x'].tensor_shape.dim.add() @@ -107,7 +138,7 @@ def VerifyPredictRequest(self, self.assertTrue('y' in result.outputs) self.assertIs(types_pb2.DT_FLOAT, result.outputs['y'].dtype) self.assertEquals(1, len(result.outputs['y'].float_val)) - self.assertEquals(3.0, result.outputs['y'].float_val[0]) + self.assertEquals(expected_output, result.outputs['y'].float_val[0]) def _GetSavedModelBundlePath(self): """Returns a path to a model in SavedModel format.""" @@ -118,6 +149,18 @@ def _GetSessionBundlePath(self): """Returns a path to a model in SessionBundle format.""" return os.path.join(self.testdata_dir, 'half_plus_two') + def _GetGoodModelConfigTemplate(self): + """Returns a path to a working configuration file template""" + return os.path.join(self.testdata_dir, 'good_model_config.txt') + + def _GetGoodModelConfigFile(self): + """Returns a path to a working configuration file""" + return os.path.join(self.testdata_dir, 'good_model_config.conf') + + def _GetBadModelConfigFile(self): + """Returns a path to a improperly formatted configuration file""" + return os.path.join(self.testdata_dir, 'bad_model_config.txt') + def _TestPredict(self, model_path, use_saved_model, enable_batching): """Helper method to test prediction. @@ -131,8 +174,9 @@ def _TestPredict(self, model_path, use_saved_model, enable_batching): model_path, use_saved_model, enable_batching) time.sleep(5) - self.VerifyPredictRequest(model_server_address) - self.VerifyPredictRequest(model_server_address, specify_output=False) + self.VerifyPredictRequest(model_server_address, expected_output=3.0) + self.VerifyPredictRequest(model_server_address, + expected_output=3.0, specify_output=False) def testPredictSessionBundle(self): """Test PredictionService.Predict implementation with SessionBundle.""" @@ -179,7 +223,47 @@ def _TestBadModelUpconvertedSavedModel(self): def _TestBadModelSessionBundle(self): """Test Predict against a bad SessionBundle model export.""" self._TestBadModel(use_saved_model=False) + + def testGoodModelConfig(self): + """Test server model configuration from file works with valid configuration""" + atexit.register(self.TerminateProcs) + model_server_address = self.RunServerWithModelConfigFile(PickUnusedPort(), + self._GetGoodModelConfigFile(), + True) # use_saved_model + time.sleep(5) + + self.VerifyPredictRequest(model_server_address, + model_name='half_plus_two', expected_output=3.0) + self.VerifyPredictRequest(model_server_address, + model_name='half_plus_two', expected_output=3.0, + specify_output=False) + + self.VerifyPredictRequest(model_server_address, + model_name='half_plus_three', expected_output=4.0) + self.VerifyPredictRequest(model_server_address, + model_name='half_plus_three', expected_output=4.0, + specify_output=False) + + def testBadModelConfig(self): + """Test server model configuration from file fails for invalid file""" + atexit.register(self.TerminateProcs) + model_server_address = self.RunServerWithModelConfigFile(PickUnusedPort(), + self._GetBadModelConfigFile(), + True, # use_saved_model + pipe=subprocess.PIPE) + last_line = None + for line in self.server_proc.stderr: + last_line = line + + error_message = 'Check failed: ::tensorflow::Status::OK() == ' \ + '(ParseProtoTextFile(file, &model_config)) ' \ + '(OK vs. Invalid argument: ' \ + 'Invalid protobuf file: \'%s\')' % self._GetBadModelConfigFile() + + self.assertNotEqual(last_line, None) + self.assertGreater(last_line.find(error_message), 0) + # self.assertEquals(self.server_proc.poll(), 1) - + if __name__ == '__main__': tf.test.main() diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 64629175c6f..5ed39d6f037 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -52,6 +52,12 @@ exports_files([ "half_plus_two/00000123/export.data-00000-of-00001", "half_plus_two/00000123/export.index", "half_plus_two/00000123/export.meta", + "saved_model_half_plus_three/00000123/saved_model.pb", + "saved_model_half_plus_three/00000123/assets/foo.txt", + "saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001", + "saved_model_half_plus_three/00000123/variables/variables.index", "bad_half_plus_two/00000123/export", "bad_half_plus_two/00000123/export.meta", + "good_model_config.txt", + "bad_model_config.txt" ]) diff --git a/tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt b/tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt new file mode 100644 index 00000000000..6649d3fd44a --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt @@ -0,0 +1 @@ +improperly formatted file \ No newline at end of file diff --git a/tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt b/tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt new file mode 100644 index 00000000000..8145a90600f --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt @@ -0,0 +1,14 @@ +model_config_list: { + + config: { + name: "half_plus_two", + base_path: "${TEST_SRCDIR}/tf_serving/external/org_tensorflow/tensorflow/cc/saved_model/testdata/half_plus_two", + model_platform: "tensorflow" + }, + config: { + name: "half_plus_three", + base_path: "${TEST_SRCDIR}/tf_serving/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three", + model_platform: "tensorflow" + } + +} \ No newline at end of file diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..20bc7d454dd8450489984bd17d92c45c3a1a96a6 GIT binary patch literal 8 PcmZQzV6bOkU~m8c0fPX5 literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.index b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.index new file mode 100644 index 0000000000000000000000000000000000000000..35e2ef7527905b228ebfd7f755bb3d06f6cc00e0 GIT binary patch literal 142 zcmZQzVB=tvV&Y(A;NT8REXqtw%1Py56k^a|F=ew*m*-&OjW-1G6&JCcXIiQeD<{{v=Y01&-#O==z%LU0rve{A>2N%z z5|DF+d>K41@X7G`Ej|?nxLEctrpB-h%|U&Y6w6YjRI5@M&Oti#Jab~xI+R@F2!H=0 zArzo;rdu;3FoR#5nLnNRTJm10B_`2&3x8^kb>GmuV|{AS@-xS=SaF9a8&IElhGsdV z=cYYLU;IW0+=a?L$M$^I28r8OD&qj8(vF-{$v14zag`?UH~bLLhwA;YVf&_k5dbx9 z#1;UpCx-CQDMI)dN}tcH>(R3k^79gLN>F~_d@+SXs6Vvyk#X!;W5ZoUodu~-sBkv; zgB6Dbu%PzH@O;NLo(vGVG(|$q^R7%g(BMbaZ2i>maAgfT;dV{8$C_uEqhOV-fXsjA z4Wy7OPw7JRiGpQ%{!YN)ogK1Azy#^Be)b<-(QCQ-2C7eV*VLa`1`-qMh(`>yq_nb3 z%taG5QX4t8ubZ~vQpxjOR0=E7f^rM$NP%mNLsG<7KNHfc?e+Hu{bNHP59FEs+;(2r z^wkD@1?w@AUDGywQ@6BG$&{Gn`vXFBXvT}XE{1|8iJ4-|^?EfIUqd%`q3vHI zqzAcvGc;QOsrfe^)V~O4rc6xl^Lif?eU{ZHyQ1wx;P-fE4caRU?Ik%G zQL+SJVq^g2pcK)tkFd_uSOHP|8BJEuDAxo$1n{tU?}jZhu3hVK?P{v^s!R)N<YN2;!M?Vp)r~zgCfHZ1;OyH5_GQgICOp6&4Ip`#<7d1I3 z;$7i;*a#s64b62dZEQ>p8?P#(3!yTKfErH)$hmR6ERn)}p*rG>mGOSD87?Q6j7y`u z9aFug(uiu3MtKS0Va|ou4lMpM9K-E7Z+XGZ^jxkhvNW`Fo?a4YWnIzqB9#g7vh%ao z!CjKect_>;En0!n03-ctthTuto@5J~LwxhPeYJi$h3za=SW_;`_1m!u*44E{1){q1 z6K^g$C-Clq$2X+}q`fomEKdu|d0?*xeH}f#BG3$haXo1JG`_3}oU#S%H&?}FwIOaW zVmJRVBF^Olh^AXs)R3YY#DTLGuSoGR;In86~KR6q)caF2G`I;`Q}5Om(szvJ$vMgRZ+ literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt new file mode 100644 index 00000000000..f9ff0366880 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt @@ -0,0 +1 @@ +asset-file-contents \ No newline at end of file diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/saved_model.pb b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..71ac858241500dae98fba4ddb4c4e3abe4581ca2 GIT binary patch literal 8658 zcmcgx-*Ven8TYYc%l_;n%}LrirwzN90lMxqcHB0!-pFnhtpas%A#V{pqG@E}|xj(B$!mOIsTl3ywn3zag}-~>vK zeBT}nX%z|{-^cg=KnMjW9vjXP7i93pJqkugfj(YuRK_Hc`U<{kTSmZj|G*e=y3}`F zhvjdO##N{u`CNBg^R+!3Ocwr52;76>V|VBWY!yn1exqm!Asee9b6N`c(09GYGN=`$ z1Z+e3Ba06MJ2(}B+C!902wEKzLZv4XLLcZe?hW`SoyP~0HCVN{!%UuJ$Xy( zu%ez@eBU^70>4vwDIc(FuoBX;hn8(3{mPge+k)kAQK{Ieg|`BGpRw_>+)Rm2uRu+4 z48IKdH7>zeSY;P9MI>eT;Jc7uL&35A;D%uN-VM^#px7ypiq?1sLYi4GY(*j{>1b8b zkvB*P9zpfFW0?E^w+Q#9w{~(THz*X9%cvig@8-aP%Fl88xgPFU+}nH|h{xCDm#Zu%YzAIKCh&dM;KU!s?3y!?U>c z)ONVgf!3hhH+*@mb|b3eS@nY0sKcW}5l=kJuNN4;xF3F0*I*CeMc`pX`7ydOU51hj z0bGdg3u}R(7`A+wEJP+39Q@N7p;IcG{g= zgPps#ca3{@Zegs!Q1{;syo6PwWe@EDU09b+KvfWJR-+J^Z_P$o3jT*~Eg zKWtus6+MjPC>J^#T>`P=I1q!zERZCIl;ztLUu~|&bw$%P*OEGlM_FwCM4)W6!fX>} z9Yze6R;jr$l~FL4;M0V(73%HehE|fX%=NjUQHV zVG=4ssk=n;Wzziq6xx?zy}L%M<^M&0h#$UK8<%&u6`C|v+vPzds;H1FLZL%Qb9quD ziqiO(*ek`tf3}q-8~d!%!3U}r5DftfGYvzK#+l?CgvcostsPUeRD`?$p(l_(>Ch~n zBWLxw_o0Y7={q!`8j_xfl;K(EuSO6Z!|d|yIM6|SwUs{0~);7jcMQ}u9Ke3sz*by_fAt4en)9=bTY$=p_dBv6+mCv7Dv{M%WIO4&XXsqOG#e{C3N+WnteVwg{R)`UB-!w{Fy(vJ8L4M? zlApoGEb25pUtno-vN=+PFHek+soX)dS>)iP6(7pXP)%!Ij|{ioTf@yFYMnxl7(LSO z5wnT&(^QEY6+{J`&RkcDDo(OsiVb^aa@&l!UFxljx#fEJKbO_-v5^-}^+K|u;Z%st zG<3B4ruAoY3 zmc2g+d={E)T0A;qs!sw~Y}U9C)uKT$s` z!f?1Id5>qMi~TT_6=ctWemKW12pqsWDK=+v!o&nlxjdQcfCRoEI5ij69BV`;gWxzC zY@Xt5k+`BWQd40Xc1GDH`{yJK3aKIm-8dy`Mq%>x&FluSl{AjNb5X$ia{>PpzCFe0 zZ9)E5f&=^nP#qX z3BtUub}3s?*(BPtsrwH0AZqJIlnloS>8z_SqXmTAvl{vGW%d9^OTt0rbg|o}s-TSW zUAGB^&pT3|U z!zL6AVjp9Y1KZu=6! z14o>w;oYgYV@#FqLfcmOWebm6@yI%8{uXAfdVGLbqx`_4Q%89&J05IHE}NjKb2eWb NWKMQV2FsA>{{bSxjHUnp literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..74cf86632b56bb7222096c7d8f59400a4fa64c57 GIT binary patch literal 12 QcmZQzV6bOkaBu)&00qMVKmY&$ literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.index b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.index new file mode 100644 index 0000000000000000000000000000000000000000..ac030a9d4018dd553bb0a736b69cf91b5bb6911a GIT binary patch literal 151 zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5=3=9m6 vK+JIPndE&C4dxv9v~U9hBU1{46I>`_)64Jf93gxl0YV`BcSE;IsrzjJ-**?> literal 0 HcmV?d00001 From 3067dde35ca7e69b986020814e970805219110d7 Mon Sep 17 00:00:00 2001 From: Kristina Chodorow Date: Mon, 13 Feb 2017 10:07:48 -0500 Subject: [PATCH 0163/8554] Updated tensorflow submodule --- tensorflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow b/tensorflow index e946a6b6397..1bbb52426bd 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit e946a6b63979a63f9e5a1d1603f6cc21d8aad1cf +Subproject commit 1bbb52426bd4f8046400731100b11e9ca767d303 From 59994127c98a09da917975f82bedcb8d9ef17941 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 8 Feb 2017 16:33:26 -0800 Subject: [PATCH 0164/8554] Merge changes from github. Change: 146973828 --- tensorflow_serving/example/inception_k8s.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/example/inception_k8s.yaml b/tensorflow_serving/example/inception_k8s.yaml index 89631768dd5..302e84e2299 100644 --- a/tensorflow_serving/example/inception_k8s.yaml +++ b/tensorflow_serving/example/inception_k8s.yaml @@ -11,7 +11,7 @@ spec: spec: containers: - name: inception-container - image: gcr.io/tensorflow-serving/inception:0.2.0 + image: gcr.io/tensorflow-serving/inception command: - /bin/sh - -c From aaea5c0cf078ef60e5f9816f6aaa729a9dc836b6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 8 Feb 2017 17:47:14 -0800 Subject: [PATCH 0165/8554] Update generated grpc protobuf files to include new classify and regress APIs. Change: 146981673 --- .../apis/prediction_service_pb2.py | 72 ++++++++++++++++++- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/apis/prediction_service_pb2.py b/tensorflow_serving/apis/prediction_service_pb2.py index 3130f0a4f38..dc16d22d0e8 100644 --- a/tensorflow_serving/apis/prediction_service_pb2.py +++ b/tensorflow_serving/apis/prediction_service_pb2.py @@ -14,6 +14,8 @@ # ============================================================================== # Generated by the protocol buffer compiler. DO NOT EDIT! # source: tensorflow_serving/apis/prediction_service.proto +# To regenerate run +# python -m grpc.tools.protoc --python_out=. --grpc_python_out=. -I. tensorflow_serving/apis/prediction_service.proto import sys _b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) @@ -27,17 +29,19 @@ _sym_db = _symbol_database.Default() +from tensorflow_serving.apis import classification_pb2 as tensorflow__serving_dot_apis_dot_classification__pb2 from tensorflow_serving.apis import get_model_metadata_pb2 as tensorflow__serving_dot_apis_dot_get__model__metadata__pb2 from tensorflow_serving.apis import predict_pb2 as tensorflow__serving_dot_apis_dot_predict__pb2 +from tensorflow_serving.apis import regression_pb2 as tensorflow__serving_dot_apis_dot_regression__pb2 DESCRIPTOR = _descriptor.FileDescriptor( name='tensorflow_serving/apis/prediction_service.proto', package='tensorflow.serving', syntax='proto3', - serialized_pb=_b('\n0tensorflow_serving/apis/prediction_service.proto\x12\x12tensorflow.serving\x1a\x30tensorflow_serving/apis/get_model_metadata.proto\x1a%tensorflow_serving/apis/predict.proto2\xd6\x01\n\x11PredictionService\x12R\n\x07Predict\x12\".tensorflow.serving.PredictRequest\x1a#.tensorflow.serving.PredictResponse\x12m\n\x10GetModelMetadata\x12+.tensorflow.serving.GetModelMetadataRequest\x1a,.tensorflow.serving.GetModelMetadataResponseB\x03\xf8\x01\x01\x62\x06proto3') + serialized_pb=_b('\n0tensorflow_serving/apis/prediction_service.proto\x12\x12tensorflow.serving\x1a,tensorflow_serving/apis/classification.proto\x1a\x30tensorflow_serving/apis/get_model_metadata.proto\x1a%tensorflow_serving/apis/predict.proto\x1a(tensorflow_serving/apis/regression.proto2\x93\x03\n\x11PredictionService\x12\x61\n\x08\x43lassify\x12).tensorflow.serving.ClassificationRequest\x1a*.tensorflow.serving.ClassificationResponse\x12X\n\x07Regress\x12%.tensorflow.serving.RegressionRequest\x1a&.tensorflow.serving.RegressionResponse\x12R\n\x07Predict\x12\".tensorflow.serving.PredictRequest\x1a#.tensorflow.serving.PredictResponse\x12m\n\x10GetModelMetadata\x12+.tensorflow.serving.GetModelMetadataRequest\x1a,.tensorflow.serving.GetModelMetadataResponseB\x03\xf8\x01\x01\x62\x06proto3') , - dependencies=[tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_predict__pb2.DESCRIPTOR,]) + dependencies=[tensorflow__serving_dot_apis_dot_classification__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_predict__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_regression__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -66,6 +70,16 @@ def __init__(self, channel): Args: channel: A grpc.Channel. """ + self.Classify = channel.unary_unary( + '/tensorflow.serving.PredictionService/Classify', + request_serializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.FromString, + ) + self.Regress = channel.unary_unary( + '/tensorflow.serving.PredictionService/Regress', + request_serializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.FromString, + ) self.Predict = channel.unary_unary( '/tensorflow.serving.PredictionService/Predict', request_serializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, @@ -84,6 +98,20 @@ class PredictionServiceServicer(object): model_servers. """ + def Classify(self, request, context): + """Classify. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Regress(self, request, context): + """Regress. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def Predict(self, request, context): """Predict -- provides access to loaded TensorFlow model. """ @@ -101,6 +129,16 @@ def GetModelMetadata(self, request, context): def add_PredictionServiceServicer_to_server(servicer, server): rpc_method_handlers = { + 'Classify': grpc.unary_unary_rpc_method_handler( + servicer.Classify, + request_deserializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.SerializeToString, + ), + 'Regress': grpc.unary_unary_rpc_method_handler( + servicer.Regress, + request_deserializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.SerializeToString, + ), 'Predict': grpc.unary_unary_rpc_method_handler( servicer.Predict, request_deserializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, @@ -127,6 +165,14 @@ class BetaPredictionServiceServicer(object): PredictionService provides access to machine-learned models loaded by model_servers. """ + def Classify(self, request, context): + """Classify. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + def Regress(self, request, context): + """Regress. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) def Predict(self, request, context): """Predict -- provides access to loaded TensorFlow model. """ @@ -147,6 +193,16 @@ class BetaPredictionServiceStub(object): PredictionService provides access to machine-learned models loaded by model_servers. """ + def Classify(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """Classify. + """ + raise NotImplementedError() + Classify.future = None + def Regress(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """Regress. + """ + raise NotImplementedError() + Regress.future = None def Predict(self, request, timeout, metadata=None, with_call=False, protocol_options=None): """Predict -- provides access to loaded TensorFlow model. """ @@ -166,16 +222,22 @@ def beta_create_PredictionService_server(servicer, pool=None, pool_size=None, de file not marked beta) for all further purposes. This function was generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" request_deserializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.FromString, ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.FromString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.FromString, } response_serializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.SerializeToString, ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.SerializeToString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.SerializeToString, } method_implementations = { + ('tensorflow.serving.PredictionService', 'Classify'): face_utilities.unary_unary_inline(servicer.Classify), ('tensorflow.serving.PredictionService', 'GetModelMetadata'): face_utilities.unary_unary_inline(servicer.GetModelMetadata), ('tensorflow.serving.PredictionService', 'Predict'): face_utilities.unary_unary_inline(servicer.Predict), + ('tensorflow.serving.PredictionService', 'Regress'): face_utilities.unary_unary_inline(servicer.Regress), } server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) return beta_implementations.server(method_implementations, options=server_options) @@ -188,16 +250,22 @@ def beta_create_PredictionService_stub(channel, host=None, metadata_transformer= file not marked beta) for all further purposes. This function was generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" request_serializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.SerializeToString, ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.SerializeToString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.SerializeToString, } response_deserializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.FromString, ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.FromString, ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.FromString, } cardinalities = { + 'Classify': cardinality.Cardinality.UNARY_UNARY, 'GetModelMetadata': cardinality.Cardinality.UNARY_UNARY, 'Predict': cardinality.Cardinality.UNARY_UNARY, + 'Regress': cardinality.Cardinality.UNARY_UNARY, } stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.PredictionService', cardinalities, options=stub_options) From 6c25856d900064eef5b686d6001df59800aaa247 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 9 Feb 2017 09:26:23 -0800 Subject: [PATCH 0166/8554] Have BatchingSession propagate RunOptions. For the timeout, we take the largest request timeout as the timeout for the batch. But if all elements of a batch have already exceeded their timeouts while waiting in the batching queue, the batch is errored out immediately and not processed. Change: 147043103 --- .../batching/batching_session.cc | 81 +++++++++++++++---- .../batching/batching_session.h | 3 + .../batching/batching_session_test.cc | 70 +++++++++++++++- 3 files changed, 137 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index 4188c7c42b3..99ab0a681a2 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -126,8 +126,14 @@ class BatchingSession : public ServingSession { const std::vector& target_node_names, std::vector* outputs) override; - // TODO(b/34971139): at the moment this method ignores run_options and - // run_metadata and behaves exactly like Run. + // RunOptions handling: + // Since multiple of these Run() calls get backed into a single call to the + // underlying Session's Run(), we select an arbitrary 'run_options' (typically + // they are the same across calls). The exception is the timeout; we take the + // largest value (after subtracting time spent in the batching queue). + // + // RunMetadata: + // We copy the batched call's RunMetadata to each non-batched call's output. Status Run(const RunOptions& run_options, const std::vector>& inputs, const std::vector& output_tensor_names, @@ -210,21 +216,21 @@ Status BatchingSession::Create( } Status BatchingSession::Run( - const RunOptions& run_options, const std::vector>& inputs, const std::vector& output_tensor_names, - const std::vector& target_node_names, std::vector* outputs, - RunMetadata* run_metadata) { - LOG(WARNING) << "Currently both run_options and run_metadata are ignored, " - << "see b/34971139"; - return Run(inputs, output_tensor_names, target_node_names, outputs); + const std::vector& target_node_names, + std::vector* outputs) { + RunMetadata run_metadata; + return Run(RunOptions(), inputs, output_tensor_names, target_node_names, + outputs, &run_metadata); } Status BatchingSession::Run( + const RunOptions& run_options, const std::vector>& inputs, const std::vector& output_tensor_names, - const std::vector& target_node_names, - std::vector* outputs) { + const std::vector& target_node_names, std::vector* outputs, + RunMetadata* run_metadata) { if (!target_node_names.empty()) { return errors::PermissionDenied( "BatchingSession does not support target nodes"); @@ -239,8 +245,8 @@ Status BatchingSession::Run( LOG(WARNING) << "Request doesn't match any declared signature. Bypassing " "batcher. Request signature is: " << TensorSignatureDebugString(signature); - return wrapped_->Run(inputs, output_tensor_names, target_node_names, - outputs); + return wrapped_->Run(run_options, inputs, output_tensor_names, + target_node_names, outputs, run_metadata); } BatchScheduler* batch_scheduler = batch_scheduler_it->second.get(); @@ -250,12 +256,15 @@ Status BatchingSession::Run( Notification done; Status status; auto task = std::unique_ptr(new BatchingSessionTask); + task->enqueue_time_micros = Env::Default()->NowMicros(); + task->run_options = run_options; TF_RETURN_IF_ERROR(ComputeInputSize(inputs, &task->zeroth_dim_size)); task->inputs = &inputs; task->output_tensor_names = &output_tensor_names; task->done = &done; task->status = &status; task->outputs = outputs; + task->run_metadata = run_metadata; TF_RETURN_IF_ERROR(batch_scheduler->Schedule(&task)); done.WaitForNotification(); @@ -457,11 +466,12 @@ void BatchingSession::ProcessBatch( return; } - Status status; + const uint64 dequeue_time_micros = Env::Default()->NowMicros(); // Regardless of the outcome, we need to propagate the status to the // individual tasks and signal that they are done. We use MakeCleanup() to // ensure that this happens no matter how we exit the method below. + Status status; auto finally = MakeCleanup([&status, &batch] { for (int i = 0; i < batch->num_tasks(); ++i) { *batch->mutable_task(i)->status = status; @@ -469,6 +479,42 @@ void BatchingSession::ProcessBatch( } }); + // Make sure we have at least one task that hasn't exceeded its timeout from + // queue time alone, and find the latest task deadline which we'll use for the + // overall batch. + bool all_tasks_timeout_exceeded = true; + uint64 batch_deadline_micros = 0; + for (int i = 0; i < batch->num_tasks(); ++i) { + const BatchingSessionTask& task = batch->task(i); + // If the caller doesn't populate RunOptions, the timeout is 0 by default. + // Interpret that as "no timeout" i.e. infinity. + const int64 task_timeout_micros = + task.run_options.timeout_in_ms() <= 0 + ? INT_MAX + : task.run_options.timeout_in_ms() * 1000; + const uint64 task_deadline_micros = + task.enqueue_time_micros + task_timeout_micros; + if (task_deadline_micros > dequeue_time_micros) { + all_tasks_timeout_exceeded = false; + if (task_deadline_micros > batch_deadline_micros) { + batch_deadline_micros = task_deadline_micros; + } + } + } + if (all_tasks_timeout_exceeded) { + status = Status(error::RESOURCE_EXHAUSTED, + "Run() timeout exceeded while waiting in batching queue"); + return; + } + + RunOptions run_options = batch->task(0).run_options; + if (batch_deadline_micros == INT_MAX) { + run_options.set_timeout_in_ms(0); + } else { + run_options.set_timeout_in_ms( + (batch_deadline_micros - dequeue_time_micros) / 1000); + } + std::vector> merged_inputs; status = MergeInputTensors(signature, *batch, &merged_inputs); if (!status.ok()) { @@ -478,8 +524,13 @@ void BatchingSession::ProcessBatch( const std::vector output_tensor_names( signature.output_tensors.begin(), signature.output_tensors.end()); std::vector combined_outputs; - status = wrapped_->Run(merged_inputs, output_tensor_names, - {} /* target node names */, &combined_outputs); + RunMetadata run_metadata; + status = wrapped_->Run(run_options, merged_inputs, output_tensor_names, + {} /* target node names */, &combined_outputs, + &run_metadata); + for (int i = 0; i < batch->num_tasks(); ++i) { + *(batch->mutable_task(i)->run_metadata) = run_metadata; + } if (!status.ok()) { return; } diff --git a/tensorflow_serving/batching/batching_session.h b/tensorflow_serving/batching/batching_session.h index 7093907819e..4426652c601 100644 --- a/tensorflow_serving/batching/batching_session.h +++ b/tensorflow_serving/batching/batching_session.h @@ -167,6 +167,8 @@ struct BatchingSessionTask : public BatchTask { size_t size() const override { return zeroth_dim_size; } // Fields populated when a task is received. + uint64 enqueue_time_micros; + RunOptions run_options; size_t zeroth_dim_size; const std::vector>* inputs; const std::vector* output_tensor_names; @@ -175,6 +177,7 @@ struct BatchingSessionTask : public BatchTask { Notification* done; Status* status; std::vector* outputs; + RunMetadata* run_metadata; }; } // namespace serving diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index 1a50646dfdd..a80dff1c555 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -36,6 +36,7 @@ namespace tensorflow { namespace serving { namespace { +using ::testing::HasSubstr; using ::testing::UnorderedElementsAre; // A wrapper around a Session that captures the batch size. @@ -49,9 +50,19 @@ class BatchSizeCapturingSession : public ServingSession { const std::vector& output_tensor_names, const std::vector& target_node_names, std::vector* outputs) override { + RunMetadata run_metadata; + return Run(RunOptions(), inputs, output_tensor_names, target_node_names, + outputs, &run_metadata); + } + + Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, RunMetadata* run_metadata) override { latest_batch_size_ = inputs[0].second.shape().dim_size(0); - return wrapped_->Run(inputs, output_tensor_names, target_node_names, - outputs); + return wrapped_->Run(run_options, inputs, output_tensor_names, + target_node_names, outputs, run_metadata); } int latest_batch_size() const { return latest_batch_size_; } @@ -392,6 +403,61 @@ TEST(BatchingSessionTest, MultipleSignatures) { EXPECT_EQ(0, schedulers[1]->NumEnqueuedTasks()); } +TEST(BatchingSessionTest, EnqueuedLongerThanTimeout) { + BatchScheduler* scheduler = nullptr; + auto create_scheduler = [&scheduler]( + std::function>)> + process_batch_callback, + std::unique_ptr>* new_scheduler) { + BasicBatchScheduler::Options options; + options.max_batch_size = 4; // fits two 2-unit tasks + options.batch_timeout_micros = 1 * 1000 * 1000; // won't trigger + options.num_batch_threads = 1; + std::unique_ptr> basic_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + options, process_batch_callback, &basic_scheduler)); + scheduler = basic_scheduler.get(); + *new_scheduler = std::move(basic_scheduler); + return Status::OK(); + }; + BatchingSessionOptions batching_session_options; + std::unique_ptr batching_session; + TF_CHECK_OK(CreateBatchingSession( + batching_session_options, {{{{"x"}, {"y"}}, create_scheduler}}, + CreateHalfPlusTwoSession(), &batching_session)); + ASSERT_FALSE(scheduler == nullptr); + + // Enqueue a request with a timeout specified via RunOptions. + Notification request_returned; + auto issue_request = [&batching_session, &request_returned] { + Tensor input = test::AsTensor({100.0f, 42.0f}, {2}); + RunOptions run_options; + run_options.set_timeout_in_ms(1); + std::vector outputs; + RunMetadata run_metadata; + const Status status = + batching_session->Run(run_options, {{"x", input}}, {"y"} /* outputs */, + {} /* target nodes */, &outputs, &run_metadata); + EXPECT_FALSE(status.ok()); + EXPECT_EQ(error::RESOURCE_EXHAUSTED, status.code()); + EXPECT_THAT( + status.error_message(), + HasSubstr("Run() timeout exceeded while waiting in batching queue")); + request_returned.Notify(); + }; + std::unique_ptr request_thread(Env::Default()->StartThread( + ThreadOptions(), "request_thread", [&] { issue_request(); })); + while (scheduler->NumEnqueuedTasks() != 1) { + Env::Default()->SleepForMicroseconds(100); + } + // Sleep for longer than the request's timeout, so that when it does finally + // get dequeued for batch processing it has already exceeded its timeout. + Env::Default()->SleepForMicroseconds(10 * 1000); + // Tear down the batcher, so that it schedules the pending batch. + batching_session = nullptr; + request_returned.WaitForNotification(); +} + } // namespace } // namespace serving } // namespace tensorflow From 03cea43505cdf70f01c09d4fdedd3d8a92bbcf0c Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 13 Feb 2017 14:14:13 -0800 Subject: [PATCH 0167/8554] Remove --logtostderr and --alsologtostderr flags in documentation. Change: 147391542 --- tensorflow_serving/g3doc/serving_advanced.md | 2 +- tensorflow_serving/g3doc/serving_basic.md | 2 +- tensorflow_serving/model_servers/main.cc | 1 - .../model_servers/tensorflow_model_server_test.py | 1 - 4 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 5e372f99a1a..9e871eb3311 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -263,7 +263,7 @@ server. $>mkdir /tmp/monitored $>cp -r /tmp/mnist_model/1 /tmp/monitored $>bazel build //tensorflow_serving/model_servers:tensorflow_model_server -$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored --logtostderr +$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored ~~~ The server will emit log messages every one second that say diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 23ed719c50e..12406b399a1 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -144,7 +144,7 @@ With that, your TensorFlow model is exported and ready to be loaded! ~~~shell $>bazel build //tensorflow_serving/model_servers:tensorflow_model_server -$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ --logtostderr +$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ ~~~ ## Test The Server diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index c30931d54c9..f1e6e8729a0 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -41,7 +41,6 @@ limitations under the License. // To specify model name (default "default"): --model_name=my_name // To specify port (default 8500): --port=my_port // To enable batching (default disabled): --enable_batching -// To log on stderr (default disabled): --alsologtostderr #include #include diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 23b04adc635..5b4dbe6952e 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -77,7 +77,6 @@ def RunServer(self, port, model_name, model_path, use_saved_model, command += ' --model_base_path=' + model_path command += ' --use_saved_model=' + str(use_saved_model).lower() command += ' --enable_batching=' + str(enable_batching).lower() - command += ' --alsologtostderr' print command self.server_proc = subprocess.Popen(shlex.split(command)) print 'Server started' From c20f6c1cfb4c065f0a3d6160cebbef9b5baf8212 Mon Sep 17 00:00:00 2001 From: isiosia <2664993@qq.com> Date: Wed, 22 Feb 2017 03:37:53 +0800 Subject: [PATCH 0168/8554] Add a gRPC client in C++ for inception serving. (#300) * Add an inception client in cc. * use the Tensor::FromProto and Tensor::SummarizeValue to output the result make the server_address/model_name/image_file configurable using Tensorflow flag --- tensorflow_serving/example/BUILD | 11 ++ .../example/inception_client.cc | 129 ++++++++++++++++++ 2 files changed, 140 insertions(+) create mode 100644 tensorflow_serving/example/inception_client.cc diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index ff7172d2795..de5d692d9c4 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -115,3 +115,14 @@ py_binary( "@org_tensorflow//tensorflow:tensorflow_py", ], ) + +cc_binary( + name = "inception_client_cc", + srcs = [ + "inception_client.cc", + ], + deps = [ + "//tensorflow_serving/apis:prediction_service_proto", + "@org_tensorflow//tensorflow/core:framework", + ], +) diff --git a/tensorflow_serving/example/inception_client.cc b/tensorflow_serving/example/inception_client.cc new file mode 100644 index 00000000000..6f8ebc69f82 --- /dev/null +++ b/tensorflow_serving/example/inception_client.cc @@ -0,0 +1,129 @@ +#include +#include + +#include +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/util/command_line_flags.h" + +using grpc::Channel; +using grpc::ClientContext; +using grpc::ClientReader; +using grpc::ClientReaderWriter; +using grpc::ClientWriter; +using grpc::Status; + + +using tensorflow::serving::PredictRequest; +using tensorflow::serving::PredictResponse; +using tensorflow::serving::PredictionService; + +typedef google::protobuf::Map< std::string, tensorflow::TensorProto > OutMap; + + +class ServingClient { + public: + ServingClient(std::shared_ptr channel) + : stub_(PredictionService::NewStub(channel)) { + } + + std::string callPredict(std::string model_name, std::string file_path){ + PredictRequest predictRequest; + PredictResponse response; + ClientContext context; + + predictRequest.mutable_model_spec()->set_name(model_name); + + google::protobuf::Map< std::string, tensorflow::TensorProto >& inputs = + *predictRequest.mutable_inputs(); + + tensorflow::TensorProto proto; + + std::ifstream imageFile(file_path, std::ios::binary); + + if (!imageFile.is_open()) { + std::cout << "Failed to open " << file_path << std::endl; + return ""; + } + + std::filebuf * pbuf = imageFile.rdbuf(); + long fileSize = pbuf->pubseekoff(0, std::ios::end, std::ios::in); + + char* image = new char[fileSize](); + + pbuf->pubseekpos(0, std::ios::in); + pbuf->sgetn(image, fileSize); + imageFile.close(); + + + proto.set_dtype(tensorflow::DataType::DT_STRING); + proto.add_string_val(image, fileSize); + + proto.mutable_tensor_shape()->add_dim()->set_size(1); + + inputs["images"] = proto; + + Status status = stub_->Predict(&context, predictRequest, &response); + + delete[] image; + + if (status.ok()) { + std::cout << "call predict ok" << std::endl; + std::cout << "outputs size is "<< response.outputs_size() << std::endl; + OutMap& map_outputs = *response.mutable_outputs(); + OutMap::iterator iter; + int output_index = 0; + + for(iter = map_outputs.begin();iter != map_outputs.end(); ++iter){ + tensorflow::TensorProto& result_tensor_proto= iter->second; + tensorflow::Tensor tensor; + bool converted = tensor.FromProto(result_tensor_proto); + if (converted) { + std::cout << "the result tensor[" << output_index << "] is:" << + std::endl << tensor.SummarizeValue(10) << std::endl; + }else { + std::cout << "the result tensor[" << output_index << + "] convert failed." << std::endl; + } + ++output_index; + } + return "Done."; + } else { + std::cout << "gRPC call return code: " + < stub_; +}; + +int main(int argc, char** argv) { + std::string server_port = "localhost:9000"; + std::string image_file = ""; + std::string model_name = "inception"; + std::vector flag_list = { + tensorflow::Flag("server_port", &server_port, + "the IP and port of the server"), + tensorflow::Flag("image_file", &image_file, + "the path to the "), + tensorflow::Flag("model_name", &model_name, "name of model") + }; + std::string usage = tensorflow::Flags::Usage(argv[0], flag_list); + const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); + if (!parse_result || image_file.empty()) { + std::cout << usage; + return -1; + } + + ServingClient guide( + grpc::CreateChannel( server_port, + grpc::InsecureChannelCredentials())); + std::cout << "calling predict using file: " << + image_file << " ..." << std::endl; + std::cout << guide.callPredict(model_name, image_file) << std::endl; + + return 0; +} From d067566ad6472220cf93330a212a10c4edcccd88 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 17 Feb 2017 14:00:41 -0800 Subject: [PATCH 0169/8554] Update no longer correct comment. Change: 147876575 --- tensorflow_serving/apis/classification.proto | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/apis/classification.proto b/tensorflow_serving/apis/classification.proto index a91c80f2d62..fa7ca50e0c3 100644 --- a/tensorflow_serving/apis/classification.proto +++ b/tensorflow_serving/apis/classification.proto @@ -19,7 +19,6 @@ message Class { // List of classes for a single item // (tensorflow.Example or tensorflow.InferenceExample.features). message Classifications { - // List of Classifications for an item sorted by decreasing score. repeated Class classes = 1; } From 6d6fd23a4115f05e811bf0c31f6b0e08114177f8 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 21 Feb 2017 11:37:37 -0800 Subject: [PATCH 0170/8554] Add missing dep to the get_model_metadata_proto_py_pb2 build target. This caused an issue when linking as a submodule - https://github.com/tensorflow/serving/issues/328 Change: 148122704 --- tensorflow_serving/apis/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index ec666746bad..26c3e33242d 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -41,6 +41,7 @@ serving_proto_library_py( srcs = ["get_model_metadata.proto"], proto_library = "get_model_metadata_proto", deps = [ + ":model_proto_py_pb2", "@org_tensorflow//tensorflow/core:protos_all_py", ], ) From 98c5865b5ea7b42f5bf7c5b440133dbff845876d Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 22 Feb 2017 11:14:50 -0800 Subject: [PATCH 0171/8554] remote testdata/half_plus_two2 which was accidentally still included in the original PR --- .../00000123/export.data-00000-of-00001 | Bin 8 -> 0 bytes .../half_plus_two2/00000123/export.index | Bin 142 -> 0 bytes .../testdata/half_plus_two2/00000123/export.meta | Bin 4266 -> 0 bytes 3 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.data-00000-of-00001 delete mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.index delete mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.meta diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.data-00000-of-00001 deleted file mode 100644 index 20bc7d454dd8450489984bd17d92c45c3a1a96a6..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 8 PcmZQzV6bOkU~m8c0fPX5 diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.index b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two2/00000123/export.index deleted file mode 100644 index 35e2ef7527905b228ebfd7f755bb3d06f6cc00e0..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 142 zcmZQzVB=tvV&Y(A;NT8REXqtw%1Py56k^a|F=ew*m*-&OjW-1G6&JCcXIiQeD<{{v=Y01&-#O==z%LU0rve{A>2N%z z5|DF+d>K41@X7G`Ej|?nxLEctrpB-h%|U&Y6w6YjRI5@M&Oti#Jab~xI+R@F2!H=0 zArzo;rdu;3FoR#5nLnNRTJm10B_`2&3x8^kb>GmuV|{AS@-xS=SaF9a8&IElhGsdV z=cYYLU;IW0+=a?L$M$^I28r8OD&qj8(vF-{$v14zag`?UH~bLLhwA;YVf&_k5dbx9 z#1;UpCx-CQDMI)dN}tcH>(R3k^79gLN>F~_d@+SXs6Vvyk#X!;W5ZoUodu~-sBkv; zgB6Dbu%PzH@O;NLo(vGVG(|$q^R7%g(BMbaZ2i>maAgfT;dV{8$C_uEqhOV-fXsjA z4Wy7OPw7JRiGpQ%{!YN)ogK1Azy#^Be)b<-(QCQ-2C7eV*VLa`1`-qMh(`>yq_nb3 z%taG5QX4t8ubZ~vQpxjOR0=E7f^rM$NP%mNLsG<7KNHfc?e+Hu{bNHP59FEs+;(2r z^wkD@1?w@AUDGywQ@6BG$&{Gn`vXFBXvT}XE{1|8iJ4-|^?EfIUqd%`q3vHI zqzAcvGc;QOsrfe^)V~O4rc6xl^Lif?eU{ZHyQ1wx;P-fE4caRU?Ik%G zQL+SJVq^g2pcK)tkFd_uSOHP|8BJEuDAxo$1n{tU?}jZhu3hVK?P{v^s!R)N<YN2;!M?Vp)r~zgCfHZ1;OyH5_GQgICOp6&4Ip`#<7d1I3 z;$7i;*a#s64b62dZEQ>p8?P#(3!yTKfErH)$hmR6ERn)}p*rG>mGOSD87?Q6j7y`u z9aFug(uiu3MtKS0Va|ou4lMpM9K-E7Z+XGZ^jxkhvNW`Fo?a4YWnIzqB9#g7vh%ao z!CjKect_>;En0!n03-ctthTuto@5J~LwxhPeYJi$h3za=SW_;`_1m!u*44E{1){q1 z6K^g$C-Clq$2X+}q`fomEKdu|d0?*xeH}f#BG3$haXo1JG`_3}oU#S%H&?}FwIOaW zVmJRVBF^Olh^AXs)R3YY#DTLGuSoGR;In86~KR6q)caF2G`I;`Q}5Om(szvJ$vMgRZ+ From fc251043cfef0f2fa4775b96ad8ca8c6c951022c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 21 Feb 2017 15:55:53 -0800 Subject: [PATCH 0172/8554] Move classifier/regressor implementations to Open Source. Change: 148156192 --- tensorflow_serving/core/test_util/BUILD | 11 + .../core/test_util/mock_session.h | 56 ++ tensorflow_serving/servables/tensorflow/BUILD | 169 ++++ .../tensorflow/classification_service.cc | 76 ++ .../tensorflow/classification_service.h | 38 + .../servables/tensorflow/classifier.cc | 424 ++++++++++ .../servables/tensorflow/classifier.h | 89 +++ .../servables/tensorflow/classifier_test.cc | 730 ++++++++++++++++++ .../tensorflow/regression_service.cc | 75 ++ .../servables/tensorflow/regression_service.h | 38 + .../servables/tensorflow/regressor.cc | 321 ++++++++ .../servables/tensorflow/regressor.h | 88 +++ .../servables/tensorflow/regressor_test.cc | 429 ++++++++++ .../servables/tensorflow/util.cc | 90 +++ .../servables/tensorflow/util.h | 55 ++ .../servables/tensorflow/util_test.cc | 181 +++++ 16 files changed, 2870 insertions(+) create mode 100644 tensorflow_serving/core/test_util/mock_session.h create mode 100644 tensorflow_serving/servables/tensorflow/classification_service.cc create mode 100644 tensorflow_serving/servables/tensorflow/classification_service.h create mode 100644 tensorflow_serving/servables/tensorflow/classifier.cc create mode 100644 tensorflow_serving/servables/tensorflow/classifier.h create mode 100644 tensorflow_serving/servables/tensorflow/classifier_test.cc create mode 100644 tensorflow_serving/servables/tensorflow/regression_service.cc create mode 100644 tensorflow_serving/servables/tensorflow/regression_service.h create mode 100644 tensorflow_serving/servables/tensorflow/regressor.cc create mode 100644 tensorflow_serving/servables/tensorflow/regressor.h create mode 100644 tensorflow_serving/servables/tensorflow/regressor_test.cc create mode 100644 tensorflow_serving/servables/tensorflow/util.cc create mode 100644 tensorflow_serving/servables/tensorflow/util.h create mode 100644 tensorflow_serving/servables/tensorflow/util_test.cc diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 0b7c3a4a186..05baa51e3c4 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -183,3 +183,14 @@ cc_library( "@protobuf//:protobuf", ], ) + +cc_library( + name = "mock_session", + testonly = 1, + hdrs = ["mock_session.h"], + deps = [ + "//external:gtest", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + ], +) diff --git a/tensorflow_serving/core/test_util/mock_session.h b/tensorflow_serving/core/test_util/mock_session.h new file mode 100644 index 00000000000..b7418922077 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_session.h @@ -0,0 +1,56 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_TEST_UTIL_MOCK_SESSION_H_ +#define TENSORFLOW_SERVING_TEST_UTIL_MOCK_SESSION_H_ + +#include +#include "tensorflow/core/public/session.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// A mock of tensorflow::Session. +class MockSession : public tensorflow::Session { + public: + MockSession() : Session() { + ON_CALL(*this, Close()).WillByDefault(::testing::Return(Status::OK())); + } + MOCK_METHOD1(Create, ::tensorflow::Status(const GraphDef& graph)); + MOCK_METHOD1(Extend, ::tensorflow::Status(const GraphDef& graph)); + MOCK_METHOD4(Run, ::tensorflow::Status( + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs)); + MOCK_METHOD4(PRunSetup, + ::tensorflow::Status(const std::vector& input_names, + const std::vector& output_names, + const std::vector& target_nodes, + string* handle)); + MOCK_METHOD4(PRun, ::tensorflow::Status( + const string& handle, + const std::vector>& inputs, + const std::vector& output_names, + std::vector* outputs)); + MOCK_METHOD0(Close, ::tensorflow::Status()); +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_TEST_UTIL_MOCK_SESSION_H_ diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index e5e1a9fcccb..6321caf065c 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -492,3 +492,172 @@ cc_test( "@org_tensorflow//tensorflow/core:test", ], ) + +cc_library( + name = "classifier", + srcs = ["classifier.cc"], + hdrs = ["classifier.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":util", + "//tensorflow_serving/apis:classification_proto", + "//tensorflow_serving/apis:classifier", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:signature", + "@org_tensorflow//tensorflow/core:all_kernels", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_library( + name = "classification_service", + srcs = ["classification_service.cc"], + hdrs = ["classification_service.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":classifier", + "//tensorflow_serving/apis:classification_proto", + "//tensorflow_serving/apis:classifier", + "//tensorflow_serving/core:servable_handle", + "//tensorflow_serving/model_servers:server_core", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:signature", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "regression_service", + srcs = ["regression_service.cc"], + hdrs = ["regression_service.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":regressor", + "//tensorflow_serving/apis:regression_proto", + "//tensorflow_serving/apis:regressor", + "//tensorflow_serving/core:servable_handle", + "//tensorflow_serving/model_servers:server_core", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:signature", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_library( + name = "regressor", + srcs = ["regressor.cc"], + hdrs = ["regressor.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":util", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/apis:regression_proto", + "//tensorflow_serving/apis:regressor", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:signature", + "@org_tensorflow//tensorflow/core:all_kernels", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "regressor_test", + size = "medium", + srcs = ["regressor_test.cc"], + deps = [ + ":regressor", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/apis:regression_proto", + "//tensorflow_serving/core/test_util:mock_session", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@protobuf//:protobuf_lite", + ], +) + +cc_test( + name = "classifier_test", + size = "medium", + srcs = ["classifier_test.cc"], + deps = [ + ":classifier", + "//tensorflow_serving/apis:classification_proto", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/core/test_util:mock_session", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", + "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@protobuf//:protobuf_lite", + ], +) + +cc_library( + name = "util", + srcs = ["util.cc"], + hdrs = ["util.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//tensorflow_serving/apis:input_proto", + "@org_tensorflow//tensorflow/core:all_kernels", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "util_test", + size = "small", + srcs = ["util_test.cc"], + deps = [ + ":util", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@org_tensorflow//tensorflow/core:testlib", + ], +) diff --git a/tensorflow_serving/servables/tensorflow/classification_service.cc b/tensorflow_serving/servables/tensorflow/classification_service.cc new file mode 100644 index 00000000000..1b021fa7b33 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/classification_service.cc @@ -0,0 +1,76 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/classification_service.h" + +#include + +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/contrib/session_bundle/signature.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow_serving/apis/classifier.h" +#include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/servables/tensorflow/classifier.h" + +namespace tensorflow { +namespace serving { + +Status TensorflowClassificationServiceImpl::Classify( + ServerCore* core, const bool use_saved_model, + const ClassificationRequest& request, ClassificationResponse* response) { + TRACELITERAL("TensorflowClassificationServiceImpl::Classify"); + // Verify Request Metadata and create a ServableRequest + if (!request.has_model_spec()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Missing ModelSpec"); + } + + std::unique_ptr classifier_interface; + if (use_saved_model) { + ServableHandle saved_model_bundle; + TF_RETURN_IF_ERROR( + core->GetServableHandle(request.model_spec(), &saved_model_bundle)); + SignatureDef signature; + TF_RETURN_IF_ERROR(GetClassificationSignatureDef( + request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( + saved_model_bundle->session.get(), &signature, &classifier_interface)); + // Run classification. + TF_RETURN_IF_ERROR( + classifier_interface->Classify(request, response->mutable_result())); + } else { + ServableHandle bundle; + TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); + Signature signature; + TF_RETURN_IF_ERROR(GetDefaultSignature(bundle->meta_graph_def, &signature)); + + if (!signature.has_classification_signature()) { + return tensorflow::Status(tensorflow::error::UNAVAILABLE, + "No Classification Signature"); + } + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( + bundle->session.get(), &signature.classification_signature(), + &classifier_interface)); + // Run classification. + TF_RETURN_IF_ERROR( + classifier_interface->Classify(request, response->mutable_result())); + } + + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/classification_service.h b/tensorflow_serving/servables/tensorflow/classification_service.h new file mode 100644 index 00000000000..731425425d9 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/classification_service.h @@ -0,0 +1,38 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CLASSIFICATION_SERVICE_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CLASSIFICATION_SERVICE_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/classification.pb.h" +#include "tensorflow_serving/model_servers/server_core.h" + +namespace tensorflow { +namespace serving { + +// Utility methods for implementation of +// tensorflow_serving/apis/classification-service.proto. +class TensorflowClassificationServiceImpl { + public: + static Status Classify(ServerCore* core, const bool use_saved_model, + const ClassificationRequest& request, + ClassificationResponse* response); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CLASSIFICATION_SERVICE_H_ diff --git a/tensorflow_serving/servables/tensorflow/classifier.cc b/tensorflow_serving/servables/tensorflow/classifier.cc new file mode 100644 index 00000000000..142a98e11af --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/classifier.cc @@ -0,0 +1,424 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/classifier.h" + +#include +#include +#include +#include +#include +#include + +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/contrib/session_bundle/signature.h" +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/apis/classification.pb.h" +#include "tensorflow_serving/apis/classifier.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/servables/tensorflow/util.h" + +namespace tensorflow { +namespace serving { +namespace { + +// Implementation of the ClassificationService using the legacy SessionBundle +// ClassificationSignature signature format. +class TensorFlowClassifier : public ClassifierInterface { + public: + explicit TensorFlowClassifier(Session* session, + const ClassificationSignature* signature) + : session_(session), signature_(signature) {} + + Status Classify(const ClassificationRequest& request, + ClassificationResult* result) override { + TRACELITERAL("TensorFlowClassifier::Classify"); + TRACELITERAL("ConvertInputTFEXamplesToTensor"); + // Setup the input Tensor to be a vector of string containing the serialized + // tensorflow.Example. + Tensor input_tensor; + TF_RETURN_IF_ERROR( + InputToSerializedExampleTensor(request.input(), &input_tensor)); + + const int num_examples = input_tensor.dim_size(0); + if (num_examples == 0) { + return errors::InvalidArgument("ClassificationRequest::input is empty."); + } + + TRACELITERAL("RunClassification"); + // Support serving models that return classes, scores or both. + std::unique_ptr classes; + if (!signature_->classes().tensor_name().empty()) { + classes.reset(new Tensor); + } + std::unique_ptr scores; + if (!signature_->scores().tensor_name().empty()) { + scores.reset(new Tensor); + } + + TF_RETURN_IF_ERROR(RunClassification(*signature_, input_tensor, session_, + classes.get(), scores.get())); + + // Validate classes output Tensor. + if (classes) { + if (classes->dtype() != DT_STRING) { + return errors::Internal("Expected classes Tensor of DT_STRING. Got: ", + DataType_Name(classes->dtype())); + } + if (classes->dim_size(0) != num_examples) { + return errors::Internal("Expected output batch size of ", num_examples, + ". Got: ", classes->dim_size(0)); + } + } + // Validate scores output Tensor. + if (scores) { + if (scores->dtype() != DT_FLOAT) { + return errors::Internal("Expected scores Tensor of DT_FLOAT. Got: ", + DataType_Name(scores->dtype())); + } + if (scores->dim_size(0) != num_examples) { + return errors::Internal("Expected output batch size of ", num_examples, + ". Got: ", scores->dim_size(0)); + } + } + // Extract the number of classes from either the class or score output + // Tensor. + int num_classes = 0; + if (classes && scores) { + // If we have both Tensors they should agree in the second dimmension. + if (classes->dim_size(1) != scores->dim_size(1)) { + return errors::Internal( + "Tensors class and score should match in dim_size(1). Got ", + classes->dim_size(1), " vs. ", scores->dim_size(1)); + } + num_classes = classes->dim_size(1); + } else if (classes) { + num_classes = classes->dim_size(1); + } else if (scores) { + num_classes = scores->dim_size(1); + } + + TRACELITERAL("ConvertToClassificationResult"); + // Convert the output to ClassificationResult format. + for (int i = 0; i < num_examples; ++i) { + serving::Classifications* classifications = result->add_classifications(); + for (int c = 0; c < num_classes; ++c) { + serving::Class* cl = classifications->add_classes(); + if (classes) { + cl->set_label((classes->matrix())(i, c)); + } + if (scores) { + cl->set_score((scores->matrix())(i, c)); + } + } + } + + return Status::OK(); + } + + private: + Session* const session_; + const ClassificationSignature* const signature_; + + TF_DISALLOW_COPY_AND_ASSIGN(TensorFlowClassifier); +}; + +// Implementation of the ClassifierInterface using SavedModel. +class SavedModelTensorFlowClassifier : public ClassifierInterface { + public: + explicit SavedModelTensorFlowClassifier(Session* session, + const SignatureDef* const signature) + : session_(session), signature_(signature) {} + + ~SavedModelTensorFlowClassifier() override = default; + + Status Classify(const ClassificationRequest& request, + ClassificationResult* result) override { + TRACELITERAL("TensorFlowClassifier::Classify"); + const int num_examples = NumInputExamples(request.input()); + if (num_examples == 0) { + return errors::InvalidArgument("ClassificationRequest::input is empty."); + } + + string input_tensor_name; + std::vector output_tensor_names; + TF_RETURN_IF_ERROR(PreProcessClassification(*signature_, &input_tensor_name, + &output_tensor_names)); + + std::vector outputs; + TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( + request.input(), input_tensor_name, output_tensor_names, session_, + &outputs)); + + TRACELITERAL("ConvertToClassificationResult"); + return PostProcessClassificationResult( + *signature_, num_examples, output_tensor_names, outputs, result); + } + + private: + Session* const session_; + const SignatureDef* const signature_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelTensorFlowClassifier); +}; + +// Implementation of the ClassificationService. +class SessionBundleClassifier : public ClassifierInterface { + public: + explicit SessionBundleClassifier(std::unique_ptr bundle) + : bundle_(std::move(bundle)) {} + + ~SessionBundleClassifier() override = default; + + Status Classify(const ClassificationRequest& request, + ClassificationResult* result) override { + // Get the default signature of the graph. Expected to be a + // classification signature. + // TODO(b/26220896): Move TensorFlowClassifier creation to construction + // time. + ClassificationSignature signature; + TF_RETURN_IF_ERROR( + GetClassificationSignature(bundle_->meta_graph_def, &signature)); + + TensorFlowClassifier classifier(bundle_->session.get(), &signature); + return classifier.Classify(request, result); + } + + private: + std::unique_ptr bundle_; + + TF_DISALLOW_COPY_AND_ASSIGN(SessionBundleClassifier); +}; + +class SavedModelClassifier : public ClassifierInterface { + public: + explicit SavedModelClassifier(std::unique_ptr bundle) + : bundle_(std::move(bundle)) {} + + ~SavedModelClassifier() override = default; + + Status Classify(const ClassificationRequest& request, + ClassificationResult* result) override { + // Get the default signature of the graph. Expected to be a + // classification signature. + // TODO(b/26220896): Move TensorFlowClassifier creation to construction + // time. + SignatureDef signature; + TF_RETURN_IF_ERROR(GetClassificationSignatureDef( + request.model_spec(), bundle_->meta_graph_def, &signature)); + SavedModelTensorFlowClassifier classifier(bundle_->session.get(), + &signature); + return classifier.Classify(request, result); + } + + private: + std::unique_ptr bundle_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelClassifier); +}; + +} // namespace + +Status CreateClassifierFromBundle( + std::unique_ptr bundle, + std::unique_ptr* service) { + service->reset(new SessionBundleClassifier(std::move(bundle))); + return Status::OK(); +} + +Status CreateClassifierFromSavedModelBundle( + std::unique_ptr bundle, + std::unique_ptr* service) { + service->reset(new SavedModelClassifier(std::move(bundle))); + return Status::OK(); +} + +Status CreateFlyweightTensorFlowClassifier( + Session* session, const ClassificationSignature* const signature, + std::unique_ptr* service) { + service->reset(new TensorFlowClassifier(session, signature)); + return Status::OK(); +} + +Status CreateFlyweightTensorFlowClassifier( + Session* session, const SignatureDef* signature, + std::unique_ptr* service) { + service->reset(new SavedModelTensorFlowClassifier(session, signature)); + return Status::OK(); +} + +Status GetClassificationSignatureDef(const ModelSpec& model_spec, + const MetaGraphDef& meta_graph_def, + SignatureDef* signature) { + const string signature_name = model_spec.signature_name().empty() + ? kDefaultServingSignatureDefKey + : model_spec.signature_name(); + auto iter = meta_graph_def.signature_def().find(signature_name); + if (iter == meta_graph_def.signature_def().end()) { + return errors::InvalidArgument(strings::StrCat( + "No signature was found with the name: ", signature_name)); + } + if (iter->second.method_name() != kClassifyMethodName) { + return errors::InvalidArgument(strings::StrCat( + "Expected classification signature method_name to be ", + kClassifyMethodName, ". Was: ", iter->second.method_name())); + } + *signature = iter->second; + return Status::OK(); +} + +Status PreProcessClassification(const SignatureDef& signature, + string* input_tensor_name, + std::vector* output_tensor_names) { + if (signature.method_name() != kClassifyMethodName) { + return errors::InvalidArgument(strings::StrCat( + "Expected classification signature method_name to be ", + kClassifyMethodName, ". Was: ", signature.method_name())); + } + if (signature.inputs().size() != 1) { + return errors::InvalidArgument( + strings::StrCat("Expected one input Tensor.")); + } + if (signature.outputs().size() != 1 && signature.outputs().size() != 2) { + return errors::InvalidArgument( + strings::StrCat("Expected one or two output Tensors, found ", + signature.outputs().size())); + } + + auto input_iter = signature.inputs().find(kClassifyInputs); + if (input_iter == signature.inputs().end()) { + return errors::FailedPrecondition( + "No classification inputs found in SignatureDef: ", + signature.DebugString()); + } + *input_tensor_name = input_iter->second.name(); + + auto classes_iter = signature.outputs().find(kClassifyOutputClasses); + auto scores_iter = signature.outputs().find(kClassifyOutputScores); + if (classes_iter == signature.outputs().end() && + scores_iter == signature.outputs().end()) { + return errors::FailedPrecondition(strings::StrCat( + "Expected classification signature outputs to contain at least one of ", + "\"", kClassifyOutputClasses, "\" or \"", kClassifyOutputScores, + "\". Signature was: ", signature.DebugString())); + } + if (classes_iter != signature.outputs().end()) { + output_tensor_names->push_back(classes_iter->second.name()); + } + if (scores_iter != signature.outputs().end()) { + output_tensor_names->push_back(scores_iter->second.name()); + } + return Status::OK(); +} + +Status PostProcessClassificationResult( + const SignatureDef& signature, int num_examples, + const std::vector& output_tensor_names, + const std::vector& output_tensors, ClassificationResult* result) { + if (output_tensors.size() != output_tensor_names.size()) { + return errors::InvalidArgument( + strings::StrCat("Expected ", output_tensor_names.size(), + " output tensor(s). Got: ", output_tensors.size())); + } + + auto classes_iter = signature.outputs().find(kClassifyOutputClasses); + string classes_tensor_name; + if (classes_iter != signature.outputs().end()) { + classes_tensor_name = classes_iter->second.name(); + } + auto scores_iter = signature.outputs().find(kClassifyOutputScores); + string scores_tensor_name; + if (scores_iter != signature.outputs().end()) { + scores_tensor_name = scores_iter->second.name(); + } + + const Tensor* classes = nullptr; + const Tensor* scores = nullptr; + for (int i = 0; i < output_tensors.size(); ++i) { + if (output_tensor_names[i] == classes_tensor_name) { + classes = &output_tensors[i]; + } else if (output_tensor_names[i] == scores_tensor_name) { + scores = &output_tensors[i]; + } + } + + // Validate classes output Tensor. + if (classes) { + if (classes->dtype() != DT_STRING) { + return errors::InvalidArgument( + "Expected classes Tensor of DT_STRING. Got: ", + DataType_Name(classes->dtype())); + } + if (classes->dim_size(0) != num_examples) { + return errors::InvalidArgument("Expected classes output batch size of ", + num_examples, + ". Got: ", classes->dim_size(0)); + } + } + // Validate scores output Tensor. + if (scores) { + if (scores->dtype() != DT_FLOAT) { + return errors::InvalidArgument( + "Expected scores Tensor of DT_FLOAT. Got: ", + DataType_Name(scores->dtype())); + } + if (scores->dim_size(0) != num_examples) { + return errors::InvalidArgument("Expected scores output batch size of ", + num_examples, + ". Got: ", scores->dim_size(0)); + } + } + // Extract the number of classes from either the class or score output + // Tensor. + int num_classes = 0; + if (classes && scores) { + // If we have both Tensors they should agree in the second dimmension. + if (classes->dim_size(1) != scores->dim_size(1)) { + return errors::InvalidArgument( + "Tensors class and score should match in dim_size(1). Got ", + classes->dim_size(1), " vs. ", scores->dim_size(1)); + } + num_classes = classes->dim_size(1); + } else if (classes) { + num_classes = classes->dim_size(1); + } else if (scores) { + num_classes = scores->dim_size(1); + } + + // Convert the output to ClassificationResult format. + for (int i = 0; i < num_examples; ++i) { + serving::Classifications* classifications = result->add_classifications(); + for (int c = 0; c < num_classes; ++c) { + serving::Class* cl = classifications->add_classes(); + if (classes) { + cl->set_label((classes->matrix())(i, c)); + } + if (scores) { + cl->set_score((scores->matrix())(i, c)); + } + } + } + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/classifier.h b/tensorflow_serving/servables/tensorflow/classifier.h new file mode 100644 index 00000000000..bb10f0a5b0d --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/classifier.h @@ -0,0 +1,89 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// TensorFlow implementation of the ClassifierInterface. + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CLASSIFIER_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CLASSIFIER_H_ + +#include + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/classifier.h" + +namespace tensorflow { +namespace serving { + +// Create a new ClassifierInterface backed by a TensorFlow Session. +// Requires the SessionBundle manifest to have a ClassificationSignature +// as the default signature. +Status CreateClassifierFromBundle( + std::unique_ptr bundle, + std::unique_ptr* service); + +// Create a new ClassifierInterface backed by a TensorFlow SavedModel. +// Requires that the default SignatureDef be compatible with classification. +Status CreateClassifierFromSavedModelBundle( + std::unique_ptr bundle, + std::unique_ptr* service); + +// Create a new ClassifierInterface backed by a TensorFlow Session using the +// specified ClassificationSignature. Does not take ownership of the Session. +// Useful in contexts where we need to avoid copying, e.g. if created per +// request. The caller must ensure that the session and signature live at least +// as long as the service. +Status CreateFlyweightTensorFlowClassifier( + Session* session, const ClassificationSignature* signature, + std::unique_ptr* service); + +// Create a new ClassifierInterface backed by a TensorFlow Session using the +// specified SignatureDef. Does not take ownership of the Session. +// Useful in contexts where we need to avoid copying, e.g. if created per +// request. The caller must ensure that the session and signature live at least +// as long as the service. +Status CreateFlyweightTensorFlowClassifier( + Session* session, const SignatureDef* signature, + std::unique_ptr* service); + +// Get a classification signature from the meta_graph_def that's either: +// 1) The signature that model_spec explicitly specifies to use. +// 2) The default serving signature. +// If neither exist, or there were other issues, an error status is returned. +Status GetClassificationSignatureDef(const ModelSpec& model_spec, + const MetaGraphDef& meta_graph_def, + SignatureDef* signature); + +// Validate a SignatureDef to make sure it's compatible with classification, and +// if so, populate the input and output tensor names. +// +// NOTE: output_tensor_names may already have elements in it (e.g. when building +// a full list of outputs from multiple signatures), and this function will just +// append to the vector. +Status PreProcessClassification(const SignatureDef& signature, + string* input_tensor_name, + std::vector* output_tensor_names); + +// Validate all results and populate a ClassificationResult. +Status PostProcessClassificationResult( + const SignatureDef& signature, int num_examples, + const std::vector& output_tensor_names, + const std::vector& output_tensors, ClassificationResult* result); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CLASSIFIER_H_ diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc new file mode 100644 index 00000000000..698bbcf4632 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -0,0 +1,730 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/classifier.h" + +#include +#include +#include +#include +#include + +#include "google/protobuf/map.h" +#include "tensorflow/contrib/session_bundle/bundle_shim.h" +#include "tensorflow/contrib/session_bundle/manifest.pb.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/example/feature.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow_serving/apis/classification.pb.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/core/test_util/mock_session.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::_; +using test_util::EqualsProto; +using test_util::MockSession; + +const char kInputTensor[] = "input:0"; +const char kClassTensor[] = "output:0"; +const char kOutputPlusOneClassTensor[] = "outputPlusOne:0"; +const char kClassFeature[] = "class"; +const char kScoreTensor[] = "score:0"; +const char kScoreFeature[] = "score"; + +const char kOutputPlusOneSignature[] = "output_plus_one"; +const char kInvalidNamedSignature[] = "invalid_regression_signature"; + +// Fake Session used for testing TensorFlowClassifier. +// Assumes the input Tensor "input:0" has serialized tensorflow::Example values. +// Copies the "class" bytes feature from each Example to be the classification +// class for that example. +class FakeSession : public tensorflow::Session { + public: + ~FakeSession() override = default; + Status Create(const GraphDef& graph) override { + return errors::Unimplemented("not available in fake"); + } + Status Extend(const GraphDef& graph) override { + return errors::Unimplemented("not available in fake"); + } + + Status Close() override { + return errors::Unimplemented("not available in fake"); + } + + Status Run(const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs) override { + if (inputs.size() != 1 || inputs[0].first != kInputTensor) { + return errors::Internal("Expected one input Tensor."); + } + for (const auto& output_name : output_names) { + if (output_name != kClassTensor && output_name != kScoreTensor && + output_name != kOutputPlusOneClassTensor) { + return errors::Internal("Unsupported output Tensor: ", output_name); + } + } + const Tensor& input = inputs[0].second; + std::vector examples; + TF_RETURN_IF_ERROR(GetExamples(input, &examples)); + Tensor classes; + Tensor scores; + TF_RETURN_IF_ERROR( + GetClassTensor(examples, output_names, &classes, &scores)); + for (const auto& output_name : output_names) { + if (output_name == kClassTensor) { + outputs->push_back(classes); + } else if (output_name == kScoreTensor || + output_name == kOutputPlusOneClassTensor) { + outputs->push_back(scores); + } + } + + return Status::OK(); + } + + // Parses TensorFlow Examples from a string Tensor. + static Status GetExamples(const Tensor& input, + std::vector* examples) { + examples->clear(); + const int batch_size = input.dim_size(0); + const auto& flat_input = input.flat(); + for (int i = 0; i < batch_size; ++i) { + Example example; + if (!example.ParseFromString(flat_input(i))) { + return errors::Internal("failed to parse example"); + } + examples->push_back(example); + } + return Status::OK(); + } + + // Gets the Feature from an Example with the given name. Returns empty + // Feature if the name does not exist. + static Feature GetFeature(const Example& example, const string& name) { + const auto it = example.features().feature().find(name); + if (it != example.features().feature().end()) { + return it->second; + } + return Feature(); + } + + // Returns the number of individual elements in a Feature. + static int FeatureSize(const Feature& feature) { + if (feature.has_float_list()) { + return feature.float_list().value_size(); + } else if (feature.has_int64_list()) { + return feature.int64_list().value_size(); + } else if (feature.has_bytes_list()) { + return feature.bytes_list().value_size(); + } + return 0; + } + + // Creates a Tensor by copying the "class" feature from each Example. + // Requires each Example have an bytes feature called "class" which is of the + // same non-zero length. + static Status GetClassTensor(const std::vector& examples, + const std::vector& output_names, + Tensor* classes, Tensor* scores) { + if (examples.empty()) { + return errors::Internal("empty example list"); + } + + auto iter = std::find(output_names.begin(), output_names.end(), + kOutputPlusOneClassTensor); + const float offset = iter == output_names.end() ? 0 : 1; + + const int batch_size = examples.size(); + const int num_classes = FeatureSize(GetFeature(examples[0], kClassFeature)); + *classes = Tensor(DT_STRING, TensorShape({batch_size, num_classes})); + *scores = Tensor(DT_FLOAT, TensorShape({batch_size, num_classes})); + auto classes_matrix = classes->matrix(); + auto scores_matrix = scores->matrix(); + + for (int i = 0; i < batch_size; ++i) { + const Feature classes_feature = GetFeature(examples[i], kClassFeature); + if (FeatureSize(classes_feature) != num_classes) { + return errors::Internal("incorrect number of classes in feature: ", + classes_feature.DebugString()); + } + const Feature scores_feature = GetFeature(examples[i], kScoreFeature); + if (FeatureSize(scores_feature) != num_classes) { + return errors::Internal("incorrect number of scores in feature: ", + scores_feature.DebugString()); + } + for (int c = 0; c < num_classes; ++c) { + classes_matrix(i, c) = classes_feature.bytes_list().value(c); + scores_matrix(i, c) = scores_feature.float_list().value(c) + offset; + } + } + return Status::OK(); + } +}; + +// Add a named signature to the mutable signatures* parameter. +// If is_classification is false, will add a regression signature, which is +// invalid in classification requests. +void AddNamedSignature(const string& input_tensor_name, + const string& output_scores_tensor_name, + const string& signature_name, + const bool is_classification, Signatures* signatures) { + tensorflow::serving::Signature named_signature; + if (is_classification) { + named_signature.mutable_classification_signature() + ->mutable_input() + ->set_tensor_name(input_tensor_name); + named_signature.mutable_classification_signature() + ->mutable_classes() + ->set_tensor_name(kClassTensor); + named_signature.mutable_classification_signature() + ->mutable_scores() + ->set_tensor_name(output_scores_tensor_name); + } else { + named_signature.mutable_regression_signature() + ->mutable_input() + ->set_tensor_name(input_tensor_name); + named_signature.mutable_regression_signature() + ->mutable_output() + ->set_tensor_name(output_scores_tensor_name); + } + signatures->mutable_named_signatures()->insert( + protobuf::MapPair( + signature_name, named_signature)); +} + +// Parameter is 'bool use_saved_model'. +class ClassifierTest : public ::testing::TestWithParam { + public: + void SetUp() override { + bundle_.reset(new SessionBundle); + meta_graph_def_ = &bundle_->meta_graph_def; + fake_session_ = new FakeSession(); + bundle_->session.reset(fake_session_); + + // Setup some defaults for our signature. + tensorflow::serving::Signatures signatures; + auto signature = signatures.mutable_default_signature() + ->mutable_classification_signature(); + signature->mutable_input()->set_tensor_name(kInputTensor); + signature->mutable_classes()->set_tensor_name(kClassTensor); + signature->mutable_scores()->set_tensor_name(kScoreTensor); + + AddNamedSignature(kInputTensor, kOutputPlusOneClassTensor, + kOutputPlusOneSignature, true /* is_classification */, + &signatures); + AddNamedSignature(kInputTensor, kOutputPlusOneClassTensor, + kInvalidNamedSignature, false /* is_classification */, + &signatures); + TF_ASSERT_OK( + tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); + } + + protected: + // Return an example with the feature "output" = [output]. + Example example(const std::vector>& class_scores) { + Feature classes_feature; + Feature scores_feature; + for (const auto& class_score : class_scores) { + classes_feature.mutable_bytes_list()->add_value(class_score.first); + scores_feature.mutable_float_list()->add_value(class_score.second); + } + Example example; + auto* features = example.mutable_features()->mutable_feature(); + (*features)[kClassFeature] = classes_feature; + (*features)[kScoreFeature] = scores_feature; + return example; + } + + Status Create() { + if (GetParam()) { + std::unique_ptr saved_model(new SavedModelBundle); + internal::ConvertSessionBundleToSavedModelBundle(*bundle_, + saved_model.get()); + return CreateClassifierFromSavedModelBundle(std::move(saved_model), + &classifier_); + } else { + return CreateClassifierFromBundle(std::move(bundle_), &classifier_); + } + } + + // Variables used to create the classifier. + tensorflow::MetaGraphDef* meta_graph_def_; + FakeSession* fake_session_; + std::unique_ptr bundle_; + + // Classifier valid after calling create. + std::unique_ptr classifier_; + + // Convenience variables. + ClassificationRequest request_; + ClassificationResult result_; +}; + +TEST_P(ClassifierTest, ExampleList) { + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + TF_ASSERT_OK(classifier_->Classify(request_, &result_)); + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " score: 4 " + " } " + " classes { " + " label: 'tres' " + " score: 3 " + " } " + " } ")); +} + +TEST_P(ClassifierTest, ExampleListWithContext) { + TF_ASSERT_OK(Create()); + auto* list_and_context = + request_.mutable_input()->mutable_example_list_with_context(); + // Context gets copied to each example. + *list_and_context->mutable_context() = example({{"dos", 2}, {"uno", 1}}); + // Add empty examples to recieve the context. + list_and_context->add_examples(); + list_and_context->add_examples(); + TF_ASSERT_OK(classifier_->Classify(request_, &result_)); + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } ")); +} + +TEST_P(ClassifierTest, ExampleListWithContext_DuplicateFeatures) { + TF_ASSERT_OK(Create()); + auto* list_and_context = + request_.mutable_input()->mutable_example_list_with_context(); + // Context gets copied to each example. + *list_and_context->mutable_context() = example({{"uno", 1}, {"dos", 2}}); + // Add an empty example, after merge it should be equal to the context. + list_and_context->add_examples(); + // Add an example with a duplicate feature. Technically this behavior is + // undefined so here we are ensuring we don't crash. + *list_and_context->add_examples() = example({{"tres", 3}, {"cuatro", 4}}); + TF_ASSERT_OK(classifier_->Classify(request_, &result_)); + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " } " + " classifications { " + " classes { " + " label: 'tres' " + " score: 3 " + " } " + " classes { " + " label: 'cuatro' " + " score: 4 " + " } " + " } ")); +} + +TEST_P(ClassifierTest, ClassesOnly) { + tensorflow::serving::Signatures signatures; + auto signature = signatures.mutable_default_signature() + ->mutable_classification_signature(); + signature->mutable_input()->set_tensor_name(kInputTensor); + signature->mutable_classes()->set_tensor_name(kClassTensor); + // No scores Tensor. + TF_ASSERT_OK(tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + TF_ASSERT_OK(classifier_->Classify(request_, &result_)); + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " } " + " classes { " + " label: 'uno' " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " } " + " classes { " + " label: 'tres' " + " } " + " } ")); +} + +TEST_P(ClassifierTest, ScoresOnly) { + tensorflow::serving::Signatures signatures; + auto signature = signatures.mutable_default_signature() + ->mutable_classification_signature(); + signature->mutable_input()->set_tensor_name(kInputTensor); + // No classes Tensor. + signature->mutable_scores()->set_tensor_name(kScoreTensor); + TF_ASSERT_OK(tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + TF_ASSERT_OK(classifier_->Classify(request_, &result_)); + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " score: 2 " + " } " + " classes { " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " score: 4 " + " } " + " classes { " + " score: 3 " + " } " + " } ")); +} + +TEST_P(ClassifierTest, ValidNamedSignature) { + TF_ASSERT_OK(Create()); + request_.mutable_model_spec()->set_signature_name(kOutputPlusOneSignature); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + TF_ASSERT_OK(classifier_->Classify(request_, &result_)); + + // GetParam() is 'is_saved_model' in this test. If using saved_model, this + // test should use the kOutputPlusOneSignature named signature. Otherwise, + // when using session_bundle, the signature_name in the model_spec will be + // ignored and the default signature will be used. + if (GetParam()) { + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 3 " + " } " + " classes { " + " label: 'uno' " + " score: 2 " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " score: 5 " + " } " + " classes { " + " label: 'tres' " + " score: 4 " + " } " + " } ")); + } else { + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " score: 4 " + " } " + " classes { " + " label: 'tres' " + " score: 3 " + " } " + " } ")); + } +} + +TEST_P(ClassifierTest, InvalidNamedSignature) { + TF_ASSERT_OK(Create()); + request_.mutable_model_spec()->set_signature_name(kInvalidNamedSignature); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + const Status status = classifier_->Classify(request_, &result_); + + // GetParam() is 'is_saved_model' in this test. If using saved_model, this + // test should fail because the named_signature requested is actually a + // regression signature. When using session_bundle, the signature_name + // will be ignored and the default signature will be used. + if (GetParam()) { + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } else { + TF_ASSERT_OK(status); + EXPECT_THAT(result_, EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " score: 4 " + " } " + " classes { " + " label: 'tres' " + " score: 3 " + " } " + " } ")); + } +} + +TEST_P(ClassifierTest, MissingClassificationSignature) { + tensorflow::serving::Signatures signatures; + signatures.mutable_default_signature(); + TF_ASSERT_OK(tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}}); + // TODO(b/26220896): This error should move to construction time. + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + // Old SessionBundle code treats a missing signature as a FAILED_PRECONDITION + // but new SavedModel code treats it as an INVALID_ARGUMENT (signature + // specified in the request was invalid). + if (GetParam()) { + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } else { + EXPECT_EQ(::tensorflow::error::FAILED_PRECONDITION, status.code()) + << status; + } +} + +TEST_P(ClassifierTest, EmptyInput) { + TF_ASSERT_OK(Create()); + // Touch input. + request_.mutable_input(); + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("ClassificationRequest::input is empty")); +} + +TEST_P(ClassifierTest, EmptyExampleList) { + TF_ASSERT_OK(Create()); + // Touch ExampleList. + request_.mutable_input()->mutable_example_list(); + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("ClassificationRequest::input is empty")); +} + +TEST_P(ClassifierTest, EmptyExampleListWithContext) { + TF_ASSERT_OK(Create()); + // Touch ExampleListWithContext, context populated but no Examples. + *request_.mutable_input() + ->mutable_example_list_with_context() + ->mutable_context() = example({{"dos", 2}}); + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("ClassificationRequest::input is empty")); +} + +TEST_P(ClassifierTest, RunsFails) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly( + ::testing::Return(errors::Internal("Run totally failed"))); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}}); + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Run totally failed")); +} + +TEST_P(ClassifierTest, ClassesIncorrectTensorBatchSize) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + // This Tensor only has one batch item but we will have two inputs. + Tensor classes(DT_STRING, TensorShape({1, 2})); + Tensor scores(DT_FLOAT, TensorShape({2, 2})); + std::vector outputs = {classes, scores}; + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("batch size")); +} + +TEST_P(ClassifierTest, ClassesIncorrectTensorType) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + // This Tensor is the wrong type for class. + Tensor classes(DT_FLOAT, TensorShape({2, 2})); + Tensor scores(DT_FLOAT, TensorShape({2, 2})); + std::vector outputs = {classes, scores}; + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Expected classes Tensor of DT_STRING")); +} + +TEST_P(ClassifierTest, ScoresIncorrectTensorBatchSize) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + Tensor classes(DT_STRING, TensorShape({2, 2})); + // This Tensor only has one batch item but we will have two inputs. + Tensor scores(DT_FLOAT, TensorShape({1, 2})); + std::vector outputs = {classes, scores}; + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("batch size")); +} + +TEST_P(ClassifierTest, ScoresIncorrectTensorType) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + Tensor classes(DT_STRING, TensorShape({2, 2})); + // This Tensor is the wrong type for class. + Tensor scores(DT_STRING, TensorShape({2, 2})); + std::vector outputs = {classes, scores}; + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Expected scores Tensor of DT_FLOAT")); +} + +TEST_P(ClassifierTest, MismatchedNumberOfTensorClasses) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + Tensor classes(DT_STRING, TensorShape({2, 2})); + // Scores Tensor has three scores but classes only has two labels. + Tensor scores(DT_FLOAT, TensorShape({2, 3})); + std::vector outputs = {classes, scores}; + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + + const Status status = classifier_->Classify(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT( + status.ToString(), + ::testing::HasSubstr( + "Tensors class and score should match in dim_size(1). Got 2 vs. 3")); +} + +// Test all ClassifierTest test cases with both SessionBundle and SavedModel. +INSTANTIATE_TEST_CASE_P(UseSavedModel, ClassifierTest, ::testing::Bool()); + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/regression_service.cc b/tensorflow_serving/servables/tensorflow/regression_service.cc new file mode 100644 index 00000000000..8e5f53bf1b3 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/regression_service.cc @@ -0,0 +1,75 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/regression_service.h" + +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/contrib/session_bundle/signature.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow_serving/apis/regressor.h" +#include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/servables/tensorflow/regressor.h" + +namespace tensorflow { +namespace serving { + +Status TensorflowRegressionServiceImpl::Regress( + ServerCore* core, const bool use_saved_model, + const RegressionRequest& request, RegressionResponse* response) { + TRACELITERAL("TensorflowRegressionServiceImpl::Regress"); + // Verify Request Metadata and create a ServableRequest + if (!request.has_model_spec()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "Missing ModelSpec"); + } + + std::unique_ptr regressor_interface; + if (use_saved_model) { + ServableHandle saved_model_bundle; + TF_RETURN_IF_ERROR( + core->GetServableHandle(request.model_spec(), &saved_model_bundle)); + SignatureDef signature; + TF_RETURN_IF_ERROR(GetRegressionSignatureDef( + request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); + + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( + saved_model_bundle->session.get(), &signature, ®ressor_interface)); + // Run regression + TF_RETURN_IF_ERROR( + regressor_interface->Regress(request, response->mutable_result())); + } else { + ServableHandle bundle; + TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); + Signature signature; + TF_RETURN_IF_ERROR(GetDefaultSignature(bundle->meta_graph_def, &signature)); + + if (!signature.has_regression_signature()) { + return tensorflow::Status(tensorflow::error::UNAVAILABLE, + "No Regression Signature"); + } + // Create a regressor interface for the session bundle + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( + bundle->session.get(), &signature.regression_signature(), + ®ressor_interface)); + // Run regression + TF_RETURN_IF_ERROR( + regressor_interface->Regress(request, response->mutable_result())); + } + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/regression_service.h b/tensorflow_serving/servables/tensorflow/regression_service.h new file mode 100644 index 00000000000..17b6063f8c9 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/regression_service.h @@ -0,0 +1,38 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_REGRESSION_SERVICE_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_REGRESSION_SERVICE_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/regression.pb.h" +#include "tensorflow_serving/model_servers/server_core.h" + +namespace tensorflow { +namespace serving { + +// Utility methods for implementation of +// tensorflow_serving/apis/regression-service.proto. +class TensorflowRegressionServiceImpl final { + public: + static Status Regress(ServerCore* core, const bool use_saved_model, + const RegressionRequest& request, + RegressionResponse* response); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_REGRESSION_SERVICE_H_ diff --git a/tensorflow_serving/servables/tensorflow/regressor.cc b/tensorflow_serving/servables/tensorflow/regressor.cc new file mode 100644 index 00000000000..37f4f192532 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/regressor.cc @@ -0,0 +1,321 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/regressor.h" + +#include +#include +#include +#include +#include +#include + +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/contrib/session_bundle/signature.h" +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/regression.pb.h" +#include "tensorflow_serving/apis/regressor.h" +#include "tensorflow_serving/servables/tensorflow/util.h" + +namespace tensorflow { +namespace serving { +namespace { + +// Implementation of the RegressorInterface using the legacy SessionBundle +// RegressionSignature signature format. +class TensorFlowRegressor : public RegressorInterface { + public: + explicit TensorFlowRegressor(Session* session, + const RegressionSignature* const signature) + : session_(session), signature_(signature) {} + + ~TensorFlowRegressor() override = default; + + Status Regress(const RegressionRequest& request, + RegressionResult* result) override { + TRACELITERAL("TensorFlowRegressor::Regress"); + TRACELITERAL("ConvertInputTFEXamplesToTensor"); + // Setup the input Tensor to be a vector of string containing the serialized + // tensorflow.Example. + Tensor input_tensor; + TF_RETURN_IF_ERROR( + InputToSerializedExampleTensor(request.input(), &input_tensor)); + + const int num_examples = input_tensor.dim_size(0); + if (num_examples == 0) { + return errors::InvalidArgument("RegressionRequest::input is empty."); + } + + TRACELITERAL("RunRegression"); + Tensor output; + TF_RETURN_IF_ERROR( + RunRegression(*signature_, input_tensor, session_, &output)); + + if (output.dtype() != DT_FLOAT) { + return errors::Internal("Expected output Tensor of DT_FLOAT. Got: ", + DataType_Name(output.dtype())); + } + + if (output.NumElements() != num_examples) { + return errors::Internal("Expected output batch size to be ", num_examples, + ". Got: ", output.NumElements()); + } + + TRACELITERAL("ConvertToRegressionResult"); + for (int i = 0; i < num_examples; ++i) { + result->add_regressions()->set_value(output.flat()(i)); + } + return Status::OK(); + } + + private: + Session* const session_; + const RegressionSignature* const signature_; + + TF_DISALLOW_COPY_AND_ASSIGN(TensorFlowRegressor); +}; + +// Implementation of the RegressorInterface using SavedModel. +class SavedModelTensorFlowRegressor : public RegressorInterface { + public: + explicit SavedModelTensorFlowRegressor(Session* session, + const SignatureDef* const signature) + : session_(session), signature_(signature) {} + + ~SavedModelTensorFlowRegressor() override = default; + + Status Regress(const RegressionRequest& request, + RegressionResult* result) override { + TRACELITERAL("SavedModelTensorFlowRegressor::Regress"); + const int num_examples = NumInputExamples(request.input()); + if (num_examples == 0) { + return errors::InvalidArgument("RegressionRequest::input is empty."); + } + + string input_tensor_name; + std::vector output_tensor_names; + TF_RETURN_IF_ERROR(PreProcessRegression(*signature_, &input_tensor_name, + &output_tensor_names)); + + std::vector outputs; + TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( + request.input(), input_tensor_name, output_tensor_names, session_, + &outputs)); + + TRACELITERAL("ConvertToRegressionResult"); + return PostProcessRegressionResult(*signature_, num_examples, + output_tensor_names, outputs, result); + } + + private: + Session* const session_; + const SignatureDef* const signature_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelTensorFlowRegressor); +}; + +// Implementation of the RegressorInterface +class SessionBundleRegressor : public RegressorInterface { + public: + explicit SessionBundleRegressor(std::unique_ptr bundle) + : bundle_(std::move(bundle)) {} + + ~SessionBundleRegressor() override = default; + + Status Regress(const RegressionRequest& request, + RegressionResult* result) override { + RegressionSignature signature; + TF_RETURN_IF_ERROR( + GetRegressionSignature(bundle_->meta_graph_def, &signature)); + + TensorFlowRegressor regressor(bundle_->session.get(), &signature); + return regressor.Regress(request, result); + } + + private: + std::unique_ptr bundle_; + + TF_DISALLOW_COPY_AND_ASSIGN(SessionBundleRegressor); +}; + +class SavedModelRegressor : public RegressorInterface { + public: + explicit SavedModelRegressor(std::unique_ptr bundle) + : bundle_(std::move(bundle)) {} + + ~SavedModelRegressor() override = default; + + Status Regress(const RegressionRequest& request, + RegressionResult* result) override { + SignatureDef signature; + TF_RETURN_IF_ERROR(GetRegressionSignatureDef( + request.model_spec(), bundle_->meta_graph_def, &signature)); + SavedModelTensorFlowRegressor regressor(bundle_->session.get(), &signature); + return regressor.Regress(request, result); + } + + private: + std::unique_ptr bundle_; + + TF_DISALLOW_COPY_AND_ASSIGN(SavedModelRegressor); +}; + +} // namespace + +Status CreateRegressorFromBundle(std::unique_ptr bundle, + std::unique_ptr* service) { + service->reset(new SessionBundleRegressor(std::move(bundle))); + return Status::OK(); +} + +Status CreateRegressorFromSavedModelBundle( + std::unique_ptr bundle, + std::unique_ptr* service) { + service->reset(new SavedModelRegressor(std::move(bundle))); + return Status::OK(); +} + +Status CreateFlyweightTensorFlowRegressor( + Session* session, const RegressionSignature* const signature, + std::unique_ptr* service) { + service->reset(new TensorFlowRegressor(session, signature)); + return Status::OK(); +} + +Status CreateFlyweightTensorFlowRegressor( + Session* session, const SignatureDef* signature, + std::unique_ptr* service) { + service->reset(new SavedModelTensorFlowRegressor(session, signature)); + return Status::OK(); +} + +Status GetRegressionSignatureDef(const ModelSpec& model_spec, + const MetaGraphDef& meta_graph_def, + SignatureDef* signature) { + const string signature_name = model_spec.signature_name().empty() + ? kDefaultServingSignatureDefKey + : model_spec.signature_name(); + auto iter = meta_graph_def.signature_def().find(signature_name); + if (iter == meta_graph_def.signature_def().end()) { + return errors::InvalidArgument(strings::StrCat( + "No signature was found with the name: ", signature_name)); + } + if (iter->second.method_name() != kRegressMethodName) { + return errors::InvalidArgument(strings::StrCat( + "Expected regression signature method_name to be ", kRegressMethodName, + ". Was: ", iter->second.method_name())); + } + *signature = iter->second; + return Status::OK(); +} + +Status PreProcessRegression(const SignatureDef& signature, + string* input_tensor_name, + std::vector* output_tensor_names) { + if (signature.method_name() != kRegressMethodName) { + return errors::InvalidArgument(strings::StrCat( + "Expected regression signature method_name to be ", kRegressMethodName, + ". Was: ", signature.method_name())); + } + if (signature.inputs().size() != 1) { + return errors::InvalidArgument( + strings::StrCat("Expected one input Tensor.")); + } + if (signature.outputs().size() != 1) { + return errors::InvalidArgument( + strings::StrCat("Expected one output Tensor.")); + } + + auto input_iter = signature.inputs().find(kRegressInputs); + if (input_iter == signature.inputs().end()) { + return errors::FailedPrecondition( + "No regression inputs found in SignatureDef: ", + signature.DebugString()); + } + *input_tensor_name = input_iter->second.name(); + + auto output_iter = signature.outputs().find(kRegressOutputs); + if (output_iter == signature.outputs().end()) { + return errors::FailedPrecondition( + "No regression outputs found in SignatureDef: ", + signature.DebugString()); + } + output_tensor_names->push_back(output_iter->second.name()); + return Status::OK(); +} + +Status PostProcessRegressionResult( + const SignatureDef& signature, int num_examples, + const std::vector& output_tensor_names, + const std::vector& output_tensors, RegressionResult* result) { + if (output_tensors.size() != output_tensor_names.size()) { + return errors::InvalidArgument( + "Expected output_tensors and output_tensor_names to have the same " + "size."); + } + + auto output_iter = signature.outputs().find(kRegressOutputs); + if (output_iter == signature.outputs().end()) { + return errors::FailedPrecondition( + "No regression outputs found in SignatureDef: ", + signature.DebugString()); + } + const string output_tensor_name = output_iter->second.name(); + const Tensor* output_tensor = nullptr; + for (int i = 0; i < output_tensor_names.size(); ++i) { + if (output_tensor_names[i] == output_tensor_name) { + output_tensor = &output_tensors[i]; + break; + } + } + + // Ensure the regression score output is shaped how we expect. + // There should be one float Tensor of shape, + // [batch_size, num_recommendations]. + if (output_tensor == nullptr) { + return errors::InvalidArgument(strings::StrCat( + "Could not find output tensor '", output_tensor_name, "'")); + } + if (num_examples != output_tensor->dim_size(0)) { + return errors::InvalidArgument(strings::StrCat( + "Input batch size did not match output batch size: ", num_examples, + " vs. ", output_tensor->dim_size(0))); + } + if (output_tensor->dtype() != DT_FLOAT) { + return errors::InvalidArgument("Expected output Tensor of DT_FLOAT. Got: ", + DataType_Name(output_tensor->dtype())); + } + + if (output_tensor->NumElements() != num_examples) { + return errors::InvalidArgument("Expected output batch size to be ", + num_examples, + ". Got: ", output_tensor->NumElements()); + } + for (int i = 0; i < num_examples; ++i) { + result->add_regressions()->set_value(output_tensor->flat()(i)); + } + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/regressor.h b/tensorflow_serving/servables/tensorflow/regressor.h new file mode 100644 index 00000000000..659cc3bc1ea --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/regressor.h @@ -0,0 +1,88 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// TensorFlow implementation of the RegressorInterface. + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_REGRESSOR_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_REGRESSOR_H_ + +#include + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/regressor.h" + +namespace tensorflow { +namespace serving { + +// Create a new RegressorInterface backed by a TensorFlow Session. +// Requires the SessionBundle manifest have a RegressionSignature +// as the default signature. +Status CreateRegressorFromBundle(std::unique_ptr bundle, + std::unique_ptr* service); + +// Create a new RegressorInterface backed by a TensorFlow SavedModel. +// Requires that the default SignatureDef be compatible with Regression. +Status CreateRegressorFromSavedModelBundle( + std::unique_ptr bundle, + std::unique_ptr* service); + +// Create a new RegressorInterface backed by a TensorFlow Session using the +// specified RegressionSignature. Does not take ownership of the Session. +// Useful in contexts where we need to avoid copying, e.g. if created per +// request. The caller must ensure that the session and signature live at least +// as long as the service. +Status CreateFlyweightTensorFlowRegressor( + Session* session, const RegressionSignature* signature, + std::unique_ptr* service); + +// Create a new RegressorInterface backed by a TensorFlow Session using the +// specified SignatureDef. Does not take ownership of the Session. +// Useful in contexts where we need to avoid copying, e.g. if created per +// request. The caller must ensure that the session and signature live at least +// as long as the service. +Status CreateFlyweightTensorFlowRegressor( + Session* session, const SignatureDef* signature, + std::unique_ptr* service); + +// Get a regression signature from the meta_graph_def that's either: +// 1) The signature that model_spec explicitly specifies to use. +// 2) The default serving signature. +// If neither exist, or there were other issues, an error status is returned. +Status GetRegressionSignatureDef(const ModelSpec& model_spec, + const MetaGraphDef& meta_graph_def, + SignatureDef* signature); + +// Validate a SignatureDef to make sure it's compatible with Regression, and if +// so, populate the input and output tensor names. +// +// NOTE: output_tensor_names may already have elements in it (e.g. when building +// a full list of outputs from multiple signatures), and this function will just +// append to the vector. +Status PreProcessRegression(const SignatureDef& signature, + string* input_tensor_name, + std::vector* output_tensor_names); + +// Validate all results and populate a RegressionResult. +Status PostProcessRegressionResult( + const SignatureDef& signature, int num_examples, + const std::vector& output_tensor_names, + const std::vector& output_tensors, RegressionResult* result); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_REGRESSOR_H_ diff --git a/tensorflow_serving/servables/tensorflow/regressor_test.cc b/tensorflow_serving/servables/tensorflow/regressor_test.cc new file mode 100644 index 00000000000..8117f9ab3ab --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/regressor_test.cc @@ -0,0 +1,429 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/regressor.h" + +#include +#include +#include +#include + +#include "google/protobuf/map.h" +#include "tensorflow/contrib/session_bundle/bundle_shim.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/example/feature.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/platform/mutex.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/regression.pb.h" +#include "tensorflow_serving/core/test_util/mock_session.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::_; +using test_util::EqualsProto; +using test_util::MockSession; + +const char kInputTensor[] = "input:0"; +const char kOutputTensor[] = "output:0"; +const char kOutputPlusOneTensor[] = "outputPlusOne:0"; +const char kOutputFeature[] = "output"; + +const char kOutputPlusOneSignature[] = "output_plus_one"; +const char kInvalidNamedSignature[] = "invalid_classification_signature"; + +// Fake Session used for testing TensorFlowRegressor +// Assumes the input Tensor "input:0" has serialized tensorflow::Example values. +// Copies the "output" float feature from each Example. +class FakeSession : public tensorflow::Session { + public: + ~FakeSession() override = default; + Status Create(const GraphDef& graph) override { + return errors::Unimplemented("not available in fake"); + } + Status Extend(const GraphDef& graph) override { + return errors::Unimplemented("not available in fake"); + } + + Status Close() override { + return errors::Unimplemented("not available in fake"); + } + + Status Run(const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs) override { + if (inputs.size() != 1 || inputs[0].first != kInputTensor) { + return errors::Internal("Expected one input Tensor."); + } + if (output_names.size() != 1 || (output_names[0] != kOutputTensor && + output_names[0] != kOutputPlusOneTensor)) { + return errors::Internal("Expected one output Tensor."); + } + const Tensor& input = inputs[0].second; + std::vector examples; + TF_RETURN_IF_ERROR(GetExamples(input, &examples)); + Tensor output; + TF_RETURN_IF_ERROR(GetOutputTensor(examples, output_names[0], &output)); + outputs->push_back(output); + return Status::OK(); + } + + // Parses TensorFlow Examples from a string Tensor. + static Status GetExamples(const Tensor& input, + std::vector* examples) { + examples->clear(); + const int batch_size = input.dim_size(0); + const auto& flat_input = input.flat(); + for (int i = 0; i < batch_size; ++i) { + Example example; + if (!example.ParseFromString(flat_input(i))) { + return errors::Internal("failed to parse example"); + } + examples->push_back(example); + } + return Status::OK(); + } + + // Gets the Feature from an Example with the given name. Returns empty + // Feature if the name does not exist. + static Feature GetFeature(const Example& example, const string& name) { + const auto it = example.features().feature().find(name); + if (it != example.features().feature().end()) { + return it->second; + } + return Feature(); + } + + // Creates a Tensor by copying the "output" feature from each Example. + // Requires each Example have an bytes feature called "class" which is of the + // same non-zero length. + static Status GetOutputTensor(const std::vector& examples, + const string& output_tensor_name, + Tensor* tensor) { + if (examples.empty()) { + return errors::Internal("empty example list"); + } + const int batch_size = examples.size(); + *tensor = Tensor(DT_FLOAT, TensorShape({batch_size, 1})); + auto output_matrix = tensor->matrix(); + + const float offset = output_tensor_name == kOutputPlusOneTensor ? 1 : 0; + for (int i = 0; i < batch_size; ++i) { + const Feature feature = GetFeature(examples[i], kOutputFeature); + if (feature.float_list().value_size() != 1) { + return errors::Internal("incorrect number of values in output feature"); + } + output_matrix(i, 0) = feature.float_list().value(0) + offset; + } + return Status::OK(); + } +}; + +// Add a named signature to the mutable signatures* parameter. +// If is_regression is false, will add a classification signature, which is +// invalid in regression requests. +void AddNamedSignature(const string& input_tensor_name, + const string& output_tensor_name, + const string& signature_name, const bool is_regression, + Signatures* signatures) { + tensorflow::serving::Signature named_signature; + if (is_regression) { + named_signature.mutable_regression_signature() + ->mutable_input() + ->set_tensor_name(input_tensor_name); + named_signature.mutable_regression_signature() + ->mutable_output() + ->set_tensor_name(output_tensor_name); + } else { + named_signature.mutable_classification_signature() + ->mutable_input() + ->set_tensor_name(input_tensor_name); + named_signature.mutable_classification_signature() + ->mutable_classes() + ->set_tensor_name(output_tensor_name); + } + signatures->mutable_named_signatures()->insert( + protobuf::MapPair( + signature_name, named_signature)); +} + +// Parameter is 'bool use_saved_model'. +class RegressorTest : public ::testing::TestWithParam { + public: + void SetUp() override { + bundle_.reset(new SessionBundle); + meta_graph_def_ = &bundle_->meta_graph_def; + fake_session_ = new FakeSession(); + bundle_->session.reset(fake_session_); + + // Setup some defaults for our signature. + tensorflow::serving::Signatures signatures; + auto default_signature = + signatures.mutable_default_signature()->mutable_regression_signature(); + default_signature->mutable_input()->set_tensor_name(kInputTensor); + default_signature->mutable_output()->set_tensor_name(kOutputTensor); + + AddNamedSignature(kInputTensor, kOutputPlusOneTensor, + kOutputPlusOneSignature, true /* is_regression */, + &signatures); + AddNamedSignature(kInputTensor, kOutputPlusOneTensor, + kInvalidNamedSignature, false /* is_regression */, + &signatures); + TF_ASSERT_OK( + tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); + } + + protected: + // Return an example with the feature "output" = [output]. + Example example_with_output(const float output) { + Feature feature; + feature.mutable_float_list()->add_value(output); + Example example; + (*example.mutable_features()->mutable_feature())["output"] = feature; + return example; + } + + Status Create() { + if (GetParam()) { + std::unique_ptr saved_model(new SavedModelBundle); + internal::ConvertSessionBundleToSavedModelBundle(*bundle_, + saved_model.get()); + return CreateRegressorFromSavedModelBundle(std::move(saved_model), + ®ressor_); + } else { + return CreateRegressorFromBundle(std::move(bundle_), ®ressor_); + } + } + + // Variables used to create the regression model + tensorflow::MetaGraphDef* meta_graph_def_; + FakeSession* fake_session_; + std::unique_ptr bundle_; + + // Regression model valid after calling create. + std::unique_ptr regressor_; + + // Convenience variables. + RegressionRequest request_; + RegressionResult result_; +}; + +TEST_P(RegressorTest, BasicExampleList) { + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example_with_output(2.0); + *examples->Add() = example_with_output(3.0); + TF_ASSERT_OK(regressor_->Regress(request_, &result_)); + EXPECT_THAT(result_, EqualsProto(" regressions { " + " value: 2.0 " + " } " + " regressions { " + " value: 3.0 " + " } ")); +} + +TEST_P(RegressorTest, BasicExampleListWithContext) { + TF_ASSERT_OK(Create()); + auto* list_with_context = + request_.mutable_input()->mutable_example_list_with_context(); + // Add two empty examples. + list_with_context->add_examples(); + list_with_context->add_examples(); + // Add the context which contains the output predictions. + *list_with_context->mutable_context() = example_with_output(3.0); + TF_ASSERT_OK(regressor_->Regress(request_, &result_)); + EXPECT_THAT(result_, EqualsProto(" regressions { " + " value: 3.0 " + " } " + " regressions { " + " value: 3.0 " + " } ")); +} + +TEST_P(RegressorTest, ValidNamedSignature) { + TF_ASSERT_OK(Create()); + request_.mutable_model_spec()->set_signature_name(kOutputPlusOneSignature); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example_with_output(2.0); + *examples->Add() = example_with_output(3.0); + TF_ASSERT_OK(regressor_->Regress(request_, &result_)); + // GetParam() is 'is_saved_model' in this test. If using saved_model, this + // test should use the kOutputPlusOneSignature named signature. Otherwise, + // when using session_bundle, the signature_name in the model_spec will be + // ignored and the default signature will be used. + if (GetParam()) { + EXPECT_THAT(result_, EqualsProto(" regressions { " + " value: 3.0 " + " } " + " regressions { " + " value: 4.0 " + " } ")); + } else { + EXPECT_THAT(result_, EqualsProto(" regressions { " + " value: 2.0 " + " } " + " regressions { " + " value: 3.0 " + " } ")); + } +} + +TEST_P(RegressorTest, InvalidNamedSignature) { + TF_ASSERT_OK(Create()); + request_.mutable_model_spec()->set_signature_name(kInvalidNamedSignature); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example_with_output(2.0); + *examples->Add() = example_with_output(3.0); + const Status status = regressor_->Regress(request_, &result_); + + // GetParam() is 'is_saved_model' in this test. If using saved_model, this + // test should fail because the named_signature requested is actually a + // classification signature. When using session_bundle, the signature_name + // will be ignored and the default signature will be used. + if (GetParam()) { + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } else { + TF_ASSERT_OK(status); + EXPECT_THAT(result_, EqualsProto(" regressions { " + " value: 2.0 " + " } " + " regressions { " + " value: 3.0 " + " } ")); + } +} + +TEST_P(RegressorTest, EmptyInput) { + TF_ASSERT_OK(Create()); + // Touch input. + request_.mutable_input(); + const Status status = regressor_->Regress(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("RegressionRequest::input is empty")); +} + +TEST_P(RegressorTest, EmptyExampleList) { + TF_ASSERT_OK(Create()); + request_.mutable_input()->mutable_example_list(); + const Status status = regressor_->Regress(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("RegressionRequest::input is empty")); +} + +TEST_P(RegressorTest, EmptyExampleListWithContext) { + TF_ASSERT_OK(Create()); + // Add a ExampleListWithContext which has context but no examples. + *request_.mutable_input() + ->mutable_example_list_with_context() + ->mutable_context() = example_with_output(3); + const Status status = regressor_->Regress(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("RegressionRequest::input is empty")); +} + +TEST_P(RegressorTest, RunsFails) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly( + ::testing::Return(errors::Internal("Run totally failed"))); + TF_ASSERT_OK(Create()); + *request_.mutable_input()->mutable_example_list()->mutable_examples()->Add() = + example_with_output(2.0); + const Status status = regressor_->Regress(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Run totally failed")); +} + +TEST_P(RegressorTest, UnexpectedOutputTensorSize) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + std::vector outputs = {Tensor(DT_FLOAT, TensorShape({2}))}; + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + TF_ASSERT_OK(Create()); + *request_.mutable_input()->mutable_example_list()->mutable_examples()->Add() = + example_with_output(2.0); + const Status status = regressor_->Regress(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("output batch size")); +} + +TEST_P(RegressorTest, UnexpectedOutputTensorType) { + MockSession* mock = new MockSession; + bundle_->session.reset(mock); + // We expect a FLOAT output type; test returning a STRING. + std::vector outputs = {Tensor(DT_STRING, TensorShape({1}))}; + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + TF_ASSERT_OK(Create()); + *request_.mutable_input()->mutable_example_list()->mutable_examples()->Add() = + example_with_output(2.0); + const Status status = regressor_->Regress(request_, &result_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Expected output Tensor of DT_FLOAT")); +} + +TEST_P(RegressorTest, MissingRegressionSignature) { + tensorflow::serving::Signatures signatures; + signatures.mutable_default_signature(); + TF_ASSERT_OK(tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); + TF_ASSERT_OK(Create()); + Feature feature; + feature.mutable_bytes_list()->add_value("uno"); + Example example; + (*example.mutable_features()->mutable_feature())["class"] = feature; + *request_.mutable_input()->mutable_example_list()->mutable_examples()->Add() = + example; + // TODO(b/26220896): This error should move to construction time. + const Status status = regressor_->Regress(request_, &result_); + ASSERT_FALSE(status.ok()); + // Old SessionBundle code treats a missing signature as a FAILED_PRECONDITION + // but new SavedModel code treats it as an INVALID_ARGUMENT (signature + // specified in the request was invalid). + if (GetParam()) { + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } else { + EXPECT_EQ(::tensorflow::error::FAILED_PRECONDITION, status.code()) + << status; + } +} + +// Test all RegressorTest test cases with both SessionBundle and SavedModel. +INSTANTIATE_TEST_CASE_P(UseSavedModel, RegressorTest, ::testing::Bool()); + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/util.cc b/tensorflow_serving/servables/tensorflow/util.cc new file mode 100644 index 00000000000..b455ea0d7fa --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/util.cc @@ -0,0 +1,90 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/util.h" + +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/apis/input.pb.h" + +namespace tensorflow { +namespace serving { + +int NumInputExamples(const Input& input) { + switch (input.kind_case()) { + case Input::KindCase::kExampleList: + return input.example_list().examples_size(); + case Input::KindCase::kExampleListWithContext: + return input.example_list_with_context().examples_size(); + default: + break; + } + return 0; +} + +Status InputToSerializedExampleTensor(const Input& input, Tensor* examples) { + const int n = NumInputExamples(input); + *examples = Tensor(DT_STRING, TensorShape({n})); + switch (input.kind_case()) { + case Input::KindCase::KIND_NOT_SET: + break; + + case Input::KindCase::kExampleList: { + auto input_vec = examples->vec(); + int input_vec_index = 0; + for (const auto& entry : input.example_list().examples()) { + input_vec(input_vec_index++) = entry.SerializeAsString(); + } + break; + } + + case Input::KindCase::kExampleListWithContext: { + const string context = + input.example_list_with_context().context().SerializeAsString(); + auto input_vec = examples->vec(); + int input_vec_index = 0; + for (const auto& entry : input.example_list_with_context().examples()) { + // Avoid the need for repeated serialization of context by simply + // appending the Example serialization to the pre-serialized context. + input_vec(input_vec_index++) = + strings::StrCat(context, entry.SerializeAsString()); + } + } break; + + default: + return errors::Unimplemented("Input with kind ", input.kind_case(), + " not supported."); + } + return Status::OK(); +} + +Status PerformOneShotTensorComputation( + const Input& input, const string& input_tensor_name, + const std::vector& output_tensor_names, Session* session, + std::vector* outputs) { + // Setup the input Tensor to be a vector of string containing the serialized + // tensorflow.Example. + Tensor input_tensor; + TF_RETURN_IF_ERROR(InputToSerializedExampleTensor(input, &input_tensor)); + + return session->Run({{input_tensor_name, input_tensor}}, output_tensor_names, + {}, outputs); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/util.h b/tensorflow_serving/servables/tensorflow/util.h new file mode 100644 index 00000000000..9cfc0192da2 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/util.h @@ -0,0 +1,55 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_UTIL_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_UTIL_H_ + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/public/session.h" +#include "tensorflow_serving/apis/input.pb.h" + +namespace tensorflow { +namespace serving { + +// Returns the number of examples in the Input. +int NumInputExamples(const Input& input); + +// InputToSerializedExampleTensor populates a string Tensor of serialized +// Examples. +// If input has n Examples returns a string Tensor with shape {n}. +// +// Cases: +// - Input kind unset: Tensor of shape {0} +// - Input::example_list: Serializes each example. +// - Input::example_list_with_context: Serializes each example merged with the +// context. +// - Other: non-OK Status. +// +// Note: does not perform any structural validation (e.g., if an example list is +// empty it will return a Tensor of shape {0}). +Status InputToSerializedExampleTensor(const Input& input, Tensor* examples); + +// Issues a single Session::Run() call with 'input' to produce 'outputs'. +// Equivalent to InputToSerializedExampleTensor() followed by Session::Run(). +Status PerformOneShotTensorComputation( + const Input& input, const string& input_tensor_name, + const std::vector& output_tensor_names, Session* session, + std::vector* outputs); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_UTIL_H_ diff --git a/tensorflow_serving/servables/tensorflow/util_test.cc b/tensorflow_serving/servables/tensorflow/util_test.cc new file mode 100644 index 00000000000..0bd44117990 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/util_test.cc @@ -0,0 +1,181 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/util.h" + +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/example/feature.pb.h" +#include "tensorflow/core/framework/tensor_shape.h" +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using test_util::EqualsProto; + +class InputUtilTest : public ::testing::Test { + protected: + // A few known Examples to use. + Example example_A() { + Feature feature; + feature.mutable_int64_list()->add_value(11); + Example example; + (*example.mutable_features()->mutable_feature())["a"] = feature; + return example; + } + + Example example_B() { + Feature feature; + feature.mutable_int64_list()->add_value(22); + Example example; + (*example.mutable_features()->mutable_feature())["b"] = feature; + return example; + } + + Example example_C() { + Feature feature; + feature.mutable_int64_list()->add_value(33); + Example example; + (*example.mutable_features()->mutable_feature())["c"] = feature; + return example; + } + + Input input_; + Tensor tensor_; +}; + +TEST_F(InputUtilTest, Empty_KindNotSet) { + EXPECT_EQ(0, NumInputExamples(input_)); + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), + tensor_); +} + +TEST_F(InputUtilTest, Empty_ExampleList) { + input_.mutable_example_list(); + + EXPECT_EQ(0, NumInputExamples(input_)); + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), + tensor_); +} + +TEST_F(InputUtilTest, Empty_ExampleListWithContext) { + input_.mutable_example_list_with_context(); + + EXPECT_EQ(0, NumInputExamples(input_)); + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), + tensor_); +} + +TEST_F(InputUtilTest, ExampleList) { + *input_.mutable_example_list()->mutable_examples()->Add() = example_A(); + *input_.mutable_example_list()->mutable_examples()->Add() = example_B(); + + EXPECT_EQ(2, NumInputExamples(input_)); + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + const auto vec = tensor_.flat(); + ASSERT_EQ(vec.size(), 2); + Example serialized_example; + ASSERT_TRUE(serialized_example.ParseFromString(vec(0))); + EXPECT_THAT(serialized_example, EqualsProto(example_A())); + ASSERT_TRUE(serialized_example.ParseFromString(vec(1))); + EXPECT_THAT(serialized_example, EqualsProto(example_B())); +} + +TEST_F(InputUtilTest, ExampleListWithContext) { + auto* examples = + input_.mutable_example_list_with_context()->mutable_examples(); + *examples->Add() = example_A(); + *examples->Add() = example_B(); + *input_.mutable_example_list_with_context()->mutable_context() = example_C(); + + EXPECT_EQ(2, NumInputExamples(input_)); + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + const auto vec = tensor_.flat(); + ASSERT_EQ(vec.size(), 2); + { + Example serialized_example; + ASSERT_TRUE(serialized_example.ParseFromString(vec(0))); + EXPECT_THAT(serialized_example.features().feature().at("c"), EqualsProto( + example_C().features().feature().at("c"))); + EXPECT_THAT(serialized_example.features().feature().at("a"), EqualsProto( + example_A().features().feature().at("a"))); + } + { + Example serialized_example; + ASSERT_TRUE(serialized_example.ParseFromString(vec(1))); + EXPECT_THAT(serialized_example.features().feature().at("c"), EqualsProto( + example_C().features().feature().at("c"))); + EXPECT_THAT(serialized_example.features().feature().at("b"), EqualsProto( + example_B().features().feature().at("b"))); + } +} + +TEST_F(InputUtilTest, ExampleListWithContext_NoContext) { + auto* examples = + input_.mutable_example_list_with_context()->mutable_examples(); + *examples->Add() = example_A(); + *examples->Add() = example_B(); + + EXPECT_EQ(2, NumInputExamples(input_)); + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + const auto vec = tensor_.flat(); + ASSERT_EQ(vec.size(), 2); + { + Example serialized_example; + ASSERT_TRUE(serialized_example.ParseFromString(vec(0))); + EXPECT_THAT(serialized_example, EqualsProto(example_A())); + } + { + Example serialized_example; + ASSERT_TRUE(serialized_example.ParseFromString(vec(1))); + EXPECT_THAT(serialized_example, EqualsProto(example_B())); + } +} + +TEST_F(InputUtilTest, ExampleListWithContext_OnlyContext) { + // Ensure that if there are no examples there is no output (even if the + // context is specified). + *input_.mutable_example_list_with_context()->mutable_context() = example_C(); + + EXPECT_EQ(0, NumInputExamples(input_)); + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), + tensor_); +} + +TEST_F(InputUtilTest, RequestNumExamplesStreamz) { + Input input_1; + *input_1.mutable_example_list()->mutable_examples()->Add() = example_A(); + *input_1.mutable_example_list()->mutable_examples()->Add() = example_B(); + EXPECT_EQ(2, NumInputExamples(input_1)); + Tensor tensor_1; + TF_ASSERT_OK(InputToSerializedExampleTensor(input_1, &tensor_1)); + + Input input_2; + *input_2.mutable_example_list()->mutable_examples()->Add() = example_C(); + EXPECT_EQ(1, NumInputExamples(input_2)); + Tensor tensor_2; + TF_ASSERT_OK(InputToSerializedExampleTensor(input_2, &tensor_2)); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From 61caec94059a34af2aab87861c3107683595cfbe Mon Sep 17 00:00:00 2001 From: Li Lao Date: Wed, 22 Feb 2017 11:19:53 -0800 Subject: [PATCH 0173/8554] Remove the support for using SessionBundle in TensorflowRegressionServiceImpl and TensorflowClassificationServiceImpl. Change: 148244577 --- .../tensorflow/classification_service.cc | 46 +++++------------- .../tensorflow/classification_service.h | 3 +- .../tensorflow/regression_service.cc | 47 +++++-------------- .../servables/tensorflow/regression_service.h | 3 +- 4 files changed, 28 insertions(+), 71 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/classification_service.cc b/tensorflow_serving/servables/tensorflow/classification_service.cc index 1b021fa7b33..476848b1059 100644 --- a/tensorflow_serving/servables/tensorflow/classification_service.cc +++ b/tensorflow_serving/servables/tensorflow/classification_service.cc @@ -29,8 +29,8 @@ namespace tensorflow { namespace serving { Status TensorflowClassificationServiceImpl::Classify( - ServerCore* core, const bool use_saved_model, - const ClassificationRequest& request, ClassificationResponse* response) { + ServerCore* core, const ClassificationRequest& request, + ClassificationResponse* response) { TRACELITERAL("TensorflowClassificationServiceImpl::Classify"); // Verify Request Metadata and create a ServableRequest if (!request.has_model_spec()) { @@ -38,38 +38,18 @@ Status TensorflowClassificationServiceImpl::Classify( "Missing ModelSpec"); } - std::unique_ptr classifier_interface; - if (use_saved_model) { - ServableHandle saved_model_bundle; - TF_RETURN_IF_ERROR( - core->GetServableHandle(request.model_spec(), &saved_model_bundle)); - SignatureDef signature; - TF_RETURN_IF_ERROR(GetClassificationSignatureDef( - request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); - TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( - saved_model_bundle->session.get(), &signature, &classifier_interface)); - // Run classification. - TF_RETURN_IF_ERROR( - classifier_interface->Classify(request, response->mutable_result())); - } else { - ServableHandle bundle; - TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); - Signature signature; - TF_RETURN_IF_ERROR(GetDefaultSignature(bundle->meta_graph_def, &signature)); - - if (!signature.has_classification_signature()) { - return tensorflow::Status(tensorflow::error::UNAVAILABLE, - "No Classification Signature"); - } - TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( - bundle->session.get(), &signature.classification_signature(), - &classifier_interface)); - // Run classification. - TF_RETURN_IF_ERROR( - classifier_interface->Classify(request, response->mutable_result())); - } + ServableHandle saved_model_bundle; + TF_RETURN_IF_ERROR( + core->GetServableHandle(request.model_spec(), &saved_model_bundle)); + SignatureDef signature; + TF_RETURN_IF_ERROR(GetClassificationSignatureDef( + request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); - return Status::OK(); + std::unique_ptr classifier_interface; + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( + saved_model_bundle->session.get(), &signature, &classifier_interface)); + // Run classification. + return classifier_interface->Classify(request, response->mutable_result()); } } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/classification_service.h b/tensorflow_serving/servables/tensorflow/classification_service.h index 731425425d9..b06eae85cc0 100644 --- a/tensorflow_serving/servables/tensorflow/classification_service.h +++ b/tensorflow_serving/servables/tensorflow/classification_service.h @@ -27,8 +27,7 @@ namespace serving { // tensorflow_serving/apis/classification-service.proto. class TensorflowClassificationServiceImpl { public: - static Status Classify(ServerCore* core, const bool use_saved_model, - const ClassificationRequest& request, + static Status Classify(ServerCore* core, const ClassificationRequest& request, ClassificationResponse* response); }; diff --git a/tensorflow_serving/servables/tensorflow/regression_service.cc b/tensorflow_serving/servables/tensorflow/regression_service.cc index 8e5f53bf1b3..f1d35696328 100644 --- a/tensorflow_serving/servables/tensorflow/regression_service.cc +++ b/tensorflow_serving/servables/tensorflow/regression_service.cc @@ -27,8 +27,8 @@ namespace tensorflow { namespace serving { Status TensorflowRegressionServiceImpl::Regress( - ServerCore* core, const bool use_saved_model, - const RegressionRequest& request, RegressionResponse* response) { + ServerCore* core, const RegressionRequest& request, + RegressionResponse* response) { TRACELITERAL("TensorflowRegressionServiceImpl::Regress"); // Verify Request Metadata and create a ServableRequest if (!request.has_model_spec()) { @@ -36,39 +36,18 @@ Status TensorflowRegressionServiceImpl::Regress( "Missing ModelSpec"); } - std::unique_ptr regressor_interface; - if (use_saved_model) { - ServableHandle saved_model_bundle; - TF_RETURN_IF_ERROR( - core->GetServableHandle(request.model_spec(), &saved_model_bundle)); - SignatureDef signature; - TF_RETURN_IF_ERROR(GetRegressionSignatureDef( - request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); - - TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( - saved_model_bundle->session.get(), &signature, ®ressor_interface)); - // Run regression - TF_RETURN_IF_ERROR( - regressor_interface->Regress(request, response->mutable_result())); - } else { - ServableHandle bundle; - TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); - Signature signature; - TF_RETURN_IF_ERROR(GetDefaultSignature(bundle->meta_graph_def, &signature)); + ServableHandle saved_model_bundle; + TF_RETURN_IF_ERROR( + core->GetServableHandle(request.model_spec(), &saved_model_bundle)); + SignatureDef signature; + TF_RETURN_IF_ERROR(GetRegressionSignatureDef( + request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); - if (!signature.has_regression_signature()) { - return tensorflow::Status(tensorflow::error::UNAVAILABLE, - "No Regression Signature"); - } - // Create a regressor interface for the session bundle - TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( - bundle->session.get(), &signature.regression_signature(), - ®ressor_interface)); - // Run regression - TF_RETURN_IF_ERROR( - regressor_interface->Regress(request, response->mutable_result())); - } - return Status::OK(); + std::unique_ptr regressor_interface; + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( + saved_model_bundle->session.get(), &signature, ®ressor_interface)); + // Run regression + return regressor_interface->Regress(request, response->mutable_result()); } } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/regression_service.h b/tensorflow_serving/servables/tensorflow/regression_service.h index 17b6063f8c9..c4980db6dd0 100644 --- a/tensorflow_serving/servables/tensorflow/regression_service.h +++ b/tensorflow_serving/servables/tensorflow/regression_service.h @@ -27,8 +27,7 @@ namespace serving { // tensorflow_serving/apis/regression-service.proto. class TensorflowRegressionServiceImpl final { public: - static Status Regress(ServerCore* core, const bool use_saved_model, - const RegressionRequest& request, + static Status Regress(ServerCore* core, const RegressionRequest& request, RegressionResponse* response); }; From 1be938eb83ea165f41ab1e91c70cde9a28f96100 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 23 Feb 2017 16:17:02 -0800 Subject: [PATCH 0174/8554] Merge changes from github. Make some modifications to make code compatible with Google style. Change: 148405560 --- tensorflow_serving/example/BUILD | 13 ++ .../example/inception_client.cc | 138 ++++++++++++++++++ tensorflow_serving/model_servers/BUILD | 6 + tensorflow_serving/model_servers/main.cc | 73 ++++++--- .../tensorflow_model_server_test.py | 114 ++++++++++++++- .../servables/tensorflow/testdata/BUILD | 6 + .../tensorflow/testdata/bad_model_config.txt | 1 + .../tensorflow/testdata/good_model_config.txt | 12 ++ .../00000123/assets/foo.txt | 1 + .../00000123/saved_model.pb | Bin 0 -> 8658 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 12 bytes .../00000123/variables/variables.index | Bin 0 -> 151 bytes 12 files changed, 339 insertions(+), 25 deletions(-) create mode 100644 tensorflow_serving/example/inception_client.cc create mode 100644 tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/saved_model.pb create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.index diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index ff7172d2795..85bb83412df 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -115,3 +115,16 @@ py_binary( "@org_tensorflow//tensorflow:tensorflow_py", ], ) + +cc_binary( + name = "inception_client_cc", + srcs = [ + "inception_client.cc", + ], + deps = [ + "//tensorflow_serving/apis:prediction_service_proto", + "@grpc//:grpc++", + "@org_tensorflow//tensorflow/core:framework", + "@protobuf//:protobuf_lite", + ], +) diff --git a/tensorflow_serving/example/inception_client.cc b/tensorflow_serving/example/inception_client.cc new file mode 100644 index 00000000000..4b1d183ae87 --- /dev/null +++ b/tensorflow_serving/example/inception_client.cc @@ -0,0 +1,138 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include +#include + +#include "grpc++/create_channel.h" +#include "grpc++/security/credentials.h" +#include "google/protobuf/map.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/types.h" +#include "tensorflow/core/util/command_line_flags.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" + +using grpc::Channel; +using grpc::ClientContext; +using grpc::Status; + +using tensorflow::serving::PredictRequest; +using tensorflow::serving::PredictResponse; +using tensorflow::serving::PredictionService; + +typedef google::protobuf::Map OutMap; + +class ServingClient { + public: + ServingClient(std::shared_ptr channel) + : stub_(PredictionService::NewStub(channel)) {} + + tensorflow::string callPredict(const tensorflow::string& model_name, + const tensorflow::string& file_path) { + PredictRequest predictRequest; + PredictResponse response; + ClientContext context; + + predictRequest.mutable_model_spec()->set_name(model_name); + + google::protobuf::Map& inputs = + *predictRequest.mutable_inputs(); + + tensorflow::TensorProto proto; + + std::ifstream imageFile(file_path, std::ios::binary); + + if (!imageFile.is_open()) { + std::cout << "Failed to open " << file_path << std::endl; + return ""; + } + + std::filebuf* pbuf = imageFile.rdbuf(); + auto fileSize = pbuf->pubseekoff(0, std::ios::end, std::ios::in); + + char* image = new char[fileSize](); + + pbuf->pubseekpos(0, std::ios::in); + pbuf->sgetn(image, fileSize); + imageFile.close(); + + proto.set_dtype(tensorflow::DataType::DT_STRING); + proto.add_string_val(image, fileSize); + + proto.mutable_tensor_shape()->add_dim()->set_size(1); + + inputs["images"] = proto; + + Status status = stub_->Predict(&context, predictRequest, &response); + + delete[] image; + + if (status.ok()) { + std::cout << "call predict ok" << std::endl; + std::cout << "outputs size is " << response.outputs_size() << std::endl; + OutMap& map_outputs = *response.mutable_outputs(); + OutMap::iterator iter; + int output_index = 0; + + for (iter = map_outputs.begin(); iter != map_outputs.end(); ++iter) { + tensorflow::TensorProto& result_tensor_proto = iter->second; + tensorflow::Tensor tensor; + bool converted = tensor.FromProto(result_tensor_proto); + if (converted) { + std::cout << "the result tensor[" << output_index + << "] is:" << std::endl + << tensor.SummarizeValue(10) << std::endl; + } else { + std::cout << "the result tensor[" << output_index + << "] convert failed." << std::endl; + } + ++output_index; + } + return "Done."; + } else { + std::cout << "gRPC call return code: " << status.error_code() << ": " + << status.error_message() << std::endl; + return "gRPC failed."; + } + } + + private: + std::unique_ptr stub_; +}; + +int main(int argc, char** argv) { + tensorflow::string server_port = "localhost:9000"; + tensorflow::string image_file = ""; + tensorflow::string model_name = "inception"; + std::vector flag_list = { + tensorflow::Flag("server_port", &server_port, + "the IP and port of the server"), + tensorflow::Flag("image_file", &image_file, "the path to the "), + tensorflow::Flag("model_name", &model_name, "name of model")}; + tensorflow::string usage = tensorflow::Flags::Usage(argv[0], flag_list); + const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); + if (!parse_result || image_file.empty()) { + std::cout << usage; + return -1; + } + + ServingClient guide( + grpc::CreateChannel(server_port, grpc::InsecureChannelCredentials())); + std::cout << "calling predict using file: " << image_file << " ..." + << std::endl; + std::cout << guide.callPredict(model_name, image_file) << std::endl; + + return 0; +} diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index a5c16c2baf7..90c841734ce 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -144,9 +144,15 @@ py_test( ":tensorflow_model_server", "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export", "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:bad_model_config.txt", + "//tensorflow_serving/servables/tensorflow/testdata:good_model_config.txt", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/assets/foo.txt", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/saved_model.pb", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/variables/variables.index", "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], tags = [ diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index f1e6e8729a0..1b0d2c6ac47 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -114,6 +114,23 @@ using tensorflow::serving::PredictionService; namespace { +tensorflow::Status ParseProtoTextFile(const string& file, + google::protobuf::Message* message) { + std::unique_ptr file_data; + TF_RETURN_IF_ERROR( + tensorflow::Env::Default()->NewReadOnlyMemoryRegionFromFile(file, + &file_data)); + string file_data_str(static_cast(file_data->data()), + file_data->length()); + if (tensorflow::protobuf::TextFormat::ParseFromString(file_data_str, + message)) { + return tensorflow::Status::OK(); + } else { + return tensorflow::errors::InvalidArgument("Invalid protobuf file: '", file, + "'"); + } +} + tensorflow::Status LoadCustomModelConfig( const ::google::protobuf::Any& any, EventBus* servable_event_bus, @@ -141,6 +158,13 @@ ModelServerConfig BuildSingleModelConfig( return config; } +ModelServerConfig BuildModelConfigFromFile(const string& file) { + LOG(INFO) << "Building from config file: " << file; + + ModelServerConfig model_config; + TF_CHECK_OK(ParseProtoTextFile(file, &model_config)); + return model_config; +} int DeadlineToTimeoutMillis(const gpr_timespec deadline) { return gpr_time_to_millis( gpr_time_sub(gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), @@ -237,15 +261,8 @@ void RunServer(int port, std::unique_ptr core, // Parses an ascii PlatformConfigMap protobuf from 'file'. tensorflow::serving::PlatformConfigMap ParsePlatformConfigMap( const string& file) { - std::unique_ptr file_data; - TF_CHECK_OK( // Crash ok - tensorflow::Env::Default()->NewReadOnlyMemoryRegionFromFile(file, - &file_data)); - string file_data_str(static_cast(file_data->data()), - file_data->length()); tensorflow::serving::PlatformConfigMap platform_config_map; - QCHECK(tensorflow::protobuf::TextFormat::ParseFromString( // Crash ok - file_data_str, &platform_config_map)); + TF_CHECK_OK(ParseProtoTextFile(file, &platform_config_map)); return platform_config_map; } @@ -262,26 +279,38 @@ int main(int argc, char** argv) { // thread pools will be auto configured. tensorflow::int64 tensorflow_session_parallelism = 0; string platform_config_file = ""; + string model_config_file; tensorflow::string model_version_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Name( FileSystemStoragePathSourceConfig::LATEST_VERSION); std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), - tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("model_config_file", &model_config_file, + "If non-empty, read an ascii ModelServerConfig " + "protobuf from the supplied file name, and serve the " + "models in that file. (If used, --model_name, " + "--model_base_path and --model_version_policy " + "are ignored.)"), + tensorflow::Flag("model_name", &model_name, + "name of model (ignored " + "if --model_config_file flag is set"), + tensorflow::Flag("model_base_path", &model_base_path, + "path to export (ignored if --model_config_file flag " + "is set, otherwise required)"), tensorflow::Flag( "model_version_policy", &model_version_policy, - "The version policy which determines the number of model versions to " - "be served at the same time. The default value is LATEST_VERSION, " - "which will serve only the latest version. See " - "file_system_storage_path_source.proto for the list of possible " - "VersionPolicy."), + "The version policy which determines the number of model " + "versions to be served at the same time. The default " + "value is LATEST_VERSION, which will serve only the " + "latest version. " + "See file_system_storage_path_source.proto for " + "the list of possible VersionPolicy. (Ignored if " + "--model_config_file flag is set)"), tensorflow::Flag("file_system_poll_wait_seconds", &file_system_poll_wait_seconds, "interval in seconds between each poll of the file " "system for new model version"), - tensorflow::Flag("model_base_path", &model_base_path, - "path to export (required)"), tensorflow::Flag("use_saved_model", &use_saved_model, "If true, use SavedModel in the server; otherwise, use " "SessionBundle. It is used by tensorflow serving team " @@ -301,7 +330,7 @@ int main(int argc, char** argv) { "ignored.)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); - if (!parse_result || model_base_path.empty()) { + if (!parse_result || (model_base_path.empty() && model_config_file.empty())) { std::cout << usage; return -1; } @@ -321,8 +350,14 @@ int main(int argc, char** argv) { // For ServerCore Options, we leave servable_state_monitor_creator unspecified // so the default servable_state_monitor_creator will be used. ServerCore::Options options; - options.model_server_config = BuildSingleModelConfig( - model_name, model_base_path, parsed_version_policy); + + // model server config + if (model_config_file.empty()) { + options.model_server_config = BuildSingleModelConfig( + model_name, model_base_path, parsed_version_policy); + } else { + options.model_server_config = BuildModelConfigFromFile(model_config_file); + } if (platform_config_file.empty()) { SessionBundleConfig session_bundle_config; diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 5b4dbe6952e..eee38ba0b52 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -51,15 +51,36 @@ def PickUnusedPort(): class TensorflowModelServerTest(tf.test.TestCase): """This class defines integration test cases for tensorflow_model_server.""" - def __TestSrcDirPath(self, relative_path): + def __TestSrcDirPath(self, relative_path=''): return os.path.join(os.environ['TEST_SRCDIR'], 'tf_serving/tensorflow_serving', relative_path) + def __BuildModelConfigFile(self): + """Write a config file to disk for use in tests. + + Substitutes placeholder for test directory with test directory path + in the configuration template file and writes it out to another file + used by the test. + """ + with open(self._GetGoodModelConfigTemplate(), 'r') as template_file: + config = template_file.read().replace('${TEST_HALF_PLUS_TWO_DIR}', + self._GetSavedModelBundlePath()) + config = config.replace('${TEST_HALF_PLUS_THREE_DIR}', + self._GetSavedModelHalfPlusThreePath()) + with open(self._GetGoodModelConfigFile(), 'w') as config_file: + config_file.write(config) + def setUp(self): """Sets up integration test parameters.""" self.binary_dir = self.__TestSrcDirPath('model_servers') self.testdata_dir = self.__TestSrcDirPath('servables/tensorflow/testdata') + self.temp_dir = tf.test.get_temp_dir() self.server_proc = None + self.__BuildModelConfigFile() + + def tearDown(self): + """Deletes created configuration file.""" + os.remove(self._GetGoodModelConfigFile()) def TerminateProcs(self): """Terminate all processes.""" @@ -82,14 +103,33 @@ def RunServer(self, port, model_name, model_path, use_saved_model, print 'Server started' return 'localhost:' + str(port) + def RunServerWithModelConfigFile(self, + port, + model_config_file, + use_saved_model, + pipe=None): + """Run tensorflow_model_server using test config.""" + print 'Starting test server...' + command = os.path.join(self.binary_dir, 'tensorflow_model_server') + command += ' --port=' + str(port) + command += ' --model_config_file=' + model_config_file + command += ' --use_saved_model=' + str(use_saved_model).lower() + + print command + self.server_proc = subprocess.Popen(shlex.split(command), stderr=pipe) + print 'Server started' + return 'localhost:' + str(port) + def VerifyPredictRequest(self, model_server_address, + expected_output, + model_name='default', specify_output=True): """Send PredictionService.Predict request and verify output.""" print 'Sending Predict request...' # Prepare request request = predict_pb2.PredictRequest() - request.model_spec.name = 'default' + request.model_spec.name = model_name request.inputs['x'].dtype = types_pb2.DT_FLOAT request.inputs['x'].float_val.append(2.0) dim = request.inputs['x'].tensor_shape.dim.add() @@ -106,17 +146,33 @@ def VerifyPredictRequest(self, self.assertTrue('y' in result.outputs) self.assertIs(types_pb2.DT_FLOAT, result.outputs['y'].dtype) self.assertEquals(1, len(result.outputs['y'].float_val)) - self.assertEquals(3.0, result.outputs['y'].float_val[0]) + self.assertEquals(expected_output, result.outputs['y'].float_val[0]) def _GetSavedModelBundlePath(self): """Returns a path to a model in SavedModel format.""" return os.path.join(os.environ['TEST_SRCDIR'], 'tf_serving/external/org_tensorflow/tensorflow/', 'cc/saved_model/testdata/half_plus_two') + def _GetSavedModelHalfPlusThreePath(self): + """Returns a path to a half_plus_three model in SavedModel format.""" + return os.path.join(self.testdata_dir, 'saved_model_half_plus_three') + def _GetSessionBundlePath(self): """Returns a path to a model in SessionBundle format.""" return os.path.join(self.testdata_dir, 'half_plus_two') + def _GetGoodModelConfigTemplate(self): + """Returns a path to a working configuration file template.""" + return os.path.join(self.testdata_dir, 'good_model_config.txt') + + def _GetGoodModelConfigFile(self): + """Returns a path to a working configuration file.""" + return os.path.join(self.temp_dir, 'good_model_config.conf') + + def _GetBadModelConfigFile(self): + """Returns a path to a improperly formatted configuration file.""" + return os.path.join(self.testdata_dir, 'bad_model_config.txt') + def _TestPredict(self, model_path, use_saved_model, enable_batching): """Helper method to test prediction. @@ -130,8 +186,9 @@ def _TestPredict(self, model_path, use_saved_model, enable_batching): model_path, use_saved_model, enable_batching) time.sleep(5) - self.VerifyPredictRequest(model_server_address) - self.VerifyPredictRequest(model_server_address, specify_output=False) + self.VerifyPredictRequest(model_server_address, expected_output=3.0) + self.VerifyPredictRequest( + model_server_address, expected_output=3.0, specify_output=False) def testPredictSessionBundle(self): """Test PredictionService.Predict implementation with SessionBundle.""" @@ -167,7 +224,7 @@ def _TestBadModel(self, use_saved_model): enable_batching=False) time.sleep(5) with self.assertRaises(face.AbortionError) as error: - self.VerifyPredictRequest(model_server_address) + self.VerifyPredictRequest(model_server_address, expected_output=3.0) self.assertIs(beta_interfaces.StatusCode.FAILED_PRECONDITION, error.exception.code) @@ -179,6 +236,51 @@ def _TestBadModelSessionBundle(self): """Test Predict against a bad SessionBundle model export.""" self._TestBadModel(use_saved_model=False) + def testGoodModelConfig(self): + """Test server configuration from file works with valid configuration.""" + atexit.register(self.TerminateProcs) + model_server_address = self.RunServerWithModelConfigFile( + PickUnusedPort(), self._GetGoodModelConfigFile(), + True) # use_saved_model + time.sleep(5) + + self.VerifyPredictRequest( + model_server_address, model_name='half_plus_two', expected_output=3.0) + self.VerifyPredictRequest( + model_server_address, + model_name='half_plus_two', + expected_output=3.0, + specify_output=False) + + self.VerifyPredictRequest( + model_server_address, model_name='half_plus_three', expected_output=4.0) + self.VerifyPredictRequest( + model_server_address, + model_name='half_plus_three', + expected_output=4.0, + specify_output=False) + + def testBadModelConfig(self): + """Test server model configuration from file fails for invalid file.""" + atexit.register(self.TerminateProcs) + self.RunServerWithModelConfigFile( + PickUnusedPort(), + self._GetBadModelConfigFile(), + True, # use_saved_model + pipe=subprocess.PIPE) + last_line = None + for line in self.server_proc.stderr: + last_line = line + + error_message = ( + 'Check failed: ::tensorflow::Status::OK() == ' + '(ParseProtoTextFile(file, &model_config)) ' + '(OK vs. Invalid argument: ' + 'Invalid protobuf file: \'%s\')') % self._GetBadModelConfigFile() + + self.assertNotEqual(last_line, None) + self.assertGreater(last_line.find(error_message), 0) + # self.assertEquals(self.server_proc.poll(), 1) if __name__ == '__main__': tf.test.main() diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 64629175c6f..69bf0e8c882 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -52,6 +52,12 @@ exports_files([ "half_plus_two/00000123/export.data-00000-of-00001", "half_plus_two/00000123/export.index", "half_plus_two/00000123/export.meta", + "saved_model_half_plus_three/00000123/saved_model.pb", + "saved_model_half_plus_three/00000123/assets/foo.txt", + "saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001", + "saved_model_half_plus_three/00000123/variables/variables.index", "bad_half_plus_two/00000123/export", "bad_half_plus_two/00000123/export.meta", + "good_model_config.txt", + "bad_model_config.txt", ]) diff --git a/tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt b/tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt new file mode 100644 index 00000000000..6649d3fd44a --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/bad_model_config.txt @@ -0,0 +1 @@ +improperly formatted file \ No newline at end of file diff --git a/tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt b/tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt new file mode 100644 index 00000000000..df60a377afc --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/good_model_config.txt @@ -0,0 +1,12 @@ +model_config_list: { + config: { + name: "half_plus_two", + base_path: "${TEST_HALF_PLUS_TWO_DIR}", + model_platform: "tensorflow" + }, + config: { + name: "half_plus_three", + base_path: "${TEST_HALF_PLUS_THREE_DIR}", + model_platform: "tensorflow" + } +} diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt new file mode 100644 index 00000000000..f9ff0366880 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/assets/foo.txt @@ -0,0 +1 @@ +asset-file-contents \ No newline at end of file diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/saved_model.pb b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..71ac858241500dae98fba4ddb4c4e3abe4581ca2 GIT binary patch literal 8658 zcmcgx-*Ven8TYYc%l_;n%}LrirwzN90lMxqcHB0!-pFnhtpas%A#V{pqG@E}|xj(B$!mOIsTl3ywn3zag}-~>vK zeBT}nX%z|{-^cg=KnMjW9vjXP7i93pJqkugfj(YuRK_Hc`U<{kTSmZj|G*e=y3}`F zhvjdO##N{u`CNBg^R+!3Ocwr52;76>V|VBWY!yn1exqm!Asee9b6N`c(09GYGN=`$ z1Z+e3Ba06MJ2(}B+C!902wEKzLZv4XLLcZe?hW`SoyP~0HCVN{!%UuJ$Xy( zu%ez@eBU^70>4vwDIc(FuoBX;hn8(3{mPge+k)kAQK{Ieg|`BGpRw_>+)Rm2uRu+4 z48IKdH7>zeSY;P9MI>eT;Jc7uL&35A;D%uN-VM^#px7ypiq?1sLYi4GY(*j{>1b8b zkvB*P9zpfFW0?E^w+Q#9w{~(THz*X9%cvig@8-aP%Fl88xgPFU+}nH|h{xCDm#Zu%YzAIKCh&dM;KU!s?3y!?U>c z)ONVgf!3hhH+*@mb|b3eS@nY0sKcW}5l=kJuNN4;xF3F0*I*CeMc`pX`7ydOU51hj z0bGdg3u}R(7`A+wEJP+39Q@N7p;IcG{g= zgPps#ca3{@Zegs!Q1{;syo6PwWe@EDU09b+KvfWJR-+J^Z_P$o3jT*~Eg zKWtus6+MjPC>J^#T>`P=I1q!zERZCIl;ztLUu~|&bw$%P*OEGlM_FwCM4)W6!fX>} z9Yze6R;jr$l~FL4;M0V(73%HehE|fX%=NjUQHV zVG=4ssk=n;Wzziq6xx?zy}L%M<^M&0h#$UK8<%&u6`C|v+vPzds;H1FLZL%Qb9quD ziqiO(*ek`tf3}q-8~d!%!3U}r5DftfGYvzK#+l?CgvcostsPUeRD`?$p(l_(>Ch~n zBWLxw_o0Y7={q!`8j_xfl;K(EuSO6Z!|d|yIM6|SwUs{0~);7jcMQ}u9Ke3sz*by_fAt4en)9=bTY$=p_dBv6+mCv7Dv{M%WIO4&XXsqOG#e{C3N+WnteVwg{R)`UB-!w{Fy(vJ8L4M? zlApoGEb25pUtno-vN=+PFHek+soX)dS>)iP6(7pXP)%!Ij|{ioTf@yFYMnxl7(LSO z5wnT&(^QEY6+{J`&RkcDDo(OsiVb^aa@&l!UFxljx#fEJKbO_-v5^-}^+K|u;Z%st zG<3B4ruAoY3 zmc2g+d={E)T0A;qs!sw~Y}U9C)uKT$s` z!f?1Id5>qMi~TT_6=ctWemKW12pqsWDK=+v!o&nlxjdQcfCRoEI5ij69BV`;gWxzC zY@Xt5k+`BWQd40Xc1GDH`{yJK3aKIm-8dy`Mq%>x&FluSl{AjNb5X$ia{>PpzCFe0 zZ9)E5f&=^nP#qX z3BtUub}3s?*(BPtsrwH0AZqJIlnloS>8z_SqXmTAvl{vGW%d9^OTt0rbg|o}s-TSW zUAGB^&pT3|U z!zL6AVjp9Y1KZu=6! z14o>w;oYgYV@#FqLfcmOWebm6@yI%8{uXAfdVGLbqx`_4Q%89&J05IHE}NjKb2eWb NWKMQV2FsA>{{bSxjHUnp literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..74cf86632b56bb7222096c7d8f59400a4fa64c57 GIT binary patch literal 12 QcmZQzV6bOkaBu)&00qMVKmY&$ literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.index b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_three/00000123/variables/variables.index new file mode 100644 index 0000000000000000000000000000000000000000..ac030a9d4018dd553bb0a736b69cf91b5bb6911a GIT binary patch literal 151 zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5=3=9m6 vK+JIPndE&C4dxv9v~U9hBU1{46I>`_)64Jf93gxl0YV`BcSE;IsrzjJ-**?> literal 0 HcmV?d00001 From 974500e5a03cbd6651f0655e5773d3309043cda1 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Mon, 27 Feb 2017 13:32:14 -0800 Subject: [PATCH 0175/8554] Remove DCHECKS within ServableStateMonitor, which were leaking abstractions. Change: 148687282 --- tensorflow_serving/core/servable_state_monitor.cc | 6 ------ 1 file changed, 6 deletions(-) diff --git a/tensorflow_serving/core/servable_state_monitor.cc b/tensorflow_serving/core/servable_state_monitor.cc index 93f295b5a9d..480d7648d52 100644 --- a/tensorflow_serving/core/servable_state_monitor.cc +++ b/tensorflow_serving/core/servable_state_monitor.cc @@ -28,17 +28,11 @@ void EraseLiveStatesEntry( const int64 version = state_and_time.state.id.version; auto servable_map_it = live_states->find(servable_name); if (servable_map_it == live_states->end()) { - DCHECK(!state_and_time.state.health.ok()) - << "Servable: " << state_and_time - << " is not in error and directly went to state kEnd."; return; } auto& version_map = servable_map_it->second; auto version_map_it = version_map.find(version); if (version_map_it == version_map.end()) { - DCHECK(!state_and_time.state.health.ok()) - << "Servable: " << state_and_time - << " is not in error and directly went to state kEnd."; return; } From 7c61febfde175c1d97470f85c7b74d2dd61e510d Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 2 Mar 2017 10:40:12 -0800 Subject: [PATCH 0176/8554] Fix issue where Predict calls to Classification/Regression signatures were not properly aliasing Tensor names. It's more appropriate to treat them like other Predict calls rather than special casing them and using raw tensor names. Change: 149019979 --- tensorflow_serving/servables/tensorflow/BUILD | 3 +- .../servables/tensorflow/predict_impl.cc | 26 ++++--------- .../servables/tensorflow/predict_impl_test.cc | 38 ++++++++++--------- 3 files changed, 28 insertions(+), 39 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 6321caf065c..9276d619fc4 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -401,8 +401,6 @@ cc_library( srcs = ["predict_impl.cc"], hdrs = ["predict_impl.h"], deps = [ - ":get_model_metadata_impl", - "//tensorflow_serving/apis:get_model_metadata_proto", "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/model_servers:server_core", @@ -479,6 +477,7 @@ cc_test( "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], deps = [ ":predict_impl", diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index b0f3b250e9a..f46a2998e4f 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -160,24 +160,18 @@ Status PreProcessPrediction(const SignatureDef& signature, } for (auto& input : request.inputs()) { const string& alias = input.first; - // When using a Prediction signature, tensors are aliased and the name is - // retrieved from the Tensor value, otherwise the alias (key) is the name. - string tensor_name = alias; - if (signature.method_name() == kPredictMethodName) { - auto iter = signature.inputs().find(alias); - if (iter == signature.inputs().end()) { - return tensorflow::Status( - tensorflow::error::INVALID_ARGUMENT, - "input tensor alias not found in signature: " + alias); - } - tensor_name = iter->second.name(); + auto iter = signature.inputs().find(alias); + if (iter == signature.inputs().end()) { + return tensorflow::Status( + tensorflow::error::INVALID_ARGUMENT, + "input tensor alias not found in signature: " + alias); } Tensor tensor; if (!tensor.FromProto(input.second)) { return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, "tensor parsing error: " + alias); } - inputs->emplace_back(std::make_pair(tensor_name, tensor)); + inputs->emplace_back(std::make_pair(iter->second.name(), tensor)); } // Prepare run target. @@ -204,13 +198,7 @@ Status PreProcessPrediction(const SignatureDef& signature, if (output_tensor_names->empty()) { for (auto& iter : signature.outputs()) { output_tensor_names->emplace_back(iter.second.name()); - // When using a Prediction signature, the tensor output alias is the key - // in the map, otherwise we don't use aliases and just go by actual tensor - // names. - const string alias = signature.method_name() == kPredictMethodName - ? iter.first - : iter.second.name(); - output_tensor_aliases->emplace_back(alias); + output_tensor_aliases->emplace_back(iter.first); } } return Status::OK(); diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 2dd06ccf822..ebf2a523281 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include "tensorflow/cc/saved_model/signature_constants.h" #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow_serving/core/availability_preserving_policy.h" @@ -41,18 +42,20 @@ const char kOutputTensorKey[] = "y"; class PredictImplTest : public ::testing::TestWithParam { public: static void SetUpTestCase() { - TF_ASSERT_OK(CreateServerCore( - "/servables/tensorflow/testdata/half_plus_two", false, &server_core_)); TF_ASSERT_OK( - CreateServerCore("/servables/tensorflow/testdata/bad_half_plus_two", - false, &server_core_bad_model_)); - - TF_ASSERT_OK( - CreateServerCore("/servables/tensorflow/testdata/half_plus_two", true, - &saved_model_server_core_)); - TF_ASSERT_OK( - CreateServerCore("/servables/tensorflow/testdata/bad_half_plus_two", - true, &saved_model_server_core_bad_model_)); + CreateServerCore(test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/half_plus_two"), + false, &server_core_)); + const string bad_half_plus_two_path = test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/bad_half_plus_two"); + TF_ASSERT_OK(CreateServerCore(bad_half_plus_two_path, false, + &server_core_bad_model_)); + + TF_ASSERT_OK(CreateServerCore(test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two"), + true, &saved_model_server_core_)); + TF_ASSERT_OK(CreateServerCore(bad_half_plus_two_path, true, + &saved_model_server_core_bad_model_)); } static void TearDownTestCase() { @@ -68,7 +71,7 @@ class PredictImplTest : public ::testing::TestWithParam { ModelServerConfig config; auto model_config = config.mutable_model_config_list()->add_config(); model_config->set_name(kTestModelName); - model_config->set_base_path(test_util::TestSrcDirPath(model_path)); + model_config->set_base_path(model_path); model_config->set_model_platform(kTensorFlowModelPlatform); // For ServerCore Options, we leave servable_state_monitor_creator @@ -288,13 +291,12 @@ TEST_P(PredictImplTest, PredictionWithNamedRegressionSignature) { ModelSpec* model_spec = request.mutable_model_spec(); model_spec->set_name(kTestModelName); model_spec->mutable_version()->set_value(kTestModelVersion); - model_spec->set_signature_name("regress"); + model_spec->set_signature_name("regress_x2_to_y3"); TensorProto tensor_proto; tensor_proto.add_float_val(2.0); tensor_proto.set_dtype(tensorflow::DT_FLOAT); - (*request.mutable_inputs())["Placeholder:0"] = tensor_proto; - + (*request.mutable_inputs())[kRegressInputs] = tensor_proto; TensorflowPredictor predictor(GetParam()); // This request is expected to work with SavedModel, but not SessionBundle. if (GetParam()) { @@ -308,15 +310,15 @@ TEST_P(PredictImplTest, PredictionWithNamedRegressionSignature) { return; } TensorProto output_tensor_proto; - output_tensor_proto.add_float_val(3); + output_tensor_proto.add_float_val(4); output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); output_tensor_proto.mutable_tensor_shape(); PredictResponse expected_response; - (*expected_response.mutable_outputs())["Add:0"] = output_tensor_proto; + (*expected_response.mutable_outputs())[kRegressOutputs] = output_tensor_proto; EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } -// Test all ClassifierTest test cases with both SessionBundle and SavedModel. +// Test all PredictImplTest test cases with both SessionBundle and SavedModel. INSTANTIATE_TEST_CASE_P(UseSavedModel, PredictImplTest, ::testing::Bool()); } // namespace From e86172dab4315ddca587bd3135dd7ca7a8f5e815 Mon Sep 17 00:00:00 2001 From: perdasilva Date: Mon, 6 Mar 2017 20:34:02 +0100 Subject: [PATCH 0177/8554] Adds public visibility to predict_impl library (#339) --- tensorflow_serving/servables/tensorflow/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 9276d619fc4..0e2ae24725e 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -400,6 +400,9 @@ cc_library( name = "predict_impl", srcs = ["predict_impl.cc"], hdrs = ["predict_impl.h"], + visibility = [ + "//visibility:public", + ], deps = [ "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", From c068a4614857fc180cbd64275b9b95cbba92db68 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Mon, 6 Mar 2017 12:12:34 -0800 Subject: [PATCH 0178/8554] Adds a static test helper to create ServerCore. Change: 149328267 --- tensorflow_serving/core/test_util/BUILD | 1 + .../model_servers/server_core_test.cc | 3 +- .../test_util/server_core_test_util.cc | 63 +++++++++++-------- .../test_util/server_core_test_util.h | 9 ++- 4 files changed, 46 insertions(+), 30 deletions(-) diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 05baa51e3c4..1bec2837dfd 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -72,6 +72,7 @@ cc_library( "//tensorflow_serving/core:storage_path", "@org_tensorflow//tensorflow/core:lib", ], + alwayslink = 1, ) load("//tensorflow_serving:serving.bzl", "serving_proto_library") diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index b25876c47f3..8e33f25ada2 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -85,7 +85,8 @@ TEST_P(ServerCoreTest, ErroringModel) { source_adapter_config.set_error_message("injected error"); ::google::protobuf::Any source_adapter_config_any; source_adapter_config_any.PackFrom(source_adapter_config); - (*(*options.platform_config_map.mutable_platform_configs())[kFakePlatform] + (*(*options.platform_config_map + .mutable_platform_configs())[test_util::kFakePlatform] .mutable_source_adapter_config()) = source_adapter_config_any; options.model_server_config = GetTestModelServerConfigForFakePlatform(); std::unique_ptr server_core; diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index f6dc472d520..6805de0b80d 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -29,7 +29,41 @@ namespace tensorflow { namespace serving { namespace test_util { -constexpr char ServerCoreTest::kFakePlatform[]; +namespace { + +ServerCore::Options GetDefaultOptions(const bool use_saved_model) { + ServerCore::Options options; + options.file_system_poll_wait_seconds = 0; + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + options.custom_model_config_loader = + [](const ::google::protobuf::Any& any, EventBus* event_bus, + UniquePtrWithDeps* manager) -> Status { + return Status::OK(); + }; + + options.platform_config_map = + CreateTensorFlowPlatformConfigMap(SessionBundleConfig(), use_saved_model); + ::google::protobuf::Any fake_source_adapter_config; + fake_source_adapter_config.PackFrom( + test_util::FakeLoaderSourceAdapterConfig()); + (*(*options.platform_config_map.mutable_platform_configs())[kFakePlatform] + .mutable_source_adapter_config()) = fake_source_adapter_config; + + return options; +} + +} // namespace + +Status CreateServerCore(const ModelServerConfig& config, + std::unique_ptr* server_core) { + ServerCore::Options options = GetDefaultOptions(true /*use_saved_model */); + options.model_server_config = config; + return ServerCore::Create(std::move(options), server_core); +} ModelServerConfig ServerCoreTest::GetTestModelServerConfigForFakePlatform() { ModelServerConfig config = GetTestModelServerConfigForTensorflowPlatform(); @@ -56,39 +90,16 @@ ServerCoreTest::GetTestModelServerConfigForTensorflowPlatform() { } ServerCore::Options ServerCoreTest::GetDefaultOptions() { - ServerCore::Options options; - options.file_system_poll_wait_seconds = 0; - // Reduce the number of initial load threads to be num_load_threads to avoid - // timing out in tests. - options.num_initial_load_threads = options.num_load_threads; - options.aspired_version_policy = - std::unique_ptr(new AvailabilityPreservingPolicy); - options.custom_model_config_loader = []( - const ::google::protobuf::Any& any, EventBus* event_bus, - UniquePtrWithDeps* manager) -> Status { - return Status::OK(); - }; - // Model platforms. const TestType test_type = GetTestType(); const bool use_saved_model = test_type == SAVED_MODEL || test_type == SAVED_MODEL_BACKWARD_COMPATIBILITY; - options.platform_config_map = - CreateTensorFlowPlatformConfigMap(SessionBundleConfig(), use_saved_model); - ::google::protobuf::Any fake_source_adapter_config; - fake_source_adapter_config.PackFrom( - test_util::FakeLoaderSourceAdapterConfig()); - (*(*options.platform_config_map.mutable_platform_configs())[kFakePlatform] - .mutable_source_adapter_config()) = fake_source_adapter_config; - - return options; + return test_util::GetDefaultOptions(use_saved_model); } Status ServerCoreTest::CreateServerCore( const ModelServerConfig& config, std::unique_ptr* server_core) { - ServerCore::Options options = GetDefaultOptions(); - options.model_server_config = config; - return ServerCore::Create(std::move(options), server_core); + return test_util::CreateServerCore(config, server_core); } } // namespace test_util diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index 3ec419da221..86557b316c6 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -26,6 +26,8 @@ namespace test_util { constexpr char kTestModelName[] = "test_model"; constexpr int kTestModelVersion = 123; +// The name of the platform associated with FakeLoaderSourceAdapter. +constexpr char kFakePlatform[] = "fake_servable"; // ServerCoreTest is parameterized based on the TestType enum defined below. // TODO(b/32248363): remove the parameter and TestType after we switch Model @@ -45,9 +47,6 @@ class ServerCoreTest : public ::testing::TestWithParam { }; protected: - // The name of the platform associated with FakeLoaderSourceAdapter. - static constexpr char kFakePlatform[] = "fake_servable"; - // Returns ModelServerConfig that contains test model for the fake platform. ModelServerConfig GetTestModelServerConfigForFakePlatform(); @@ -67,6 +66,10 @@ class ServerCoreTest : public ::testing::TestWithParam { TestType GetTestType() { return static_cast(GetParam()); } }; +// Creates a ServerCore object with sane defaults. +Status CreateServerCore(const ModelServerConfig& config, + std::unique_ptr* server_core); + } // namespace test_util } // namespace serving } // namespace tensorflow From d4c5a6fc9aac22d162a60df3b1072f88d1ab0e53 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 6 Mar 2017 16:24:04 -0800 Subject: [PATCH 0179/8554] Fix the inception repo workspace name to be consistent with what's specified in the actual repo. Eliminate the printed warning. Change: 149360061 --- tensorflow_serving/example/BUILD | 4 ++-- tensorflow_serving/workspace.bzl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 85bb83412df..3b34164170c 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -80,7 +80,7 @@ py_binary( "inception_export.py", ], deps = [ - "@inception_model//inception", + "@inception//inception", "@org_tensorflow//tensorflow:tensorflow_py", "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", ], @@ -92,7 +92,7 @@ py_binary( "inception_saved_model.py", ], deps = [ - "@inception_model//inception", + "@inception//inception", "@org_tensorflow//tensorflow:tensorflow_py", "@org_tensorflow//tensorflow/python/saved_model:builder", "@org_tensorflow//tensorflow/python/saved_model:constants", diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index 93d9a79fee2..18cd9469bc0 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -8,7 +8,7 @@ load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') # as a submodule, it'll likely be '__workspace_dir__ + "/serving"' def tf_serving_workspace(): native.local_repository( - name = "inception_model", + name = "inception", path = "tf_models/inception", ) From 3bc9708b8e542d982339042a13fd8b3edd06c77d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 6 Mar 2017 17:04:47 -0800 Subject: [PATCH 0180/8554] Update install instruction links. Fixes #8142. Change: 149364578 --- .../core/aspired_versions_manager.cc | 3 +- tensorflow_serving/core/basic_manager.cc | 23 ++- tensorflow_serving/core/basic_manager_test.cc | 158 ++++++++++-------- .../core/dynamic_source_router_test.cc | 2 +- tensorflow_serving/core/static_manager.h | 17 +- tensorflow_serving/g3doc/setup.md | 4 +- .../hashmap/hashmap_source_adapter_test.cc | 2 +- .../servables/tensorflow/classifier_test.cc | 4 +- .../servables/tensorflow/regressor_test.cc | 4 +- .../file_system_storage_path_source.cc | 4 +- 10 files changed, 121 insertions(+), 100 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index ea458d6e6fb..c867b03d089 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -376,7 +376,8 @@ void AspiredVersionsManager::FlushServables() { state_snapshot.state == LoaderHarness::State::kError) && !state_snapshot.additional_state->is_aspired) { VLOG(1) << "Removing " << state_snapshot.id << "from BasicManager"; - basic_manager_->StopManagingServable(state_snapshot.id); + // TODO(b/35997855): Don't just ignore the ::tensorflow::Status object! + basic_manager_->StopManagingServable(state_snapshot.id).IgnoreError(); } } } diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 320a64ef2b2..777ea9a0376 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -253,17 +253,20 @@ void BasicManager::UnloadAllServables() { for (auto it = managed_map_.begin(); it != managed_map_.end(); ++it) { LoaderHarness* const harness = it->second.get(); if (harness->state() == LoaderHarness::State::kReady) { - harness->UnloadRequested(); - harness->StartQuiescing(); - harness->DoneQuiescing(); - harness->Unload(); + // TODO(b/35997855): Don't just ignore the ::tensorflow::Status object! + harness->UnloadRequested().IgnoreError(); + harness->StartQuiescing().IgnoreError(); + harness->DoneQuiescing().IgnoreError(); + harness->Unload().IgnoreError(); } if (harness->state() == LoaderHarness::State::kQuiescing) { - harness->DoneQuiescing(); - harness->Unload(); + // TODO(b/35997855): Don't just ignore the ::tensorflow::Status object! + harness->DoneQuiescing().IgnoreError(); + harness->Unload().IgnoreError(); } if (harness->state() == LoaderHarness::State::kQuiesced) { - harness->Unload(); + // TODO(b/35997855): Don't just ignore the ::tensorflow::Status object! + harness->Unload().IgnoreError(); } } } @@ -667,8 +670,10 @@ Status BasicManager::ApproveUnload(LoaderHarness* harness) { Status BasicManager::ReserveResources(LoaderHarness* harness, mutex_lock* mu_lock) { while (true) { - resource_tracker_->RecomputeUsedResources( - GetLoadersCurrentlyUsingResources()); + // TODO(b/35997855): Don't just ignore the ::tensorflow::Status object! + resource_tracker_ + ->RecomputeUsedResources(GetLoadersCurrentlyUsingResources()) + .IgnoreError(); bool resources_reserved; // We retry reserving resources because it may involve transiently failing // operations like file-reads. diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 15eb23bd09d..97cdac15df7 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -102,7 +102,7 @@ class BasicManagerTest : public ::testing::TestWithParam { for (const char* servable_name : {kServableName, kServableName2}) { for (int i = 1; i <= kNumVersionsPerServable; ++i) { const ServableId id = {servable_name, i}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); loaded_servables.insert(id); @@ -150,7 +150,7 @@ TEST_P(BasicManagerTest, ServableHandleNotFoundMissingVersion) { TEST_P(BasicManagerTest, ServableHandleLatest) { const ServableId id = {kServableName, kNumVersionsPerServable + 1}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -172,7 +172,7 @@ TEST_P(BasicManagerTest, AlreadyManagedError) { // Tests the case where the latest version of a servable available is 0. TEST_P(BasicManagerTest, ServableHandleLatestVersionIsZero) { const ServableId id = {kServableName3, 1}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -193,7 +193,7 @@ TEST_P(BasicManagerTest, StopManagingUnknownId) { TEST_P(BasicManagerTest, StopManagingActiveServable) { const ServableId id = {kServableName3, 1}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_EXPECT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -203,7 +203,7 @@ TEST_P(BasicManagerTest, StopManagingActiveServable) { TEST_P(BasicManagerTest, StopManagingDisabledServable) { const ServableId id = {kServableName3, 1}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_EXPECT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -228,7 +228,7 @@ TEST_P(BasicManagerTest, DontStopManagingOnError) { const ServableId id = {kServableName, 7}; const Status error_status = errors::Internal("An error."); std::unique_ptr loader(new FakeLoader(7, error_status)); - basic_manager_->ManageServable({id, std::move(loader)}); + TF_CHECK_OK(basic_manager_->ManageServable({id, std::move(loader)})); basic_manager_->LoadServable(id, [error_status](const Status& status) { EXPECT_EQ(error_status, status); }); @@ -260,7 +260,7 @@ TEST_P(BasicManagerTest, UpdateServingMapServableHandleLatest) { // manager, as opposed to kServableName which already has 2 loaded. const ServableId id0 = {kServableName3, 0}; // Servable is int64 with value 0. - basic_manager_->ManageServable(CreateServable(id0)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id0))); basic_manager_->LoadServable( id0, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -276,8 +276,8 @@ TEST_P(BasicManagerTest, UpdateServingMapServableHandleLatest) { .WillByDefault(Return(Status::OK())); ON_CALL(*notify_to_unload, Load()).WillByDefault(Return(Status::OK())); const ServableId id1 = {kServableName3, 1}; - basic_manager_->ManageServable( - {id1, std::unique_ptr(notify_to_unload)}); + TF_CHECK_OK(basic_manager_->ManageServable( + {id1, std::unique_ptr(notify_to_unload)})); basic_manager_->LoadServable( id1, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -340,7 +340,8 @@ TEST_P(BasicManagerTest, ListAvailableServableIds) { const ServableId id = {kServableName, 7}; std::unique_ptr loader( new FakeLoader(7, errors::Internal("An error."))); - basic_manager_->ManageServable(CreateServableData(id, std::move(loader))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(id, std::move(loader)))); basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_EQ(errors::Internal("An error."), status); }); @@ -386,7 +387,8 @@ TEST_P(BasicManagerTest, GetAvailableServableHandles) { const ServableId id = {kServableName, 7}; std::unique_ptr loader( new FakeLoader(7, errors::Internal("An error."))); - basic_manager_->ManageServable(CreateServableData(id, std::move(loader))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(id, std::move(loader)))); basic_manager_->LoadServable(id, [](const Status& status) { EXPECT_EQ(errors::Internal("An error."), status); }); @@ -456,10 +458,10 @@ TEST_P(BasicManagerTest, GetManagedServableStateSnapshot) { } TEST_P(BasicManagerTest, GetManagedServableStateSnapshotsWithAdditionalState) { - basic_manager_->ManageServableWithAdditionalState( - CreateServable({kServableName3, 0}), std::unique_ptr(new int(0))); - basic_manager_->ManageServableWithAdditionalState( - CreateServable({kServableName3, 1}), std::unique_ptr(new int(1))); + TF_CHECK_OK(basic_manager_->ManageServableWithAdditionalState( + CreateServable({kServableName3, 0}), std::unique_ptr(new int(0)))); + TF_CHECK_OK(basic_manager_->ManageServableWithAdditionalState( + CreateServable({kServableName3, 1}), std::unique_ptr(new int(1)))); const std::vector> expected = { {{kServableName3, 0}, LoaderHarness::State::kNew, {0}}, {{kServableName3, 1}, LoaderHarness::State::kNew, {1}}}; @@ -473,13 +475,15 @@ TEST_P(BasicManagerTest, MultipleManageCallsUsesFirstServable) { std::unique_ptr first_loader( new FakeLoader(1, errors::Internal("An error."))); - basic_manager_->ManageServable( - CreateServableData(id, std::move(first_loader))); + basic_manager_ + ->ManageServable(CreateServableData(id, std::move(first_loader))) + .IgnoreError(); // Different servable returned. std::unique_ptr second_loader( new FakeLoader(2, errors::Internal("An error."))); - basic_manager_->ManageServable( - CreateServableData(id, std::move(second_loader))); + basic_manager_ + ->ManageServable(CreateServableData(id, std::move(second_loader))) + .IgnoreError(); ServableHandle handle; TF_ASSERT_OK(basic_manager_->GetServableHandle( @@ -491,8 +495,8 @@ TEST_P(BasicManagerTest, MultipleManageCallsUsesFirstServable) { // erroneous servable. TEST_P(BasicManagerTest, ErroneousServable) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable( - ServableData>(id, errors::Unknown("error"))); + TF_CHECK_OK(basic_manager_->ManageServable( + ServableData>(id, errors::Unknown("error")))); ServableHandle handle; Status status = basic_manager_->GetServableHandle( @@ -510,8 +514,8 @@ TEST_P(BasicManagerTest, ErroneousServable) { // thread, and not a request thread. TEST_P(BasicManagerTest, DestructOnNonServingThread) { const ServableId id = {kServableName, 7}; - basic_manager_->ManageServable( - CreateServableData(id, std::unique_ptr(new FakeLoader(7)))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(id, std::unique_ptr(new FakeLoader(7))))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -551,8 +555,8 @@ TEST_P(BasicManagerTest, DestructOnNonServingThread) { TEST_P(BasicManagerTest, AdditionalState) { const ServableId id = {kServableName, 3}; std::unique_ptr state(new int(1)); - basic_manager_->ManageServableWithAdditionalState(CreateServable(id), - std::move(state)); + TF_CHECK_OK(basic_manager_->ManageServableWithAdditionalState( + CreateServable(id), std::move(state))); EXPECT_EQ(1, *basic_manager_->GetAdditionalServableState(id)); EXPECT_EQ(nullptr, basic_manager_->GetAdditionalServableState(id)); @@ -560,7 +564,7 @@ TEST_P(BasicManagerTest, AdditionalState) { TEST_P(BasicManagerTest, NoAdditionalState) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); // Will return nullptr when there is no metadata set. EXPECT_EQ(nullptr, basic_manager_->GetAdditionalServableState(id)); @@ -578,7 +582,7 @@ TEST_P(BasicManagerTest, OutOfOrderLoadServable) { TEST_P(BasicManagerTest, MultipleLoadServables) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -592,7 +596,7 @@ TEST_P(BasicManagerTest, MultipleLoadServables) { TEST_P(BasicManagerTest, MultipleUnloadServables) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( @@ -620,7 +624,7 @@ TEST_P(BasicManagerTest, UnloadWithoutManage) { TEST_P(BasicManagerTest, UnloadWithoutLoad) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->UnloadServable(id, [](const Status& status) { EXPECT_FALSE(status.ok()); EXPECT_EQ(error::FAILED_PRECONDITION, status.code()); @@ -630,8 +634,8 @@ TEST_P(BasicManagerTest, UnloadWithoutLoad) { TEST_P(BasicManagerTest, EventBusErroneousVersion) { const ServableId id = {kServableName, 3}; - basic_manager_->ManageServable( - ServableData>(id, errors::Unknown("error"))); + TF_CHECK_OK(basic_manager_->ManageServable( + ServableData>(id, errors::Unknown("error")))); const ServableState expected_published_state = { id, ServableState::ManagerState::kEnd, errors::Unknown("error")}; @@ -643,7 +647,7 @@ TEST_P(BasicManagerTest, EventBusErrorOnLoad) { const ServableId id = {kServableName, 7}; std::unique_ptr loader( new FakeLoader(7, errors::Internal("Error on load."))); - basic_manager_->ManageServable({id, std::move(loader)}); + TF_CHECK_OK(basic_manager_->ManageServable({id, std::move(loader)})); const ServableState start_state = {id, ServableState::ManagerState::kStart, Status::OK()}; @@ -663,7 +667,8 @@ TEST_P(BasicManagerTest, EventBusErrorOnLoad) { TEST_P(BasicManagerTest, EventBusServableLifecycle) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_CHECK_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); const ServableState start_state = {id, ServableState::ManagerState::kStart, Status::OK()}; @@ -741,7 +746,7 @@ TEST_P(BasicManagerTest, NoEventBus) { const ServableId id = {kServableName, 7}; std::unique_ptr loader(new FakeLoader(7)); - manager->ManageServable({id, std::move(loader)}); + TF_CHECK_OK(manager->ManageServable({id, std::move(loader)})); manager->LoadServable(id, [](const Status& status) { TF_ASSERT_OK(status); }); manager->UnloadServable(id, [](const Status& status) { TF_ASSERT_OK(status); }); @@ -757,7 +762,7 @@ TEST_P(BasicManagerTest, LoadsThenUnloads) { const ServableId id = {kServableName3, i}; servables.insert(id); load_executor.Schedule([this, id, &servables]() { - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); }); @@ -791,7 +796,7 @@ TEST_P(BasicManagerTest, InterleavedLoadsAndUnloads) { for (int i = 0; i < 20; ++i) { executor.Schedule([this, i]() { const ServableId id = {kServableName3, i}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); Notification load_done; basic_manager_->LoadServable(id, [&load_done](const Status& status) { TF_ASSERT_OK(status); @@ -832,14 +837,14 @@ TEST_F(SetNumLoadThreadsBasicManagerTest, ThreadPoolSwapped) { }; const ServableId id0 = {kServableName3, 0}; - basic_manager_->ManageServable(CreateServable(id0)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id0))); basic_manager_->LoadServable(id0, load_done_fn); manager_test_access.SetNumLoadThreads(0); EXPECT_EQ(0, manager_test_access.num_load_threads()); const ServableId id1 = {kServableName3, 1}; - basic_manager_->ManageServable(CreateServable(id1)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id1))); basic_manager_->LoadServable(id1, load_done_fn); // Force the manager to finish before deleting the notifications. @@ -860,7 +865,7 @@ TEST_F(SetNumLoadThreadsBasicManagerTest, ThreadPoolsNotAliveSimultaneously) { }; const ServableId id0 = {kServableName3, 0}; - basic_manager_->ManageServable(CreateServable(id0)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id0))); Notification notify_for_setting; Notification continue_load; basic_manager_->LoadServable(id0, [&](const Status& status) { @@ -880,7 +885,7 @@ TEST_F(SetNumLoadThreadsBasicManagerTest, ThreadPoolsNotAliveSimultaneously) { executor.Schedule([&]() { const ServableId id1 = {kServableName3, 1}; - basic_manager_->ManageServable(CreateServable(id1)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id1))); continue_load.Notify(); basic_manager_->LoadServable( id1, [&](const Status& status) { data_race_fn(status); }); @@ -905,7 +910,7 @@ TEST_F(SetNumLoadThreadsBasicManagerTest, FastLoad) { for (int i = 0; i < 20; ++i) { executor.Schedule([this, i]() { const ServableId id = {kServableName3, i}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); // We don't wait for load to be done here because we want to test that @@ -940,7 +945,7 @@ TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { kNumThreads); for (int i = 0; i < 4; ++i) { load_executor.Schedule([this, id, i, &statuses, &status_mu]() { - basic_manager_->ManageServable(CreateServable(id)); + basic_manager_->ManageServable(CreateServable(id)).IgnoreError(); basic_manager_->LoadServable( id, [i, &statuses, &status_mu](const Status& status) { mutex_lock l(status_mu); @@ -970,7 +975,7 @@ TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { TEST_P(BasicManagerTest, ConcurrentUnloadsOnlyOneSucceeds) { const ServableId id = {kServableName3, 0}; - basic_manager_->ManageServable(CreateServable(id)); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id))); basic_manager_->LoadServable( id, [](const Status& status) { TF_ASSERT_OK(status); }); // At this point, all loads may not have completed, so we wait for them. @@ -1021,7 +1026,8 @@ TEST_P(BasicManagerTest, ConcurrentUnloadsOnlyOneSucceeds) { TEST_P(BasicManagerTest, RetryOnLoadErrorFinallySucceeds) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_CHECK_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); EXPECT_CALL(*loader, Load()) .WillOnce(Return(errors::Internal("Load error."))) .WillRepeatedly(Return(Status::OK())); @@ -1032,7 +1038,8 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorFinallySucceeds) { TEST_P(BasicManagerTest, RetryOnLoadErrorFinallyFails) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_CHECK_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); EXPECT_CALL(*loader, Load()) .WillRepeatedly(Return(errors::Internal("Load error."))); basic_manager_->LoadServable(id, [](const Status& status) { @@ -1044,7 +1051,8 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorFinallyFails) { TEST_P(BasicManagerTest, RetryOnLoadErrorCancelledLoad) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_CHECK_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); Notification load_called; Notification load_should_return; @@ -1071,7 +1079,8 @@ TEST_P(BasicManagerTest, RetryOnLoadErrorCancelledLoad) { TEST_P(BasicManagerTest, LoadAfterCancelledLoad) { const ServableId id = {kServableName, 7}; test_util::MockLoader* loader = new NiceMock(); - basic_manager_->ManageServable({id, std::unique_ptr(loader)}); + TF_CHECK_OK( + basic_manager_->ManageServable({id, std::unique_ptr(loader)})); Notification load_called; Notification load_should_return; @@ -1179,7 +1188,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, ConcurrentLoads) { for (int i = 0; i < kNumLoaders; ++i) { std::unique_ptr loader(new BarrierLoader(&barrier)); const ServableId id = {"barrier", i}; - basic_manager_->ManageServable(CreateServableData(id, std::move(loader))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(id, std::move(loader)))); basic_manager_->LoadServable( id, [](const Status& status) { TF_EXPECT_OK(status); }); } @@ -1198,8 +1208,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, InsufficientResources) { return Status::OK(); })); EXPECT_CALL(*hogging_loader, Load()).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable( - CreateServableData(hogging_id, std::unique_ptr(hogging_loader))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(hogging_id, std::unique_ptr(hogging_loader)))); Notification hogging_loaded; basic_manager_->LoadServable(hogging_id, [&hogging_loaded](const Status& status) { @@ -1216,8 +1226,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, InsufficientResources) { *estimate = CreateResourceQuantity(1); return Status::OK(); })); - basic_manager_->ManageServable(CreateServableData( - rejected_id, std::unique_ptr(rejected_loader))); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServableData( + rejected_id, std::unique_ptr(rejected_loader)))); Notification rejection_received; Status rejected_status; basic_manager_->LoadServable( @@ -1251,8 +1261,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { })); EXPECT_CALL(*failing_loader, Load()) .WillOnce(Return(errors::Unknown("Load failure"))); - basic_manager_->ManageServable( - CreateServableData(failing_id, std::unique_ptr(failing_loader))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(failing_id, std::unique_ptr(failing_loader)))); Notification failing_failed; basic_manager_->LoadServable(failing_id, [&failing_failed](const Status& status) { @@ -1273,8 +1283,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedIfLoadFails) { return Status::OK(); })); EXPECT_CALL(*succeeding_loader, Load()).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); basic_manager_->LoadServable( succeeding_id, [](const Status& status) { TF_EXPECT_OK(status); }); } @@ -1301,8 +1311,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, })) .RetiresOnSaturation(); } - basic_manager_->ManageServable(CreateServableData( - overestimating_id, std::unique_ptr(overestimating_loader))); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServableData( + overestimating_id, std::unique_ptr(overestimating_loader)))); Notification overestimating_loaded; basic_manager_->LoadServable(overestimating_id, [&overestimating_loaded](const Status& status) { @@ -1323,8 +1333,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, return Status::OK(); })); EXPECT_CALL(*succeeding_loader, Load()).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); basic_manager_->LoadServable( succeeding_id, [](const Status& status) { TF_EXPECT_OK(status); }); } @@ -1339,8 +1349,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedAfterUnload) { })); Notification load_done; EXPECT_CALL(*unloading_loader, Load()).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - unloading_id, std::unique_ptr(unloading_loader))); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServableData( + unloading_id, std::unique_ptr(unloading_loader)))); basic_manager_->LoadServable(unloading_id, [&load_done](const Status& status) { TF_EXPECT_OK(status); @@ -1375,8 +1385,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, ResourcesReleasedAfterUnload) { return Status::OK(); })); EXPECT_CALL(*succeeding_loader, Load()).WillOnce(Return(Status::OK())); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); basic_manager_->LoadServable( succeeding_id, [](const Status& status) { TF_EXPECT_OK(status); }); @@ -1400,8 +1410,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { })); // Load won't be called because resources are not enough to load it. EXPECT_CALL(*denied_loader, Load()).Times(0); - basic_manager_->ManageServable( - CreateServableData(denied_id, std::unique_ptr(denied_loader))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(denied_id, std::unique_ptr(denied_loader)))); // A second loader that succeeds. const ServableId succeeding_id = {"succeeding", 0}; @@ -1412,8 +1422,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, FirstLoadDeniedSecondOneApproved) { *estimate = CreateResourceQuantity(10); return Status::OK(); })); - basic_manager_->ManageServable(CreateServableData( - succeeding_id, std::unique_ptr(succeeding_loader))); + TF_CHECK_OK(basic_manager_->ManageServable(CreateServableData( + succeeding_id, std::unique_ptr(succeeding_loader)))); Status denied_load_status; // Place the first servable into a load request decision phase. @@ -1464,8 +1474,8 @@ TEST_F(ResourceConstrainedBasicManagerTest, EventBusErrorOnEstimateResources) { test_util::MockLoader* loader = new NiceMock; EXPECT_CALL(*loader, EstimateResources(_)) .WillOnce(Return(errors::Internal("Error on estimate resources."))); - basic_manager_->ManageServable( - CreateServableData(id, std::unique_ptr(loader))); + TF_CHECK_OK(basic_manager_->ManageServable( + CreateServableData(id, std::unique_ptr(loader)))); basic_manager_->LoadServable( id, [](const Status& status) { EXPECT_FALSE(status.ok()); }); WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, id, @@ -1503,8 +1513,8 @@ TEST(EstimateResourcesRetriedTest, Succeeds) { .WillOnce(Return(errors::Internal("Error on estimate resources."))) .WillOnce(Return(Status::OK())); EXPECT_CALL(*loader, Load()).WillRepeatedly(Return(Status::OK())); - basic_manager->ManageServable( - CreateServableData(id, std::unique_ptr(loader))); + TF_CHECK_OK(basic_manager->ManageServable( + CreateServableData(id, std::unique_ptr(loader)))); basic_manager->LoadServable( id, [](const Status& status) { EXPECT_TRUE(status.ok()); }); WaitUntilServableManagerStateIsOneOf( @@ -1539,8 +1549,8 @@ TEST(EstimateResourcesRetriedTest, Fails) { .WillOnce(Return(errors::Internal("Error on estimate resources."))) .WillOnce(Return(errors::Internal("Error on estimate resources."))) .WillRepeatedly(Return(Status::OK())); - basic_manager->ManageServable( - CreateServableData(id, std::unique_ptr(loader))); + TF_CHECK_OK(basic_manager->ManageServable( + CreateServableData(id, std::unique_ptr(loader)))); basic_manager->LoadServable( id, [](const Status& status) { EXPECT_FALSE(status.ok()); }); WaitUntilServableManagerStateIsOneOf(servable_state_monitor, id, diff --git a/tensorflow_serving/core/dynamic_source_router_test.cc b/tensorflow_serving/core/dynamic_source_router_test.cc index 832c599ce5f..d875de79a33 100644 --- a/tensorflow_serving/core/dynamic_source_router_test.cc +++ b/tensorflow_serving/core/dynamic_source_router_test.cc @@ -125,7 +125,7 @@ TEST(DynamicSourceRouterTest, Reconfigure) { router->SetAspiredVersions("bar", {ServableData({"bar", 7}, "data")}); - router->UpdateRoutes({{"bar", 0}}); + TF_CHECK_OK(router->UpdateRoutes({{"bar", 0}})); // Now, the routes of "foo" and "bar" should be swapped. EXPECT_CALL(*targets[1], SetAspiredVersions( diff --git a/tensorflow_serving/core/static_manager.h b/tensorflow_serving/core/static_manager.h index 481bff1c767..f0f21809f40 100644 --- a/tensorflow_serving/core/static_manager.h +++ b/tensorflow_serving/core/static_manager.h @@ -66,13 +66,16 @@ Status StaticManagerBuilder::AddServable(const ServableId& id, TF_RETURN_IF_ERROR(health_); DCHECK(basic_manager_ != nullptr); - basic_manager_->ManageServable(CreateServableData( - id, std::unique_ptr(new SimpleLoader( - [&servable](std::unique_ptr* const returned_servable) { - *returned_servable = std::move(servable); - return Status::OK(); - }, - SimpleLoader::EstimateNoResources())))); + // TODO(b/35997855): Don't just ignore the ::tensorflow::Status object! + basic_manager_ + ->ManageServable(CreateServableData( + id, std::unique_ptr(new SimpleLoader( + [&servable](std::unique_ptr* const returned_servable) { + *returned_servable = std::move(servable); + return Status::OK(); + }, + SimpleLoader::EstimateNoResources())))) + .IgnoreError(); Status load_status; Notification load_done; basic_manager_->LoadServable(id, [&](const Status& status) { diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 58028e35292..c0be883ce20 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -83,8 +83,8 @@ cd tensorflow cd .. ~~~ -Consult the [TensorFlow install instructions]( -https://github.com/tensorflow/tensorflow/blob/master/tensorflow/g3doc/get_started/os_setup.md) +Consult the +[TensorFlow install instructions](https://www.tensorflow.org/install/) if you encounter any issues with setting up TensorFlow or its dependencies. diff --git a/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc b/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc index 997359ef432..2ffdcbb73fe 100644 --- a/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc +++ b/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc @@ -55,7 +55,7 @@ Status WriteHashmapToFile(const HashmapSourceAdapterConfig::Format format, const string& key = entry.first; const string& value = entry.second; const string line = strings::StrCat(key, ",", value, "\n"); - file->Append(line); + TF_CHECK_OK(file->Append(line)); } break; } diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc index 698bbcf4632..10ef40e50d7 100644 --- a/tensorflow_serving/servables/tensorflow/classifier_test.cc +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -263,8 +263,8 @@ class ClassifierTest : public ::testing::TestWithParam { Status Create() { if (GetParam()) { std::unique_ptr saved_model(new SavedModelBundle); - internal::ConvertSessionBundleToSavedModelBundle(*bundle_, - saved_model.get()); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); return CreateClassifierFromSavedModelBundle(std::move(saved_model), &classifier_); } else { diff --git a/tensorflow_serving/servables/tensorflow/regressor_test.cc b/tensorflow_serving/servables/tensorflow/regressor_test.cc index 8117f9ab3ab..e57148be6ff 100644 --- a/tensorflow_serving/servables/tensorflow/regressor_test.cc +++ b/tensorflow_serving/servables/tensorflow/regressor_test.cc @@ -208,8 +208,8 @@ class RegressorTest : public ::testing::TestWithParam { Status Create() { if (GetParam()) { std::unique_ptr saved_model(new SavedModelBundle); - internal::ConvertSessionBundleToSavedModelBundle(*bundle_, - saved_model.get()); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); return CreateRegressorFromSavedModelBundle(std::move(saved_model), ®ressor_); } else { diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 2437dd7fb15..16797ed985f 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -272,7 +272,9 @@ Status FileSystemStoragePathSource::UpdateConfig( } if (aspired_versions_callback_) { - UnaspireServables(GetDeletedServables(config_, normalized_config)); + // TODO(b/35997855): Don't just ignore the ::tensorflow::Status object! + UnaspireServables(GetDeletedServables(config_, normalized_config)) + .IgnoreError(); } config_ = normalized_config; From 3c85e8b71ad7a419c1973a40c0fc565418014ecf Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 6 Mar 2017 18:34:03 -0800 Subject: [PATCH 0181/8554] Implement Classify and Regress API calls in model server. Change: 149372660 --- tensorflow_serving/model_servers/BUILD | 2 + tensorflow_serving/model_servers/main.cc | 22 +++++- .../tensorflow_model_server_test.py | 74 ++++++++++++++++++- 3 files changed, 90 insertions(+), 8 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 90c841734ce..eb3e83b4b22 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -110,7 +110,9 @@ SUPPORTED_TENSORFLOW_OPS = [ TENSORFLOW_DEPS = [ "@org_tensorflow//tensorflow/core:tensorflow", + "//tensorflow_serving/servables/tensorflow:classification_service", "//tensorflow_serving/servables/tensorflow:get_model_metadata_impl", + "//tensorflow_serving/servables/tensorflow:regression_service", "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:predict_impl", diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 1b0d2c6ac47..3a78b396637 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -70,8 +70,10 @@ limitations under the License. #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" #include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/classification_service.h" #include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" +#include "tensorflow_serving/servables/tensorflow/regression_service.h" namespace grpc { class ServerCompletionQueue; @@ -92,6 +94,8 @@ using tensorflow::serving::ServableState; using tensorflow::serving::ServerCore; using tensorflow::serving::SessionBundleConfig; using tensorflow::serving::Target; +using tensorflow::serving::TensorflowClassificationServiceImpl; +using tensorflow::serving::TensorflowRegressionServiceImpl; using tensorflow::serving::TensorflowPredictor; using tensorflow::serving::UniquePtrWithDeps; using tensorflow::string; @@ -226,15 +230,25 @@ class PredictionServiceImpl final : public PredictionService::Service { grpc::Status Classify(ServerContext* context, const ClassificationRequest* request, ClassificationResponse* response) override { - return ToGRPCStatus(tensorflow::errors::Unimplemented( - "Classify API is not implemented")); + const grpc::Status status = + ToGRPCStatus(TensorflowClassificationServiceImpl::Classify( + core_.get(), *request, response)); + if (!status.ok()) { + VLOG(1) << "Classify request failed: " << status.error_message(); + } + return status; } grpc::Status Regress(ServerContext* context, const RegressionRequest* request, RegressionResponse* response) override { - return ToGRPCStatus(tensorflow::errors::Unimplemented( - "Regress API is not implemented")); + const grpc::Status status = + ToGRPCStatus(TensorflowRegressionServiceImpl::Regress( + core_.get(), *request, response)); + if (!status.ok()) { + VLOG(1) << "Regress request failed: " << status.error_message(); + } + return status; } private: diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index eee38ba0b52..d144714d715 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -34,8 +34,10 @@ from tensorflow.core.framework import types_pb2 from tensorflow.python.platform import flags +from tensorflow_serving.apis import classification_pb2 from tensorflow_serving.apis import predict_pb2 from tensorflow_serving.apis import prediction_service_pb2 +from tensorflow_serving.apis import regression_pb2 FLAGS = flags.FLAGS @@ -173,6 +175,71 @@ def _GetBadModelConfigFile(self): """Returns a path to a improperly formatted configuration file.""" return os.path.join(self.testdata_dir, 'bad_model_config.txt') + def testClassify(self): + """Test PredictionService.Classify implementation.""" + model_path = self._GetSavedModelBundlePath() + use_saved_model = True + enable_batching = False + + atexit.register(self.TerminateProcs) + model_server_address = self.RunServer(PickUnusedPort(), 'default', + model_path, use_saved_model, + enable_batching) + time.sleep(5) + + print 'Sending Classify request...' + # Prepare request + request = classification_pb2.ClassificationRequest() + request.model_spec.name = 'default' + request.model_spec.signature_name = 'classify_x_to_y' + + example = request.input.example_list.examples.add() + example.features.feature['x'].float_list.value.extend([2.0]) + + # Send request + host, port = model_server_address.split(':') + channel = implementations.insecure_channel(host, int(port)) + stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) + result = stub.Classify(request, 5.0) # 5 secs timeout + # Verify response + self.assertEquals(1, len(result.result.classifications)) + self.assertEquals(1, len(result.result.classifications[0].classes)) + expected_output = 3.0 + self.assertEquals(expected_output, + result.result.classifications[0].classes[0].score) + + def testRegress(self): + """Test PredictionService.Regress implementation.""" + model_path = self._GetSavedModelBundlePath() + use_saved_model = True + enable_batching = False + + atexit.register(self.TerminateProcs) + model_server_address = self.RunServer(PickUnusedPort(), 'default', + model_path, use_saved_model, + enable_batching) + time.sleep(5) + + print 'Sending Regress request...' + # Prepare request + request = regression_pb2.RegressionRequest() + request.model_spec.name = 'default' + request.model_spec.signature_name = 'regress_x_to_y' + + example = request.input.example_list.examples.add() + example.features.feature['x'].float_list.value.extend([2.0]) + + # Send request + host, port = model_server_address.split(':') + channel = implementations.insecure_channel(host, int(port)) + stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) + result = stub.Regress(request, 5.0) # 5 secs timeout + # Verify response + self.assertEquals(1, len(result.result.regressions)) + expected_output = 3.0 + self.assertEquals(expected_output, + result.result.regressions[0].value) + def _TestPredict(self, model_path, use_saved_model, enable_batching): """Helper method to test prediction. @@ -273,10 +340,9 @@ def testBadModelConfig(self): last_line = line error_message = ( - 'Check failed: ::tensorflow::Status::OK() == ' - '(ParseProtoTextFile(file, &model_config)) ' - '(OK vs. Invalid argument: ' - 'Invalid protobuf file: \'%s\')') % self._GetBadModelConfigFile() + 'Non-OK-status: ParseProtoTextFile(file, &model_config) status: ' + 'Invalid argument: Invalid protobuf file: \'%s\'') % ( + self._GetBadModelConfigFile()) self.assertNotEqual(last_line, None) self.assertGreater(last_line.find(error_message), 0) From ac6894d3bbeba14b930867315211124a3ca5ed98 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 7 Mar 2017 09:08:03 -0800 Subject: [PATCH 0182/8554] Add batching parameters flag to ModelServer. Change: 149425310 --- tensorflow_serving/model_servers/BUILD | 1 + tensorflow_serving/model_servers/main.cc | 34 +++++++++---- .../tensorflow_model_server_test.py | 48 +++++++++---------- .../servables/tensorflow/testdata/BUILD | 1 + .../tensorflow/testdata/batching_config.txt | 4 ++ 5 files changed, 55 insertions(+), 33 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/testdata/batching_config.txt diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index eb3e83b4b22..08d70c1474b 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -147,6 +147,7 @@ py_test( "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export", "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export.meta", "//tensorflow_serving/servables/tensorflow/testdata:bad_model_config.txt", + "//tensorflow_serving/servables/tensorflow/testdata:batching_config.txt", "//tensorflow_serving/servables/tensorflow/testdata:good_model_config.txt", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 3a78b396637..2c36eb5647c 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -41,6 +41,7 @@ limitations under the License. // To specify model name (default "default"): --model_name=my_name // To specify port (default 8500): --port=my_port // To enable batching (default disabled): --enable_batching +// To override the default batching parameters: --batching_parameters_file #include #include @@ -162,13 +163,13 @@ ModelServerConfig BuildSingleModelConfig( return config; } -ModelServerConfig BuildModelConfigFromFile(const string& file) { - LOG(INFO) << "Building from config file: " << file; - - ModelServerConfig model_config; - TF_CHECK_OK(ParseProtoTextFile(file, &model_config)); - return model_config; +template +ProtoType ReadProtoFromFile(const string& file) { + ProtoType proto; + TF_CHECK_OK(ParseProtoTextFile(file, &proto)); + return proto; } + int DeadlineToTimeoutMillis(const gpr_timespec deadline) { return gpr_time_to_millis( gpr_time_sub(gpr_convert_clock_type(deadline, GPR_CLOCK_MONOTONIC), @@ -285,6 +286,7 @@ tensorflow::serving::PlatformConfigMap ParsePlatformConfigMap( int main(int argc, char** argv) { tensorflow::int32 port = 8500; bool enable_batching = false; + tensorflow::string batching_parameters_file; tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; @@ -300,6 +302,10 @@ int main(int argc, char** argv) { std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), + tensorflow::Flag("batching_parameters_file", &batching_parameters_file, + "If non-empty, read an ascii BatchingParameters " + "protobuf from the supplied file name and use the " + "contained values instead of the defaults."), tensorflow::Flag("model_config_file", &model_config_file, "If non-empty, read an ascii ModelServerConfig " "protobuf from the supplied file name, and serve the " @@ -370,7 +376,8 @@ int main(int argc, char** argv) { options.model_server_config = BuildSingleModelConfig( model_name, model_base_path, parsed_version_policy); } else { - options.model_server_config = BuildModelConfigFromFile(model_config_file); + options.model_server_config = + ReadProtoFromFile(model_config_file); } if (platform_config_file.empty()) { @@ -379,8 +386,17 @@ int main(int argc, char** argv) { if (enable_batching) { BatchingParameters* batching_parameters = session_bundle_config.mutable_batching_parameters(); - batching_parameters->mutable_thread_pool_name()->set_value( - "model_server_batch_threads"); + if (batching_parameters_file.empty()) { + batching_parameters->mutable_thread_pool_name()->set_value( + "model_server_batch_threads"); + } else { + *batching_parameters = + ReadProtoFromFile(batching_parameters_file); + } + } else if (!batching_parameters_file.empty()) { + CHECK(false) // Crash ok + << "You supplied --batching_parameters_file without " + "--enable_batching"; } session_bundle_config.mutable_session_config() diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index d144714d715..67492bdec2d 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -91,7 +91,7 @@ def TerminateProcs(self): self.server_proc.terminate() def RunServer(self, port, model_name, model_path, use_saved_model, - enable_batching): + batching_parameters_file=''): """Run tensorflow_model_server using test config.""" print 'Starting test server...' command = os.path.join(self.binary_dir, 'tensorflow_model_server') @@ -99,7 +99,9 @@ def RunServer(self, port, model_name, model_path, use_saved_model, command += ' --model_name=' + model_name command += ' --model_base_path=' + model_path command += ' --use_saved_model=' + str(use_saved_model).lower() - command += ' --enable_batching=' + str(enable_batching).lower() + if batching_parameters_file: + command += ' --enable_batching' + command += ' --batching_parameters_file=' + batching_parameters_file print command self.server_proc = subprocess.Popen(shlex.split(command)) print 'Server started' @@ -175,16 +177,18 @@ def _GetBadModelConfigFile(self): """Returns a path to a improperly formatted configuration file.""" return os.path.join(self.testdata_dir, 'bad_model_config.txt') + def _GetBatchingParametersFile(self): + """Returns a path to a batching configuration file.""" + return os.path.join(self.testdata_dir, 'batching_config.txt') + def testClassify(self): """Test PredictionService.Classify implementation.""" model_path = self._GetSavedModelBundlePath() use_saved_model = True - enable_batching = False atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', - model_path, use_saved_model, - enable_batching) + model_path, use_saved_model) time.sleep(5) print 'Sending Classify request...' @@ -212,12 +216,10 @@ def testRegress(self): """Test PredictionService.Regress implementation.""" model_path = self._GetSavedModelBundlePath() use_saved_model = True - enable_batching = False atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', - model_path, use_saved_model, - enable_batching) + model_path, use_saved_model) time.sleep(5) print 'Sending Regress request...' @@ -240,18 +242,20 @@ def testRegress(self): self.assertEquals(expected_output, result.result.regressions[0].value) - def _TestPredict(self, model_path, use_saved_model, enable_batching): + def _TestPredict(self, model_path, use_saved_model, + batching_parameters_file=''): """Helper method to test prediction. Args: model_path: Path to the model on disk. use_saved_model: Whether the model server should use SavedModel. - enable_batching: Whether model server should use BatchingSession + batching_parameters_file: Batching parameters file to use (if left empty, + batching is not enabled). """ atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', model_path, use_saved_model, - enable_batching) + batching_parameters_file) time.sleep(5) self.VerifyPredictRequest(model_server_address, expected_output=3.0) self.VerifyPredictRequest( @@ -259,26 +263,25 @@ def _TestPredict(self, model_path, use_saved_model, enable_batching): def testPredictSessionBundle(self): """Test PredictionService.Predict implementation with SessionBundle.""" - self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False, - enable_batching=False) + self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False) def testPredictBatchingSessionBundle(self): """Test PredictionService.Predict implementation with SessionBundle.""" - self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False, - enable_batching=True) + self._TestPredict(self._GetSessionBundlePath(), + use_saved_model=False, + batching_parameters_file= + self._GetBatchingParametersFile()) def testPredictSavedModel(self): """Test PredictionService.Predict implementation with SavedModel.""" - self._TestPredict(self._GetSavedModelBundlePath(), use_saved_model=True, - enable_batching=False) + self._TestPredict(self._GetSavedModelBundlePath(), use_saved_model=True) def testPredictUpconvertedSavedModel(self): """Test PredictionService.Predict implementation. Using a SessionBundle converted to a SavedModel. """ - self._TestPredict(self._GetSessionBundlePath(), use_saved_model=True, - enable_batching=False) + self._TestPredict(self._GetSessionBundlePath(), use_saved_model=True) def _TestBadModel(self, use_saved_model): """Helper method to test against a bad model export.""" @@ -287,8 +290,7 @@ def _TestBadModel(self, use_saved_model): # case of SavedModel, the export will get up-converted to a SavedModel. model_server_address = self.RunServer( PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'bad_half_plus_two'), use_saved_model, - enable_batching=False) + os.path.join(self.testdata_dir, 'bad_half_plus_two'), use_saved_model) time.sleep(5) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address, expected_output=3.0) @@ -340,9 +342,7 @@ def testBadModelConfig(self): last_line = line error_message = ( - 'Non-OK-status: ParseProtoTextFile(file, &model_config) status: ' - 'Invalid argument: Invalid protobuf file: \'%s\'') % ( - self._GetBadModelConfigFile()) + 'Invalid protobuf file: \'%s\'') % self._GetBadModelConfigFile() self.assertNotEqual(last_line, None) self.assertGreater(last_line.find(error_message), 0) diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 69bf0e8c882..8c2982ea68b 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -60,4 +60,5 @@ exports_files([ "bad_half_plus_two/00000123/export.meta", "good_model_config.txt", "bad_model_config.txt", + "batching_config.txt", ]) diff --git a/tensorflow_serving/servables/tensorflow/testdata/batching_config.txt b/tensorflow_serving/servables/tensorflow/testdata/batching_config.txt new file mode 100644 index 00000000000..ef53c87fc60 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/batching_config.txt @@ -0,0 +1,4 @@ +max_batch_size { value: 128 } +batch_timeout_micros { value: 0 } +max_enqueued_batches { value: 1000000 } +num_batch_threads { value: 8 } From f24c7cca18531cbd508dbde187507a7b1a7df2ef Mon Sep 17 00:00:00 2001 From: Li Lao Date: Tue, 7 Mar 2017 12:22:40 -0800 Subject: [PATCH 0183/8554] Update serving_basic.md to be consistent with mnist_saved_model.py. Change: 149450172 --- tensorflow_serving/g3doc/serving_basic.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 12406b399a1..ff18b40c7b1 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -40,12 +40,15 @@ so that it can be loaded later for inference. from tensorflow.python.saved_model import builder as saved_model_builder ... export_path_base = sys.argv[-1] +export_path = os.path.join( + compat.as_bytes(export_path_base), + compat.as_bytes(str(FLAGS.model_version))) print 'Exporting trained model to', export_path builder = saved_model_builder.SavedModelBuilder(export_path) builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ - 'predict': + 'predict_images': prediction_signature, signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: classification_signature, From 681aa4303227234011bc22fb32e95e7ad457da00 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 7 Mar 2017 13:44:33 -0800 Subject: [PATCH 0184/8554] Have BatchingSession use non-crashy tensor concat/split utils. Change: 149460640 --- .../batching/batching_session.cc | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index 99ab0a681a2..f9abc99e238 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -376,7 +376,14 @@ Status BatchingSession::MergeInputTensors( return errors::Internal( "One or more tasks does not conform to batch signature"); } - merged_inputs->push_back({tensor_name, tensor::Concat(tensors->second)}); + Tensor concated; + const Status concat_status = tensor::TryConcat(tensors->second, &concated); + DCHECK(concat_status.ok()) << concat_status.ToString(); + if (!concat_status.ok()) { + return errors::Internal("Tensor concat operation failed: ", + concat_status.ToString()); + } + merged_inputs->push_back({tensor_name, concated}); } return Status::OK(); @@ -426,8 +433,14 @@ Status BatchingSession::SplitOutputTensors( "0th dimension sizes of the input tensors"); } - std::vector split_tensor = - tensor::Split(tensor, task_sizes_plus_optional_padding); + std::vector split_tensor; + const Status split_status = tensor::TrySplit( + tensor, task_sizes_plus_optional_padding, &split_tensor); + DCHECK(split_status.ok()) << split_status.ToString(); + if (!split_status.ok()) { + return errors::Internal("Tensor split operation failed: ", + split_status.ToString()); + } DCHECK_EQ(split_tensor.size(), task_sizes_plus_optional_padding.size()); if (split_tensor.size() != task_sizes_plus_optional_padding.size()) { return errors::Internal( From ffdb3b46c2927a5994e31bdef944f286363228fb Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Wed, 8 Mar 2017 21:07:40 -0800 Subject: [PATCH 0185/8554] Add documentation for use of SignatureDefs with SavedModel in TensorFlow Serving. Change: 149617468 --- tensorflow_serving/g3doc/signature_defs.md | 160 +++++++++++++++++++++ 1 file changed, 160 insertions(+) create mode 100644 tensorflow_serving/g3doc/signature_defs.md diff --git a/tensorflow_serving/g3doc/signature_defs.md b/tensorflow_serving/g3doc/signature_defs.md new file mode 100644 index 00000000000..8e0348cb3c4 --- /dev/null +++ b/tensorflow_serving/g3doc/signature_defs.md @@ -0,0 +1,160 @@ +# SignatureDefs in SavedModel for TensorFlow Serving + +[TOC] + +## Objective + +This document provides examples for the intended usage of SignatureDefs in SavedModel +that map to TensorFlow Serving's APIs. + +## Overview + +A +[SignatureDef](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto) +defines the signature of a computation supported in a TensorFlow graph. +SignatureDefs aim to provide generic support to identify inputs and outputs of a +function and can be specified when building a +[SavedModel](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/builder.py). + +## Background + +[TF-Exporter](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/session_bundle/README.md) +and +[SessionBundle](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/session_bundle/session_bundle.h) +used +[Signatures](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/session_bundle/manifest.proto) +which are similar in concept but required users to distinguish between named and +default signatures in order for them to be retrieved correctly upon a load. For +those who previously used TF-Exporter/SessionBundle, `Signatures` in TF-Exporter +will be replaced by `SignatureDefs` in SavedModel. + +## SignatureDef Structure + +A SignatureDef requires specification of: + +* `inputs` as a map of string to TensorInfo. +* `outputs` as a map of string to TensorInfo. +* `method_name` (which corresponds to a supported method name in the loading + tool/system). + +Note that +[TensorInfo](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto#L194) +itself requires specification of name, dtype and tensor shape. While tensor +information is already present in the graph, it is useful to explicitly have the +TensorInfo defined as part of the SignatureDef since tools can then perform +signature validation, etc. without having to read the graph definition. + +## Related constants and utils + +For ease of reuse and sharing across tools and systems, commonly used constants +related to SignatureDefs that will be supported in TensorFlow Serving are defined as +constants. Specifically: + +* [Signature constants in + Python](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py). +* [Signature constants in + C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/signature_constants.h). + +In addition, SavedModel provides a +[util](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py) +to help build a signature-def: link. + +## Sample structures + +TensorFlow Serving provides high level APIs for performing inference. To enable these APIs, +models must include one or more SignatureDefs that define the exact TensorFlow +nodes to use for input and output. See below for examples of the specific +SignatureDefs that TensorFlow Serving supports for each API. + +Note that TensorFlow Serving depends on the keys of each TensorInfo (in the inputs and +outputs of the SignatureDef), as well as the method_name of the SignatureDef. +The actual contents of the TensorInfo are specific to your graph. + +### Classification SignatureDef + +~~~proto +signature_def: { + key : "my_classification_signature" + value: { + inputs: { + key : "inputs" + value: { + name: "tf_example:0" + dtype: DT_STRING + tensor_shape: ... + } + } + outputs: { + key : "classes" + value: { + name: "index_to_string:0" + dtype: DT_STRING + tensor_shape: ... + } + } + outputs: { + key : "scores" + value: { + name: "TopKV2:0" + dtype: DT_FLOAT + tensor_shape: ... + } + } + method_name: "tensorflow/serving/classify" + } +} +~~~ + +### Predict SignatureDef + +~~~proto +signature_def: { + key : "my_prediction_signature" + value: { + inputs: { + key : "images" + value: { + name: "x:0" + dtype: ... + tensor_shape: ... + } + } + outputs: { + key : "scores" + value: { + name: "y:0" + dtype: ... + tensor_shape: ... + } + } + method_name: "tensorflow/serving/predict" + } +} +~~~ + +### Regression SignatureDef + +~~~proto +signature_def: { + key : "my_regression_signature" + value: { + inputs: { + key : "inputs" + value: { + name: "x_input_examples_tensor_0" + dtype: ... + tensor_shape: ... + } + } + outputs: { + key : "outputs" + value: { + name: "y_outputs_0" + dtype: DT_FLOAT + tensor_shape: ... + } + } + method_name: "tensorflow/serving/regress" + } +} +~~~ From 75251e06a248df6c4fd19dd6eac21ae879fe7206 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Thu, 9 Mar 2017 10:29:33 -0800 Subject: [PATCH 0186/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index e946a6b6397..100552f943c 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit e946a6b63979a63f9e5a1d1603f6cc21d8aad1cf +Subproject commit 100552f943c78cbf90aad521f9981df9b5e3c738 diff --git a/tf_models b/tf_models index 2fd3dcf3f31..1bfe902acd6 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 2fd3dcf3f31707820126a4d9ce595e6a1547385d +Subproject commit 1bfe902acd6d60cbc31a174fb5e9ae9a550ed33d From 53e8d2120fb84dc14138e6cbf4b870cca17ac7b3 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 10 Mar 2017 07:21:40 -0800 Subject: [PATCH 0187/8554] Rename the new TryConcat()/TrySplit() to Concat()/Split() and mark the existing Concat()/Split() as deprecated. This is the first change in a sequence that will remove the crashy Concat()/Split(). Change: 149755537 --- tensorflow_serving/batching/batching_session.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index f9abc99e238..449988ff3b6 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -377,7 +377,7 @@ Status BatchingSession::MergeInputTensors( "One or more tasks does not conform to batch signature"); } Tensor concated; - const Status concat_status = tensor::TryConcat(tensors->second, &concated); + const Status concat_status = tensor::Concat(tensors->second, &concated); DCHECK(concat_status.ok()) << concat_status.ToString(); if (!concat_status.ok()) { return errors::Internal("Tensor concat operation failed: ", @@ -434,8 +434,8 @@ Status BatchingSession::SplitOutputTensors( } std::vector split_tensor; - const Status split_status = tensor::TrySplit( - tensor, task_sizes_plus_optional_padding, &split_tensor); + const Status split_status = + tensor::Split(tensor, task_sizes_plus_optional_padding, &split_tensor); DCHECK(split_status.ok()) << split_status.ToString(); if (!split_status.ok()) { return errors::Internal("Tensor split operation failed: ", From 6607adffa580a63b5a23a95f18bb20461dab50e5 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 10 Mar 2017 10:09:30 -0800 Subject: [PATCH 0188/8554] Add a classification signature to the test saved_model_half_plus_two_model. Test using this classification signature using the Predict API. Change: 149769547 --- tensorflow_serving/core/basic_manager_test.cc | 3 +- .../servables/tensorflow/predict_impl_test.cc | 39 +++++++++++++++++++ 2 files changed, 41 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 97cdac15df7..b0880fd07ea 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -536,7 +536,8 @@ TEST_P(BasicManagerTest, DestructOnNonServingThread) { id, [](const Status& status) { TF_ASSERT_OK(status); }); WaitUntilServableManagerStateIsOneOf( servable_state_monitor_, id, {ServableState::ManagerState::kEnd}); - basic_manager_->StopManagingServable(id); + // TODO(b/35997855): Don't just ignore this status! + basic_manager_->StopManagingServable(id).IgnoreError(); // The servable has been deleted in this thread if there is no // thread-pool for load/unload. if (thread_pool_sizes_.num_load_threads == 0) { diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index ebf2a523281..2e23c315b1a 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -318,6 +318,45 @@ TEST_P(PredictImplTest, PredictionWithNamedRegressionSignature) { EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } +// Test querying a model with a classification signature. Predict calls work +// with predict, classify, and regress signatures when using SavedModel, but +// will only work with a generic signature when using legacy SessionBundle. +TEST_P(PredictImplTest, PredictionWithNamedClassificationSignature) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + model_spec->set_signature_name("classify_x2_to_y3"); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())[kClassifyInputs] = tensor_proto; + + TensorflowPredictor predictor(GetParam()); + // This request is expected to work with SavedModel, but not SessionBundle. + const bool using_session_bundle = !GetParam(); + if (using_session_bundle) { + ASSERT_EQ( + tensorflow::error::INVALID_ARGUMENT, + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) + .code()); + return; + } + TF_ASSERT_OK( + predictor.Predict(GetRunOptions(), GetServerCore(), request, &response)); + TensorProto output_tensor_proto; + output_tensor_proto.add_float_val(4); + output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); + output_tensor_proto.mutable_tensor_shape(); + PredictResponse expected_response; + (*expected_response.mutable_outputs())[kClassifyOutputScores] = + output_tensor_proto; + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + // Test all PredictImplTest test cases with both SessionBundle and SavedModel. INSTANTIATE_TEST_CASE_P(UseSavedModel, PredictImplTest, ::testing::Bool()); From e7f4edd511194f4c3755415124f02fce6e002539 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Fri, 10 Mar 2017 11:05:47 -0800 Subject: [PATCH 0189/8554] Internal change. Change: 149776204 --- tensorflow_serving/g3doc/signature_defs.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow_serving/g3doc/signature_defs.md b/tensorflow_serving/g3doc/signature_defs.md index 8e0348cb3c4..32ac3b95fd4 100644 --- a/tensorflow_serving/g3doc/signature_defs.md +++ b/tensorflow_serving/g3doc/signature_defs.md @@ -1,7 +1,5 @@ # SignatureDefs in SavedModel for TensorFlow Serving -[TOC] - ## Objective This document provides examples for the intended usage of SignatureDefs in SavedModel From 3f3051e1e6b47a8d829f91120f5978897ca333b6 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Mon, 13 Mar 2017 16:08:54 -0700 Subject: [PATCH 0190/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 100552f943c..12a98726e76 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 100552f943c78cbf90aad521f9981df9b5e3c738 +Subproject commit 12a98726e769e988f6368a029ec2f5b0ac3ccbd4 diff --git a/tf_models b/tf_models index 1bfe902acd6..b0ee52c89bf 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 1bfe902acd6d60cbc31a174fb5e9ae9a550ed33d +Subproject commit b0ee52c89bfdb4f012664c1325520f8d0b803017 From aae71de3b5686db4d4e33019bfd9cc29377e927d Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 13 Mar 2017 15:42:43 -0800 Subject: [PATCH 0191/8554] Create a utility for currying Session::Run() inputs. Change: 150008967 --- .../core/test_util/mock_session.h | 7 + tensorflow_serving/servables/tensorflow/BUILD | 31 ++++ .../servables/tensorflow/curried_session.cc | 76 +++++++++ .../servables/tensorflow/curried_session.h | 74 +++++++++ .../tensorflow/curried_session_test.cc | 144 ++++++++++++++++++ 5 files changed, 332 insertions(+) create mode 100644 tensorflow_serving/servables/tensorflow/curried_session.cc create mode 100644 tensorflow_serving/servables/tensorflow/curried_session.h create mode 100644 tensorflow_serving/servables/tensorflow/curried_session_test.cc diff --git a/tensorflow_serving/core/test_util/mock_session.h b/tensorflow_serving/core/test_util/mock_session.h index b7418922077..3bd3200311f 100644 --- a/tensorflow_serving/core/test_util/mock_session.h +++ b/tensorflow_serving/core/test_util/mock_session.h @@ -36,6 +36,13 @@ class MockSession : public tensorflow::Session { const std::vector& output_names, const std::vector& target_nodes, std::vector* outputs)); + MOCK_METHOD6(Run, + ::tensorflow::Status( + const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs, RunMetadata* run_metadata)); MOCK_METHOD4(PRunSetup, ::tensorflow::Status(const std::vector& input_names, const std::vector& output_names, diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 9276d619fc4..33854c148da 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -173,6 +173,7 @@ cc_library( ], deps = [ ":bundle_factory_util", + ":curried_session", ":session_bundle_config_proto", "//tensorflow_serving/batching:batching_session", "//tensorflow_serving/batching:shared_batch_scheduler", @@ -396,6 +397,36 @@ cc_library( ], ) +cc_library( + name = "curried_session", + srcs = ["curried_session.cc"], + hdrs = ["curried_session.h"], + deps = [ + ":serving_session", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + ], +) + +cc_test( + name = "curried_session_test", + size = "small", + srcs = ["curried_session_test.cc"], + deps = [ + ":curried_session", + "//tensorflow_serving/core/test_util:mock_session", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + ], +) + cc_library( name = "predict_impl", srcs = ["predict_impl.cc"], diff --git a/tensorflow_serving/servables/tensorflow/curried_session.cc b/tensorflow_serving/servables/tensorflow/curried_session.cc new file mode 100644 index 00000000000..b46f5034ae5 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/curried_session.cc @@ -0,0 +1,76 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/curried_session.h" + +namespace tensorflow { +namespace serving { + +CurriedSession::CurriedSession( + std::unique_ptr wrapped, + const std::vector>& curried_inputs) + : wrapped_(std::move(wrapped)), curried_inputs_(curried_inputs) { + for (const auto& entry : curried_inputs) { + const string& name = entry.first; + curried_input_names_.insert(name); + } +} + +Status CurriedSession::Run(const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs) { + TF_RETURN_IF_ERROR(ValidateExplicitInputsDontMatchCurriedInputs(inputs)); + const std::vector> combined_inputs = + AddCurriedInputs(inputs); + return wrapped_->Run(combined_inputs, output_tensor_names, target_node_names, + outputs); +} + +Status CurriedSession::Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, + RunMetadata* run_metadata) { + TF_RETURN_IF_ERROR(ValidateExplicitInputsDontMatchCurriedInputs(inputs)); + const std::vector> combined_inputs = + AddCurriedInputs(inputs); + return wrapped_->Run(run_options, combined_inputs, output_tensor_names, + target_node_names, outputs, run_metadata); +} + +Status CurriedSession::ValidateExplicitInputsDontMatchCurriedInputs( + const std::vector>& explicit_inputs) const { + for (const auto& entry : explicit_inputs) { + const string& name = entry.first; + if (curried_input_names_.find(name) != curried_input_names_.end()) { + return errors::InvalidArgument( + "Explicit Run() input has same name as curried input ", name); + } + } + return Status::OK(); +} + +std::vector> CurriedSession::AddCurriedInputs( + const std::vector>& explicit_inputs) const { + std::vector> combined_inputs = explicit_inputs; + std::copy(curried_inputs_.begin(), curried_inputs_.end(), + std::back_inserter(combined_inputs)); + return combined_inputs; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/curried_session.h b/tensorflow_serving/servables/tensorflow/curried_session.h new file mode 100644 index 00000000000..e42d596fd71 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/curried_session.h @@ -0,0 +1,74 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CURRIED_SESSION_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CURRIED_SESSION_H_ + +#include "tensorflow_serving/servables/tensorflow/serving_session.h" + +namespace tensorflow { +namespace serving { + +// A session that wraps another session, while injecting a fixed set of +// additional input tensors into each Run() call. Useful for injecting static +// configuration tensors into a session without requiring the caller to be aware +// of them. +// +// It is an error to call Run() with an input that has the same name as one of +// the curried inputs. +class CurriedSession : public ServingSession { + public: + CurriedSession(std::unique_ptr wrapped, + const std::vector>& curried_inputs); + ~CurriedSession() override = default; + + Status Run(const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs) override; + + Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_tensor_names, + const std::vector& target_node_names, + std::vector* outputs, RunMetadata* run_metadata) override; + + private: + // Verifies no overlap between the tensor names in 'explicit_inputs' and + // 'curried_inputs_'. + Status ValidateExplicitInputsDontMatchCurriedInputs( + const std::vector>& explicit_inputs) const; + + // Adds 'curried_inputs_' to 'explicit_inputs'. + std::vector> AddCurriedInputs( + const std::vector>& explicit_inputs) const; + + // The session to which Run() calls are forwarded (after appending the + // curried inputs). + const std::unique_ptr wrapped_; + + // The inputs that get appended to 'inputs' in each Run() call. + const std::vector> curried_inputs_; + + // The tensor names from 'curried_inputs_'. + std::set curried_input_names_; + + TF_DISALLOW_COPY_AND_ASSIGN(CurriedSession); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CURRIED_SESSION_H_ diff --git a/tensorflow_serving/servables/tensorflow/curried_session_test.cc b/tensorflow_serving/servables/tensorflow/curried_session_test.cc new file mode 100644 index 00000000000..355a8695fac --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/curried_session_test.cc @@ -0,0 +1,144 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/curried_session.h" + +#include "tensorflow/core/framework/tensor_testutil.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/core/test_util/mock_session.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::_; +using ::testing::ElementsAre; +using ::testing::HasSubstr; +using ::testing::Pair; +using ::testing::Return; + +using test_util::EqualsProto; + +MATCHER_P(EqualsTensor, value, "") { + return arg.DebugString() == value.DebugString(); +} + +TEST(RegressorTest, ZeroCurriedInputs) { + const Tensor input = test::AsScalar(0); + + test_util::MockSession* mock = new test_util::MockSession; + auto curried = std::unique_ptr( + new CurriedSession(std::unique_ptr(mock), {})); + + EXPECT_CALL(*mock, Run(ElementsAre(Pair("input", EqualsTensor(input))), + ElementsAre("output"), ElementsAre("target"), _)) + .WillOnce(Return(Status::OK())); + std::vector outputs; + TF_ASSERT_OK( + curried->Run({{"input", input}}, {"output"}, {"target"}, &outputs)); +} + +TEST(RegressorTest, Basic) { + const Tensor input_a = test::AsScalar(0); + const Tensor input_b = test::AsScalar(1); + const Tensor curried_0 = test::AsScalar(2); + const Tensor curried_1 = test::AsScalar(3); + + test_util::MockSession* mock = new test_util::MockSession; + auto curried = std::unique_ptr( + new CurriedSession(std::unique_ptr(mock), + {{"curried_0", curried_0}, {"curried_1", curried_1}})); + + EXPECT_CALL(*mock, + Run(ElementsAre(Pair("input_a", EqualsTensor(input_a)), + Pair("input_b", EqualsTensor(input_b)), + Pair("curried_0", EqualsTensor(curried_0)), + Pair("curried_1", EqualsTensor(curried_1))), + ElementsAre("output_a", "output_b"), + ElementsAre("target_a", "target_b"), _)) + .WillOnce(Return(Status::OK())); + std::vector outputs; + TF_ASSERT_OK(curried->Run({{"input_a", input_a}, {"input_b", input_b}}, + {"output_a", "output_b"}, {"target_a", "target_b"}, + &outputs)); +} + +TEST(RegressorTest, WithOptions) { + RunOptions run_options; + run_options.set_timeout_in_ms(42); + + const Tensor input_a = test::AsScalar(0); + const Tensor input_b = test::AsScalar(1); + const Tensor curried_0 = test::AsScalar(2); + const Tensor curried_1 = test::AsScalar(3); + + test_util::MockSession* mock = new test_util::MockSession; + auto curried = std::unique_ptr( + new CurriedSession(std::unique_ptr(mock), + {{"curried_0", curried_0}, {"curried_1", curried_1}})); + + EXPECT_CALL(*mock, + Run(EqualsProto(run_options), + ElementsAre(Pair("input_a", EqualsTensor(input_a)), + Pair("input_b", EqualsTensor(input_b)), + Pair("curried_0", EqualsTensor(curried_0)), + Pair("curried_1", EqualsTensor(curried_1))), + ElementsAre("output_a", "output_b"), + ElementsAre("target_a", "target_b"), _, _)) + .WillOnce(Return(Status::OK())); + std::vector outputs; + RunMetadata run_metadata; + TF_ASSERT_OK(curried->Run(run_options, + {{"input_a", input_a}, {"input_b", input_b}}, + {"output_a", "output_b"}, {"target_a", "target_b"}, + &outputs, &run_metadata)); +} + +TEST(RegressorTest, ExplicitInputsMatchCurriedInputs) { + const Tensor t0 = test::AsScalar(0); + const Tensor t1 = test::AsScalar(1); + const Tensor t2 = test::AsScalar(2); + + test_util::MockSession* mock = new test_util::MockSession; + auto curried = std::unique_ptr(new CurriedSession( + std::unique_ptr(mock), {{"t0", t0}, {"t1", t1}})); + + EXPECT_CALL(*mock, Run(_, _, _, _)).Times(0); + std::vector outputs; + const Status status = + curried->Run({{"t1", t1}, {"t2", t2}}, {"output"}, {"target"}, &outputs); + ASSERT_FALSE(status.ok()); + EXPECT_THAT( + status.ToString(), + HasSubstr("Explicit Run() input has same name as curried input t1")); +} + +TEST(RegressorTest, PropagateError) { + test_util::MockSession* mock = new test_util::MockSession; + auto curried = std::unique_ptr( + new CurriedSession(std::unique_ptr(mock), {})); + + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillOnce(Return(errors::Unknown("Tensor clog"))); + std::vector outputs; + const Status status = curried->Run({}, {}, {}, &outputs); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), HasSubstr("Tensor clog")); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From 9cd456d81de30e672308b13d78413c0e69fa1dd1 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 13 Mar 2017 16:24:13 -0800 Subject: [PATCH 0192/8554] Adds an experimental feature to append extra tensors to every Session::Run() call. Change: 150013771 --- .../tensorflow/saved_model_bundle_factory.cc | 26 ++++++++++++++++ .../saved_model_bundle_factory_test.cc | 30 ++++++++++++++++++- .../tensorflow/session_bundle_config.proto | 6 ++++ 3 files changed, 61 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc index f96e3c1d762..c4c95eeef0c 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -19,8 +19,10 @@ limitations under the License. #include "tensorflow/contrib/session_bundle/bundle_shim.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow/core/protobuf/named_tensor.pb.h" #include "tensorflow/core/public/session_options.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" +#include "tensorflow_serving/servables/tensorflow/curried_session.h" namespace tensorflow { namespace serving { @@ -37,6 +39,22 @@ std::vector GetSignatureDefs(const SavedModelBundle& bundle) { return signature_defs; } +// Parses a repeated field of NamedTensorProtos into a corresponding list of +// name/tensor pairs. +Status ParseFixedInputTensors( + const proto2::RepeatedPtrField& protos, + std::vector>* parsed) { + for (const NamedTensorProto& proto : protos) { + Tensor tensor; + if (!tensor.FromProto(proto.tensor())) { + return errors::InvalidArgument("Unable to parse tensor proto: ", + proto.tensor().ShortDebugString()); + } + parsed->push_back({proto.name(), tensor}); + } + return Status::OK(); +} + } // namespace Status SavedModelBundleFactory::Create( @@ -62,6 +80,14 @@ Status SavedModelBundleFactory::CreateSavedModelBundle( TF_RETURN_IF_ERROR(LoadSessionBundleOrSavedModelBundle( GetSessionOptions(config_), GetRunOptions(config_), path, {kSavedModelTagServe}, bundle->get())); + if (!config_.experimental_fixed_input_tensors().empty()) { + LOG(INFO) << "Wrapping session to inject fixed input tensors"; + std::vector> fixed_input_tensors; + TF_RETURN_IF_ERROR(ParseFixedInputTensors( + config_.experimental_fixed_input_tensors(), &fixed_input_tensors)); + (*bundle)->session.reset( + new CurriedSession(std::move((*bundle)->session), fixed_input_tensors)); + } if (config_.has_batching_parameters()) { LOG(INFO) << "Wrapping session to perform batch processing"; if (batch_scheduler_ == nullptr) { diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index ac26c951fb3..1d2267f4c4b 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -24,8 +24,10 @@ limitations under the License. #include #include #include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/protobuf/named_tensor.pb.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/public/version.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test.h" @@ -56,7 +58,7 @@ class SavedModelBundleFactoryTest : public test_util::BundleFactoryTest { virtual ~SavedModelBundleFactoryTest() = default; - private: + protected: Status CreateSession(const SessionBundleConfig& config, std::unique_ptr* session) const override { return CreateSessionFromPath(config, export_dir_, session); @@ -65,6 +67,32 @@ class SavedModelBundleFactoryTest : public test_util::BundleFactoryTest { TEST_F(SavedModelBundleFactoryTest, Basic) { TestBasic(); } +TEST_F(SavedModelBundleFactoryTest, FixedInputTensors) { + Tensor fixed_input = test::AsTensor({100.0f, 42.0f}, {2}); + NamedTensorProto fixed_input_proto; + fixed_input_proto.set_name("x:0"); + fixed_input.AsProtoField(fixed_input_proto.mutable_tensor()); + + SessionBundleConfig config; + *config.add_experimental_fixed_input_tensors() = fixed_input_proto; + std::unique_ptr session; + TF_ASSERT_OK(CreateSession(config, &session)); + + // half plus two: output should be input / 2 + 2. + const Tensor expected_output = + test::AsTensor({100.0f / 2 + 2, 42.0f / 2 + 2}, {2}); + + const std::vector> non_fixed_inputs = {}; + const std::vector output_names = {"y:0"}; + const std::vector empty_targets; + std::vector outputs; + TF_ASSERT_OK( + session->Run(non_fixed_inputs, output_names, empty_targets, &outputs)); + ASSERT_EQ(1, outputs.size()); + const Tensor& single_output = outputs.at(0); + test::ExpectTensorEqual(expected_output, single_output); +} + TEST_F(SavedModelBundleFactoryTest, Batching) { TestBatching(); } TEST_F(SavedModelBundleFactoryTest, EstimateResourceRequirementWithGoodExport) { diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto index 76e8aa86c30..602819c88e0 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto +++ b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto @@ -2,6 +2,7 @@ syntax = "proto3"; import "google/protobuf/wrappers.proto"; import "tensorflow/core/protobuf/config.proto"; +import "tensorflow/core/protobuf/named_tensor.proto"; package tensorflow.serving; @@ -39,6 +40,11 @@ message SessionBundleConfig { // correspond to the index of the tensorflow::ThreadPoolOptionProto defined as // part of `session_config.session_inter_op_thread_pool`. google.protobuf.Int32Value session_run_load_threadpool_index = 4; + + // EXPERIMENTAL. THIS FIELD MAY CHANGE OR GO AWAY. USE WITH CAUTION. + // + // Input tensors to append to every Session::Run() call. + repeated NamedTensorProto experimental_fixed_input_tensors = 778; } // Batching parameters. Each individual parameter is optional. If omitted, the From 50bb75e8d1684b8d081cd24af957da69221261e7 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 13 Mar 2017 19:07:07 -0800 Subject: [PATCH 0193/8554] Return error if shape is wrong for classifier call. Change: 150025160 --- tensorflow_serving/servables/tensorflow/classifier.cc | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/classifier.cc b/tensorflow_serving/servables/tensorflow/classifier.cc index 142a98e11af..dd32a150db6 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.cc +++ b/tensorflow_serving/servables/tensorflow/classifier.cc @@ -80,6 +80,11 @@ class TensorFlowClassifier : public ClassifierInterface { // Validate classes output Tensor. if (classes) { + if (classes->dims() != 2) { + return errors::InvalidArgument( + "Expected Tensor shape: [batch_size num_classes] but got ", + classes->shape().DebugString()); + } if (classes->dtype() != DT_STRING) { return errors::Internal("Expected classes Tensor of DT_STRING. Got: ", DataType_Name(classes->dtype())); @@ -363,6 +368,11 @@ Status PostProcessClassificationResult( // Validate classes output Tensor. if (classes) { + if (classes->dims() != 2) { + return errors::InvalidArgument( + "Expected Tensor shape: [batch_size num_classes] but got ", + classes->shape().DebugString()); + } if (classes->dtype() != DT_STRING) { return errors::InvalidArgument( "Expected classes Tensor of DT_STRING. Got: ", From 738f3daacddb5c10a96d6c8aec7f991bc63722de Mon Sep 17 00:00:00 2001 From: Yao Lu Date: Wed, 22 Mar 2017 00:57:10 +0800 Subject: [PATCH 0194/8554] Update README.md (#364) Fix the format error of Markdown in readme file --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4ddfd7516e..3ea0278e69d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#TensorFlow Serving +# TensorFlow Serving [![Build Status](http://ci.tensorflow.org/buildStatus/icon?job=serving-master-cpu)](http://ci.tensorflow.org/job/serving-master-cpu) @@ -37,18 +37,18 @@ points; perhaps the most useful ways to extend the system are: [contribution guidelines](CONTRIBUTING.md).** **We use [GitHub issues](https://github.com/tensorflow/serving/issues) for -tracking requests and bugs. +tracking requests and bugs.** # Download and Setup See [install instructions](tensorflow_serving/g3doc/setup.md). -##Tutorials +## Tutorials * [Basic tutorial](tensorflow_serving/g3doc/serving_basic.md) * [Advanced tutorial](tensorflow_serving/g3doc/serving_advanced.md) -##For more information +## For more information * [Serving architecture overview](tensorflow_serving/g3doc/architecture_overview.md) * [TensorFlow website](http://tensorflow.org) From 273074d45db34e0ba5b6ea98b302514d2dceaee8 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Wed, 15 Mar 2017 10:50:49 -0800 Subject: [PATCH 0195/8554] Merge changes from github. Change: 150221528 --- tensorflow_serving/servables/tensorflow/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 33854c148da..e707971cf64 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -431,6 +431,9 @@ cc_library( name = "predict_impl", srcs = ["predict_impl.cc"], hdrs = ["predict_impl.h"], + visibility = [ + "//visibility:public", + ], deps = [ "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", From 7e90f53516d307707389a8a580c5a4041ff032ad Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 15 Mar 2017 12:20:35 -0800 Subject: [PATCH 0196/8554] Update API documentation Change: 150232894 --- tensorflow_serving/apis/classification.proto | 8 +++----- tensorflow_serving/apis/regression.proto | 8 +++----- 2 files changed, 6 insertions(+), 10 deletions(-) diff --git a/tensorflow_serving/apis/classification.proto b/tensorflow_serving/apis/classification.proto index fa7ca50e0c3..2161b3ac880 100644 --- a/tensorflow_serving/apis/classification.proto +++ b/tensorflow_serving/apis/classification.proto @@ -16,15 +16,13 @@ message Class { float score = 2; } -// List of classes for a single item -// (tensorflow.Example or tensorflow.InferenceExample.features). +// List of classes for a single item (tensorflow.Example). message Classifications { repeated Class classes = 1; } -// For tensorflow.Example this will contain one result. -// For tensorflow.InferenceExample this will contain one result for each -// InferenceExample::features and in the same order as the features. +// Contains one result per input example, in the same order as the input in +// ClassificationRequest. message ClassificationResult { repeated Classifications classifications = 1; } diff --git a/tensorflow_serving/apis/regression.proto b/tensorflow_serving/apis/regression.proto index 48d8513cfe9..8864300cd8f 100644 --- a/tensorflow_serving/apis/regression.proto +++ b/tensorflow_serving/apis/regression.proto @@ -7,15 +7,13 @@ import "tensorflow_serving/apis/model.proto"; package tensorflow.serving; -// Regression result for a single item -// (tensorflow.Example or tensorflow.InferenceExample.features). +// Regression result for a single item (tensorflow.Example). message Regression { float value = 1; } -// For tensorflow.Example this will contain one result. -// For tensorflow.InferenceExample this will contain one result for each -// InferenceExample::features. +// Contains one result per input example, in the same order as the input in +// RegressionRequest. message RegressionResult { repeated Regression regressions = 1; } From 87af5e4288cc60aa928e8a5abbe433a8ca3d7e8a Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Wed, 15 Mar 2017 16:36:07 -0800 Subject: [PATCH 0197/8554] Creates ServerCores in tests with separate loading threadpool. Change: 150266857 --- .../test_util/server_core_test_util.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 6805de0b80d..ced73ed791e 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -31,6 +31,14 @@ namespace test_util { namespace { +void AddSessionRunLoadThreadPool(SessionBundleConfig* const bundle_config) { + auto* const session_config = bundle_config->mutable_session_config(); + session_config->add_session_inter_op_thread_pool(); + // The second pool will be used for loading. + session_config->add_session_inter_op_thread_pool()->set_num_threads(4); + bundle_config->mutable_session_run_load_threadpool_index()->set_value(1); +} + ServerCore::Options GetDefaultOptions(const bool use_saved_model) { ServerCore::Options options; options.file_system_poll_wait_seconds = 0; @@ -45,8 +53,11 @@ ServerCore::Options GetDefaultOptions(const bool use_saved_model) { return Status::OK(); }; + SessionBundleConfig bundle_config; + AddSessionRunLoadThreadPool(&bundle_config); + options.platform_config_map = - CreateTensorFlowPlatformConfigMap(SessionBundleConfig(), use_saved_model); + CreateTensorFlowPlatformConfigMap(bundle_config, use_saved_model); ::google::protobuf::Any fake_source_adapter_config; fake_source_adapter_config.PackFrom( test_util::FakeLoaderSourceAdapterConfig()); From 5d1cc352cfc733596f8624f74305953ad17bcd6c Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Wed, 15 Mar 2017 18:07:47 -0800 Subject: [PATCH 0198/8554] Adds lazy annotations for tensorflow.serving.Input. Change: 150274074 --- tensorflow_serving/apis/input.proto | 4 +- tensorflow_serving/apis/internal/BUILD | 22 ++++++ .../apis/internal/serialized_input.proto | 45 ++++++++++++ tensorflow_serving/servables/tensorflow/BUILD | 1 + .../servables/tensorflow/classifier.cc | 7 +- .../servables/tensorflow/classifier_test.cc | 6 +- .../servables/tensorflow/regressor.cc | 7 +- .../servables/tensorflow/regressor_test.cc | 6 +- .../servables/tensorflow/util.cc | 41 +++++++---- .../servables/tensorflow/util.h | 5 +- .../servables/tensorflow/util_test.cc | 71 +++++++++++++------ 11 files changed, 156 insertions(+), 59 deletions(-) create mode 100644 tensorflow_serving/apis/internal/BUILD create mode 100644 tensorflow_serving/apis/internal/serialized_input.proto diff --git a/tensorflow_serving/apis/input.proto b/tensorflow_serving/apis/input.proto index d79af2e5fe1..2fffb526fcf 100644 --- a/tensorflow_serving/apis/input.proto +++ b/tensorflow_serving/apis/input.proto @@ -70,7 +70,7 @@ message ExampleListWithContext { message Input { oneof kind { - ExampleList example_list = 1; - ExampleListWithContext example_list_with_context = 2; + ExampleList example_list = 1 [lazy = true]; + ExampleListWithContext example_list_with_context = 2 [lazy = true]; } } diff --git a/tensorflow_serving/apis/internal/BUILD b/tensorflow_serving/apis/internal/BUILD new file mode 100644 index 00000000000..6ce2e32a915 --- /dev/null +++ b/tensorflow_serving/apis/internal/BUILD @@ -0,0 +1,22 @@ +# Internal implementation details of serving APIs. + +package( + default_visibility = [ + "//tensorflow_serving:internal", + ], + features = ["-layering_check"], +) + +licenses(["notice"]) # Apache 2.0 + +exports_files(["LICENSE"]) + +load("//tensorflow_serving:serving.bzl", "serving_proto_library") + +serving_proto_library( + name = "serialized_input_proto", + srcs = ["serialized_input.proto"], + cc_api_version = 2, + deps = [ + ], +) diff --git a/tensorflow_serving/apis/internal/serialized_input.proto b/tensorflow_serving/apis/internal/serialized_input.proto new file mode 100644 index 00000000000..36c026a76c8 --- /dev/null +++ b/tensorflow_serving/apis/internal/serialized_input.proto @@ -0,0 +1,45 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +// Serialized counterparts of the messages in input.proto. These protos enable +// us to keep the original tensorflow.serving.Input's structure but with the +// tensorflow.Examples in their serialized form. When combined with lazy +// parsing, this improves performance by allowing us to skip a redundant +// deserialization/serialization loop. +// +// WARNING: These are internal implementation details and not part of the public +// API. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +package tensorflow.serving.internal; + +message SerializedExampleList { + repeated bytes examples = 1; +} + +message SerializedExampleListWithContext { + repeated bytes examples = 1; + bytes context = 2; +} + +message SerializedInput { + oneof kind { + SerializedExampleList example_list = 1; + SerializedExampleListWithContext example_list_with_context = 2; + } +} diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index e707971cf64..180c819e848 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -670,6 +670,7 @@ cc_library( ], deps = [ "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis/internal:serialized_input_proto", "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", diff --git a/tensorflow_serving/servables/tensorflow/classifier.cc b/tensorflow_serving/servables/tensorflow/classifier.cc index dd32a150db6..ae9c1e92990 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.cc +++ b/tensorflow_serving/servables/tensorflow/classifier.cc @@ -159,10 +159,6 @@ class SavedModelTensorFlowClassifier : public ClassifierInterface { Status Classify(const ClassificationRequest& request, ClassificationResult* result) override { TRACELITERAL("TensorFlowClassifier::Classify"); - const int num_examples = NumInputExamples(request.input()); - if (num_examples == 0) { - return errors::InvalidArgument("ClassificationRequest::input is empty."); - } string input_tensor_name; std::vector output_tensor_names; @@ -170,9 +166,10 @@ class SavedModelTensorFlowClassifier : public ClassifierInterface { &output_tensor_names)); std::vector outputs; + int num_examples; TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( request.input(), input_tensor_name, output_tensor_names, session_, - &outputs)); + &outputs, &num_examples)); TRACELITERAL("ConvertToClassificationResult"); return PostProcessClassificationResult( diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc index 10ef40e50d7..b69da132215 100644 --- a/tensorflow_serving/servables/tensorflow/classifier_test.cc +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -572,7 +572,7 @@ TEST_P(ClassifierTest, EmptyInput) { const Status status = classifier_->Classify(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("ClassificationRequest::input is empty")); + ::testing::HasSubstr("Invalid argument: Input is empty")); } TEST_P(ClassifierTest, EmptyExampleList) { @@ -582,7 +582,7 @@ TEST_P(ClassifierTest, EmptyExampleList) { const Status status = classifier_->Classify(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("ClassificationRequest::input is empty")); + ::testing::HasSubstr("Invalid argument: Input is empty")); } TEST_P(ClassifierTest, EmptyExampleListWithContext) { @@ -594,7 +594,7 @@ TEST_P(ClassifierTest, EmptyExampleListWithContext) { const Status status = classifier_->Classify(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("ClassificationRequest::input is empty")); + ::testing::HasSubstr("Invalid argument: Input is empty")); } TEST_P(ClassifierTest, RunsFails) { diff --git a/tensorflow_serving/servables/tensorflow/regressor.cc b/tensorflow_serving/servables/tensorflow/regressor.cc index 37f4f192532..b54477760b6 100644 --- a/tensorflow_serving/servables/tensorflow/regressor.cc +++ b/tensorflow_serving/servables/tensorflow/regressor.cc @@ -107,10 +107,6 @@ class SavedModelTensorFlowRegressor : public RegressorInterface { Status Regress(const RegressionRequest& request, RegressionResult* result) override { TRACELITERAL("SavedModelTensorFlowRegressor::Regress"); - const int num_examples = NumInputExamples(request.input()); - if (num_examples == 0) { - return errors::InvalidArgument("RegressionRequest::input is empty."); - } string input_tensor_name; std::vector output_tensor_names; @@ -118,9 +114,10 @@ class SavedModelTensorFlowRegressor : public RegressorInterface { &output_tensor_names)); std::vector outputs; + int num_examples; TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( request.input(), input_tensor_name, output_tensor_names, session_, - &outputs)); + &outputs, &num_examples)); TRACELITERAL("ConvertToRegressionResult"); return PostProcessRegressionResult(*signature_, num_examples, diff --git a/tensorflow_serving/servables/tensorflow/regressor_test.cc b/tensorflow_serving/servables/tensorflow/regressor_test.cc index e57148be6ff..c7550435f93 100644 --- a/tensorflow_serving/servables/tensorflow/regressor_test.cc +++ b/tensorflow_serving/servables/tensorflow/regressor_test.cc @@ -326,7 +326,7 @@ TEST_P(RegressorTest, EmptyInput) { const Status status = regressor_->Regress(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("RegressionRequest::input is empty")); + ::testing::HasSubstr("Invalid argument: Input is empty")); } TEST_P(RegressorTest, EmptyExampleList) { @@ -335,7 +335,7 @@ TEST_P(RegressorTest, EmptyExampleList) { const Status status = regressor_->Regress(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("RegressionRequest::input is empty")); + ::testing::HasSubstr("Invalid argument: Input is empty")); } TEST_P(RegressorTest, EmptyExampleListWithContext) { @@ -347,7 +347,7 @@ TEST_P(RegressorTest, EmptyExampleListWithContext) { const Status status = regressor_->Regress(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("RegressionRequest::input is empty")); + ::testing::HasSubstr("Invalid argument: Input is empty")); } TEST_P(RegressorTest, RunsFails) { diff --git a/tensorflow_serving/servables/tensorflow/util.cc b/tensorflow_serving/servables/tensorflow/util.cc index b455ea0d7fa..69cdb221cc7 100644 --- a/tensorflow_serving/servables/tensorflow/util.cc +++ b/tensorflow_serving/servables/tensorflow/util.cc @@ -21,11 +21,14 @@ limitations under the License. #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/internal/serialized_input.pb.h" namespace tensorflow { namespace serving { +namespace { -int NumInputExamples(const Input& input) { +// Returns the number of examples in the Input. +int NumInputExamples(const internal::SerializedInput& input) { switch (input.kind_case()) { case Input::KindCase::kExampleList: return input.example_list().examples_size(); @@ -36,39 +39,48 @@ int NumInputExamples(const Input& input) { } return 0; } +} // namespace Status InputToSerializedExampleTensor(const Input& input, Tensor* examples) { - const int n = NumInputExamples(input); - *examples = Tensor(DT_STRING, TensorShape({n})); - switch (input.kind_case()) { + const string serialized_input_str = input.SerializeAsString(); + internal::SerializedInput serialized_input; + if (!serialized_input.ParseFromString(serialized_input_str)) { + return errors::Internal("Error parsing serialized input."); + } + const int64 num_examples = NumInputExamples(serialized_input); + if (num_examples == 0) { + return errors::InvalidArgument("Input is empty."); + } + *examples = Tensor(DT_STRING, TensorShape({num_examples})); + switch (serialized_input.kind_case()) { case Input::KindCase::KIND_NOT_SET: break; case Input::KindCase::kExampleList: { auto input_vec = examples->vec(); int input_vec_index = 0; - for (const auto& entry : input.example_list().examples()) { - input_vec(input_vec_index++) = entry.SerializeAsString(); + for (const auto& entry : serialized_input.example_list().examples()) { + input_vec(input_vec_index++) = entry; } break; } case Input::KindCase::kExampleListWithContext: { - const string context = - input.example_list_with_context().context().SerializeAsString(); + const string& context = + serialized_input.example_list_with_context().context(); auto input_vec = examples->vec(); int input_vec_index = 0; - for (const auto& entry : input.example_list_with_context().examples()) { + for (const auto& entry : + serialized_input.example_list_with_context().examples()) { // Avoid the need for repeated serialization of context by simply // appending the Example serialization to the pre-serialized context. - input_vec(input_vec_index++) = - strings::StrCat(context, entry.SerializeAsString()); + input_vec(input_vec_index++) = strings::StrCat(context, entry); } } break; default: - return errors::Unimplemented("Input with kind ", input.kind_case(), - " not supported."); + return errors::Unimplemented( + "Input with kind ", serialized_input.kind_case(), " not supported."); } return Status::OK(); } @@ -76,11 +88,12 @@ Status InputToSerializedExampleTensor(const Input& input, Tensor* examples) { Status PerformOneShotTensorComputation( const Input& input, const string& input_tensor_name, const std::vector& output_tensor_names, Session* session, - std::vector* outputs) { + std::vector* outputs, int* num_input_examples) { // Setup the input Tensor to be a vector of string containing the serialized // tensorflow.Example. Tensor input_tensor; TF_RETURN_IF_ERROR(InputToSerializedExampleTensor(input, &input_tensor)); + *num_input_examples = input_tensor.dim_size(0); return session->Run({{input_tensor_name, input_tensor}}, output_tensor_names, {}, outputs); diff --git a/tensorflow_serving/servables/tensorflow/util.h b/tensorflow_serving/servables/tensorflow/util.h index 9cfc0192da2..5e797edbe23 100644 --- a/tensorflow_serving/servables/tensorflow/util.h +++ b/tensorflow_serving/servables/tensorflow/util.h @@ -24,9 +24,6 @@ limitations under the License. namespace tensorflow { namespace serving { -// Returns the number of examples in the Input. -int NumInputExamples(const Input& input); - // InputToSerializedExampleTensor populates a string Tensor of serialized // Examples. // If input has n Examples returns a string Tensor with shape {n}. @@ -47,7 +44,7 @@ Status InputToSerializedExampleTensor(const Input& input, Tensor* examples); Status PerformOneShotTensorComputation( const Input& input, const string& input_tensor_name, const std::vector& output_tensor_names, Session* session, - std::vector* outputs); + std::vector* outputs, int* num_input_examples); } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/util_test.cc b/tensorflow_serving/servables/tensorflow/util_test.cc index 0bd44117990..89ce28ca965 100644 --- a/tensorflow_serving/servables/tensorflow/util_test.cc +++ b/tensorflow_serving/servables/tensorflow/util_test.cc @@ -26,6 +26,7 @@ namespace tensorflow { namespace serving { namespace { +using ::testing::HasSubstr; using test_util::EqualsProto; class InputUtilTest : public ::testing::Test { @@ -47,9 +48,9 @@ class InputUtilTest : public ::testing::Test { return example; } - Example example_C() { + Example example_C(const int64 value = 33) { Feature feature; - feature.mutable_int64_list()->add_value(33); + feature.mutable_int64_list()->add_value(value); Example example; (*example.mutable_features()->mutable_feature())["c"] = feature; return example; @@ -60,36 +61,33 @@ class InputUtilTest : public ::testing::Test { }; TEST_F(InputUtilTest, Empty_KindNotSet) { - EXPECT_EQ(0, NumInputExamples(input_)); - TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); - test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), - tensor_); + const Status status = InputToSerializedExampleTensor(input_, &tensor_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.error_message(), HasSubstr("Input is empty")); } TEST_F(InputUtilTest, Empty_ExampleList) { input_.mutable_example_list(); - EXPECT_EQ(0, NumInputExamples(input_)); - TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); - test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), - tensor_); + const Status status = InputToSerializedExampleTensor(input_, &tensor_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.error_message(), HasSubstr("Input is empty")); } TEST_F(InputUtilTest, Empty_ExampleListWithContext) { input_.mutable_example_list_with_context(); - EXPECT_EQ(0, NumInputExamples(input_)); - TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); - test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), - tensor_); + const Status status = InputToSerializedExampleTensor(input_, &tensor_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.error_message(), HasSubstr("Input is empty")); } TEST_F(InputUtilTest, ExampleList) { *input_.mutable_example_list()->mutable_examples()->Add() = example_A(); *input_.mutable_example_list()->mutable_examples()->Add() = example_B(); - EXPECT_EQ(2, NumInputExamples(input_)); TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + EXPECT_EQ(2, tensor_.NumElements()); const auto vec = tensor_.flat(); ASSERT_EQ(vec.size(), 2); Example serialized_example; @@ -106,8 +104,8 @@ TEST_F(InputUtilTest, ExampleListWithContext) { *examples->Add() = example_B(); *input_.mutable_example_list_with_context()->mutable_context() = example_C(); - EXPECT_EQ(2, NumInputExamples(input_)); TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + EXPECT_EQ(2, tensor_.NumElements()); const auto vec = tensor_.flat(); ASSERT_EQ(vec.size(), 2); { @@ -128,14 +126,42 @@ TEST_F(InputUtilTest, ExampleListWithContext) { } } +// Tests whether individual examples do override the context. +TEST_F(InputUtilTest, ExampleListWithOverridingContext) { + auto* examples = + input_.mutable_example_list_with_context()->mutable_examples(); + *examples->Add() = example_A(); + *examples->Add() = example_C(64); + *input_.mutable_example_list_with_context()->mutable_context() = example_C(); + + TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + EXPECT_EQ(2, tensor_.NumElements()); + const auto vec = tensor_.flat(); + ASSERT_EQ(vec.size(), 2); + { + Example serialized_example; + ASSERT_TRUE(serialized_example.ParseFromString(vec(0))); + EXPECT_THAT(serialized_example.features().feature().at("c"), + EqualsProto(example_C().features().feature().at("c"))); + EXPECT_THAT(serialized_example.features().feature().at("a"), + EqualsProto(example_A().features().feature().at("a"))); + } + { + Example serialized_example; + ASSERT_TRUE(serialized_example.ParseFromString(vec(1))); + EXPECT_THAT(serialized_example.features().feature().at("c"), + EqualsProto(example_C(64).features().feature().at("c"))); + } +} + TEST_F(InputUtilTest, ExampleListWithContext_NoContext) { auto* examples = input_.mutable_example_list_with_context()->mutable_examples(); *examples->Add() = example_A(); *examples->Add() = example_B(); - EXPECT_EQ(2, NumInputExamples(input_)); TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); + EXPECT_EQ(2, tensor_.NumElements()); const auto vec = tensor_.flat(); ASSERT_EQ(vec.size(), 2); { @@ -155,25 +181,24 @@ TEST_F(InputUtilTest, ExampleListWithContext_OnlyContext) { // context is specified). *input_.mutable_example_list_with_context()->mutable_context() = example_C(); - EXPECT_EQ(0, NumInputExamples(input_)); - TF_ASSERT_OK(InputToSerializedExampleTensor(input_, &tensor_)); - test::ExpectTensorEqual(test::AsTensor({}, TensorShape({0})), - tensor_); + const Status status = InputToSerializedExampleTensor(input_, &tensor_); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.error_message(), HasSubstr("Input is empty")); } TEST_F(InputUtilTest, RequestNumExamplesStreamz) { Input input_1; *input_1.mutable_example_list()->mutable_examples()->Add() = example_A(); *input_1.mutable_example_list()->mutable_examples()->Add() = example_B(); - EXPECT_EQ(2, NumInputExamples(input_1)); Tensor tensor_1; TF_ASSERT_OK(InputToSerializedExampleTensor(input_1, &tensor_1)); + EXPECT_EQ(2, tensor_1.NumElements()); Input input_2; *input_2.mutable_example_list()->mutable_examples()->Add() = example_C(); - EXPECT_EQ(1, NumInputExamples(input_2)); Tensor tensor_2; TF_ASSERT_OK(InputToSerializedExampleTensor(input_2, &tensor_2)); + EXPECT_EQ(1, tensor_2.NumElements()); } } // namespace From 9fe41be1a5518f99fbb0ae7790e36603b4f57254 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 16 Mar 2017 07:48:51 -0800 Subject: [PATCH 0199/8554] Make SourceAdapter::Adapt() public and add an AdaptOneVersion() utility, to facilitate using a SourceAdapter in a synchronous fashion (e.g. outside of a source->manager chain). Change: 150324258 --- tensorflow_serving/core/source_adapter.h | 24 +++++++++++++++---- .../core/source_adapter_test.cc | 9 +++++++ 2 files changed, 28 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/core/source_adapter.h b/tensorflow_serving/core/source_adapter.h index c7d9999f92a..839896eb355 100644 --- a/tensorflow_serving/core/source_adapter.h +++ b/tensorflow_serving/core/source_adapter.h @@ -66,17 +66,20 @@ class SourceAdapter : public TargetBase, public Source { void SetAspiredVersionsCallback( typename Source::AspiredVersionsCallback callback) final; - protected: - // This is an abstract class. - SourceAdapter() = default; - - private: // Given an InputType-based aspired-versions request, produces a corresponding // OutputType-based request. virtual std::vector> Adapt( const StringPiece servable_name, std::vector> versions) = 0; + // Adapts a single servable data item. (Implemented on top of Adapt().) + ServableData AdaptOneVersion(ServableData input); + + protected: + // This is an abstract class. + SourceAdapter() = default; + + private: // The callback for emitting OutputType-based aspired-version lists. typename Source::AspiredVersionsCallback outgoing_callback_; @@ -180,6 +183,17 @@ void SourceAdapter::SetAspiredVersionsCallback( outgoing_callback_set_.Notify(); } +template +ServableData SourceAdapter::AdaptOneVersion( + ServableData input) { + const StringPiece servable_name(input.id().name); + std::vector> input_versions = {input}; + std::vector> output_versions = + Adapt(servable_name, input_versions); + DCHECK_EQ(1, output_versions.size()); + return std::move(output_versions[0]); +} + template UnarySourceAdapter::~UnarySourceAdapter() {} diff --git a/tensorflow_serving/core/source_adapter_test.cc b/tensorflow_serving/core/source_adapter_test.cc index 8a9253fae54..1cc3503dee0 100644 --- a/tensorflow_serving/core/source_adapter_test.cc +++ b/tensorflow_serving/core/source_adapter_test.cc @@ -58,6 +58,15 @@ class LimitedAdapter final : public SourceAdapter { TF_DISALLOW_COPY_AND_ASSIGN(LimitedAdapter); }; +TEST(SourceAdapterTest, AdaptOneVersion) { + test_util::FakeStoragePathSourceAdapter adapter("baz"); + ServableData output = + adapter.AdaptOneVersion(ServableData({"foo", 42}, "bar")); + EXPECT_EQ("foo", output.id().name); + EXPECT_EQ(42, output.id().version); + EXPECT_EQ("bar/baz", output.DataOrDie()); +} + TEST(SourceAdapterTest, SetAspiredVersionsBlocksUntilTargetConnected) { LimitedAdapter adapter; std::unique_ptr target( From a7373c23f4a0b61d91a7dfbdd5c46d9fa72d9f50 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 16 Mar 2017 08:07:56 -0800 Subject: [PATCH 0200/8554] Add a CachingManager::LoaderFactory implementation that looks for a given servable at a location given by a path prefix concatenated with the servable bame. That implementation ought to be suitable for many use cases, and it also provides a concrete example of how to build a LoaderFactory. Also streamlines the way the API handles the ServableData> objects, and encapsulates the error handling inside the ServableData (which gets propagated to the underlying BasicManager). Change: 150326516 --- tensorflow_serving/core/BUILD | 2 + tensorflow_serving/core/caching_manager.cc | 32 +++++++++-- tensorflow_serving/core/caching_manager.h | 37 +++++++++--- .../core/caching_manager_test.cc | 57 ++++++++++++++----- 4 files changed, 101 insertions(+), 27 deletions(-) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 066fef86c80..323c1b3af69 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -586,6 +586,7 @@ cc_library( ":servable_data", ":servable_handle", ":servable_id", + ":source_adapter", "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/core:lib", ], @@ -602,6 +603,7 @@ cc_test( ":servable_state", ":servable_state_monitor", ":simple_loader", + "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/core/test_util:manager_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:event_bus", diff --git a/tensorflow_serving/core/caching_manager.cc b/tensorflow_serving/core/caching_manager.cc index 703dc62cd30..3856c50ba7b 100644 --- a/tensorflow_serving/core/caching_manager.cc +++ b/tensorflow_serving/core/caching_manager.cc @@ -86,8 +86,8 @@ Status CachingManager::GetUntypedServableHandleForId( } // Build the servable data corresponding to the servable-id. - std::unique_ptr>> loader_data; - TF_RETURN_IF_ERROR(loader_factory_->CreateLoader(servable_id, &loader_data)); + ServableData> loader_data = + loader_factory_->CreateLoader(servable_id); // Load the servable corresponding to the servable-id. For multiple concurrent // requests enforces that exactly one thread performs the load operation with @@ -101,8 +101,8 @@ Status CachingManager::GetUntypedServableHandleForId( } Status CachingManager::LoadServable( - std::unique_ptr>> loader_data) { - const ServableId servable_id = loader_data->id(); + ServableData> loader_data) { + const ServableId servable_id = loader_data.id(); std::shared_ptr servable_id_mu; { @@ -145,7 +145,7 @@ Status CachingManager::LoadServable( // automatically available in the caching-manager as well (via the basic // manager). const Status manage_status = - basic_manager_->ManageServable(std::move(*loader_data)); + basic_manager_->ManageServable(std::move(loader_data)); if (!manage_status.ok()) { const string error_msg = strings::StrCat( "Internal error: unable to transfer servable to 'basic_manager_': ", @@ -194,5 +194,27 @@ std::vector CachingManager::ListAvailableServableIds() const { return basic_manager_->ListAvailableServableIds(); } +PathPrefixLoaderFactory::PathPrefixLoaderFactory( + const string& path_prefix, + std::unique_ptr adapter) + : path_prefix_(path_prefix), adapter_(std::move(adapter)) {} + +ServableData> PathPrefixLoaderFactory::CreateLoader( + const ServableId& id) { + if (id.version != 0) { + return ServableData>( + id, + errors::FailedPrecondition("PathPrefixLoaderFactory only supports " + "single-version servables at version 0")); + } + const StoragePath servable_path = io::JoinPath(path_prefix_, id.name); + return adapter_->AdaptOneVersion({id, servable_path}); +} + +int64 PathPrefixLoaderFactory::GetLatestVersion( + const string& servable_name) const { + return 0; +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/caching_manager.h b/tensorflow_serving/core/caching_manager.h index a1283c56e96..434d0dfc762 100644 --- a/tensorflow_serving/core/caching_manager.h +++ b/tensorflow_serving/core/caching_manager.h @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow_serving/core/basic_manager.h" #include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/source_adapter.h" namespace tensorflow { namespace serving { @@ -82,11 +83,10 @@ class CachingManager : public Manager { virtual ~LoaderFactory() = default; // Creates servable data consisting of the loader corresponding to the - // servable-id. - virtual Status CreateLoader( - const ServableId& servable_id, - std::unique_ptr>>* - loader_data) = 0; + // servable-id. Any errors can be reported by embedding them in the returned + // ServableData item. + virtual ServableData> CreateLoader( + const ServableId& servable_id) = 0; // Returns the latest version corresponding to the servable name. virtual int64 GetLatestVersion(const string& servable_name) const = 0; @@ -129,8 +129,7 @@ class CachingManager : public Manager { // exactly one thread performs the load operation using the wrapped // basic-manager. All other requests block until the load completes and then // trivially succeed. - Status LoadServable( - std::unique_ptr>> loader_data) + Status LoadServable(ServableData> loader_data) LOCKS_EXCLUDED(load_mutex_map_mu_); // Returns the size of the load_mutex_map_. @@ -157,6 +156,30 @@ class CachingManager : public Manager { TF_DISALLOW_COPY_AND_ASSIGN(CachingManager); }; +// A simple LoaderFactory that looks for a servable at a path formed by +// concatenating a fixed path prefix with the servable's name. It assumes that +// a given servable only has one version, namely version 0. +class PathPrefixLoaderFactory : public CachingManager::LoaderFactory { + public: + PathPrefixLoaderFactory(const string& path_prefix, + std::unique_ptr adapter); + ~PathPrefixLoaderFactory() override = default; + + ServableData> CreateLoader( + const ServableId& id) override; + + int64 GetLatestVersion(const string& servable_name) const override; + + private: + // The prefix of the path to the servables. + const string path_prefix_; + + // An adapter for creating a loader from a given path. + const std::unique_ptr adapter_; + + TF_DISALLOW_COPY_AND_ASSIGN(PathPrefixLoaderFactory); +}; + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/caching_manager_test.cc b/tensorflow_serving/core/caching_manager_test.cc index cdd01672f58..6c3de1e33db 100644 --- a/tensorflow_serving/core/caching_manager_test.cc +++ b/tensorflow_serving/core/caching_manager_test.cc @@ -30,6 +30,7 @@ limitations under the License. #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/simple_loader.h" +#include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/core/test_util/manager_test_util.h" #include "tensorflow_serving/util/event_bus.h" #include "tensorflow_serving/util/optional.h" @@ -39,6 +40,7 @@ namespace tensorflow { namespace serving { namespace { +using ::testing::HasSubstr; using ::testing::UnorderedElementsAre; using ::testing::UnorderedElementsAreArray; @@ -51,9 +53,14 @@ class StringLoaderFactory : public CachingManager::LoaderFactory { ~StringLoaderFactory() override = default; - Status CreateLoader(const ServableId& id, - std::unique_ptr>>* - loaded_data) override { + ServableData> CreateLoader( + const ServableId& id) override { + // Update state to indicate a new loader was created. + { + mutex_lock l(mu_); + num_loaders_dispensed_++; + } + auto servable_creator = [&](std::unique_ptr* servable) { servable->reset(new string); **servable = strings::StrCat(id.name, "-", id.version); @@ -62,12 +69,7 @@ class StringLoaderFactory : public CachingManager::LoaderFactory { std::unique_ptr loader; loader.reset(new SimpleLoader( servable_creator, SimpleLoader::EstimateNoResources())); - loaded_data->reset( - new ServableData>(id, std::move(loader))); - // Update state to indicate a new loader was created. - mutex_lock l(mu_); - num_loaders_dispensed_++; - return Status::OK(); + return ServableData>(id, std::move(loader)); } // Returns the latest version corresponding to the servable name. @@ -109,18 +111,15 @@ class ErrorLoaderFactory : public CachingManager::LoaderFactory { ErrorLoaderFactory() = default; ~ErrorLoaderFactory() override = default; - Status CreateLoader(const ServableId& id, - std::unique_ptr>>* - loaded_data) override { + ServableData> CreateLoader( + const ServableId& id) override { auto servable_creator = [&](std::unique_ptr* servable) { return errors::Unknown("error loader-factory"); }; std::unique_ptr loader; loader.reset(new SimpleLoader( servable_creator, SimpleLoader::EstimateNoResources())); - loaded_data->reset( - new ServableData>(id, std::move(loader))); - return Status::OK(); + return ServableData>(id, std::move(loader)); } int64 GetLatestVersion(const string& request_name) const override { @@ -523,6 +522,34 @@ TEST_P(CachingManagerTest, ConcurrentIntersectingRequests) { /////////////////////////////////////////////////////////////////////////////// +TEST(PathPrefixLoaderFactoryTest, Basic) { + auto adapter = std::unique_ptr( + new test_util::FakeLoaderSourceAdapter("suffix")); + PathPrefixLoaderFactory factory("prefix", std::move(adapter)); + + ServableData> loader_data = + factory.CreateLoader({"servable_name", 0}); + TF_ASSERT_OK(loader_data.status()); + std::unique_ptr loader = loader_data.ConsumeDataOrDie(); + TF_ASSERT_OK(loader->Load()); + EXPECT_EQ("prefix/servable_name/suffix", *loader->servable().get()); + + EXPECT_EQ(0, factory.GetLatestVersion("blah")); +} + +TEST(PathPrefixLoaderFactoryTest, VersionOtherThanZeroYieldsError) { + auto adapter = std::unique_ptr( + new test_util::FakeLoaderSourceAdapter("suffix")); + PathPrefixLoaderFactory factory("prefix", std::move(adapter)); + + ServableData> loader_data = + factory.CreateLoader({"servable_name", 42}); + ASSERT_FALSE(loader_data.status().ok()); + EXPECT_THAT(loader_data.status().ToString(), + HasSubstr("PathPrefixLoaderFactory only supports single-version " + "servables at version 0")); +} + } // namespace } // namespace serving } // namespace tensorflow From 97b4c3a8e5451f8b97bf2c496e1c5aecd914ef0e Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 16 Mar 2017 08:25:48 -0800 Subject: [PATCH 0201/8554] Make CachingManager public. Change: 150328283 --- tensorflow_serving/core/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 323c1b3af69..337154c3bc1 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -579,6 +579,9 @@ cc_library( name = "caching_manager", srcs = ["caching_manager.cc"], hdrs = ["caching_manager.h"], + visibility = [ + "//visibility:public", + ], deps = [ ":basic_manager", ":loader", From 2208226f294c143f304150c834c588abaaa711af Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 16 Mar 2017 09:36:37 -0800 Subject: [PATCH 0202/8554] Fix open source compilation issue. Change: 150337237 --- .../servables/tensorflow/saved_model_bundle_factory.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc index c4c95eeef0c..d0979c734fe 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -42,7 +42,7 @@ std::vector GetSignatureDefs(const SavedModelBundle& bundle) { // Parses a repeated field of NamedTensorProtos into a corresponding list of // name/tensor pairs. Status ParseFixedInputTensors( - const proto2::RepeatedPtrField& protos, + const protobuf::RepeatedPtrField& protos, std::vector>* parsed) { for (const NamedTensorProto& proto : protos) { Tensor tensor; From c582ece52b5a24f7d4c735b7b7b442602971be1f Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 16 Mar 2017 11:10:12 -0800 Subject: [PATCH 0203/8554] Define MultiInference API protos. Change: 150351162 --- tensorflow_serving/apis/BUILD | 26 ++++++++++++ tensorflow_serving/apis/inference.proto | 56 +++++++++++++++++++++++++ 2 files changed, 82 insertions(+) create mode 100644 tensorflow_serving/apis/inference.proto diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 26c3e33242d..2d745e23bfd 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -157,6 +157,32 @@ serving_proto_library_py( ], ) +serving_proto_library( + name = "inference_proto", + srcs = ["inference.proto"], + cc_api_version = 2, + go_api_version = 2, + java_api_version = 2, + deps = [ + ":classification_proto", + ":input_proto", + ":model_proto", + ":regression_proto", + ], +) + +serving_proto_library_py( + name = "inference_py_pb2", + srcs = ["inference.proto"], + proto_library = "inference_proto", + deps = [ + ":classification_proto_py_pb2", + ":input_proto_py_pb2", + ":model_proto_py_pb2", + ":regression_proto_py_pb2", + ], +) + serving_proto_library( name = "regression_proto", srcs = ["regression.proto"], diff --git a/tensorflow_serving/apis/inference.proto b/tensorflow_serving/apis/inference.proto new file mode 100644 index 00000000000..1eac7f9743f --- /dev/null +++ b/tensorflow_serving/apis/inference.proto @@ -0,0 +1,56 @@ +// This file contains messages for various machine learning inferences +// such as regression and classification. +// +// In many applications more than one type of inference is desired for a single +// input. For example, given meteorologic data an application may want to +// perform a classification to determine if we should expect rain, snow or sun +// and also perform a regression to predict the temperature. +// Sharing the single input data between two inference tasks can be accomplished +// using MultiInferenceRequest and MultiInferenceResponse. + +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/classification.proto"; +import "tensorflow_serving/apis/input.proto"; +import "tensorflow_serving/apis/model.proto"; +import "tensorflow_serving/apis/regression.proto"; + +package tensorflow.serving; + +// Inference request such as classification, regression, etc... +message InferenceTask { + ModelSpec model_spec = 1; + + // Signature's method_name. Should be one of the method names defined in + // third_party/tensorflow/python/saved_model/signature_constants.py. + // e.g. "tensorflow/serving/classify". + string method_name = 2; +} + +// Inference result, matches the type of request or is an error. +message InferenceResult { + ModelSpec model_spec = 1; + + oneof result { + ClassificationResult classification_result = 2; + RegressionResult regression_result = 3; + } +} + +// Inference request containing one or more requests. +message MultiInferenceRequest { + // Inference tasks. + repeated InferenceTask tasks = 1; + + // Input data. + Input input = 2; +} + +// Inference request containing one or more responses. +message MultiInferenceResponse { + // List of results; one for each InferenceTask in the request, returned in the + // same order as the request. + repeated InferenceResult results = 1; +} From 235e5f11bf808c4c92c44846b6a0cc5f8dca6410 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 16 Mar 2017 19:36:25 -0800 Subject: [PATCH 0204/8554] Eliminate source_adapter_test_util, since now its functionality can be inlined using SourceAdapter::AdaptOneVersion(). Change: 150404132 --- tensorflow_serving/core/BUILD | 1 - .../aspired_versions_manager_builder_test.cc | 1 - tensorflow_serving/core/test_util/BUILD | 14 --- .../core/test_util/source_adapter_test_util.h | 85 ------------------- tensorflow_serving/servables/hashmap/BUILD | 1 - .../hashmap/hashmap_source_adapter_test.cc | 3 +- tensorflow_serving/servables/tensorflow/BUILD | 2 - .../saved_model_bundle_source_adapter_test.cc | 4 +- .../session_bundle_source_adapter_test.cc | 4 +- 9 files changed, 5 insertions(+), 110 deletions(-) delete mode 100644 tensorflow_serving/core/test_util/source_adapter_test_util.h diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 337154c3bc1..f095d7c9d45 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -374,7 +374,6 @@ cc_test( "//tensorflow_serving/core/test_util:availability_test_util", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/core/test_util:fake_storage_path_source_adapter", - "//tensorflow_serving/core/test_util:source_adapter_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:event_bus", ], diff --git a/tensorflow_serving/core/aspired_versions_manager_builder_test.cc b/tensorflow_serving/core/aspired_versions_manager_builder_test.cc index 4fe48b9a9f5..dbe71597b8b 100644 --- a/tensorflow_serving/core/aspired_versions_manager_builder_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_builder_test.cc @@ -27,7 +27,6 @@ limitations under the License. #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/core/test_util/fake_storage_path_source_adapter.h" -#include "tensorflow_serving/core/test_util/source_adapter_test_util.h" #include "tensorflow_serving/util/event_bus.h" namespace tensorflow { diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 1bec2837dfd..0dd5778cbc6 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -109,20 +109,6 @@ cc_library( ], ) -cc_library( - name = "source_adapter_test_util", - testonly = 1, - hdrs = ["source_adapter_test_util.h"], - visibility = ["//visibility:public"], - deps = [ - "//external:gtest", - "//tensorflow_serving/core:loader", - "//tensorflow_serving/core:servable_data", - "//tensorflow_serving/core:servable_id", - "@org_tensorflow//tensorflow/core:lib", - ], -) - cc_library( name = "availability_test_util", testonly = 1, diff --git a/tensorflow_serving/core/test_util/source_adapter_test_util.h b/tensorflow_serving/core/test_util/source_adapter_test_util.h deleted file mode 100644 index 25f98347162..00000000000 --- a/tensorflow_serving/core/test_util/source_adapter_test_util.h +++ /dev/null @@ -1,85 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_SOURCE_ADAPTER_TEST_UTIL_H_ -#define TENSORFLOW_SERVING_CORE_TEST_UTIL_SOURCE_ADAPTER_TEST_UTIL_H_ - -#include -#include -#include - -#include -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/stringpiece.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/core/loader.h" -#include "tensorflow_serving/core/servable_data.h" -#include "tensorflow_serving/core/servable_id.h" - -namespace tensorflow { -namespace serving { -template -class SourceAdapter; -} // namespace serving -} // namespace tensorflow - -namespace tensorflow { -namespace serving { -namespace test_util { - -// Takes a SourceAdapter, and arranges to push a single data item of type -// InputType through its aspired-versions API. Returns the resulting -// ServableData object (which has a trivial servable id of {"", 0}). -// -// Assumes 'adapter->SetAspiredVersionsCallback()' has not yet been called. -// Assumes 'adapter->SetAspiredVersions()' is synchronous and internally calls -// the outgoing aspired-versions callback. -// -// Mutates 'adapter' and leaves it in an unspecified state. -template -ServableData RunSourceAdapter( - const InputType& in, SourceAdapter* adapter); - -////////// -// Implementation details follow. API users need not read. - -template -ServableData RunSourceAdapter( - const InputType& in, SourceAdapter* adapter) { - ServableId servable_id = {"", 0}; - std::vector> servable_data; - bool outgoing_callback_called = false; - adapter->SetAspiredVersionsCallback( - [&servable_id, &servable_data, &outgoing_callback_called]( - const StringPiece servable_name, - std::vector> versions) { - outgoing_callback_called = true; - CHECK_EQ(servable_id.name, servable_name); - servable_data = std::move(versions); - }); - adapter->SetAspiredVersions(servable_id.name, - {CreateServableData(servable_id, in)}); - CHECK(outgoing_callback_called) - << "Supplied adapter appears to have asynchronous behavior"; - CHECK_EQ(1, servable_data.size()); - CHECK_EQ(servable_id, servable_data[0].id()); - return std::move(servable_data[0]); -} - -} // namespace test_util -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_SOURCE_ADAPTER_TEST_UTIL_H_ diff --git a/tensorflow_serving/servables/hashmap/BUILD b/tensorflow_serving/servables/hashmap/BUILD index e34ad29efb8..11105cf5cd7 100644 --- a/tensorflow_serving/servables/hashmap/BUILD +++ b/tensorflow_serving/servables/hashmap/BUILD @@ -47,7 +47,6 @@ cc_test( ":hashmap_source_adapter_proto", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:servable_data", - "//tensorflow_serving/core/test_util:source_adapter_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:any_ptr", "@org_tensorflow//tensorflow/core:lib", diff --git a/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc b/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc index 2ffdcbb73fe..3b67e6ce05a 100644 --- a/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc +++ b/tensorflow_serving/servables/hashmap/hashmap_source_adapter_test.cc @@ -31,7 +31,6 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_data.h" -#include "tensorflow_serving/core/test_util/source_adapter_test_util.h" #include "tensorflow_serving/servables/hashmap/hashmap_source_adapter.pb.h" #include "tensorflow_serving/util/any_ptr.h" @@ -78,7 +77,7 @@ TEST(HashmapSourceAdapter, Basic) { auto adapter = std::unique_ptr(new HashmapSourceAdapter(config)); ServableData> loader_data = - test_util::RunSourceAdapter(file, adapter.get()); + adapter->AdaptOneVersion({{"", 0}, file}); TF_ASSERT_OK(loader_data.status()); std::unique_ptr loader = loader_data.ConsumeDataOrDie(); diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 180c819e848..5f3b5f6ae48 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -250,7 +250,6 @@ cc_test( ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:servable_data", - "//tensorflow_serving/core/test_util:source_adapter_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", @@ -302,7 +301,6 @@ cc_test( ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:servable_data", - "//tensorflow_serving/core/test_util:source_adapter_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc index a6a0d44e3ec..8dc6087d7f6 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc @@ -27,7 +27,6 @@ limitations under the License. #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_data.h" -#include "tensorflow_serving/core/test_util/source_adapter_test_util.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" @@ -50,7 +49,8 @@ class SavedModelBundleSourceAdapterTest : public ::testing::Test { std::unique_ptr adapter; TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(config, &adapter)); ServableData> loader_data = - test_util::RunSourceAdapter(export_dir, adapter.get()); + adapter->AdaptOneVersion( + ServableData({"", 0}, export_dir)); TF_ASSERT_OK(loader_data.status()); loader = loader_data.ConsumeDataOrDie(); diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc index 5b1336a0be7..e83a7b87322 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_source_adapter_test.cc @@ -27,7 +27,6 @@ limitations under the License. #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_data.h" -#include "tensorflow_serving/core/test_util/source_adapter_test_util.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" @@ -55,7 +54,8 @@ class SessionBundleSourceAdapterTest : public ::testing::Test { std::unique_ptr adapter; TF_CHECK_OK(SessionBundleSourceAdapter::Create(config, &adapter)); ServableData> loader_data = - test_util::RunSourceAdapter(export_dir_, adapter.get()); + adapter->AdaptOneVersion( + ServableData({"", 0}, export_dir_)); TF_ASSERT_OK(loader_data.status()); loader = loader_data.ConsumeDataOrDie(); From caacd0908176887d78d448f14eca5bf58b862a5f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 20 Mar 2017 10:42:30 -0800 Subject: [PATCH 0205/8554] Fix a use-after-move bug. Change: 150653941 --- tensorflow_serving/model_servers/server_core.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 799a84eb703..a934518e5a6 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -156,10 +156,11 @@ Status ServerCore::Create(Options options, // server_core_config (which contains aspired_version_policy) below. std::unique_ptr aspired_version_policy = std::move(options.aspired_version_policy); + auto model_server_config = options.model_server_config; server_core->reset(new ServerCore(std::move(options))); TF_RETURN_IF_ERROR( (*server_core)->Initialize(std::move(aspired_version_policy))); - return (*server_core)->ReloadConfig(options.model_server_config); + return (*server_core)->ReloadConfig(model_server_config); } // ************************************************************************ From 5f7e01713d133b0727e0671277e01948e1441ad9 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 20 Mar 2017 13:42:45 -0800 Subject: [PATCH 0206/8554] Move MultiInference implementation to open source. Change: 150678462 --- tensorflow_serving/servables/tensorflow/BUILD | 73 ++++- .../servables/tensorflow/multi_inference.cc | 127 ++++++++ .../servables/tensorflow/multi_inference.h | 48 +++ .../tensorflow/multi_inference_test.cc | 284 ++++++++++++++++++ 4 files changed, 521 insertions(+), 11 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/multi_inference.cc create mode 100644 tensorflow_serving/servables/tensorflow/multi_inference.h create mode 100644 tensorflow_serving/servables/tensorflow/multi_inference_test.cc diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 5f3b5f6ae48..05050934aac 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -549,6 +549,30 @@ cc_library( ], ) +cc_test( + name = "classifier_test", + size = "medium", + srcs = ["classifier_test.cc"], + deps = [ + ":classifier", + "//tensorflow_serving/apis:classification_proto", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/core/test_util:mock_session", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", + "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:tensorflow", + "@org_tensorflow//tensorflow/core:test", + "@protobuf//:protobuf_lite", + ], +) + cc_library( name = "classification_service", srcs = ["classification_service.cc"], @@ -635,27 +659,54 @@ cc_test( ], ) +cc_library( + name = "multi_inference", + srcs = ["multi_inference.cc"], + hdrs = ["multi_inference.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "//tensorflow_serving/apis:inference_proto", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/servables/tensorflow:classifier", + "//tensorflow_serving/servables/tensorflow:regressor", + "//tensorflow_serving/servables/tensorflow:util", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/core:all_kernels", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:tensorflow", + "@protobuf//:protobuf", + ], +) + cc_test( - name = "classifier_test", + name = "multi_inference_test", size = "medium", - srcs = ["classifier_test.cc"], + srcs = ["multi_inference_test.cc"], + data = [ + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + ], deps = [ - ":classifier", + ":multi_inference", "//tensorflow_serving/apis:classification_proto", "//tensorflow_serving/apis:input_proto", - "//tensorflow_serving/apis:model_proto", - "//tensorflow_serving/core/test_util:mock_session", + "//tensorflow_serving/apis:regression_proto", + "//tensorflow_serving/core:availability_preserving_policy", "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", + "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", "//tensorflow_serving/test_util", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", - "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", - "@org_tensorflow//tensorflow/core:core_cpu", - "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:protobuf_lite", ], ) diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.cc b/tensorflow_serving/servables/tensorflow/multi_inference.cc new file mode 100644 index 00000000000..6bb5a6cc94e --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/multi_inference.cc @@ -0,0 +1,127 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/multi_inference.h" + +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/tracing.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/servables/tensorflow/classifier.h" +#include "tensorflow_serving/servables/tensorflow/regressor.h" +#include "tensorflow_serving/servables/tensorflow/util.h" + +namespace tensorflow { +namespace serving { + +Status TensorFlowMultiInferenceRunner::Infer( + const MultiInferenceRequest& request, MultiInferenceResponse* response) { + TRACELITERAL("TensorFlowMultiInferenceRunner::Infer"); + + string model_name = ""; + string input_tensor_name = ""; + std::set signature_names; + std::set output_tensor_name_set; + for (const auto& task : request.tasks()) { + if (task.model_spec().name().empty()) { + return errors::InvalidArgument( + "Found ModelSpec with an empty model name."); + } + if (model_name.empty()) { + model_name = task.model_spec().name(); + } else if (model_name != task.model_spec().name()) { + return errors::InvalidArgument( + "All ModelSpecs in a MultiInferenceRequest must access the same " + "model name."); + } + + const string signature_name = task.model_spec().signature_name().empty() + ? kDefaultServingSignatureDefKey + : task.model_spec().signature_name(); + + if (signature_names.find(signature_name) != signature_names.end()) { + return errors::InvalidArgument(strings::StrCat( + "Duplicate evaluation of signature: ", signature_name)); + } + signature_names.insert(signature_name); + + auto iter = meta_graph_def_->signature_def().find(signature_name); + if (iter == meta_graph_def_->signature_def().end()) { + return errors::InvalidArgument(strings::StrCat( + "Requested signature not found in model graph: ", signature_name)); + } + string input_name; + std::vector output_names; + + if (task.method_name() == kClassifyMethodName) { + TF_RETURN_IF_ERROR( + PreProcessClassification(iter->second, &input_name, &output_names)); + } else if (task.method_name() == kRegressMethodName) { + TF_RETURN_IF_ERROR( + PreProcessRegression(iter->second, &input_name, &output_names)); + } else { + return errors::Unimplemented("Unsupported signature method_name: ", + task.method_name()); + } + if (input_tensor_name.empty()) { + input_tensor_name = input_name; + } else if (input_tensor_name != input_name) { + return errors::InvalidArgument( + "Input tensor must be the same for all Signatures."); + } + + for (const auto& output_tensor_name : output_names) { + output_tensor_name_set.insert(output_tensor_name); + } + } + + const std::vector output_tensor_names(output_tensor_name_set.begin(), + output_tensor_name_set.end()); + + std::vector outputs; + int num_examples; + TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( + request.input(), input_tensor_name, output_tensor_names, session_, + &outputs, &num_examples)); + + TRACELITERAL("PostProcessResults"); + for (const auto& task : request.tasks()) { + const string signature_name = task.model_spec().signature_name().empty() + ? kDefaultServingSignatureDefKey + : task.model_spec().signature_name(); + auto iter = meta_graph_def_->signature_def().find(signature_name); + if (iter == meta_graph_def_->signature_def().end()) { + return errors::InvalidArgument(strings::StrCat( + "Requested signature not found in model graph: ", signature_name)); + } + if (task.method_name() == kClassifyMethodName) { + TF_RETURN_IF_ERROR(PostProcessClassificationResult( + iter->second, num_examples, output_tensor_names, outputs, + response->add_results()->mutable_classification_result())); + } else if (task.method_name() == kRegressMethodName) { + TF_RETURN_IF_ERROR(PostProcessRegressionResult( + iter->second, num_examples, output_tensor_names, outputs, + response->add_results()->mutable_regression_result())); + } else { + return errors::InvalidArgument("Unrecognized signature method_name: ", + task.method_name()); + } + } + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.h b/tensorflow_serving/servables/tensorflow/multi_inference.h new file mode 100644 index 00000000000..533d7469f5f --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/multi_inference.h @@ -0,0 +1,48 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef LEARNING_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ +#define LEARNING_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ + +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/inference.pb.h" + +namespace tensorflow { +namespace serving { + +// TensorFlow implementation of the MultiInference. +// Only supports Models in the SavedModel format. +class TensorFlowMultiInferenceRunner { + public: + TensorFlowMultiInferenceRunner(Session* session, MetaGraphDef* meta_graph_def) + : session_(session), meta_graph_def_(meta_graph_def) {} + + // Run inference and return the inference results in the same order as the + // InferenceTasks in the request. + Status Infer(const MultiInferenceRequest& request, + MultiInferenceResponse* response); + + virtual ~TensorFlowMultiInferenceRunner() = default; + + private: + Session* const session_; + MetaGraphDef* const meta_graph_def_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // LEARNING_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ diff --git a/tensorflow_serving/servables/tensorflow/multi_inference_test.cc b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc new file mode 100644 index 00000000000..5f5e0eea5d2 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc @@ -0,0 +1,284 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/multi_inference.h" + +#include +#include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/example/feature.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/regression.pb.h" +#include "tensorflow_serving/apis/classification.pb.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; + +class MultiInferenceTest : public ::testing::Test { + public: + static void SetUpTestCase() { TF_ASSERT_OK(CreateServerCore(&server_core_)); } + + static void TearDownTestCase() { server_core_.reset(); } + + protected: + static Status CreateServerCore(std::unique_ptr* server_core) { + ModelServerConfig config; + auto model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two")); + + model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + options.model_server_config = config; + options.platform_config_map = CreateTensorFlowPlatformConfigMap( + SessionBundleConfig(), true /* use_saved_model */); + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + return ServerCore::Create(std::move(options), server_core); + } + + ServerCore* GetServerCore() { return server_core_.get(); } + + Status GetInferenceRunner( + std::unique_ptr* inference_runner) { + ServableHandle bundle; + ModelSpec model_spec; + model_spec.set_name(kTestModelName); + TF_RETURN_IF_ERROR(GetServerCore()->GetServableHandle(model_spec, &bundle)); + + inference_runner->reset(new TensorFlowMultiInferenceRunner( + bundle->session.get(), &bundle->meta_graph_def)); + return Status::OK(); + } + + private: + static std::unique_ptr server_core_; +}; + +std::unique_ptr MultiInferenceTest::server_core_; + +//////////////////////////////////////////////////////////////////////////////// +// Test Helpers + +void AddInput(const std::vector>& feature_kv, + MultiInferenceRequest* request) { + auto* example = + request->mutable_input()->mutable_example_list()->add_examples(); + auto* features = example->mutable_features()->mutable_feature(); + for (const auto& feature : feature_kv) { + (*features)[feature.first].mutable_float_list()->add_value(feature.second); + } +} + +void PopulateTask(const string& signature_name, const string& method_name, + InferenceTask* task) { + ModelSpec model_spec; + model_spec.set_name(kTestModelName); + model_spec.set_signature_name(signature_name); + *task->mutable_model_spec() = model_spec; + task->set_method_name(method_name); +} + +void ExpectStatusError(const Status& status, + const tensorflow::error::Code expected_code, + const string& message_substring) { + ASSERT_EQ(expected_code, status.code()); + EXPECT_THAT(status.error_message(), ::testing::HasSubstr(message_substring)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Tests + +TEST_F(MultiInferenceTest, MissingInputTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(inference_runner->Infer(request, &response), + tensorflow::error::INVALID_ARGUMENT, "Input is empty"); +} + +TEST_F(MultiInferenceTest, UndefinedSignatureTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("ThisSignatureDoesNotExist", kRegressMethodName, + request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(inference_runner->Infer(request, &response), + tensorflow::error::INVALID_ARGUMENT, "signature not found"); +} + +// Two ModelSpecs, accessing different models. +TEST_F(MultiInferenceTest, InconsistentModelSpecsInRequestTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + // Valid signature. + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + + // Add invalid Task to request. + ModelSpec model_spec; + model_spec.set_name("ModelDoesNotExist"); + model_spec.set_signature_name("regress_x_to_y"); + auto* task = request.add_tasks(); + *task->mutable_model_spec() = model_spec; + task->set_method_name(kRegressMethodName); + + MultiInferenceResponse response; + ExpectStatusError(inference_runner->Infer(request, &response), + tensorflow::error::INVALID_ARGUMENT, + "must access the same model name"); +} + +TEST_F(MultiInferenceTest, EvaluateDuplicateSignaturesTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + // Add the same task again (error). + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(inference_runner->Infer(request, &response), + tensorflow::error::INVALID_ARGUMENT, + "Duplicate evaluation of signature: regress_x_to_y"); +} + +TEST_F(MultiInferenceTest, UsupportedSignatureTypeTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("serving_default", kPredictMethodName, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(inference_runner->Infer(request, &response), + tensorflow::error::UNIMPLEMENTED, "Unsupported signature"); +} + +TEST_F(MultiInferenceTest, SignaturesWithDifferentInputsTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}, {"x2", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + PopulateTask("regress_x2_to_y3", kRegressMethodName, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(inference_runner->Infer(request, &response), + tensorflow::error::INVALID_ARGUMENT, + "Input tensor must be the same"); +} + +TEST_F(MultiInferenceTest, ValidSingleSignatureTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + + MultiInferenceResponse expected_response; + auto* regression_result = + expected_response.add_results()->mutable_regression_result(); + regression_result->add_regressions()->set_value(3.0); + + MultiInferenceResponse response; + TF_ASSERT_OK(inference_runner->Infer(request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +TEST_F(MultiInferenceTest, MultipleValidRegressSignaturesTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + PopulateTask("regress_x_to_y2", kRegressMethodName, request.add_tasks()); + + // regress_x_to_y is y = 0.5x + 2. + MultiInferenceResponse expected_response; + auto* regression_result_1 = + expected_response.add_results()->mutable_regression_result(); + regression_result_1->add_regressions()->set_value(3.0); + // regress_x_to_y2 is y2 = 0.5x + 3. + auto* regression_result_2 = + expected_response.add_results()->mutable_regression_result(); + regression_result_2->add_regressions()->set_value(4.0); + + MultiInferenceResponse response; + TF_ASSERT_OK(inference_runner->Infer(request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +TEST_F(MultiInferenceTest, RegressAndClassifySignaturesTest) { + std::unique_ptr inference_runner; + TF_ASSERT_OK(GetInferenceRunner(&inference_runner)); + + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); + PopulateTask("classify_x_to_y", kClassifyMethodName, request.add_tasks()); + + MultiInferenceResponse expected_response; + auto* regression_result = + expected_response.add_results()->mutable_regression_result(); + regression_result->add_regressions()->set_value(3.0); + auto* classification_result = + expected_response.add_results()->mutable_classification_result(); + classification_result->add_classifications()->add_classes()->set_score(3.0); + + MultiInferenceResponse response; + TF_ASSERT_OK(inference_runner->Infer(request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +} // namespace +} // namespace serving +} // namespace tensorflow From 4d0a571ff9c15b937f58d3d5e97a5310b5decf2b Mon Sep 17 00:00:00 2001 From: Tom Stall Date: Thu, 23 Mar 2017 13:58:29 -0700 Subject: [PATCH 0207/8554] Fix issue with inception_saved_model build --- tensorflow_serving/example/BUILD | 4 ++-- tensorflow_serving/workspace.bzl | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 3b34164170c..85bb83412df 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -80,7 +80,7 @@ py_binary( "inception_export.py", ], deps = [ - "@inception//inception", + "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", ], @@ -92,7 +92,7 @@ py_binary( "inception_saved_model.py", ], deps = [ - "@inception//inception", + "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", "@org_tensorflow//tensorflow/python/saved_model:builder", "@org_tensorflow//tensorflow/python/saved_model:constants", diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index 18cd9469bc0..a465eb7f137 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -7,9 +7,10 @@ load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') # workspace_dir is the absolute path to the TensorFlow Serving repo. If linked # as a submodule, it'll likely be '__workspace_dir__ + "/serving"' def tf_serving_workspace(): - native.local_repository( - name = "inception", + native.new_local_repository( + name = "inception_model", path = "tf_models/inception", + build_file = "tf_models/inception/inception/BUILD", ) tf_workspace(path_prefix = "", tf_repo_name = "org_tensorflow") From bfc40e95026ebdb6e7ad13ebc261d169ac1b01a8 Mon Sep 17 00:00:00 2001 From: Christi Kaes Date: Wed, 29 Mar 2017 16:59:40 -0400 Subject: [PATCH 0208/8554] Fix gcloud docker image push command (#383) Based on documentation at: https://cloud.google.com/container-registry/docs/pushing --- tensorflow_serving/g3doc/serving_inception.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 514495fd5a8..4a9c1f84765 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -213,7 +213,7 @@ $ docker tag $USER/inception_serving gcr.io/tensorflow-serving/inception Next we push the image to the Registry, ```shell -$ gcloud docker push gcr.io/tensorflow-serving/inception +$ gcloud docker -- push gcr.io/tensorflow-serving/inception ``` ### Create Kubernetes Deployment and Service From 00842c0d4b7d19a46ef7940152b567f0c198d7a3 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 22 Mar 2017 15:33:03 -0800 Subject: [PATCH 0209/8554] Merge changes from github. Change: 150944627 --- README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index d4ddfd7516e..3ea0278e69d 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -#TensorFlow Serving +# TensorFlow Serving [![Build Status](http://ci.tensorflow.org/buildStatus/icon?job=serving-master-cpu)](http://ci.tensorflow.org/job/serving-master-cpu) @@ -37,18 +37,18 @@ points; perhaps the most useful ways to extend the system are: [contribution guidelines](CONTRIBUTING.md).** **We use [GitHub issues](https://github.com/tensorflow/serving/issues) for -tracking requests and bugs. +tracking requests and bugs.** # Download and Setup See [install instructions](tensorflow_serving/g3doc/setup.md). -##Tutorials +## Tutorials * [Basic tutorial](tensorflow_serving/g3doc/serving_basic.md) * [Advanced tutorial](tensorflow_serving/g3doc/serving_advanced.md) -##For more information +## For more information * [Serving architecture overview](tensorflow_serving/g3doc/architecture_overview.md) * [TensorFlow website](http://tensorflow.org) From 6701289d173baaf6f81520fce402a8ebc973f89d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 23 Mar 2017 13:35:22 -0800 Subject: [PATCH 0210/8554] no-op Change: 151055240 --- tensorflow_serving/core/BUILD | 1 + tensorflow_serving/core/manager_test.cc | 52 +--------- tensorflow_serving/core/test_util/BUILD | 13 +++ .../test_util/servable_handle_test_util.h | 99 +++++++++++++++++++ 4 files changed, 118 insertions(+), 47 deletions(-) create mode 100644 tensorflow_serving/core/test_util/servable_handle_test_util.h diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index f095d7c9d45..e880d38d2b4 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -322,6 +322,7 @@ cc_test( srcs = ["manager_test.cc"], deps = [ ":manager", + "//tensorflow_serving/core/test_util:servable_handle_test_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/util:any_ptr", "@org_tensorflow//tensorflow/core:lib", diff --git a/tensorflow_serving/core/manager_test.cc b/tensorflow_serving/core/manager_test.cc index b2747e121f9..a5520c9f4ae 100644 --- a/tensorflow_serving/core/manager_test.cc +++ b/tensorflow_serving/core/manager_test.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include "tensorflow/core/lib/core/errors.h" +#include "tensorflow_serving/core/test_util/servable_handle_test_util.h" #include "tensorflow_serving/util/any_ptr.h" namespace tensorflow { @@ -132,56 +133,13 @@ TEST(ManagerTest, ErrorReturnsNullHandle) { EXPECT_EQ(nullptr, handle.get()); } -// Wraps a pointer as a servable. Does a bit of type-gymnastics since servable -// handles can only be constructed by managers. -template -ServableHandle WrapAsHandle(const ServableId& id, T* t) { - // Perform some type gymnastics to create a handle that points to 't'. - class DummyHandle : public UntypedServableHandle { - public: - explicit DummyHandle(const ServableId& id, T* servable) - : id_(id), servable_(servable) {} - - AnyPtr servable() override { return servable_; } - - const ServableId& id() const override { return id_; } - - private: - const ServableId id_; - T* servable_; - }; - - // Always returns the same servable. - class DummyManager : public TestManager { - public: - explicit DummyManager(const ServableId& id, T* servable) - : id_(id), servable_(servable) {} - - Status GetUntypedServableHandle( - const ServableRequest& request, - std::unique_ptr* result) override { - result->reset(new DummyHandle(id_, servable_)); - return Status::OK(); - } - - private: - const ServableId id_; - T* servable_; - }; - - DummyManager manager{id, t}; - ServableHandle handle; - TF_CHECK_OK(manager.GetServableHandle({"Dummy", 0}, &handle)); - return handle; -} - TEST(ServableHandleTest, PointerOps) { TestServable servables[2]; ServableHandle handles[2]; const ServableId id = {"servable", 7}; - handles[0] = WrapAsHandle(id, &servables[0]); - handles[1] = WrapAsHandle(id, &servables[1]); + handles[0] = test_util::WrapAsHandle(id, &servables[0]); + handles[1] = test_util::WrapAsHandle(id, &servables[1]); // Equality. EXPECT_EQ(handles[0], handles[0]); @@ -203,8 +161,8 @@ TEST(ServableHandleTest, Id) { ServableHandle handles[2]; const ServableId id = {"servable", 7}; - handles[0] = WrapAsHandle(id, &servables[0]); - handles[1] = WrapAsHandle(id, &servables[1]); + handles[0] = test_util::WrapAsHandle(id, &servables[0]); + handles[1] = test_util::WrapAsHandle(id, &servables[1]); EXPECT_EQ(id, handles[0].id()); EXPECT_EQ(id, handles[1].id()); diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 0dd5778cbc6..a2cc04eb979 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -133,6 +133,19 @@ cc_library( ], ) +cc_library( + name = "servable_handle_test_util", + testonly = 1, + hdrs = ["servable_handle_test_util.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow_serving/core:manager", + "//tensorflow_serving/core:servable_handle", + "//tensorflow_serving/core:servable_id", + "@org_tensorflow//tensorflow/core:lib", + ], +) + cc_library( name = "mock_log_collector", testonly = 1, diff --git a/tensorflow_serving/core/test_util/servable_handle_test_util.h b/tensorflow_serving/core/test_util/servable_handle_test_util.h new file mode 100644 index 00000000000..46550ba294b --- /dev/null +++ b/tensorflow_serving/core/test_util/servable_handle_test_util.h @@ -0,0 +1,99 @@ +/* Copyright 2016 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_SERVABLE_HANDLE_TEST_UTIL_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_SERVABLE_HANDLE_TEST_UTIL_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/core/servable_id.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +// Wraps a pointer as a servable. +// +// The ownership of @p t is *not* transferred to this function; t must outlive +// the returned ServableHandle. +// +// Does a bit of type-gymnastics since ServableHandles can only be constructed +// by Managers. +template +static ServableHandle WrapAsHandle(const ServableId& id, T* t) { + // A basic Handle that wraps a ServableId and a pointer to a servable. + // + // Exactly the same objects that are used to construct the DummyHandle are + // returned when the appropriate getter functions are invoked. + class DummyHandle : public UntypedServableHandle { + public: + explicit DummyHandle(const ServableId& id, T* servable) + : id_(id), servable_(servable) {} + + AnyPtr servable() override { return servable_; } + + const ServableId& id() const override { return id_; } + + private: + const ServableId id_; + T* servable_; + }; + + // A Manager that always returns the same servable when + // GetUntypedServableHandle is invoked. + class DummyManager : public Manager { + public: + explicit DummyManager(const ServableId& id, T* servable) + : id_(id), servable_(servable) {} + + // Resets the UntypedServableHandle to a new DummyHandle that wraps the + // Servable and ServableId which this DummyManager was created with. + // + // Always returns OK status. + Status GetUntypedServableHandle( + const ServableRequest& request, + std::unique_ptr* result) override { + result->reset(new DummyHandle(id_, servable_)); + return Status::OK(); + } + + // Unimplemented: always returns an empty map. + std::map> + GetAvailableUntypedServableHandles() const override { + return {}; + } + + // Unimplemented: always returns an empty vector. + std::vector ListAvailableServableIds() const override { + return {}; + } + + private: + const ServableId id_; + T* servable_; + }; + + DummyManager manager{id, t}; + ServableHandle handle; + TF_CHECK_OK(manager.GetServableHandle({"Dummy", 0}, &handle)); + return handle; +} + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_SERVABLE_HANDLE_TEST_UTIL_H_ From 9a3b10e776e65133e47d06d020f66515e2b326d1 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 24 Mar 2017 15:10:54 -0800 Subject: [PATCH 0211/8554] Move the core API & implementation of the TF-Serving batching library to tensorflow/contrib/. (Leave shims in the old location for now.) Change: 151184823 --- tensorflow_serving/batching/BUILD | 72 +- tensorflow_serving/batching/README.md | 22 +- .../batching/basic_batch_scheduler.h | 246 +------ .../basic_batch_scheduler_benchmark.cc | 442 ------------ .../batching/basic_batch_scheduler_test.cc | 90 --- tensorflow_serving/batching/batch_scheduler.h | 258 +------ .../batching/batch_scheduler_retrier_test.cc | 2 +- .../batching/batch_scheduler_test.cc | 117 --- .../batching/shared_batch_scheduler.h | 682 +----------------- .../batching/shared_batch_scheduler_test.cc | 593 --------------- .../streaming_batch_scheduler_test.cc | 2 +- tensorflow_serving/core/BUILD | 7 +- .../core/aspired_versions_manager.h | 2 +- .../aspired_versions_manager_benchmark.cc | 2 +- .../core/servable_state_monitor_test.cc | 2 +- tensorflow_serving/sources/storage_path/BUILD | 2 +- .../file_system_storage_path_source.h | 2 +- tensorflow_serving/test_util/BUILD | 14 - .../test_util/fake_clock_env.cc | 90 --- tensorflow_serving/test_util/fake_clock_env.h | 76 -- tensorflow_serving/util/BUILD | 26 +- tensorflow_serving/util/event_bus_test.cc | 2 +- .../util/fast_read_dynamic_ptr_benchmark.cc | 2 +- tensorflow_serving/util/observer_test.cc | 2 +- tensorflow_serving/util/periodic_function.cc | 102 --- tensorflow_serving/util/periodic_function.h | 132 ---- .../util/periodic_function_test.cc | 225 ------ 27 files changed, 42 insertions(+), 3172 deletions(-) delete mode 100644 tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc delete mode 100644 tensorflow_serving/batching/basic_batch_scheduler_test.cc delete mode 100644 tensorflow_serving/batching/batch_scheduler_test.cc delete mode 100644 tensorflow_serving/batching/shared_batch_scheduler_test.cc delete mode 100644 tensorflow_serving/test_util/fake_clock_env.cc delete mode 100644 tensorflow_serving/test_util/fake_clock_env.h delete mode 100644 tensorflow_serving/util/periodic_function.cc delete mode 100644 tensorflow_serving/util/periodic_function.h delete mode 100644 tensorflow_serving/util/periodic_function_test.cc diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index 17d30c0d7b4..02ab0e94914 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -20,28 +20,17 @@ filegroup( ), ) +# TODO(b/36404166): Remove this shim after migrating all clients. cc_library( name = "batch_scheduler", hdrs = ["batch_scheduler.h"], visibility = ["//visibility:public"], deps = [ - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - -cc_test( - name = "batch_scheduler_test", - srcs = [ - "batch_scheduler_test.cc", - ], - deps = [ - ":batch_scheduler", - "//tensorflow_serving/core/test_util:test_main", - "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", ], ) +# TODO(b/36404166): Remove this shim after migrating all clients. cc_library( name = "shared_batch_scheduler", hdrs = ["shared_batch_scheduler.h"], @@ -49,27 +38,11 @@ cc_library( "//visibility:public", ], deps = [ - ":batch_scheduler", - "//tensorflow_serving/util:periodic_function", - "@org_tensorflow//tensorflow/core:lib", - ], -) - -cc_test( - name = "shared_batch_scheduler_test", - srcs = [ - "shared_batch_scheduler_test.cc", - ], - deps = [ - ":shared_batch_scheduler", - "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:test", + "@org_tensorflow//tensorflow/contrib/batching:shared_batch_scheduler", ], ) +# TODO(b/36404166): Remove this shim after migrating all clients. cc_library( name = "basic_batch_scheduler", hdrs = ["basic_batch_scheduler.h"], @@ -77,36 +50,7 @@ cc_library( "//visibility:public", ], deps = [ - ":shared_batch_scheduler", - ], -) - -cc_test( - name = "basic_batch_scheduler_test", - srcs = [ - "basic_batch_scheduler_test.cc", - ], - deps = [ - ":basic_batch_scheduler", - ":batch_scheduler", - "//tensorflow_serving/core/test_util:test_main", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:test", - ], -) - -cc_test( - name = "basic_batch_scheduler_benchmark", - srcs = ["basic_batch_scheduler_benchmark.cc"], - tags = [ - "local", - "manual", - ], - deps = [ - ":basic_batch_scheduler", - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", - "@org_tensorflow//tensorflow/core:test", + "@org_tensorflow//tensorflow/contrib/batching:basic_batch_scheduler", ], ) @@ -131,7 +75,7 @@ cc_test( deps = [ ":streaming_batch_scheduler", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", + "@org_tensorflow//tensorflow/contrib/batching/test_util:fake_clock_env", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", @@ -200,7 +144,7 @@ cc_test( deps = [ ":batch_scheduler_retrier", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", + "@org_tensorflow//tensorflow/contrib/batching/test_util:fake_clock_env", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", diff --git a/tensorflow_serving/batching/README.md b/tensorflow_serving/batching/README.md index 1c3916a0639..c6103219192 100644 --- a/tensorflow_serving/batching/README.md +++ b/tensorflow_serving/batching/README.md @@ -8,16 +8,18 @@ ## Introduction -Batching individual model inference requests together can be important for -performance. In particular, batching is necessary to unlock the high throughput -promised by hardware accelerators such as GPUs. TensorFlow Serving provides a -library for batching requests and scheduling the batches. The library is not -tied to GPUs, per se, and can be used for any situation that benefits from -processing groups of small tasks in tandem (but this document assumes GPUs to -simplify exposition). Its code does not depend on the core TensorFlow Serving -code, so it can be used separately from a TensorFlow Serving deployment if -desired. It offers a specific TensorFlow Session API, as well as general APIs -that are not tied to TensorFlow. +While serving a TensorFlow model, batching individual model inference requests +together can be important for performance. In particular, batching is necessary +to unlock the high throughput promised by hardware accelerators such as GPUs. +This is a library for batching requests and scheduling the batches. The library +is not tied to GPUs, per se, and can be used for any situation that benefits +from processing groups of small tasks in tandem (but this document assumes GPUs +to simplify exposition). It offers a specific TensorFlow Session API, as well as +lower-level APIs that can be used to batch at other granularities. + +The library is currently split across two locations: +(1) tensorflow/contrib/batching (core API and implementation), and +(2) tensorflow_serving/batching (higher-level and experimental code). The library offers several alternative classes to choose from. The reason for the choices is that there are many reasonable ways to perform batching. No diff --git a/tensorflow_serving/batching/basic_batch_scheduler.h b/tensorflow_serving/batching/basic_batch_scheduler.h index 1124a49b7e8..9999ddc05f6 100644 --- a/tensorflow_serving/batching/basic_batch_scheduler.h +++ b/tensorflow_serving/batching/basic_batch_scheduler.h @@ -16,249 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ #define TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ -#include -#include -#include -#include -#include - -#include "tensorflow_serving/batching/shared_batch_scheduler.h" - -namespace tensorflow { -namespace serving { - -// A BatchScheduler implementation geared toward handling a single request type -// running on a specific set of hardware resources. A typical scenario is one in -// which all requests invoke the same machine-learned model on one GPU. -// -// If there are, say, two GPUs and two models each bound to one of the GPUs, one -// could use two BasicBatchScheduler instances to schedule the two model/GPU -// combinations independently. If multiple models must share a given GPU or -// other hardware resource, consider using SharedBatchScheduler instead. -// -// -// PARAMETERS AND BEHAVIOR: -// -// BasicBatchScheduler runs a fixed pool of threads, which it uses to process -// batches of tasks. It enforces a maximum batch size, and enqueues a bounded -// number of tasks. If the queue is nearly empty, such that a full batch cannot -// be formed, when a thread becomes free, it anyway schedules a batch -// immediately if a task has been in the queue for longer than a given timeout -// parameter. If the timeout parameter is set to 0, then the batch threads will -// always be kept busy (unless there are zero tasks waiting to be processed). -// -// For online serving, it is recommended to set the maximum number of enqueued -// batches worth of tasks equal to the number of batch threads, which allows -// enqueuing of enough tasks s.t. if every thread becomes available it can be -// kept busy, but no more. For bulk processing jobs and throughput-oriented -// benchmarks, you may want to set it much higher. -// -// When Schedule() is called, if the queue is full the call will fail with an -// UNAVAILABLE error (after which the client may retry again later). If the call -// succeeds, the maximum time the task will spend in the queue before being -// placed in a batch and assigned to a thread for processing, is the greater of: -// - the maximum time to process ceil(max_enqueued_batches/num_batch_threads) -// (1 in the recommended configuration) batches of previously-submitted tasks -// - the configured timeout parameter (which can be 0, as mentioned above) -// -// Unlike StreamingBatchScheduler, when BasicBatchScheduler assigns a batch to a -// thread, it closes the batch. The process-batch callback may assume that every -// batch it receives is closed at the outset. -// -// -// RECOMMENDED USE-CASES: -// -// BasicBatchScheduler is suitable for use-cases that feature a single kind of -// request (e.g. a server performing inference with a single machine-learned -// model, possibly evolving over time), with loose versioning semantics. -// Concretely, the following conditions should hold: -// -// A. All requests batched onto a given resource (e.g. a hardware accelerator, -// or a pool accelerators) are of the same type. For example, they all -// invoke the same machine-learned model. -// -// These variations are permitted: -// - The model may reside in a single servable, or it may be spread across -// multiple servables that are used in unison (e.g. a vocabulary lookup -// table servable and a tensorflow session servable). -// - The model's servable(s) may be static, or they may evolve over time -// (successive servable versions). -// - Zero or more of the servables are used in the request thread; the rest -// are used in the batch thread. In our running example, the vocabulary -// lookups and tensorflow runs may both be performed in the batch thread, -// or alternatively the vocabulary lookup may occur in the request thread -// with only the tensorflow run performed in the batch thread. -// -// In contrast, BasicBatchScheduler is not a good fit if the server -// hosts multiple distinct models running on a pool accelerators, with each -// request specifying which model it wants to use. BasicBatchScheduler -// has no facility to time-multiplex the batch threads across multiple -// models in a principled way. More basically, it cannot ensure that a given -// batch doesn't contain a mixture of requests for different models. -// -// B. Requests do not specify a particular version of the servable(s) that must -// be used. Instead, each request is content to use the "latest" version. -// -// BasicBatchScheduler does not constrain which requests get grouped -// together into a batch, so using this scheduler there is no way to achieve -// cohesion of versioned requests to version-specific batches. -// -// C. No servable version coordination needs to be performed between the -// request threads and the batch threads. Often, servables are only used in -// the batch threads, in which case this condition trivially holds. If -// servables are used in both threads, then the use-case must tolerate -// version skew across the servables used in the two kinds of threads. -// -// -// EXAMPLE USE-CASE FLOW: -// -// For such use-cases, request processing via BasicBatchScheduler generally -// follows this flow (given for illustration; variations are possible): -// 1. Optionally perform some pre-processing on each request in the request -// threads. -// 2. Route the requests to the batch scheduler, as batching::Task objects. -// (Since all requests are of the same type and are not versioned, the -// scheduler is free to group them into batches arbitrarily.) -// 3. Merge the requests into a single batched representation B. -// 4. Obtain handles to the servable(s) needed to process B. The simplest -// approach is to obtain the latest version of each servable. Alternatively, -// if cross-servable consistency is required (e.g. the vocabulary lookup -// table's version number must match that of the tensorflow session), -// identify an appropriate version number and obtain the servable handles -// accordingly. -// 5. Process B using the obtained servable handles, and split the result into -// individual per-request units. -// 6. Perform any post-processing in the batch thread and/or request thread. -// -// -// PERFORMANCE TUNING: See README.md. -// -template -class BasicBatchScheduler : public BatchScheduler { - public: - // TODO(b/25089730): Tune defaults based on best practices as they develop. - // (Keep them mirrored to the ones in SharedBatchScheduler::QueueOptions and - // SharedBatchScheduler::Options.) - struct Options { - // The maximum size of each batch. - // - // The scheduler may form batches of any size between 1 and this number - // (inclusive). If there is a need to quantize the batch sizes, i.e. only - // submit batches whose size is in a small set of allowed sizes, that can be - // done by adding padding in the process-batch callback. - int max_batch_size = 1000; - - // If a task has been enqueued for this amount of time (in microseconds), - // and a thread is available, the scheduler will immediately form a batch - // from enqueued tasks and assign the batch to the thread for processing, - // even if the batch's size is below 'max_batch_size'. - // - // This parameter offers a way to bound queue latency, so that a task isn't - // stuck in the queue indefinitely waiting for enough tasks to arrive to - // make a full batch. (The latency bound is given in the class documentation - // above.) - // - // The goal is to smooth out batch sizes under low request rates, and thus - // avoid latency spikes. - int64 batch_timeout_micros = 0; - - // The name to use for the pool of batch threads. - string thread_pool_name = {"batch_threads"}; - - // The number of threads to use to process batches. - // Must be >= 1, and should be tuned carefully. - int num_batch_threads = port::NumSchedulableCPUs(); - - // The maximum allowable number of enqueued (accepted by Schedule() but - // not yet being processed on a batch thread) tasks in terms of batches. - // If this limit is reached, Schedule() will return an UNAVAILABLE error. - // See the class documentation above for guidelines on how to tune this - // parameter. - int max_enqueued_batches = 10; - - // The following options are typically only overridden by test code. - - // The environment to use. - Env* env = Env::Default(); - }; - static Status Create(const Options& options, - std::function>)> - process_batch_callback, - std::unique_ptr* scheduler); - - ~BasicBatchScheduler() override = default; - - Status Schedule(std::unique_ptr* task) override; - size_t NumEnqueuedTasks() const override; - size_t SchedulingCapacity() const override; - - private: - explicit BasicBatchScheduler( - std::unique_ptr> shared_scheduler_queue); - - // This class is merely a thin wrapper around a SharedBatchScheduler with a - // single queue. - std::unique_ptr> shared_scheduler_queue_; - - TF_DISALLOW_COPY_AND_ASSIGN(BasicBatchScheduler); -}; - -////////// -// Implementation details follow. API users need not read. - -template -Status BasicBatchScheduler::Create( - const Options& options, - std::function>)> - process_batch_callback, - std::unique_ptr* scheduler) { - typename SharedBatchScheduler::Options shared_scheduler_options; - shared_scheduler_options.thread_pool_name = options.thread_pool_name; - shared_scheduler_options.num_batch_threads = options.num_batch_threads; - shared_scheduler_options.env = options.env; - std::shared_ptr> shared_scheduler; - TF_RETURN_IF_ERROR(SharedBatchScheduler::Create( - shared_scheduler_options, &shared_scheduler)); - - typename SharedBatchScheduler::QueueOptions - shared_scheduler_queue_options; - shared_scheduler_queue_options.max_batch_size = options.max_batch_size; - shared_scheduler_queue_options.batch_timeout_micros = - options.batch_timeout_micros; - shared_scheduler_queue_options.max_enqueued_batches = - options.max_enqueued_batches; - std::unique_ptr> shared_scheduler_queue; - TF_RETURN_IF_ERROR(shared_scheduler->AddQueue(shared_scheduler_queue_options, - process_batch_callback, - &shared_scheduler_queue)); - - scheduler->reset( - new BasicBatchScheduler(std::move(shared_scheduler_queue))); - return Status::OK(); -} - -template -Status BasicBatchScheduler::Schedule( - std::unique_ptr* task) { - return shared_scheduler_queue_->Schedule(task); -} - -template -size_t BasicBatchScheduler::NumEnqueuedTasks() const { - return shared_scheduler_queue_->NumEnqueuedTasks(); -} - -template -size_t BasicBatchScheduler::SchedulingCapacity() const { - return shared_scheduler_queue_->SchedulingCapacity(); -} - -template -BasicBatchScheduler::BasicBatchScheduler( - std::unique_ptr> shared_scheduler_queue) - : shared_scheduler_queue_(std::move(shared_scheduler_queue)) {} - -} // namespace serving -} // namespace tensorflow +// TODO(b/36404166): Remove this shim after migrating all clients. +#include "tensorflow/contrib/batching/basic_batch_scheduler.h" #endif // TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc b/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc deleted file mode 100644 index 40d6315eba9..00000000000 --- a/tensorflow_serving/batching/basic_batch_scheduler_benchmark.cc +++ /dev/null @@ -1,442 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// Benchmarks for performance (throughput and latency) of BasicBatchScheduler -// under various rates of task injection. -// -// Run with: -// bazel run -c opt --dynamic_mode=off \ -// tensorflow_serving/batching:basic_batch_scheduler_benchmark -- -// --benchmarks=. -// For a longer run time and more consistent results for the throughput -// benchmark, consider a min time e.g.: --benchmark_min_time=60.0 - -#include "tensorflow/core/lib/histogram/histogram.h" -#include "tensorflow/core/platform/init_main.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/test.h" -#include "tensorflow/core/platform/test_benchmark.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" - -namespace tensorflow { -namespace serving { -namespace { - -using ::tensorflow::histogram::Histogram; - -// An abstract class for injecting load into a system at a specific rate. -class LoadInjector { - public: - virtual ~LoadInjector() = default; - - // Run 'injector' 'num_injection' times, with average inter-injection spacing - // as 'average_injection_interval_micros' (in microseconds). - virtual void InjectLoad(std::function injector, int num_injections, - int64 average_injection_interval_micros) const = 0; -}; - -// A load injector that uses uniform inter-injection spacing, i.e. each pair of -// injections is separated in time by 'average_injection_interval_micros' (as -// best as possible). -class UniformLoadInjector : public LoadInjector { - public: - UniformLoadInjector() = default; - ~UniformLoadInjector() override = default; - - void InjectLoad(std::function injector, int num_injections, - int64 average_injection_interval_micros) const override; - - private: - TF_DISALLOW_COPY_AND_ASSIGN(UniformLoadInjector); -}; - -void UniformLoadInjector::InjectLoad( - std::function injector, const int num_injections, - const int64 average_injection_interval_micros) const { - int num_injections_performed = 0; - const int64 start_time_micros = Env::Default()->NowMicros(); - while (num_injections_performed < num_injections) { - // Inject. - injector(); - ++num_injections_performed; - - // Wait until it's time for the next injection. - const int64 next_injection_time_micros = - start_time_micros + - (num_injections_performed * average_injection_interval_micros); - int64 now_micros = Env::Default()->NowMicros(); - while (now_micros < next_injection_time_micros) { - const int64 kSleepThresholdMicros = 1000; - if (next_injection_time_micros - now_micros >= kSleepThresholdMicros) { - Env::Default()->SleepForMicroseconds(1 /* minimum time */); - } - now_micros = Env::Default()->NowMicros(); - } - } -} - -class BenchmarkBatchTask : public BatchTask { - public: - BenchmarkBatchTask(); - - BenchmarkBatchTask(const BenchmarkBatchTask&) = delete; - BenchmarkBatchTask& operator=(const BenchmarkBatchTask&) = delete; - - ~BenchmarkBatchTask() override = default; - - size_t size() const override { return 1; } - - uint64 start_time_micros() const { return start_time_micros_; } - - private: - // The time at which the task was created, in microseconds. - const uint64 start_time_micros_; -}; - -BenchmarkBatchTask::BenchmarkBatchTask() - : start_time_micros_(Env::Default()->NowMicros()) {} - -// The state and logic associated with a throughput benchmark, which injects a -// large number of tasks into a batch scheduler and measures the total time to -// process all the tasks. -class ThroughputBenchmark { - public: - explicit ThroughputBenchmark( - const BasicBatchScheduler::Options& - scheduler_options); - - ThroughputBenchmark(const ThroughputBenchmark&) = delete; - ThroughputBenchmark& operator=(const ThroughputBenchmark&) = delete; - - // Perform the benchmark run, based on the parameters supplied to the ctor. - void RunBenchmark(int iters); - - private: - // Resets all mutable state, including the scheduler. - void ResetState(); - - // Processes a batch of tasks. (Invoked by 'scheduler_' on one of its batch - // threads.) - void ProcessBatch(std::unique_ptr> batch); - - // Parameters for the BasicBatchScheduler being benchmarked. - const BasicBatchScheduler::Options scheduler_options_; - - // The BasicBatchScheduler being benchmarked. - std::unique_ptr> scheduler_; -}; - -ThroughputBenchmark::ThroughputBenchmark( - const BasicBatchScheduler::Options& scheduler_options) - : scheduler_options_(scheduler_options) {} - -void ThroughputBenchmark::RunBenchmark(int iters) { - CHECK_GE(iters, 1); - - testing::StopTiming(); - ResetState(); - - // Have each iteration issue a reasonably large number of tasks, to ensure our - // measurements reflect steady-state behavior. - const int kNumTasksPerIteration = 100 * 1000; - - testing::ItemsProcessed(iters * kNumTasksPerIteration); - testing::UseRealTime(); - testing::StartTiming(); - - // Schedule 'num_iterations_*kNumTasksPerIteration' tasks. - for (int i = 0; i < iters; ++i) { - for (int j = 0; j < kNumTasksPerIteration; ++j) { - auto task = std::unique_ptr(new BenchmarkBatchTask); - TF_CHECK_OK(scheduler_->Schedule(&task)); - } - } - - // Wait for the scheduler to process all tasks. - scheduler_.reset(); - testing::StopTiming(); -} - -void ThroughputBenchmark::ResetState() { - auto process_batch_callback = - [this](std::unique_ptr> batch) { - ProcessBatch(std::move(batch)); - }; - TF_CHECK_OK(BasicBatchScheduler::Create( - scheduler_options_, process_batch_callback, &scheduler_)); -} - -void ThroughputBenchmark::ProcessBatch( - std::unique_ptr> batch) { - // No-op. -} - -// The state and logic associated with a latency benchmark, which injects tasks -// into a batch scheduler at a controlled rate and measures the distribution of -// task completion latencies. -// -// Reports the measurements to std::cout (not LOG(INFO)), like the throughput -// measurements. -class LatencyBenchmark { - public: - LatencyBenchmark( - const BasicBatchScheduler::Options& scheduler_options, - int64 task_injection_interval_micros, int batch_cpu_cost); - - LatencyBenchmark(const LatencyBenchmark&) = delete; - LatencyBenchmark& operator=(const LatencyBenchmark&) = delete; - - // Perform the benchmark run, based on the parameters supplied to the ctor. - void RunBenchmark(); - - private: - // Resets all mutable state, including the scheduler and latency measurements. - void ResetState() LOCKS_EXCLUDED(mu_); - - // Processes a batch of tasks. (Invoked by 'scheduler_' on one of its batch - // threads.) - void ProcessBatch(std::unique_ptr> batch); - - // Performs one batch's dummy CPU work. - void PerformBatchCpuWork() const; - - // Parameters for the BasicBatchScheduler being benchmarked. - const BasicBatchScheduler::Options scheduler_options_; - - // The time interval between successively injected tasks, in microseconds. - // A large interval corresponds to a slow rate of task injection, and vice- - // versa. - const int64 task_injection_interval_micros_; - - // The amount of work to do while processing one batch of tasks. (The cost is - // independent of the number of tasks in the batch.) - const int batch_cpu_cost_; - - // The BasicBatchScheduler being benchmarked. - std::unique_ptr> scheduler_; - - mutable mutex mu_; - - // A histogram of the task latencies, i.e. queue time plus processing time, in - // milliseconds. - Histogram task_latency_millis_histogram_ GUARDED_BY(mu_); - - // A histogram of the batch sizes. - Histogram batch_size_histogram_ GUARDED_BY(mu_); -}; - -LatencyBenchmark::LatencyBenchmark( - const BasicBatchScheduler::Options& scheduler_options, - int64 task_injection_interval_micros, int batch_cpu_cost) - : scheduler_options_(scheduler_options), - task_injection_interval_micros_(task_injection_interval_micros), - batch_cpu_cost_(batch_cpu_cost) {} - -void LatencyBenchmark::RunBenchmark() { - ResetState(); - - // Arrange to inject tasks at the specified rate, for a fixed total time - // duration. - const int kTimeDurationMicros = 100 * 1000 * 1000 /* 100 seconds */; - const int kNumTasks = kTimeDurationMicros / task_injection_interval_micros_; - CHECK_GE(kNumTasks, 100000) - << "Not enough tasks to report meaningful 99.9% latency"; - - const int64 start_time_micros = Env::Default()->NowMicros(); - - // Inject the tasks. - UniformLoadInjector injector; - injector.InjectLoad( - [this] { - auto task = std::unique_ptr(new BenchmarkBatchTask); - TF_CHECK_OK(scheduler_->Schedule(&task)); - }, - kNumTasks, task_injection_interval_micros_); - - // Be sure we were able to more-or-less match our target injection rate. - const int64 target_injection_time_micros = - kNumTasks * task_injection_interval_micros_; - const int64 actual_injection_time_micros = - Env::Default()->NowMicros() - start_time_micros; - if (actual_injection_time_micros > 1.1 * target_injection_time_micros) { - LOG(FATAL) << "Unable to inject tasks at the requested rate"; - } - - // Wait for the scheduler to process all injected tasks. - scheduler_.reset(); - - // Be sure the scheduler was able to process the tasks at close to the - // injection rate. If not, our latency measurements will be dominated by queue - // waiting time - const int64 actual_processing_time_micros = - Env::Default()->NowMicros() - start_time_micros; - if (actual_processing_time_micros > 1.01 * actual_injection_time_micros) { - LOG(FATAL) << "Unable to keep up with task injection rate"; - } - - // Report benchmark measurements. - { - mutex_lock l(mu_); - std::cout << "\t" - << "99.9% latency: " - << task_latency_millis_histogram_.Percentile(99.9) << "ms" - << "\t" - << "99% batch size: " << batch_size_histogram_.Percentile(99) - << std::endl; - } -} - -void LatencyBenchmark::ResetState() { - auto process_batch_callback = - [this](std::unique_ptr> batch) { - ProcessBatch(std::move(batch)); - }; - TF_CHECK_OK(BasicBatchScheduler::Create( - scheduler_options_, process_batch_callback, &scheduler_)); - - { - mutex_lock l(mu_); - task_latency_millis_histogram_.Clear(); - batch_size_histogram_.Clear(); - } -} - -void LatencyBenchmark::ProcessBatch( - std::unique_ptr> batch) { - PerformBatchCpuWork(); - const uint64 batch_completion_time = Env::Default()->NowMicros(); - - { - mutex_lock l(mu_); - batch_size_histogram_.Add(batch->num_tasks()); - } - - for (int i = 0; i < batch->num_tasks(); ++i) { - const BenchmarkBatchTask& task = batch->task(i); - - const uint64 task_latency_micros = - batch_completion_time - task.start_time_micros(); - - { - mutex_lock l(mu_); - task_latency_millis_histogram_.Add(task_latency_micros / 1000.0); - } - } -} - -void LatencyBenchmark::PerformBatchCpuWork() const { - int dummy = 1; - for (int i = 0; i < batch_cpu_cost_; ++i) { - dummy += dummy * 2; - } - CHECK_NE(dummy, 0); -} - -static void RunThroughputBenchmark(int iters, int64 batch_timeout_micros, - int num_batch_threads) { - BasicBatchScheduler::Options scheduler_options; - const int kMaxBatchSize = 100; - scheduler_options.max_batch_size = kMaxBatchSize; - scheduler_options.batch_timeout_micros = batch_timeout_micros; - scheduler_options.num_batch_threads = num_batch_threads; - scheduler_options.max_enqueued_batches = INT_MAX; // Unbounded queue. - ThroughputBenchmark benchmark(scheduler_options); - benchmark.RunBenchmark(iters); -} - -static void ThroughputBM_ZeroTimeout(int iters, int num_batch_threads) { - RunThroughputBenchmark(iters, 0 /* 0 ms timeout */, num_batch_threads); -} -BENCHMARK(ThroughputBM_ZeroTimeout) - ->Arg(1) - ->Arg(2) - ->Arg(4) - ->Arg(8) - ->Arg(16) - ->Arg(32) - ->Arg(64); - -static void ThroughputBM_SmallTimeout(int iters, int num_batch_threads) { - RunThroughputBenchmark(iters, 1 * 1000 /* 1 ms timeout */, num_batch_threads); -} -BENCHMARK(ThroughputBM_SmallTimeout) - ->Arg(1) - ->Arg(2) - ->Arg(4) - ->Arg(8) - ->Arg(16) - ->Arg(32) - ->Arg(64); - -static void ThroughputBM_LargeTimeout(int iters, int num_batch_threads) { - RunThroughputBenchmark(iters, 50 * 1000 /* 50 ms timeout */, - num_batch_threads); -} -BENCHMARK(ThroughputBM_LargeTimeout) - ->Arg(1) - ->Arg(2) - ->Arg(4) - ->Arg(8) - ->Arg(16) - ->Arg(32) - ->Arg(64); - -static void RunLatencyBenchmark(int64 task_injection_interval_micros, - int64 batch_timeout_micros) { - BasicBatchScheduler::Options scheduler_options; - const int kMaxBatchSize = 100; - scheduler_options.max_batch_size = kMaxBatchSize; - scheduler_options.batch_timeout_micros = batch_timeout_micros; - const int kNumBatchThreads = 2; - scheduler_options.num_batch_threads = kNumBatchThreads; - scheduler_options.max_enqueued_batches = INT_MAX; // Unbounded queue. - const int kBatchCpuCost = 10 * 1000 * 1000; - LatencyBenchmark benchmark(scheduler_options, task_injection_interval_micros, - kBatchCpuCost); - benchmark.RunBenchmark(); -} - -static void RunLatencyBenchmarks() { - for (const int64 batch_timeout_micros : {0, 1 * 1000, 2 * 1000, 5 * 1000}) { - for (const int64 task_injection_interval_micros : {1000, 50, 20}) { - std::cout << "Latency benchmark w/ batch timeout " - << batch_timeout_micros / 1000.0 << "ms" - << "; " - << "task injection rate " - << 1000000.0 / task_injection_interval_micros << "/sec" - << "\t..."; - RunLatencyBenchmark(task_injection_interval_micros, batch_timeout_micros); - } - std::cout << std::endl; - } -} - -} // namespace -} // namespace serving -} // namespace tensorflow - -int main(int argc, char** argv) { - tensorflow::port::InitMain(argv[0], &argc, &argv); - std::setprecision(5); - - // Run latency benchmarks (outside of tensorflow benchmark framework). - tensorflow::serving::RunLatencyBenchmarks(); - - // Run throughput benchmarks (via tensorflow benchmark framework). - tensorflow::testing::RunBenchmarks(); - - return 0; -} diff --git a/tensorflow_serving/batching/basic_batch_scheduler_test.cc b/tensorflow_serving/batching/basic_batch_scheduler_test.cc deleted file mode 100644 index 5c5ef65ca94..00000000000 --- a/tensorflow_serving/batching/basic_batch_scheduler_test.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/batching/basic_batch_scheduler.h" - -#include - -#include -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/batching/batch_scheduler.h" - -namespace tensorflow { -namespace serving { -namespace { - -class FakeTask : public BatchTask { - public: - explicit FakeTask(size_t size) : size_(size) {} - - ~FakeTask() override = default; - - size_t size() const override { return size_; } - - private: - const size_t size_; - - TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); -}; - -// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' -// on that task. Returns the resulting status. -Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { - std::unique_ptr task(new FakeTask(task_size)); - Status status = scheduler->Schedule(&task); - // Schedule() should have consumed 'task' iff it returned Status::OK. - CHECK_EQ(status.ok(), task == nullptr); - return status; -} - -// Since BasicBatchScheduler is implemented as a thin wrapper around -// SharedBatchScheduler, we only do some basic testing. More comprehensive -// testing is done in shared_batch_scheduler_test.cc. - -TEST(BasicBatchSchedulerTest, Basic) { - bool callback_called = false; - auto callback = [&callback_called](std::unique_ptr> batch) { - callback_called = true; - ASSERT_TRUE(batch->IsClosed()); - ASSERT_EQ(2, batch->num_tasks()); - EXPECT_EQ(3, batch->task(0).size()); - EXPECT_EQ(5, batch->task(1).size()); - }; - { - BasicBatchScheduler::Options options; - options.max_batch_size = 10; - options.batch_timeout_micros = 100 * 1000; // 100 milliseconds - options.num_batch_threads = 1; - options.max_enqueued_batches = 3; - std::unique_ptr> scheduler; - TF_ASSERT_OK( - BasicBatchScheduler::Create(options, callback, &scheduler)); - EXPECT_EQ(0, scheduler->NumEnqueuedTasks()); - EXPECT_EQ(3 * 10, scheduler->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(3, scheduler.get())); - EXPECT_EQ(1, scheduler->NumEnqueuedTasks()); - EXPECT_EQ((3 * 10) - 3, scheduler->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(5, scheduler.get())); - EXPECT_EQ(2, scheduler->NumEnqueuedTasks()); - EXPECT_EQ((3 * 10) - (3 + 5), scheduler->SchedulingCapacity()); - } - EXPECT_TRUE(callback_called); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/batching/batch_scheduler.h b/tensorflow_serving/batching/batch_scheduler.h index 76f8ad499fa..73929928823 100644 --- a/tensorflow_serving/batching/batch_scheduler.h +++ b/tensorflow_serving/batching/batch_scheduler.h @@ -13,264 +13,10 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// Abstractions for processing small tasks in a batched fashion, to reduce -// processing times and costs that can be amortized across multiple tasks. -// -// The core class is BatchScheduler, which groups tasks into batches. -// -// BatchScheduler encapsulates logic for aggregating multiple tasks into a -// batch, and kicking off processing of a batch on a thread pool it manages. -// -// This file defines an abstract BatchScheduler class. - #ifndef TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ #define TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ -#include -#include -#include -#include -#include -#include - -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/logging.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace serving { - -// The abstract superclass for a unit of work to be done as part of a batch. -// -// An implementing subclass typically contains (or points to): -// (a) input data; -// (b) a thread-safe completion signal (e.g. a Notification); -// (c) a place to store the outcome (success, or some error), upon completion; -// (d) a place to store the output data, upon success. -// -// Items (b), (c) and (d) are typically non-owned pointers to data homed -// elsewhere, because a task's ownership gets transferred to a BatchScheduler -// (see below) and it may be deleted as soon as it is done executing. -class BatchTask { - public: - virtual ~BatchTask() = default; - - // Returns the size of the task, in terms of how much it contributes to the - // size of a batch. (A batch's size is the sum of its task sizes.) - virtual size_t size() const = 0; -}; - -// A thread-safe collection of BatchTasks, to be executed together in some -// fashion. -// -// At a given time, a batch is either "open" or "closed": an open batch can -// accept new tasks; a closed one cannot. A batch is monotonic: initially it is -// open and tasks can be added to it; then it is closed and its set of tasks -// remains fixed for the remainder of its life. A closed batch cannot be re- -// opened. Tasks can never be removed from a batch. -// -// Type parameter TaskType must be a subclass of BatchTask. -template -class Batch { - public: - Batch() = default; - ~Batch(); // Blocks until the batch is closed. - - // Appends 'task' to the batch. After calling AddTask(), the newly-added task - // can be accessed via task(num_tasks()-1) or mutable_task(num_tasks()-1). - // Dies if the batch is closed. - void AddTask(std::unique_ptr task); - - // Removes the most recently added task. Returns nullptr if the batch is - // empty. - std::unique_ptr RemoveTask(); - - // Returns the number of tasks in the batch. - int num_tasks() const; - - // Returns true iff the batch contains 0 tasks. - bool empty() const; - - // Returns a reference to the ith task (in terms of insertion order). - const TaskType& task(int i) const; - - // Returns a pointer to the ith task (in terms of insertion order). - TaskType* mutable_task(int i); - - // Returns the sum of the task sizes. - size_t size() const; - - // Returns true iff the batch is currently closed. - bool IsClosed() const; - - // Blocks until the batch is closed. - void WaitUntilClosed() const; - - // Marks the batch as closed. Dies if called more than once. - void Close(); - - private: - mutable mutex mu_; - - // The tasks in the batch. - std::vector> tasks_ GUARDED_BY(mu_); - - // The sum of the sizes of the tasks in 'tasks_'. - size_t size_ GUARDED_BY(mu_) = 0; - - // Whether the batch has been closed. - Notification closed_; - - TF_DISALLOW_COPY_AND_ASSIGN(Batch); -}; - -// An abstract batch scheduler class. Collects individual tasks into batches, -// and processes each batch on a pool of "batch threads" that it manages. The -// actual logic for processing a batch is accomplished via a callback. -// -// Type parameter TaskType must be a subclass of BatchTask. -template -class BatchScheduler { - public: - virtual ~BatchScheduler() = default; - - // Submits a task to be processed as part of a batch. - // - // Ownership of '*task' is transferred to the callee iff the method returns - // Status::OK. In that case, '*task' is left as nullptr. Otherwise, '*task' is - // left as-is. - // - // If no batch processing capacity is available to process this task at the - // present time, and any task queue maintained by the implementing subclass is - // full, this method returns an UNAVAILABLE error code. The client may retry - // later. - // - // Other problems, such as the task size being larger than the maximum batch - // size, yield other, permanent error types. - // - // In all cases, this method returns "quickly" without blocking for any - // substantial amount of time. If the method returns Status::OK, the task is - // processed asynchronously, and any errors that occur during the processing - // of the batch that includes the task can be reported to 'task'. - virtual Status Schedule(std::unique_ptr* task) = 0; - - // Returns the number of tasks that have been scheduled (i.e. accepted by - // Schedule()), but have yet to be handed to a thread for execution as part of - // a batch. Note that this returns the number of tasks, not the aggregate task - // size (so if there is one task of size 3 and one task of size 5, this method - // returns 2 rather than 8). - virtual size_t NumEnqueuedTasks() const = 0; - - // Returns a guaranteed number of size 1 tasks that can be Schedule()d without - // getting an UNAVAILABLE error. In a typical implementation, returns the - // available space on a queue. - // - // There are two important caveats: - // 1. The guarantee does not extend to varying-size tasks due to possible - // internal fragmentation of batches. - // 2. The guarantee only holds in a single-thread environment or critical - // section, i.e. if an intervening thread cannot call Schedule(). - // - // This method is useful for monitoring, or for guaranteeing a future slot in - // the schedule (but being mindful about the caveats listed above). - virtual size_t SchedulingCapacity() const = 0; -}; - -////////// -// Implementation details follow. API users need not read. - -template -Batch::~Batch() { - WaitUntilClosed(); -} - -template -void Batch::AddTask(std::unique_ptr task) { - DCHECK(!IsClosed()); - { - mutex_lock l(mu_); - size_ += task->size(); - tasks_.push_back(std::move(task)); - } -} - -template -std::unique_ptr Batch::RemoveTask() { - { - mutex_lock l(mu_); - if (tasks_.empty()) { - return nullptr; - } - std::unique_ptr task = std::move(tasks_.back()); - tasks_.pop_back(); - return task; - } -} - -template -int Batch::num_tasks() const { - { - mutex_lock l(mu_); - return tasks_.size(); - } -} - -template -bool Batch::empty() const { - { - mutex_lock l(mu_); - return tasks_.empty(); - } -} - -template -const TaskType& Batch::task(int i) const { - DCHECK_GE(i, 0); - { - mutex_lock l(mu_); - DCHECK_LT(i, tasks_.size()); - return *tasks_[i].get(); - } -} - -template -TaskType* Batch::mutable_task(int i) { - DCHECK_GE(i, 0); - { - mutex_lock l(mu_); - DCHECK_LT(i, tasks_.size()); - return tasks_[i].get(); - } -} - -template -size_t Batch::size() const { - { - mutex_lock l(mu_); - return size_; - } -} - -template -bool Batch::IsClosed() const { - return const_cast(&closed_)->HasBeenNotified(); -} - -template -void Batch::WaitUntilClosed() const { - const_cast(&closed_)->WaitForNotification(); -} - -template -void Batch::Close() { - closed_.Notify(); -} - -} // namespace serving -} // namespace tensorflow +// TODO(b/36404166): Remove this shim after migrating all clients. +#include "tensorflow/contrib/batching/batch_scheduler.h" #endif // TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/batch_scheduler_retrier_test.cc b/tensorflow_serving/batching/batch_scheduler_retrier_test.cc index 84afd80b377..acaf7f68bd3 100644 --- a/tensorflow_serving/batching/batch_scheduler_retrier_test.cc +++ b/tensorflow_serving/batching/batch_scheduler_retrier_test.cc @@ -18,13 +18,13 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" #include "tensorflow/core/lib/core/error_codes.pb.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/batching/batch_scheduler_test.cc b/tensorflow_serving/batching/batch_scheduler_test.cc deleted file mode 100644 index 6efe313653b..00000000000 --- a/tensorflow_serving/batching/batch_scheduler_test.cc +++ /dev/null @@ -1,117 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/batching/batch_scheduler.h" - -#include -#include -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/macros.h" - -namespace tensorflow { -namespace serving { -namespace { - -class FakeTask : public BatchTask { - public: - explicit FakeTask(size_t size) : size_(size) {} - - ~FakeTask() override = default; - - size_t size() const override { return size_; } - - private: - const size_t size_; - - TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); -}; - -TEST(BatchTest, Basic) { - Batch batch; - - EXPECT_EQ(0, batch.num_tasks()); - EXPECT_TRUE(batch.empty()); - EXPECT_EQ(0, batch.size()); - EXPECT_FALSE(batch.IsClosed()); - - auto task0 = new FakeTask(3); - batch.AddTask(std::unique_ptr(task0)); - - EXPECT_EQ(1, batch.num_tasks()); - EXPECT_FALSE(batch.empty()); - EXPECT_EQ(task0->size(), batch.size()); - EXPECT_EQ(task0->size(), batch.task(0).size()); - EXPECT_FALSE(batch.IsClosed()); - - auto task1 = new FakeTask(7); - batch.AddTask(std::unique_ptr(task1)); - - EXPECT_EQ(2, batch.num_tasks()); - EXPECT_FALSE(batch.empty()); - EXPECT_EQ(task0->size() + task1->size(), batch.size()); - EXPECT_EQ(task1->size(), batch.task(1).size()); - EXPECT_EQ(task1->size(), batch.mutable_task(1)->size()); - EXPECT_FALSE(batch.IsClosed()); - - batch.Close(); - EXPECT_TRUE(batch.IsClosed()); - - EXPECT_EQ(2, batch.num_tasks()); - EXPECT_FALSE(batch.empty()); - EXPECT_EQ(task0->size() + task1->size(), batch.size()); - EXPECT_EQ(task0->size(), batch.task(0).size()); - EXPECT_EQ(task1->size(), batch.task(1).size()); - - EXPECT_EQ(7, batch.RemoveTask()->size()); - EXPECT_EQ(3, batch.RemoveTask()->size()); - EXPECT_TRUE(batch.empty()); -} - -TEST(BatchTest, WaitUntilClosed) { - Batch batch; - batch.AddTask(std::unique_ptr(new FakeTask(3))); - EXPECT_FALSE(batch.IsClosed()); - - std::unique_ptr close_thread( - Env::Default()->StartThread(ThreadOptions(), "test", [&batch]() { - Env::Default()->SleepForMicroseconds(100); - batch.Close(); - })); - batch.WaitUntilClosed(); - EXPECT_TRUE(batch.IsClosed()); -} - -TEST(BatchTest, DeletionBlocksUntilClosed) { - Batch* batch = new Batch; - batch->AddTask(std::unique_ptr(new FakeTask(3))); - EXPECT_FALSE(batch->IsClosed()); - - Notification do_delete, deleted; - std::unique_ptr delete_thread(Env::Default()->StartThread( - ThreadOptions(), "test", [&batch, &do_delete, &deleted]() { - do_delete.WaitForNotification(); - delete batch; - deleted.Notify(); - })); - do_delete.Notify(); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(deleted.HasBeenNotified()); - batch->Close(); - deleted.WaitForNotification(); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/batching/shared_batch_scheduler.h b/tensorflow_serving/batching/shared_batch_scheduler.h index 32351dc0ff4..9d03f46f86f 100644 --- a/tensorflow_serving/batching/shared_batch_scheduler.h +++ b/tensorflow_serving/batching/shared_batch_scheduler.h @@ -16,685 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ #define TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ -#include -#include -#include -#include -#include -#include -#include -#include - -#include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/cpu_info.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/batching/batch_scheduler.h" -#include "tensorflow_serving/util/periodic_function.h" - -namespace tensorflow { -namespace serving { -namespace internal { -template -class Queue; -} // namespace internal -} // namespace serving -} // namespace tensorflow - -namespace tensorflow { -namespace serving { - -// A batch scheduler for server instances that service multiple request types -// (e.g. multiple machine-learned models, or multiple versions of a model served -// concurrently), or even multiple distinct tasks for a given request. The -// scheduler multiplexes batches of different kinds of tasks onto a fixed-size -// thread pool (each batch contains tasks of a single type), in a carefully -// controlled manner. A common configuration is to set the number of threads -// equal to the number of hardware accelerator units, in which case the -// scheduler takes care of multiplexing the task types onto the shared hardware, -// in a manner that is both fair and efficient. -// -// Semantically, SharedBatchScheduler behaves like having N instances of -// BasicBatchScheduler (see basic_batch_scheduler.h), one per task type. The -// difference is that under the covers there is a single shared thread pool, -// instead of N independent ones, with their sharing deliberately coordinated. -// -// SharedBatchScheduler does not implement the BatchScheduler API; rather, it -// presents an abstraction of "queues", where each queue coresponds to one type -// of task. Tasks submitted to a given queue are placed in their own batches, -// and cannot be mixed with other tasks. Queues can be added and deleted -// dynamically, to accommodate e.g. versions of a model being brought up and -// down over the lifetime of a server. -// -// The batch thread pool round-robins through the queues, running one batch -// from a queue and then moving to the next queue. Each queue behaves like a -// BasicBatchScheduler instance, in the sense that it has maximum batch size and -// timeout parameters, which govern when a batch is eligible to be processed. -// -// Each queue is independently configured with a maximum size (in terms of the -// maximum number of batches worth of enqueued tasks). For online serving, it is -// recommended that the queue sizes be configured such that the sum of the sizes -// of the active queues roughly equal the number of batch threads. (The idea is -// that if all threads become available at roughly the same time, there will be -// enough enqueued work for them to take on, but no more.) -// -// If queue sizes are configured in the manner suggested above, the maximum time -// a task can spend in a queue before being placed in a batch and assigned to a -// thread for processing, is the greater of: -// - the maximum time to process one batch of tasks from any active queue -// - the configured timeout parameter for the task's queue (which can be 0) -// -// For bulk processing jobs and throughput-oriented benchmarks, you may want to -// set the maximum queue size to a large value. -// -// TODO(b/26539183): Support queue servicing policies other than round-robin. -// E.g. let each queue specify a "share" (an int >= 1), so e.g. with queues A -// and B having shares 1 and 2 respectively, the servicing pattern is ABBABB... -// -// -// PERFORMANCE TUNING: See README.md. -// -template -class SharedBatchScheduler - : public std::enable_shared_from_this> { - public: - // TODO(b/25089730): Tune defaults based on best practices as they develop. - struct Options { - // The name to use for the pool of batch threads. - string thread_pool_name = {"batch_threads"}; - - // The number of threads to use to process batches. - // Must be >= 1, and should be tuned carefully. - int num_batch_threads = port::NumSchedulableCPUs(); - - // The environment to use. - // (Typically only overridden by test code.) - Env* env = Env::Default(); - }; - // Ownership is shared between the caller of Create() and any queues created - // via AddQueue(). - static Status Create( - const Options& options, - std::shared_ptr>* scheduler); - - ~SharedBatchScheduler(); - - // Adds a queue to which tasks may be submitted. The returned queue implements - // the BatchScheduler API. Each queue has its own set of scheduling options, - // and its own callback to process batches of tasks submitted to the queue. - // - // The returned queue's destructor blocks until all tasks submitted to it have - // been processed. - struct QueueOptions { - // The maximum size of each batch. - // - // The scheduler may form batches of any size between 1 and this number - // (inclusive). If there is a need to quantize the batch sizes, i.e. only - // submit batches whose size is in a small set of allowed sizes, that can be - // done by adding padding in the process-batch callback. - int max_batch_size = 1000; - - // If a task has been enqueued for this amount of time (in microseconds), - // and a thread is available, the scheduler will immediately form a batch - // from enqueued tasks and assign the batch to the thread for processing, - // even if the batch's size is below 'max_batch_size'. - // - // This parameter offers a way to bound queue latency, so that a task isn't - // stuck in the queue indefinitely waiting for enough tasks to arrive to - // make a full batch. (The latency bound is given in the class documentation - // above.) - // - // The goal is to smooth out batch sizes under low request rates, and thus - // avoid latency spikes. - int64 batch_timeout_micros = 0; - - // The maximum allowable number of enqueued (accepted by Schedule() but - // not yet being processed on a batch thread) tasks in terms of batches. - // If this limit is reached, Schedule() will return an UNAVAILABLE error. - // See the class documentation above for guidelines on how to tune this - // parameter. - int max_enqueued_batches = 10; - }; - Status AddQueue(const QueueOptions& options, - std::function>)> - process_batch_callback, - std::unique_ptr>* queue); - - private: - explicit SharedBatchScheduler(const Options& options); - - // The code executed in 'batch_threads_'. Obtains a batch to process from the - // queue pointed to by 'next_queue_to_schedule_', and processes it. If that - // queue declines to provide a batch to process, moves onto the next queue. If - // no queues provide a batch to process, just sleeps briefly and exits. - void ThreadLogic(); - - const Options options_; - - mutex mu_; - - // A list of queues. (We use std::list instead of std::vector to ensure that - // iterators are not invalidated by adding/removing elements. It also offers - // efficient removal of elements from the middle.) - using QueueList = std::list>>; - - // All "active" queues, i.e. ones that either: - // - have not been removed, or - // - have been removed but are not yet empty. - QueueList queues_ GUARDED_BY(mu_); - - // An iterator over 'queues_', pointing to the queue from which the next - // available batch thread should grab work. - typename QueueList::iterator next_queue_to_schedule_ GUARDED_BY(mu_); - - // Used by idle batch threads to wait for work to enter the system. Notified - // whenever a batch becomes schedulable. - condition_variable schedulable_batch_cv_; - - // Threads that process batches obtained from the queues. - std::vector> batch_threads_; - - TF_DISALLOW_COPY_AND_ASSIGN(SharedBatchScheduler); -}; - -////////// -// Implementation details follow. API users need not read. - -namespace internal { - -// A task queue for SharedBatchScheduler. Accepts tasks and accumulates them -// into batches, and dispenses those batches to be processed via a "pull" -// interface. The queue's behavior is governed by maximum batch size, timeout -// and maximum queue length parameters; see their documentation in -// SharedBatchScheduler. -// -// The queue is implemented as a deque of batches, with these invariants: -// - The number of batches is between 1 and 'options_.max_enqueued_batches'. -// - The back-most batch is open; the rest are closed. -// -// Submitted tasks are added to the open batch. If that batch doesn't have room -// but the queue isn't full, then that batch is closed and a new open batch is -// started. -// -// Batch pull requests are handled by dequeuing the front-most batch if it is -// closed. If the front-most batch is open (i.e. the queue contains only one -// batch) and has reached the timeout, it is immediately closed and returned; -// otherwise no batch is returned for the request. -template -class Queue { - public: - using ProcessBatchCallback = - std::function>)>; - using SchedulableBatchCallback = std::function; - Queue(const typename SharedBatchScheduler::QueueOptions& options, - Env* env, ProcessBatchCallback process_batch_callback, - SchedulableBatchCallback schdulable_batch_callback); - - // Illegal to destruct unless the queue is empty. - ~Queue(); - - // Submits a task to the queue, with the same semantics as - // BatchScheduler::Schedule(). - Status Schedule(std::unique_ptr* task); - - // Returns the number of enqueued tasks, with the same semantics as - // BatchScheduler::NumEnqueuedTasks(). - size_t NumEnqueuedTasks() const; - - // Returns the queue capacity, with the same semantics as - // BatchScheduler::SchedulingCapacity(). - size_t SchedulingCapacity() const; - - // Called by a thread that is ready to process a batch, to request one from - // this queue. Either returns a batch that is ready to be processed, or - // nullptr if the queue declines to schedule a batch at this time. If it - // returns a batch, the batch is guaranteed to be closed. - std::unique_ptr> ScheduleBatch(); - - // Processes a batch that has been returned earlier by ScheduleBatch(). - void ProcessBatch(std::unique_ptr> batch); - - // Determines whether the queue is empty, i.e. has no tasks waiting or being - // processed. - bool IsEmpty() const; - - // Marks the queue closed, and waits until it is empty. - void CloseAndWaitUntilEmpty(); - - bool closed() const { - mutex_lock l(mu_); - return closed_; - } - - private: - // Same as IsEmpty(), but assumes the caller already holds a lock on 'mu_'. - bool IsEmptyInternal() const EXCLUSIVE_LOCKS_REQUIRED(mu_); - - // Closes the open batch residing at the back of 'batches_', and inserts a - // fresh open batch behind it. - void StartNewBatch() EXCLUSIVE_LOCKS_REQUIRED(mu_); - - // Determines whether the open batch residing at the back of 'batches_' is - // currently schedulable. - bool IsOpenBatchSchedulable() const EXCLUSIVE_LOCKS_REQUIRED(mu_); - - const typename SharedBatchScheduler::QueueOptions options_; - - // The environment to use. - Env* env_; - - // A callback invoked to processes a batch of work units. Always invoked from - // a batch thread. - ProcessBatchCallback process_batch_callback_; - - // A callback invoked to notify the scheduler that a new batch has become - // schedulable. - SchedulableBatchCallback schedulable_batch_callback_; - - mutable mutex mu_; - - // Whether this queue can accept new tasks. This variable is monotonic: it - // starts as false, and then at some point gets set to true and remains true - // for the duration of this object's life. - bool closed_ GUARDED_BY(mu_) = false; - - // The enqueued batches. See the invariants in the class comments above. - std::deque>> batches_ GUARDED_BY(mu_); - - // The time at which the first task was added to the open (back-most) batch - // in 'batches_'. Valid iff that batch contains at least one task. - uint64 open_batch_start_time_micros_ GUARDED_BY(mu_); - - // Whether this queue contains a batch that is eligible to be scheduled. Used - // to keep track of when to call 'schedulable_batch_callback_'. - bool schedulable_batch_ GUARDED_BY(mu_) = false; - - // The number of batches currently being processed by batch threads. - // Incremented in ScheduleBatch() and decremented in ProcessBatch(). - int num_batches_being_processed_ GUARDED_BY(mu_) = 0; - - // Used by CloseAndWaitUntilEmpty() to wait until the queue is empty, for the - // case in which the queue is not empty when CloseAndWaitUntilEmpty() starts. - // When ProcessBatch() dequeues the last batch and makes the queue empty, if - // 'empty_notification_' is non-null it calls 'empty_notification_->Notify()'. - Notification* empty_notification_ GUARDED_BY(mu_) = nullptr; - - TF_DISALLOW_COPY_AND_ASSIGN(Queue); -}; - -// A RAII-style object that points to a Queue and implements -// the BatchScheduler API. To be handed out to clients who call AddQueue(). -template -class QueueHandle : public BatchScheduler { - public: - QueueHandle(std::shared_ptr> scheduler, - Queue* queue); - ~QueueHandle() override; - - Status Schedule(std::unique_ptr* task) override; - size_t NumEnqueuedTasks() const override; - size_t SchedulingCapacity() const override; - - private: - // The scheduler that owns 'queue_'. - std::shared_ptr> scheduler_; - - // The queue this handle wraps. Owned by 'scheduler_', which keeps it alive at - // least until this class's destructor closes it. - Queue* queue_; - - TF_DISALLOW_COPY_AND_ASSIGN(QueueHandle); -}; - -} // namespace internal - -template -Status SharedBatchScheduler::Create( - const Options& options, - std::shared_ptr>* scheduler) { - if (options.num_batch_threads < 1) { - return errors::InvalidArgument("num_batch_threads must be positive; was ", - options.num_batch_threads); - } - scheduler->reset(new SharedBatchScheduler(options)); - return Status::OK(); -} - -template -SharedBatchScheduler::~SharedBatchScheduler() { - // Wait until the batch threads finish clearing out and deleting the closed - // queues. - for (;;) { - { - mutex_lock l(mu_); - if (queues_.empty()) { - break; - } - } - const int64 kSleepTimeMicros = 100; - options_.env->SleepForMicroseconds(kSleepTimeMicros); - } - // Delete the batch threads before allowing state the threads may access (e.g. - // 'mu_') to be deleted. - batch_threads_.clear(); -} - -template -Status SharedBatchScheduler::AddQueue( - const QueueOptions& options, - std::function>)> - process_batch_callback, - std::unique_ptr>* queue) { - if (options.max_batch_size <= 0) { - return errors::InvalidArgument("max_batch_size must be positive; was ", - options.max_batch_size); - } - if (options.batch_timeout_micros < 0) { - return errors::InvalidArgument( - "batch_timeout_micros must be non-negative; was ", - options.batch_timeout_micros); - } - if (options.max_enqueued_batches < 0) { - return errors::InvalidArgument( - "max_enqueued_batches must be non-negative; was ", - options.max_enqueued_batches); - } - - auto schedulable_batch_callback = [this] { - mutex_lock l(mu_); - schedulable_batch_cv_.notify_one(); - }; - auto internal_queue = - std::unique_ptr>(new internal::Queue( - options, options_.env, process_batch_callback, - schedulable_batch_callback)); - auto handle = std::unique_ptr>( - new internal::QueueHandle(this->shared_from_this(), - internal_queue.get())); - { - mutex_lock l(mu_); - queues_.push_back(std::move(internal_queue)); - if (next_queue_to_schedule_ == queues_.end()) { - next_queue_to_schedule_ = queues_.begin(); - } - } - *queue = std::move(handle); - return Status::OK(); -} - -template -SharedBatchScheduler::SharedBatchScheduler(const Options& options) - : options_(options), next_queue_to_schedule_(queues_.end()) { - // Kick off the batch threads. - PeriodicFunction::Options periodic_fn_options; - periodic_fn_options.thread_name_prefix = - strings::StrCat(options.thread_pool_name, "_"); - for (int i = 0; i < options.num_batch_threads; ++i) { - std::unique_ptr thread(new PeriodicFunction( - [this] { this->ThreadLogic(); }, - 0 /* function invocation interval time */, periodic_fn_options)); - batch_threads_.push_back(std::move(thread)); - } -} - -template -void SharedBatchScheduler::ThreadLogic() { - // A batch to process next (or nullptr if no work to do). - std::unique_ptr> batch_to_process; - // The queue with which 'batch_to_process' is associated. - internal::Queue* queue_for_batch = nullptr; - { - mutex_lock l(mu_); - - const int num_queues = queues_.size(); - for (int num_queues_tried = 0; - batch_to_process == nullptr && num_queues_tried < num_queues; - ++num_queues_tried) { - DCHECK(next_queue_to_schedule_ != queues_.end()); - - // If a closed queue responds to ScheduleBatch() with nullptr, the queue - // will never yield any further batches so we can drop it. To avoid a - // race, we take a snapshot of the queue's closedness state *before* - // calling ScheduleBatch(). - const bool queue_closed = (*next_queue_to_schedule_)->closed(); - - // Ask '*next_queue_to_schedule_' if it wants us to process a batch. - batch_to_process = (*next_queue_to_schedule_)->ScheduleBatch(); - if (batch_to_process != nullptr) { - queue_for_batch = next_queue_to_schedule_->get(); - } - - // Advance 'next_queue_to_schedule_'. - if (queue_closed && (*next_queue_to_schedule_)->IsEmpty() && - batch_to_process == nullptr) { - // We've encountered a closed queue with no work to do. Drop it. - DCHECK_NE(queue_for_batch, next_queue_to_schedule_->get()); - next_queue_to_schedule_ = queues_.erase(next_queue_to_schedule_); - } else { - ++next_queue_to_schedule_; - } - if (next_queue_to_schedule_ == queues_.end() && !queues_.empty()) { - // We've hit the end. Wrap to the first queue. - next_queue_to_schedule_ = queues_.begin(); - } - } - - if (batch_to_process == nullptr) { - // We couldn't find any work to do. Wait until a new batch becomes - // schedulable, or some time has elapsed, before checking again. - const int64 kTimeoutMillis = 1; // The smallest accepted granule of time. - WaitForMilliseconds(&l, &schedulable_batch_cv_, kTimeoutMillis); - return; - } - } - - queue_for_batch->ProcessBatch(std::move(batch_to_process)); -} - -namespace internal { - -template -Queue::Queue( - const typename SharedBatchScheduler::QueueOptions& options, - Env* env, ProcessBatchCallback process_batch_callback, - SchedulableBatchCallback schedulable_batch_callback) - : options_(options), - env_(env), - process_batch_callback_(process_batch_callback), - schedulable_batch_callback_(schedulable_batch_callback) { - // Create an initial, open batch. - batches_.emplace_back(new Batch); -} - -template -Queue::~Queue() { - mutex_lock l(mu_); - DCHECK(IsEmptyInternal()); - - // Close the (empty) open batch, so its destructor doesn't block. - batches_.back()->Close(); -} - -template -Status Queue::Schedule(std::unique_ptr* task) { - if ((*task)->size() > options_.max_batch_size) { - return errors::InvalidArgument("Task size ", (*task)->size(), - " is larger than maximum batch size ", - options_.max_batch_size); - } - - bool notify_of_schedulable_batch = false; - { - mutex_lock l(mu_); - - DCHECK(!closed_); - - if (batches_.back()->size() + (*task)->size() > options_.max_batch_size) { - if (batches_.size() >= options_.max_enqueued_batches) { - return errors::Unavailable( - "The batch scheduling queue to which this task was submitted is " - "full"); - } - StartNewBatch(); - } - if (batches_.back()->empty()) { - open_batch_start_time_micros_ = env_->NowMicros(); - } - batches_.back()->AddTask(std::move(*task)); - - if (!schedulable_batch_) { - if (batches_.size() > 1 || IsOpenBatchSchedulable()) { - schedulable_batch_ = true; - notify_of_schedulable_batch = true; - } - } - } - - if (notify_of_schedulable_batch) { - schedulable_batch_callback_(); - } - - return Status::OK(); -} - -template -size_t Queue::NumEnqueuedTasks() const { - mutex_lock l(mu_); - size_t num_enqueued_tasks = 0; - for (const auto& batch : batches_) { - num_enqueued_tasks += batch->num_tasks(); - } - return num_enqueued_tasks; -} - -template -size_t Queue::SchedulingCapacity() const { - mutex_lock l(mu_); - const int num_new_batches_schedulable = - options_.max_enqueued_batches - batches_.size(); - const int open_batch_capacity = - options_.max_batch_size - batches_.back()->size(); - return (num_new_batches_schedulable * options_.max_batch_size) + - open_batch_capacity; -} - -template -std::unique_ptr> Queue::ScheduleBatch() { - // The batch to schedule, which we may populate below. (If left as nullptr, - // that means we are electing not to schedule a batch at this time.) - std::unique_ptr> batch_to_schedule; - - { - mutex_lock l(mu_); - - // Consider closing the open batch at this time, to schedule it. - if (batches_.size() == 1 && IsOpenBatchSchedulable()) { - StartNewBatch(); - } - - if (batches_.size() >= 2) { - // There is at least one closed batch that is ready to be scheduled. - ++num_batches_being_processed_; - batch_to_schedule = std::move(batches_.front()); - batches_.pop_front(); - } else { - schedulable_batch_ = false; - } - } - - return batch_to_schedule; -} - -template -void Queue::ProcessBatch(std::unique_ptr> batch) { - process_batch_callback_(std::move(batch)); - - { - mutex_lock l(mu_); - --num_batches_being_processed_; - if (empty_notification_ != nullptr && IsEmptyInternal()) { - empty_notification_->Notify(); - } - } -} - -template -bool Queue::IsEmpty() const { - mutex_lock l(mu_); - return IsEmptyInternal(); -} - -template -void Queue::CloseAndWaitUntilEmpty() { - Notification empty; - { - mutex_lock l(mu_); - closed_ = true; - if (IsEmptyInternal()) { - empty.Notify(); - } else { - // Arrange for ProcessBatch() to notify when the queue becomes empty. - empty_notification_ = ∅ - } - } - empty.WaitForNotification(); -} - -template -bool Queue::IsEmptyInternal() const { - return num_batches_being_processed_ == 0 && batches_.size() == 1 && - batches_.back()->empty(); -} - -template -void Queue::StartNewBatch() { - batches_.back()->Close(); - batches_.emplace_back(new Batch); -} - -template -bool Queue::IsOpenBatchSchedulable() const { - Batch* open_batch = batches_.back().get(); - if (open_batch->empty()) { - return false; - } - return closed_ || open_batch->size() >= options_.max_batch_size || - env_->NowMicros() >= - open_batch_start_time_micros_ + options_.batch_timeout_micros; -} - -template -QueueHandle::QueueHandle( - std::shared_ptr> scheduler, - Queue* queue) - : scheduler_(scheduler), queue_(queue) {} - -template -QueueHandle::~QueueHandle() { - queue_->CloseAndWaitUntilEmpty(); -} - -template -Status QueueHandle::Schedule(std::unique_ptr* task) { - return queue_->Schedule(task); -} - -template -size_t QueueHandle::NumEnqueuedTasks() const { - return queue_->NumEnqueuedTasks(); -} - -template -size_t QueueHandle::SchedulingCapacity() const { - return queue_->SchedulingCapacity(); -} - -} // namespace internal - -} // namespace serving -} // namespace tensorflow +// TODO(b/36404166): Remove this shim after migrating all clients. +#include "tensorflow/contrib/batching/shared_batch_scheduler.h" #endif // TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/shared_batch_scheduler_test.cc b/tensorflow_serving/batching/shared_batch_scheduler_test.cc deleted file mode 100644 index be236818906..00000000000 --- a/tensorflow_serving/batching/shared_batch_scheduler_test.cc +++ /dev/null @@ -1,593 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/batching/shared_batch_scheduler.h" - -#include -#include -#include "tensorflow/core/lib/core/error_codes.pb.h" -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" - -using ::testing::ElementsAre; -using ::testing::IsEmpty; -using ::testing::UnorderedElementsAre; - -namespace tensorflow { -namespace serving { -namespace { - -class FakeTask : public BatchTask { - public: - explicit FakeTask(size_t size) : size_(size) {} - - ~FakeTask() override = default; - - size_t size() const override { return size_; } - - private: - const size_t size_; - - TF_DISALLOW_COPY_AND_ASSIGN(FakeTask); -}; - -// Creates a FakeTask of size 'task_size', and calls 'scheduler->Schedule()' on -// that task. Returns the resulting status. -Status ScheduleTask(size_t task_size, BatchScheduler* scheduler) { - std::unique_ptr task(new FakeTask(task_size)); - Status status = scheduler->Schedule(&task); - // Schedule() should have consumed 'task' iff it returned Status::OK. - CHECK_EQ(status.ok(), task == nullptr); - return status; -} - -// Creates a thread that waits on 'start' and then advances the fake clock in -// 'env' in a loop until 'stop' is notified. Useful for allowing objects that -// use the clock to be destroyed. -std::unique_ptr CreateFakeClockAdvancerThread( - test_util::FakeClockEnv* env, Notification* start, Notification* stop) { - return std::unique_ptr( - Env::Default()->StartThread({}, "FakeClockAdvancerThread", - [env, start, stop] { - start->WaitForNotification(); - while (!stop->HasBeenNotified()) { - env->AdvanceByMicroseconds(10); - Env::Default()->SleepForMicroseconds(10); - } - })); -} - -TEST(SharedBatchSchedulerTest, Basic) { - for (int num_batch_threads : {1, 2, 3}) { - for (const bool delete_scheduler_early : {false, true}) { - for (const bool delete_queue_1_early : {false, true}) { - bool queue_0_callback_called = false; - auto queue_0_callback = - [&queue_0_callback_called](std::unique_ptr> batch) { - queue_0_callback_called = true; - ASSERT_TRUE(batch->IsClosed()); - ASSERT_EQ(3, batch->num_tasks()); - EXPECT_EQ(1, batch->task(0).size()); - EXPECT_EQ(3, batch->task(1).size()); - EXPECT_EQ(5, batch->task(2).size()); - }; - bool queue_1_callback_called = false; - auto queue_1_callback = - [&queue_1_callback_called](std::unique_ptr> batch) { - queue_1_callback_called = true; - ASSERT_TRUE(batch->IsClosed()); - ASSERT_EQ(2, batch->num_tasks()); - EXPECT_EQ(2, batch->task(0).size()); - EXPECT_EQ(4, batch->task(1).size()); - }; - { - SharedBatchScheduler::Options options; - options.num_batch_threads = num_batch_threads; - std::shared_ptr> scheduler; - TF_ASSERT_OK( - SharedBatchScheduler::Create(options, &scheduler)); - - // Create two queues. - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 10 * 1000 * 1000; // 10 seconds - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue_0; - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); - std::unique_ptr> queue_1; - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); - - if (delete_scheduler_early) { - // Delete our copy of the scheduler. The queues should keep it alive - // under the covers. - scheduler = nullptr; - } - - // Submit tasks to the two queues, and (optionally) remove the queues. - TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); - TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); - TF_ASSERT_OK(ScheduleTask(3, queue_0.get())); - TF_ASSERT_OK(ScheduleTask(4, queue_1.get())); - if (delete_queue_1_early) { - queue_1 = nullptr; - } - TF_ASSERT_OK(ScheduleTask(5, queue_0.get())); - } - EXPECT_TRUE(queue_0_callback_called); - EXPECT_TRUE(queue_1_callback_called); - } - } - } -} - -TEST(SharedBatchSchedulerTest, ObeyBatchSizeConstraint) { - // Set up a callback that captures the batches' task sizes. - mutex mu; - std::vector> callback_data; - auto callback = [&mu, - &callback_data](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - std::vector batch_data; - for (int i = 0; i < batch->num_tasks(); ++i) { - batch_data.push_back(batch->mutable_task(i)->size()); - } - { - mutex_lock l(mu); - callback_data.push_back(batch_data); - } - }; - - // Run a batch scheduler and inject some tasks. - { - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 10 * 1000 * 1000; // 10 seconds - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // First batch. - TF_ASSERT_OK(ScheduleTask(3, queue.get())); - TF_ASSERT_OK(ScheduleTask(5, queue.get())); - - // Second batch (due to size overage). - TF_ASSERT_OK(ScheduleTask(3 /* (3+5) + 3 > 10 */, queue.get())); - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - TF_ASSERT_OK(ScheduleTask(6, queue.get())); - - // (Empty third batch, since the second batch exactly hit the size limit, - // which should never get sent to the callback.) - } - - // Expect a certain grouping of the tasks into batches. - EXPECT_THAT(callback_data, - UnorderedElementsAre(ElementsAre(3, 5), ElementsAre(3, 1, 6))); -} - -TEST(SharedBatchSchedulerTest, ObeysTimeout) { - // Set up a fake clock, which only advances when we explicitly tell it to. - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - Notification first_batch_processed, second_batch_processed, - third_batch_processed; - auto callback = - [&first_batch_processed, &second_batch_processed, - &third_batch_processed](std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - if (batch->size() == 1) { - first_batch_processed.Notify(); - } else if (batch->size() == 2) { - second_batch_processed.Notify(); - } else if (batch->size() == 3) { - third_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 4; - queue_options.batch_timeout_micros = 10; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // Create an underfull batch, and ensure that it gets processed when the - // clock hits the timeout. - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - env.AdvanceByMicroseconds(9); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(first_batch_processed.HasBeenNotified()); - env.AdvanceByMicroseconds(1); - first_batch_processed.WaitForNotification(); - - // Start creating a batch, while leaving the clock well below the timeout. - // Then submit a new task that overflows into the next batch, causing - // the original batch to close. - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(second_batch_processed.HasBeenNotified()); - TF_ASSERT_OK(ScheduleTask(3, queue.get())); - second_batch_processed.WaitForNotification(); - - // Allow the third batch to hit its timeout, and ensure it gets closed at - // the right time. - env.AdvanceByMicroseconds(9); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(third_batch_processed.HasBeenNotified()); - env.AdvanceByMicroseconds(1); - third_batch_processed.WaitForNotification(); - - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(SharedBatchSchedulerTest, ObeysTimeoutWithRealClock) { - Notification first_batch_processed, second_batch_processed; - auto callback = [&first_batch_processed, &second_batch_processed]( - std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - if (batch->size() == 1) { - first_batch_processed.Notify(); - } else if (batch->size() == 2) { - second_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 100 * 1000; // 100 milliseconds - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // Submit a single task that doesn't fill up the batch. - // Ensure that it gets processed due to the timeout. - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - first_batch_processed.WaitForNotification(); - - // Do it again. - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - second_batch_processed.WaitForNotification(); -} - -TEST(SharedBatchSchedulerTest, - WithZeroTimeoutBatchesScheduledAsSoonAsThreadIsAvailable) { - // Set up a fake clock, and never advance the time. - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - Notification first_batch_processed, second_batch_processed; - auto callback = [&first_batch_processed, &second_batch_processed]( - std::unique_ptr> batch) { - ASSERT_TRUE(batch->IsClosed()); - if (batch->size() == 1) { - first_batch_processed.Notify(); - } else if (batch->size() == 2) { - second_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - // Set a large batch size, so that we don't hit the batch size limit. - queue_options.max_batch_size = 100; - // Process a batch as soon as a thread is available. - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - first_batch_processed.WaitForNotification(); - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - second_batch_processed.WaitForNotification(); - - // Shut everything down. - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(SharedBatchSchedulerTest, Fairness) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - Notification queue_0_first_batch_scheduled, queue_0_first_batch_proceed, - queue_0_second_batch_scheduled; - auto queue_0_callback = [&queue_0_first_batch_scheduled, - &queue_0_first_batch_proceed, - &queue_0_second_batch_scheduled]( - std::unique_ptr> batch) { - if (!queue_0_first_batch_scheduled.HasBeenNotified()) { - queue_0_first_batch_scheduled.Notify(); - queue_0_first_batch_proceed.WaitForNotification(); - } else if (!queue_0_second_batch_scheduled.HasBeenNotified()) { - queue_0_second_batch_scheduled.Notify(); - } - }; - - Notification queue_1_first_batch_scheduled, queue_1_first_batch_proceed; - auto queue_1_callback = - [&queue_1_first_batch_scheduled, - &queue_1_first_batch_proceed](std::unique_ptr> batch) { - queue_1_first_batch_scheduled.Notify(); - queue_1_first_batch_proceed.WaitForNotification(); - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 1; - queue_options.max_enqueued_batches = 100 /* give plenty of room */; - std::vector>> queues(2); - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_0_callback, &queues[0])); - TF_ASSERT_OK( - scheduler->AddQueue(queue_options, queue_1_callback, &queues[1])); - - // Enqueue a batch-filling task to queue 0, and wait for it to get - // scheduled. - TF_ASSERT_OK(ScheduleTask(10, queues[0].get())); - env.AdvanceByMicroseconds(1); - queue_0_first_batch_scheduled.WaitForNotification(); - - // Enqueue two more batch-filling tasks to queue 0. - TF_ASSERT_OK(ScheduleTask(10, queues[0].get())); - TF_ASSERT_OK(ScheduleTask(10, queues[0].get())); - - // Enqueue one task to queue 1, and then advance the clock so it becomes - // eligible for scheduling due to the timeout. Ensure that the queue 1 batch - // gets scheduled before the next queue 0 one. - TF_ASSERT_OK(ScheduleTask(1, queues[1].get())); - env.AdvanceByMicroseconds(1); - queue_0_first_batch_proceed.Notify(); - queue_1_first_batch_scheduled.WaitForNotification(); - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(queue_0_second_batch_scheduled.HasBeenNotified()); - - // Shut everything down. - queue_1_first_batch_proceed.Notify(); - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -TEST(SharedBatchSchedulerTest, ConstMethods) { - for (const int max_enqueued_batches : {1, 2, 5}) { - Notification processing, proceed; - auto callback = [&processing, - &proceed](std::unique_ptr> batch) { - if (!processing.HasBeenNotified()) { - processing.Notify(); - } - proceed.WaitForNotification(); - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 2; - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = max_enqueued_batches; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - EXPECT_EQ(0, queue->NumEnqueuedTasks()); - EXPECT_EQ(max_enqueued_batches * 2, queue->SchedulingCapacity()); - - // Get one batch going on the thread, and keep the thread blocked until - // we're done testing the maximum queue length. - TF_ASSERT_OK(ScheduleTask(2, queue.get())); - processing.WaitForNotification(); - EXPECT_EQ(0, queue->NumEnqueuedTasks()); - - // We should be able to enqueue 'max_enqueued_batches'*2 tasks without - // issue. - for (int i = 0; i < max_enqueued_batches; ++i) { - EXPECT_EQ(i * 2, queue->NumEnqueuedTasks()); - EXPECT_EQ((max_enqueued_batches - i) * 2, queue->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - EXPECT_EQ((i * 2) + 1, queue->NumEnqueuedTasks()); - EXPECT_EQ((max_enqueued_batches - i) * 2 - 1, - queue->SchedulingCapacity()); - TF_ASSERT_OK(ScheduleTask(1, queue.get())); - } - EXPECT_EQ(max_enqueued_batches * 2, queue->NumEnqueuedTasks()); - EXPECT_EQ(0, queue->SchedulingCapacity()); - - // Attempting to enqueue one more task should yield an UNAVAILABLE error. - Status status = ScheduleTask(1, queue.get()); - ASSERT_FALSE(status.ok()); - EXPECT_EQ(error::UNAVAILABLE, status.code()); - EXPECT_EQ(max_enqueued_batches * 2, queue->NumEnqueuedTasks()); - EXPECT_EQ(0, queue->SchedulingCapacity()); - - proceed.Notify(); - } -} - -TEST(SharedBatchSchedulerTest, OneFullQueueDoesntBlockOtherQueues) { - Notification queue_0_processing, queue_0_proceed; - auto queue_0_callback = [&queue_0_processing, &queue_0_proceed]( - std::unique_ptr> batch) { - if (!queue_0_processing.HasBeenNotified()) { - queue_0_processing.Notify(); - queue_0_proceed.WaitForNotification(); - } - }; - - Notification queue_1_first_batch_processed, queue_1_second_batch_processed, - queue_1_third_batch_processed; - auto queue_1_callback = - [&queue_1_first_batch_processed, &queue_1_second_batch_processed, - &queue_1_third_batch_processed](std::unique_ptr> batch) { - if (batch->size() == 1) { - queue_1_first_batch_processed.Notify(); - } else if (batch->size() == 2) { - queue_1_second_batch_processed.Notify(); - } else if (batch->size() == 3) { - queue_1_third_batch_processed.Notify(); - } else { - EXPECT_TRUE(false) << "Unexpected batch size"; - } - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 2; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue_0; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_0_callback, &queue_0)); - std::unique_ptr> queue_1; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, queue_1_callback, &queue_1)); - - // Clog up queue 0. - TF_ASSERT_OK(ScheduleTask(1, queue_0.get())); - queue_0_processing.WaitForNotification(); - Status queue_0_status; - do { - queue_0_status = ScheduleTask(1, queue_0.get()); - } while (queue_0_status.ok()); - EXPECT_EQ(error::UNAVAILABLE, queue_0_status.code()); - - // Ensure that queue 1 still behaves normally, and lets us process tasks. - TF_ASSERT_OK(ScheduleTask(1, queue_1.get())); - queue_1_first_batch_processed.WaitForNotification(); - TF_ASSERT_OK(ScheduleTask(2, queue_1.get())); - queue_1_second_batch_processed.WaitForNotification(); - TF_ASSERT_OK(ScheduleTask(3, queue_1.get())); - queue_1_third_batch_processed.WaitForNotification(); - - // Let poor queue 0 drain. - queue_0_proceed.Notify(); -} - -TEST(SharedBatchSchedulerTest, QueueDestructorBlocksUntilAllTasksProcessed) { - test_util::FakeClockEnv env(Env::Default()); - Notification start_teardown, stop_teardown; - std::unique_ptr teardown_thread = - CreateFakeClockAdvancerThread(&env, &start_teardown, &stop_teardown); - - { - int current_batch = 0; - Notification first_callback_started; - const int kMaxEnqueuedBatches = 3; - std::vector callback_proceed(kMaxEnqueuedBatches); - auto callback = - [¤t_batch, &first_callback_started, - &callback_proceed](std::unique_ptr> batch) { - if (current_batch == 0) { - first_callback_started.Notify(); - } - callback_proceed[current_batch].WaitForNotification(); - ++current_batch; - }; - - SharedBatchScheduler::Options options; - options.num_batch_threads = 1; - options.env = &env; - std::shared_ptr> scheduler; - TF_ASSERT_OK(SharedBatchScheduler::Create(options, &scheduler)); - SharedBatchScheduler::QueueOptions queue_options; - queue_options.max_batch_size = 10; - queue_options.batch_timeout_micros = 0; - queue_options.max_enqueued_batches = 2; - std::unique_ptr> queue; - TF_ASSERT_OK(scheduler->AddQueue(queue_options, callback, &queue)); - - // Clog up the queue. - int num_enqueued_batches = 0; - TF_ASSERT_OK(ScheduleTask(10, queue.get())); - ++num_enqueued_batches; - env.AdvanceByMicroseconds(1); - first_callback_started.WaitForNotification(); - for (int i = 0; i < 2; ++i) { - TF_ASSERT_OK(ScheduleTask(10, queue.get())); - ++num_enqueued_batches; - } - EXPECT_EQ(kMaxEnqueuedBatches, num_enqueued_batches); - EXPECT_EQ(error::UNAVAILABLE, ScheduleTask(10, queue.get()).code()); - - // Destroy the queue. The destructor should block until all tasks have been - // processed. - Notification destroy_queue_thread_started, queue_destroyed; - std::unique_ptr destroy_queue_thread(Env::Default()->StartThread( - {}, "DestroyQueueThread", - [&queue, &destroy_queue_thread_started, &queue_destroyed] { - destroy_queue_thread_started.Notify(); - queue = nullptr; - queue_destroyed.Notify(); - })); - destroy_queue_thread_started.WaitForNotification(); - for (int i = 0; i < num_enqueued_batches; ++i) { - Env::Default()->SleepForMicroseconds(10 * 1000 /* 10 milliseconds */); - EXPECT_FALSE(queue_destroyed.HasBeenNotified()); - callback_proceed[i].Notify(); - } - - start_teardown.Notify(); - } - stop_teardown.Notify(); -} - -} // namespace -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/batching/streaming_batch_scheduler_test.cc b/tensorflow_serving/batching/streaming_batch_scheduler_test.cc index fefacd8a472..3f7ac4470d3 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler_test.cc +++ b/tensorflow_serving/batching/streaming_batch_scheduler_test.cc @@ -19,10 +19,10 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" #include "tensorflow/core/lib/core/error_codes.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" using ::testing::ElementsAre; using ::testing::IsEmpty; diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index e880d38d2b4..61d080e4be1 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -404,7 +404,7 @@ cc_test( deps = [ ":servable_state_monitor", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", + "@org_tensorflow//tensorflow/contrib/batching/test_util:fake_clock_env", ], ) @@ -502,7 +502,7 @@ cc_library( ":target", "//tensorflow_serving/util:event_bus", "//tensorflow_serving/util:optional", - "//tensorflow_serving/util:periodic_function", + "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -541,7 +541,7 @@ cc_test( ":servable_handle", ":simple_loader", "//tensorflow_serving/core/test_util:manager_test_util", - "//tensorflow_serving/util:periodic_function", + "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", @@ -571,7 +571,6 @@ cc_test( deps = [ ":static_manager", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", ], ) diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index db4342f18b3..a4b74cbb8cc 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -21,6 +21,7 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/util/periodic_function.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/hash/hash.h" @@ -37,7 +38,6 @@ limitations under the License. #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/target.h" #include "tensorflow_serving/util/optional.h" -#include "tensorflow_serving/util/periodic_function.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc index 6a73ec9edee..4e109201c2f 100644 --- a/tensorflow_serving/core/aspired_versions_manager_benchmark.cc +++ b/tensorflow_serving/core/aspired_versions_manager_benchmark.cc @@ -26,6 +26,7 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/util/periodic_function.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/threadpool.h" @@ -48,7 +49,6 @@ limitations under the License. #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/simple_loader.h" #include "tensorflow_serving/core/test_util/manager_test_util.h" -#include "tensorflow_serving/util/periodic_function.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/core/servable_state_monitor_test.cc b/tensorflow_serving/core/servable_state_monitor_test.cc index cb48eed42b9..e59a796551c 100644 --- a/tensorflow_serving/core/servable_state_monitor_test.cc +++ b/tensorflow_serving/core/servable_state_monitor_test.cc @@ -17,9 +17,9 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/logging.h" -#include "tensorflow_serving/test_util/fake_clock_env.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/sources/storage_path/BUILD b/tensorflow_serving/sources/storage_path/BUILD index 01aa40b2b13..b327b8a5af9 100644 --- a/tensorflow_serving/sources/storage_path/BUILD +++ b/tensorflow_serving/sources/storage_path/BUILD @@ -76,7 +76,7 @@ cc_library( "//tensorflow_serving/core:servable_id", "//tensorflow_serving/core:source", "//tensorflow_serving/core:storage_path", - "//tensorflow_serving/util:periodic_function", + "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", ], diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h index 15cbeaf4dec..672c954e99f 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h @@ -18,13 +18,13 @@ limitations under the License. #include +#include "tensorflow/contrib/batching/util/periodic_function.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/source.h" #include "tensorflow_serving/core/storage_path.h" #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" -#include "tensorflow_serving/util/periodic_function.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/test_util/BUILD b/tensorflow_serving/test_util/BUILD index 8f551cbdf6c..2908a9aab9b 100644 --- a/tensorflow_serving/test_util/BUILD +++ b/tensorflow_serving/test_util/BUILD @@ -32,17 +32,3 @@ cc_library( "@protobuf//:protobuf", ], ) - -cc_library( - name = "fake_clock_env", - testonly = 1, - srcs = ["fake_clock_env.cc"], - hdrs = ["fake_clock_env.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) diff --git a/tensorflow_serving/test_util/fake_clock_env.cc b/tensorflow_serving/test_util/fake_clock_env.cc deleted file mode 100644 index 85351510c4b..00000000000 --- a/tensorflow_serving/test_util/fake_clock_env.cc +++ /dev/null @@ -1,90 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/test_util/fake_clock_env.h" - -#include - -namespace tensorflow { -namespace serving { -namespace test_util { - -FakeClockEnv::FakeClockEnv(Env* wrapped) : EnvWrapper(wrapped) {} - -void FakeClockEnv::AdvanceByMicroseconds(int micros) { - { - mutex_lock l(mu_); - current_time_ += micros; - for (auto it = sleeping_threads_.begin(); it != sleeping_threads_.end();) { - if (current_time_ >= it->wake_time) { - it->wake_notification->Notify(); - it = sleeping_threads_.erase(it); - } else { - ++it; - } - } - } -} - -void FakeClockEnv::BlockUntilSleepingThread(uint64 wake_time) { - for (;;) { - { - mutex_lock l(mu_); - for (auto it = sleeping_threads_.begin(); it != sleeping_threads_.end(); - ++it) { - if (it->wake_time == wake_time) { - return; - } - } - } - EnvWrapper::SleepForMicroseconds(100); - } -} - -void FakeClockEnv::BlockUntilThreadsAsleep(int num_threads) { - for (;;) { - { - mutex_lock l(mu_); - if (num_threads <= sleeping_threads_.size()) { - return; - } - } - EnvWrapper::SleepForMicroseconds(100); - } -} - -uint64 FakeClockEnv::NowMicros() { - { - mutex_lock l(mu_); - return current_time_; - } -} - -void FakeClockEnv::SleepForMicroseconds(int64 micros) { - if (micros == 0) { - return; - } - - Notification wake_notification; - { - mutex_lock l(mu_); - sleeping_threads_.push_back({current_time_ + micros, &wake_notification}); - } - wake_notification.WaitForNotification(); -} - -} // namespace test_util -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/test_util/fake_clock_env.h b/tensorflow_serving/test_util/fake_clock_env.h deleted file mode 100644 index c4179d3ef5e..00000000000 --- a/tensorflow_serving/test_util/fake_clock_env.h +++ /dev/null @@ -1,76 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_TEST_UTIL_FAKE_CLOCK_ENV_H_ -#define TENSORFLOW_SERVING_TEST_UTIL_FAKE_CLOCK_ENV_H_ - -#include -#include -#include - -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/lib/core/status.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace serving { -namespace test_util { - -// An Env implementation with a fake clock for NowMicros() and -// SleepForMicroseconds(). The clock doesn't advance on its own; it advances via -// an explicit Advance() method. -// All other Env virtual methods pass through to a wrapped Env. -class FakeClockEnv : public EnvWrapper { - public: - explicit FakeClockEnv(Env* wrapped); - ~FakeClockEnv() override = default; - - // Advance the clock by a certain number of microseconds. - void AdvanceByMicroseconds(int micros); - - // Blocks until there is a sleeping thread that is scheduled to wake up at - // the given (absolute) time. - void BlockUntilSleepingThread(uint64 wake_time); - - // Blocks until there are at least num_threads sleeping. - void BlockUntilThreadsAsleep(int num_threads); - - // Methods that this class implements. - uint64 NowMicros() override; - void SleepForMicroseconds(int64 micros) override; - - private: - mutex mu_; - - uint64 current_time_ GUARDED_BY(mu_) = 0; - - struct SleepingThread { - uint64 wake_time; - Notification* wake_notification; - }; - std::vector sleeping_threads_ GUARDED_BY(mu_); - - TF_DISALLOW_COPY_AND_ASSIGN(FakeClockEnv); -}; - -} // namespace test_util -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_TEST_UTIL_FAKE_CLOCK_ENV_H_ diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 1637afb8d05..2fdeeff2c98 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -82,8 +82,8 @@ cc_test( srcs = ["observer_test.cc"], deps = [ ":observer", - ":periodic_function", "//tensorflow_serving/core/test_util:test_main", + "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -95,7 +95,7 @@ cc_test( deps = [ ":event_bus", "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", + "@org_tensorflow//tensorflow/contrib/batching/test_util:fake_clock_env", ], ) @@ -123,7 +123,7 @@ cc_test( srcs = ["fast_read_dynamic_ptr_benchmark.cc"], deps = [ ":fast_read_dynamic_ptr", - ":periodic_function", + "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", @@ -153,26 +153,6 @@ cc_test( ], ) -cc_library( - name = "periodic_function", - srcs = ["periodic_function.cc"], - hdrs = ["periodic_function.h"], - deps = [ - "@org_tensorflow//tensorflow/core:lib", - ], -) - -cc_test( - name = "periodic_function_test", - srcs = ["periodic_function_test.cc"], - deps = [ - ":periodic_function", - "//tensorflow_serving/core/test_util:test_main", - "//tensorflow_serving/test_util:fake_clock_env", - "@org_tensorflow//tensorflow/core:tensorflow", - ], -) - cc_test( name = "any_ptr_test", srcs = [ diff --git a/tensorflow_serving/util/event_bus_test.cc b/tensorflow_serving/util/event_bus_test.cc index 79fdcfd790c..5a815c29b2d 100644 --- a/tensorflow_serving/util/event_bus_test.cc +++ b/tensorflow_serving/util/event_bus_test.cc @@ -16,7 +16,7 @@ limitations under the License. #include "tensorflow_serving/util/event_bus.h" #include -#include "tensorflow_serving/test_util/fake_clock_env.h" +#include "tensorflow/contrib/batching/test_util/fake_clock_env.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc b/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc index cb468a95484..2e2987f34ef 100644 --- a/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc +++ b/tensorflow_serving/util/fast_read_dynamic_ptr_benchmark.cc @@ -38,6 +38,7 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/util/periodic_function.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/threadpool.h" #include "tensorflow/core/platform/env.h" @@ -46,7 +47,6 @@ limitations under the License. #include "tensorflow/core/platform/test_benchmark.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/util/fast_read_dynamic_ptr.h" -#include "tensorflow_serving/util/periodic_function.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/util/observer_test.cc b/tensorflow_serving/util/observer_test.cc index 501e5a3e79a..593362419dd 100644 --- a/tensorflow_serving/util/observer_test.cc +++ b/tensorflow_serving/util/observer_test.cc @@ -16,8 +16,8 @@ limitations under the License. #include "tensorflow_serving/util/observer.h" #include +#include "tensorflow/contrib/batching/util/periodic_function.h" #include "tensorflow/core/platform/env.h" -#include "tensorflow_serving/util/periodic_function.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/util/periodic_function.cc b/tensorflow_serving/util/periodic_function.cc deleted file mode 100644 index 4d52bf5f6bc..00000000000 --- a/tensorflow_serving/util/periodic_function.cc +++ /dev/null @@ -1,102 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/util/periodic_function.h" - -#include - -#include "tensorflow/core/lib/strings/strcat.h" -#include "tensorflow/core/platform/logging.h" - -namespace tensorflow { -namespace serving { - -PeriodicFunction::PeriodicFunction(const std::function& function, - const int64 interval_micros, - const Options& options) - : function_(function), - interval_micros_([interval_micros]() -> int64 { - if (interval_micros < 0) { - const string error = strings::StrCat( - " The value of 'interval_micros' should be >= 0: ", - interval_micros, ". "); - DCHECK(false) << error; - LOG(WARNING) << error << "Resetting it to 0."; - return 0; - } - return interval_micros; - }()), - options_(options) { - thread_.reset(options_.env->StartThread( - options_.thread_options, options_.thread_name_prefix, [this]() { - // Record the starting time here instead of in RunLoop. That way, if - // there is a delay starting RunLoop, that does not affect the timing - // of - // the first function. (Such a delay can often happen in tests where - // the test simulates a large time delay immediately after calling - // Start.) - RunLoop(options_.env->NowMicros()); - })); -} - -PeriodicFunction::~PeriodicFunction() { - NotifyStop(); - - // Waits for thread_ to complete and clean up. - thread_.reset(); -} - -void PeriodicFunction::NotifyStop() { - if (!stop_thread_.HasBeenNotified()) { - stop_thread_.Notify(); - } -} - -void PeriodicFunction::RunLoop(const int64 start) { - { - if (options_.startup_delay_micros > 0) { - const int64 deadline = start + options_.startup_delay_micros; - options_.env->SleepForMicroseconds(deadline - start); - } - - while (!stop_thread_.HasBeenNotified()) { - VLOG(3) << "Running function."; - const int64 begin = options_.env->NowMicros(); - function_(); - - // Take the max() here to guard against time going backwards which - // sometimes happens in multiproc machines. - const int64 end = - std::max(static_cast(options_.env->NowMicros()), begin); - - // The deadline is relative to when the last function started. - const int64 deadline = begin + interval_micros_; - - // We want to sleep until 'deadline'. - if (deadline > end) { - if (end > begin) { - VLOG(3) << "Reducing interval_micros from " << interval_micros_ - << " to " << (deadline - end); - } - options_.env->SleepForMicroseconds(deadline - end); - } else { - VLOG(3) << "Function took longer than interval_micros, so not sleeping"; - } - } - } -} - -} // namespace serving -} // namespace tensorflow diff --git a/tensorflow_serving/util/periodic_function.h b/tensorflow_serving/util/periodic_function.h deleted file mode 100644 index e489a8fe0cb..00000000000 --- a/tensorflow_serving/util/periodic_function.h +++ /dev/null @@ -1,132 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -// PeriodicFunction will periodically call the given function with a specified -// period in a background thread. After Start() returns, the thread is -// guaranteed to have started. The destruction of the class causes the -// background thread to be destroyed as well. Start() should not be called more -// than once. -// -// PeriodicFunction runs the function as soon as any previous run both is -// complete and was started more than "interval_micros" earlier. Thus, runs are -// both serialized, and normally have a period of "interval_micros" if no run -// exceeds the time. -// -// Note that, if the function takes longer than two interval_micross to finish, -// then PeriodicFunction will "skip" at least one call to the function. For -// instance, if the period is 50ms and the function starts runs at time 0 for -// 150ms, then the function will immediately start executing again at time 150, -// but there will be no function runs corresponding to times 50 or 100. This is -// especially important to remember when using an environment with a simulated -// clock: advancing simulated time atomically over N interval_micross will not -// cause the function to be called N times. -// -// This object is thread-safe. -// -// Example: -// -// class Foo { -// public: -// Foo() : periodic_function_([this]() { Bar(); }, -// 1000 /* 1000us == 1ms*/) { -// } -// -// private: -// void Bar() { ... } -// -// PeriodicFunction periodic_function_; -// }; - -#ifndef TENSORFLOW_SERVING_UTIL_PERIODIC_FUNCTION_H_ -#define TENSORFLOW_SERVING_UTIL_PERIODIC_FUNCTION_H_ - -#include -#include -#include - -#include "tensorflow/core/lib/core/notification.h" -#include "tensorflow/core/platform/env.h" -#include "tensorflow/core/platform/macros.h" -#include "tensorflow/core/platform/mutex.h" -#include "tensorflow/core/platform/thread_annotations.h" -#include "tensorflow/core/platform/types.h" - -namespace tensorflow { -namespace serving { - -namespace internal { -class PeriodicFunctionTestAccess; -} - -class PeriodicFunction { - public: - // Provides the ability to customize several aspects of the PeriodicFunction. - // Passed to constructor of PeriodicFunction. - struct Options { - Options() {} - - // Any standard thread options, such as stack size, should - // be passed via "thread_options". - ThreadOptions thread_options; - - // Specifies the thread name prefix (see the description in class - // Thread). - string thread_name_prefix = "periodic_function"; - - // The environment to use. Does not take ownership, but must remain alive - // for as long as the PeriodicFunction exists. - Env* env = Env::Default(); - - // Specifies the length of sleep before the first invocation of the - // function. - // This can be used for adding a random jitter to avoid synchronous behavior - // across multiple periodic functions. - int64 startup_delay_micros = 0; - }; - - // Also starts the background thread which will be calling the function. - PeriodicFunction(const std::function& function, int64 interval_micros, - const Options& options = Options()); - - ~PeriodicFunction(); - - private: - friend class internal::PeriodicFunctionTestAccess; - - // Notifies the background thread to stop. - void NotifyStop(); - - // (Blocking.) Loops forever calling "function_" every "interval_micros_". - void RunLoop(int64 start) LOCKS_EXCLUDED(mutex_); - - const std::function function_; // Actual client function - const int64 interval_micros_; // Interval between calls. - const Options options_; - - // Protects state below. - mutable mutex mutex_; - // Used to notify the thread to stop. - Notification stop_thread_; - - // Thread for running "function_" - std::unique_ptr thread_ GUARDED_BY(mutex_) = nullptr; - - TF_DISALLOW_COPY_AND_ASSIGN(PeriodicFunction); -}; - -} // namespace serving -} // namespace tensorflow - -#endif // TENSORFLOW_SERVING_UTIL_PERIODIC_FUNCTION_H_ diff --git a/tensorflow_serving/util/periodic_function_test.cc b/tensorflow_serving/util/periodic_function_test.cc deleted file mode 100644 index df81f04f1ff..00000000000 --- a/tensorflow_serving/util/periodic_function_test.cc +++ /dev/null @@ -1,225 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#include "tensorflow_serving/util/periodic_function.h" - -#include -#include - -#include -#include "tensorflow_serving/test_util/fake_clock_env.h" - -namespace tensorflow { -namespace serving { - -namespace internal { - -class PeriodicFunctionTestAccess { - public: - explicit PeriodicFunctionTestAccess(PeriodicFunction* periodic_function) - : periodic_function_(periodic_function) {} - - void NotifyStop() { periodic_function_->NotifyStop(); } - - private: - PeriodicFunction* const periodic_function_; -}; - -} // namespace internal - -namespace { - -using test_util::FakeClockEnv; - -void StopPeriodicFunction(PeriodicFunction* periodic_function, - FakeClockEnv* fake_clock_env, - const uint64 pf_interval_micros) { - fake_clock_env->BlockUntilThreadsAsleep(1); - internal::PeriodicFunctionTestAccess(periodic_function).NotifyStop(); - fake_clock_env->AdvanceByMicroseconds(pf_interval_micros); -} - -TEST(PeriodicFunctionTest, ObeyInterval) { - const int64 kPeriodMicros = 2; - const int kCalls = 10; - - int actual_calls = 0; - { - FakeClockEnv fake_clock_env(Env::Default()); - PeriodicFunction::Options options; - options.env = &fake_clock_env; - PeriodicFunction periodic_function([&actual_calls]() { ++actual_calls; }, - kPeriodMicros, options); - - for (int i = 0; i < kCalls; ++i) { - fake_clock_env.BlockUntilThreadsAsleep(1); - fake_clock_env.AdvanceByMicroseconds(kPeriodMicros); - } - StopPeriodicFunction(&periodic_function, &fake_clock_env, kPeriodMicros); - } - - // The function gets called kCalls+1 times: once at time 0, once at time - // kPeriodMicros, once at time kPeriodMicros*2, up to once at time - // kPeriodMicros*kCalls. - ASSERT_EQ(actual_calls, kCalls + 1); -} - -TEST(PeriodicFunctionTest, ObeyStartupDelay) { - const int64 kDelayMicros = 10; - const int64 kPeriodMicros = kDelayMicros / 10; - - int actual_calls = 0; - { - PeriodicFunction::Options options; - options.startup_delay_micros = kDelayMicros; - FakeClockEnv fake_clock_env(Env::Default()); - options.env = &fake_clock_env; - PeriodicFunction periodic_function([&actual_calls]() { ++actual_calls; }, - kPeriodMicros, options); - - // Wait for the thread to start up. - fake_clock_env.BlockUntilThreadsAsleep(1); - // Function shouldn't have been called yet. - EXPECT_EQ(0, actual_calls); - // Give enough time for startup delay to expire. - fake_clock_env.AdvanceByMicroseconds(kDelayMicros); - StopPeriodicFunction(&periodic_function, &fake_clock_env, kDelayMicros); - } - - // Function should have been called at least once. - EXPECT_EQ(1, actual_calls); -} - -// Test for race in calculating the first time the callback should fire. -TEST(PeriodicFunctionTest, StartupDelayRace) { - const int64 kDelayMicros = 10; - const int64 kPeriodMicros = kDelayMicros / 10; - - mutex mu; - int counter = 0; - std::unique_ptr listener(new Notification); - - FakeClockEnv fake_clock_env(Env::Default()); - PeriodicFunction::Options options; - options.env = &fake_clock_env; - options.startup_delay_micros = kDelayMicros; - PeriodicFunction periodic_function( - [&mu, &counter, &listener]() { - mutex_lock l(mu); - counter++; - listener->Notify(); - }, - kPeriodMicros, options); - - fake_clock_env.BlockUntilThreadsAsleep(1); - fake_clock_env.AdvanceByMicroseconds(kDelayMicros); - listener->WaitForNotification(); - { - mutex_lock l(mu); - EXPECT_EQ(1, counter); - // A notification can only be notified once. - listener.reset(new Notification); - } - fake_clock_env.BlockUntilThreadsAsleep(1); - fake_clock_env.AdvanceByMicroseconds(kPeriodMicros); - listener->WaitForNotification(); - { - mutex_lock l(mu); - EXPECT_EQ(2, counter); - } - StopPeriodicFunction(&periodic_function, &fake_clock_env, kPeriodMicros); -} - -// If this test hangs forever, its probably a deadlock caused by setting the -// PeriodicFunction's interval to 0ms. -TEST(PeriodicFunctionTest, MinInterval) { - PeriodicFunction periodic_function( - []() { Env::Default()->SleepForMicroseconds(20 * 1000); }, 0); -} - -class PeriodicFunctionWithFakeClockEnvTest : public testing::Test { - protected: - const int64 kPeriodMicros = 50; - PeriodicFunctionWithFakeClockEnvTest() - : fake_clock_env_(Env::Default()), - counter_(0), - pf_( - [this]() { - mutex_lock l(counter_mu_); - ++counter_; - }, - kPeriodMicros, GetPeriodicFunctionOptions()) {} - - PeriodicFunction::Options GetPeriodicFunctionOptions() { - PeriodicFunction::Options options; - options.thread_name_prefix = "ignore"; - options.env = &fake_clock_env_; - return options; - } - - void SetUp() override { - // Note: counter_ gets initially incremented at time 0. - ASSERT_TRUE(AwaitCount(1)); - } - - void TearDown() override { - StopPeriodicFunction(&pf_, &fake_clock_env_, kPeriodMicros); - } - - // The FakeClockEnv tests below advance simulated time and then expect the - // PeriodicFunction thread to run its function. This method helps the tests - // wait for the thread to execute, and then check the count matches the - // expectation. - bool AwaitCount(int expected_counter) { - fake_clock_env_.BlockUntilThreadsAsleep(1); - { - mutex_lock lock(counter_mu_); - return counter_ == expected_counter; - } - } - - FakeClockEnv fake_clock_env_; - mutex counter_mu_; - int counter_; - PeriodicFunction pf_; -}; - -TEST_F(PeriodicFunctionWithFakeClockEnvTest, FasterThanRealTime) { - fake_clock_env_.AdvanceByMicroseconds(kPeriodMicros / 2); - for (int i = 2; i < 7; ++i) { - fake_clock_env_.AdvanceByMicroseconds( - kPeriodMicros); // advance past a tick - EXPECT_TRUE(AwaitCount(i)); - } -} - -TEST_F(PeriodicFunctionWithFakeClockEnvTest, SlowerThanRealTime) { - Env::Default()->SleepForMicroseconds( - 125 * 1000); // wait for any unexpected breakage - EXPECT_TRUE(AwaitCount(1)); -} - -TEST(PeriodicFunctionDeathTest, BadInterval) { - EXPECT_DEBUG_DEATH(PeriodicFunction periodic_function([]() {}, -1), - ".* should be >= 0"); - - EXPECT_DEBUG_DEATH(PeriodicFunction periodic_function( - []() {}, -1, PeriodicFunction::Options()), - ".* should be >= 0"); -} - -} // namespace -} // namespace serving -} // namespace tensorflow From 405b8ab4a921a7bbcf1c426a4df869682b78af24 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 28 Mar 2017 14:13:19 -0800 Subject: [PATCH 0212/8554] Adds utilities for comparing resource objects, and getting and setting quantities within resource objects. Change: 151505432 --- .../model_servers/server_core.cc | 9 +- tensorflow_serving/resources/resource_util.cc | 313 ++++++++++----- tensorflow_serving/resources/resource_util.h | 58 ++- .../resources/resource_util_test.cc | 359 +++++++++++++++++- 4 files changed, 615 insertions(+), 124 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index a934518e5a6..0d0448a7cb2 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -544,11 +544,10 @@ Status ServerCore::CreateResourceTracker( auto resource_util = std::unique_ptr(new ResourceUtil(resource_util_options)); ResourceAllocation total_resources; - ResourceAllocation::Entry* main_memory_resource = - total_resources.add_resource_quantities(); - main_memory_resource->mutable_resource()->set_device(device_types::kMain); - main_memory_resource->mutable_resource()->set_kind(resource_kinds::kRamBytes); - main_memory_resource->set_quantity(options_.total_model_memory_limit_bytes); + resource_util->SetQuantity( + resource_util->CreateBoundResource(device_types::kMain, + resource_kinds::kRamBytes), + options_.total_model_memory_limit_bytes, &total_resources); const tensorflow::Status status = ResourceTracker::Create( total_resources, std::move(resource_util), resource_tracker); if (!status.ok()) { diff --git a/tensorflow_serving/resources/resource_util.cc b/tensorflow_serving/resources/resource_util.cc index 993711db7e1..f5a34c17ba8 100644 --- a/tensorflow_serving/resources/resource_util.cc +++ b/tensorflow_serving/resources/resource_util.cc @@ -29,6 +29,24 @@ namespace serving { namespace { +// Performs a direct equality comparison of 'lhs' and 'rhs'. +bool RawResourcesEqual(const Resource& lhs, const Resource& rhs) { + if (lhs.device() != rhs.device()) { + return false; + } + + if (lhs.has_device_instance() != rhs.has_device_instance()) { + return false; + } + if (lhs.has_device_instance()) { + if (lhs.device_instance().value() != rhs.device_instance().value()) { + return false; + } + } + + return lhs.kind() == rhs.kind(); +} + // Returns a copy of 'devices', stripped of any entries whose value is 0. std::map StripDevicesWithZeroInstances( const std::map& devices) { @@ -41,26 +59,13 @@ std::map StripDevicesWithZeroInstances( return result; } -// Obtains the quantity associated with 'resource' in 'allocation'. If none is -// found, returns 0. -uint64 GetQuantityForResource(const Resource& resource, - const ResourceAllocation& allocation) { - for (const ResourceAllocation::Entry& entry : - allocation.resource_quantities()) { - if (entry.resource() == resource) { - return entry.quantity(); - } - } - return 0; -} - // Returns a pointer to the entry associated with 'resource' in 'allocation'. If // none is found, returns nullptr. ResourceAllocation::Entry* FindMutableEntry(const Resource& resource, ResourceAllocation* allocation) { for (ResourceAllocation::Entry& entry : *allocation->mutable_resource_quantities()) { - if (entry.resource() == resource) { + if (RawResourcesEqual(entry.resource(), resource)) { return &entry; } } @@ -87,75 +92,39 @@ ResourceUtil::ResourceUtil(const Options& options) Status ResourceUtil::VerifyValidity( const ResourceAllocation& allocation) const { - // We use 'validated_entries' to look for duplicates. - ResourceAllocation validated_entries; - for (const auto& entry : allocation.resource_quantities()) { - auto it = devices_.find(entry.resource().device()); - if (it == devices_.end()) { - return errors::InvalidArgument( - "Invalid resource allocation: Invalid device ", - entry.resource().device(), " in resource allocation\n", - allocation.DebugString()); - } - const uint32 num_instances = it->second; - if (entry.resource().has_device_instance() && - entry.resource().device_instance().value() >= num_instances) { - return errors::InvalidArgument( - "Invalid resource allocation: Invalid device instance ", - entry.resource().device(), ":", - entry.resource().device_instance().value(), - " in resource allocation\n", allocation.DebugString()); - } - - if (FindMutableEntry(entry.resource(), &validated_entries) != nullptr) { - return errors::InvalidArgument( - "Invalid resource allocation: Repeated resource\n", - entry.resource().DebugString(), "in allocation\n", - allocation.DebugString()); - } + return VerifyValidityInternal(allocation, DCHECKFailOption::kDoNotDCHECKFail); +} - *validated_entries.add_resource_quantities() = entry; - } - return Status::OK(); +Status ResourceUtil::VerifyResourceValidity(const Resource& resource) const { + return VerifyResourceValidityInternal(resource, + DCHECKFailOption::kDoNotDCHECKFail); } ResourceAllocation ResourceUtil::Normalize( const ResourceAllocation& allocation) const { - const Status validity = VerifyValidity(allocation); - DCHECK_EQ(Status::OK(), validity); - if (!validity.ok()) { - LOG(ERROR) << validity; + if (!VerifyValidityInternal(allocation, DCHECKFailOption::kDoDCHECKFail) + .ok()) { return allocation; } ResourceAllocation normalized; - for (const auto& entry : allocation.resource_quantities()) { + for (const ResourceAllocation::Entry& entry : + allocation.resource_quantities()) { if (entry.quantity() == 0) { continue; } ResourceAllocation::Entry* normalized_entry = normalized.add_resource_quantities(); - *normalized_entry = entry; - if (entry.resource().has_device_instance()) { - continue; - } - const uint32 num_instances = - devices_.find(entry.resource().device())->second; - if (num_instances == 1) { - normalized_entry->mutable_resource() - ->mutable_device_instance() - ->set_value(0); - } + *normalized_entry->mutable_resource() = NormalizeResource(entry.resource()); + normalized_entry->set_quantity(entry.quantity()); } return normalized; } bool ResourceUtil::IsNormalized(const ResourceAllocation& allocation) const { - const Status validity = VerifyValidity(allocation); - DCHECK_EQ(Status::OK(), validity); - if (!validity.ok()) { - LOG(ERROR) << validity; + if (!VerifyValidityInternal(allocation, DCHECKFailOption::kDoDCHECKFail) + .ok()) { return false; } @@ -163,15 +132,7 @@ bool ResourceUtil::IsNormalized(const ResourceAllocation& allocation) const { if (entry.quantity() == 0) { return false; } - - if (entry.resource().has_device_instance()) { - continue; - } - // For singleton devices (ones that have one instance), the resource should - // be bound to the single device in the normalized representation. - const uint32 num_instances = - devices_.find(entry.resource().device())->second; - if (num_instances == 1) { + if (!IsResourceNormalized(entry.resource())) { return false; } } @@ -182,6 +143,45 @@ bool ResourceUtil::IsBound(const ResourceAllocation& allocation) const { return IsBoundNormalized(Normalize(allocation)); } +Resource ResourceUtil::CreateBoundResource(const string& device, + const string& kind, + uint32 device_instance) const { + DCHECK(devices_.find(device) != devices_.end()); + Resource resource; + resource.set_device(device); + resource.set_kind(kind); + resource.mutable_device_instance()->set_value(device_instance); + return resource; +} + +uint64 ResourceUtil::GetQuantity(const Resource& resource, + const ResourceAllocation& allocation) const { + DCHECK(devices_.find(resource.device()) != devices_.end()); + for (const ResourceAllocation::Entry& entry : + allocation.resource_quantities()) { + if (ResourcesEqual(entry.resource(), resource)) { + return entry.quantity(); + } + } + return 0; +} + +void ResourceUtil::SetQuantity(const Resource& resource, uint64 quantity, + ResourceAllocation* allocation) const { + DCHECK(devices_.find(resource.device()) != devices_.end()); + for (int i = 0; i < allocation->resource_quantities().size(); ++i) { + ResourceAllocation::Entry* entry = + allocation->mutable_resource_quantities(i); + if (ResourcesEqual(entry->resource(), resource)) { + entry->set_quantity(quantity); + return; + } + } + ResourceAllocation::Entry* new_entry = allocation->add_resource_quantities(); + *new_entry->mutable_resource() = resource; + new_entry->set_quantity(quantity); +} + void ResourceUtil::Add(const ResourceAllocation& to_add, ResourceAllocation* base) const { *base = Normalize(*base); @@ -194,6 +194,17 @@ bool ResourceUtil::Subtract(const ResourceAllocation& to_subtract, return SubtractNormalized(Normalize(to_subtract), base); } +bool ResourceUtil::Equal(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const { + return EqualNormalized(Normalize(lhs), Normalize(rhs)); +} + +bool ResourceUtil::ResourcesEqual(const Resource& lhs, + const Resource& rhs) const { + return ResourcesEqualNormalized(NormalizeResource(lhs), + NormalizeResource(rhs)); +} + bool ResourceUtil::LessThanOrEqual(const ResourceAllocation& lhs, const ResourceAllocation& rhs) const { return LessThanOrEqualNormalized(Normalize(lhs), Normalize(rhs)); @@ -215,6 +226,89 @@ bool ResourceUtil::IsBoundNormalized( return true; } +Status ResourceUtil::VerifyValidityInternal( + const ResourceAllocation& allocation, + DCHECKFailOption dcheck_fail_option) const { + const Status result = [this, &allocation]() -> Status { + // We use 'validated_entries' to look for duplicates. + ResourceAllocation validated_entries; + for (const auto& entry : allocation.resource_quantities()) { + TF_RETURN_IF_ERROR(VerifyResourceValidityInternal( + entry.resource(), DCHECKFailOption::kDoNotDCHECKFail)); + + if (FindMutableEntry(entry.resource(), &validated_entries) != nullptr) { + return errors::InvalidArgument( + "Invalid resource allocation: Repeated resource\n", + entry.resource().DebugString(), "in allocation\n", + allocation.DebugString()); + } + + *validated_entries.add_resource_quantities() = entry; + } + return Status::OK(); + }(); + + if (dcheck_fail_option == DCHECKFailOption::kDoDCHECKFail) { + TF_DCHECK_OK(result); + } + if (!result.ok()) { + LOG(ERROR) << result; + } + + return result; +} + +Status ResourceUtil::VerifyResourceValidityInternal( + const Resource& resource, DCHECKFailOption dcheck_fail_option) const { + const Status result = [this, &resource]() -> Status { + auto it = devices_.find(resource.device()); + if (it == devices_.end()) { + return errors::InvalidArgument( + "Invalid resource allocation: Invalid device ", resource.device()); + } + const uint32 num_instances = it->second; + if (resource.has_device_instance() && + resource.device_instance().value() >= num_instances) { + return errors::InvalidArgument( + "Invalid resource allocation: Invalid device instance ", + resource.device(), ":", resource.device_instance().value()); + } + return Status::OK(); + }(); + + if (dcheck_fail_option == DCHECKFailOption::kDoDCHECKFail) { + TF_DCHECK_OK(result); + } + if (!result.ok()) { + LOG(ERROR) << result; + } + + return result; +} + +Resource ResourceUtil::NormalizeResource(const Resource& resource) const { + Resource normalized = resource; + if (!normalized.has_device_instance()) { + const uint32 num_instances = devices_.find(normalized.device())->second; + if (num_instances == 1) { + normalized.mutable_device_instance()->set_value(0); + } + } + return normalized; +} + +bool ResourceUtil::IsResourceNormalized(const Resource& resource) const { + if (!VerifyResourceValidityInternal(resource, DCHECKFailOption::kDoDCHECKFail) + .ok()) { + return false; + } + + // For singleton devices (ones that have one instance), the resource should + // be bound to the single device in the normalized representation. + return resource.has_device_instance() || + devices_.find(resource.device())->second > 1; +} + void ResourceUtil::AddNormalized(const ResourceAllocation& to_add, ResourceAllocation* base) const { DCHECK(IsNormalized(to_add)); @@ -260,12 +354,54 @@ bool ResourceUtil::SubtractNormalized(const ResourceAllocation& to_subtract, return true; } +bool ResourceUtil::EqualNormalized(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const { + if (!VerifyValidityInternal(lhs, DCHECKFailOption::kDoDCHECKFail).ok() || + !VerifyValidityInternal(rhs, DCHECKFailOption::kDoDCHECKFail).ok()) { + return false; + } + DCHECK(IsNormalized(lhs)); + DCHECK(IsNormalized(rhs)); + + if (lhs.resource_quantities().size() != rhs.resource_quantities().size()) { + return false; + } + + for (const ResourceAllocation::Entry& lhs_entry : lhs.resource_quantities()) { + bool matched = false; + for (const ResourceAllocation::Entry& rhs_entry : + rhs.resource_quantities()) { + if (ResourcesEqual(lhs_entry.resource(), rhs_entry.resource()) && + lhs_entry.quantity() == rhs_entry.quantity()) { + matched = true; + break; + } + } + if (!matched) { + return false; + } + } + + return true; +} + +bool ResourceUtil::ResourcesEqualNormalized(const Resource& lhs, + const Resource& rhs) const { + if (!VerifyResourceValidityInternal(lhs, DCHECKFailOption::kDoDCHECKFail) + .ok() || + !VerifyResourceValidityInternal(rhs, DCHECKFailOption::kDoDCHECKFail) + .ok()) { + return false; + } + DCHECK(IsResourceNormalized(lhs)); + DCHECK(IsResourceNormalized(rhs)); + return RawResourcesEqual(lhs, rhs); +} + bool ResourceUtil::LessThanOrEqualNormalized( const ResourceAllocation& lhs, const ResourceAllocation& rhs) const { - const Status validity = VerifyValidity(lhs); - DCHECK_EQ(Status::OK(), validity); - if (!validity.ok()) { - LOG(ERROR) << validity; + if (!VerifyValidityInternal(lhs, DCHECKFailOption::kDoDCHECKFail).ok() || + !VerifyValidityInternal(rhs, DCHECKFailOption::kDoDCHECKFail).ok()) { return false; } DCHECK(IsNormalized(lhs)); @@ -296,7 +432,7 @@ bool ResourceUtil::LessThanOrEqualNormalized( for (int instance = 0; instance < num_instances; ++instance) { bound_resource.mutable_device_instance()->set_value(instance); if (lhs_entry.quantity() <= - GetQuantityForResource(bound_resource, subtracted_rhs)) { + GetQuantity(bound_resource, subtracted_rhs)) { found_room = true; break; } @@ -311,10 +447,8 @@ bool ResourceUtil::LessThanOrEqualNormalized( ResourceAllocation ResourceUtil::OverbindNormalized( const ResourceAllocation& allocation) const { - const Status validity = VerifyValidity(allocation); - DCHECK_EQ(Status::OK(), validity); - if (!validity.ok()) { - LOG(ERROR) << validity; + if (!VerifyValidityInternal(allocation, DCHECKFailOption::kDoDCHECKFail) + .ok()) { return allocation; } DCHECK(IsNormalized(allocation)); @@ -343,22 +477,5 @@ ResourceAllocation ResourceUtil::OverbindNormalized( return result; } -bool operator==(const Resource& a, const Resource& b) { - if (a.device() != b.device()) { - return false; - } - - if (a.has_device_instance() != b.has_device_instance()) { - return false; - } - if (a.has_device_instance()) { - if (a.device_instance().value() != b.device_instance().value()) { - return false; - } - } - - return a.kind() == b.kind(); -} - } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/resources/resource_util.h b/tensorflow_serving/resources/resource_util.h index fe18114d1bd..b90c880759f 100644 --- a/tensorflow_serving/resources/resource_util.h +++ b/tensorflow_serving/resources/resource_util.h @@ -49,6 +49,10 @@ class ResourceUtil { // have undefined behavior otherwise), and guarantee to produce valid outputs. Status VerifyValidity(const ResourceAllocation& allocation) const; + // Verifies whether 'resource' is valid, i.e. it only refers to valid devices, + // i.e. those supplied via Options. + Status VerifyResourceValidity(const Resource& resource) const; + // Converts 'allocation' to normal form, meaning: // 1. It has no entries with quantity 0. // 2. Resources of a device that has exactly one instance are bound to that @@ -63,6 +67,23 @@ class ResourceUtil { // 2. An allocation is bound iff every entry is bound. bool IsBound(const ResourceAllocation& allocation) const; + // Creates a bound resource with the given values. For single-instance + // resources (which is a common case, e.g. main memory) the 'instance' + // argument can be omitted. + Resource CreateBoundResource(const string& device, const string& kind, + uint32 device_instance = 0) const; + + // Gets the quantity of 'resource' present in 'allocation'. Returns 0 if + // 'resource' is not mentioned in 'allocation', since unmentioned resources + // are implicitly zero. + uint64 GetQuantity(const Resource& resource, + const ResourceAllocation& allocation) const; + + // Sets the quantity of 'resource' to 'quantity' in 'allocation', overwriting + // any existing quantity. + void SetQuantity(const Resource& resource, uint64 quantity, + ResourceAllocation* allocation) const; + // Adds 'to_add' to 'base'. // // Keeps bound and unbound entries separate. For example, adding @@ -78,6 +99,15 @@ class ResourceUtil { bool Subtract(const ResourceAllocation& to_subtract, ResourceAllocation* base) const; + // Determines whether two ResourceAllocation objects are identical (modulo + // normalization). + bool Equal(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const; + + // Determines whether two Resource objects are identical (modulo + // normalization). + bool ResourcesEqual(const Resource& lhs, const Resource& rhs) const; + // Takes a (bound or unbound) allocation 'lhs' and a *bound* allocation 'rhs'. // Returns true iff for each entry in 'lhs', either: // 1. The entry is bound and its quantity is <= the corresponding one in @@ -103,6 +133,24 @@ class ResourceUtil { ResourceAllocation Overbind(const ResourceAllocation& allocation) const; private: + enum class DCHECKFailOption { kDoDCHECKFail, kDoNotDCHECKFail }; + + // Wraps VerifyValidity() with error logging and the option to DCHECK-fail. + Status VerifyValidityInternal(const ResourceAllocation& allocation, + DCHECKFailOption dcheck_fail_option) const; + + // Wraps VerifyResourceValidity() with error logging and the option to + // DCHECK-fail. + Status VerifyResourceValidityInternal( + const Resource& resource, DCHECKFailOption dcheck_fail_option) const; + + // Converts 'resource' to normal form, i.e. ensures that if the device has + // exactly one instance, the resource is bound to that instance. + Resource NormalizeResource(const Resource& resource) const; + + // Determines whether 'resource' is normalized. Assumes 'resource' is valid. + bool IsResourceNormalized(const Resource& resource) const; + // Like IsBound(), but assumes the input is normalized. bool IsBoundNormalized(const ResourceAllocation& allocation) const; @@ -116,6 +164,13 @@ class ResourceUtil { bool SubtractNormalized(const ResourceAllocation& to_subtract, ResourceAllocation* base) const; + // Like Equal(), but assumes the input is normalized. + bool EqualNormalized(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const; + + // Like ResourcesEqual(), but assumes the input is normalized. + bool ResourcesEqualNormalized(const Resource& lhs, const Resource& rhs) const; + // Like LessThanOrEqual(), but assumes the input is normalized. bool LessThanOrEqualNormalized(const ResourceAllocation& lhs, const ResourceAllocation& rhs) const; @@ -130,9 +185,6 @@ class ResourceUtil { TF_DISALLOW_COPY_AND_ASSIGN(ResourceUtil); }; -// Determines whether two Resource protos are equal. -bool operator==(const Resource& a, const Resource& b); - } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/resources/resource_util_test.cc b/tensorflow_serving/resources/resource_util_test.cc index a76c3a0e128..dd78ad62fe3 100644 --- a/tensorflow_serving/resources/resource_util_test.cc +++ b/tensorflow_serving/resources/resource_util_test.cc @@ -168,6 +168,38 @@ TEST_F(ResourceUtilTest, VerifyValidity) { .ok()); } +TEST_F(ResourceUtilTest, VerifyResourceValidity) { + // Unbound. + TF_EXPECT_OK(util_.VerifyResourceValidity( + CreateProto("device: 'main' " + "kind: 'processing' "))); + + // Bound to a valid instance. + TF_EXPECT_OK(util_.VerifyResourceValidity( + CreateProto("device: 'gpu' " + "device_instance { value: 0 } " + "kind: 'ram' "))); + TF_EXPECT_OK(util_.VerifyResourceValidity( + CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'ram' "))); + + // Non-existent device. + EXPECT_FALSE(util_ + .VerifyResourceValidity( + CreateProto("device: 'nonexistent_device' " + "kind: 'processing' ")) + .ok()); + + // Bound to an invalid instance. + EXPECT_FALSE(util_ + .VerifyResourceValidity( + CreateProto("device: 'gpu' " + "device_instance { value: 2 } " + "kind: 'ram' ")) + .ok()); +} + TEST_F(ResourceUtilTest, Normalize) { EXPECT_THAT(util_.Normalize(CreateProto("")), EqualsProto("")); @@ -340,6 +372,174 @@ TEST_F(ResourceUtilTest, IsBound) { "} "))); } +TEST_F(ResourceUtilTest, CreateBoundResource) { + EXPECT_THAT(util_.CreateBoundResource("gpu", "ram", 2), + EqualsProto("device: 'gpu' " + "device_instance { value: 2 } " + "kind: 'ram' ")); + EXPECT_THAT( + util_.CreateBoundResource("gpu", "ram" /* , implicit instance = 0 */), + EqualsProto("device: 'gpu' " + "device_instance { value: 0 } " + "kind: 'ram' ")); +} + +TEST_F(ResourceUtilTest, GetQuantity) { + const auto allocation = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 32 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 100 " + "} "); + EXPECT_EQ(32, util_.GetQuantity(CreateProto("device: 'main' " + "kind: 'ram' "), + allocation)); + EXPECT_EQ(32, util_.GetQuantity( + CreateProto("device: 'main' " + "device_instance { value: 0 } " + "kind: 'ram' "), + allocation)); + EXPECT_EQ(100, util_.GetQuantity( + CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'processing' "), + allocation)); + EXPECT_EQ( + 0, util_.GetQuantity(CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'ram' "), + allocation)); +} + +TEST_F(ResourceUtilTest, SetQuantity) { + const auto initial_allocation = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'ram' " + " } " + " quantity: 32 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 100 " + "} "); + + { + ResourceAllocation allocation = initial_allocation; + util_.SetQuantity(CreateProto("device: 'main' " + "device_instance { value: 0 } " + "kind: 'ram' "), + 64, &allocation); + EXPECT_THAT(allocation, EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'ram' " + " } " + " quantity: 64 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 100 " + "} ")); + } + + { + ResourceAllocation allocation = initial_allocation; + util_.SetQuantity(CreateProto("device: 'main' " + "kind: 'ram' "), + 64, &allocation); + EXPECT_THAT(allocation, EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'ram' " + " } " + " quantity: 64 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 100 " + "} ")); + } + + { + ResourceAllocation allocation = initial_allocation; + util_.SetQuantity(CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'processing' "), + 200, &allocation); + EXPECT_THAT(allocation, EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'ram' " + " } " + " quantity: 32 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 200 " + "} ")); + } + + { + ResourceAllocation allocation = initial_allocation; + util_.SetQuantity(CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'ram' "), + 128, &allocation); + EXPECT_THAT(allocation, EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'ram' " + " } " + " quantity: 32 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 100 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'ram' " + " } " + " quantity: 128 " + "} ")); + } +} + TEST_F(ResourceUtilTest, AddEmpty) { auto base = CreateProto(""); const auto to_add = CreateProto(""); @@ -687,6 +887,147 @@ TEST_F(ResourceUtilTest, SubtractBoundAndUnbound) { "} ")); } +TEST_F(ResourceUtilTest, Equal) { + const std::vector values = { + CreateProto(""), + CreateProto("resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 10 " + "} "), + CreateProto("resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 20 " + "} "), + CreateProto("resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 20 " + "} "), + CreateProto("resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 32 " + "} "), + CreateProto("resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 20 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 32 " + "} ")}; + + for (int i = 0; i < values.size(); ++i) { + for (int j = 0; j < values.size(); ++j) { + EXPECT_EQ(i == j, util_.Equal(values[i], values[j])) << i << " vs. " << j; + EXPECT_EQ(j == i, util_.Equal(values[j], values[i])) << j << " vs. " << i; + } + } +} + +TEST_F(ResourceUtilTest, EqualOrderInsensitive) { + const auto a = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 32 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 20 " + "} "); + const auto b = CreateProto( + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 1 } " + " kind: 'processing' " + " } " + " quantity: 20 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 32 " + "} "); + EXPECT_TRUE(util_.Equal(a, b)); + EXPECT_TRUE(util_.Equal(b, a)); +} + +TEST_F(ResourceUtilTest, ResourcesEqual) { + EXPECT_TRUE(util_.ResourcesEqual(CreateProto("device: 'gpu' " + "kind: 'ram' "), + CreateProto("device: 'gpu' " + "kind: 'ram' "))); + EXPECT_TRUE( + util_.ResourcesEqual(CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'ram' "), + CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'ram' "))); + EXPECT_TRUE( + util_.ResourcesEqual(CreateProto("device: 'main' " + "kind: 'ram' "), + CreateProto("device: 'main' " + "device_instance { value: 0 } " + "kind: 'ram' "))); + EXPECT_FALSE(util_.ResourcesEqual(CreateProto("device: 'gpu' " + "kind: 'ram' "), + CreateProto("device: 'main' " + "kind: 'ram' "))); + EXPECT_FALSE( + util_.ResourcesEqual(CreateProto("device: 'gpu' " + "kind: 'ram' "), + CreateProto("device: 'gpu' " + "kind: 'processing' "))); + EXPECT_FALSE( + util_.ResourcesEqual(CreateProto("device: 'gpu' " + "device_instance { value: 0 } " + "kind: 'ram' "), + CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'ram' "))); + EXPECT_FALSE( + util_.ResourcesEqual(CreateProto("device: 'gpu' " + "kind: 'ram' "), + CreateProto("device: 'gpu' " + "device_instance { value: 1 } " + "kind: 'ram' "))); +} + TEST_F(ResourceUtilTest, LessThanOrEqualEmpty) { const auto a = CreateProto(""); EXPECT_TRUE(util_.LessThanOrEqual(a, a)); @@ -1095,24 +1436,6 @@ TEST_F(ResourceUtilTest, Overbind) { "} ")); } -TEST_F(ResourceUtilTest, ResourceEquality) { - std::vector distinct_protos = { - CreateProto(""), CreateProto("device: 'main' " - "kind: 'ram' "), - CreateProto("device: 'gpu' " - "kind: 'ram' "), - CreateProto("device: 'gpu' " - "kind: 'processing' "), - CreateProto("device: 'gpu' " - " device_instance { value: 1 } " - "kind: 'processing' ")}; - for (int i = 0; i < distinct_protos.size(); ++i) { - for (int j = 0; j < distinct_protos.size(); ++j) { - EXPECT_EQ(i == j, operator==(distinct_protos[i], distinct_protos[j])); - } - } -} - } // namespace } // namespace serving } // namespace tensorflow From b92f829b535950db2ed708e9303f9778a2605b95 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 28 Mar 2017 16:21:33 -0800 Subject: [PATCH 0213/8554] Propagate ModelServer RPC deadline to Session::Run() calls for Classify and Regress, in addition to Predict. Change: 151520986 --- tensorflow_serving/model_servers/main.cc | 12 +- tensorflow_serving/servables/tensorflow/BUILD | 2 + .../tensorflow/classification_service.cc | 7 +- .../tensorflow/classification_service.h | 4 +- .../servables/tensorflow/classifier.cc | 30 +++-- .../servables/tensorflow/classifier.h | 5 +- .../servables/tensorflow/classifier_test.cc | 112 ++++++++++++++---- .../servables/tensorflow/multi_inference.cc | 7 +- .../servables/tensorflow/multi_inference.h | 3 +- .../tensorflow/multi_inference_test.cc | 18 +-- .../tensorflow/regression_service.cc | 7 +- .../servables/tensorflow/regression_service.h | 4 +- .../servables/tensorflow/regressor.cc | 29 +++-- .../servables/tensorflow/regressor.h | 5 +- .../servables/tensorflow/regressor_test.cc | 76 ++++++++++-- .../servables/tensorflow/util.cc | 8 +- .../servables/tensorflow/util.h | 3 +- 17 files changed, 245 insertions(+), 87 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 2c36eb5647c..92d7fd76de7 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -231,9 +231,13 @@ class PredictionServiceImpl final : public PredictionService::Service { grpc::Status Classify(ServerContext* context, const ClassificationRequest* request, ClassificationResponse* response) override { + tensorflow::RunOptions run_options = tensorflow::RunOptions(); + // By default, this is infinite which is the same default as RunOptions. + run_options.set_timeout_in_ms( + DeadlineToTimeoutMillis(context->raw_deadline())); const grpc::Status status = ToGRPCStatus(TensorflowClassificationServiceImpl::Classify( - core_.get(), *request, response)); + run_options, core_.get(), *request, response)); if (!status.ok()) { VLOG(1) << "Classify request failed: " << status.error_message(); } @@ -243,9 +247,13 @@ class PredictionServiceImpl final : public PredictionService::Service { grpc::Status Regress(ServerContext* context, const RegressionRequest* request, RegressionResponse* response) override { + tensorflow::RunOptions run_options = tensorflow::RunOptions(); + // By default, this is infinite which is the same default as RunOptions. + run_options.set_timeout_in_ms( + DeadlineToTimeoutMillis(context->raw_deadline())); const grpc::Status status = ToGRPCStatus(TensorflowRegressionServiceImpl::Regress( - core_.get(), *request, response)); + run_options, core_.get(), *request, response)); if (!status.ok()) { VLOG(1) << "Regress request failed: " << status.error_message(); } diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 05050934aac..1cb87990d3d 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -561,6 +561,7 @@ cc_test( "//tensorflow_serving/core/test_util:mock_session", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", @@ -648,6 +649,7 @@ cc_test( "//tensorflow_serving/core/test_util:mock_session", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", "@org_tensorflow//tensorflow/core:core_cpu", diff --git a/tensorflow_serving/servables/tensorflow/classification_service.cc b/tensorflow_serving/servables/tensorflow/classification_service.cc index 476848b1059..4116fca2763 100644 --- a/tensorflow_serving/servables/tensorflow/classification_service.cc +++ b/tensorflow_serving/servables/tensorflow/classification_service.cc @@ -29,8 +29,8 @@ namespace tensorflow { namespace serving { Status TensorflowClassificationServiceImpl::Classify( - ServerCore* core, const ClassificationRequest& request, - ClassificationResponse* response) { + const RunOptions& run_options, ServerCore* core, + const ClassificationRequest& request, ClassificationResponse* response) { TRACELITERAL("TensorflowClassificationServiceImpl::Classify"); // Verify Request Metadata and create a ServableRequest if (!request.has_model_spec()) { @@ -47,7 +47,8 @@ Status TensorflowClassificationServiceImpl::Classify( std::unique_ptr classifier_interface; TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( - saved_model_bundle->session.get(), &signature, &classifier_interface)); + run_options, saved_model_bundle->session.get(), &signature, + &classifier_interface)); // Run classification. return classifier_interface->Classify(request, response->mutable_result()); } diff --git a/tensorflow_serving/servables/tensorflow/classification_service.h b/tensorflow_serving/servables/tensorflow/classification_service.h index b06eae85cc0..9e2dab45c4c 100644 --- a/tensorflow_serving/servables/tensorflow/classification_service.h +++ b/tensorflow_serving/servables/tensorflow/classification_service.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_CLASSIFICATION_SERVICE_H_ #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow_serving/apis/classification.pb.h" #include "tensorflow_serving/model_servers/server_core.h" @@ -27,7 +28,8 @@ namespace serving { // tensorflow_serving/apis/classification-service.proto. class TensorflowClassificationServiceImpl { public: - static Status Classify(ServerCore* core, const ClassificationRequest& request, + static Status Classify(const RunOptions& run_options, ServerCore* core, + const ClassificationRequest& request, ClassificationResponse* response); }; diff --git a/tensorflow_serving/servables/tensorflow/classifier.cc b/tensorflow_serving/servables/tensorflow/classifier.cc index ae9c1e92990..af31b2b47f1 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.cc +++ b/tensorflow_serving/servables/tensorflow/classifier.cc @@ -150,9 +150,10 @@ class TensorFlowClassifier : public ClassifierInterface { // Implementation of the ClassifierInterface using SavedModel. class SavedModelTensorFlowClassifier : public ClassifierInterface { public: - explicit SavedModelTensorFlowClassifier(Session* session, + explicit SavedModelTensorFlowClassifier(const RunOptions& run_options, + Session* session, const SignatureDef* const signature) - : session_(session), signature_(signature) {} + : run_options_(run_options), session_(session), signature_(signature) {} ~SavedModelTensorFlowClassifier() override = default; @@ -168,8 +169,8 @@ class SavedModelTensorFlowClassifier : public ClassifierInterface { std::vector outputs; int num_examples; TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( - request.input(), input_tensor_name, output_tensor_names, session_, - &outputs, &num_examples)); + run_options_, request.input(), input_tensor_name, output_tensor_names, + session_, &outputs, &num_examples)); TRACELITERAL("ConvertToClassificationResult"); return PostProcessClassificationResult( @@ -177,6 +178,7 @@ class SavedModelTensorFlowClassifier : public ClassifierInterface { } private: + const RunOptions run_options_; Session* const session_; const SignatureDef* const signature_; @@ -213,8 +215,9 @@ class SessionBundleClassifier : public ClassifierInterface { class SavedModelClassifier : public ClassifierInterface { public: - explicit SavedModelClassifier(std::unique_ptr bundle) - : bundle_(std::move(bundle)) {} + SavedModelClassifier(const RunOptions& run_options, + std::unique_ptr bundle) + : run_options_(run_options), bundle_(std::move(bundle)) {} ~SavedModelClassifier() override = default; @@ -227,12 +230,13 @@ class SavedModelClassifier : public ClassifierInterface { SignatureDef signature; TF_RETURN_IF_ERROR(GetClassificationSignatureDef( request.model_spec(), bundle_->meta_graph_def, &signature)); - SavedModelTensorFlowClassifier classifier(bundle_->session.get(), - &signature); + SavedModelTensorFlowClassifier classifier( + run_options_, bundle_->session.get(), &signature); return classifier.Classify(request, result); } private: + const RunOptions run_options_; std::unique_ptr bundle_; TF_DISALLOW_COPY_AND_ASSIGN(SavedModelClassifier); @@ -248,9 +252,9 @@ Status CreateClassifierFromBundle( } Status CreateClassifierFromSavedModelBundle( - std::unique_ptr bundle, + const RunOptions& run_options, std::unique_ptr bundle, std::unique_ptr* service) { - service->reset(new SavedModelClassifier(std::move(bundle))); + service->reset(new SavedModelClassifier(run_options, std::move(bundle))); return Status::OK(); } @@ -262,9 +266,11 @@ Status CreateFlyweightTensorFlowClassifier( } Status CreateFlyweightTensorFlowClassifier( - Session* session, const SignatureDef* signature, + const RunOptions& run_options, Session* session, + const SignatureDef* signature, std::unique_ptr* service) { - service->reset(new SavedModelTensorFlowClassifier(session, signature)); + service->reset( + new SavedModelTensorFlowClassifier(run_options, session, signature)); return Status::OK(); } diff --git a/tensorflow_serving/servables/tensorflow/classifier.h b/tensorflow_serving/servables/tensorflow/classifier.h index bb10f0a5b0d..14d796daa36 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.h +++ b/tensorflow_serving/servables/tensorflow/classifier.h @@ -38,7 +38,7 @@ Status CreateClassifierFromBundle( // Create a new ClassifierInterface backed by a TensorFlow SavedModel. // Requires that the default SignatureDef be compatible with classification. Status CreateClassifierFromSavedModelBundle( - std::unique_ptr bundle, + const RunOptions& run_options, std::unique_ptr bundle, std::unique_ptr* service); // Create a new ClassifierInterface backed by a TensorFlow Session using the @@ -56,7 +56,8 @@ Status CreateFlyweightTensorFlowClassifier( // request. The caller must ensure that the session and signature live at least // as long as the service. Status CreateFlyweightTensorFlowClassifier( - Session* session, const SignatureDef* signature, + const RunOptions& run_options, Session* session, + const SignatureDef* signature, std::unique_ptr* service); // Get a classification signature from the meta_graph_def that's either: diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc index b69da132215..67848b5dbe3 100644 --- a/tensorflow_serving/servables/tensorflow/classifier_test.cc +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -38,6 +38,7 @@ limitations under the License. #include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/core/test_util/mock_session.h" #include "tensorflow_serving/test_util/test_util.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -63,6 +64,8 @@ const char kInvalidNamedSignature[] = "invalid_regression_signature"; // class for that example. class FakeSession : public tensorflow::Session { public: + explicit FakeSession(optional expected_timeout) + : expected_timeout_(expected_timeout) {} ~FakeSession() override = default; Status Create(const GraphDef& graph) override { return errors::Unimplemented("not available in fake"); @@ -79,6 +82,22 @@ class FakeSession : public tensorflow::Session { const std::vector& output_names, const std::vector& target_nodes, std::vector* outputs) override { + if (expected_timeout_) { + LOG(FATAL) << "Run() without RunOptions not expected to be called"; + } + RunMetadata run_metadata; + return Run(RunOptions(), inputs, output_names, target_nodes, outputs, + &run_metadata); + } + + Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs, RunMetadata* run_metadata) override { + if (expected_timeout_) { + CHECK_EQ(*expected_timeout_, run_options.timeout_in_ms()); + } if (inputs.size() != 1 || inputs[0].first != kInputTensor) { return errors::Internal("Expected one input Tensor."); } @@ -184,6 +203,9 @@ class FakeSession : public tensorflow::Session { } return Status::OK(); } + + private: + const optional expected_timeout_; }; // Add a named signature to the mutable signatures* parameter. @@ -223,7 +245,12 @@ class ClassifierTest : public ::testing::TestWithParam { void SetUp() override { bundle_.reset(new SessionBundle); meta_graph_def_ = &bundle_->meta_graph_def; - fake_session_ = new FakeSession(); + optional expected_timeout = GetRunOptions().timeout_in_ms(); + if (!GetParam()) { + // For SessionBundle we don't propagate the timeout. + expected_timeout = nullopt; + } + fake_session_ = new FakeSession(expected_timeout); bundle_->session.reset(fake_session_); // Setup some defaults for our signature. @@ -265,8 +292,8 @@ class ClassifierTest : public ::testing::TestWithParam { std::unique_ptr saved_model(new SavedModelBundle); TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( *bundle_, saved_model.get())); - return CreateClassifierFromSavedModelBundle(std::move(saved_model), - &classifier_); + return CreateClassifierFromSavedModelBundle( + GetRunOptions(), std::move(saved_model), &classifier_); } else { return CreateClassifierFromBundle(std::move(bundle_), &classifier_); } @@ -283,6 +310,13 @@ class ClassifierTest : public ::testing::TestWithParam { // Convenience variables. ClassificationRequest request_; ClassificationResult result_; + + private: + RunOptions GetRunOptions() const { + RunOptions run_options; + run_options.set_timeout_in_ms(42); + return run_options; + } }; TEST_P(ClassifierTest, ExampleList) { @@ -600,9 +634,15 @@ TEST_P(ClassifierTest, EmptyExampleListWithContext) { TEST_P(ClassifierTest, RunsFails) { MockSession* mock = new MockSession; bundle_->session.reset(mock); - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillRepeatedly( - ::testing::Return(errors::Internal("Run totally failed"))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly( + ::testing::Return(errors::Internal("Run totally failed"))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly( + ::testing::Return(errors::Internal("Run totally failed"))); + } TF_ASSERT_OK(Create()); auto* examples = request_.mutable_input()->mutable_example_list()->mutable_examples(); @@ -619,9 +659,15 @@ TEST_P(ClassifierTest, ClassesIncorrectTensorBatchSize) { Tensor classes(DT_STRING, TensorShape({1, 2})); Tensor scores(DT_FLOAT, TensorShape({2, 2})); std::vector outputs = {classes, scores}; - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), - ::testing::Return(Status::OK()))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + } TF_ASSERT_OK(Create()); auto* examples = request_.mutable_input()->mutable_example_list()->mutable_examples(); @@ -640,9 +686,15 @@ TEST_P(ClassifierTest, ClassesIncorrectTensorType) { Tensor classes(DT_FLOAT, TensorShape({2, 2})); Tensor scores(DT_FLOAT, TensorShape({2, 2})); std::vector outputs = {classes, scores}; - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), - ::testing::Return(Status::OK()))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + } TF_ASSERT_OK(Create()); auto* examples = request_.mutable_input()->mutable_example_list()->mutable_examples(); @@ -662,9 +714,15 @@ TEST_P(ClassifierTest, ScoresIncorrectTensorBatchSize) { // This Tensor only has one batch item but we will have two inputs. Tensor scores(DT_FLOAT, TensorShape({1, 2})); std::vector outputs = {classes, scores}; - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), - ::testing::Return(Status::OK()))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + } TF_ASSERT_OK(Create()); auto* examples = request_.mutable_input()->mutable_example_list()->mutable_examples(); @@ -683,9 +741,15 @@ TEST_P(ClassifierTest, ScoresIncorrectTensorType) { // This Tensor is the wrong type for class. Tensor scores(DT_STRING, TensorShape({2, 2})); std::vector outputs = {classes, scores}; - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), - ::testing::Return(Status::OK()))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + } TF_ASSERT_OK(Create()); auto* examples = request_.mutable_input()->mutable_example_list()->mutable_examples(); @@ -705,9 +769,15 @@ TEST_P(ClassifierTest, MismatchedNumberOfTensorClasses) { // Scores Tensor has three scores but classes only has two labels. Tensor scores(DT_FLOAT, TensorShape({2, 3})); std::vector outputs = {classes, scores}; - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), - ::testing::Return(Status::OK()))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + } TF_ASSERT_OK(Create()); auto* examples = request_.mutable_input()->mutable_example_list()->mutable_examples(); diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.cc b/tensorflow_serving/servables/tensorflow/multi_inference.cc index 6bb5a6cc94e..050b33f17dc 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference.cc @@ -28,7 +28,8 @@ namespace tensorflow { namespace serving { Status TensorFlowMultiInferenceRunner::Infer( - const MultiInferenceRequest& request, MultiInferenceResponse* response) { + const RunOptions& run_options, const MultiInferenceRequest& request, + MultiInferenceResponse* response) { TRACELITERAL("TensorFlowMultiInferenceRunner::Infer"); string model_name = ""; @@ -94,8 +95,8 @@ Status TensorFlowMultiInferenceRunner::Infer( std::vector outputs; int num_examples; TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( - request.input(), input_tensor_name, output_tensor_names, session_, - &outputs, &num_examples)); + run_options, request.input(), input_tensor_name, output_tensor_names, + session_, &outputs, &num_examples)); TRACELITERAL("PostProcessResults"); for (const auto& task : request.tasks()) { diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.h b/tensorflow_serving/servables/tensorflow/multi_inference.h index 533d7469f5f..d4c44b36ba8 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.h +++ b/tensorflow_serving/servables/tensorflow/multi_inference.h @@ -32,7 +32,8 @@ class TensorFlowMultiInferenceRunner { // Run inference and return the inference results in the same order as the // InferenceTasks in the request. - Status Infer(const MultiInferenceRequest& request, + Status Infer(const RunOptions& run_options, + const MultiInferenceRequest& request, MultiInferenceResponse* response); virtual ~TensorFlowMultiInferenceRunner() = default; diff --git a/tensorflow_serving/servables/tensorflow/multi_inference_test.cc b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc index 5f5e0eea5d2..0a1a33999b2 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference_test.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc @@ -129,7 +129,7 @@ TEST_F(MultiInferenceTest, MissingInputTest) { PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); MultiInferenceResponse response; - ExpectStatusError(inference_runner->Infer(request, &response), + ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "Input is empty"); } @@ -143,7 +143,7 @@ TEST_F(MultiInferenceTest, UndefinedSignatureTest) { request.add_tasks()); MultiInferenceResponse response; - ExpectStatusError(inference_runner->Infer(request, &response), + ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "signature not found"); } @@ -166,7 +166,7 @@ TEST_F(MultiInferenceTest, InconsistentModelSpecsInRequestTest) { task->set_method_name(kRegressMethodName); MultiInferenceResponse response; - ExpectStatusError(inference_runner->Infer(request, &response), + ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "must access the same model name"); } @@ -182,7 +182,7 @@ TEST_F(MultiInferenceTest, EvaluateDuplicateSignaturesTest) { PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); MultiInferenceResponse response; - ExpectStatusError(inference_runner->Infer(request, &response), + ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "Duplicate evaluation of signature: regress_x_to_y"); } @@ -196,7 +196,7 @@ TEST_F(MultiInferenceTest, UsupportedSignatureTypeTest) { PopulateTask("serving_default", kPredictMethodName, request.add_tasks()); MultiInferenceResponse response; - ExpectStatusError(inference_runner->Infer(request, &response), + ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::UNIMPLEMENTED, "Unsupported signature"); } @@ -210,7 +210,7 @@ TEST_F(MultiInferenceTest, SignaturesWithDifferentInputsTest) { PopulateTask("regress_x2_to_y3", kRegressMethodName, request.add_tasks()); MultiInferenceResponse response; - ExpectStatusError(inference_runner->Infer(request, &response), + ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "Input tensor must be the same"); } @@ -229,7 +229,7 @@ TEST_F(MultiInferenceTest, ValidSingleSignatureTest) { regression_result->add_regressions()->set_value(3.0); MultiInferenceResponse response; - TF_ASSERT_OK(inference_runner->Infer(request, &response)); + TF_ASSERT_OK(inference_runner->Infer(RunOptions(), request, &response)); EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } @@ -253,7 +253,7 @@ TEST_F(MultiInferenceTest, MultipleValidRegressSignaturesTest) { regression_result_2->add_regressions()->set_value(4.0); MultiInferenceResponse response; - TF_ASSERT_OK(inference_runner->Infer(request, &response)); + TF_ASSERT_OK(inference_runner->Infer(RunOptions(), request, &response)); EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } @@ -275,7 +275,7 @@ TEST_F(MultiInferenceTest, RegressAndClassifySignaturesTest) { classification_result->add_classifications()->add_classes()->set_score(3.0); MultiInferenceResponse response; - TF_ASSERT_OK(inference_runner->Infer(request, &response)); + TF_ASSERT_OK(inference_runner->Infer(RunOptions(), request, &response)); EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } diff --git a/tensorflow_serving/servables/tensorflow/regression_service.cc b/tensorflow_serving/servables/tensorflow/regression_service.cc index f1d35696328..1c616914041 100644 --- a/tensorflow_serving/servables/tensorflow/regression_service.cc +++ b/tensorflow_serving/servables/tensorflow/regression_service.cc @@ -27,8 +27,8 @@ namespace tensorflow { namespace serving { Status TensorflowRegressionServiceImpl::Regress( - ServerCore* core, const RegressionRequest& request, - RegressionResponse* response) { + const RunOptions& run_options, ServerCore* core, + const RegressionRequest& request, RegressionResponse* response) { TRACELITERAL("TensorflowRegressionServiceImpl::Regress"); // Verify Request Metadata and create a ServableRequest if (!request.has_model_spec()) { @@ -45,7 +45,8 @@ Status TensorflowRegressionServiceImpl::Regress( std::unique_ptr regressor_interface; TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( - saved_model_bundle->session.get(), &signature, ®ressor_interface)); + run_options, saved_model_bundle->session.get(), &signature, + ®ressor_interface)); // Run regression return regressor_interface->Regress(request, response->mutable_result()); } diff --git a/tensorflow_serving/servables/tensorflow/regression_service.h b/tensorflow_serving/servables/tensorflow/regression_service.h index c4980db6dd0..52770357949 100644 --- a/tensorflow_serving/servables/tensorflow/regression_service.h +++ b/tensorflow_serving/servables/tensorflow/regression_service.h @@ -17,6 +17,7 @@ limitations under the License. #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_REGRESSION_SERVICE_H_ #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow_serving/apis/regression.pb.h" #include "tensorflow_serving/model_servers/server_core.h" @@ -27,7 +28,8 @@ namespace serving { // tensorflow_serving/apis/regression-service.proto. class TensorflowRegressionServiceImpl final { public: - static Status Regress(ServerCore* core, const RegressionRequest& request, + static Status Regress(const RunOptions& run_options, ServerCore* core, + const RegressionRequest& request, RegressionResponse* response); }; diff --git a/tensorflow_serving/servables/tensorflow/regressor.cc b/tensorflow_serving/servables/tensorflow/regressor.cc index b54477760b6..dc044716a44 100644 --- a/tensorflow_serving/servables/tensorflow/regressor.cc +++ b/tensorflow_serving/servables/tensorflow/regressor.cc @@ -98,9 +98,10 @@ class TensorFlowRegressor : public RegressorInterface { // Implementation of the RegressorInterface using SavedModel. class SavedModelTensorFlowRegressor : public RegressorInterface { public: - explicit SavedModelTensorFlowRegressor(Session* session, + explicit SavedModelTensorFlowRegressor(const RunOptions& run_options, + Session* session, const SignatureDef* const signature) - : session_(session), signature_(signature) {} + : run_options_(run_options), session_(session), signature_(signature) {} ~SavedModelTensorFlowRegressor() override = default; @@ -116,8 +117,8 @@ class SavedModelTensorFlowRegressor : public RegressorInterface { std::vector outputs; int num_examples; TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( - request.input(), input_tensor_name, output_tensor_names, session_, - &outputs, &num_examples)); + run_options_, request.input(), input_tensor_name, output_tensor_names, + session_, &outputs, &num_examples)); TRACELITERAL("ConvertToRegressionResult"); return PostProcessRegressionResult(*signature_, num_examples, @@ -125,6 +126,7 @@ class SavedModelTensorFlowRegressor : public RegressorInterface { } private: + const RunOptions run_options_; Session* const session_; const SignatureDef* const signature_; @@ -157,8 +159,9 @@ class SessionBundleRegressor : public RegressorInterface { class SavedModelRegressor : public RegressorInterface { public: - explicit SavedModelRegressor(std::unique_ptr bundle) - : bundle_(std::move(bundle)) {} + SavedModelRegressor(const RunOptions& run_options, + std::unique_ptr bundle) + : run_options_(run_options), bundle_(std::move(bundle)) {} ~SavedModelRegressor() override = default; @@ -167,11 +170,13 @@ class SavedModelRegressor : public RegressorInterface { SignatureDef signature; TF_RETURN_IF_ERROR(GetRegressionSignatureDef( request.model_spec(), bundle_->meta_graph_def, &signature)); - SavedModelTensorFlowRegressor regressor(bundle_->session.get(), &signature); + SavedModelTensorFlowRegressor regressor(run_options_, + bundle_->session.get(), &signature); return regressor.Regress(request, result); } private: + const RunOptions run_options_; std::unique_ptr bundle_; TF_DISALLOW_COPY_AND_ASSIGN(SavedModelRegressor); @@ -186,9 +191,9 @@ Status CreateRegressorFromBundle(std::unique_ptr bundle, } Status CreateRegressorFromSavedModelBundle( - std::unique_ptr bundle, + const RunOptions& run_options, std::unique_ptr bundle, std::unique_ptr* service) { - service->reset(new SavedModelRegressor(std::move(bundle))); + service->reset(new SavedModelRegressor(run_options, std::move(bundle))); return Status::OK(); } @@ -200,9 +205,11 @@ Status CreateFlyweightTensorFlowRegressor( } Status CreateFlyweightTensorFlowRegressor( - Session* session, const SignatureDef* signature, + const RunOptions& run_options, Session* session, + const SignatureDef* signature, std::unique_ptr* service) { - service->reset(new SavedModelTensorFlowRegressor(session, signature)); + service->reset( + new SavedModelTensorFlowRegressor(run_options, session, signature)); return Status::OK(); } diff --git a/tensorflow_serving/servables/tensorflow/regressor.h b/tensorflow_serving/servables/tensorflow/regressor.h index 659cc3bc1ea..a1d560778e0 100644 --- a/tensorflow_serving/servables/tensorflow/regressor.h +++ b/tensorflow_serving/servables/tensorflow/regressor.h @@ -37,7 +37,7 @@ Status CreateRegressorFromBundle(std::unique_ptr bundle, // Create a new RegressorInterface backed by a TensorFlow SavedModel. // Requires that the default SignatureDef be compatible with Regression. Status CreateRegressorFromSavedModelBundle( - std::unique_ptr bundle, + const RunOptions& run_options, std::unique_ptr bundle, std::unique_ptr* service); // Create a new RegressorInterface backed by a TensorFlow Session using the @@ -55,7 +55,8 @@ Status CreateFlyweightTensorFlowRegressor( // request. The caller must ensure that the session and signature live at least // as long as the service. Status CreateFlyweightTensorFlowRegressor( - Session* session, const SignatureDef* signature, + const RunOptions& run_options, Session* session, + const SignatureDef* signature, std::unique_ptr* service); // Get a regression signature from the meta_graph_def that's either: diff --git a/tensorflow_serving/servables/tensorflow/regressor_test.cc b/tensorflow_serving/servables/tensorflow/regressor_test.cc index c7550435f93..46ce82dd0f0 100644 --- a/tensorflow_serving/servables/tensorflow/regressor_test.cc +++ b/tensorflow_serving/servables/tensorflow/regressor_test.cc @@ -36,6 +36,7 @@ limitations under the License. #include "tensorflow_serving/apis/regression.pb.h" #include "tensorflow_serving/core/test_util/mock_session.h" #include "tensorflow_serving/test_util/test_util.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -58,6 +59,8 @@ const char kInvalidNamedSignature[] = "invalid_classification_signature"; // Copies the "output" float feature from each Example. class FakeSession : public tensorflow::Session { public: + explicit FakeSession(optional expected_timeout) + : expected_timeout_(expected_timeout) {} ~FakeSession() override = default; Status Create(const GraphDef& graph) override { return errors::Unimplemented("not available in fake"); @@ -74,6 +77,22 @@ class FakeSession : public tensorflow::Session { const std::vector& output_names, const std::vector& target_nodes, std::vector* outputs) override { + if (expected_timeout_) { + LOG(FATAL) << "Run() without RunOptions not expected to be called"; + } + RunMetadata run_metadata; + return Run(RunOptions(), inputs, output_names, target_nodes, outputs, + &run_metadata); + } + + Status Run(const RunOptions& run_options, + const std::vector>& inputs, + const std::vector& output_names, + const std::vector& target_nodes, + std::vector* outputs, RunMetadata* run_metadata) override { + if (expected_timeout_) { + CHECK_EQ(*expected_timeout_, run_options.timeout_in_ms()); + } if (inputs.size() != 1 || inputs[0].first != kInputTensor) { return errors::Internal("Expected one input Tensor."); } @@ -139,6 +158,9 @@ class FakeSession : public tensorflow::Session { } return Status::OK(); } + + private: + const optional expected_timeout_; }; // Add a named signature to the mutable signatures* parameter. @@ -175,7 +197,12 @@ class RegressorTest : public ::testing::TestWithParam { void SetUp() override { bundle_.reset(new SessionBundle); meta_graph_def_ = &bundle_->meta_graph_def; - fake_session_ = new FakeSession(); + optional expected_timeout = GetRunOptions().timeout_in_ms(); + if (!GetParam()) { + // For SessionBundle we don't propagate the timeout. + expected_timeout = nullopt; + } + fake_session_ = new FakeSession(expected_timeout); bundle_->session.reset(fake_session_); // Setup some defaults for our signature. @@ -210,8 +237,8 @@ class RegressorTest : public ::testing::TestWithParam { std::unique_ptr saved_model(new SavedModelBundle); TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( *bundle_, saved_model.get())); - return CreateRegressorFromSavedModelBundle(std::move(saved_model), - ®ressor_); + return CreateRegressorFromSavedModelBundle( + GetRunOptions(), std::move(saved_model), ®ressor_); } else { return CreateRegressorFromBundle(std::move(bundle_), ®ressor_); } @@ -228,6 +255,13 @@ class RegressorTest : public ::testing::TestWithParam { // Convenience variables. RegressionRequest request_; RegressionResult result_; + + private: + RunOptions GetRunOptions() const { + RunOptions run_options; + run_options.set_timeout_in_ms(42); + return run_options; + } }; TEST_P(RegressorTest, BasicExampleList) { @@ -353,9 +387,15 @@ TEST_P(RegressorTest, EmptyExampleListWithContext) { TEST_P(RegressorTest, RunsFails) { MockSession* mock = new MockSession; bundle_->session.reset(mock); - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillRepeatedly( - ::testing::Return(errors::Internal("Run totally failed"))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly( + ::testing::Return(errors::Internal("Run totally failed"))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillRepeatedly( + ::testing::Return(errors::Internal("Run totally failed"))); + } TF_ASSERT_OK(Create()); *request_.mutable_input()->mutable_example_list()->mutable_examples()->Add() = example_with_output(2.0); @@ -368,9 +408,15 @@ TEST_P(RegressorTest, UnexpectedOutputTensorSize) { MockSession* mock = new MockSession; bundle_->session.reset(mock); std::vector outputs = {Tensor(DT_FLOAT, TensorShape({2}))}; - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillOnce(::testing::DoAll(::testing::SetArgPointee<3>(outputs), - ::testing::Return(Status::OK()))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + } TF_ASSERT_OK(Create()); *request_.mutable_input()->mutable_example_list()->mutable_examples()->Add() = example_with_output(2.0); @@ -384,9 +430,15 @@ TEST_P(RegressorTest, UnexpectedOutputTensorType) { bundle_->session.reset(mock); // We expect a FLOAT output type; test returning a STRING. std::vector outputs = {Tensor(DT_STRING, TensorShape({1}))}; - EXPECT_CALL(*mock, Run(_, _, _, _)) - .WillOnce(::testing::DoAll(::testing::SetArgPointee<3>(outputs), - ::testing::Return(Status::OK()))); + if (GetParam()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + } else { + EXPECT_CALL(*mock, Run(_, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<3>(outputs), + ::testing::Return(Status::OK()))); + } TF_ASSERT_OK(Create()); *request_.mutable_input()->mutable_example_list()->mutable_examples()->Add() = example_with_output(2.0); diff --git a/tensorflow_serving/servables/tensorflow/util.cc b/tensorflow_serving/servables/tensorflow/util.cc index 69cdb221cc7..3831df0a48d 100644 --- a/tensorflow_serving/servables/tensorflow/util.cc +++ b/tensorflow_serving/servables/tensorflow/util.cc @@ -86,7 +86,8 @@ Status InputToSerializedExampleTensor(const Input& input, Tensor* examples) { } Status PerformOneShotTensorComputation( - const Input& input, const string& input_tensor_name, + const RunOptions& run_options, const Input& input, + const string& input_tensor_name, const std::vector& output_tensor_names, Session* session, std::vector* outputs, int* num_input_examples) { // Setup the input Tensor to be a vector of string containing the serialized @@ -95,8 +96,9 @@ Status PerformOneShotTensorComputation( TF_RETURN_IF_ERROR(InputToSerializedExampleTensor(input, &input_tensor)); *num_input_examples = input_tensor.dim_size(0); - return session->Run({{input_tensor_name, input_tensor}}, output_tensor_names, - {}, outputs); + RunMetadata run_metadata; + return session->Run(run_options, {{input_tensor_name, input_tensor}}, + output_tensor_names, {}, outputs, &run_metadata); } } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/util.h b/tensorflow_serving/servables/tensorflow/util.h index 5e797edbe23..832ced2544b 100644 --- a/tensorflow_serving/servables/tensorflow/util.h +++ b/tensorflow_serving/servables/tensorflow/util.h @@ -42,7 +42,8 @@ Status InputToSerializedExampleTensor(const Input& input, Tensor* examples); // Issues a single Session::Run() call with 'input' to produce 'outputs'. // Equivalent to InputToSerializedExampleTensor() followed by Session::Run(). Status PerformOneShotTensorComputation( - const Input& input, const string& input_tensor_name, + const RunOptions& run_options, const Input& input, + const string& input_tensor_name, const std::vector& output_tensor_names, Session* session, std::vector* outputs, int* num_input_examples); From 25735dc6fddcd3ed9f90bcfd7d80fdf7aa90365c Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 29 Mar 2017 10:46:25 -0800 Subject: [PATCH 0214/8554] Update TF-Serving code to point to new location of low-level batch scheduling library. Change: 151604241 --- tensorflow_serving/batching/batch_scheduler_retrier.h | 2 +- tensorflow_serving/batching/batching_session.h | 4 ++-- .../batching/streaming_batch_scheduler.h | 2 +- tensorflow_serving/batching/test_util/BUILD | 2 +- .../batching/test_util/puppet_batch_scheduler.h | 2 +- tensorflow_serving/servables/tensorflow/BUILD | 10 +++++----- .../servables/tensorflow/bundle_factory_util.cc | 2 +- .../servables/tensorflow/bundle_factory_util.h | 2 +- .../servables/tensorflow/bundle_factory_util_test.cc | 2 +- .../servables/tensorflow/saved_model_bundle_factory.h | 2 +- .../servables/tensorflow/session_bundle_factory.h | 2 +- 11 files changed, 16 insertions(+), 16 deletions(-) diff --git a/tensorflow_serving/batching/batch_scheduler_retrier.h b/tensorflow_serving/batching/batch_scheduler_retrier.h index 4c2a197edad..f62f31a9272 100644 --- a/tensorflow_serving/batching/batch_scheduler_retrier.h +++ b/tensorflow_serving/batching/batch_scheduler_retrier.h @@ -21,10 +21,10 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/batch_scheduler.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/macros.h" -#include "tensorflow_serving/batching/batch_scheduler.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/batching/batching_session.h b/tensorflow_serving/batching/batching_session.h index 4426652c601..4d2e0ddf2d8 100644 --- a/tensorflow_serving/batching/batching_session.h +++ b/tensorflow_serving/batching/batching_session.h @@ -26,10 +26,10 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/basic_batch_scheduler.h" +#include "tensorflow/contrib/batching/batch_scheduler.h" #include "tensorflow/core/protobuf/meta_graph.pb.h" #include "tensorflow/core/public/session.h" -#include "tensorflow_serving/batching/basic_batch_scheduler.h" -#include "tensorflow_serving/batching/batch_scheduler.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/batching/streaming_batch_scheduler.h b/tensorflow_serving/batching/streaming_batch_scheduler.h index b883678d800..4341f10a21c 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler.h +++ b/tensorflow_serving/batching/streaming_batch_scheduler.h @@ -23,6 +23,7 @@ limitations under the License. #include #include +#include "tensorflow/contrib/batching/batch_scheduler.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/notification.h" #include "tensorflow/core/lib/core/status.h" @@ -34,7 +35,6 @@ limitations under the License. #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/thread_annotations.h" #include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/batching/batch_scheduler.h" #include "tensorflow_serving/batching/batch_scheduler_retrier.h" #include "tensorflow_serving/util/optional.h" diff --git a/tensorflow_serving/batching/test_util/BUILD b/tensorflow_serving/batching/test_util/BUILD index 86f96f15750..2de50699795 100644 --- a/tensorflow_serving/batching/test_util/BUILD +++ b/tensorflow_serving/batching/test_util/BUILD @@ -26,7 +26,7 @@ cc_library( hdrs = ["puppet_batch_scheduler.h"], visibility = ["//visibility:private"], deps = [ - "//tensorflow_serving/batching:batch_scheduler", + "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", "@org_tensorflow//tensorflow/core:tensorflow", ], ) diff --git a/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h b/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h index 5cb70d96766..4410a82ca3b 100644 --- a/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h +++ b/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h @@ -24,7 +24,7 @@ limitations under the License. #include #include -#include "tensorflow_serving/batching/batch_scheduler.h" +#include "tensorflow/contrib/batching/batch_scheduler.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 1cb87990d3d..d8b8c9a546b 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -47,12 +47,12 @@ cc_library( deps = [ ":serving_session", ":session_bundle_config_proto", - "//tensorflow_serving/batching:batch_scheduler", "//tensorflow_serving/batching:batching_session", - "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/util:file_probing_env", + "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", + "@org_tensorflow//tensorflow/contrib/batching:shared_batch_scheduler", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", @@ -70,11 +70,11 @@ cc_test( ":bundle_factory_util", ":session_bundle_config_proto", "//tensorflow_serving/batching:batching_session", - "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", "//tensorflow_serving/util/test_util:mock_file_probing_env", + "@org_tensorflow//tensorflow/contrib/batching:shared_batch_scheduler", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", @@ -131,8 +131,8 @@ cc_library( ":bundle_factory_util", ":session_bundle_config_proto", "//tensorflow_serving/batching:batching_session", - "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resources_proto", + "@org_tensorflow//tensorflow/contrib/batching:shared_batch_scheduler", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", "@org_tensorflow//tensorflow/core:core_cpu", @@ -176,10 +176,10 @@ cc_library( ":curried_session", ":session_bundle_config_proto", "//tensorflow_serving/batching:batching_session", - "//tensorflow_serving/batching:shared_batch_scheduler", "//tensorflow_serving/resources:resources_proto", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", + "@org_tensorflow//tensorflow/contrib/batching:shared_batch_scheduler", "@org_tensorflow//tensorflow/contrib/session_bundle:bundle_shim", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc index d14120efe12..46f0b8e192e 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc @@ -16,11 +16,11 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" #include "google/protobuf/wrappers.pb.h" +#include "tensorflow/contrib/batching/batch_scheduler.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/types.h" -#include "tensorflow_serving/batching/batch_scheduler.h" #include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/servables/tensorflow/serving_session.h" diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.h b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h index ad19f301a19..ee0ebda5d2a 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.h @@ -16,12 +16,12 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_UTIL_H_ #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_BUNDLE_FACTORY_UTIL_H_ +#include "tensorflow/contrib/batching/shared_batch_scheduler.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/public/session.h" #include "tensorflow/core/public/session_options.h" #include "tensorflow_serving/batching/batching_session.h" -#include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/util/file_probing_env.h" diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc index 7331768f44b..cd39f31db18 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util_test.cc @@ -23,6 +23,7 @@ limitations under the License. #include "google/protobuf/wrappers.pb.h" #include #include +#include "tensorflow/contrib/batching/shared_batch_scheduler.h" #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" @@ -32,7 +33,6 @@ limitations under the License. #include "tensorflow/core/public/session_options.h" #include "tensorflow/core/public/version.h" #include "tensorflow_serving/batching/batching_session.h" -#include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h index 65f25aebc69..253499ab311 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h @@ -17,10 +17,10 @@ limitations under the License. #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_BUNDLE_FACTORY_H_ #include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/contrib/batching/shared_batch_scheduler.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow_serving/batching/batching_session.h" -#include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h index 3037e8f73c6..38f3f830368 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory.h @@ -16,10 +16,10 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SESSION_BUNDLE_FACTORY_H_ #define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SESSION_BUNDLE_FACTORY_H_ +#include "tensorflow/contrib/batching/shared_batch_scheduler.h" #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/batching/batching_session.h" -#include "tensorflow_serving/batching/shared_batch_scheduler.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" From 906ccfdfeaaa44e0d50f7fb204a594d95e59c58b Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 30 Mar 2017 08:23:22 -0800 Subject: [PATCH 0215/8554] Remove tensorflow-serving batch library shims that point to tensorflow/contrib/batching/. Change: 151710128 --- tensorflow_serving/batching/BUILD | 42 ++----------------- .../batching/basic_batch_scheduler.h | 22 ---------- tensorflow_serving/batching/batch_scheduler.h | 22 ---------- .../batching/shared_batch_scheduler.h | 22 ---------- 4 files changed, 4 insertions(+), 104 deletions(-) delete mode 100644 tensorflow_serving/batching/basic_batch_scheduler.h delete mode 100644 tensorflow_serving/batching/batch_scheduler.h delete mode 100644 tensorflow_serving/batching/shared_batch_scheduler.h diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index 02ab0e94914..1fc4fff8234 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -20,49 +20,15 @@ filegroup( ), ) -# TODO(b/36404166): Remove this shim after migrating all clients. -cc_library( - name = "batch_scheduler", - hdrs = ["batch_scheduler.h"], - visibility = ["//visibility:public"], - deps = [ - "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", - ], -) - -# TODO(b/36404166): Remove this shim after migrating all clients. -cc_library( - name = "shared_batch_scheduler", - hdrs = ["shared_batch_scheduler.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - "@org_tensorflow//tensorflow/contrib/batching:shared_batch_scheduler", - ], -) - -# TODO(b/36404166): Remove this shim after migrating all clients. -cc_library( - name = "basic_batch_scheduler", - hdrs = ["basic_batch_scheduler.h"], - visibility = [ - "//visibility:public", - ], - deps = [ - "@org_tensorflow//tensorflow/contrib/batching:basic_batch_scheduler", - ], -) - cc_library( name = "streaming_batch_scheduler", srcs = ["streaming_batch_scheduler.cc"], hdrs = ["streaming_batch_scheduler.h"], visibility = ["//visibility:public"], deps = [ - ":batch_scheduler", ":batch_scheduler_retrier", "//tensorflow_serving/util:optional", + "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -90,11 +56,11 @@ cc_library( "//visibility:public", ], deps = [ - ":basic_batch_scheduler", - ":batch_scheduler", "//tensorflow_serving/servables/tensorflow:serving_session", "//tensorflow_serving/util:cleanup", "//tensorflow_serving/util:hash", + "@org_tensorflow//tensorflow/contrib/batching:basic_batch_scheduler", + "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", @@ -131,7 +97,7 @@ cc_library( hdrs = ["batch_scheduler_retrier.h"], visibility = ["//visibility:public"], deps = [ - ":batch_scheduler", + "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/batching/basic_batch_scheduler.h b/tensorflow_serving/batching/basic_batch_scheduler.h deleted file mode 100644 index 9999ddc05f6..00000000000 --- a/tensorflow_serving/batching/basic_batch_scheduler.h +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ -#define TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ - -// TODO(b/36404166): Remove this shim after migrating all clients. -#include "tensorflow/contrib/batching/basic_batch_scheduler.h" - -#endif // TENSORFLOW_SERVING_BATCHING_BASIC_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/batch_scheduler.h b/tensorflow_serving/batching/batch_scheduler.h deleted file mode 100644 index 73929928823..00000000000 --- a/tensorflow_serving/batching/batch_scheduler.h +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ -#define TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ - -// TODO(b/36404166): Remove this shim after migrating all clients. -#include "tensorflow/contrib/batching/batch_scheduler.h" - -#endif // TENSORFLOW_SERVING_BATCHING_BATCH_SCHEDULER_H_ diff --git a/tensorflow_serving/batching/shared_batch_scheduler.h b/tensorflow_serving/batching/shared_batch_scheduler.h deleted file mode 100644 index 9d03f46f86f..00000000000 --- a/tensorflow_serving/batching/shared_batch_scheduler.h +++ /dev/null @@ -1,22 +0,0 @@ -/* Copyright 2016 Google Inc. All Rights Reserved. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -==============================================================================*/ - -#ifndef TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ -#define TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ - -// TODO(b/36404166): Remove this shim after migrating all clients. -#include "tensorflow/contrib/batching/shared_batch_scheduler.h" - -#endif // TENSORFLOW_SERVING_BATCHING_SHARED_BATCH_SCHEDULER_H_ From ee2458c8b81caf1684d3837282a9a157e8516faa Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 30 Mar 2017 12:57:22 -0800 Subject: [PATCH 0216/8554] Implement MultiInference API in Model Server and add an integration test. Change: 151746252 --- tensorflow_serving/apis/BUILD | 4 +- .../apis/prediction_service.proto | 4 + .../apis/prediction_service_pb2.py | 467 ++++++++++-------- tensorflow_serving/model_servers/BUILD | 1 + tensorflow_serving/model_servers/main.cc | 18 + .../tensorflow_model_server_test.py | 40 ++ tensorflow_serving/servables/tensorflow/BUILD | 1 + .../servables/tensorflow/multi_inference.cc | 24 + .../servables/tensorflow/multi_inference.h | 5 + 9 files changed, 348 insertions(+), 216 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 2d745e23bfd..84ca39f02a5 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -118,6 +118,7 @@ serving_proto_library( deps = [ ":classification_proto", ":get_model_metadata_proto", + ":inference_proto", ":predict_proto", ":regression_proto", ], @@ -129,6 +130,7 @@ py_library( deps = [ ":classification_proto_py_pb2", ":get_model_metadata_proto_py_pb2", + ":inference_proto_py_pb2", ":predict_proto_py_pb2", ":regression_proto_py_pb2", ], @@ -172,7 +174,7 @@ serving_proto_library( ) serving_proto_library_py( - name = "inference_py_pb2", + name = "inference_proto_py_pb2", srcs = ["inference.proto"], proto_library = "inference_proto", deps = [ diff --git a/tensorflow_serving/apis/prediction_service.proto b/tensorflow_serving/apis/prediction_service.proto index 9ed0a0096dd..44e655417fd 100644 --- a/tensorflow_serving/apis/prediction_service.proto +++ b/tensorflow_serving/apis/prediction_service.proto @@ -5,6 +5,7 @@ option cc_enable_arenas = true; import "tensorflow_serving/apis/classification.proto"; import "tensorflow_serving/apis/get_model_metadata.proto"; +import "tensorflow_serving/apis/inference.proto"; import "tensorflow_serving/apis/predict.proto"; import "tensorflow_serving/apis/regression.proto"; @@ -21,6 +22,9 @@ service PredictionService { // Predict -- provides access to loaded TensorFlow model. rpc Predict(PredictRequest) returns (PredictResponse); + // MultiInference API for multi-headed models. + rpc MultiInference(MultiInferenceRequest) returns (MultiInferenceResponse); + // GetModelMetadata - provides access to metadata for loaded models. rpc GetModelMetadata(GetModelMetadataRequest) returns (GetModelMetadataResponse); diff --git a/tensorflow_serving/apis/prediction_service_pb2.py b/tensorflow_serving/apis/prediction_service_pb2.py index dc16d22d0e8..a522541e9d9 100644 --- a/tensorflow_serving/apis/prediction_service_pb2.py +++ b/tensorflow_serving/apis/prediction_service_pb2.py @@ -31,6 +31,7 @@ from tensorflow_serving.apis import classification_pb2 as tensorflow__serving_dot_apis_dot_classification__pb2 from tensorflow_serving.apis import get_model_metadata_pb2 as tensorflow__serving_dot_apis_dot_get__model__metadata__pb2 +from tensorflow_serving.apis import inference_pb2 as tensorflow__serving_dot_apis_dot_inference__pb2 from tensorflow_serving.apis import predict_pb2 as tensorflow__serving_dot_apis_dot_predict__pb2 from tensorflow_serving.apis import regression_pb2 as tensorflow__serving_dot_apis_dot_regression__pb2 @@ -39,9 +40,9 @@ name='tensorflow_serving/apis/prediction_service.proto', package='tensorflow.serving', syntax='proto3', - serialized_pb=_b('\n0tensorflow_serving/apis/prediction_service.proto\x12\x12tensorflow.serving\x1a,tensorflow_serving/apis/classification.proto\x1a\x30tensorflow_serving/apis/get_model_metadata.proto\x1a%tensorflow_serving/apis/predict.proto\x1a(tensorflow_serving/apis/regression.proto2\x93\x03\n\x11PredictionService\x12\x61\n\x08\x43lassify\x12).tensorflow.serving.ClassificationRequest\x1a*.tensorflow.serving.ClassificationResponse\x12X\n\x07Regress\x12%.tensorflow.serving.RegressionRequest\x1a&.tensorflow.serving.RegressionResponse\x12R\n\x07Predict\x12\".tensorflow.serving.PredictRequest\x1a#.tensorflow.serving.PredictResponse\x12m\n\x10GetModelMetadata\x12+.tensorflow.serving.GetModelMetadataRequest\x1a,.tensorflow.serving.GetModelMetadataResponseB\x03\xf8\x01\x01\x62\x06proto3') + serialized_pb=_b('\n0tensorflow_serving/apis/prediction_service.proto\x12\x12tensorflow.serving\x1a,tensorflow_serving/apis/classification.proto\x1a\x30tensorflow_serving/apis/get_model_metadata.proto\x1a\'tensorflow_serving/apis/inference.proto\x1a%tensorflow_serving/apis/predict.proto\x1a(tensorflow_serving/apis/regression.proto2\xfc\x03\n\x11PredictionService\x12\x61\n\x08\x43lassify\x12).tensorflow.serving.ClassificationRequest\x1a*.tensorflow.serving.ClassificationResponse\x12X\n\x07Regress\x12%.tensorflow.serving.RegressionRequest\x1a&.tensorflow.serving.RegressionResponse\x12R\n\x07Predict\x12\".tensorflow.serving.PredictRequest\x1a#.tensorflow.serving.PredictResponse\x12g\n\x0eMultiInference\x12).tensorflow.serving.MultiInferenceRequest\x1a*.tensorflow.serving.MultiInferenceResponse\x12m\n\x10GetModelMetadata\x12+.tensorflow.serving.GetModelMetadataRequest\x1a,.tensorflow.serving.GetModelMetadataResponseB\x03\xf8\x01\x01\x62\x06proto3') , - dependencies=[tensorflow__serving_dot_apis_dot_classification__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_predict__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_regression__pb2.DESCRIPTOR,]) + dependencies=[tensorflow__serving_dot_apis_dot_classification__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_inference__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_predict__pb2.DESCRIPTOR,tensorflow__serving_dot_apis_dot_regression__pb2.DESCRIPTOR,]) _sym_db.RegisterFileDescriptor(DESCRIPTOR) @@ -50,223 +51,259 @@ DESCRIPTOR.has_options = True DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) - -import grpc -from grpc.framework.common import cardinality -from grpc.framework.interfaces.face import utilities as face_utilities -from grpc.beta import implementations as beta_implementations -from grpc.beta import interfaces as beta_interfaces - - -class PredictionServiceStub(object): - """open source marker; do not remove - PredictionService provides access to machine-learned models loaded by - model_servers. - """ - - def __init__(self, channel): - """Constructor. - - Args: - channel: A grpc.Channel. - """ - self.Classify = channel.unary_unary( - '/tensorflow.serving.PredictionService/Classify', - request_serializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.SerializeToString, - response_deserializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.FromString, - ) - self.Regress = channel.unary_unary( - '/tensorflow.serving.PredictionService/Regress', - request_serializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.SerializeToString, - response_deserializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.FromString, - ) - self.Predict = channel.unary_unary( - '/tensorflow.serving.PredictionService/Predict', - request_serializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, - response_deserializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, - ) - self.GetModelMetadata = channel.unary_unary( - '/tensorflow.serving.PredictionService/GetModelMetadata', - request_serializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.SerializeToString, - response_deserializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.FromString, - ) - - -class PredictionServiceServicer(object): - """open source marker; do not remove - PredictionService provides access to machine-learned models loaded by - model_servers. - """ - - def Classify(self, request, context): - """Classify. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Regress(self, request, context): - """Regress. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - def Predict(self, request, context): - """Predict -- provides access to loaded TensorFlow model. +try: + # THESE ELEMENTS WILL BE DEPRECATED. + # Please use the generated *_pb2_grpc.py files instead. + import grpc + from grpc.framework.common import cardinality + from grpc.framework.interfaces.face import utilities as face_utilities + from grpc.beta import implementations as beta_implementations + from grpc.beta import interfaces as beta_interfaces + + + class PredictionServiceStub(object): + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by + model_servers. """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - def GetModelMetadata(self, request, context): - """GetModelMetadata - provides access to metadata for loaded models. - """ - context.set_code(grpc.StatusCode.UNIMPLEMENTED) - context.set_details('Method not implemented!') - raise NotImplementedError('Method not implemented!') - - -def add_PredictionServiceServicer_to_server(servicer, server): - rpc_method_handlers = { - 'Classify': grpc.unary_unary_rpc_method_handler( - servicer.Classify, - request_deserializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.FromString, - response_serializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.SerializeToString, - ), - 'Regress': grpc.unary_unary_rpc_method_handler( - servicer.Regress, - request_deserializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.FromString, - response_serializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.SerializeToString, - ), - 'Predict': grpc.unary_unary_rpc_method_handler( - servicer.Predict, - request_deserializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, - response_serializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, - ), - 'GetModelMetadata': grpc.unary_unary_rpc_method_handler( - servicer.GetModelMetadata, - request_deserializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.FromString, - response_serializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.SerializeToString, - ), - } - generic_handler = grpc.method_handlers_generic_handler( - 'tensorflow.serving.PredictionService', rpc_method_handlers) - server.add_generic_rpc_handlers((generic_handler,)) - - -class BetaPredictionServiceServicer(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """open source marker; do not remove - PredictionService provides access to machine-learned models loaded by - model_servers. - """ - def Classify(self, request, context): - """Classify. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def Regress(self, request, context): - """Regress. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def Predict(self, request, context): - """Predict -- provides access to loaded TensorFlow model. - """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - def GetModelMetadata(self, request, context): - """GetModelMetadata - provides access to metadata for loaded models. + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.Classify = channel.unary_unary( + '/tensorflow.serving.PredictionService/Classify', + request_serializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.FromString, + ) + self.Regress = channel.unary_unary( + '/tensorflow.serving.PredictionService/Regress', + request_serializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.FromString, + ) + self.Predict = channel.unary_unary( + '/tensorflow.serving.PredictionService/Predict', + request_serializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, + ) + self.MultiInference = channel.unary_unary( + '/tensorflow.serving.PredictionService/MultiInference', + request_serializer=tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceResponse.FromString, + ) + self.GetModelMetadata = channel.unary_unary( + '/tensorflow.serving.PredictionService/GetModelMetadata', + request_serializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.FromString, + ) + + + class PredictionServiceServicer(object): + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by + model_servers. """ - context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) - -class BetaPredictionServiceStub(object): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This class was generated - only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" - """open source marker; do not remove - PredictionService provides access to machine-learned models loaded by - model_servers. - """ - def Classify(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """Classify. - """ - raise NotImplementedError() - Classify.future = None - def Regress(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """Regress. - """ - raise NotImplementedError() - Regress.future = None - def Predict(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """Predict -- provides access to loaded TensorFlow model. + def Classify(self, request, context): + """Classify. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Regress(self, request, context): + """Regress. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def Predict(self, request, context): + """Predict -- provides access to loaded TensorFlow model. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def MultiInference(self, request, context): + """MultiInference API for multi-headed models. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def GetModelMetadata(self, request, context): + """GetModelMetadata - provides access to metadata for loaded models. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + + def add_PredictionServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'Classify': grpc.unary_unary_rpc_method_handler( + servicer.Classify, + request_deserializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.SerializeToString, + ), + 'Regress': grpc.unary_unary_rpc_method_handler( + servicer.Regress, + request_deserializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.SerializeToString, + ), + 'Predict': grpc.unary_unary_rpc_method_handler( + servicer.Predict, + request_deserializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, + ), + 'MultiInference': grpc.unary_unary_rpc_method_handler( + servicer.MultiInference, + request_deserializer=tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceResponse.SerializeToString, + ), + 'GetModelMetadata': grpc.unary_unary_rpc_method_handler( + servicer.GetModelMetadata, + request_deserializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'tensorflow.serving.PredictionService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + class BetaPredictionServiceServicer(object): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This class was generated + only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by + model_servers. """ - raise NotImplementedError() - Predict.future = None - def GetModelMetadata(self, request, timeout, metadata=None, with_call=False, protocol_options=None): - """GetModelMetadata - provides access to metadata for loaded models. + def Classify(self, request, context): + """Classify. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + def Regress(self, request, context): + """Regress. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + def Predict(self, request, context): + """Predict -- provides access to loaded TensorFlow model. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + def MultiInference(self, request, context): + """MultiInference API for multi-headed models. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + def GetModelMetadata(self, request, context): + """GetModelMetadata - provides access to metadata for loaded models. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + + + class BetaPredictionServiceStub(object): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This class was generated + only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" + """open source marker; do not remove + PredictionService provides access to machine-learned models loaded by + model_servers. """ - raise NotImplementedError() - GetModelMetadata.future = None - - -def beta_create_PredictionService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_deserializers = { - ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.FromString, - ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.FromString, - ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, - ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.FromString, - } - response_serializers = { - ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.SerializeToString, - ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.SerializeToString, - ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, - ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.SerializeToString, - } - method_implementations = { - ('tensorflow.serving.PredictionService', 'Classify'): face_utilities.unary_unary_inline(servicer.Classify), - ('tensorflow.serving.PredictionService', 'GetModelMetadata'): face_utilities.unary_unary_inline(servicer.GetModelMetadata), - ('tensorflow.serving.PredictionService', 'Predict'): face_utilities.unary_unary_inline(servicer.Predict), - ('tensorflow.serving.PredictionService', 'Regress'): face_utilities.unary_unary_inline(servicer.Regress), - } - server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) - return beta_implementations.server(method_implementations, options=server_options) - - -def beta_create_PredictionService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): - """The Beta API is deprecated for 0.15.0 and later. - - It is recommended to use the GA API (classes and functions in this - file not marked beta) for all further purposes. This function was - generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" - request_serializers = { - ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.SerializeToString, - ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.SerializeToString, - ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, - ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.SerializeToString, - } - response_deserializers = { - ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.FromString, - ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.FromString, - ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, - ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.FromString, - } - cardinalities = { - 'Classify': cardinality.Cardinality.UNARY_UNARY, - 'GetModelMetadata': cardinality.Cardinality.UNARY_UNARY, - 'Predict': cardinality.Cardinality.UNARY_UNARY, - 'Regress': cardinality.Cardinality.UNARY_UNARY, - } - stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) - return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.PredictionService', cardinalities, options=stub_options) + def Classify(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """Classify. + """ + raise NotImplementedError() + Classify.future = None + def Regress(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """Regress. + """ + raise NotImplementedError() + Regress.future = None + def Predict(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """Predict -- provides access to loaded TensorFlow model. + """ + raise NotImplementedError() + Predict.future = None + def MultiInference(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """MultiInference API for multi-headed models. + """ + raise NotImplementedError() + MultiInference.future = None + def GetModelMetadata(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """GetModelMetadata - provides access to metadata for loaded models. + """ + raise NotImplementedError() + GetModelMetadata.future = None + + + def beta_create_PredictionService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This function was + generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" + request_deserializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.FromString, + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.FromString, + ('tensorflow.serving.PredictionService', 'MultiInference'): tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceRequest.FromString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.FromString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.FromString, + } + response_serializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.SerializeToString, + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.SerializeToString, + ('tensorflow.serving.PredictionService', 'MultiInference'): tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceResponse.SerializeToString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.SerializeToString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.SerializeToString, + } + method_implementations = { + ('tensorflow.serving.PredictionService', 'Classify'): face_utilities.unary_unary_inline(servicer.Classify), + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): face_utilities.unary_unary_inline(servicer.GetModelMetadata), + ('tensorflow.serving.PredictionService', 'MultiInference'): face_utilities.unary_unary_inline(servicer.MultiInference), + ('tensorflow.serving.PredictionService', 'Predict'): face_utilities.unary_unary_inline(servicer.Predict), + ('tensorflow.serving.PredictionService', 'Regress'): face_utilities.unary_unary_inline(servicer.Regress), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + + + def beta_create_PredictionService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This function was + generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" + request_serializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationRequest.SerializeToString, + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataRequest.SerializeToString, + ('tensorflow.serving.PredictionService', 'MultiInference'): tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceRequest.SerializeToString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictRequest.SerializeToString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionRequest.SerializeToString, + } + response_deserializers = { + ('tensorflow.serving.PredictionService', 'Classify'): tensorflow__serving_dot_apis_dot_classification__pb2.ClassificationResponse.FromString, + ('tensorflow.serving.PredictionService', 'GetModelMetadata'): tensorflow__serving_dot_apis_dot_get__model__metadata__pb2.GetModelMetadataResponse.FromString, + ('tensorflow.serving.PredictionService', 'MultiInference'): tensorflow__serving_dot_apis_dot_inference__pb2.MultiInferenceResponse.FromString, + ('tensorflow.serving.PredictionService', 'Predict'): tensorflow__serving_dot_apis_dot_predict__pb2.PredictResponse.FromString, + ('tensorflow.serving.PredictionService', 'Regress'): tensorflow__serving_dot_apis_dot_regression__pb2.RegressionResponse.FromString, + } + cardinalities = { + 'Classify': cardinality.Cardinality.UNARY_UNARY, + 'GetModelMetadata': cardinality.Cardinality.UNARY_UNARY, + 'MultiInference': cardinality.Cardinality.UNARY_UNARY, + 'Predict': cardinality.Cardinality.UNARY_UNARY, + 'Regress': cardinality.Cardinality.UNARY_UNARY, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.PredictionService', cardinalities, options=stub_options) +except ImportError: + pass # @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 08d70c1474b..f8831ab1f73 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -112,6 +112,7 @@ TENSORFLOW_DEPS = [ "@org_tensorflow//tensorflow/core:tensorflow", "//tensorflow_serving/servables/tensorflow:classification_service", "//tensorflow_serving/servables/tensorflow:get_model_metadata_impl", + "//tensorflow_serving/servables/tensorflow:multi_inference", "//tensorflow_serving/servables/tensorflow:regression_service", "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter", diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 92d7fd76de7..538d1eed347 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -73,6 +73,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/classification_service.h" #include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h" +#include "tensorflow_serving/servables/tensorflow/multi_inference.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" #include "tensorflow_serving/servables/tensorflow/regression_service.h" @@ -111,6 +112,8 @@ using tensorflow::serving::ClassificationRequest; using tensorflow::serving::ClassificationResponse; using tensorflow::serving::GetModelMetadataRequest; using tensorflow::serving::GetModelMetadataResponse; +using tensorflow::serving::MultiInferenceRequest; +using tensorflow::serving::MultiInferenceResponse; using tensorflow::serving::PredictRequest; using tensorflow::serving::PredictResponse; using tensorflow::serving::RegressionRequest; @@ -260,6 +263,21 @@ class PredictionServiceImpl final : public PredictionService::Service { return status; } + grpc::Status MultiInference(ServerContext* context, + const MultiInferenceRequest* request, + MultiInferenceResponse* response) override { + tensorflow::RunOptions run_options = tensorflow::RunOptions(); + // By default, this is infinite which is the same default as RunOptions. + run_options.set_timeout_in_ms( + DeadlineToTimeoutMillis(context->raw_deadline())); + const grpc::Status status = ToGRPCStatus( + RunMultiInference(run_options, core_.get(), *request, response)); + if (!status.ok()) { + VLOG(1) << "MultiInference request failed: " << status.error_message(); + } + return status; + } + private: std::unique_ptr core_; std::unique_ptr predictor_; diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 67492bdec2d..81b7c0a91ba 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -38,6 +38,7 @@ from tensorflow_serving.apis import predict_pb2 from tensorflow_serving.apis import prediction_service_pb2 from tensorflow_serving.apis import regression_pb2 +from tensorflow_serving.apis import inference_pb2 FLAGS = flags.FLAGS @@ -242,6 +243,45 @@ def testRegress(self): self.assertEquals(expected_output, result.result.regressions[0].value) + def testMultiInference(self): + """Test PredictionService.MultiInference implementation.""" + model_path = self._GetSavedModelBundlePath() + use_saved_model = True + enable_batching = False + + atexit.register(self.TerminateProcs) + model_server_address = self.RunServer(PickUnusedPort(), 'default', + model_path, use_saved_model, + enable_batching) + time.sleep(5) + + print 'Sending MultiInference request...' + # Prepare request + request = inference_pb2.MultiInferenceRequest() + request.tasks.add().model_spec.name = 'default' + request.tasks[0].model_spec.signature_name = 'regress_x_to_y' + request.tasks[0].method_name = 'tensorflow/serving/regress' + request.tasks.add().model_spec.name = 'default' + request.tasks[1].model_spec.signature_name = 'classify_x_to_y' + request.tasks[1].method_name = 'tensorflow/serving/classify' + + example = request.input.example_list.examples.add() + example.features.feature['x'].float_list.value.extend([2.0]) + + # Send request + host, port = model_server_address.split(':') + channel = implementations.insecure_channel(host, int(port)) + stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) + result = stub.MultiInference(request, 5.0) # 5 secs timeout + + # Verify response + self.assertEquals(2, len(result.results)) + expected_output = 3.0 + self.assertEquals(expected_output, + result.results[0].regression_result.regressions[0].value) + self.assertEquals(expected_output, result.results[ + 1].classification_result.classifications[0].classes[0].score) + def _TestPredict(self, model_path, use_saved_model, batching_parameters_file=''): """Helper method to test prediction. diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index d8b8c9a546b..3cfb2010248 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -672,6 +672,7 @@ cc_library( "//tensorflow_serving/apis:inference_proto", "//tensorflow_serving/apis:input_proto", "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/model_servers:server_core", "//tensorflow_serving/servables/tensorflow:classifier", "//tensorflow_serving/servables/tensorflow:regressor", "//tensorflow_serving/servables/tensorflow:util", diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.cc b/tensorflow_serving/servables/tensorflow/multi_inference.cc index 050b33f17dc..1068240c876 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference.cc @@ -124,5 +124,29 @@ Status TensorFlowMultiInferenceRunner::Infer( return Status::OK(); } +namespace { + +const ModelSpec& GetModelSpecFromRequest(const MultiInferenceRequest& request) { + if (request.tasks_size() > 0 && request.tasks(0).has_model_spec()) { + return request.tasks(0).model_spec(); + } + return ModelSpec::default_instance(); +} + +} // namespace + +Status RunMultiInference(const RunOptions& run_options, ServerCore* core, + const MultiInferenceRequest& request, + MultiInferenceResponse* response) { + TRACELITERAL("RunMultiInference"); + ServableHandle bundle; + TF_RETURN_IF_ERROR( + core->GetServableHandle(GetModelSpecFromRequest(request), &bundle)); + + TensorFlowMultiInferenceRunner inference_runner(bundle->session.get(), + &bundle->meta_graph_def); + return inference_runner.Infer(run_options, request, response); +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.h b/tensorflow_serving/servables/tensorflow/multi_inference.h index d4c44b36ba8..2d22a256fb9 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.h +++ b/tensorflow_serving/servables/tensorflow/multi_inference.h @@ -19,6 +19,7 @@ limitations under the License. #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/apis/inference.pb.h" +#include "tensorflow_serving/model_servers/server_core.h" namespace tensorflow { namespace serving { @@ -43,6 +44,10 @@ class TensorFlowMultiInferenceRunner { MetaGraphDef* const meta_graph_def_; }; +Status RunMultiInference(const RunOptions& run_options, ServerCore* core, + const MultiInferenceRequest& request, + MultiInferenceResponse* response); + } // namespace serving } // namespace tensorflow From 73ec32fe677b23c01bf5f9820c02ba8cbea0c47b Mon Sep 17 00:00:00 2001 From: Christi Kaes Date: Thu, 30 Mar 2017 18:23:39 -0400 Subject: [PATCH 0217/8554] Fix for command to get deployments (#384) based on documentation at: https://kubernetes.io/docs/user-guide/deployments/#creating-a-deployment --- tensorflow_serving/g3doc/serving_inception.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 4a9c1f84765..26819ea2db0 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -238,7 +238,7 @@ service "inception-service" created To view status of the deployment and pods: ```shell -$ kc get deployments +$ kubectl get deployments NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE inception-deployment 3 3 3 3 5s ``` From a7dbcc63f2b95bad6dfd5285a7158de094aecd93 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 30 Mar 2017 14:26:08 -0800 Subject: [PATCH 0218/8554] Minor documentation formatting fix. Change: 151758582 --- tensorflow_serving/g3doc/serving_inception.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 514495fd5a8..b4c9fbce46d 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -200,7 +200,7 @@ kubeconfig entry generated for inception-serving-cluster. ### Upload the Docker image Let's now push our image to the -[Google Container Registry] (https://cloud.google.com/container-registry/docs/) +[Google Container Registry](https://cloud.google.com/container-registry/docs/) so that we can run it on Google Cloud Platform. First we tag the `$USER/inception_serving` image using the Container Registry From dd7e911aa927fa846d3fc4ea5c602603fd505da5 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Thu, 30 Mar 2017 14:34:11 -0800 Subject: [PATCH 0219/8554] Minor link fix in SignatureDef documentation. Change: 151759589 --- tensorflow_serving/g3doc/signature_defs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/signature_defs.md b/tensorflow_serving/g3doc/signature_defs.md index 32ac3b95fd4..7315f00a314 100644 --- a/tensorflow_serving/g3doc/signature_defs.md +++ b/tensorflow_serving/g3doc/signature_defs.md @@ -55,7 +55,7 @@ constants. Specifically: In addition, SavedModel provides a [util](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py) -to help build a signature-def: link. +to help build a signature-def. ## Sample structures From 21459954d1ca87f799edcd06a3d1cc14cec1c2fc Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 30 Mar 2017 15:41:15 -0700 Subject: [PATCH 0220/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 12a98726e76..2a4811054a9 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 12a98726e769e988f6368a029ec2f5b0ac3ccbd4 +Subproject commit 2a4811054a9e6b83e1f5a2705a92aab50e151b13 diff --git a/tf_models b/tf_models index b0ee52c89bf..c15fada2811 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit b0ee52c89bfdb4f012664c1325520f8d0b803017 +Subproject commit c15fada28113eca32dc98d6e3bec4755d0d5b4c2 From 6562e78086891423d0e8acb0b294495c617392e2 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 31 Mar 2017 13:58:31 -0800 Subject: [PATCH 0221/8554] Update minimum Bazel version to 0.4.5. Change: 151872067 --- WORKSPACE | 2 +- tensorflow_serving/g3doc/setup.md | 8 ++++---- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index c5337768213..071d867b4d8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,4 +23,4 @@ tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:workspace.bzl", "check_version") -check_version("0.4.2") +check_version("0.4.5") diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index c0be883ce20..9b2e9bdd4a5 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -6,7 +6,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel -TensorFlow Serving requires Bazel 0.4.2 or higher. You can find the Bazel +TensorFlow Serving requires Bazel 0.4.5 or higher. You can find the Bazel installation instructions [here](http://bazel.build/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the @@ -14,13 +14,13 @@ following steps: 1. Download the relevant binary from [here](https://github.com/bazelbuild/bazel/releases). - Let's say you downloaded bazel-0.4.2-installer-linux-x86_64.sh. You would + Let's say you downloaded bazel-0.4.5-installer-linux-x86_64.sh. You would execute: ~~~shell cd ~/Downloads - chmod +x bazel-0.4.2-installer-linux-x86_64.sh - ./bazel-0.4.2-installer-linux-x86_64.sh --user + chmod +x bazel-0.4.5-installer-linux-x86_64.sh + ./bazel-0.4.5-installer-linux-x86_64.sh --user ~~~ 2. Set up your environment. Put this in your ~/.bashrc. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index ad0cc711410..07a109df8ee 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -56,7 +56,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.4.2 +ENV BAZEL_VERSION 0.4.5 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ From 9524641a4c0dad435cfc3582814fabaebaee3079 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 31 Mar 2017 18:17:25 -0800 Subject: [PATCH 0222/8554] This CL is a no-op for the public version. Change: 151893624 --- tensorflow_serving/apis/BUILD | 7 +++++++ tensorflow_serving/serving.bzl | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 84ca39f02a5..2d2693bf2af 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -22,6 +22,7 @@ filegroup( load("//tensorflow_serving:serving.bzl", "serving_proto_library") load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") +load("//tensorflow_serving:serving.bzl", "serving_go_grpc_library") serving_proto_library( name = "get_model_metadata_proto", @@ -136,6 +137,12 @@ py_library( ], ) +serving_go_grpc_library( + name = "prediction_service_grpc", + srcs = [":prediction_service_proto"], + deps = [":prediction_service_proto"], +) + serving_proto_library( name = "classification_proto", srcs = ["classification.proto"], diff --git a/tensorflow_serving/serving.bzl b/tensorflow_serving/serving.bzl index 0eed287abb1..c55e12567c8 100644 --- a/tensorflow_serving/serving.bzl +++ b/tensorflow_serving/serving.bzl @@ -24,6 +24,10 @@ def serving_proto_library(name, srcs=[], has_services=False, testonly=testonly, visibility=visibility,) +def serving_go_grpc_library(**kwargs): # pylint: disable=unused-argument + """Build the Go gRPC bindings for a service. Not yet implemented.""" + return + def serving_proto_library_py(name, proto_library, srcs=[], deps=[], visibility=None, testonly=0): # pylint: disable=unused-argument py_proto_library(name=name, srcs=srcs, From b6e34d530bf8fd3cd121e7d4628222bd32e39843 Mon Sep 17 00:00:00 2001 From: Vincent Vanhoucke Date: Fri, 7 Apr 2017 07:54:39 -0800 Subject: [PATCH 0223/8554] Move MNIST pointers to mirror hosted by the CVDF on Google Cloud. Fixes: #9031 Change: 152504901 --- tensorflow_serving/example/mnist_input_data.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/example/mnist_input_data.py b/tensorflow_serving/example/mnist_input_data.py index 6a532cb2197..3e8021a2435 100644 --- a/tensorflow_serving/example/mnist_input_data.py +++ b/tensorflow_serving/example/mnist_input_data.py @@ -25,7 +25,8 @@ import numpy from six.moves import urllib -SOURCE_URL = 'http://yann.lecun.com/exdb/mnist/' +# CVDF mirror of http://yann.lecun.com/exdb/mnist/ +SOURCE_URL = 'https://storage.googleapis.com/cvdf-datasets/mnist/' TRAIN_IMAGES = 'train-images-idx3-ubyte.gz' TRAIN_LABELS = 'train-labels-idx1-ubyte.gz' TEST_IMAGES = 't10k-images-idx3-ubyte.gz' From cf09866154b12ef215820cf3a3fada6f5e3d4fc8 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 7 Apr 2017 11:31:58 -0700 Subject: [PATCH 0224/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 2a4811054a9..f8dce81aeaf 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 2a4811054a9e6b83e1f5a2705a92aab50e151b13 +Subproject commit f8dce81aeaff40dc78d398741854ad8766806f91 diff --git a/tf_models b/tf_models index c15fada2811..405bb623bfd 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit c15fada28113eca32dc98d6e3bec4755d0d5b4c2 +Subproject commit 405bb623bfdc5a1bf97550727272f43970414560 From 495217c6b2ba0892d09759e4f5e9206b111bf1ae Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 10 Apr 2017 09:17:10 -0800 Subject: [PATCH 0225/8554] Update documentation which refers to the old dynamic_model_config. Change: 152701736 --- tensorflow_serving/g3doc/serving_advanced.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 9e871eb3311..12d49645b15 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -115,8 +115,8 @@ commonly used options: * `ModelServerConfig` that specifies models to be loaded. Models are declared either through `model_config_list`, which declares a static list of models, or - through `dynamic_model_config`, which declares a dynamic list of models that - may get updated at runtime. + through `custom_model_config`, which defines a custom way to declare a list of + models that may get updated at runtime. * `PlatformConfigMap` that maps from the name of the platform (such as `tensorflow`) to the `PlatformConfig`, which is used to create the `SourceAdapter`. `SourceAdapter` adapts `StoragePath` (the path where a model From 3c65090aaf94b9eadac4856599a38c718935f932 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 11 Apr 2017 14:46:10 -0800 Subject: [PATCH 0226/8554] Fix a bug in the classifier where if scores are set to a Tensor of the wrong rank, the server would hit a segfault. Change: 152871421 --- .../servables/tensorflow/classifier.cc | 10 +++ .../servables/tensorflow/classifier_test.cc | 79 +++++++++++++------ 2 files changed, 64 insertions(+), 25 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/classifier.cc b/tensorflow_serving/servables/tensorflow/classifier.cc index af31b2b47f1..7c1d490e970 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.cc +++ b/tensorflow_serving/servables/tensorflow/classifier.cc @@ -96,6 +96,11 @@ class TensorFlowClassifier : public ClassifierInterface { } // Validate scores output Tensor. if (scores) { + if (scores->dims() != 2) { + return errors::InvalidArgument( + "Expected Tensor shape: [batch_size num_classes] but got ", + scores->shape().DebugString()); + } if (scores->dtype() != DT_FLOAT) { return errors::Internal("Expected scores Tensor of DT_FLOAT. Got: ", DataType_Name(scores->dtype())); @@ -389,6 +394,11 @@ Status PostProcessClassificationResult( } // Validate scores output Tensor. if (scores) { + if (scores->dims() != 2) { + return errors::InvalidArgument( + "Expected Tensor shape: [batch_size num_classes] but got ", + scores->shape().DebugString()); + } if (scores->dtype() != DT_FLOAT) { return errors::InvalidArgument( "Expected scores Tensor of DT_FLOAT. Got: ", diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc index 67848b5dbe3..13e5e5d98a5 100644 --- a/tensorflow_serving/servables/tensorflow/classifier_test.cc +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -54,9 +54,11 @@ const char kOutputPlusOneClassTensor[] = "outputPlusOne:0"; const char kClassFeature[] = "class"; const char kScoreTensor[] = "score:0"; const char kScoreFeature[] = "score"; +const char kImproperlySizedScoresTensor[] = "ImproperlySizedScores:0"; const char kOutputPlusOneSignature[] = "output_plus_one"; const char kInvalidNamedSignature[] = "invalid_regression_signature"; +const char kImproperlySizedScoresSignature[] = "ImproperlySizedScoresSignature"; // Fake Session used for testing TensorFlowClassifier. // Assumes the input Tensor "input:0" has serialized tensorflow::Example values. @@ -101,12 +103,7 @@ class FakeSession : public tensorflow::Session { if (inputs.size() != 1 || inputs[0].first != kInputTensor) { return errors::Internal("Expected one input Tensor."); } - for (const auto& output_name : output_names) { - if (output_name != kClassTensor && output_name != kScoreTensor && - output_name != kOutputPlusOneClassTensor) { - return errors::Internal("Unsupported output Tensor: ", output_name); - } - } + const Tensor& input = inputs[0].second; std::vector examples; TF_RETURN_IF_ERROR(GetExamples(input, &examples)); @@ -120,6 +117,11 @@ class FakeSession : public tensorflow::Session { } else if (output_name == kScoreTensor || output_name == kOutputPlusOneClassTensor) { outputs->push_back(scores); + } else if (output_name == kImproperlySizedScoresTensor) { + // Insert a rank 3 tensor which should be an error because scores are + // expected to be rank 2. + outputs->emplace_back(DT_FLOAT, TensorShape({scores.dim_size(0), + scores.dim_size(1), 10})); } } @@ -246,7 +248,7 @@ class ClassifierTest : public ::testing::TestWithParam { bundle_.reset(new SessionBundle); meta_graph_def_ = &bundle_->meta_graph_def; optional expected_timeout = GetRunOptions().timeout_in_ms(); - if (!GetParam()) { + if (!UseSavedModel()) { // For SessionBundle we don't propagate the timeout. expected_timeout = nullopt; } @@ -267,6 +269,11 @@ class ClassifierTest : public ::testing::TestWithParam { AddNamedSignature(kInputTensor, kOutputPlusOneClassTensor, kInvalidNamedSignature, false /* is_classification */, &signatures); + + // Add a named signature where the output is not valid. + AddNamedSignature(kInputTensor, kImproperlySizedScoresTensor, + kImproperlySizedScoresSignature, + true /* is_classification */, &signatures); TF_ASSERT_OK( tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); } @@ -287,8 +294,12 @@ class ClassifierTest : public ::testing::TestWithParam { return example; } + // Whether or not to use SavedModel for this test. Simply wraps GetParam() + // with a more meaningful name. + bool UseSavedModel() { return GetParam(); } + Status Create() { - if (GetParam()) { + if (UseSavedModel()) { std::unique_ptr saved_model(new SavedModelBundle); TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( *bundle_, saved_model.get())); @@ -487,11 +498,10 @@ TEST_P(ClassifierTest, ValidNamedSignature) { *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); TF_ASSERT_OK(classifier_->Classify(request_, &result_)); - // GetParam() is 'is_saved_model' in this test. If using saved_model, this - // test should use the kOutputPlusOneSignature named signature. Otherwise, - // when using session_bundle, the signature_name in the model_spec will be - // ignored and the default signature will be used. - if (GetParam()) { + // If using saved_model, this test should use the kOutputPlusOneSignature + // named signature. Otherwise, when using session_bundle, the signature_name + // in the model_spec will be ignored and the default signature will be used. + if (UseSavedModel()) { EXPECT_THAT(result_, EqualsProto(" classifications { " " classes { " " label: 'dos' " @@ -545,11 +555,10 @@ TEST_P(ClassifierTest, InvalidNamedSignature) { *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); const Status status = classifier_->Classify(request_, &result_); - // GetParam() is 'is_saved_model' in this test. If using saved_model, this - // test should fail because the named_signature requested is actually a - // regression signature. When using session_bundle, the signature_name - // will be ignored and the default signature will be used. - if (GetParam()) { + // If using saved_model, this test should fail because the named_signature + // requested is actually a regression signature. When using session_bundle, + // the signature_name will be ignored and the default signature will be used. + if (UseSavedModel()) { ASSERT_FALSE(status.ok()); EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; } else { @@ -577,6 +586,26 @@ TEST_P(ClassifierTest, InvalidNamedSignature) { } } +TEST_P(ClassifierTest, MalformedScores) { + // If not using SavedModel, we don't use named signatures so the test is not + // actually testing the right thing. Skip it. + if (!UseSavedModel()) { + return; + } + + TF_ASSERT_OK(Create()); + request_.mutable_model_spec()->set_signature_name( + kImproperlySizedScoresSignature); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"dos", 2}, {"uno", 1}}); + *examples->Add() = example({{"cuatro", 4}, {"tres", 3}}); + const Status status = classifier_->Classify(request_, &result_); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; +} + TEST_P(ClassifierTest, MissingClassificationSignature) { tensorflow::serving::Signatures signatures; signatures.mutable_default_signature(); @@ -591,7 +620,7 @@ TEST_P(ClassifierTest, MissingClassificationSignature) { // Old SessionBundle code treats a missing signature as a FAILED_PRECONDITION // but new SavedModel code treats it as an INVALID_ARGUMENT (signature // specified in the request was invalid). - if (GetParam()) { + if (UseSavedModel()) { EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; } else { EXPECT_EQ(::tensorflow::error::FAILED_PRECONDITION, status.code()) @@ -634,7 +663,7 @@ TEST_P(ClassifierTest, EmptyExampleListWithContext) { TEST_P(ClassifierTest, RunsFails) { MockSession* mock = new MockSession; bundle_->session.reset(mock); - if (GetParam()) { + if (UseSavedModel()) { EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) .WillRepeatedly( ::testing::Return(errors::Internal("Run totally failed"))); @@ -659,7 +688,7 @@ TEST_P(ClassifierTest, ClassesIncorrectTensorBatchSize) { Tensor classes(DT_STRING, TensorShape({1, 2})); Tensor scores(DT_FLOAT, TensorShape({2, 2})); std::vector outputs = {classes, scores}; - if (GetParam()) { + if (UseSavedModel()) { EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), ::testing::Return(Status::OK()))); @@ -686,7 +715,7 @@ TEST_P(ClassifierTest, ClassesIncorrectTensorType) { Tensor classes(DT_FLOAT, TensorShape({2, 2})); Tensor scores(DT_FLOAT, TensorShape({2, 2})); std::vector outputs = {classes, scores}; - if (GetParam()) { + if (UseSavedModel()) { EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), ::testing::Return(Status::OK()))); @@ -714,7 +743,7 @@ TEST_P(ClassifierTest, ScoresIncorrectTensorBatchSize) { // This Tensor only has one batch item but we will have two inputs. Tensor scores(DT_FLOAT, TensorShape({1, 2})); std::vector outputs = {classes, scores}; - if (GetParam()) { + if (UseSavedModel()) { EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), ::testing::Return(Status::OK()))); @@ -741,7 +770,7 @@ TEST_P(ClassifierTest, ScoresIncorrectTensorType) { // This Tensor is the wrong type for class. Tensor scores(DT_STRING, TensorShape({2, 2})); std::vector outputs = {classes, scores}; - if (GetParam()) { + if (UseSavedModel()) { EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), ::testing::Return(Status::OK()))); @@ -769,7 +798,7 @@ TEST_P(ClassifierTest, MismatchedNumberOfTensorClasses) { // Scores Tensor has three scores but classes only has two labels. Tensor scores(DT_FLOAT, TensorShape({2, 3})); std::vector outputs = {classes, scores}; - if (GetParam()) { + if (UseSavedModel()) { EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) .WillRepeatedly(::testing::DoAll(::testing::SetArgPointee<4>(outputs), ::testing::Return(Status::OK()))); From c579f3343a68c45fe38f2835c4a504e47c3c3d70 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 12 Apr 2017 14:07:11 -0800 Subject: [PATCH 0227/8554] Remove local tag from tensorflow_model_server_test. Change: 152992941 --- tensorflow_serving/model_servers/BUILD | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index f8831ab1f73..cbeaaf042e3 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -159,10 +159,8 @@ py_test( "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/variables/variables.index", "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], - tags = [ - "local", - "manual", - ], + # TODO(b/36008812): Remove this tag. + tags = ["manual"], deps = [ "//tensorflow_serving/apis:prediction_service_proto_py_pb2", "@org_tensorflow//tensorflow:tensorflow_py", From b50eff5b3fa98c1577053d39469a1cea4b626727 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 13 Apr 2017 12:12:02 -0800 Subject: [PATCH 0228/8554] Read the full stderr output in tensorflow_model_server_test when checking for an error message. Change: 153096707 --- .../model_servers/tensorflow_model_server_test.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 81b7c0a91ba..313ec74358e 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -377,16 +377,11 @@ def testBadModelConfig(self): self._GetBadModelConfigFile(), True, # use_saved_model pipe=subprocess.PIPE) - last_line = None - for line in self.server_proc.stderr: - last_line = line error_message = ( 'Invalid protobuf file: \'%s\'') % self._GetBadModelConfigFile() - - self.assertNotEqual(last_line, None) - self.assertGreater(last_line.find(error_message), 0) - # self.assertEquals(self.server_proc.poll(), 1) + self.assertNotEqual(self.server_proc.stderr, None) + self.assertGreater(self.server_proc.stderr.read().find(error_message), -1) if __name__ == '__main__': tf.test.main() From 1c4fab796ce02472860c2c214267c8abeaf8bb97 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 13 Apr 2017 12:41:15 -0800 Subject: [PATCH 0229/8554] Ensure regression output tensors are either of shape [batch_size] or [batch_size, 1] to avoid potential issues when populating RegressionResults. Change: 153100407 --- .../servables/tensorflow/regressor.cc | 8 +++- .../servables/tensorflow/regressor_test.cc | 47 ++++++++++++++++--- 2 files changed, 46 insertions(+), 9 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/regressor.cc b/tensorflow_serving/servables/tensorflow/regressor.cc index dc044716a44..a6ccf1d9f1a 100644 --- a/tensorflow_serving/servables/tensorflow/regressor.cc +++ b/tensorflow_serving/servables/tensorflow/regressor.cc @@ -294,12 +294,16 @@ Status PostProcessRegressionResult( } // Ensure the regression score output is shaped how we expect. - // There should be one float Tensor of shape, - // [batch_size, num_recommendations]. if (output_tensor == nullptr) { return errors::InvalidArgument(strings::StrCat( "Could not find output tensor '", output_tensor_name, "'")); } + if (!(output_tensor->dims() == 1 || + (output_tensor->dims() == 2 && output_tensor->dim_size(1) == 1))) { + return errors::InvalidArgument( + "Expected output Tensor shape to be either [batch_size] or ", + "[batch_size, 1] but got ", output_tensor->shape().DebugString()); + } if (num_examples != output_tensor->dim_size(0)) { return errors::InvalidArgument(strings::StrCat( "Input batch size did not match output batch size: ", num_examples, diff --git a/tensorflow_serving/servables/tensorflow/regressor_test.cc b/tensorflow_serving/servables/tensorflow/regressor_test.cc index 46ce82dd0f0..7dd045494c4 100644 --- a/tensorflow_serving/servables/tensorflow/regressor_test.cc +++ b/tensorflow_serving/servables/tensorflow/regressor_test.cc @@ -49,10 +49,12 @@ using test_util::MockSession; const char kInputTensor[] = "input:0"; const char kOutputTensor[] = "output:0"; const char kOutputPlusOneTensor[] = "outputPlusOne:0"; +const char kImproperlySizedOutputTensor[] = "ImproperlySizedOutput:0"; const char kOutputFeature[] = "output"; const char kOutputPlusOneSignature[] = "output_plus_one"; const char kInvalidNamedSignature[] = "invalid_classification_signature"; +const char kImproperlySizedOutputSignature[] = "ImproperlySizedOutputSignature"; // Fake Session used for testing TensorFlowRegressor // Assumes the input Tensor "input:0" has serialized tensorflow::Example values. @@ -96,10 +98,6 @@ class FakeSession : public tensorflow::Session { if (inputs.size() != 1 || inputs[0].first != kInputTensor) { return errors::Internal("Expected one input Tensor."); } - if (output_names.size() != 1 || (output_names[0] != kOutputTensor && - output_names[0] != kOutputPlusOneTensor)) { - return errors::Internal("Expected one output Tensor."); - } const Tensor& input = inputs[0].second; std::vector examples; TF_RETURN_IF_ERROR(GetExamples(input, &examples)); @@ -145,8 +143,17 @@ class FakeSession : public tensorflow::Session { return errors::Internal("empty example list"); } const int batch_size = examples.size(); - *tensor = Tensor(DT_FLOAT, TensorShape({batch_size, 1})); - auto output_matrix = tensor->matrix(); + if (output_tensor_name == kImproperlySizedOutputTensor) { + // Insert a rank 3 tensor which should be an error because outputs are + // expected to be of shape [batch_size] or [batch_size, 1]. + *tensor = Tensor(DT_FLOAT, TensorShape({batch_size, 1, 10})); + return Status::OK(); + } + // Both tensor shapes are valid, so make one of shape [batch_size, 1] and + // the rest of shape [batch_size]. + *tensor = output_tensor_name == kOutputPlusOneTensor + ? Tensor(DT_FLOAT, TensorShape({batch_size, 1})) + : Tensor(DT_FLOAT, TensorShape({batch_size})); const float offset = output_tensor_name == kOutputPlusOneTensor ? 1 : 0; for (int i = 0; i < batch_size; ++i) { @@ -154,7 +161,7 @@ class FakeSession : public tensorflow::Session { if (feature.float_list().value_size() != 1) { return errors::Internal("incorrect number of values in output feature"); } - output_matrix(i, 0) = feature.float_list().value(0) + offset; + tensor->flat()(i) = feature.float_list().value(0) + offset; } return Status::OK(); } @@ -218,6 +225,12 @@ class RegressorTest : public ::testing::TestWithParam { AddNamedSignature(kInputTensor, kOutputPlusOneTensor, kInvalidNamedSignature, false /* is_regression */, &signatures); + + // Add a named signature where the output is not valid. + AddNamedSignature(kInputTensor, kImproperlySizedOutputTensor, + kImproperlySizedOutputSignature, true /* is_regression */, + &signatures); + TF_ASSERT_OK( tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); } @@ -353,6 +366,26 @@ TEST_P(RegressorTest, InvalidNamedSignature) { } } +TEST_P(RegressorTest, MalformedOutputs) { + // If not using SavedModel, we don't use named signatures so the test is not + // actually testing the right thing. Skip it. + if (!GetParam()) { + return; + } + + TF_ASSERT_OK(Create()); + request_.mutable_model_spec()->set_signature_name( + kImproperlySizedOutputSignature); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example_with_output(2.0); + *examples->Add() = example_with_output(3.0); + const Status status = regressor_->Regress(request_, &result_); + + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; +} + TEST_P(RegressorTest, EmptyInput) { TF_ASSERT_OK(Create()); // Touch input. From 5aa5757a425c489555ebcd835a26e2b540637d39 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Fri, 14 Apr 2017 15:56:48 -0800 Subject: [PATCH 0230/8554] Adds a ServerCore option to do custom updates of ServerRequestLogger. Change: 153222833 --- tensorflow_serving/model_servers/server_core.cc | 9 ++++++--- tensorflow_serving/model_servers/server_core.h | 7 +++++++ 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 0d0448a7cb2..c9386a149f4 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -292,6 +292,11 @@ Status ServerCore::AddModelsViaCustomModelConfig() { } Status ServerCore::MaybeUpdateServerRequestLogger() { + if (options_.server_request_logger_updater) { + return options_.server_request_logger_updater( + config_, options_.server_request_logger.get()); + } + std::map logging_config_map; for (const auto& model_config : config_.model_config_list().config()) { if (model_config.has_logging_config()) { @@ -299,9 +304,7 @@ Status ServerCore::MaybeUpdateServerRequestLogger() { {model_config.name(), model_config.logging_config()}); } } - TF_RETURN_IF_ERROR( - options_.server_request_logger->Update(logging_config_map)); - return Status::OK(); + return options_.server_request_logger->Update(logging_config_map); } Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index d3c536e8819..a9e82ef20f0 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -78,6 +78,10 @@ class ServerCore : public Manager { const ::google::protobuf::Any& any, EventBus* event_bus, UniquePtrWithDeps* manager)>; + // Function signature used to update the server_request_logger. + using ServerRequestLoggerUpdater = + std::function; + struct Options { // ModelServer configuration. ModelServerConfig model_server_config; @@ -121,6 +125,9 @@ class ServerCore : public Manager { // Logger used for logging requests hitting the server. std::unique_ptr server_request_logger; + + // If set, we use this function to update the server_request_logger. + ServerRequestLoggerUpdater server_request_logger_updater; }; virtual ~ServerCore() = default; From 5ffe1ac9215215443615cab11f16aab63913c41d Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Tue, 18 Apr 2017 10:15:02 -0800 Subject: [PATCH 0231/8554] Fix TensorFlow Serving Dockerfile.devel to use the right license. Change: 153485269 --- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 07a109df8ee..4c61f9286bd 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -61,7 +61,7 @@ WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ - curl -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE.txt && \ + curl -fSsL -o /bazel/LICENSE.txt https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ chmod +x bazel-*.sh && \ ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ cd / && \ From ce53656b38f040a357fee9cfd2ed35000ca9ae43 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Wed, 19 Apr 2017 08:18:06 -0800 Subject: [PATCH 0232/8554] Add details and relevant links to serving basic codelab. Change: 153594681 --- tensorflow_serving/g3doc/serving_basic.md | 99 ++++++++++++++++------- 1 file changed, 68 insertions(+), 31 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index ff18b40c7b1..463d006d279 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -14,12 +14,10 @@ tutorial. The code for this tutorial consists of two parts: -* A Python file -([mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_saved_model.py)) +* A Python file, [mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_saved_model.py), that trains and exports the model. -* A C++ file -[main.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc) +* A C++ file, [main.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc), which is the standard TensorFlow model server that discovers new exported models and runs a [gRPC](http://www.grpc.io) service for serving them. @@ -32,9 +30,16 @@ is in the MNIST For ML Beginners tutorial. The TensorFlow graph is launched in TensorFlow session `sess`, with the input tensor (image) as `x` and output tensor (Softmax score) as `y`. -Then we use TensorFlow Serving `SavedModelBuilder` module to export the model. -`SavedModelBuilder` saves a "snapshot" of the trained model to reliable storage -so that it can be loaded later for inference. +Then we use TensorFlow's [SavedModelBuilder module](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/builder.py) +to export the model. `SavedModelBuilder` saves a "snapshot" of the trained model +to reliable storage so that it can be loaded later for inference. + +For details on the SavedModel format, please see the documentation at +[SavedModel README.md](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/README.md). + +From [mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_saved_model.py), +the following is a short code snippet to illustrate the general process of +saving a model to disk. ~~~python from tensorflow.python.saved_model import builder as saved_model_builder @@ -58,44 +63,76 @@ builder.save() ~~~ `SavedModelBuilder.__init__` takes the following argument: -* `export_path` is the path of the export directory. `SavedModelBuilder` will -create the directory if it does not exist. In the example, we concatenate -the command line argument and `FLAGS.model_version` to obtain the export -directory. `FLAGS.model_version` specifies the **version** of the model. You -should specify a larger integer value when exporting a newer version of the same -model. Each version will be exported to a different sub-directory under the -given path. + +* `export_path` is the path of the export directory. + +`SavedModelBuilder` will create the directory if it does not exist. In the +example, we concatenate the command line argument and `FLAGS.model_version` to +obtain the export directory. `FLAGS.model_version` specifies the **version** of +the model. You should specify a larger integer value when exporting a newer +version of the same model. Each version will be exported to a different +sub-directory under the given path. You can add meta graph and variables to the builder using `SavedModelBuilder.add_meta_graph_and_variables()` with the following arguments: - * `sess` is the TensorFlow session that holds the trained model you are + +* `sess` is the TensorFlow session that holds the trained model you are exporting. - * `tags` is the set of tags with which to save the meta graph. +* `tags` is the set of tags with which to save the meta graph. In this case, + since we intend to use the graph in serving, we use the `serve` tag from + predefined SavedModel tag constants. For more details, see [tag_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) + and [related TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/tag_constants). - * `signature_def_map` specifies the map of user-supplied key for a +* `signature_def_map` specifies the map of user-supplied key for a **signature** to a tensorflow::SignatureDef to add to the meta graph. - Signature specifies what type of model is being exported, and the input/output - tensors to bind to when running inference. The special signature key - `serving_default` specifies the default serving signature. - `signature_def_utils.build_signature_def()` accepts the following arguments: + Signature specifies what type of model is being exported, and the + input/output tensors to bind to when running inference. + + The special signature key `serving_default` specifies the default serving + signature. The default serving signature def key, along with other constants + related to signatures, are defined as part of SavedModel signature constants. + For more details, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) + and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). + + Further, to help build signature defs easily, the SavedModel API provides + [signature def utils](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_def_utils). + Specifically, in the `mnist_saved_model.py` code snippet above, we use + `signature_def_utils.build_signature_def()` to build `predict_signature` and + `classification_signature`. + + As an example for how `predict_signature` is defined, the util takes the + following arguments: * `inputs={'images': tensor_info_x}` specifies the input tensor info. * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. - * `images` and `scores` are tensor alias names. They can be whatever - unique strings you want, and they will become the logical names of tensor - `x` and `y` that you refer to for tensor binding when sending prediction - requests later. For instance, if `x` refers to the tensor with name - 'long_tensor_name_foo' and `y` refers to the tensor with name - 'generated_tensor_name_bar', `builder` will store tensor logical name to - real name mapping ('images' -> 'long_tensor_name_foo' and 'scores' -> - 'generated_tensor_name_bar') and allow user to refer to these tensors with - their logical names when running inference. + Note that `tensor_info_x` and `tensor_info_y` have the structure of + `tensorflow::TensorInfo` protocol buffer defined [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto). + To easily build tensor infos, the TensorFlow SavedModel API also provides + [utils.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py), + with [related TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/utils). + + Also, note that `images` and `scores` are tensor alias names. They can be + whatever unique strings you want, and they will become the logical names + of tensor `x` and `y` that you refer to for tensor binding when sending + prediction requests later. + + For instance, if `x` refers to the tensor with name 'long_tensor_name_foo' + and `y` refers to the tensor with name 'generated_tensor_name_bar', + `builder` will store tensor logical name to real name mapping + ('images' -> 'long_tensor_name_foo') and ('scores' -> 'generated_tensor_name_bar'). + This allows the user to refer to these tensors with their logical names + when running inference. * `method_name` is the method used for the inference. For Prediction - requests, it should be set to `tensorflow/serving/predict`. + requests, it should be set to `tensorflow/serving/predict`. For other + method names, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) + and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). + + In addition to the description above, documentation related to signature def + structure and how to set up them up can be found [here](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/signature_defs.md). Let's run it! From d6a186f4783a002d297c0cf605c31b0038a04814 Mon Sep 17 00:00:00 2001 From: Derek Murray Date: Thu, 20 Apr 2017 22:22:29 -0800 Subject: [PATCH 0233/8554] Switch DirectSession to use _Arg and _Retval ops for feeding and fetching. This change reduces the overhead imposed by string processing and rendezvous invocation in the DirectSession::Run() call by 1--2 microseconds per value fed or fetched. RELNOTES: Improved DirectSession::Run() overhead and error checking. Feeding a value of the wrong type will now synchronously raise an INVALID_ARGUMENT error instead of asynchronously raising an INTERNAL error. Code that depends on the (undefined) behavior when feeding a tensor of the wrong type may need to be updated. Change: 153797943 --- tensorflow_serving/servables/tensorflow/predict_impl_test.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 2e23c315b1a..049f4523b79 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -234,7 +234,7 @@ TEST_P(PredictImplTest, InputTensorsHaveWrongType) { TensorflowPredictor predictor(GetParam()); // Input tensors are all wrong. EXPECT_EQ( - tensorflow::error::INTERNAL, + tensorflow::error::INVALID_ARGUMENT, predictor.Predict(GetRunOptions(), GetServerCore(), request, &response) .code()); } From a7f814a1950025c50b053ddbc16694124f35ee38 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Tue, 25 Apr 2017 10:49:58 -0700 Subject: [PATCH 0234/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index f8dce81aeaf..b6bafadd51a 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit f8dce81aeaff40dc78d398741854ad8766806f91 +Subproject commit b6bafadd51a471daa7157f515598e08e8f9f1611 diff --git a/tf_models b/tf_models index 405bb623bfd..39c59d13703 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 405bb623bfdc5a1bf97550727272f43970414560 +Subproject commit 39c59d137035c1b53b756b280510f1705c53fa4e From f25c34675f5bda4c95064622ac7c8a0edf33de46 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Wed, 26 Apr 2017 13:46:15 -0800 Subject: [PATCH 0235/8554] Call ServerRequestLogger::Update(...) unconditionally. - So that we can set logging-configs in the custom-model-config case too. Change: 154347390 --- tensorflow_serving/model_servers/server_core.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index c9386a149f4..d9972171948 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -342,7 +342,6 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { switch (config_.config_case()) { case ModelServerConfig::kModelConfigList: { TF_RETURN_IF_ERROR(AddModelsViaModelConfigList()); - TF_RETURN_IF_ERROR(MaybeUpdateServerRequestLogger()); break; } case ModelServerConfig::kCustomModelConfig: { @@ -355,6 +354,7 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { default: return errors::InvalidArgument("Invalid ServerModelConfig"); } + TF_RETURN_IF_ERROR(MaybeUpdateServerRequestLogger()); return Status::OK(); } From da6a36cc55d8bfd77ddd37f13486401ea71a2fd4 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 2 May 2017 17:49:52 -0700 Subject: [PATCH 0236/8554] Add license string to inception_client.cc and make some modifications to make code compatible with Google style. (#428) --- .../example/inception_client.cc | 99 ++++++++++--------- 1 file changed, 54 insertions(+), 45 deletions(-) diff --git a/tensorflow_serving/example/inception_client.cc b/tensorflow_serving/example/inception_client.cc index 6f8ebc69f82..4b1d183ae87 100644 --- a/tensorflow_serving/example/inception_client.cc +++ b/tensorflow_serving/example/inception_client.cc @@ -1,40 +1,53 @@ -#include +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + #include +#include -#include -#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" +#include "grpc++/create_channel.h" +#include "grpc++/security/credentials.h" +#include "google/protobuf/map.h" #include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/platform/types.h" #include "tensorflow/core/util/command_line_flags.h" +#include "tensorflow_serving/apis/prediction_service.grpc.pb.h" using grpc::Channel; using grpc::ClientContext; -using grpc::ClientReader; -using grpc::ClientReaderWriter; -using grpc::ClientWriter; using grpc::Status; - using tensorflow::serving::PredictRequest; using tensorflow::serving::PredictResponse; using tensorflow::serving::PredictionService; -typedef google::protobuf::Map< std::string, tensorflow::TensorProto > OutMap; - +typedef google::protobuf::Map OutMap; class ServingClient { public: ServingClient(std::shared_ptr channel) - : stub_(PredictionService::NewStub(channel)) { - } + : stub_(PredictionService::NewStub(channel)) {} - std::string callPredict(std::string model_name, std::string file_path){ + tensorflow::string callPredict(const tensorflow::string& model_name, + const tensorflow::string& file_path) { PredictRequest predictRequest; PredictResponse response; ClientContext context; predictRequest.mutable_model_spec()->set_name(model_name); - google::protobuf::Map< std::string, tensorflow::TensorProto >& inputs = + google::protobuf::Map& inputs = *predictRequest.mutable_inputs(); tensorflow::TensorProto proto; @@ -46,15 +59,14 @@ class ServingClient { return ""; } - std::filebuf * pbuf = imageFile.rdbuf(); - long fileSize = pbuf->pubseekoff(0, std::ios::end, std::ios::in); - + std::filebuf* pbuf = imageFile.rdbuf(); + auto fileSize = pbuf->pubseekoff(0, std::ios::end, std::ios::in); + char* image = new char[fileSize](); pbuf->pubseekpos(0, std::ios::in); pbuf->sgetn(image, fileSize); imageFile.close(); - proto.set_dtype(tensorflow::DataType::DT_STRING); proto.add_string_val(image, fileSize); @@ -62,36 +74,36 @@ class ServingClient { proto.mutable_tensor_shape()->add_dim()->set_size(1); inputs["images"] = proto; - + Status status = stub_->Predict(&context, predictRequest, &response); delete[] image; if (status.ok()) { std::cout << "call predict ok" << std::endl; - std::cout << "outputs size is "<< response.outputs_size() << std::endl; - OutMap& map_outputs = *response.mutable_outputs(); + std::cout << "outputs size is " << response.outputs_size() << std::endl; + OutMap& map_outputs = *response.mutable_outputs(); OutMap::iterator iter; int output_index = 0; - - for(iter = map_outputs.begin();iter != map_outputs.end(); ++iter){ - tensorflow::TensorProto& result_tensor_proto= iter->second; + + for (iter = map_outputs.begin(); iter != map_outputs.end(); ++iter) { + tensorflow::TensorProto& result_tensor_proto = iter->second; tensorflow::Tensor tensor; bool converted = tensor.FromProto(result_tensor_proto); if (converted) { - std::cout << "the result tensor[" << output_index << "] is:" << - std::endl << tensor.SummarizeValue(10) << std::endl; - }else { - std::cout << "the result tensor[" << output_index << - "] convert failed." << std::endl; + std::cout << "the result tensor[" << output_index + << "] is:" << std::endl + << tensor.SummarizeValue(10) << std::endl; + } else { + std::cout << "the result tensor[" << output_index + << "] convert failed." << std::endl; } ++output_index; } return "Done."; } else { - std::cout << "gRPC call return code: " - < flag_list = { - tensorflow::Flag("server_port", &server_port, - "the IP and port of the server"), - tensorflow::Flag("image_file", &image_file, - "the path to the "), - tensorflow::Flag("model_name", &model_name, "name of model") - }; - std::string usage = tensorflow::Flags::Usage(argv[0], flag_list); + tensorflow::Flag("server_port", &server_port, + "the IP and port of the server"), + tensorflow::Flag("image_file", &image_file, "the path to the "), + tensorflow::Flag("model_name", &model_name, "name of model")}; + tensorflow::string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || image_file.empty()) { std::cout << usage; @@ -119,10 +129,9 @@ int main(int argc, char** argv) { } ServingClient guide( - grpc::CreateChannel( server_port, - grpc::InsecureChannelCredentials())); - std::cout << "calling predict using file: " << - image_file << " ..." << std::endl; + grpc::CreateChannel(server_port, grpc::InsecureChannelCredentials())); + std::cout << "calling predict using file: " << image_file << " ..." + << std::endl; std::cout << guide.callPredict(model_name, image_file) << std::endl; return 0; From 4ba4bda6b6221694af49f1e000a7254012709fae Mon Sep 17 00:00:00 2001 From: Salas Date: Wed, 3 May 2017 09:23:39 +0800 Subject: [PATCH 0237/8554] remove deprecated calls and upgrade them to newer ones in examples (#375) --- tensorflow_serving/example/inception_export.py | 6 +++--- tensorflow_serving/example/inception_saved_model.py | 6 +++--- tensorflow_serving/example/mnist_export.py | 9 +++++---- tensorflow_serving/example/mnist_saved_model.py | 9 +++++---- .../tensorflow/testdata/export_bad_half_plus_two.py | 2 +- .../tensorflow/testdata/export_half_plus_two.py | 2 +- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index 1c5fbdf8157..e807a08c680 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -90,8 +90,8 @@ def export(): class_descriptions.append(texts[s]) class_tensor = tf.constant(class_descriptions) - classes = tf.contrib.lookup.index_to_string(tf.to_int64(indices), - mapping=class_tensor) + table = tf.contrib.lookup.index_to_string_table_from_tensor(class_tensor) + classes = table.lookup(tf.to_int64(indices)) # Restore variables from training checkpoint. variable_averages = tf.train.ExponentialMovingAverage( @@ -114,7 +114,7 @@ def export(): return # Export inference model. - init_op = tf.group(tf.initialize_all_tables(), name='init_op') + init_op = tf.group(tf.tables_initializer(), name='init_op') classification_signature = exporter.classification_signature( input_tensor=serialized_tf_example, classes_tensor=classes, diff --git a/tensorflow_serving/example/inception_saved_model.py b/tensorflow_serving/example/inception_saved_model.py index febf2bd5d19..1654c75cc74 100644 --- a/tensorflow_serving/example/inception_saved_model.py +++ b/tensorflow_serving/example/inception_saved_model.py @@ -94,8 +94,8 @@ def export(): class_descriptions.append(texts[s]) class_tensor = tf.constant(class_descriptions) - classes = tf.contrib.lookup.index_to_string( - tf.to_int64(indices), mapping=class_tensor) + table = tf.contrib.lookup.index_to_string_table_from_tensor(class_tensor) + classes = table.lookup(tf.to_int64(indices)) # Restore variables from training checkpoint. variable_averages = tf.train.ExponentialMovingAverage( @@ -152,7 +152,7 @@ def export(): method_name=signature_constants.PREDICT_METHOD_NAME) legacy_init_op = tf.group( - tf.initialize_all_tables(), name='legacy_init_op') + tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 0c008b3bb42..a5a59f63716 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -68,13 +68,14 @@ def main(_): y_ = tf.placeholder('float', shape=[None, 10]) w = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) values, indices = tf.nn.top_k(y, 10) - prediction_classes = tf.contrib.lookup.index_to_string( - tf.to_int64(indices), mapping=tf.constant([str(i) for i in range(10)])) + table = tf.contrib.lookup.index_to_string_table_from_tensor( + tf.constant([str(i) for i in range(10)])) + prediction_classes = table.lookup(tf.to_int64(indices)) for _ in range(FLAGS.training_iteration): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x: batch[0], y_: batch[1]}) @@ -92,7 +93,7 @@ def main(_): # whenever code changes. export_path = sys.argv[-1] print('Exporting trained model to %s' % export_path) - init_op = tf.group(tf.initialize_all_tables(), name='init_op') + init_op = tf.group(tf.tables_initializer(), name='init_op') saver = tf.train.Saver(sharded=True) model_exporter = exporter.Exporter(saver) model_exporter.init( diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py index 35bb2d244d2..c753c505629 100644 --- a/tensorflow_serving/example/mnist_saved_model.py +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -69,13 +69,14 @@ def main(_): y_ = tf.placeholder('float', shape=[None, 10]) w = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) values, indices = tf.nn.top_k(y, 10) - prediction_classes = tf.contrib.lookup.index_to_string( - tf.to_int64(indices), mapping=tf.constant([str(i) for i in xrange(10)])) + table = tf.contrib.lookup.index_to_string_table_from_tensor( + tf.constant([str(i) for i in xrange(10)])) + prediction_classes = table.lookup(tf.to_int64(indices)) for _ in range(FLAGS.training_iteration): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x: batch[0], y_: batch[1]}) @@ -120,7 +121,7 @@ def main(_): outputs={'scores': tensor_info_y}, method_name=signature_constants.PREDICT_METHOD_NAME) - legacy_init_op = tf.group(tf.initialize_all_tables(), name='legacy_init_op') + legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py b/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py index c44c55f357d..a062e1cc7e3 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py +++ b/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py @@ -42,7 +42,7 @@ def Export(): # Note that the model is intentionally exported without using exporter, # but using the same format. This is to avoid exporter creating default # empty signatures upon export. - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() saver = tf.train.Saver() saver.export_meta_graph( filename=os.path.join(export_path, "export.meta")) diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py index b90453d6404..a199f40bd8a 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py +++ b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py @@ -41,7 +41,7 @@ def Export(): y = tf.add(tf.multiply(a, x), b) # Run an export. - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() export = exporter.Exporter(tf.train.Saver()) export.init(named_graph_signatures={ "inputs": exporter.generic_signature({"x": x}), From b613102cb614b4fa09a0f390c05053064f9e17e2 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Mon, 1 May 2017 11:26:53 -0800 Subject: [PATCH 0238/8554] Merge changes from github. Change: 154744780 --- tensorflow_serving/example/BUILD | 4 ++-- tensorflow_serving/g3doc/serving_inception.md | 4 ++-- tensorflow_serving/workspace.bzl | 5 +++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 3b34164170c..85bb83412df 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -80,7 +80,7 @@ py_binary( "inception_export.py", ], deps = [ - "@inception//inception", + "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", ], @@ -92,7 +92,7 @@ py_binary( "inception_saved_model.py", ], deps = [ - "@inception//inception", + "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", "@org_tensorflow//tensorflow/python/saved_model:builder", "@org_tensorflow//tensorflow/python/saved_model:constants", diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index b4c9fbce46d..7cf93883f31 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -213,7 +213,7 @@ $ docker tag $USER/inception_serving gcr.io/tensorflow-serving/inception Next we push the image to the Registry, ```shell -$ gcloud docker push gcr.io/tensorflow-serving/inception +$ gcloud docker -- push gcr.io/tensorflow-serving/inception ``` ### Create Kubernetes Deployment and Service @@ -238,7 +238,7 @@ service "inception-service" created To view status of the deployment and pods: ```shell -$ kc get deployments +$ kubectl get deployments NAME DESIRED CURRENT UP-TO-DATE AVAILABLE AGE inception-deployment 3 3 3 3 5s ``` diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index 18cd9469bc0..a465eb7f137 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -7,9 +7,10 @@ load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') # workspace_dir is the absolute path to the TensorFlow Serving repo. If linked # as a submodule, it'll likely be '__workspace_dir__ + "/serving"' def tf_serving_workspace(): - native.local_repository( - name = "inception", + native.new_local_repository( + name = "inception_model", path = "tf_models/inception", + build_file = "tf_models/inception/inception/BUILD", ) tf_workspace(path_prefix = "", tf_repo_name = "org_tensorflow") From 5c65a77b43b7948498fa83c3bcff985799050a21 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 2 May 2017 18:25:22 -0800 Subject: [PATCH 0239/8554] Add copyright header to inception_k8s.yaml. Change: 154913044 --- tensorflow_serving/example/inception_k8s.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow_serving/example/inception_k8s.yaml b/tensorflow_serving/example/inception_k8s.yaml index 302e84e2299..0a5c68009ca 100644 --- a/tensorflow_serving/example/inception_k8s.yaml +++ b/tensorflow_serving/example/inception_k8s.yaml @@ -1,3 +1,18 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + apiVersion: extensions/v1beta1 kind: Deployment metadata: From 84251f42f9945668030c1694e9e78cafbba434fb Mon Sep 17 00:00:00 2001 From: Tom Stall Date: Wed, 3 May 2017 09:43:53 -0700 Subject: [PATCH 0240/8554] Add model_signature_name command line parameter to inception cpp example client (#419) --- tensorflow_serving/example/inception_client.cc | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/example/inception_client.cc b/tensorflow_serving/example/inception_client.cc index 4b1d183ae87..244dcf15f17 100644 --- a/tensorflow_serving/example/inception_client.cc +++ b/tensorflow_serving/example/inception_client.cc @@ -40,12 +40,14 @@ class ServingClient { : stub_(PredictionService::NewStub(channel)) {} tensorflow::string callPredict(const tensorflow::string& model_name, + const tensorflow::string& model_signature_name, const tensorflow::string& file_path) { PredictRequest predictRequest; PredictResponse response; ClientContext context; predictRequest.mutable_model_spec()->set_name(model_name); + predictRequest.mutable_model_spec()->set_signature_name(model_signature_name); google::protobuf::Map& inputs = *predictRequest.mutable_inputs(); @@ -116,11 +118,13 @@ int main(int argc, char** argv) { tensorflow::string server_port = "localhost:9000"; tensorflow::string image_file = ""; tensorflow::string model_name = "inception"; + tensorflow::string model_signature_name = "predict_images"; std::vector flag_list = { tensorflow::Flag("server_port", &server_port, "the IP and port of the server"), - tensorflow::Flag("image_file", &image_file, "the path to the "), - tensorflow::Flag("model_name", &model_name, "name of model")}; + tensorflow::Flag("image_file", &image_file, "the path to the image"), + tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("model_signature_name", &model_signature_name, "name of model signature")}; tensorflow::string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || image_file.empty()) { @@ -132,7 +136,7 @@ int main(int argc, char** argv) { grpc::CreateChannel(server_port, grpc::InsecureChannelCredentials())); std::cout << "calling predict using file: " << image_file << " ..." << std::endl; - std::cout << guide.callPredict(model_name, image_file) << std::endl; + std::cout << guide.callPredict(model_name, model_signature_name, image_file) << std::endl; return 0; } From 76ab71b9aeb71cc879f2a1271131c22f78458dc8 Mon Sep 17 00:00:00 2001 From: tobe Date: Fri, 5 May 2017 02:04:00 +0800 Subject: [PATCH 0241/8554] Add hdfs_file_system in BUILD file (#432) --- tensorflow_serving/model_servers/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index cbeaaf042e3..247480695ae 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -132,6 +132,7 @@ cc_binary( "@protobuf//:cc_wkt_protos", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", + "@org_tensorflow//tensorflow/core/platform/hadoop:hadoop_file_system", "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:availability_preserving_policy", From 40dbe48ac01e77c918531d22ce0da7d4b4d3ddbe Mon Sep 17 00:00:00 2001 From: MtDersvan Date: Mon, 8 May 2017 11:23:10 -0700 Subject: [PATCH 0242/8554] Fixed a typo (#434) * Fixed a typo --- tensorflow_serving/g3doc/serving_basic.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 463d006d279..d1a7e236702 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -172,8 +172,8 @@ saved_model.pb variables Each version sub-directory contains the following files: - * `saved_model.pb` is the serialized tensorflow::SavedModel. It includes the - the one or more graph definitions of the model, as well as metadata of the + * `saved_model.pb` is the serialized tensorflow::SavedModel. It includes + one or more graph definitions of the model, as well as metadata of the model such as signatures. * `variables` are files that hold the serialized variables of the graphs. From 0cac9aeca7b4cce1d0ee19df28cff09bf1df4838 Mon Sep 17 00:00:00 2001 From: Peter Prettenhofer Date: Wed, 10 May 2017 20:01:10 +0200 Subject: [PATCH 0243/8554] Install grpcio from pypi.python.org and require minimum version 1.1.3 (#351) * Pump grpcio version to minimum 1.1.3 and pull from pypi.python.org . Addresses tf_serving issue #122 --- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 4c61f9286bd..8ea5fc08c71 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -30,7 +30,7 @@ RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ RUN pip install enum34 futures mock six && \ pip install --pre 'protobuf>=3.0.0a3' && \ - pip install -i https://testpypi.python.org/simple --pre grpcio + pip install 'grpcio>=1.1.3' # Set up Bazel. From bdccd0f718956e58629154fac737caac5e66de2f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 3 May 2017 07:20:47 -0800 Subject: [PATCH 0244/8554] Add p90 latency measurement to resource types Change: 154957594 --- tensorflow_serving/resources/resource_values.cc | 1 + tensorflow_serving/resources/resource_values.h | 3 +++ 2 files changed, 4 insertions(+) diff --git a/tensorflow_serving/resources/resource_values.cc b/tensorflow_serving/resources/resource_values.cc index 6f3e05e531f..fae14e7d3fa 100644 --- a/tensorflow_serving/resources/resource_values.cc +++ b/tensorflow_serving/resources/resource_values.cc @@ -27,6 +27,7 @@ namespace resource_kinds { const char* const kNumModelSlots = "num_model_slots"; const char* const kRamBytes = "ram_in_bytes"; const char* const kProcessingMillis = "processing_in_millicores"; +const char* const kP90LatencyMicros = "p90_latency_in_microseconds"; } // namespace resource_kinds } // namespace serving diff --git a/tensorflow_serving/resources/resource_values.h b/tensorflow_serving/resources/resource_values.h index 81616087043..23e65df48f5 100644 --- a/tensorflow_serving/resources/resource_values.h +++ b/tensorflow_serving/resources/resource_values.h @@ -48,6 +48,9 @@ extern const char* const kRamBytes; // Fraction of a processing unit's cycles, in thousandths. extern const char* const kProcessingMillis; +// 90th percentile request processing latency, measured in microseconds. +extern const char* const kP90LatencyMicros; + } // namespace resource_kinds } // namespace serving From 18132d25e21f60577083d5dcb6d8368ed0faa516 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 3 May 2017 08:46:54 -0800 Subject: [PATCH 0245/8554] Merge changes from github. Change: 154966933 --- tensorflow_serving/example/inception_export.py | 6 +++--- tensorflow_serving/example/inception_saved_model.py | 6 +++--- tensorflow_serving/example/mnist_export.py | 9 +++++---- tensorflow_serving/example/mnist_saved_model.py | 9 +++++---- .../tensorflow/testdata/export_bad_half_plus_two.py | 2 +- .../tensorflow/testdata/export_half_plus_two.py | 2 +- 6 files changed, 18 insertions(+), 16 deletions(-) diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index 1c5fbdf8157..e807a08c680 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -90,8 +90,8 @@ def export(): class_descriptions.append(texts[s]) class_tensor = tf.constant(class_descriptions) - classes = tf.contrib.lookup.index_to_string(tf.to_int64(indices), - mapping=class_tensor) + table = tf.contrib.lookup.index_to_string_table_from_tensor(class_tensor) + classes = table.lookup(tf.to_int64(indices)) # Restore variables from training checkpoint. variable_averages = tf.train.ExponentialMovingAverage( @@ -114,7 +114,7 @@ def export(): return # Export inference model. - init_op = tf.group(tf.initialize_all_tables(), name='init_op') + init_op = tf.group(tf.tables_initializer(), name='init_op') classification_signature = exporter.classification_signature( input_tensor=serialized_tf_example, classes_tensor=classes, diff --git a/tensorflow_serving/example/inception_saved_model.py b/tensorflow_serving/example/inception_saved_model.py index febf2bd5d19..1654c75cc74 100644 --- a/tensorflow_serving/example/inception_saved_model.py +++ b/tensorflow_serving/example/inception_saved_model.py @@ -94,8 +94,8 @@ def export(): class_descriptions.append(texts[s]) class_tensor = tf.constant(class_descriptions) - classes = tf.contrib.lookup.index_to_string( - tf.to_int64(indices), mapping=class_tensor) + table = tf.contrib.lookup.index_to_string_table_from_tensor(class_tensor) + classes = table.lookup(tf.to_int64(indices)) # Restore variables from training checkpoint. variable_averages = tf.train.ExponentialMovingAverage( @@ -152,7 +152,7 @@ def export(): method_name=signature_constants.PREDICT_METHOD_NAME) legacy_init_op = tf.group( - tf.initialize_all_tables(), name='legacy_init_op') + tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 0c008b3bb42..4df170e5820 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -68,13 +68,14 @@ def main(_): y_ = tf.placeholder('float', shape=[None, 10]) w = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) values, indices = tf.nn.top_k(y, 10) - prediction_classes = tf.contrib.lookup.index_to_string( - tf.to_int64(indices), mapping=tf.constant([str(i) for i in range(10)])) + table = tf.contrib.lookup.index_to_string_table_from_tensor( + tf.constant([str(i) for i in range(10)])) + prediction_classes = table.lookup(tf.to_int64(indices)) for _ in range(FLAGS.training_iteration): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x: batch[0], y_: batch[1]}) @@ -92,7 +93,7 @@ def main(_): # whenever code changes. export_path = sys.argv[-1] print('Exporting trained model to %s' % export_path) - init_op = tf.group(tf.initialize_all_tables(), name='init_op') + init_op = tf.group(tf.tables_initializer(), name='init_op') saver = tf.train.Saver(sharded=True) model_exporter = exporter.Exporter(saver) model_exporter.init( diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py index 35bb2d244d2..19249e46a45 100644 --- a/tensorflow_serving/example/mnist_saved_model.py +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -69,13 +69,14 @@ def main(_): y_ = tf.placeholder('float', shape=[None, 10]) w = tf.Variable(tf.zeros([784, 10])) b = tf.Variable(tf.zeros([10])) - sess.run(tf.initialize_all_variables()) + sess.run(tf.global_variables_initializer()) y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) values, indices = tf.nn.top_k(y, 10) - prediction_classes = tf.contrib.lookup.index_to_string( - tf.to_int64(indices), mapping=tf.constant([str(i) for i in xrange(10)])) + table = tf.contrib.lookup.index_to_string_table_from_tensor( + tf.constant([str(i) for i in xrange(10)])) + prediction_classes = table.lookup(tf.to_int64(indices)) for _ in range(FLAGS.training_iteration): batch = mnist.train.next_batch(50) train_step.run(feed_dict={x: batch[0], y_: batch[1]}) @@ -120,7 +121,7 @@ def main(_): outputs={'scores': tensor_info_y}, method_name=signature_constants.PREDICT_METHOD_NAME) - legacy_init_op = tf.group(tf.initialize_all_tables(), name='legacy_init_op') + legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py b/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py index c44c55f357d..a062e1cc7e3 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py +++ b/tensorflow_serving/servables/tensorflow/testdata/export_bad_half_plus_two.py @@ -42,7 +42,7 @@ def Export(): # Note that the model is intentionally exported without using exporter, # but using the same format. This is to avoid exporter creating default # empty signatures upon export. - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() saver = tf.train.Saver() saver.export_meta_graph( filename=os.path.join(export_path, "export.meta")) diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py index b90453d6404..a199f40bd8a 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py +++ b/tensorflow_serving/servables/tensorflow/testdata/export_half_plus_two.py @@ -41,7 +41,7 @@ def Export(): y = tf.add(tf.multiply(a, x), b) # Run an export. - tf.initialize_all_variables().run() + tf.global_variables_initializer().run() export = exporter.Exporter(tf.train.Saver()) export.init(named_graph_signatures={ "inputs": exporter.generic_signature({"x": x}), From d94f646faab83011bcb6d666bac89fe150170565 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 3 May 2017 19:36:10 -0800 Subject: [PATCH 0246/8554] Add test coverage to ServerCore for a config reload that unloads a model. Change: 155040454 --- .../model_servers/server_core_test.cc | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 8e33f25ada2..7f052800d50 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -79,6 +79,22 @@ TEST_P(ServerCoreTest, ReloadConfigWaitsTillModelsAvailable) { EXPECT_EQ(available_servables.at(0), expected_id); } +TEST_P(ServerCoreTest, ReloadConfigUnloadsModels) { + const ModelServerConfig nonempty_config = + GetTestModelServerConfigForFakePlatform(); + ModelServerConfig empty_config; + empty_config.mutable_model_config_list(); + + std::unique_ptr server_core; + TF_ASSERT_OK(CreateServerCore(nonempty_config, &server_core)); + ASSERT_FALSE(server_core->ListAvailableServableIds().empty()); + + TF_ASSERT_OK(server_core->ReloadConfig(empty_config)); + // Wait for the unload to finish (ReloadConfig() doesn't block on this). + Env::Default()->SleepForMicroseconds(1 * 1000 * 1000); + EXPECT_TRUE(server_core->ListAvailableServableIds().empty()); +} + TEST_P(ServerCoreTest, ErroringModel) { ServerCore::Options options = GetDefaultOptions(); test_util::StoragePathErrorInjectingSourceAdapterConfig source_adapter_config; From e16555546c595971915557f992485b8d8210239d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 4 May 2017 12:01:49 -0800 Subject: [PATCH 0247/8554] Create a Dockerfile that can be used to build a CUDA enabled version of Tensorflow Serving. It is more involved than the Dockerfile.devel because it builds Tensorflow and TFServing in the image itself. Change: 155115422 --- .../tools/docker/Dockerfile.devel-gpu | 101 ++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 tensorflow_serving/tools/docker/Dockerfile.devel-gpu diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu new file mode 100644 index 00000000000..4c80705492c --- /dev/null +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -0,0 +1,101 @@ +FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 + +RUN apt-get update && apt-get install -y \ + build-essential \ + curl \ + git \ + libfreetype6-dev \ + libpng12-dev \ + libzmq3-dev \ + pkg-config \ + python-dev \ + python-numpy \ + python-pip \ + software-properties-common \ + swig \ + zip \ + zlib1g-dev \ + libcurl3-dev \ + && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ + python get-pip.py && \ + rm get-pip.py + +# Set up grpc + +RUN pip install enum34 futures mock six && \ + pip install --pre 'protobuf>=3.0.0a3' && \ + pip install -i https://testpypi.python.org/simple --pre grpcio + +# Set up Bazel. + +# We need to add a custom PPA to pick up JDK8, since trusty doesn't +# have an openjdk8 backport. openjdk-r is maintained by a reliable contributor: +# Matthias Klose (https://launchpad.net/~doko). It will do until +# we either update the base image beyond 14.04 or openjdk-8 is +# finally backported to trusty; see e.g. +# https://bugs.launchpad.net/trusty-backports/+bug/1368094 +RUN add-apt-repository -y ppa:openjdk-r/ppa && \ + apt-get update && \ + apt-get install -y openjdk-8-jdk openjdk-8-jre-headless && \ + apt-get clean && \ + rm -rf /var/lib/apt/lists/* + +# Running bazel inside a `docker build` command causes trouble, cf: +# https://github.com/bazelbuild/bazel/issues/134 +# The easiest solution is to set up a bazelrc file forcing --batch. +RUN echo "startup --batch" >>/root/.bazelrc +# Similarly, we need to workaround sandboxing issues: +# https://github.com/bazelbuild/bazel/issues/418 +RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ + >>/root/.bazelrc +ENV BAZELRC /root/.bazelrc +# Install the most recent bazel release. +ENV BAZEL_VERSION 0.4.5 +WORKDIR / +RUN mkdir /bazel && \ + cd /bazel && \ + curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + curl -fSsL -o /bazel/LICENSE https://raw.githubusercontent.com/bazelbuild/bazel/master/LICENSE && \ + chmod +x bazel-*.sh && \ + ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ + cd / && \ + rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh + +# Download TensorFlow Serving +RUN git clone --recurse-submodules https://github.com/tensorflow/serving && \ + cd serving && \ + git checkout + +# Build TensorFlow with the CUDA configuration +ENV CI_BUILD_PYTHON python +ENV LD_LIBRARY_PATH /usr/local/cuda/extras/CUPTI/lib64:$LD_LIBRARY_PATH +ENV TF_NEED_CUDA 1 +ENV TF_CUDA_COMPUTE_CAPABILITIES=3.0,3.5,5.2,6.0,6.1 + +# Fix paths so that CUDNN can be found +# See https://github.com/tensorflow/tensorflow/issues/8264 +RUN ls -lah /usr/local/cuda/lib64/* +RUN mkdir /usr/lib/x86_64-linux-gnu/include/ && \ + ln -s /usr/lib/x86_64-linux-gnu/include/cudnn.h /usr/lib/x86_64-linux-gnu/include/cudnn.h && \ + ln -s /usr/include/cudnn.h /usr/local/cuda/include/cudnn.h && \ + ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/local/cuda/lib64/libcudnn.so && \ + ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/local/cuda/lib64/libcudnn.so.5 + + +# Configure Tensorflow to use the GPU +WORKDIR /serving/tensorflow +RUN tensorflow/tools/ci_build/builds/configured GPU + +# Build TensorFlow Serving and Install it in /usr/local/bin +WORKDIR /serving +RUN bazel build -c opt --config=cuda \ + --crosstool_top=@local_config_cuda//crosstool:toolchain \ + tensorflow_serving/model_servers:tensorflow_model_server && \ + cp bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server /usr/local/bin/ && \ + bazel clean --expunge + +CMD ["/bin/bash"] \ No newline at end of file From bd22e67a596642fd7d22d1fe8897be18b5f4d200 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 8 May 2017 17:26:31 -0800 Subject: [PATCH 0248/8554] Fix bug in which ServerCore gets confused when asked to re-load a model that has previously been unloaded. Change: 155456053 --- .../model_servers/server_core.cc | 52 +++++++++++++++---- .../model_servers/server_core.h | 8 +-- .../model_servers/server_core_test.cc | 24 +++++++++ .../file_system_storage_path_source.h | 5 ++ .../util/fast_read_dynamic_ptr.h | 1 + 5 files changed, 77 insertions(+), 13 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index d9972171948..8eb49da68a3 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -130,6 +130,27 @@ Status UnionRoutes(const DynamicSourceRouter::Routes& a, return Status::OK(); } +// Finds all models that occur in 'new_config' but not in 'old_config'. +std::set NewModelNamesInSourceConfig( + const FileSystemStoragePathSourceConfig& old_config, + const FileSystemStoragePathSourceConfig& new_config) { + std::set old_models; + for (const FileSystemStoragePathSourceConfig::ServableToMonitor& servable : + old_config.servables()) { + const string& model_name = servable.servable_name(); + old_models.insert(model_name); + } + std::set new_models; + for (const FileSystemStoragePathSourceConfig::ServableToMonitor& servable : + new_config.servables()) { + const string& model_name = servable.servable_name(); + if (old_models.find(model_name) == old_models.end()) { + new_models.insert(model_name); + } + } + return new_models; +} + } // namespace // ************************************************************************ @@ -198,16 +219,16 @@ Status ServerCore::Initialize(std::unique_ptr policy) { return Status::OK(); } -Status ServerCore::WaitUntilConfiguredModelsAvailable() { - std::vector awaited_models; - for (const auto& model : config_.model_config_list().config()) { - awaited_models.push_back(ServableRequest::Latest(model.name())); +Status ServerCore::WaitUntilModelsAvailable(const std::set& models, + ServableStateMonitor* monitor) { + std::vector awaited_servables; + for (const string& model : models) { + awaited_servables.push_back(ServableRequest::Latest(model)); } std::map states_reached; - const bool all_models_available = - servable_state_monitor_->WaitUntilServablesReachState( - awaited_models, ServableState::ManagerState::kAvailable, - &states_reached); + const bool all_models_available = monitor->WaitUntilServablesReachState( + awaited_servables, ServableState::ManagerState::kAvailable, + &states_reached); if (!all_models_available) { string message = "Some models did not become available: {"; for (const auto& id_and_state : states_reached) { @@ -256,6 +277,18 @@ Status ServerCore::AddModelsViaModelConfigList() { } manager_.AddDependency(std::move(adapters.error_adapter)); } else { + // Create a fresh servable state monitor, to avoid getting confused if we're + // re-loading a model-version that has previously been unloaded. + ServableStateMonitor fresh_servable_state_monitor( + servable_event_bus_.get()); + + // Figure out which models are new. + const std::set new_models = NewModelNamesInSourceConfig( + storage_path_source_and_router_->source->config(), source_config); + + // Now we're ready to start reconfiguring the elements of the Source-> + // Manager pipeline ... + // First, add the new routes without removing the old ones. DynamicSourceRouter::Routes old_and_new_routes; const Status union_status = @@ -276,7 +309,8 @@ Status ServerCore::AddModelsViaModelConfigList() { TF_RETURN_IF_ERROR(ReloadRoutes(routes)); // Wait for any new models to get loaded and become available. - TF_RETURN_IF_ERROR(WaitUntilConfiguredModelsAvailable()); + TF_RETURN_IF_ERROR( + WaitUntilModelsAvailable(new_models, &fresh_servable_state_monitor)); } return Status::OK(); } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index a9e82ef20f0..3ba2ca717d2 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -232,10 +232,10 @@ class ServerCore : public Manager { const ModelServerConfig& config, DynamicSourceRouter::Routes* routes) const; - // Waits for all models from the ModelConfigList in 'config_' to be loaded. - // Returns an error if any configured model fails to load. - Status WaitUntilConfiguredModelsAvailable() - EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + // Waits until all entries in 'models' have been loaded, according to + // 'monitor'. Returns an error if any model fails to load. + Status WaitUntilModelsAvailable(const std::set& models, + ServableStateMonitor* monitor); // Creates a FileSystemStoragePathSource and connects it to the supplied // target. diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 7f052800d50..f740d168317 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -95,6 +95,30 @@ TEST_P(ServerCoreTest, ReloadConfigUnloadsModels) { EXPECT_TRUE(server_core->ListAvailableServableIds().empty()); } +TEST_P(ServerCoreTest, ReloadConfigHandlesLoadingAPreviouslyUnloadedModel) { + ModelServerConfig empty_config; + empty_config.mutable_model_config_list(); + const ModelServerConfig nonempty_config = + GetTestModelServerConfigForFakePlatform(); + + // Load, and then unload, a servable. + std::unique_ptr server_core; + TF_ASSERT_OK(CreateServerCore(nonempty_config, &server_core)); + TF_ASSERT_OK(server_core->ReloadConfig(empty_config)); + // Wait for the unload to finish (ReloadConfig() doesn't block on this). + Env::Default()->SleepForMicroseconds(1 * 1000 * 1000); + ASSERT_TRUE(server_core->ListAvailableServableIds().empty()); + + // Re-load the same servable. + TF_ASSERT_OK(server_core->ReloadConfig(nonempty_config)); + const std::vector available_servables = + server_core->ListAvailableServableIds(); + ASSERT_EQ(available_servables.size(), 1); + const ServableId expected_id = {test_util::kTestModelName, + test_util::kTestModelVersion}; + EXPECT_EQ(available_servables.at(0), expected_id); +} + TEST_P(ServerCoreTest, ErroringModel) { ServerCore::Options options = GetDefaultOptions(); test_util::StoragePathErrorInjectingSourceAdapterConfig source_adapter_config; diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h index 672c954e99f..90891471b8d 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h @@ -69,6 +69,11 @@ class FileSystemStoragePathSource : public Source { void SetAspiredVersionsCallback(AspiredVersionsCallback callback) override; + FileSystemStoragePathSourceConfig config() const { + mutex_lock l(mu_); + return config_; + } + private: friend class internal::FileSystemStoragePathSourceTestAccess; diff --git a/tensorflow_serving/util/fast_read_dynamic_ptr.h b/tensorflow_serving/util/fast_read_dynamic_ptr.h index 1afc30f8eb5..cd4d62aff89 100644 --- a/tensorflow_serving/util/fast_read_dynamic_ptr.h +++ b/tensorflow_serving/util/fast_read_dynamic_ptr.h @@ -192,6 +192,7 @@ std::unique_ptr FastReadDynamicPtr::Update(std::unique_ptr object) { // Swap the new ReleasableSharedPtr under lock. { mutex_lock lock(mutex_); + using std::swap; swap(object_, local_ptr); } From 2597532669900e881fa725f97f5c620fd228f3cb Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 8 May 2017 19:47:34 -0800 Subject: [PATCH 0249/8554] Clarify ModelServer config reloading semantics. Change: 155463849 --- tensorflow_serving/config/model_server_config.proto | 10 ++++++++++ tensorflow_serving/model_servers/server_core.h | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index 5a057248853..cc172297ddc 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -23,6 +23,10 @@ message ModelConfig { // Base path to the model, excluding the version directory. // E.g> for a model at /foo/bar/my_model/123, where 123 is the version, the // base path is /foo/bar/my_model. + // + // (This can be changed once a model is in serving, *if* the underlying data + // remains the same. Otherwise there are no guarantees about whether the old + // or new data will be used for model versions currently loaded.) string base_path = 2; // Type of model. @@ -30,14 +34,20 @@ message ModelConfig { ModelType model_type = 3 [deprecated = true]; // Type of model (e.g. "tensorflow"). + // + // (This cannot be changed once a model is in serving.) string model_platform = 4; // Version policy for the model indicating how many versions of the model to // be served at the same time. // The default option is to serve only the latest version of the model. + // + // (This can be changed once a model is in serving.) FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5; // Configures logging requests and responses, to the model. + // + // (This can be changed once a model is in serving.) LoggingConfig logging_config = 6; } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 3ba2ca717d2..dfbc07ae1a1 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -147,7 +147,8 @@ class ServerCore : public Manager { // Updates the server core with all the models and sources per the // ModelServerConfig. Like Create(), waits for all statically configured // servables to be made available before returning, and returns an error if - // any such model fails to load. + // any such model fails to load. (Does not necessarily wait for models removed + // from the config to finish unloading; that may occur asynchronously.) // // IMPORTANT: It is only legal to call this method more than once if using // ModelConfigList (versus custom model config). From 028b9319c7236381efc36bbb217fa689a3552515 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 8 May 2017 22:48:29 -0800 Subject: [PATCH 0250/8554] Eliminate heuristical sleeps from server_core_test. Change: 155472848 --- tensorflow_serving/model_servers/server_core_test.cc | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index f740d168317..b3820133b69 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -91,8 +91,9 @@ TEST_P(ServerCoreTest, ReloadConfigUnloadsModels) { TF_ASSERT_OK(server_core->ReloadConfig(empty_config)); // Wait for the unload to finish (ReloadConfig() doesn't block on this). - Env::Default()->SleepForMicroseconds(1 * 1000 * 1000); - EXPECT_TRUE(server_core->ListAvailableServableIds().empty()); + while (!server_core->ListAvailableServableIds().empty()) { + Env::Default()->SleepForMicroseconds(10 * 1000); + } } TEST_P(ServerCoreTest, ReloadConfigHandlesLoadingAPreviouslyUnloadedModel) { @@ -106,8 +107,9 @@ TEST_P(ServerCoreTest, ReloadConfigHandlesLoadingAPreviouslyUnloadedModel) { TF_ASSERT_OK(CreateServerCore(nonempty_config, &server_core)); TF_ASSERT_OK(server_core->ReloadConfig(empty_config)); // Wait for the unload to finish (ReloadConfig() doesn't block on this). - Env::Default()->SleepForMicroseconds(1 * 1000 * 1000); - ASSERT_TRUE(server_core->ListAvailableServableIds().empty()); + while (!server_core->ListAvailableServableIds().empty()) { + Env::Default()->SleepForMicroseconds(10 * 1000); + } // Re-load the same servable. TF_ASSERT_OK(server_core->ReloadConfig(nonempty_config)); From 1707ea239129339b1997b6b0dbd111f98ea82543 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 10 May 2017 11:55:08 -0800 Subject: [PATCH 0251/8554] Remove training '\' from a kubernetes yaml file that was causing issues. Change: 155664568 --- tensorflow_serving/example/inception_k8s.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/example/inception_k8s.yaml b/tensorflow_serving/example/inception_k8s.yaml index 0a5c68009ca..0471d83fbb5 100644 --- a/tensorflow_serving/example/inception_k8s.yaml +++ b/tensorflow_serving/example/inception_k8s.yaml @@ -31,7 +31,7 @@ spec: - /bin/sh - -c args: - - /serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server + - serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=inception --model_base_path=/serving/inception-export ports: - containerPort: 9000 From cbc0abe0b02ef99f0458ad0348f01bed39042d7a Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 10 May 2017 14:23:04 -0800 Subject: [PATCH 0252/8554] Missed one more instance of '/' in the yaml file. Change: 155683267 --- tensorflow_serving/example/inception_k8s.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/example/inception_k8s.yaml b/tensorflow_serving/example/inception_k8s.yaml index 0471d83fbb5..e23c3a6036e 100644 --- a/tensorflow_serving/example/inception_k8s.yaml +++ b/tensorflow_serving/example/inception_k8s.yaml @@ -32,7 +32,7 @@ spec: - -c args: - serving/bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server - --port=9000 --model_name=inception --model_base_path=/serving/inception-export + --port=9000 --model_name=inception --model_base_path=serving/inception-export ports: - containerPort: 9000 --- From 09cd2a59b1da74b0efadf74d91e150110896d3a4 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 11 May 2017 10:05:59 -0800 Subject: [PATCH 0253/8554] Improve predict error messages when input/output aliases are not found in the signature. Change: 155770116 --- .../servables/tensorflow/predict_impl.cc | 23 +++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index f46a2998e4f..d2afe49c91b 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -129,6 +129,21 @@ Status SessionBundlePredict(const RunOptions& run_options, ServerCore* core, return Status::OK(); } +// Returns the keys in the map as a comma delimited string. Useful for debugging +// or when returning error messages. +// e.g. returns "key1, key2, key3". +string MapKeysToString(const google::protobuf::Map& map) { + string result = ""; + for (const auto& i : map) { + if (result.empty()) { + result += i.first; + } else { + result += ", " + i.first; + } + } + return result; +} + // Validate a SignatureDef to make sure it's compatible with prediction, and // if so, populate the input and output tensor names. Status PreProcessPrediction(const SignatureDef& signature, @@ -164,7 +179,9 @@ Status PreProcessPrediction(const SignatureDef& signature, if (iter == signature.inputs().end()) { return tensorflow::Status( tensorflow::error::INVALID_ARGUMENT, - "input tensor alias not found in signature: " + alias); + strings::StrCat("input tensor alias not found in signature: ", alias, + ". Inputs expected to be in the set {", + MapKeysToString(signature.inputs()), "}.")); } Tensor tensor; if (!tensor.FromProto(input.second)) { @@ -183,7 +200,9 @@ Status PreProcessPrediction(const SignatureDef& signature, if (iter == signature.outputs().end()) { return tensorflow::Status( tensorflow::error::INVALID_ARGUMENT, - "output tensor alias not found in signature: " + alias); + strings::StrCat("output tensor alias not found in signature: ", alias, + " Outputs expected to be in the set {", + MapKeysToString(signature.outputs()), "}.")); } if (seen_outputs.find(alias) != seen_outputs.end()) { return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, From 6419c8178f292e1d9da09a4134bb6e3fe306d1a5 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 11 May 2017 12:07:30 -0800 Subject: [PATCH 0254/8554] Beef up unit test coverage of AspiredVersionManager for reverting to a smaller version number. Change: 155786228 --- .../core/aspired_versions_manager_test.cc | 86 ++++++++++++++++--- 1 file changed, 73 insertions(+), 13 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index 814e82fd9e8..edb36868119 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -44,6 +44,7 @@ using ::testing::Invoke; using ::testing::InvokeWithoutArgs; using ::testing::NiceMock; using ::testing::Return; +using ::testing::UnorderedElementsAre; using ::testing::UnorderedElementsAreArray; using test_util::FakeLoader; using test_util::WaitUntilServableManagerStateIsOneOf; @@ -393,14 +394,6 @@ TEST_P(AspiredVersionsManagerTest, AspiredRemovedFull) { } TEST_P(AspiredVersionsManagerTest, AspiredRemovedPartial) { - { - ServableHandle handle; - const Status status = manager_->GetServableHandle( - ServableRequest::Specific(kServableName, 1), &handle); - TF_ASSERT_OK(status); - EXPECT_EQ(1, *handle); - } - std::vector>> aspired_versions; aspired_versions.push_back(CreateAspiredVersion({kServableName, 0})); manager_->GetAspiredVersionsCallback()(kServableName, @@ -412,11 +405,78 @@ TEST_P(AspiredVersionsManagerTest, AspiredRemovedPartial) { {kServableName, 1}, {ServableState::ManagerState::kEnd}); - ServableHandle missing_handle; - const Status missing_status = manager_->GetServableHandle( - ServableRequest::Specific(kServableName, 1), &missing_handle); - ASSERT_FALSE(missing_status.ok()); - EXPECT_EQ(error::NOT_FOUND, missing_status.code()); + // Version 0 should remain available in the manager. + ServableHandle v0_handle; + const Status v0_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 0), &v0_handle); + TF_ASSERT_OK(v0_status); + EXPECT_EQ(0, *v0_handle); + + // Version 1 should no longer be available. + ServableHandle v1_handle; + const Status v1_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 1), &v1_handle); + ASSERT_FALSE(v1_status.ok()); + EXPECT_EQ(error::NOT_FOUND, v1_status.code()); +} + +TEST_P(AspiredVersionsManagerTest, RevertToSmallerVersionNumber) { + // Initially, versions 0 and 1 of kServableName are loaded. + std::set initial_versions; + for (const ServableId& id : manager_->ListAvailableServableIds()) { + if (id.name == kServableName) { + initial_versions.insert(id.version); + } + } + ASSERT_THAT(initial_versions, UnorderedElementsAre(0, 1)); + + // Unload version 0, s.t. only version 1 is loaded. + std::vector>> initial_aspired_versions; + initial_aspired_versions.push_back(CreateAspiredVersion({kServableName, 1})); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(initial_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + InvokePolicyAndExecuteAction(); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 0}, + {ServableState::ManagerState::kEnd}); + FlushServables(); + + // Now, switch to version 0 (dropping version 1). + std::vector>> new_aspired_versions; + new_aspired_versions.push_back(CreateAspiredVersion({kServableName, 0})); + manager_->GetAspiredVersionsCallback()(kServableName, + std::move(new_aspired_versions)); + HandlePendingAspiredVersionsRequests(); + Notification done_transitioning; + std::unique_ptr transition_servables( + Env::Default()->StartThread({}, "TransitionServables", [&]() { + while (!done_transitioning.HasBeenNotified()) { + InvokePolicyAndExecuteAction(); + Env::Default()->SleepForMicroseconds(1000 /* 1 ms */); + } + })); + WaitUntilServableManagerStateIsOneOf( + servable_state_monitor_, {kServableName, 0}, + {ServableState::ManagerState::kAvailable}); + WaitUntilServableManagerStateIsOneOf(servable_state_monitor_, + {kServableName, 1}, + {ServableState::ManagerState::kEnd}); + done_transitioning.Notify(); + + // Version 0 should be available. + ServableHandle v0_handle; + const Status v0_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 0), &v0_handle); + TF_ASSERT_OK(v0_status); + EXPECT_EQ(0, *v0_handle); + + // Version 1 should not be available. + ServableHandle v1_handle; + const Status v1_status = manager_->GetServableHandle( + ServableRequest::Specific(kServableName, 1), &v1_handle); + ASSERT_FALSE(v1_status.ok()); + EXPECT_EQ(error::NOT_FOUND, v1_status.code()); } TEST_P(AspiredVersionsManagerTest, AspiredAndManageStateLoad) { From be208d59e4bcfe4566ed64fc80211aa533f9929f Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 15 May 2017 12:08:19 -0800 Subject: [PATCH 0255/8554] Add a test for a config reload that changes a model's base path. Change: 156093213 --- .../model_servers/server_core_test.cc | 40 ++++++++++++++++++ .../model_servers/test_util/BUILD | 6 +++ .../test_util/server_core_test_util.cc | 8 ++++ .../test_util/server_core_test_util.h | 5 +++ .../00000123/export.data-00000-of-00001 | Bin 0 -> 8 bytes .../00000123/export.index | Bin 0 -> 142 bytes .../00000123/export.meta | Bin 0 -> 4266 bytes .../00000124/export.data-00000-of-00001 | Bin 0 -> 8 bytes .../00000124/export.index | Bin 0 -> 142 bytes .../00000124/export.meta | Bin 0 -> 4266 bytes 10 files changed, 59 insertions(+) create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000123/export.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000123/export.index create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000123/export.meta create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000124/export.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000124/export.index create mode 100644 tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000124/export.meta diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index b3820133b69..a083ebb1702 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -121,6 +121,46 @@ TEST_P(ServerCoreTest, ReloadConfigHandlesLoadingAPreviouslyUnloadedModel) { EXPECT_EQ(available_servables.at(0), expected_id); } +TEST_P(ServerCoreTest, ReloadConfigChangeModelBasePath) { + // Create two configs that differ only in the model's base path. One base path + // has a single version test_util::kTestModelVersion, and one has two versions + // test_util::kTestModelVersion and test_util::kTestModelLargerVersion. + const ModelServerConfig one_version_config = + GetTestModelServerConfigForFakePlatform(); + ModelServerConfig two_version_config = + GetTestModelServerConfigForFakePlatform(); + SwitchToHalfPlusTwoWith2Versions(&two_version_config); + + // Start with the one-version path. + std::unique_ptr server_core; + TF_ASSERT_OK(CreateServerCore(one_version_config, &server_core)); + std::vector available_servables = + server_core->ListAvailableServableIds(); + ASSERT_EQ(1, available_servables.size()); + EXPECT_EQ(test_util::kTestModelVersion, available_servables.at(0).version); + + // Switch to the two-version path. + TF_ASSERT_OK(server_core->ReloadConfig(two_version_config)); + // Wait for the new base path set-up to propagate through the Source and + // Manager to (ReloadConfig() doesn't block on this). + do { + Env::Default()->SleepForMicroseconds(10 * 1000); + available_servables = server_core->ListAvailableServableIds(); + } while (available_servables.empty() || + available_servables.at(0).version != + test_util::kTestModelLargerVersion); + + // Revert to the one-version path. + TF_ASSERT_OK(server_core->ReloadConfig(one_version_config)); + // Wait for the new base path set-up to propagate through the Source and + // Manager to (ReloadConfig() doesn't block on this). + do { + Env::Default()->SleepForMicroseconds(10 * 1000); + available_servables = server_core->ListAvailableServableIds(); + } while (available_servables.empty() || + available_servables.at(0).version != test_util::kTestModelVersion); +} + TEST_P(ServerCoreTest, ErroringModel) { ServerCore::Options options = GetDefaultOptions(); test_util::StoragePathErrorInjectingSourceAdapterConfig source_adapter_config; diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 64114270c99..71921c47bfe 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -51,6 +51,12 @@ cc_library( "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two_2_versions/00000123/export.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two_2_versions/00000123/export.index", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two_2_versions/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two_2_versions/00000124/export.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two_2_versions/00000124/export.index", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two_2_versions/00000124/export.meta", "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], visibility = [ diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index ced73ed791e..1805bf8677c 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -100,6 +100,14 @@ ServerCoreTest::GetTestModelServerConfigForTensorflowPlatform() { return config; } +void ServerCoreTest::SwitchToHalfPlusTwoWith2Versions( + ModelServerConfig* config) { + CHECK_EQ(1, config->model_config_list().config().size()); + auto model = config->mutable_model_config_list()->mutable_config(0); + model->set_base_path(test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/half_plus_two_2_versions")); +} + ServerCore::Options ServerCoreTest::GetDefaultOptions() { // Model platforms. const TestType test_type = GetTestType(); diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index 86557b316c6..f8ffe135863 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -26,6 +26,7 @@ namespace test_util { constexpr char kTestModelName[] = "test_model"; constexpr int kTestModelVersion = 123; +constexpr int kTestModelLargerVersion = 124; // The name of the platform associated with FakeLoaderSourceAdapter. constexpr char kFakePlatform[] = "fake_servable"; @@ -54,6 +55,10 @@ class ServerCoreTest : public ::testing::TestWithParam { // platform. ModelServerConfig GetTestModelServerConfigForTensorflowPlatform(); + // Mutates 'config' by changing the model's base path to point to a variant + // of half-plus-two that has two versions instead of one. + void SwitchToHalfPlusTwoWith2Versions(ModelServerConfig* config); + // Creates some reasonable default ServerCore options for tests. ServerCore::Options GetDefaultOptions(); diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000123/export.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000123/export.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..20bc7d454dd8450489984bd17d92c45c3a1a96a6 GIT binary patch literal 8 PcmZQzV6bOkU~m8c0fPX5 literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000123/export.index b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000123/export.index new file mode 100644 index 0000000000000000000000000000000000000000..35e2ef7527905b228ebfd7f755bb3d06f6cc00e0 GIT binary patch literal 142 zcmZQzVB=tvV&Y(A;NT8REXqtw%1Py56k^a|F=ew*m*-&OjW-1G6&JCcXIiQeD<{{v=Y01&-#O==z%LU0rve{A>2N%z z5|DF+d>K41@X7G`Ej|?nxLEctrpB-h%|U&Y6w6YjRI5@M&Oti#Jab~xI+R@F2!H=0 zArzo;rdu;3FoR#5nLnNRTJm10B_`2&3x8^kb>GmuV|{AS@-xS=SaF9a8&IElhGsdV z=cYYLU;IW0+=a?L$M$^I28r8OD&qj8(vF-{$v14zag`?UH~bLLhwA;YVf&_k5dbx9 z#1;UpCx-CQDMI)dN}tcH>(R3k^79gLN>F~_d@+SXs6Vvyk#X!;W5ZoUodu~-sBkv; zgB6Dbu%PzH@O;NLo(vGVG(|$q^R7%g(BMbaZ2i>maAgfT;dV{8$C_uEqhOV-fXsjA z4Wy7OPw7JRiGpQ%{!YN)ogK1Azy#^Be)b<-(QCQ-2C7eV*VLa`1`-qMh(`>yq_nb3 z%taG5QX4t8ubZ~vQpxjOR0=E7f^rM$NP%mNLsG<7KNHfc?e+Hu{bNHP59FEs+;(2r z^wkD@1?w@AUDGywQ@6BG$&{Gn`vXFBXvT}XE{1|8iJ4-|^?EfIUqd%`q3vHI zqzAcvGc;QOsrfe^)V~O4rc6xl^Lif?eU{ZHyQ1wx;P-fE4caRU?Ik%G zQL+SJVq^g2pcK)tkFd_uSOHP|8BJEuDAxo$1n{tU?}jZhu3hVK?P{v^s!R)N<YN2;!M?Vp)r~zgCfHZ1;OyH5_GQgICOp6&4Ip`#<7d1I3 z;$7i;*a#s64b62dZEQ>p8?P#(3!yTKfErH)$hmR6ERn)}p*rG>mGOSD87?Q6j7y`u z9aFug(uiu3MtKS0Va|ou4lMpM9K-E7Z+XGZ^jxkhvNW`Fo?a4YWnIzqB9#g7vh%ao z!CjKect_>;En0!n03-ctthTuto@5J~LwxhPeYJi$h3za=SW_;`_1m!u*44E{1){q1 z6K^g$C-Clq$2X+}q`fomEKdu|d0?*xeH}f#BG3$haXo1JG`_3}oU#S%H&?}FwIOaW zVmJRVBF^Olh^AXs)R3YY#DTLGuSoGR;In86~KR6q)caF2G`I;`Q}5Om(szvJ$vMgRZ+ literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000124/export.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000124/export.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..20bc7d454dd8450489984bd17d92c45c3a1a96a6 GIT binary patch literal 8 PcmZQzV6bOkU~m8c0fPX5 literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000124/export.index b/tensorflow_serving/servables/tensorflow/testdata/half_plus_two_2_versions/00000124/export.index new file mode 100644 index 0000000000000000000000000000000000000000..35e2ef7527905b228ebfd7f755bb3d06f6cc00e0 GIT binary patch literal 142 zcmZQzVB=tvV&Y(A;NT8REXqtw%1Py56k^a|F=ew*m*-&OjW-1G6&JCcXIiQeD<{{v=Y01&-#O==z%LU0rve{A>2N%z z5|DF+d>K41@X7G`Ej|?nxLEctrpB-h%|U&Y6w6YjRI5@M&Oti#Jab~xI+R@F2!H=0 zArzo;rdu;3FoR#5nLnNRTJm10B_`2&3x8^kb>GmuV|{AS@-xS=SaF9a8&IElhGsdV z=cYYLU;IW0+=a?L$M$^I28r8OD&qj8(vF-{$v14zag`?UH~bLLhwA;YVf&_k5dbx9 z#1;UpCx-CQDMI)dN}tcH>(R3k^79gLN>F~_d@+SXs6Vvyk#X!;W5ZoUodu~-sBkv; zgB6Dbu%PzH@O;NLo(vGVG(|$q^R7%g(BMbaZ2i>maAgfT;dV{8$C_uEqhOV-fXsjA z4Wy7OPw7JRiGpQ%{!YN)ogK1Azy#^Be)b<-(QCQ-2C7eV*VLa`1`-qMh(`>yq_nb3 z%taG5QX4t8ubZ~vQpxjOR0=E7f^rM$NP%mNLsG<7KNHfc?e+Hu{bNHP59FEs+;(2r z^wkD@1?w@AUDGywQ@6BG$&{Gn`vXFBXvT}XE{1|8iJ4-|^?EfIUqd%`q3vHI zqzAcvGc;QOsrfe^)V~O4rc6xl^Lif?eU{ZHyQ1wx;P-fE4caRU?Ik%G zQL+SJVq^g2pcK)tkFd_uSOHP|8BJEuDAxo$1n{tU?}jZhu3hVK?P{v^s!R)N<YN2;!M?Vp)r~zgCfHZ1;OyH5_GQgICOp6&4Ip`#<7d1I3 z;$7i;*a#s64b62dZEQ>p8?P#(3!yTKfErH)$hmR6ERn)}p*rG>mGOSD87?Q6j7y`u z9aFug(uiu3MtKS0Va|ou4lMpM9K-E7Z+XGZ^jxkhvNW`Fo?a4YWnIzqB9#g7vh%ao z!CjKect_>;En0!n03-ctthTuto@5J~LwxhPeYJi$h3za=SW_;`_1m!u*44E{1){q1 z6K^g$C-Clq$2X+}q`fomEKdu|d0?*xeH}f#BG3$haXo1JG`_3}oU#S%H&?}FwIOaW zVmJRVBF^Olh^AXs)R3YY#DTLGuSoGR;In86~KR6q)caF2G`I;`Q}5Om(szvJ$vMgRZ+ literal 0 HcmV?d00001 From d1eef334fbc23a375a07511df62172c4f957096a Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 17 May 2017 10:16:54 -0800 Subject: [PATCH 0256/8554] Use new mirror.bazel.build hostname Change: 156329689 --- WORKSPACE | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 071d867b4d8..7a510f9440c 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -9,11 +9,11 @@ local_repository( # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. http_archive( name = "io_bazel_rules_closure", - sha256 = "60fc6977908f999b23ca65698c2bb70213403824a84f7904310b6000d78be9ce", - strip_prefix = "rules_closure-5ca1dab6df9ad02050f7ba4e816407f88690cf7d", + sha256 = "4be8a887f6f38f883236e77bb25c2da10d506f2bf1a8e5d785c0f35574c74ca4", + strip_prefix = "rules_closure-aac19edc557aec9b603cd7ffe359401264ceff0d", urls = [ - "http://bazel-mirror.storage.googleapis.com/github.com/bazelbuild/rules_closure/archive/5ca1dab6df9ad02050f7ba4e816407f88690cf7d.tar.gz", # 2017-02-03 - "https://github.com/bazelbuild/rules_closure/archive/5ca1dab6df9ad02050f7ba4e816407f88690cf7d.tar.gz", + "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz", # 2017-05-10 + "https://github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz", ], ) From fb96a78ebcf5bad9772cc77beee92515f80beff7 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 17 May 2017 10:55:46 -0800 Subject: [PATCH 0257/8554] Add PYTHON_BIN_PATH declaration in bazel.rc. Change: 156334983 --- tools/bazel.rc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/bazel.rc b/tools/bazel.rc index f342de3aef5..9397f97379e 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -4,6 +4,8 @@ build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true build --force_python=py2 build --python2_path=/usr/bin/python +build --action_env PYTHON_BIN_PATH="/usr/bin/python" + build --define PYTHON_BIN_PATH=/usr/bin/python test --define PYTHON_BIN_PATH=/usr/bin/python run --define PYTHON_BIN_PATH=/usr/bin/python From f675ef5b260e3e5c4c230c8d4ae2ba00ad1bc0c8 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 17 May 2017 12:09:40 -0700 Subject: [PATCH 0258/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index b6bafadd51a..c03d5cc6642 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit b6bafadd51a471daa7157f515598e08e8f9f1611 +Subproject commit c03d5cc66421b97770ee2b63595c806a90173ef4 diff --git a/tf_models b/tf_models index 39c59d13703..71bf3d47fc8 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 39c59d137035c1b53b756b280510f1705c53fa4e +Subproject commit 71bf3d47fc8b064ec6b677f121d9b044e0f75c7a From d787dbe7575c027650c91b2f0a22aaa2c7609dfe Mon Sep 17 00:00:00 2001 From: Noah Fiedel Date: Tue, 23 May 2017 12:36:39 -0800 Subject: [PATCH 0259/8554] no-op Change: 156903927 --- .../model_servers/tensorflow_model_server_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 313ec74358e..8e9f0e1df59 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -44,9 +44,9 @@ def PickUnusedPort(): - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) s.bind(('localhost', 0)) - _, port = s.getsockname() + port = s.getsockname()[1] s.close() return port From 92aa67923513a02e20499081397648c857bf8d54 Mon Sep 17 00:00:00 2001 From: Brennan Saeta Date: Fri, 26 May 2017 10:04:04 -0800 Subject: [PATCH 0260/8554] Add session.list_devices() API In order to debug a TensorFlow cluster or check whether devices are available in a local session (e.g. GPU drivers are loaded), this change adds a `sess.list_devices` API to list all devices within the cluster. This CL implements the list_devices() feature via extensions to the TensorFlow C API, and the corresponding additions to the session.h session class and corresponding subclasses for both direct sessions, grpc_sessions, tensorflow_serving, and others. Additionally, in order to accomidate ClusterSpec propagation clusters, Master::ListDevices now also includes a session_handle in order to identify the appropriate master_session on which it should list the available devices. (With ClusterSpec propagation, different sessions can have different servers with different device capabilities.) This CL adds a ListDevices() API to MasterSession. It is most efficient to implement this API call there, because the MasterSession already has a list of devices. Additionally, this change upgrades the implementation of Master::ListDevices() to delegate to the MasterSession if a session handle is specified, and to return an error if no corresponding session is found. Change: 157239656 --- .../batching/batching_session.cc | 30 ++++++++++++------- .../batching/batching_session_test.cc | 4 +++ .../core/test_util/mock_session.h | 5 ++++ .../servables/tensorflow/classifier_test.cc | 4 +++ .../servables/tensorflow/curried_session.cc | 4 +++ .../servables/tensorflow/curried_session.h | 2 ++ .../servables/tensorflow/regressor_test.cc | 4 +++ .../servables/tensorflow/serving_session.h | 3 ++ 8 files changed, 45 insertions(+), 11 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index 449988ff3b6..cff81f9a955 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -140,6 +140,8 @@ class BatchingSession : public ServingSession { const std::vector& target_node_names, std::vector* outputs, RunMetadata* run_metadata) override; + Status ListDevices(std::vector* response) override; + private: explicit BatchingSession(const BatchingSessionOptions& options); @@ -271,6 +273,10 @@ Status BatchingSession::Run( return status; } +Status BatchingSession::ListDevices(std::vector* response) { + return wrapped_->ListDevices(response); +} + BatchingSession::BatchingSession(const BatchingSessionOptions& options) : options_(options) {} @@ -581,17 +587,19 @@ Status CreateBasicBatchingSession( } } - auto scheduler_creator = [schedule_options]( - std::function>)> - process_batch_callback, - std::unique_ptr>* batch_scheduler) { - std::unique_ptr> - basic_batch_scheduler; - TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( - schedule_options, process_batch_callback, &basic_batch_scheduler)); - *batch_scheduler = std::move(basic_batch_scheduler); - return Status::OK(); - }; + auto scheduler_creator = + [schedule_options]( + std::function>)> + process_batch_callback, + std::unique_ptr>* + batch_scheduler) { + std::unique_ptr> + basic_batch_scheduler; + TF_RETURN_IF_ERROR(BasicBatchScheduler::Create( + schedule_options, process_batch_callback, &basic_batch_scheduler)); + *batch_scheduler = std::move(basic_batch_scheduler); + return Status::OK(); + }; return CreateBatchingSession(batching_session_options, {{signature, scheduler_creator}}, std::move(session), batching_session); diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index a80dff1c555..a173aead2d8 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -65,6 +65,10 @@ class BatchSizeCapturingSession : public ServingSession { target_node_names, outputs, run_metadata); } + Status ListDevices(std::vector* response) override { + return wrapped_->ListDevices(response); + } + int latest_batch_size() const { return latest_batch_size_; } private: diff --git a/tensorflow_serving/core/test_util/mock_session.h b/tensorflow_serving/core/test_util/mock_session.h index 3bd3200311f..64017439fa7 100644 --- a/tensorflow_serving/core/test_util/mock_session.h +++ b/tensorflow_serving/core/test_util/mock_session.h @@ -53,6 +53,11 @@ class MockSession : public tensorflow::Session { const std::vector>& inputs, const std::vector& output_names, std::vector* outputs)); + + MOCK_METHOD1(ListDevices, + ::tensorflow::Status( + std::vector<::tensorflow::DeviceAttributes>* response)); + MOCK_METHOD0(Close, ::tensorflow::Status()); }; diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc index 13e5e5d98a5..582ea12dd75 100644 --- a/tensorflow_serving/servables/tensorflow/classifier_test.cc +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -80,6 +80,10 @@ class FakeSession : public tensorflow::Session { return errors::Unimplemented("not available in fake"); } + Status ListDevices(std::vector* response) override { + return errors::Unimplemented("not available in fake"); + } + Status Run(const std::vector>& inputs, const std::vector& output_names, const std::vector& target_nodes, diff --git a/tensorflow_serving/servables/tensorflow/curried_session.cc b/tensorflow_serving/servables/tensorflow/curried_session.cc index b46f5034ae5..8f8393df947 100644 --- a/tensorflow_serving/servables/tensorflow/curried_session.cc +++ b/tensorflow_serving/servables/tensorflow/curried_session.cc @@ -52,6 +52,10 @@ Status CurriedSession::Run(const RunOptions& run_options, target_node_names, outputs, run_metadata); } +Status CurriedSession::ListDevices(std::vector* response) { + return wrapped_->ListDevices(response); +} + Status CurriedSession::ValidateExplicitInputsDontMatchCurriedInputs( const std::vector>& explicit_inputs) const { for (const auto& entry : explicit_inputs) { diff --git a/tensorflow_serving/servables/tensorflow/curried_session.h b/tensorflow_serving/servables/tensorflow/curried_session.h index e42d596fd71..7bd7f9b756e 100644 --- a/tensorflow_serving/servables/tensorflow/curried_session.h +++ b/tensorflow_serving/servables/tensorflow/curried_session.h @@ -45,6 +45,8 @@ class CurriedSession : public ServingSession { const std::vector& target_node_names, std::vector* outputs, RunMetadata* run_metadata) override; + Status ListDevices(std::vector* response) override; + private: // Verifies no overlap between the tensor names in 'explicit_inputs' and // 'curried_inputs_'. diff --git a/tensorflow_serving/servables/tensorflow/regressor_test.cc b/tensorflow_serving/servables/tensorflow/regressor_test.cc index 7dd045494c4..d9cb5219b05 100644 --- a/tensorflow_serving/servables/tensorflow/regressor_test.cc +++ b/tensorflow_serving/servables/tensorflow/regressor_test.cc @@ -75,6 +75,10 @@ class FakeSession : public tensorflow::Session { return errors::Unimplemented("not available in fake"); } + Status ListDevices(std::vector* response) override { + return errors::Unimplemented("not available in fake"); + } + Status Run(const std::vector>& inputs, const std::vector& output_names, const std::vector& target_nodes, diff --git a/tensorflow_serving/servables/tensorflow/serving_session.h b/tensorflow_serving/servables/tensorflow/serving_session.h index 0106524ef2a..561c8ff051a 100644 --- a/tensorflow_serving/servables/tensorflow/serving_session.h +++ b/tensorflow_serving/servables/tensorflow/serving_session.h @@ -70,6 +70,9 @@ class ServingSessionWrapper : public ServingSession { return wrapped_->Run(run_options, inputs, output_tensor_names, target_node_names, outputs, run_metadata); } + Status ListDevices(std::vector* response) override { + return wrapped_->ListDevices(response); + } private: std::unique_ptr wrapped_; From ee2144cab6b1d09cde916223c7edfee374b2ba0e Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 30 May 2017 15:19:26 -0700 Subject: [PATCH 0261/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index c03d5cc6642..7fda1bb1177 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit c03d5cc66421b97770ee2b63595c806a90173ef4 +Subproject commit 7fda1bb1177c69fa7bf80d20d5c5e7aaa25816e7 diff --git a/tf_models b/tf_models index 71bf3d47fc8..8b760f85e90 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 71bf3d47fc8b064ec6b677f121d9b044e0f75c7a +Subproject commit 8b760f85e902f11a5dd0060a32ed054061a95e82 From c3a8074f061ec49773f831435dc0ef207d3b3678 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 2 Jun 2017 09:26:42 -0800 Subject: [PATCH 0262/8554] Remove outdated reference to mnist_inference in setup.md Change: 157845952 --- tensorflow_serving/g3doc/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 9b2e9bdd4a5..10fe58994a7 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -103,7 +103,7 @@ Binaries are placed in the bazel-bin directory, and can be run using a command like: ~~~shell -./bazel-bin/tensorflow_serving/example/mnist_inference +bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server ~~~ To test your installation, execute: From 35515d61e9d43f66abde18dd8cf0529785ed4318 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 2 Jun 2017 13:35:14 -0800 Subject: [PATCH 0263/8554] Merge changes from github. Change: 157876783 --- tensorflow_serving/example/inception_client.cc | 14 ++++++++++---- tensorflow_serving/g3doc/serving_basic.md | 4 ++-- tensorflow_serving/model_servers/BUILD | 1 + tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- 4 files changed, 14 insertions(+), 7 deletions(-) diff --git a/tensorflow_serving/example/inception_client.cc b/tensorflow_serving/example/inception_client.cc index 4b1d183ae87..34ab4217366 100644 --- a/tensorflow_serving/example/inception_client.cc +++ b/tensorflow_serving/example/inception_client.cc @@ -40,12 +40,15 @@ class ServingClient { : stub_(PredictionService::NewStub(channel)) {} tensorflow::string callPredict(const tensorflow::string& model_name, + const tensorflow::string& model_signature_name, const tensorflow::string& file_path) { PredictRequest predictRequest; PredictResponse response; ClientContext context; predictRequest.mutable_model_spec()->set_name(model_name); + predictRequest.mutable_model_spec()->set_signature_name( + model_signature_name); google::protobuf::Map& inputs = *predictRequest.mutable_inputs(); @@ -116,11 +119,14 @@ int main(int argc, char** argv) { tensorflow::string server_port = "localhost:9000"; tensorflow::string image_file = ""; tensorflow::string model_name = "inception"; + tensorflow::string model_signature_name = "predict_images"; std::vector flag_list = { tensorflow::Flag("server_port", &server_port, "the IP and port of the server"), - tensorflow::Flag("image_file", &image_file, "the path to the "), - tensorflow::Flag("model_name", &model_name, "name of model")}; + tensorflow::Flag("image_file", &image_file, "the path to the image"), + tensorflow::Flag("model_name", &model_name, "name of model"), + tensorflow::Flag("model_signature_name", &model_signature_name, + "name of model signature")}; tensorflow::string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || image_file.empty()) { @@ -132,7 +138,7 @@ int main(int argc, char** argv) { grpc::CreateChannel(server_port, grpc::InsecureChannelCredentials())); std::cout << "calling predict using file: " << image_file << " ..." << std::endl; - std::cout << guide.callPredict(model_name, image_file) << std::endl; - + std::cout << guide.callPredict(model_name, model_signature_name, image_file) + << std::endl; return 0; } diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 463d006d279..d1a7e236702 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -172,8 +172,8 @@ saved_model.pb variables Each version sub-directory contains the following files: - * `saved_model.pb` is the serialized tensorflow::SavedModel. It includes the - the one or more graph definitions of the model, as well as metadata of the + * `saved_model.pb` is the serialized tensorflow::SavedModel. It includes + one or more graph definitions of the model, as well as metadata of the model such as signatures. * `variables` are files that hold the serialized variables of the graphs. diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index cbeaaf042e3..247480695ae 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -132,6 +132,7 @@ cc_binary( "@protobuf//:cc_wkt_protos", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", + "@org_tensorflow//tensorflow/core/platform/hadoop:hadoop_file_system", "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:availability_preserving_policy", diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 4c61f9286bd..8ea5fc08c71 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -30,7 +30,7 @@ RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ RUN pip install enum34 futures mock six && \ pip install --pre 'protobuf>=3.0.0a3' && \ - pip install -i https://testpypi.python.org/simple --pre grpcio + pip install 'grpcio>=1.1.3' # Set up Bazel. From afc298368ca707d77503a2ee82a6af844750d02b Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 5 Jun 2017 10:21:36 -0800 Subject: [PATCH 0264/8554] Fix race in server_core_test.cc. The problem was that ListAvailableServableIds() stops returning the servable when it's removed from the serving map, even though it may not have finished unloading. Later, when ServerCore requests monitor->WaitUntilServablesReachState(), if it's "too quick" it sees the kEnd from the unload and concludes that new load didn't succeed. Change: 158037660 --- .../model_servers/server_core_test.cc | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index a083ebb1702..7b1f8efdbc5 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -84,6 +84,8 @@ TEST_P(ServerCoreTest, ReloadConfigUnloadsModels) { GetTestModelServerConfigForFakePlatform(); ModelServerConfig empty_config; empty_config.mutable_model_config_list(); + const ServableId servable_id = {test_util::kTestModelName, + test_util::kTestModelVersion}; std::unique_ptr server_core; TF_ASSERT_OK(CreateServerCore(nonempty_config, &server_core)); @@ -91,9 +93,9 @@ TEST_P(ServerCoreTest, ReloadConfigUnloadsModels) { TF_ASSERT_OK(server_core->ReloadConfig(empty_config)); // Wait for the unload to finish (ReloadConfig() doesn't block on this). - while (!server_core->ListAvailableServableIds().empty()) { - Env::Default()->SleepForMicroseconds(10 * 1000); - } + test_util::WaitUntilServableManagerStateIsOneOf( + *server_core->servable_state_monitor(), servable_id, + {ServableState::ManagerState::kEnd}); } TEST_P(ServerCoreTest, ReloadConfigHandlesLoadingAPreviouslyUnloadedModel) { @@ -101,15 +103,17 @@ TEST_P(ServerCoreTest, ReloadConfigHandlesLoadingAPreviouslyUnloadedModel) { empty_config.mutable_model_config_list(); const ModelServerConfig nonempty_config = GetTestModelServerConfigForFakePlatform(); + const ServableId servable_id = {test_util::kTestModelName, + test_util::kTestModelVersion}; // Load, and then unload, a servable. std::unique_ptr server_core; TF_ASSERT_OK(CreateServerCore(nonempty_config, &server_core)); TF_ASSERT_OK(server_core->ReloadConfig(empty_config)); // Wait for the unload to finish (ReloadConfig() doesn't block on this). - while (!server_core->ListAvailableServableIds().empty()) { - Env::Default()->SleepForMicroseconds(10 * 1000); - } + test_util::WaitUntilServableManagerStateIsOneOf( + *server_core->servable_state_monitor(), servable_id, + {ServableState::ManagerState::kEnd}); // Re-load the same servable. TF_ASSERT_OK(server_core->ReloadConfig(nonempty_config)); From d2e0a0bad6ec7bb769b2e8fc87f2382a4b6331f7 Mon Sep 17 00:00:00 2001 From: Noah Fiedel Date: Tue, 6 Jun 2017 11:13:16 -0800 Subject: [PATCH 0265/8554] Reduce test flakiness by changing from post-server-startup sleep to a function that polls and waits until the server is ready. This significantly reduces test time in the average case, while accommodating cases where the server takes longer to startup. Before, flakiness was around 0.5%. After, flakiness is around 0.13%. (10K test runs, 13 failures). Change: 158172529 --- tensorflow_serving/model_servers/BUILD | 2 +- .../tensorflow_model_server_test.py | 79 +++++++++++++------ 2 files changed, 57 insertions(+), 24 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 247480695ae..20200d224c7 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -142,7 +142,7 @@ cc_binary( py_test( name = "tensorflow_model_server_test", - size = "large", + size = "medium", srcs = ["tensorflow_model_server_test.py"], data = [ ":tensorflow_model_server", diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 8e9f0e1df59..56e038ff7fe 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -14,7 +14,6 @@ # ============================================================================== #!/usr/bin/env python2.7 - """Tests for tensorflow_model_server.""" import atexit @@ -42,6 +41,10 @@ FLAGS = flags.FLAGS +RPC_TIMEOUT = 5.0 +CHANNEL_WAIT_TIMEOUT = 5.0 +WAIT_FOR_SERVER_READY_INT_SECS = 60 + def PickUnusedPort(): s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) @@ -51,6 +54,25 @@ def PickUnusedPort(): return port +def WaitForServerReady(port): + """Waits for a server on the localhost to become ready.""" + for _ in range(0, WAIT_FOR_SERVER_READY_INT_SECS): + time.sleep(1) + request = predict_pb2.PredictRequest() + request.model_spec.name = 'intentionally_missing_model' + + try: + # Send empty request to missing model + channel = implementations.insecure_channel('localhost', port) + stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) + stub.Predict(request, RPC_TIMEOUT) + except face.AbortionError as error: + # Missing model error will have details containing 'Servable' + if 'Servable' in error.details: + print 'Server is ready' + break + + class TensorflowModelServerTest(tf.test.TestCase): """This class defines integration test cases for tensorflow_model_server.""" @@ -91,8 +113,13 @@ def TerminateProcs(self): if self.server_proc is not None: self.server_proc.terminate() - def RunServer(self, port, model_name, model_path, use_saved_model, - batching_parameters_file=''): + def RunServer(self, + port, + model_name, + model_path, + use_saved_model, + batching_parameters_file='', + wait_for_server_ready=True): """Run tensorflow_model_server using test config.""" print 'Starting test server...' command = os.path.join(self.binary_dir, 'tensorflow_model_server') @@ -106,13 +133,16 @@ def RunServer(self, port, model_name, model_path, use_saved_model, print command self.server_proc = subprocess.Popen(shlex.split(command)) print 'Server started' + if wait_for_server_ready: + WaitForServerReady(port) return 'localhost:' + str(port) def RunServerWithModelConfigFile(self, port, model_config_file, use_saved_model, - pipe=None): + pipe=None, + wait_for_server_ready=True): """Run tensorflow_model_server using test config.""" print 'Starting test server...' command = os.path.join(self.binary_dir, 'tensorflow_model_server') @@ -123,6 +153,8 @@ def RunServerWithModelConfigFile(self, print command self.server_proc = subprocess.Popen(shlex.split(command), stderr=pipe) print 'Server started' + if wait_for_server_ready: + WaitForServerReady(port) return 'localhost:' + str(port) def VerifyPredictRequest(self, @@ -146,7 +178,7 @@ def VerifyPredictRequest(self, host, port = model_server_address.split(':') channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) - result = stub.Predict(request, 5.0) # 5 secs timeout + result = stub.Predict(request, RPC_TIMEOUT) # 5 secs timeout # Verify response self.assertTrue('y' in result.outputs) self.assertIs(types_pb2.DT_FLOAT, result.outputs['y'].dtype) @@ -190,7 +222,6 @@ def testClassify(self): atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', model_path, use_saved_model) - time.sleep(5) print 'Sending Classify request...' # Prepare request @@ -205,7 +236,7 @@ def testClassify(self): host, port = model_server_address.split(':') channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) - result = stub.Classify(request, 5.0) # 5 secs timeout + result = stub.Classify(request, RPC_TIMEOUT) # 5 secs timeout # Verify response self.assertEquals(1, len(result.result.classifications)) self.assertEquals(1, len(result.result.classifications[0].classes)) @@ -221,7 +252,6 @@ def testRegress(self): atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', model_path, use_saved_model) - time.sleep(5) print 'Sending Regress request...' # Prepare request @@ -236,12 +266,11 @@ def testRegress(self): host, port = model_server_address.split(':') channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) - result = stub.Regress(request, 5.0) # 5 secs timeout + result = stub.Regress(request, RPC_TIMEOUT) # 5 secs timeout # Verify response self.assertEquals(1, len(result.result.regressions)) expected_output = 3.0 - self.assertEquals(expected_output, - result.result.regressions[0].value) + self.assertEquals(expected_output, result.result.regressions[0].value) def testMultiInference(self): """Test PredictionService.MultiInference implementation.""" @@ -253,7 +282,6 @@ def testMultiInference(self): model_server_address = self.RunServer(PickUnusedPort(), 'default', model_path, use_saved_model, enable_batching) - time.sleep(5) print 'Sending MultiInference request...' # Prepare request @@ -272,7 +300,7 @@ def testMultiInference(self): host, port = model_server_address.split(':') channel = implementations.insecure_channel(host, int(port)) stub = prediction_service_pb2.beta_create_PredictionService_stub(channel) - result = stub.MultiInference(request, 5.0) # 5 secs timeout + result = stub.MultiInference(request, RPC_TIMEOUT) # 5 secs timeout # Verify response self.assertEquals(2, len(result.results)) @@ -282,7 +310,9 @@ def testMultiInference(self): self.assertEquals(expected_output, result.results[ 1].classification_result.classifications[0].classes[0].score) - def _TestPredict(self, model_path, use_saved_model, + def _TestPredict(self, + model_path, + use_saved_model, batching_parameters_file=''): """Helper method to test prediction. @@ -296,7 +326,6 @@ def _TestPredict(self, model_path, use_saved_model, model_server_address = self.RunServer(PickUnusedPort(), 'default', model_path, use_saved_model, batching_parameters_file) - time.sleep(5) self.VerifyPredictRequest(model_server_address, expected_output=3.0) self.VerifyPredictRequest( model_server_address, expected_output=3.0, specify_output=False) @@ -307,10 +336,10 @@ def testPredictSessionBundle(self): def testPredictBatchingSessionBundle(self): """Test PredictionService.Predict implementation with SessionBundle.""" - self._TestPredict(self._GetSessionBundlePath(), - use_saved_model=False, - batching_parameters_file= - self._GetBatchingParametersFile()) + self._TestPredict( + self._GetSessionBundlePath(), + use_saved_model=False, + batching_parameters_file=self._GetBatchingParametersFile()) def testPredictSavedModel(self): """Test PredictionService.Predict implementation with SavedModel.""" @@ -328,10 +357,14 @@ def _TestBadModel(self, use_saved_model): atexit.register(self.TerminateProcs) # Both SessionBundle and SavedModel use the same bad model path, but in the # case of SavedModel, the export will get up-converted to a SavedModel. + # As the bad model will prevent the server from becoming ready, we set the + # wait_for_server_ready param to False to avoid blocking/timing out. model_server_address = self.RunServer( - PickUnusedPort(), 'default', - os.path.join(self.testdata_dir, 'bad_half_plus_two'), use_saved_model) - time.sleep(5) + PickUnusedPort(), + 'default', + os.path.join(self.testdata_dir, 'bad_half_plus_two'), + use_saved_model, + wait_for_server_ready=False) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address, expected_output=3.0) self.assertIs(beta_interfaces.StatusCode.FAILED_PRECONDITION, @@ -351,7 +384,6 @@ def testGoodModelConfig(self): model_server_address = self.RunServerWithModelConfigFile( PickUnusedPort(), self._GetGoodModelConfigFile(), True) # use_saved_model - time.sleep(5) self.VerifyPredictRequest( model_server_address, model_name='half_plus_two', expected_output=3.0) @@ -383,5 +415,6 @@ def testBadModelConfig(self): self.assertNotEqual(self.server_proc.stderr, None) self.assertGreater(self.server_proc.stderr.read().find(error_message), -1) + if __name__ == '__main__': tf.test.main() From 3144bfc2c5a0e15d7b265b47adeb57242a308ad6 Mon Sep 17 00:00:00 2001 From: xuchen Date: Mon, 12 Jun 2017 21:53:05 +0800 Subject: [PATCH 0266/8554] FIx service selector (#468) FIx service selector --- tensorflow_serving/example/inception_k8s.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/example/inception_k8s.yaml b/tensorflow_serving/example/inception_k8s.yaml index e23c3a6036e..c45c5334ef3 100644 --- a/tensorflow_serving/example/inception_k8s.yaml +++ b/tensorflow_serving/example/inception_k8s.yaml @@ -47,5 +47,5 @@ spec: - port: 9000 targetPort: 9000 selector: - run: inception-service + app: inception-server type: LoadBalancer From 9ec5fc145d6f6543a26adc32f3d46747cfc77f96 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Fri, 9 Jun 2017 07:42:35 -0800 Subject: [PATCH 0267/8554] Update TensorFlow Serving examples to use tf.saved_model APIs. Change: 158522097 --- tensorflow_serving/example/BUILD | 14 ---- .../example/inception_saved_model.py | 71 ++++++++++--------- .../example/mnist_saved_model.py | 65 ++++++++--------- 3 files changed, 69 insertions(+), 81 deletions(-) diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 85bb83412df..37d22efe3b0 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -49,13 +49,6 @@ py_binary( deps = [ ":mnist_input_data", "@org_tensorflow//tensorflow:tensorflow_py", - "@org_tensorflow//tensorflow/python/saved_model:builder", - "@org_tensorflow//tensorflow/python/saved_model:constants", - "@org_tensorflow//tensorflow/python/saved_model:loader", - "@org_tensorflow//tensorflow/python/saved_model:signature_constants", - "@org_tensorflow//tensorflow/python/saved_model:signature_def_utils", - "@org_tensorflow//tensorflow/python/saved_model:tag_constants", - "@org_tensorflow//tensorflow/python/saved_model:utils", ], ) @@ -94,13 +87,6 @@ py_binary( deps = [ "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", - "@org_tensorflow//tensorflow/python/saved_model:builder", - "@org_tensorflow//tensorflow/python/saved_model:constants", - "@org_tensorflow//tensorflow/python/saved_model:loader", - "@org_tensorflow//tensorflow/python/saved_model:signature_constants", - "@org_tensorflow//tensorflow/python/saved_model:signature_def_utils", - "@org_tensorflow//tensorflow/python/saved_model:tag_constants", - "@org_tensorflow//tensorflow/python/saved_model:utils", ], ) diff --git a/tensorflow_serving/example/inception_saved_model.py b/tensorflow_serving/example/inception_saved_model.py index 1654c75cc74..86b20c460d0 100644 --- a/tensorflow_serving/example/inception_saved_model.py +++ b/tensorflow_serving/example/inception_saved_model.py @@ -26,12 +26,6 @@ import tensorflow as tf -from tensorflow.python.saved_model import builder as saved_model_builder -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.saved_model import signature_def_utils -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.saved_model import utils -from tensorflow.python.util import compat from inception import inception_model tf.app.flags.DEFINE_string('checkpoint_dir', '/tmp/inception_train', @@ -119,46 +113,53 @@ def export(): # Export inference model. output_path = os.path.join( - compat.as_bytes(FLAGS.output_dir), - compat.as_bytes(str(FLAGS.model_version))) + tf.compat.as_bytes(FLAGS.output_dir), + tf.compat.as_bytes(str(FLAGS.model_version))) print 'Exporting trained model to', output_path - builder = saved_model_builder.SavedModelBuilder(output_path) + builder = tf.saved_model.builder.SavedModelBuilder(output_path) # Build the signature_def_map. - classify_inputs_tensor_info = utils.build_tensor_info( + classify_inputs_tensor_info = tf.saved_model.utils.build_tensor_info( serialized_tf_example) - classes_output_tensor_info = utils.build_tensor_info(classes) - scores_output_tensor_info = utils.build_tensor_info(values) - - classification_signature = signature_def_utils.build_signature_def( - inputs={ - signature_constants.CLASSIFY_INPUTS: classify_inputs_tensor_info - }, - outputs={ - signature_constants.CLASSIFY_OUTPUT_CLASSES: - classes_output_tensor_info, - signature_constants.CLASSIFY_OUTPUT_SCORES: - scores_output_tensor_info - }, - method_name=signature_constants.CLASSIFY_METHOD_NAME) - - predict_inputs_tensor_info = utils.build_tensor_info(jpegs) - prediction_signature = signature_def_utils.build_signature_def( - inputs={'images': predict_inputs_tensor_info}, - outputs={ - 'classes': classes_output_tensor_info, - 'scores': scores_output_tensor_info - }, - method_name=signature_constants.PREDICT_METHOD_NAME) + classes_output_tensor_info = tf.saved_model.utils.build_tensor_info( + classes) + scores_output_tensor_info = tf.saved_model.utils.build_tensor_info(values) + + classification_signature = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs={ + tf.saved_model.signature_constants.CLASSIFY_INPUTS: + classify_inputs_tensor_info + }, + outputs={ + tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES: + classes_output_tensor_info, + tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES: + scores_output_tensor_info + }, + method_name=tf.saved_model.signature_constants. + CLASSIFY_METHOD_NAME)) + + predict_inputs_tensor_info = tf.saved_model.utils.build_tensor_info(jpegs) + prediction_signature = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs={'images': predict_inputs_tensor_info}, + outputs={ + 'classes': classes_output_tensor_info, + 'scores': scores_output_tensor_info + }, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME + )) legacy_init_op = tf.group( tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( - sess, [tag_constants.SERVING], + sess, [tf.saved_model.tag_constants.SERVING], signature_def_map={ 'predict_images': prediction_signature, - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + tf.saved_model.signature_constants. + DEFAULT_SERVING_SIGNATURE_DEF_KEY: classification_signature, }, legacy_init_op=legacy_init_op) diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py index 19249e46a45..795b3865e9b 100644 --- a/tensorflow_serving/example/mnist_saved_model.py +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -31,12 +31,6 @@ import tensorflow as tf -from tensorflow.python.saved_model import builder as saved_model_builder -from tensorflow.python.saved_model import signature_constants -from tensorflow.python.saved_model import signature_def_utils -from tensorflow.python.saved_model import tag_constants -from tensorflow.python.saved_model import utils -from tensorflow.python.util import compat from tensorflow_serving.example import mnist_input_data tf.app.flags.DEFINE_integer('training_iteration', 1000, @@ -93,41 +87,48 @@ def main(_): # whenever code changes. export_path_base = sys.argv[-1] export_path = os.path.join( - compat.as_bytes(export_path_base), - compat.as_bytes(str(FLAGS.model_version))) + tf.compat.as_bytes(export_path_base), + tf.compat.as_bytes(str(FLAGS.model_version))) print 'Exporting trained model to', export_path - builder = saved_model_builder.SavedModelBuilder(export_path) + builder = tf.saved_model.builder.SavedModelBuilder(export_path) # Build the signature_def_map. - classification_inputs = utils.build_tensor_info(serialized_tf_example) - classification_outputs_classes = utils.build_tensor_info(prediction_classes) - classification_outputs_scores = utils.build_tensor_info(values) - - classification_signature = signature_def_utils.build_signature_def( - inputs={signature_constants.CLASSIFY_INPUTS: classification_inputs}, - outputs={ - signature_constants.CLASSIFY_OUTPUT_CLASSES: - classification_outputs_classes, - signature_constants.CLASSIFY_OUTPUT_SCORES: - classification_outputs_scores - }, - method_name=signature_constants.CLASSIFY_METHOD_NAME) - - tensor_info_x = utils.build_tensor_info(x) - tensor_info_y = utils.build_tensor_info(y) - - prediction_signature = signature_def_utils.build_signature_def( - inputs={'images': tensor_info_x}, - outputs={'scores': tensor_info_y}, - method_name=signature_constants.PREDICT_METHOD_NAME) + classification_inputs = tf.saved_model.utils.build_tensor_info( + serialized_tf_example) + classification_outputs_classes = tf.saved_model.utils.build_tensor_info( + prediction_classes) + classification_outputs_scores = tf.saved_model.utils.build_tensor_info(values) + + classification_signature = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs={ + tf.saved_model.signature_constants.CLASSIFY_INPUTS: + classification_inputs + }, + outputs={ + tf.saved_model.signature_constants.CLASSIFY_OUTPUT_CLASSES: + classification_outputs_classes, + tf.saved_model.signature_constants.CLASSIFY_OUTPUT_SCORES: + classification_outputs_scores + }, + method_name=tf.saved_model.signature_constants.CLASSIFY_METHOD_NAME)) + + tensor_info_x = tf.saved_model.utils.build_tensor_info(x) + tensor_info_y = tf.saved_model.utils.build_tensor_info(y) + + prediction_signature = ( + tf.saved_model.signature_def_utils.build_signature_def( + inputs={'images': tensor_info_x}, + outputs={'scores': tensor_info_y}, + method_name=tf.saved_model.signature_constants.PREDICT_METHOD_NAME)) legacy_init_op = tf.group(tf.tables_initializer(), name='legacy_init_op') builder.add_meta_graph_and_variables( - sess, [tag_constants.SERVING], + sess, [tf.saved_model.tag_constants.SERVING], signature_def_map={ 'predict_images': prediction_signature, - signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: classification_signature, }, legacy_init_op=legacy_init_op) From 95cedd98da877e720bd48aa0e9c6e313612c2f84 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 9 Jun 2017 15:47:18 -0800 Subject: [PATCH 0268/8554] Update outdated comment now that all functions in the Prediction API support signature_name. Change: 158580011 --- tensorflow_serving/apis/model.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/apis/model.proto b/tensorflow_serving/apis/model.proto index e74cf383895..14ec3f0f48e 100644 --- a/tensorflow_serving/apis/model.proto +++ b/tensorflow_serving/apis/model.proto @@ -17,6 +17,6 @@ message ModelSpec { google.protobuf.Int64Value version = 2; // A named signature to evaluate. If unspecified, the default signature will - // be used. Note that only MultiInference will initially support this. + // be used. string signature_name = 3; } From c0284256f6abd473f81db35109f7bc27b1db3433 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 12 Jun 2017 13:39:29 -0800 Subject: [PATCH 0269/8554] Include Go language support for the log-related proto buffers. Change: 158766273 --- tensorflow_serving/config/BUILD | 2 ++ tensorflow_serving/core/BUILD | 1 + 2 files changed, 3 insertions(+) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 78ada7438ed..74682b4629a 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -46,6 +46,7 @@ serving_proto_library( name = "log_collector_config_proto", srcs = ["log_collector_config.proto"], cc_api_version = 2, + go_api_version = 2, deps = [ ], ) @@ -54,6 +55,7 @@ serving_proto_library( name = "logging_config_proto", srcs = ["logging_config.proto"], cc_api_version = 2, + go_api_version = 2, deps = [ ":log_collector_config_proto", ], diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 61d080e4be1..baa5df129db 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -733,6 +733,7 @@ serving_proto_library( name = "logging_proto", srcs = ["logging.proto"], cc_api_version = 2, + go_api_version = 2, visibility = [ "//visibility:public", ], From fe0060efde17e58bd93e202f950064ba6a49b0e0 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 12 Jun 2017 17:11:19 -0800 Subject: [PATCH 0270/8554] Add sequence length bucketization to SharedBatchScheduler use case example list, in batching guide. Change: 158793348 --- tensorflow_serving/batching/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/batching/README.md b/tensorflow_serving/batching/README.md index c6103219192..03869e2fa44 100644 --- a/tensorflow_serving/batching/README.md +++ b/tensorflow_serving/batching/README.md @@ -173,7 +173,8 @@ multiple versions of a model offered concurrently). In another scenario, a single request gets broken down into sub-requests involving multiple distinct servables (e.g. a recommender system might have a triggering model that decides whether to formulate a recommendation, followed by a model that selects the -actual recommendation). +actual recommendation). A third scenario is bucketizing sequence model requests +to batch together requests of similar length, to minimize padding. Generally speaking, using a separate batch scheduler for each kind of request or sub-task does not work well if they share a common underlying compute From cc46ded88288a282fcd010074445bd4aa6c47d70 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 14 Jun 2017 05:48:37 -0800 Subject: [PATCH 0271/8554] Merge changes from github. Change: 158972472 --- tensorflow_serving/example/inception_client.cc | 1 + tensorflow_serving/example/inception_k8s.yaml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/example/inception_client.cc b/tensorflow_serving/example/inception_client.cc index 34ab4217366..f2895f2b8c9 100644 --- a/tensorflow_serving/example/inception_client.cc +++ b/tensorflow_serving/example/inception_client.cc @@ -127,6 +127,7 @@ int main(int argc, char** argv) { tensorflow::Flag("model_name", &model_name, "name of model"), tensorflow::Flag("model_signature_name", &model_signature_name, "name of model signature")}; + tensorflow::string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || image_file.empty()) { diff --git a/tensorflow_serving/example/inception_k8s.yaml b/tensorflow_serving/example/inception_k8s.yaml index e23c3a6036e..c45c5334ef3 100644 --- a/tensorflow_serving/example/inception_k8s.yaml +++ b/tensorflow_serving/example/inception_k8s.yaml @@ -47,5 +47,5 @@ spec: - port: 9000 targetPort: 9000 selector: - run: inception-service + app: inception-server type: LoadBalancer From 69d2067fced5184098c28414b5e1201af9770bb7 Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Wed, 14 Jun 2017 07:00:03 -0700 Subject: [PATCH 0272/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 7fda1bb1177..4c0052dc4b7 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 7fda1bb1177c69fa7bf80d20d5c5e7aaa25816e7 +Subproject commit 4c0052dc4b7c49a876166113b49960a57f7db939 diff --git a/tf_models b/tf_models index 8b760f85e90..7ad450b8430 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 8b760f85e902f11a5dd0060a32ed054061a95e82 +Subproject commit 7ad450b84309b82bccb0e8f2e40e9559f33cd258 From 545764504de91cd721edc74bbbf9d56ef6f89866 Mon Sep 17 00:00:00 2001 From: Evgeny Date: Thu, 15 Jun 2017 10:06:31 +0700 Subject: [PATCH 0273/8554] docker: updated ubuntu to 16.04, removed ppa of JDK8, updated bazel to 0.5.1, removed unnecessary hacks for bazel, updated python modules (#484) --- .../tools/docker/Dockerfile.devel | 34 +++---------------- 1 file changed, 5 insertions(+), 29 deletions(-) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 8ea5fc08c71..f7392be5e8f 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -1,4 +1,4 @@ -FROM ubuntu:14.04 +FROM ubuntu:16.04 MAINTAINER Jeremiah Harmsen @@ -18,45 +18,21 @@ RUN apt-get update && apt-get install -y \ zip \ zlib1g-dev \ libcurl3-dev \ + openjdk-8-jdk\ + openjdk-8-jre-headless \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py && \ - rm get-pip.py - # Set up grpc -RUN pip install enum34 futures mock six && \ - pip install --pre 'protobuf>=3.0.0a3' && \ - pip install 'grpcio>=1.1.3' +RUN pip install mock grpcio # Set up Bazel. -# We need to add a custom PPA to pick up JDK8, since trusty doesn't -# have an openjdk8 backport. openjdk-r is maintained by a reliable contributor: -# Matthias Klose (https://launchpad.net/~doko). It will do until -# we either update the base image beyond 14.04 or openjdk-8 is -# finally backported to trusty; see e.g. -# https://bugs.launchpad.net/trusty-backports/+bug/1368094 -RUN add-apt-repository -y ppa:openjdk-r/ppa && \ - apt-get update && \ - apt-get install -y openjdk-8-jdk openjdk-8-jre-headless && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Running bazel inside a `docker build` command causes trouble, cf: -# https://github.com/bazelbuild/bazel/issues/134 -# The easiest solution is to set up a bazelrc file forcing --batch. -RUN echo "startup --batch" >>/root/.bazelrc -# Similarly, we need to workaround sandboxing issues: -# https://github.com/bazelbuild/bazel/issues/418 -RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ - >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.4.5 +ENV BAZEL_VERSION 0.5.1 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ From f03667949589dc32584275b639ca954fa96e53b5 Mon Sep 17 00:00:00 2001 From: Catherine Sawatzky Date: Thu, 15 Jun 2017 09:36:51 -0400 Subject: [PATCH 0274/8554] Add locate to installed packages in Dockerfile (#457) * Add locate to installed packages in Dockerfile to support locate command in tensorflow configure script * Change locate to mlocate --- tensorflow_serving/tools/docker/Dockerfile.devel | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index f7392be5e8f..7b8402e6e4f 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ + mlocate \ pkg-config \ python-dev \ python-numpy \ From 6623f04f6bd713870f605e600e29f6ec5160bf96 Mon Sep 17 00:00:00 2001 From: Wolff Dobson Date: Tue, 20 Jun 2017 12:44:06 -0700 Subject: [PATCH 0275/8554] Leftnav files (#486) --- tensorflow_serving/g3doc/leftnav_files | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 tensorflow_serving/g3doc/leftnav_files diff --git a/tensorflow_serving/g3doc/leftnav_files b/tensorflow_serving/g3doc/leftnav_files new file mode 100644 index 00000000000..146e204d6a1 --- /dev/null +++ b/tensorflow_serving/g3doc/leftnav_files @@ -0,0 +1,9 @@ +### TensorFlow Serving +architecture_overview.md +setup.md +serving_basic.md +serving_advanced.md +serving_inception.md +custom_servable.md +custom_source.md +docker.md \ No newline at end of file From fe04745f76576782a6a34c6aed753af9052f44c6 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 20 Jun 2017 09:33:26 -0800 Subject: [PATCH 0276/8554] Make some core TensorFlow Serving class comments Doxygen friendly. Change: 159579172 --- .../core/aspired_version_policy.h | 52 ++-- .../core/aspired_versions_manager.h | 89 ++++--- .../core/aspired_versions_manager_builder.h | 82 +++--- tensorflow_serving/core/basic_manager.h | 246 +++++++++--------- tensorflow_serving/core/caching_manager.h | 34 +-- tensorflow_serving/core/loader.h | 169 ++++++------ tensorflow_serving/core/loader_harness.h | 186 ++++++------- tensorflow_serving/core/manager.h | 35 +-- tensorflow_serving/core/servable_handle.h | 72 ++--- .../core/servable_state_monitor.h | 107 ++++---- tensorflow_serving/core/source.h | 102 ++++---- tensorflow_serving/core/source_adapter.h | 47 ++-- .../model_servers/server_core.h | 89 +++---- .../servables/tensorflow/serving_session.h | 10 +- .../file_system_storage_path_source.h | 44 ++-- tensorflow_serving/util/any_ptr.h | 100 +++---- tensorflow_serving/util/event_bus.h | 119 +++++---- tensorflow_serving/util/executor.h | 10 +- 18 files changed, 806 insertions(+), 787 deletions(-) diff --git a/tensorflow_serving/core/aspired_version_policy.h b/tensorflow_serving/core/aspired_version_policy.h index b7f6447aa42..cd819d8433b 100644 --- a/tensorflow_serving/core/aspired_version_policy.h +++ b/tensorflow_serving/core/aspired_version_policy.h @@ -28,59 +28,59 @@ limitations under the License. namespace tensorflow { namespace serving { -// A snapshot of a servable's state and aspiredness. +/// A snapshot of a servable's state and aspiredness. struct AspiredServableStateSnapshot final { ServableId id; LoaderHarness::State state; bool is_aspired; }; -// An interface for the policy to be applied for transitioning servable versions -// in a servable stream. -// -// Policies should be entirely stateless and idempotent. Asking the same policy -// multiple times for the next action, for an identical vector of -// AspiredServableStateSnapshots, should return the same result. -// -// If additional state is required to implement a Policy, such state shall be -// shared via AspiredServableStateSnapshots. Depending on the kind of state, the -// most likely candidates for originating or tracking state are Sources or the -// Harness and Manager. +/// An interface for the policy to be applied for transitioning servable +/// versions in a servable stream. +/// +/// Policies should be entirely stateless and idempotent. Asking the same policy +/// multiple times for the next action, for an identical vector of +/// AspiredServableStateSnapshots, should return the same result. +/// +/// If additional state is required to implement a Policy, such state shall be +/// shared via AspiredServableStateSnapshots. Depending on the kind of state, +/// the most likely candidates for originating or tracking state are Sources or +/// the Harness and Manager. class AspiredVersionPolicy { public: - // The different actions that could be recommended by a policy. + /// The different actions that could be recommended by a policy. enum class Action : int { - // Call load on the servable. + /// Call load on the servable. kLoad, - // Call unload on the servable. + /// Call unload on the servable. kUnload, }; virtual ~AspiredVersionPolicy() = default; - // Action and the id of the servable associated with it. + /// Action and the id of the servable associated with it. struct ServableAction final { Action action; ServableId id; string DebugString() const { - return strings::StrCat("{ action: ", static_cast(action), " id: ", - id.DebugString(), " }"); + return strings::StrCat("{ action: ", static_cast(action), + " id: ", id.DebugString(), " }"); } }; - // Takes in a vector of state snapshots of all versions of a servable stream - // and returns an action to be performed for a particular servable version, - // depending only on the states of all the versions. - // - // If no action is to be performed, we don't return an action, meaning - // that the servable stream is up to date. + /// Takes in a vector of state snapshots of all versions of a servable stream + /// and returns an action to be performed for a particular servable version, + /// depending only on the states of all the versions. + /// + /// If no action is to be performed, we don't return an action, meaning + /// that the servable stream is up to date. virtual optional GetNextAction( const std::vector& all_versions) const = 0; protected: - // Returns the aspired ServableId with the highest version that matches - // kNew state, if any exists. + /// Returns the aspired ServableId with the highest version that matches + /// kNew state, if any exists. static optional GetHighestAspiredNewServableId( const std::vector& all_versions); diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index a4b74cbb8cc..43a4bb1a634 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -58,65 +58,68 @@ namespace test_util { class AspiredVersionsManagerTestAccess; } // namespace test_util -// A manager that implements the Target API which uses aspired-versions -// callbacks to dictate which servable versions to load. This manager also uses -// that API to infer which ones to unload: If a given servable version is -// currently loaded, and is omitted from an aspired-versions callback invocation -// pertaining to its servable stream, this manager interprets that omission as -// an implicit instruction to unload the version. See below for details. -// -// (The implicit-unload semantics facilitates stateless Source implementations, -// whereby a given iteration of the Source's logic simply decides which versions -// of a servable ought to be loaded, without needing to know what it has decided -// in the past.) -// -// This manager makes transitions between versions of a servable stream using a -// configured AspiredVersionPolicy. The manager prefers unloading before loading -// to free up resources in the server when deciding among transitions suggested -// by the policy. +/// A manager that implements the Target API which uses aspired-versions +/// callbacks to dictate which servable versions to load. This manager also uses +/// that API to infer which ones to unload: If a given servable version is +/// currently loaded, and is omitted from an aspired-versions callback +/// invocation pertaining to its servable stream, this manager interprets that +/// omission as an implicit instruction to unload the version. See below for +/// details. +/// +/// (The implicit-unload semantics facilitates stateless Source implementations, +/// whereby a given iteration of the Source's logic simply decides which +/// versions of a servable ought to be loaded, without needing to know what it +/// has decided in the past.) +/// +/// This manager makes transitions between versions of a servable stream using a +/// configured AspiredVersionPolicy. The manager prefers unloading before +/// loading to free up resources in the server when deciding among transitions +/// suggested by the policy. class AspiredVersionsManager : public Manager, public Target> { public: + /// Config options and pluggable objects that will be used by the + /// AspiredVersionsManager. struct Options { - // The resource tracker to use while managing servable resources. Optional. - // If left as nullptr, we do not validate servable resource usage. + /// The resource tracker to use while managing servable resources. Optional. + /// If left as nullptr, we do not validate servable resource usage. std::unique_ptr resource_tracker; - // The periodicity, in microseconds, of the thread which manages the state - // of the servables. Default: 100 milliseconds. If this is set less than or - // equal to 0, we don't run this thread at all. + /// The periodicity, in microseconds, of the thread which manages the state + /// of the servables. Default: 100 milliseconds. If this is set less than or + /// equal to 0, we don't run this thread at all. int64 manage_state_interval_micros = 100 * 1000; - // EventBus to publish servable state changes. This is optional, if unset, - // we don't publish. + /// EventBus to publish servable state changes. This is optional, if unset, + /// we don't publish. EventBus* servable_event_bus = nullptr; - // The AspiredVersionPolicy to use for the manager. Must be non-null. + /// The AspiredVersionPolicy to use for the manager. Must be non-null. std::unique_ptr aspired_version_policy; - // The number of threads in the thread-pool used to load servables. - // - // If set as 0, we don't use a thread-pool, and servable loads are performed - // serially in the manager's main work loop. + /// The number of threads in the thread-pool used to load servables. + /// + /// If set as 0, we don't use a thread-pool, and servable loads are + /// performed serially in the manager's main work loop. uint32 num_load_threads = 0; - // The number of threads in the thread-pool used to unload servables. - // - // If set as 0, we don't use a thread-pool, and servable unloads are - // performed serially in the manager's main work loop. + /// The number of threads in the thread-pool used to unload servables. + /// + /// If set as 0, we don't use a thread-pool, and servable unloads are + /// performed serially in the manager's main work loop. uint32 num_unload_threads = 0; - // Maximum number of times we retry loading a servable, after the first - // failure, before we give up. + /// Maximum number of times we retry loading a servable, after the first + /// failure, before we give up. uint32 max_num_load_retries = 5; - // The interval, in microseconds, between each servable load retry. If set - // negative, we don't wait. - // Default: 1 minute. + /// The interval, in microseconds, between each servable load retry. If set + /// negative, we don't wait. + /// Default: 1 minute. int64 load_retry_interval_micros = 1LL * 60 * 1000 * 1000; - // The environment to use for starting threads in the thread-pool or for - // sleeping. + /// The environment to use for starting threads in the thread-pool or for + /// sleeping. Env* env = Env::Default(); }; static Status Create(Options options, @@ -125,9 +128,11 @@ class AspiredVersionsManager : public Manager, std::vector ListAvailableServableIds() const override; - // Returns a callback to set the list of aspired versions for a particular - // servable stream, using Loaders. AspiredVersionsManager's semantics with - // respect to this callback are as follows: + /// \brief Returns a callback to set the list of aspired versions for a + /// particular servable stream, using Loaders. + // + // AspiredVersionsManager's semantics with respect to this callback are as + // follows: // // 1. OMITTING A VERSION INSTRUCTS THE MANAGER TO UNLOAD IT // diff --git a/tensorflow_serving/core/aspired_versions_manager_builder.h b/tensorflow_serving/core/aspired_versions_manager_builder.h index 9f8be093ea9..566128791ec 100644 --- a/tensorflow_serving/core/aspired_versions_manager_builder.h +++ b/tensorflow_serving/core/aspired_versions_manager_builder.h @@ -28,31 +28,31 @@ limitations under the License. namespace tensorflow { namespace serving { -// Builds an AspiredVersionsManager with options and sources connected to it. -// It takes over the ownership of the sources and the returned manager handles -// the destruction of itself and its dependencies. Both single sources and -// source/source-adapter chains are accepted, i.e. you can use sources that -// directly supply loaders (Source) or composites that -// consist of Source + some chain of SourceAdapter, ..., -// SourceAdapter<..., std::unique_ptr>. The builder connects the chain -// for you. -// -// Usage: -// ... -// AspiredVersionsManagerBuilder::Options options = ManagerOptions(); -// std::unique_ptr builder; -// TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( -// std::move(options), &builder)); -// builder->AddSource(std::move(some_source)); -// builder->AddSourceChain( -// std::move(source), std::move(source_adapter1), -// std::move(source_adapter2)); -// std::unique_ptr manager = builder->Build(); -// ... -// -// NOTE: A builder can only be used to build a single AspiredVersionsManager. -// -// This class is not thread-safe. +/// Builds an AspiredVersionsManager with options and sources connected to it. +/// It takes over the ownership of the sources and the returned manager handles +/// the destruction of itself and its dependencies. Both single sources and +/// source/source-adapter chains are accepted, i.e. you can use sources that +/// directly supply loaders (Source) or composites that +/// consist of Source + some chain of SourceAdapter, ..., +/// SourceAdapter<..., std::unique_ptr>. The builder connects the chain +/// for you. +/// +/// Usage: +/// ... +/// AspiredVersionsManagerBuilder::Options options = ManagerOptions(); +/// std::unique_ptr builder; +/// TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( +/// std::move(options), &builder)); +/// builder->AddSource(std::move(some_source)); +/// builder->AddSourceChain( +/// std::move(source), std::move(source_adapter1), +/// std::move(source_adapter2)); +/// std::unique_ptr manager = builder->Build(); +/// ... +/// +/// NOTE: A builder can only be used to build a single AspiredVersionsManager. +/// +/// This class is not thread-safe. class AspiredVersionsManagerBuilder { public: using Options = AspiredVersionsManager::Options; @@ -61,30 +61,30 @@ class AspiredVersionsManagerBuilder { ~AspiredVersionsManagerBuilder() = default; - // Connects the source to the AspiredVersionsManager being built and takes - // over its ownership. - // - // REQUIRES: Template type S be convertible to - // Source>. + /// Connects the source to the AspiredVersionsManager being built and takes + /// over its ownership. + /// + /// REQUIRES: Template type S be convertible to + /// Source>. template void AddSource(std::unique_ptr source); - // Connects a chain comprising a source and a chain of source adapters, s.t. - // the final adapter in the chain emits Loaders for the manager. The final - // adapter is connected to the manager. We take ownership of the whole chain. - // - // REQUIRES: At least one source adapter. - // - // Usage: - // builder->AddSourceChain( - // std::move(source), std::move(source_adapter1), - // std::move(source_adapter2)); + /// Connects a chain comprising a source and a chain of source adapters, s.t. + /// the final adapter in the chain emits Loaders for the manager. The final + /// adapter is connected to the manager. We take ownership of the whole chain. + /// + /// REQUIRES: At least one source adapter. + /// + /// Usage: + /// builder->AddSourceChain( + /// std::move(source), std::move(source_adapter1), + /// std::move(source_adapter2)); template void AddSourceChain(std::unique_ptr source, std::unique_ptr first_source_adapter, std::unique_ptr... remaining_source_adapters); - // Builds the AspiredVersionsManager and returns it as the Manager interface. + /// Builds the AspiredVersionsManager and returns it as the Manager interface. std::unique_ptr Build(); private: diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index 2ae0ade27c1..d60b33fc748 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -48,62 +48,64 @@ namespace test_util { class BasicManagerTestAccess; } // namespace test_util -// Helps manage the lifecycle of servables including loading, serving and -// unloading them. The manager accepts servables in the form of Loaders. -// -// We start managing a servable through one of the ManageServable* methods. You -// can go on to load the servable after this by calling LoadServable. Loading -// will also make the servable available to serve. Once you decide to unload it, -// you can call UnloadServable on it, which will make it unavailable to serve, -// then unload the servable. -// -// Servables are retained until StopManagingServable() is called. This allows a -// higher level manager with more information to decide when it's safe to forget -// about a servable. -// -// BasicManager tracks the resources (e.g. RAM) used by loaded servables, and -// only allows loading new servables that fit within the overall resource pool. -// -// BasicManager can be configured to use a thread-pool to do it's load and -// unloads. This makes the {Load,Unload}Servable() methods schedule the -// load/unloads rather than executing them synchronously. If there are more -// pending load/unloads than threads in the thread pool, they are processed in -// FIFO order. -// -// In the presence of loaders that over-estimate their servables' resource needs -// and/or only bind their servables' resources to device instances, load/unload -// concurrency can be reduced below the thread-pool size. That is because we may -// have to wait for one servable's load/unload to finish to pin down the -// resource availability for loading another servable. -// -// REQUIRES: -// 1. Order of method calls - -// ManageServable*() -> LoadServable() -> UnloadServable() -> -// StopManagingServable(). -// 2. Do not schedule concurrent load and unloads of the same servable. -// 3. Do not call load or unload multiple times on the same servable. -// -// This class is thread-safe. -// -// Example usage: -// -// const ServableId id = {kServableName, 0}; -// std::unique_ptr loader = ...; -// ... -// BasicManager manager; -// TF_CHECK_OK(manager.ManageServable( -// CreateServableData(id, std::move(loader)))); -// TF_CHECK_OK(manager.LoadServable(id)); -// -// ... -// TF_CHECK_OK(manager.GetServableHandle( -// ServableRequest::Latest(kServableName), &handle)); -// ... -// -// TF_CHECK_OK(manager.UnloadServable(id)); -// TF_CHECK_OK(manager.StopManagingServable(id)); +/// Helps manage the lifecycle of servables including loading, serving and +/// unloading them. The manager accepts servables in the form of Loaders. +/// +/// We start managing a servable through one of the ManageServable* methods. You +/// can go on to load the servable after this by calling LoadServable. Loading +/// will also make the servable available to serve. Once you decide to unload +/// it, you can call UnloadServable on it, which will make it unavailable to +/// serve, then unload the servable. +/// +/// Servables are retained until StopManagingServable() is called. This allows a +/// higher level manager with more information to decide when it's safe to +/// forget about a servable. +/// +/// BasicManager tracks the resources (e.g. RAM) used by loaded servables, and +/// only allows loading new servables that fit within the overall resource pool. +/// +/// BasicManager can be configured to use a thread-pool to do it's load and +/// unloads. This makes the {Load,Unload}Servable() methods schedule the +/// load/unloads rather than executing them synchronously. If there are more +/// pending load/unloads than threads in the thread pool, they are processed in +/// FIFO order. +/// +/// In the presence of loaders that over-estimate their servables' resource +/// needs and/or only bind their servables' resources to device instances, +/// load/unload concurrency can be reduced below the thread-pool size. That is +/// because we may have to wait for one servable's load/unload to finish to pin +/// down the resource availability for loading another servable. +/// +/// REQUIRES: +/// 1. Order of method calls - +/// ManageServable*() -> LoadServable() -> UnloadServable() -> +/// StopManagingServable(). +/// 2. Do not schedule concurrent load and unloads of the same servable. +/// 3. Do not call load or unload multiple times on the same servable. +/// +/// This class is thread-safe. +/// +/// Example usage: +/// +/// const ServableId id = {kServableName, 0}; +/// std::unique_ptr loader = ...; +/// ... +/// BasicManager manager; +/// TF_CHECK_OK(manager.ManageServable( +/// CreateServableData(id, std::move(loader)))); +/// TF_CHECK_OK(manager.LoadServable(id)); +/// +/// ... +/// TF_CHECK_OK(manager.GetServableHandle( +/// ServableRequest::Latest(kServableName), &handle)); +/// ... +/// +/// TF_CHECK_OK(manager.UnloadServable(id)); +/// TF_CHECK_OK(manager.StopManagingServable(id)); class BasicManager : public Manager { public: + /// Config options and pluggable objects that will be used by the + /// BasicManager. struct Options { // The resource tracker to use while managing servable resources. Optional. // If left as nullptr, we do not validate servable resource usage. @@ -139,8 +141,8 @@ class BasicManager : public Manager { }; static Status Create(Options options, std::unique_ptr* manager); - // If configured to use a load/unload thread-pool, waits until all scheduled - // loads and unloads have finished and then destroys the set of threads. + /// If configured to use a load/unload thread-pool, waits until all scheduled + /// loads and unloads have finished and then destroys the set of threads. ~BasicManager() override; std::vector ListAvailableServableIds() const override; @@ -152,100 +154,100 @@ class BasicManager : public Manager { std::map> GetAvailableUntypedServableHandles() const override; - // Starts managing the servable. - // - // Returns an error if given a servable that is already being managed. - // - // If 'servable' is in an error state, this method does *not* return an error. - // Instead, the manager accepts the servable, puts it in state kError (with a - // notification sent to the event bus), and then immediately stops managing - // it. This behavior facilitates uniform handling of errors that occur in - // sources (e.g. invalid file path to servable data) and ones that occur in - // the manager (e.g. insufficient resources to load servable). + /// Starts managing the servable. + /// + /// Returns an error if given a servable that is already being managed. + /// + /// If 'servable' is in an error state, this method does *not* return an + /// error. Instead, the manager accepts the servable, puts it in state kError + /// (with a notification sent to the event bus), and then immediately stops + /// managing it. This behavior facilitates uniform handling of errors that + /// occur in sources (e.g. invalid file path to servable data) and ones that + /// occur in the manager (e.g. insufficient resources to load servable). Status ManageServable(ServableData> servable); - // Similar to the above method, but callers, usually other managers built on - // top of this one, can associate additional state with the servable. - // Additional state may be ACL or lifetime metadata for that servable. The - // ownership of the state is transferred to this class. + /// Similar to the above method, but callers, usually other managers built on + /// top of this one, can associate additional state with the servable. + /// Additional state may be ACL or lifetime metadata for that servable. The + /// ownership of the state is transferred to this class. template Status ManageServableWithAdditionalState( ServableData> servable, std::unique_ptr additional_state); - // Tells the manager to stop managing this servable. Requires that the - // servable is currently being managed and that its state is one of {kNew, - // kError, kDisabled}. + /// Tells the manager to stop managing this servable. Requires that the + /// servable is currently being managed and that its state is one of {kNew, + /// kError, kDisabled}. Status StopManagingServable(const ServableId& id); - // Returns the names of all the servables managed by this manager. The names - // will be duplicate-free and not in any particular order. + /// @return the names of all the servables managed by this manager. The names + /// will be duplicate-free and not in any particular order. std::vector GetManagedServableNames() const; - // Returns the state snapshots of all the servables of a particular stream, - // managed by this manager. - // - // T is the additional-state type, if any. + /// @return the state snapshots of all the servables of a particular stream, + /// managed by this manager. + /// + /// T is the additional-state type, if any. template std::vector> GetManagedServableStateSnapshots( const string& servable_name) const; - // Returns the state snapshot of a particular servable-id managed by this - // manager if available. - // - // REQUIRES: This manager should have been managing this servable already, - // else we return nullopt. + /// @return the state snapshot of a particular servable-id managed by this + /// manager if available. + /// + /// REQUIRES: This manager should have been managing this servable already, + /// else we return nullopt. template optional> GetManagedServableStateSnapshot( const ServableId& id); - // Returns the additional state for the servable. Returns nullptr if there is - // no additional state setup or if there is a type mismatch between what was - // setup and what is being asked for. - // - // REQUIRES: This manager should have been managing this servable already, - // else we return nullptr. + /// @return the additional state for the servable. Returns nullptr if there is + /// no additional state setup or if there is a type mismatch between what was + /// setup and what is being asked for. + /// + /// REQUIRES: This manager should have been managing this servable already, + /// else we return nullptr. template T* GetAdditionalServableState(const ServableId& id); - // Callback called at the end of {Load,Unload}Servable(). We pass in the - // status of the operation to the callback. + /// Callback called at the end of {Load,Unload}Servable(). We pass in the + /// status of the operation to the callback. using DoneCallback = std::function; - // Loads the servable with this id, and updates the serving map too. Calls - // 'done_callback' with ok iff the servable was loaded successfully, else - // returns an error status. - // - // If using a thread-pool, this method transitions the servable harness to - // kLoading state, schedules the load and returns, otherwise it - // completes the load before returning. - // - // REQUIRES: This manager should have been managing this servable already, for - // it to be loaded, else we call 'done_callback' with an error status. Do not - // call this multiple times on the same servable. Only one of those will - // succeed and the rest will fail with an error status. + /// Loads the servable with this id, and updates the serving map too. Calls + /// 'done_callback' with ok iff the servable was loaded successfully, else + /// returns an error status. + /// + /// If using a thread-pool, this method transitions the servable harness to + /// kLoading state, schedules the load and returns, otherwise it + /// completes the load before returning. + /// + /// REQUIRES: This manager should have been managing this servable already, + /// for it to be loaded, else we call 'done_callback' with an error status. Do + /// not call this multiple times on the same servable. Only one of those will + /// succeed and the rest will fail with an error status. void LoadServable(const ServableId& id, DoneCallback done_callback); - // Cancels retrying the servable load during LoadServable(). Does nothing if - // the servable isn't managed. - // - // If the retries are cancelled, the servable goes into a state dependent on - // the last Load() called on it. If the last Load() was successful, it will be - // in state kReady, else in kError. + /// Cancels retrying the servable load during LoadServable(). Does nothing if + /// the servable isn't managed. + /// + /// If the retries are cancelled, the servable goes into a state dependent on + /// the last Load() called on it. If the last Load() was successful, it will + /// be in state kReady, else in kError. void CancelLoadServableRetry(const ServableId& id); - // Unloads the servable with this id, and updates the serving map too. Calls - // 'done_callback' with ok iff the servable was unloaded successfully, else - // returns an error status. - // - // If using a thread-pool, this method transitions the servable harness to - // kQuiescing state, schedules the unload and returns, otherwise it completes - // the unload before returning. - // - // REQUIRES: This manager should have loaded and made this servable available, - // for it to be unloaded, else calls 'done_callback' with an error status. Do - // not call this multiple times on the same servable. Only one of those will - // succeed and the rest will fail with an error status. + /// Unloads the servable with this id, and updates the serving map too. Calls + /// 'done_callback' with ok iff the servable was unloaded successfully, else + /// returns an error status. + /// + /// If using a thread-pool, this method transitions the servable harness to + /// kQuiescing state, schedules the unload and returns, otherwise it completes + /// the unload before returning. + /// + /// REQUIRES: This manager should have loaded and made this servable + /// available, for it to be unloaded, else calls 'done_callback' with an error + /// status. Do not call this multiple times on the same servable. Only one of + /// those will succeed and the rest will fail with an error status. void UnloadServable(const ServableId& id, DoneCallback done_callback); private: diff --git a/tensorflow_serving/core/caching_manager.h b/tensorflow_serving/core/caching_manager.h index 434d0dfc762..eef6dd09483 100644 --- a/tensorflow_serving/core/caching_manager.h +++ b/tensorflow_serving/core/caching_manager.h @@ -32,18 +32,20 @@ namespace test_util { class CachingManagerTestAccess; } // namespace test_util -// A manager that manages and loads servables on-demand. Upon receiving the -// request for a servable name and optional version, the manager checks if it -// already has the requested servable loaded. If not, it initiates the load -// operation and then serves the request. -// -// The manager blocks on the load operation and returns the handle when the -// servable has been loaded, or upon error. +/// A manager that manages and loads servables on-demand. Upon receiving the +/// request for a servable name and optional version, the manager checks if it +/// already has the requested servable loaded. If not, it initiates the load +/// operation and then serves the request. +/// +/// The manager blocks on the load operation and returns the handle when the +/// servable has been loaded, or upon error. // // TODO(b/25449742): Add support for evictions of loaded servables from the // caching-manager. class CachingManager : public Manager { public: + /// Config options and pluggable objects that will be used by the + /// CachingManager. struct Options { // The resource tracker to use while managing servable resources. Optional. // If left as nullptr, we do not validate servable resource usage. @@ -76,19 +78,19 @@ class CachingManager : public Manager { Env* env = Env::Default(); }; - // An abstraction for a loader-factory to map from a servable request to the - // corresponding loader. + /// An abstraction for a loader-factory to map from a servable request to the + /// corresponding loader. class LoaderFactory { public: virtual ~LoaderFactory() = default; - // Creates servable data consisting of the loader corresponding to the - // servable-id. Any errors can be reported by embedding them in the returned - // ServableData item. + /// Creates servable data consisting of the loader corresponding to the + /// servable-id. Any errors can be reported by embedding them in the + /// returned ServableData item. virtual ServableData> CreateLoader( const ServableId& servable_id) = 0; - // Returns the latest version corresponding to the servable name. + /// Returns the latest version corresponding to the servable name. virtual int64 GetLatestVersion(const string& servable_name) const = 0; }; @@ -156,9 +158,9 @@ class CachingManager : public Manager { TF_DISALLOW_COPY_AND_ASSIGN(CachingManager); }; -// A simple LoaderFactory that looks for a servable at a path formed by -// concatenating a fixed path prefix with the servable's name. It assumes that -// a given servable only has one version, namely version 0. +/// A simple LoaderFactory that looks for a servable at a path formed by +/// concatenating a fixed path prefix with the servable's name. It assumes that +/// a given servable only has one version, namely version 0. class PathPrefixLoaderFactory : public CachingManager::LoaderFactory { public: PathPrefixLoaderFactory(const string& path_prefix, diff --git a/tensorflow_serving/core/loader.h b/tensorflow_serving/core/loader.h index 1b5b4347480..0e379932b30 100644 --- a/tensorflow_serving/core/loader.h +++ b/tensorflow_serving/core/loader.h @@ -27,102 +27,105 @@ limitations under the License. namespace tensorflow { namespace serving { -// A standardized abstraction for an object that manages the lifecycle of a -// servable, including loading and unloading it. Servables are arbitrary objects -// that serve algorithms or data that often, though not necessarily, use a -// machine-learned model. -// -// A Loader for a servable object represents one instance of a stream of -// servable versions, all sharing a common name (e.g. "my_servable") and -// increasing version numbers, typically representing updated model parameters -// learned from fresh training data. -// -// A Loader should start in an unloaded state, meaning that no work has been -// done to prepare to perform operations. A typical instance that has not yet -// been loaded contains merely a pointer to a location from which its data can -// be loaded (e.g. a file-system path or network location). Construction and -// destruction of instances should be fairly cheap. Expensive initialization -// operations should be done in Load(). -// -// Subclasses may optionally store a pointer to the Source that originated it, -// for accessing state shared across multiple servable objects in a given -// servable stream. -// -// Implementations need to ensure that the methods they expose are thread-safe, -// or carefully document and/or coordinate their thread-safety properties with -// their clients to ensure correctness. -// Servables do not need to worry about concurrent execution of Load()/Unload() -// as the caller will ensure that does not happen. +/// A standardized abstraction for an object that manages the lifecycle of a +/// servable, including loading and unloading it. Servables are arbitrary +/// objects that serve algorithms or data that often, though not necessarily, +/// use a machine-learned model. +/// +/// A Loader for a servable object represents one instance of a stream of +/// servable versions, all sharing a common name (e.g. "my_servable") and +/// increasing version numbers, typically representing updated model parameters +/// learned from fresh training data. +/// +/// A Loader should start in an unloaded state, meaning that no work has been +/// done to prepare to perform operations. A typical instance that has not yet +/// been loaded contains merely a pointer to a location from which its data can +/// be loaded (e.g. a file-system path or network location). Construction and +/// destruction of instances should be fairly cheap. Expensive initialization +/// operations should be done in Load(). +/// +/// Subclasses may optionally store a pointer to the Source that originated it, +/// for accessing state shared across multiple servable objects in a given +/// servable stream. +/// +/// Implementations need to ensure that the methods they expose are thread-safe, +/// or carefully document and/or coordinate their thread-safety properties with +/// their clients to ensure correctness. +/// Servables do not need to worry about concurrent execution of Load()/Unload() +/// as the caller will ensure that does not happen. class Loader { public: - // The destructor will never be called on a Loader whose servable is currently - // loaded, i.e. between (successful) calls to Load() and Unload(). + /// The destructor will never be called on a Loader whose servable is + /// currently loaded, i.e. between (successful) calls to Load() and Unload(). virtual ~Loader() = default; - // Returns an estimate of the resources the servable will consume once loaded. - // If the servable has already been loaded, returns an estimate of the actual - // resource usage. - // - // IMPORTANT: This method's implementation must obey following requirements, - // which enable the serving system to reason correctly about which servables - // can be loaded safely: - // 1. The estimate must represent an upper bound on the actual value. - // 2. Prior to load, the estimate may include resources that are not bound - // to any specific device instance, e.g. RAM on one of the two GPUs. - // 3. While loaded, for any devices with multiple instances (e.g. two GPUs), - // the estimate must specify the instance to which each resource is bound. - // 4. The estimate must be monotonically non-increasing, i.e. it cannot - // increase over time. + /// Estimates the resources a servable will use. + /// + /// IMPORTANT: This method's implementation must obey following requirements, + /// which enable the serving system to reason correctly about which servables + /// can be loaded safely: + /// 1. The estimate must represent an upper bound on the actual value. + /// 2. Prior to load, the estimate may include resources that are not bound + /// to any specific device instance, e.g. RAM on one of the two GPUs. + /// 3. While loaded, for any devices with multiple instances (e.g. two GPUs), + /// the estimate must specify the instance to which each resource is + /// bound. + /// 4. The estimate must be monotonically non-increasing, i.e. it cannot + /// increase over time. + /// + /// @return an estimate of the resources the servable will consume once + /// loaded. If the servable has already been loaded, returns an estimate of + /// the actual resource usage. virtual Status EstimateResources(ResourceAllocation* estimate) const = 0; - // Fetches any data that needs to be loaded before using the servable returned - // by servable(). May use no more resources than the estimate reported by - // EstimateResources(). + /// Fetches any data that needs to be loaded before using the servable + /// returned by servable(). May use no more resources than the estimate + /// reported by EstimateResources(). virtual Status Load() = 0; - // Frees any resources allocated during Load() (except perhaps for resources - // shared across servables that are still needed for other active ones). - // The loader does not need to return to the "new" state (i.e. Load() cannot - // be called after Unload()). + /// Frees any resources allocated during Load() (except perhaps for resources + /// shared across servables that are still needed for other active ones). + /// The loader does not need to return to the "new" state (i.e. Load() cannot + /// be called after Unload()). virtual void Unload() = 0; - // Returns an opaque interface to the underlying servable object. - // The caller should know the precise type of the interface in order to make - // actual use of it. For example: - // - // CustomLoader implementation: - // - // class CustomLoader : public Loader { - // public: - // ... - // Status Load() override { - // servable_ = ...; - // } - // - // AnyPtr servable() override { return servable_; } - // - // private: - // CustomServable* servable_ = nullptr; - // }; - // - // Serving user request: - // - // ServableHandle handle = ... - // CustomServable* servable = handle.get(); - // servable->... - // - // If servable() is called after successful Load() and before Unload(), it - // returns a valid, non-null AnyPtr object. If called before a successful - // Load() call or after Unload(), it returns null AnyPtr. + /// Returns an opaque interface to the underlying servable object. + /// The caller should know the precise type of the interface in order to make + /// actual use of it. For example: + /// + /// CustomLoader implementation: + /// + /// class CustomLoader : public Loader { + /// public: + /// ... + /// Status Load() override { + /// servable_ = ...; + /// } + /// + /// AnyPtr servable() override { return servable_; } + /// + /// private: + /// CustomServable* servable_ = nullptr; + /// }; + /// + /// Serving user request: + /// + /// ServableHandle handle = ... + /// CustomServable* servable = handle.get(); + /// servable->... + /// + /// If servable() is called after successful Load() and before Unload(), it + /// returns a valid, non-null AnyPtr object. If called before a successful + /// Load() call or after Unload(), it returns null AnyPtr. virtual AnyPtr servable() = 0; }; -// A Loader that is oblivious to resources. Its EstimateResources() method -// returns 0, thus effectively disabling resource-based safety checks in the -// serving system. -// -// Loaders that are experimental, or run in environments that do not need the -// resource safety checks, can subclass ResourceUnsafeLoader instead of Loader. +/// A Loader that is oblivious to resources. Its EstimateResources() method +/// returns 0, thus effectively disabling resource-based safety checks in the +/// serving system. +/// +/// Loaders that are experimental, or run in environments that do not need the +/// resource safety checks, can subclass ResourceUnsafeLoader instead of Loader. class ResourceUnsafeLoader : public Loader { public: Status EstimateResources(ResourceAllocation* estimate) const final { diff --git a/tensorflow_serving/core/loader_harness.h b/tensorflow_serving/core/loader_harness.h index b65b391574c..34180354cf2 100644 --- a/tensorflow_serving/core/loader_harness.h +++ b/tensorflow_serving/core/loader_harness.h @@ -34,72 +34,74 @@ namespace serving { template struct ServableStateSnapshot; -// LoaderHarness is a widget the Manager uses to hold on to and talk to a Loader -// while it owns it. It tracks the overall state of a Servable such that Manager -// can determine which state transitions to make at what times. -// -// A manager implementation can also add some additional state with each -// harness. For example, a manager could put ACL or lifecycle metadata here. The -// ownership is maintained by the harness. -// -// This class is thread-safe. +/// LoaderHarness is a widget the Manager uses to hold on to and talk to a +/// Loader while it owns it. It tracks the overall state of a Servable such that +/// Manager can determine which state transitions to make at what times. +/// +/// A manager implementation can also add some additional state with each +/// harness. For example, a manager could put ACL or lifecycle metadata here. +/// The ownership is maintained by the harness. +/// +/// This class is thread-safe. class LoaderHarness final { public: - // State of the underlying servable, from the perspective of the LoaderHarness - // and for the purpose of communication between it and a Manager. Not - // equivalent to the semantic servable states in servable_state.h. - // - // Valid transitions: - // kNew-->kLoading-->kReady-->kQuiescing-->kQuiesced-->kUnloading-->kDisabled - // as well as: any_state-->kError. + /// State of the underlying servable, from the perspective of the + /// LoaderHarness and for the purpose of communication between it and a + /// Manager. Not equivalent to the semantic servable states in + /// servable_state.h. + /// + /// Valid transitions: + /// kNew-->kLoading-->kReady-->kQuiescing-->kQuiesced-->kUnloading-->kDisabled + /// as well as: any_state-->kError. enum class State { - // Initial state. + /// Initial state. kNew, - // The manager has been requested to load this servable. + /// The manager has been requested to load this servable. kLoadRequested, - // The servable has been approved for loading, e.g. resources have been set - // aside for it. + /// The servable has been approved for loading, e.g. resources have been set + /// aside for it. kLoadApproved, - // 'loader_->Load()' has been called. + /// 'loader_->Load()' has been called. kLoading, - // 'loader_->Load()' has succeeded. + /// 'loader_->Load()' has succeeded. kReady, - // The manager has been requested to unload this servable. + /// The manager has been requested to unload this servable. kUnloadRequested, - // The servable is going to be made unavailable for serving. + /// The servable is going to be made unavailable for serving. kQuiescing, - // The servable has been made unavailable for serving. + /// The servable has been made unavailable for serving. kQuiesced, - // 'loader_->Unload()' has been called. + /// 'loader_->Unload()' has been called. kUnloading, - // 'loader_->Unload()' has finished. + /// 'loader_->Unload()' has finished. kDisabled, - // An error has occurred, either during 'loader_->Load()' or outside of the - // harness (and was reported to the harness via a call to Error()). + /// An error has occurred, either during 'loader_->Load()' or outside of the + /// harness (and was reported to the harness via a call to Error()). kError }; + /// Options to configure a LoaderHarness. struct Options { Options() {} - // Maximum number of times we retry loading a servable, after the first - // failure, before we give up. + /// Maximum number of times we retry loading a servable, after the first + /// failure, before we give up. uint32 max_num_load_retries = 0; - // The interval, in microseconds, between each servable load retry. + /// The interval, in microseconds, between each servable load retry. uint64 load_retry_interval_micros = 0; - // An (optional) function to call upon transitioning to state kError. + /// An (optional) function to call upon transitioning to state kError. std::function error_callback; }; @@ -107,8 +109,8 @@ class LoaderHarness final { LoaderHarness(const ServableId& id, std::unique_ptr loader, const Options& options = Options()); - // Constructor to create a harness with additional state, which a manager - // needs. + /// Constructor to create a harness with additional state, which a manager + /// needs. template LoaderHarness(const ServableId& id, std::unique_ptr loader, std::unique_ptr additional_state, @@ -118,95 +120,95 @@ class LoaderHarness final { additional_state_(std::move(additional_state)), options_(options) {} - // Legal to destruct iff current state is one of kNew, kDisabled or kError. - // Check-fails if violated. + /// Legal to destruct iff current state is one of kNew, kDisabled or kError. + /// Check-fails if violated. ~LoaderHarness(); - // Returns the identifier of underlying Servable. + /// Returns the identifier of underlying Servable. ServableId id() const { return id_; } - // Returns the current state of underlying Servable. + /// Returns the current state of underlying Servable. State state() const LOCKS_EXCLUDED(mu_); - // Returns a pointer to the wrapped loader. - // Ownership remains with this class. + /// Returns a pointer to the wrapped loader. + /// Ownership remains with this class. Loader* loader() const { return loader_.get(); } - // Returns the current overall state snapshot of the underlying Servable. + /// Returns the current overall state snapshot of the underlying Servable. template ServableStateSnapshot loader_state_snapshot() const LOCKS_EXCLUDED(mu_); - // Transitions the state of the harness to kLoadRequested iff its current - // state is kNew. The test-and-change is done transactionally, so this method - // can be used to ensure that at most one Load() request can proceed. + /// Transitions the state of the harness to kLoadRequested iff its current + /// state is kNew. The test-and-change is done transactionally, so this method + /// can be used to ensure that at most one Load() request can proceed. Status LoadRequested() LOCKS_EXCLUDED(mu_); - // Transitions to kLoadApproved. - // - // REQUIRES: State is kLoadRequested when called. Otherwise DCHECK-fails, - // transitions to state kError and invokes 'options_.error_callback'. + /// Transitions to kLoadApproved. + /// + /// REQUIRES: State is kLoadRequested when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. Status LoadApproved() LOCKS_EXCLUDED(mu_); - // Transitions to kLoading, delegates to Servable::Load(), then transitions - // either to kReady if Load() succeeds, or to kError (and invokes 'options_. - // error_callback') if Load() fails. This call may take a long time. - // - // We retry the Servable::Load() according to the options set during - // construction of this class. We stop retrying and give up if 1. we have - // reached max_num_load_retries or, 2. if cancel_load() is set to true. - // - // REQUIRES: State is kLoadApproved when called. Otherwise DCHECK-fails, - // transitions to state kError and invokes 'options_.error_callback'. + /// Transitions to kLoading, delegates to Servable::Load(), then transitions + /// either to kReady if Load() succeeds, or to kError (and invokes 'options_. + /// error_callback') if Load() fails. This call may take a long time. + /// + /// We retry the Servable::Load() according to the options set during + /// construction of this class. We stop retrying and give up if 1. we have + /// reached max_num_load_retries or, 2. if cancel_load() is set to true. + /// + /// REQUIRES: State is kLoadApproved when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. Status Load() LOCKS_EXCLUDED(mu_); - // Transitions the state of the harness to kUnloadRequested iff its current - // state is kReady. The test-and-change is done transactionally, so this - // method can be used to ensure that at most one Load() request can proceed. + /// Transitions the state of the harness to kUnloadRequested iff its current + /// state is kReady. The test-and-change is done transactionally, so this + /// method can be used to ensure that at most one Load() request can proceed. Status UnloadRequested() LOCKS_EXCLUDED(mu_); - // Cancels retrying the load of the servable. This is best-effort, and does - // not preempt a Load() which is already happening, only subsequent calls. - // - // If the retries are cancelled, the servable goes into a state dependent on - // the last Load() called on it. If the last Load() was successful, it will be - // in state kReady, else in kError. + /// Cancels retrying the load of the servable. This is best-effort, and does + /// not preempt a Load() which is already happening, only subsequent calls. + /// + /// If the retries are cancelled, the servable goes into a state dependent on + /// the last Load() called on it. If the last Load() was successful, it will + /// be in state kReady, else in kError. void set_cancel_load_retry(bool value) LOCKS_EXCLUDED(mu_); bool cancel_load_retry() LOCKS_EXCLUDED(mu_); - // Transitions to kUnloading, delegates to Servable::Unload(), then - // transitions to kDisabled when Unload() is done. - // - // REQUIRES: State is kQuiesced when called. Otherwise DCHECK-fails, - // transitions to state kError and invokes 'options_.error_callback'. + /// Transitions to kUnloading, delegates to Servable::Unload(), then + /// transitions to kDisabled when Unload() is done. + /// + /// REQUIRES: State is kQuiesced when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. Status Unload() LOCKS_EXCLUDED(mu_); - // Transitions the state to kQuiescing, which means that we would like to not - // give out any more handles to this servable. - // - // REQUIRES: State is kUnloadRequested when called. Otherwise DCHECK-fails, - // transitions to state kError and invokes 'options_.error_callback'. + /// Transitions the state to kQuiescing, which means that we would like to not + /// give out any more handles to this servable. + /// + /// REQUIRES: State is kUnloadRequested when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. Status StartQuiescing() LOCKS_EXCLUDED(mu_); - // Transitions the state to kQuiesced, which means that there are no more live - // handles to this servable available in the frontend. At this point, we can - // unload this object. - // - // REQUIRES: State is kQuiescing when called. Otherwise DCHECK-fails, - // transitions to state kError and invokes 'options_.error_callback'. + /// Transitions the state to kQuiesced, which means that there are no more + /// live handles to this servable available in the frontend. At this point, we + /// can unload this object. + /// + /// REQUIRES: State is kQuiescing when called. Otherwise DCHECK-fails, + /// transitions to state kError and invokes 'options_.error_callback'. Status DoneQuiescing() LOCKS_EXCLUDED(mu_); - // Transitions the state to kError and invokes 'options_.error_callback'. + /// Transitions the state to kError and invokes 'options_.error_callback'. void Error(const Status& status) LOCKS_EXCLUDED(mu_); - // Whether anything has gone wrong with this servable. If state is kError, - // this will be non-OK. If not OK, the error could be something that occurred - // in a Source or SourceAdapter, in the Loader, in the Manager, or elsewhere. - // All errors pertaining to the servable are reported here, regardless of - // origin. + /// Whether anything has gone wrong with this servable. If state is kError, + /// this will be non-OK. If not OK, the error could be something that occurred + /// in a Source or SourceAdapter, in the Loader, in the Manager, or elsewhere. + /// All errors pertaining to the servable are reported here, regardless of + /// origin. Status status() const LOCKS_EXCLUDED(mu_); - // Gets the additional state. Returns nullptr if the type mismatches or if it - // wasn't set. + /// Gets the additional state. Returns nullptr if the type mismatches or if it + /// wasn't set. template T* additional_state() { return additional_state_.get(); diff --git a/tensorflow_serving/core/manager.h b/tensorflow_serving/core/manager.h index 6db9854d741..4436818bf91 100644 --- a/tensorflow_serving/core/manager.h +++ b/tensorflow_serving/core/manager.h @@ -33,8 +33,9 @@ limitations under the License. namespace tensorflow { namespace serving { -// A query for a specific loaded servable object. The request can either specify -// a specific version number, or simply opt to use the latest loaded version. +/// A query for a specific loaded servable object. The request can either +/// specify a specific version number, or simply opt to use the latest loaded +/// version. struct ServableRequest { // Initialization factories, for convenience and readability. static ServableRequest Specific(const string& name, const int64 version); @@ -51,30 +52,30 @@ struct ServableRequest { string DebugString() const; }; -// Manager is responsible for loading, unloading, lookup and lifetime -// management of all Servable objects via their Loaders. +/// Manager is responsible for loading, unloading, lookup and lifetime +/// management of all Servable objects via their Loaders. class Manager { public: virtual ~Manager() = default; - // Gets a list of all available servable ids, i.e. each of these can - // be retrieved using GetServableHandle. + /// Gets a list of all available servable ids, i.e. each of these can + /// be retrieved using GetServableHandle. virtual std::vector ListAvailableServableIds() const = 0; - // Returns a map of all the currently available servables of a particular type - // T. The map is from the servable's id to its corresponding handle. - // - // IMPORTANT: The caller should not hold onto the handles for a long time, - // because holding them will delay servable loading and unloading. + /// Returns a map of all the currently available servables of a particular + /// type T. The map is from the servable's id to its corresponding handle. + /// + /// IMPORTANT: The caller should not hold onto the handles for a long time, + /// because holding them will delay servable loading and unloading. template std::map> GetAvailableServableHandles() const; - // Returns a ServableHandle given a ServableRequest. Returns error if no such - // Servable is available -- e.g. not yet loaded, has been quiesced/unloaded, - // etc. Callers may assume that an OK status indicates a non-null handle. - // - // IMPORTANT: The caller should not hold onto the handles for a long time, - // because holding them will delay servable loading and unloading. + /// Returns a ServableHandle given a ServableRequest. Returns error if no such + /// Servable is available -- e.g. not yet loaded, has been quiesced/unloaded, + /// etc. Callers may assume that an OK status indicates a non-null handle. + /// + /// IMPORTANT: The caller should not hold onto the handles for a long time, + /// because holding them will delay servable loading and unloading. template Status GetServableHandle(const ServableRequest& request, ServableHandle* const handle); diff --git a/tensorflow_serving/core/servable_handle.h b/tensorflow_serving/core/servable_handle.h index e77de60c523..27bab01ce97 100644 --- a/tensorflow_serving/core/servable_handle.h +++ b/tensorflow_serving/core/servable_handle.h @@ -28,11 +28,11 @@ limitations under the License. namespace tensorflow { namespace serving { -// A non-templatized handle to a servable, used internally in the -// Manager to retrieve a type-erased servable object from the Loader. -// The handle keeps the underlying object alive as long as the handle is alive. -// The frontend should not hold onto it for a long time, because holding it can -// delay servable reloading. +/// A non-templatized handle to a servable, used internally in the +/// Manager to retrieve a type-erased servable object from the Loader. +/// The handle keeps the underlying object alive as long as the handle is alive. +/// The frontend should not hold onto it for a long time, because holding it can +/// delay servable reloading. class UntypedServableHandle { public: virtual ~UntypedServableHandle() = default; @@ -42,34 +42,34 @@ class UntypedServableHandle { virtual AnyPtr servable() = 0; }; -// A smart pointer to the underlying servable object T retrieved from the -// Loader. Frontend code gets these handles from the ServableManager. The -// handle keeps the underlying object alive as long as the handle is alive. The -// frontend should not hold onto it for a long time, because holding it can -// delay servable reloading. -// -// The T returned from the handle is generally shared among multiple requests, -// which means any mutating changes made to T must preserve correctness -// vis-a-vis the application logic. Moreover, in the presence of multiple -// request threads, thread-safe usage of T must be ensured. -// -// T is expected to be a value type, and is internally stored as a pointer. -// Using a pointer type for T will fail to compile, since it would be a mistake -// to do so in most situations. -// -// Example Use: -// // Define or use an existing servable: -// class MyServable { -// public: -// void MyMethod(); -// }; -// -// // Get your handle from a manager. -// ServableHandle handle; -// TF_RETURN_IF_ERROR(manager->GetServableHandle(id, &handle)); -// -// // Use your handle as a smart-pointer: -// handle->MyMethod(); +/// A smart pointer to the underlying servable object T retrieved from the +/// Loader. Frontend code gets these handles from the ServableManager. The +/// handle keeps the underlying object alive as long as the handle is alive. The +/// frontend should not hold onto it for a long time, because holding it can +/// delay servable reloading. +/// +/// The T returned from the handle is generally shared among multiple requests, +/// which means any mutating changes made to T must preserve correctness +/// vis-a-vis the application logic. Moreover, in the presence of multiple +/// request threads, thread-safe usage of T must be ensured. +/// +/// T is expected to be a value type, and is internally stored as a pointer. +/// Using a pointer type for T will fail to compile, since it would be a mistake +/// to do so in most situations. +/// +/// Example Use: +/// // Define or use an existing servable: +/// class MyServable { +/// public: +/// void MyMethod(); +/// }; +/// +/// // Get your handle from a manager. +/// ServableHandle handle; +/// TF_RETURN_IF_ERROR(manager->GetServableHandle(id, &handle)); +/// +/// // Use your handle as a smart-pointer: +/// handle->MyMethod(); template class ServableHandle { public: @@ -77,7 +77,7 @@ class ServableHandle { "Servables are implicitly passed as pointers, please use T " "instead of T*."); - // ServableHandle is null by default. + /// ServableHandle is null by default. ServableHandle() = default; const ServableId& id() const { return untyped_handle_->id(); } @@ -108,8 +108,8 @@ class ServableHandle { T* servable_ = nullptr; }; -// An implementation of UntypedServableHandle using shared_ptr to do -// ref-counting on the Loader that owns the Servable. +/// An implementation of UntypedServableHandle using shared_ptr to do +/// ref-counting on the Loader that owns the Servable. class SharedPtrHandle final : public UntypedServableHandle { public: ~SharedPtrHandle() override = default; diff --git a/tensorflow_serving/core/servable_state_monitor.h b/tensorflow_serving/core/servable_state_monitor.h index f496d2c19bf..bd9e256c743 100644 --- a/tensorflow_serving/core/servable_state_monitor.h +++ b/tensorflow_serving/core/servable_state_monitor.h @@ -33,15 +33,15 @@ limitations under the License. namespace tensorflow { namespace serving { -// A utility that listens to an EventBus, and keeps track of the -// state of each servable mentioned on the bus. The intended use case is to -// track the states of servables in a Manager. -// -// Offers an interface for querying the servable states. It may be useful as the -// basis for dashboards, as well as for testing a manager. -// -// IMPORTANT: You must create this monitor before arranging for events to be -// published on the event bus, e.g. giving the event bus to a Manager. +/// A utility that listens to an EventBus, and keeps track of the +/// state of each servable mentioned on the bus. The intended use case is to +/// track the states of servables in a Manager. +/// +/// Offers an interface for querying the servable states. It may be useful as +/// the basis for dashboards, as well as for testing a manager. +/// +/// IMPORTANT: You must create this monitor before arranging for events to be +/// published on the event bus, e.g. giving the event bus to a Manager. class ServableStateMonitor { public: struct ServableStateAndTime { @@ -49,14 +49,14 @@ class ServableStateMonitor { ServableStateAndTime(ServableState servable_state, const uint64 event_time) : state(std::move(servable_state)), event_time_micros(event_time) {} - // State of the servable. + /// State of the servable. ServableState state; - // Time at which servable state event was published. + /// Time at which servable state event was published. uint64 event_time_micros; - // Returns a string representation of this struct useful for debugging or - // logging. + /// Returns a string representation of this struct useful for debugging or + /// logging. string DebugString() const; }; @@ -69,8 +69,8 @@ class ServableStateMonitor { struct Options { Options() {} - // Upper bound for the number of events captured in the bounded log. If set - // to 0, logging is disabled. + /// Upper bound for the number of events captured in the bounded log. If set + /// to 0, logging is disabled. uint64 max_count_log_events = 0; }; using BoundedLog = std::deque; @@ -79,54 +79,55 @@ class ServableStateMonitor { const Options& options = Options()); virtual ~ServableStateMonitor(); - // Returns the current state of one servable, or nullopt if that servable is - // not being tracked. + /// Returns the current state of one servable, or nullopt if that servable is + /// not being tracked. optional GetState(const ServableId& servable_id) const LOCKS_EXCLUDED(mu_); - // Returns the current state and time of one servable, or nullopt if that - // servable is not being tracked. + /// Returns the current state and time of one servable, or nullopt if that + /// servable is not being tracked. optional GetStateAndTime( const ServableId& servable_id) const LOCKS_EXCLUDED(mu_); - // Returns the current states of all tracked versions of the given servable, - // if any. + /// Returns the current states of all tracked versions of the given servable, + /// if any. VersionMap GetVersionStates(const string& servable_name) const LOCKS_EXCLUDED(mu_); - // Returns the current states of all tracked versions of all servables. + /// Returns the current states of all tracked versions of all servables. ServableMap GetAllServableStates() const LOCKS_EXCLUDED(mu_); - // Returns the current states of all versions of all servables which have not - // transitioned to state ServableState::ManagerState::kEnd. + /// Returns the current states of all versions of all servables which have not + /// transitioned to state ServableState::ManagerState::kEnd. ServableMap GetLiveServableStates() const LOCKS_EXCLUDED(mu_); - // Returns the current bounded log of handled servable state events. + /// Returns the current bounded log of handled servable state events. BoundedLog GetBoundedLog() const LOCKS_EXCLUDED(mu_); - // Notifies when all of the servables have reached the 'goal_state'. - // - // Servables can be specified in two ways: - // 1. As specific versions of a servable stream name. In this case, we check - // whether the specific version has reached the 'goal_state' or kEnd. - // 2. As latest versions, in which case any version for a servable stream - // name will be matched against the 'goal_state' or kEnd. - // - // We call the 'notifier_fn' when both conditions are true - - // 1. All of the specific servable requests have either reached the - // 'goal_state' or kEnd. - // 2. All of the latest servable requests have reached 'goal_state' or kEnd. - // The 'notifier_fn' will be called only once, and not repeatedly. - // - // The 'reached_goal_state' argument is set as true iff all of the specific - // servables have reached 'goal_state'. So callers should verify that - // 'reached_goal_state' is true in the 'notifier_fn'. - // - // The 'states_reached' argument is populated with the servable's id and the - // state it reached. The state would be 'goal_state' if 'reached_goal_state' - // is true, else it will contain one or more servables in kEnd state. For - // latest servable requests, the servable id will be the id of the servable in - // the stream which reached the state. + /// Notifies when all of the servables have reached the 'goal_state'. + /// + /// Servables can be specified in two ways: + /// 1. As specific versions of a servable stream name. In this case, we + /// check whether the specific version has reached the 'goal_state' or kEnd. + /// 2. As latest versions, in which case any version for a servable stream + /// name will be matched against the 'goal_state' or kEnd. + /// + /// We call the 'notifier_fn' when both conditions are true - + /// 1. All of the specific servable requests have either reached the + /// 'goal_state' or kEnd. + /// 2. All of the latest servable requests have reached 'goal_state' or + /// kEnd. + /// The 'notifier_fn' will be called only once, and not repeatedly. + /// + /// The 'reached_goal_state' argument is set as true iff all of the specific + /// servables have reached 'goal_state'. So callers should verify that + /// 'reached_goal_state' is true in the 'notifier_fn'. + /// + /// The 'states_reached' argument is populated with the servable's id and the + /// state it reached. The state would be 'goal_state' if 'reached_goal_state' + /// is true, else it will contain one or more servables in kEnd state. For + /// latest servable requests, the servable id will be the id of the servable + /// in the stream which reached the state. using ServableStateNotifierFn = std::function& states_reached)>; @@ -135,11 +136,11 @@ class ServableStateMonitor { ServableState::ManagerState goal_state, const ServableStateNotifierFn& notifier_fn) LOCKS_EXCLUDED(mu_); - // Similar to NotifyWhenServablesReachState(...), but instead of notifying, we - // wait until the 'goal_state' or kEnd is reached. - // - // To understand the return value and the return parameter 'states_reached', - // please read the documentation on NotifyWhenServablesReachState(...). + /// Similar to NotifyWhenServablesReachState(...), but instead of notifying, + /// we wait until the 'goal_state' or kEnd is reached. + /// + /// To understand the return value and the return parameter 'states_reached', + /// please read the documentation on NotifyWhenServablesReachState(...). bool WaitUntilServablesReachState( const std::vector& servables, ServableState::ManagerState goal_state, diff --git a/tensorflow_serving/core/source.h b/tensorflow_serving/core/source.h index 083f8c66d8a..e9347968123 100644 --- a/tensorflow_serving/core/source.h +++ b/tensorflow_serving/core/source.h @@ -28,64 +28,66 @@ limitations under the License. namespace tensorflow { namespace serving { -// An abstraction for a module that sources servables to load, or, more -// precisely, handles to data that can be used to load those servables. -// Examples of such data handles are: -// - a file-system path to a serialized vocabulary map -// - a handle to an incoming RPC that specifies a machine-learned model to load -// - a Loader (see loader.h) -// The data handles are generally assumed to be small. -// -// A Source monitors some external resource (e.g. file system, RPC calls) to -// find out about new servables and/or new versions of servables and/or the -// need to unload servable versions. It uses the provided callback to instruct -// a Target module (e.g. AspiredVersionsManager) which version(s) of a given -// servable to load. Furthermore, depending on the semantics of the Target -// module, the Source implicitly instructs it which ones to unload by omitting -// those servables. -// -// A common case is that a Source emits versions for exactly one servable. An -// even simpler case is that a servable has a single, static version for the -// lifetime of the server. -// -// Sources can house state that is shared among multiple emitted servables, e.g. -// 1. A shared thread pool or other resource that multiple servables use. -// 2. A shared read-only data structure that multiple servables use, to avoid -// the time and space overhead of replicating the data structure in each -// servable instance. -// Shared state whose initialization time and size is negligible (e.g. thread -// pools) can be created eagerly by the source, which then embeds a pointer to -// it in each emitted ServableData item. Creation of expensive or large shared -// state should be deferred to the first applicable Loader::Load() call, i.e. -// governed by the manager. Symmetrically, the Loader::Unload() call to the -// final servable using the expensive/large shared state should tear it down. +/// An abstraction for a module that sources servables to load, or, more +/// precisely, handles to data that can be used to load those servables. +/// Examples of such data handles are: +/// - a file-system path to a serialized vocabulary map +/// - a handle to an incoming RPC that specifies a machine-learned model to +/// load +/// - a Loader (see loader.h) +/// The data handles are generally assumed to be small. +/// +/// A Source monitors some external resource (e.g. file system, RPC calls) to +/// find out about new servables and/or new versions of servables and/or the +/// need to unload servable versions. It uses the provided callback to instruct +/// a Target module (e.g. AspiredVersionsManager) which version(s) of a given +/// servable to load. Furthermore, depending on the semantics of the Target +/// module, the Source implicitly instructs it which ones to unload by omitting +/// those servables. +/// +/// A common case is that a Source emits versions for exactly one servable. An +/// even simpler case is that a servable has a single, static version for the +/// lifetime of the server. +/// +/// Sources can house state that is shared among multiple emitted servables, +/// e.g. +/// 1. A shared thread pool or other resource that multiple servables use. +/// 2. A shared read-only data structure that multiple servables use, to avoid +/// the time and space overhead of replicating the data structure in each +/// servable instance. +/// Shared state whose initialization time and size is negligible (e.g. thread +/// pools) can be created eagerly by the source, which then embeds a pointer to +/// it in each emitted ServableData item. Creation of expensive or large shared +/// state should be deferred to the first applicable Loader::Load() call, i.e. +/// governed by the manager. Symmetrically, the Loader::Unload() call to the +/// final servable using the expensive/large shared state should tear it down. template class Source { public: virtual ~Source() = default; - // A callback for a Source to supply version(s) of a servable to a Target, to - // be loaded. - // - // A single invocation of the callback pertains to a single servable stream - // (given by 'servable_name'). All versions supplied in a call must be for the - // servable identified in 'servable_name'. Invocations on different servable - // streams are orthogonal to one another. - // - // Multiple invocations may supply servable-data objects with identical - // ids (i.e. same servable name and version). Such servable-data objects are - // treated as semantically equivalent. The recipient will ultimately retain - // one and discard the rest. - // - // If a servable version V is supplied in a first invocation, and subsequently - // omitted from a second invocation, the implication of omitting V depends on - // the semantics of the Target of the callback. Certain Targets will interpret - // V's omission as an implicit instruction to unload V. Each Target must - // document its semantics in this regard. + /// A callback for a Source to supply version(s) of a servable to a Target, to + /// be loaded. + /// + /// A single invocation of the callback pertains to a single servable stream + /// (given by 'servable_name'). All versions supplied in a call must be for + /// the servable identified in 'servable_name'. Invocations on different + /// servable streams are orthogonal to one another. + /// + /// Multiple invocations may supply servable-data objects with identical + /// ids (i.e. same servable name and version). Such servable-data objects are + /// treated as semantically equivalent. The recipient will ultimately retain + /// one and discard the rest. + /// + /// If a servable version V is supplied in a first invocation, and + /// subsequently omitted from a second invocation, the implication of omitting + /// V depends on the semantics of the Target of the callback. Certain Targets + /// will interpret V's omission as an implicit instruction to unload V. Each + /// Target must document its semantics in this regard. using AspiredVersionsCallback = std::function> versions)>; - // Supplies an AspiredVersionsCallback to use. Can be called at most once. + /// Supplies an AspiredVersionsCallback to use. Can be called at most once. virtual void SetAspiredVersionsCallback(AspiredVersionsCallback callback) = 0; }; diff --git a/tensorflow_serving/core/source_adapter.h b/tensorflow_serving/core/source_adapter.h index 839896eb355..30cbcc470a1 100644 --- a/tensorflow_serving/core/source_adapter.h +++ b/tensorflow_serving/core/source_adapter.h @@ -35,44 +35,45 @@ limitations under the License. namespace tensorflow { namespace serving { -// An abstraction for a module that receives aspired-version callbacks with data -// of type InputType and converts them into calls with data of type OutputType. -// -// A common example uses InputType=StoragePath, OutputType=unique_ptr, -// in which case the module "converts" each incoming storage path into a loader -// capable of loading a (particular type of) servable based on the path. -// -// SourceAdapters are typically stateless. However, as with all Sources they can -// house state that is shared among multiple emitted servables. See the -// discussion in source.h. -// -// Implementing subclasses supply an implementation of the Adapt() virtual -// method, which converts a servable version list from InputType to OutputType. -// -// IMPORTANT: Every leaf derived class must call Detach() at the top of its -// destructor. (See documentation on TargetBase::Detach() in target.h.) Doing so -// ensures that no Adapt() calls are in flight during destruction of member -// variables. +/// An abstraction for a module that receives aspired-version callbacks with +/// data of type InputType and converts them into calls with data of type +/// OutputType. +/// +/// A common example uses InputType=StoragePath, OutputType=unique_ptr, +/// in which case the module "converts" each incoming storage path into a loader +/// capable of loading a (particular type of) servable based on the path. +/// +/// SourceAdapters are typically stateless. However, as with all Sources they +/// can house state that is shared among multiple emitted servables. See the +/// discussion in source.h. +/// +/// Implementing subclasses supply an implementation of the Adapt() virtual +/// method, which converts a servable version list from InputType to OutputType. +/// +/// IMPORTANT: Every leaf derived class must call Detach() at the top of its +/// destructor. (See documentation on TargetBase::Detach() in target.h.) Doing +/// so ensures that no Adapt() calls are in flight during destruction of member +/// variables. template class SourceAdapter : public TargetBase, public Source { public: ~SourceAdapter() override = 0; - // This method is implemented in terms of Adapt(), which the implementing - // subclass must supply. + /// This method is implemented in terms of Adapt(), which the implementing + /// subclass must supply. void SetAspiredVersions(const StringPiece servable_name, std::vector> versions) final; void SetAspiredVersionsCallback( typename Source::AspiredVersionsCallback callback) final; - // Given an InputType-based aspired-versions request, produces a corresponding - // OutputType-based request. + /// Given an InputType-based aspired-versions request, produces a + /// corresponding OutputType-based request. virtual std::vector> Adapt( const StringPiece servable_name, std::vector> versions) = 0; - // Adapts a single servable data item. (Implemented on top of Adapt().) + /// Adapts a single servable data item. (Implemented on top of Adapt().) ServableData AdaptOneVersion(ServableData input); protected: diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index dfbc07ae1a1..d5ea3933084 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -13,19 +13,6 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// ServerCore contains state and helper methods enabling the building of -// ModelServers that support multiple interfaces. All functionality in -// ServerCore is independent of any domain specific APIs and independent of -// platforms. -// -// In terms of state, ServerCore is initialized with and retains a static -// ModelServerConfig, from which it bootstraps an AspiredVersionsManager and -// auxiliary data structures to support efficient serving. -// -// Interfaces built above ServerCore, e.g. RPC service implementations, will -// remain stateless and will perform all lookups of servables (models) via -// ServerCore. - #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_SERVER_CORE_H_ #define TENSORFLOW_SERVING_MODEL_SERVERS_SERVER_CORE_H_ @@ -63,25 +50,38 @@ namespace test_util { class ServerCoreTestAccess; } // namespace test_util +/// ServerCore contains state and helper methods enabling the building of +/// ModelServers that support multiple interfaces. All functionality in +/// ServerCore is independent of any domain specific APIs and independent of +/// platforms. +/// +/// In terms of state, ServerCore is initialized with and retains a static +/// ModelServerConfig, from which it bootstraps an AspiredVersionsManager and +/// auxiliary data structures to support efficient serving. +/// +/// Interfaces built above ServerCore, e.g. RPC service implementations, will +/// remain stateless and will perform all lookups of servables (models) via +/// ServerCore. class ServerCore : public Manager { public: using ServableStateMonitorCreator = std::function* event_bus, std::unique_ptr* monitor)>; - // A function that's responsible for instantiating and connecting the - // necessary custom sources and source adapters to the manager based on a - // passed in config (any). - // The expected pattern is that ownership of the created sources/source - // adapters can be transferred to the manager. + /// A function that's responsible for instantiating and connecting the + /// necessary custom sources and source adapters to the manager based on a + /// passed in config (any). + /// The expected pattern is that ownership of the created sources/source + /// adapters can be transferred to the manager. using CustomModelConfigLoader = std::function* event_bus, UniquePtrWithDeps* manager)>; - // Function signature used to update the server_request_logger. + /// Function signature used to update the server_request_logger. using ServerRequestLoggerUpdater = std::function; + /// Options for configuring a ServerCore object. struct Options { // ModelServer configuration. ModelServerConfig model_server_config; @@ -132,41 +132,42 @@ class ServerCore : public Manager { virtual ~ServerCore() = default; - // Creates a ServerCore instance with all the models and sources per the - // ModelServerConfig. - // - // For models statically configured with ModelConfigList, waits for them - // to be made available (or hit an error) for serving before returning. - // Returns an error status if any such model fails to load. + /// Creates a ServerCore instance with all the models and sources per the + /// ModelServerConfig. + /// + /// For models statically configured with ModelConfigList, waits for them + /// to be made available (or hit an error) for serving before returning. + /// Returns an error status if any such model fails to load. static Status Create(Options options, std::unique_ptr* core); std::vector ListAvailableServableIds() const override { return manager_->ListAvailableServableIds(); } - // Updates the server core with all the models and sources per the - // ModelServerConfig. Like Create(), waits for all statically configured - // servables to be made available before returning, and returns an error if - // any such model fails to load. (Does not necessarily wait for models removed - // from the config to finish unloading; that may occur asynchronously.) - // - // IMPORTANT: It is only legal to call this method more than once if using - // ModelConfigList (versus custom model config). + /// Updates the server core with all the models and sources per the + /// ModelServerConfig. Like Create(), waits for all statically configured + /// servables to be made available before returning, and returns an error if + /// any such model fails to load. (Does not necessarily wait for models + /// removed from the config to finish unloading; that may occur + /// asynchronously.) + /// + /// IMPORTANT: It is only legal to call this method more than once if using + /// ModelConfigList (versus custom model config). virtual Status ReloadConfig(const ModelServerConfig& config) LOCKS_EXCLUDED(config_mu_); - // Returns ServableStateMonitor that can be used to query servable states. + /// Returns ServableStateMonitor that can be used to query servable states. virtual const ServableStateMonitor* servable_state_monitor() const { return servable_state_monitor_.get(); } - // Returns a ServableHandle given a ServableRequest. Returns error if no such - // Servable is available -- e.g. not yet loaded, has been quiesced/unloaded, - // etc. Callers may assume that an OK status indicates a non-null handle. - // - // IMPORTANT: The caller should only hold on to a handle for a short time, for - // example for the duration of a single request. Holding a handle for a long - // period of time will prevent servable loading and unloading. + /// Returns a ServableHandle given a ServableRequest. Returns error if no such + /// Servable is available -- e.g. not yet loaded, has been quiesced/unloaded, + /// etc. Callers may assume that an OK status indicates a non-null handle. + /// + /// IMPORTANT: The caller should only hold on to a handle for a short time, + /// for example for the duration of a single request. Holding a handle for a + /// long period of time will prevent servable loading and unloading. template Status GetServableHandle(const ModelSpec& model_spec, ServableHandle* const handle) { @@ -185,9 +186,9 @@ class ServerCore : public Manager { return Status::OK(); } - // Writes the log for the particular request, response and metadata, if we - // decide to sample it and if request-logging was configured for the - // particular model. + /// Writes the log for the particular request, response and metadata, if we + /// decide to sample it and if request-logging was configured for the + /// particular model. Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, const LogMetadata& log_metadata) { return options_.server_request_logger->Log(request, response, log_metadata); diff --git a/tensorflow_serving/servables/tensorflow/serving_session.h b/tensorflow_serving/servables/tensorflow/serving_session.h index 561c8ff051a..25f749e451f 100644 --- a/tensorflow_serving/servables/tensorflow/serving_session.h +++ b/tensorflow_serving/servables/tensorflow/serving_session.h @@ -27,9 +27,9 @@ limitations under the License. namespace tensorflow { namespace serving { -// A Session that blocks state-changing methods such as Close(), while allowing -// Run() for read-only access (not enforced). Useful for Session implementations -// that intend to be read-only and only implement Run(). +/// A Session that blocks state-changing methods such as Close(), while allowing +/// Run() for read-only access (not enforced). Useful for Session +/// implementations that intend to be read-only and only implement Run(). class ServingSession : public Session { public: ServingSession() = default; @@ -43,8 +43,8 @@ class ServingSession : public Session { // (Subclasses just implement Run().) }; -// A ServingSession that wraps a given Session, and blocks all calls other than -// Run(). +/// A ServingSession that wraps a given Session, and blocks all calls other than +/// Run(). class ServingSessionWrapper : public ServingSession { public: explicit ServingSessionWrapper(std::unique_ptr wrapped) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h index 90891471b8d..7cc53dd42a4 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.h @@ -37,34 +37,34 @@ class FileSystemStoragePathSourceTestAccess; namespace tensorflow { namespace serving { -// A storage path source that aspires versions for a given set of servables. For -// each servable, it monitors a given file-system base path. It identifies base- -// path children whose name is a number (e.g. 123) and emits the path -// corresponding to the largest number as the servable's single aspired version. -// (To do the file-system monitoring, it uses a background thread that polls the -// file system periodically.) -// -// For example, if a configured servable's base path is /foo/bar, and a file- -// system poll reveals child paths /foo/bar/baz, /foo/bar/123 and /foo/bar/456, -// the aspired-versions callback is called with {{456, "/foo/bar/456"}}. If, at -// any time, the base path is found to contain no numerical children, the -// aspired-versions callback is called with an empty versions list. -// -// The configured set of servables to monitor can be updated at any time by -// calling UpdateConfig(). If any servables were present in the old config but -// not in the new one, the source will immediately aspire zero versions for that -// servable (causing it to be unloaded in the Manager that ultimately consumes -// the aspired-versions calls). +/// A storage path source that aspires versions for a given set of servables. +/// For each servable, it monitors a given file-system base path. It identifies +/// base-path children whose name is a number (e.g. 123) and emits the path +/// corresponding to the largest number as the servable's single aspired +/// version. (To do the file-system monitoring, it uses a background thread that +/// polls the file system periodically.) +/// +/// For example, if a configured servable's base path is /foo/bar, and a file- +/// system poll reveals child paths /foo/bar/baz, /foo/bar/123 and /foo/bar/456, +/// the aspired-versions callback is called with {456, "/foo/bar/456"}. If, at +/// any time, the base path is found to contain no numerical children, the +/// aspired-versions callback is called with an empty versions list. +/// +/// The configured set of servables to monitor can be updated at any time by +/// calling UpdateConfig(). If any servables were present in the old config but +/// not in the new one, the source will immediately aspire zero versions for +/// that servable (causing it to be unloaded in the Manager that ultimately +/// consumes the aspired-versions calls). class FileSystemStoragePathSource : public Source { public: static Status Create(const FileSystemStoragePathSourceConfig& config, std::unique_ptr* result); ~FileSystemStoragePathSource() override; - // Supplies a new config to use. The set of servables to monitor can be - // changed at any time (see class comment for more information), but it is - // illegal to change the file-system polling period once - // SetAspiredVersionsCallback() has been called. + /// Supplies a new config to use. The set of servables to monitor can be + /// changed at any time (see class comment for more information), but it is + /// illegal to change the file-system polling period once + /// SetAspiredVersionsCallback() has been called. Status UpdateConfig(const FileSystemStoragePathSourceConfig& config); void SetAspiredVersionsCallback(AspiredVersionsCallback callback) override; diff --git a/tensorflow_serving/util/any_ptr.h b/tensorflow_serving/util/any_ptr.h index da243f79a23..1ac269f0341 100644 --- a/tensorflow_serving/util/any_ptr.h +++ b/tensorflow_serving/util/any_ptr.h @@ -22,55 +22,55 @@ limitations under the License. namespace tensorflow { namespace serving { -// A (sort of) type-safe void*. Appears as null if a caller attempts to use it -// as the wrong type. -// -// Example use: -// -// // A function that returns an AnyPtr: -// AnyPtr StringOrInt() { -// if (use_string) { -// return AnyPtr(&some_string); -// } else { -// return AnyPtr(&some_int); -// } -// } -// -// // Use an AnyPtr at the correct type: -// AnyPtr ptr = StringOrInt(); -// if (ptr.get() != nullptr) { -// DoSomethingWithInt(*ptr.get()); -// } else if (ptr.get() != nullptr) { -// DoSomethingWithString(*ptr.get()); -// } else { -// // Handle error. -// } -// -// Typical best practice for this class is to use it when two disjoint pieces of -// code must agree on type, but intermediate code is type agnostic. Large chains -// of conditionals that handle a multitude of types is discouraged as an -// anti-pattern. -// -// Note that this will appear null even if T is somewhere on the underlying -// type's inheritance hierarchy, if you must use the object at some other type -// you must do so explicitly when constructing an AnyPtr, like so: -// -// SomeObject object; -// AnyPtr any_ptr(static_cast(&object)); -// SomeInterface* interface = any_ptr.get(); -// -// This class is a value type; It can be copied or assigned. It performs no -// internal allocations and should be relatively cheap to copy or return by -// value. +/// A (sort of) type-safe void*. Appears as null if a caller attempts to use it +/// as the wrong type. +/// +/// Example use: +/// +/// // A function that returns an AnyPtr: +/// AnyPtr StringOrInt() { +/// if (use_string) { +/// return AnyPtr(&some_string); +/// } else { +/// return AnyPtr(&some_int); +/// } +/// } +/// +/// // Use an AnyPtr at the correct type: +/// AnyPtr ptr = StringOrInt(); +/// if (ptr.get() != nullptr) { +/// DoSomethingWithInt(*ptr.get()); +/// } else if (ptr.get() != nullptr) { +/// DoSomethingWithString(*ptr.get()); +/// } else { +/// // Handle error. +/// } +/// +/// Typical best practice for this class is to use it when two disjoint pieces +/// of code must agree on type, but intermediate code is type agnostic. Large +/// chains of conditionals that handle a multitude of types is discouraged as an +/// anti-pattern. +/// +/// Note that this will appear null even if T is somewhere on the underlying +/// type's inheritance hierarchy, if you must use the object at some other type +/// you must do so explicitly when constructing an AnyPtr, like so: +/// +/// SomeObject object; +/// AnyPtr any_ptr(static_cast(&object)); +/// SomeInterface* interface = any_ptr.get(); +/// +/// This class is a value type; It can be copied or assigned. It performs no +/// internal allocations and should be relatively cheap to copy or return by +/// value. class AnyPtr { public: - // AnyPtr is void and null by default. + /// AnyPtr is void and null by default. AnyPtr() : type_id_(FastTypeId()), ptr_(nullptr) {} - // Implicit construction from nullptr. + /// Implicit construction from nullptr. AnyPtr(std::nullptr_t) : AnyPtr() {} // NOLINT - // Construct from a pointer to any type. + /// Construct from a pointer to any type. template AnyPtr(T* ptr) // NOLINT : type_id_(FastTypeId()), @@ -80,7 +80,7 @@ class AnyPtr { // non-const T. ptr_(const_cast(reinterpret_cast(ptr))) {} - // Returns the underlying pointer if it is of type T, otherwise null. + /// Accessor for the underlying pointer if it is of type T, otherwise null. template T* get() const { if (type_id_ != FastTypeId()) { @@ -104,15 +104,15 @@ class AnyPtr { void* ptr_; }; -// Like AnyPtr, but owns the pointed-to object (calls delete upon destruction). -// This class is move-only, like std::unique_ptr. +/// Like AnyPtr, but owns the pointed-to object (calls delete upon destruction). +/// This class is move-only, like std::unique_ptr. class UniqueAnyPtr { public: - // UniqueAnyPtr is void and null by default. + /// UniqueAnyPtr is void and null by default. UniqueAnyPtr() = default; UniqueAnyPtr(std::nullptr_t) : UniqueAnyPtr() {} // NOLINT - // Construct from a unique pointer to any type. + /// Construct from a unique pointer to any type. template explicit UniqueAnyPtr(std::unique_ptr ptr) : ptr_(ptr.release()), deleter_(DeleterForType()) {} @@ -131,13 +131,13 @@ class UniqueAnyPtr { return *this; } - // Returns the underlying pointer if it is of type T, otherwise null. + /// Accessor for the underlying pointer if it is of type T, otherwise null. template T* get() const { return ptr_.get(); } - // Returns the underlying pointer as an AnyPtr. + /// Accessor for the underlying pointer as an AnyPtr. const AnyPtr& as_any_ptr() const { return ptr_; } void swap(UniqueAnyPtr& other) { diff --git a/tensorflow_serving/util/event_bus.h b/tensorflow_serving/util/event_bus.h index 167547e095e..4e677d70380 100644 --- a/tensorflow_serving/util/event_bus.h +++ b/tensorflow_serving/util/event_bus.h @@ -13,31 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// EventBus enables basic publish / subscribe semantics for events amongst one -// or more publishers and subscribers. The purpose of EventBus for serving is -// to de-couple the code for such events, and is optimized for that use-case. -// -// EventBus is thread-safe. However, if any subscriber callback calls any method -// in the EventBus, it will deadlock. -// -// EventBus and Subscriptions can be destroyed safely in any order. There is a -// strict requirement for memory safety that a Subscription must be destroyed -// before any of the objects or memory that a subscriber's callback accesses. -// -// Important scaling and threading limitations: -// -// Scaling: -// The EventBus is not currently optimized for high scale, either in the number -// of subscribers or frequency of events. For such use-cases, consider alternate -// implementations or upgrades to this class. -// -// Threading: -// Subscribers are notified serially on the event publisher's thread. Thus, the -// amount of work done in a subscriber's callback should be very minimal. -// // TODO(b/25725560): Consider having a thread pool for invoking callbacks. -// -// This implementation is single-binary and does not communicate across tasks. #ifndef TENSORFLOW_SERVING_UTIL_EVENT_BUS_H_ #define TENSORFLOW_SERVING_UTIL_EVENT_BUS_H_ @@ -57,24 +33,47 @@ limitations under the License. namespace tensorflow { namespace serving { -// Note that the types used for typename E in EventBus must be moveable and -// thread-safe. Future implementations may read Events of type E in multiple -// threads. +/// EventBus enables basic publish / subscribe semantics for events amongst one +/// or more publishers and subscribers. The purpose of EventBus for serving is +/// to de-couple the code for such events, and is optimized for that use-case. +/// +/// EventBus and Subscriptions can be destroyed safely in any order. There is a +/// strict requirement for memory safety that a Subscription must be destroyed +/// before any of the objects or memory that a subscriber's callback accesses. +/// +/// Important scaling and threading limitations: +/// +/// Scaling: +/// The EventBus is not currently optimized for high scale, either in the number +/// of subscribers or frequency of events. For such use-cases, consider +/// alternate implementations or upgrades to this class. +/// +/// Threading: +/// EventBus is thread-safe. However, if any subscriber callback calls any +/// method in the EventBus, it will deadlock. Subscribers are notified serially +/// on the event publisher's thread. Thus, the amount of work done in a +/// subscriber's callback should be very minimal. +/// +/// This implementation is single-binary and does not communicate across tasks. +/// +/// Note that the types used for typename E in EventBus must be moveable and +/// thread-safe. Future implementations may read Events of type E in multiple +/// threads. template class EventBus : public std::enable_shared_from_this> { static_assert(std::is_move_assignable::value, "E must be moveable"); public: - // Subscription is an RAII object and tracks the lifecycle of a single - // subscription to an EventBus. Upon destruction, it automatically - // Unsubscribes from the originating EventBus. - // - // Note that Subscription only maintains weak_ptr references to the EventBus, - // such that the EventBus and Subscriptions can be safely destructed in any - // order. Subscription is not an owner of EventBus. + /// Subscription is an RAII object and tracks the lifecycle of a single + /// subscription to an EventBus. Upon destruction, it automatically + /// Unsubscribes from the originating EventBus. + /// + /// Note that Subscription only maintains weak_ptr references to the EventBus, + /// such that the EventBus and Subscriptions can be safely destructed in any + /// order. Subscription is not an owner of EventBus. class Subscription { public: - // Unsubscribes the subscriber. + /// Unsubscribes the subscriber. ~Subscription(); private: @@ -93,45 +92,45 @@ class EventBus : public std::enable_shared_from_this> { Env* env = Env::Default(); }; - // Creates an EventBus and returns a shared_ptr to it. This is the only - // allowed public mechanism for creating an EventBus so that we can track - // references to an EventBus uniformly. + /// Creates an EventBus and returns a shared_ptr to it. This is the only + /// allowed public mechanism for creating an EventBus so that we can track + /// references to an EventBus uniformly. static std::shared_ptr CreateEventBus(const Options& options = {}); ~EventBus() = default; - // Event and the publish time associated with it. + /// Event and the publish time associated with it. struct EventAndTime { const E& event; uint64 event_time_micros; }; - // The function type for EventBus Callbacks to be implemented by clients. - // Important Warnings: - // * Callbacks must not themselves callback to the EventBus for any purpose - // including subscribing, publishing or unsubscribing. This will cause a - // circular deadlock. - // * Callbacks must do very little work as they are invoked on the - // publisher's thread. Any costly work should be performed asynchronously. + /// The function type for EventBus Callbacks to be implemented by clients. + /// Important Warnings: + /// * Callbacks must not themselves callback to the EventBus for any purpose + /// including subscribing, publishing or unsubscribing. This will cause a + /// circular deadlock. + /// * Callbacks must do very little work as they are invoked on the + /// publisher's thread. Any costly work should be performed asynchronously. using Callback = std::function; - // Subscribes to all events on the EventBus. - // - // Returns a Subscription RAII object that can be used to unsubscribe, or will - // automatically unsubscribe on destruction. Returns a unique_ptr so that we - // can use the subscription's address to Unsubscribe. - // - // Important contract for unsubscribing (deleting the RAII object): - // * Unsubscribing (deleting the RAII object) may block while currently - // scheduled callback invocation(s) finish. - // * Once it returns no callback invocations will occur. - // Callers' destructors must use the sequence: - // (1) Unsubscribe. - // (2) Tear down anything that the callback references. + /// Subscribes to all events on the EventBus. + /// + /// Returns a Subscription RAII object that can be used to unsubscribe, or + /// will automatically unsubscribe on destruction. Returns a unique_ptr so + /// that we can use the subscription's address to Unsubscribe. + /// + /// Important contract for unsubscribing (deleting the RAII object): + /// * Unsubscribing (deleting the RAII object) may block while currently + /// scheduled callback invocation(s) finish. + /// * Once it returns no callback invocations will occur. + /// Callers' destructors must use the sequence: + /// (1) Unsubscribe. + /// (2) Tear down anything that the callback references. std::unique_ptr Subscribe(const Callback& callback) LOCKS_EXCLUDED(mutex_) TF_MUST_USE_RESULT; - // Publishes an event to all subscribers. + /// Publishes an event to all subscribers. void Publish(const E& event) LOCKS_EXCLUDED(mutex_); private: diff --git a/tensorflow_serving/util/executor.h b/tensorflow_serving/util/executor.h index 9d2f0ac6970..a88452ebfa9 100644 --- a/tensorflow_serving/util/executor.h +++ b/tensorflow_serving/util/executor.h @@ -21,15 +21,15 @@ limitations under the License. namespace tensorflow { namespace serving { -// An abstract object that can execute closures. -// -// Implementations of executor must be thread-safe. +/// An abstract object that can execute closures. +/// +/// Implementations of executor must be thread-safe. class Executor { public: virtual ~Executor() = default; - // Schedule the specified 'fn' for execution in this executor. Depending on - // the subclass implementation, this may block in some situations. + /// Schedule the specified 'fn' for execution in this executor. Depending on + /// the subclass implementation, this may block in some situations. virtual void Schedule(std::function fn) = 0; }; From 08312dbb3bac863e61c28004a3987addb413b9ce Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 21 Jun 2017 14:54:03 -0800 Subject: [PATCH 0277/8554] Internal change. Change: 159756874 --- tensorflow_serving/batching/BUILD | 2 -- tensorflow_serving/servables/tensorflow/BUILD | 2 -- tensorflow_serving/util/BUILD | 2 -- 3 files changed, 6 deletions(-) diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index 1fc4fff8234..7dcceede9a7 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -7,8 +7,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 3cfb2010248..b974f974b7f 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 2fdeeff2c98..9b82eb3170d 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( From e18231d879826ce72bff7ceddcb58b78bf5c5b9b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 22 Jun 2017 07:28:00 -0800 Subject: [PATCH 0278/8554] Internal change. Change: 159829097 --- tensorflow_serving/apis/BUILD | 2 -- tensorflow_serving/apis/internal/BUILD | 2 -- tensorflow_serving/batching/test_util/BUILD | 2 -- tensorflow_serving/config/BUILD | 2 -- tensorflow_serving/core/BUILD | 2 -- tensorflow_serving/core/test_util/BUILD | 2 -- tensorflow_serving/example/BUILD | 2 -- tensorflow_serving/model_servers/BUILD | 2 -- tensorflow_serving/model_servers/test_util/BUILD | 2 -- tensorflow_serving/resources/BUILD | 2 -- tensorflow_serving/servables/hashmap/BUILD | 2 -- tensorflow_serving/servables/tensorflow/testdata/BUILD | 2 -- tensorflow_serving/sources/storage_path/BUILD | 2 -- tensorflow_serving/test_util/BUILD | 2 -- tensorflow_serving/util/test_util/BUILD | 2 -- 15 files changed, 30 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 2d2693bf2af..62cd8392865 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -7,8 +7,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/apis/internal/BUILD b/tensorflow_serving/apis/internal/BUILD index 6ce2e32a915..1e46511dec1 100644 --- a/tensorflow_serving/apis/internal/BUILD +++ b/tensorflow_serving/apis/internal/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - load("//tensorflow_serving:serving.bzl", "serving_proto_library") serving_proto_library( diff --git a/tensorflow_serving/batching/test_util/BUILD b/tensorflow_serving/batching/test_util/BUILD index 2de50699795..9bf13315bd6 100644 --- a/tensorflow_serving/batching/test_util/BUILD +++ b/tensorflow_serving/batching/test_util/BUILD @@ -7,8 +7,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 74682b4629a..c7122d52122 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -7,8 +7,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index baa5df129db..70def4e30d4 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index a2cc04eb979..70b02df045a 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 37d22efe3b0..19150a86301 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -7,8 +7,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - load("//tensorflow_serving:serving.bzl", "serving_proto_library") filegroup( diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 20200d224c7..313e35f9113 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 71921c47bfe..b79deeded5d 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/resources/BUILD b/tensorflow_serving/resources/BUILD index 6c5522b8749..d94b1b0206d 100644 --- a/tensorflow_serving/resources/BUILD +++ b/tensorflow_serving/resources/BUILD @@ -7,8 +7,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/servables/hashmap/BUILD b/tensorflow_serving/servables/hashmap/BUILD index 11105cf5cd7..a3a614bc92f 100644 --- a/tensorflow_serving/servables/hashmap/BUILD +++ b/tensorflow_serving/servables/hashmap/BUILD @@ -7,8 +7,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 8c2982ea68b..5cfbac11907 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -12,8 +12,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/sources/storage_path/BUILD b/tensorflow_serving/sources/storage_path/BUILD index b327b8a5af9..44da570369d 100644 --- a/tensorflow_serving/sources/storage_path/BUILD +++ b/tensorflow_serving/sources/storage_path/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/test_util/BUILD b/tensorflow_serving/test_util/BUILD index 2908a9aab9b..8ab767118a6 100644 --- a/tensorflow_serving/test_util/BUILD +++ b/tensorflow_serving/test_util/BUILD @@ -6,8 +6,6 @@ package(default_visibility = [ licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - filegroup( name = "all_files", srcs = glob( diff --git a/tensorflow_serving/util/test_util/BUILD b/tensorflow_serving/util/test_util/BUILD index 9196d1aa1f1..676554a200b 100644 --- a/tensorflow_serving/util/test_util/BUILD +++ b/tensorflow_serving/util/test_util/BUILD @@ -9,8 +9,6 @@ package( licenses(["notice"]) # Apache 2.0 -exports_files(["LICENSE"]) - cc_library( name = "mock_file_probing_env", testonly = 1, From 013c401519e7a9945168b6bc028b60d5bd6273dd Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Fri, 23 Jun 2017 13:37:02 -0800 Subject: [PATCH 0279/8554] Add deprecation comment for *_export.py examples. Use *_saved_model.py instead. Change: 159995425 --- tensorflow_serving/example/inception_export.py | 3 +++ tensorflow_serving/example/mnist_export.py | 3 +++ 2 files changed, 6 insertions(+) diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py index e807a08c680..5db25de18dd 100644 --- a/tensorflow_serving/example/inception_export.py +++ b/tensorflow_serving/example/inception_export.py @@ -16,6 +16,9 @@ #!/usr/bin/env python2.7 """Export inception model given existing training checkpoints. +Note the export format for this example is SessionBundle, which is deprecated. +Please use SavedModel instead. Specifically, see: `inception_saved_model.py`. + The model is exported with proper signatures that can be loaded by standard tensorflow_model_server. """ diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 4df170e5820..7c2c1879ff0 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -17,6 +17,9 @@ """Train and export a simple Softmax Regression TensorFlow model. +Note the export format for this example is SessionBundle, which is deprecated. +Please use SavedModel instead. Specifically, see: `mnist_saved_model.py`. + The model is from the TensorFlow "MNIST For ML Beginner" tutorial. This program simply follows all its training instructions, and uses TensorFlow Serving exporter to export the trained model with proper signatures that can be From 9450b7cecfe6e059a5ba8792ffb77742f717af8a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 27 Jun 2017 16:09:22 -0800 Subject: [PATCH 0280/8554] Remove some unused code Change: 160348624 --- tensorflow_serving/core/aspired_versions_manager_test.cc | 1 - tensorflow_serving/core/caching_manager_test.cc | 1 - tensorflow_serving/core/loader_harness_test.cc | 3 --- tensorflow_serving/core/source_adapter_test.cc | 1 - tensorflow_serving/core/source_router_test.cc | 1 - tensorflow_serving/core/static_source_router_test.cc | 1 - tensorflow_serving/model_servers/main.cc | 4 ---- tensorflow_serving/resources/resource_tracker_test.cc | 1 - tensorflow_serving/servables/tensorflow/bundle_factory_test.h | 3 --- .../servables/tensorflow/session_bundle_factory_test.cc | 2 -- .../storage_path/file_system_storage_path_source_test.cc | 1 - .../sources/storage_path/static_storage_path_source_test.cc | 1 - 12 files changed, 20 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index edb36868119..f73a0d9ba37 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -39,7 +39,6 @@ namespace serving { namespace { using ::testing::_; -using ::testing::DoAll; using ::testing::Invoke; using ::testing::InvokeWithoutArgs; using ::testing::NiceMock; diff --git a/tensorflow_serving/core/caching_manager_test.cc b/tensorflow_serving/core/caching_manager_test.cc index 6c3de1e33db..4126a36b35b 100644 --- a/tensorflow_serving/core/caching_manager_test.cc +++ b/tensorflow_serving/core/caching_manager_test.cc @@ -41,7 +41,6 @@ namespace serving { namespace { using ::testing::HasSubstr; -using ::testing::UnorderedElementsAre; using ::testing::UnorderedElementsAreArray; // A simple loader-factory that concatenates requested servable name and diff --git a/tensorflow_serving/core/loader_harness_test.cc b/tensorflow_serving/core/loader_harness_test.cc index 7e4508b5be7..7b9539c5bff 100644 --- a/tensorflow_serving/core/loader_harness_test.cc +++ b/tensorflow_serving/core/loader_harness_test.cc @@ -36,12 +36,9 @@ using ::testing::_; using ::testing::HasSubstr; using ::testing::InvokeWithoutArgs; using ::testing::InSequence; -using ::testing::IsNull; using ::testing::NiceMock; using ::testing::Return; -using ::testing::ReturnRef; using ::testing::StrictMock; -using test_util::EqualsProto; // Walks 'harness' through a sequence of transitions from kReady to kDisabled. void QuiesceAndUnload(LoaderHarness* const harness) { diff --git a/tensorflow_serving/core/source_adapter_test.cc b/tensorflow_serving/core/source_adapter_test.cc index 1cc3503dee0..4eddf99ee49 100644 --- a/tensorflow_serving/core/source_adapter_test.cc +++ b/tensorflow_serving/core/source_adapter_test.cc @@ -33,7 +33,6 @@ limitations under the License. using ::testing::ElementsAre; using ::testing::Eq; using ::testing::IsEmpty; -using ::testing::Return; using ::testing::StrictMock; namespace tensorflow { diff --git a/tensorflow_serving/core/source_router_test.cc b/tensorflow_serving/core/source_router_test.cc index 053e23d4809..69f63a6de03 100644 --- a/tensorflow_serving/core/source_router_test.cc +++ b/tensorflow_serving/core/source_router_test.cc @@ -31,7 +31,6 @@ using ::testing::_; using ::testing::ElementsAre; using ::testing::Eq; using ::testing::IsEmpty; -using ::testing::Return; using ::testing::StrictMock; namespace tensorflow { diff --git a/tensorflow_serving/core/static_source_router_test.cc b/tensorflow_serving/core/static_source_router_test.cc index 094ee122abb..121ad6aadba 100644 --- a/tensorflow_serving/core/static_source_router_test.cc +++ b/tensorflow_serving/core/static_source_router_test.cc @@ -31,7 +31,6 @@ limitations under the License. using ::testing::ElementsAre; using ::testing::Eq; -using ::testing::Return; using ::testing::StrictMock; namespace tensorflow { diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 538d1eed347..7e33f10bc96 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -90,12 +90,10 @@ using tensorflow::serving::FileSystemStoragePathSourceConfig; using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy; using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy_Name; using tensorflow::serving::GetModelMetadataImpl; -using tensorflow::serving::Loader; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; using tensorflow::serving::ServerCore; using tensorflow::serving::SessionBundleConfig; -using tensorflow::serving::Target; using tensorflow::serving::TensorflowClassificationServiceImpl; using tensorflow::serving::TensorflowRegressionServiceImpl; using tensorflow::serving::TensorflowPredictor; @@ -104,10 +102,8 @@ using tensorflow::string; using grpc::InsecureServerCredentials; using grpc::Server; -using grpc::ServerAsyncResponseWriter; using grpc::ServerBuilder; using grpc::ServerContext; -using grpc::ServerCompletionQueue; using tensorflow::serving::ClassificationRequest; using tensorflow::serving::ClassificationResponse; using tensorflow::serving::GetModelMetadataRequest; diff --git a/tensorflow_serving/resources/resource_tracker_test.cc b/tensorflow_serving/resources/resource_tracker_test.cc index 0e6c5336a66..48c2fdcd01d 100644 --- a/tensorflow_serving/resources/resource_tracker_test.cc +++ b/tensorflow_serving/resources/resource_tracker_test.cc @@ -33,7 +33,6 @@ using ::tensorflow::serving::test_util::EqualsProto; using ::testing::_; using ::testing::Invoke; using ::testing::NiceMock; -using ::testing::Return; namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_test.h b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h index 173df46ef29..c2c7fd5cb9b 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_test.h +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_test.h @@ -36,9 +36,6 @@ namespace tensorflow { namespace serving { namespace test_util { -using ::testing::DoAll; -using ::testing::Return; -using ::testing::SetArgPointee; using ::testing::_; using test_util::EqualsProto; diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc index a195f5b7876..06145856655 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/session_bundle_factory_test.cc @@ -37,8 +37,6 @@ namespace tensorflow { namespace serving { namespace { -using test_util::EqualsProto; - class SessionBundleFactoryTest : public test_util::BundleFactoryTest { public: SessionBundleFactoryTest() diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index 55c08508fa5..3fa36304244 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -36,7 +36,6 @@ using ::testing::ElementsAre; using ::testing::Eq; using ::testing::IsEmpty; using ::testing::StrictMock; -using ::testing::Return; namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/sources/storage_path/static_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/static_storage_path_source_test.cc index 98a568328fc..acf6380a495 100644 --- a/tensorflow_serving/sources/storage_path/static_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/static_storage_path_source_test.cc @@ -26,7 +26,6 @@ limitations under the License. using ::testing::ElementsAre; using ::testing::Eq; -using ::testing::Return; using ::testing::StrictMock; namespace tensorflow { From d54d1d81bbc5ce2d7fc9ad596dc699ebd2f850d8 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 28 Jun 2017 11:04:38 -0800 Subject: [PATCH 0281/8554] Add missing inclusions and remove unused BUILD dependencies Change: 160435107 --- tensorflow_serving/core/BUILD | 4 +-- .../core/aspired_versions_manager.h | 2 ++ tensorflow_serving/servables/tensorflow/BUILD | 32 +++---------------- .../servables/tensorflow/predict_impl_test.cc | 1 + tensorflow_serving/util/BUILD | 1 - 5 files changed, 9 insertions(+), 31 deletions(-) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 70def4e30d4..e1ef7755b8c 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -491,6 +491,7 @@ cc_library( ":aspired_version_policy", ":basic_manager", ":loader", + ":loader_harness", ":manager", ":servable_data", ":servable_handle", @@ -539,7 +540,6 @@ cc_test( ":servable_handle", ":simple_loader", "//tensorflow_serving/core/test_util:manager_test_util", - "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", @@ -802,13 +802,11 @@ cc_test( deps = [ ":log_collector", ":logging_proto", - ":request_logger", ":server_request_logger", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/config:logging_config_proto", "//tensorflow_serving/core/test_util:fake_log_collector", - "//tensorflow_serving/core/test_util:mock_log_collector", "//tensorflow_serving/core/test_util:mock_request_logger", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 43a4bb1a634..704710ddb8f 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -33,10 +33,12 @@ limitations under the License. #include "tensorflow_serving/core/basic_manager.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/manager.h" +#include "tensorflow_serving/core/servable_data.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_id.h" #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/target.h" +#include "tensorflow_serving/util/event_bus.h" #include "tensorflow_serving/util/optional.h" namespace tensorflow { diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index b974f974b7f..bd14e238ec1 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -78,7 +78,6 @@ cc_test( "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@protobuf//:cc_wkt_protos", ], @@ -136,7 +135,6 @@ cc_library( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -156,7 +154,6 @@ cc_test( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@protobuf//:cc_wkt_protos", ], @@ -182,7 +179,6 @@ cc_library( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -201,10 +197,8 @@ cc_test( ":session_bundle_config_proto", "//tensorflow_serving/core/test_util:test_main", "@org_tensorflow//tensorflow/cc/saved_model:loader", - "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@protobuf//:cc_wkt_protos", ], @@ -218,7 +212,6 @@ cc_library( "//visibility:public", ], deps = [ - ":serving_session", ":session_bundle_factory", ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:loader", @@ -229,7 +222,6 @@ cc_library( "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], alwayslink = 1, ) @@ -253,7 +245,6 @@ cc_test( "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", ], ) @@ -277,7 +268,6 @@ cc_library( "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], alwayslink = 1, ) @@ -304,7 +294,6 @@ cc_test( "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", ], ) @@ -355,7 +344,6 @@ cc_library( "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -370,12 +358,10 @@ cc_test( "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", - "//tensorflow_serving/util:unique_ptr_with_deps", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@org_tensorflow//tensorflow/core:testlib", ], @@ -402,7 +388,6 @@ cc_library( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -418,7 +403,6 @@ cc_test( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", ], ) @@ -511,13 +495,17 @@ cc_test( ], deps = [ ":predict_impl", + ":saved_model_bundle_source_adapter_proto", + ":session_bundle_config_proto", + ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:availability_preserving_policy", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/model_servers:model_platform_types", "//tensorflow_serving/model_servers:platform_config_util", "//tensorflow_serving/model_servers:server_core", - "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:test", ], ) @@ -539,11 +527,9 @@ cc_library( "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -566,7 +552,6 @@ cc_test( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@protobuf//:protobuf_lite", ], @@ -627,11 +612,9 @@ cc_library( "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:signature", - "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) @@ -653,7 +636,6 @@ cc_test( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@protobuf//:protobuf_lite", ], @@ -676,10 +658,8 @@ cc_library( "//tensorflow_serving/servables/tensorflow:util", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:tensorflow", "@protobuf//:protobuf", ], ) @@ -706,7 +686,6 @@ cc_test( "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", ], ) @@ -740,7 +719,6 @@ cc_test( "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", "@org_tensorflow//tensorflow/core:test", "@org_tensorflow//tensorflow/core:testlib", ], diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 049f4523b79..46d276a0e1c 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -23,6 +23,7 @@ limitations under the License. #include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 9b82eb3170d..da565b3a282 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -263,7 +263,6 @@ cc_library( hdrs = ["class_registration_util.h"], deps = [ "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:cc_wkt_protos", ], ) From 90343e22780c34943dce15cb7175270d8e007355 Mon Sep 17 00:00:00 2001 From: Noah Fiedel Date: Thu, 29 Jun 2017 12:09:21 -0800 Subject: [PATCH 0282/8554] Add clarifying comments to Regression & Classification SignatureDefs, along with detailed notes on the Predict SignatureDef. Change: 160563072 --- tensorflow_serving/g3doc/signature_defs.md | 27 ++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tensorflow_serving/g3doc/signature_defs.md b/tensorflow_serving/g3doc/signature_defs.md index 7315f00a314..efb8b0ccf8d 100644 --- a/tensorflow_serving/g3doc/signature_defs.md +++ b/tensorflow_serving/g3doc/signature_defs.md @@ -70,6 +70,11 @@ The actual contents of the TensorInfo are specific to your graph. ### Classification SignatureDef +Classification SignatureDefs support structured calls to TensorFlow Serving's +Classification API. These prescribe that there must be an `inputs` Tensor, and +that there are two optional output Tensors: `classes` and `scores`, at least one +of which must be present. + ~~~proto signature_def: { key : "my_classification_signature" @@ -105,6 +110,24 @@ signature_def: { ### Predict SignatureDef +Predict SignatureDefs support calls to TensorFlow Serving's Predict API. These +signatures allow you to flexibly support arbitrarily many input and output +Tensors. For the example below, the signature `my_prediction_signature` has a +single logical input Tensor `images` that are mapped to the actual Tensor in +your graph `x:0`. + +Predict SignatureDefs enable portability across models. This means that you can +swap in different SavedModels, possibly with different underlying Tensor names +(e.g. instead of `x:0` perhaps you have a new alternate model with a Tensor +`z:0`), while your clients can stay online continuously querying the old and new +versions of this model without client-side changes. + +Predict SignatureDefs also allow you to add optional additional Tensors to the +outputs, that you can explicitly query. Let's say that in addition to the output +key below of `scores`, you also wanted to fetch a pooling layer for debugging or +other purposes. In that case, you would simply add an additional Tensor with a +key like `pool` and appropriate value. + ~~~proto signature_def: { key : "my_prediction_signature" @@ -132,6 +155,10 @@ signature_def: { ### Regression SignatureDef +Regression SignatureDefs support structured calls to TensorFlow Serving's +Regression API. These prescribe that there must be exactly one `inputs` Tensor, +and one `outputs` Tensor. + ~~~proto signature_def: { key : "my_regression_signature" From a5e06e7a2fd5ac96136bfc77e6cdd08cb5b4996f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 29 Jun 2017 22:30:27 -0800 Subject: [PATCH 0283/8554] Automated rollback of change 160028181 Change: 160616014 --- tensorflow_serving/servables/tensorflow/BUILD | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index bd14e238ec1..93f3366308c 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -523,10 +523,10 @@ cc_library( "//tensorflow_serving/apis:classifier", "//tensorflow_serving/apis:input_proto", "//tensorflow_serving/apis:model_proto", - "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:loader_lite", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", + "@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_lite", + "@org_tensorflow//tensorflow/contrib/session_bundle:signature_lite", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", @@ -700,12 +700,10 @@ cc_library( deps = [ "//tensorflow_serving/apis:input_proto", "//tensorflow_serving/apis/internal:serialized_input_proto", - "@org_tensorflow//tensorflow/core:all_kernels", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@org_tensorflow//tensorflow/core:tensorflow", ], ) From 55ab4ca5b347d033fbc9d8c229cd8bb93a0f9ec6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 7 Jul 2017 07:27:43 -0800 Subject: [PATCH 0284/8554] Remove resource keys that aren't resources in the way we intend (they are more like performance metrics). Change: 161204085 --- tensorflow_serving/resources/resource_values.cc | 1 - tensorflow_serving/resources/resource_values.h | 3 --- 2 files changed, 4 deletions(-) diff --git a/tensorflow_serving/resources/resource_values.cc b/tensorflow_serving/resources/resource_values.cc index fae14e7d3fa..6f3e05e531f 100644 --- a/tensorflow_serving/resources/resource_values.cc +++ b/tensorflow_serving/resources/resource_values.cc @@ -27,7 +27,6 @@ namespace resource_kinds { const char* const kNumModelSlots = "num_model_slots"; const char* const kRamBytes = "ram_in_bytes"; const char* const kProcessingMillis = "processing_in_millicores"; -const char* const kP90LatencyMicros = "p90_latency_in_microseconds"; } // namespace resource_kinds } // namespace serving diff --git a/tensorflow_serving/resources/resource_values.h b/tensorflow_serving/resources/resource_values.h index 23e65df48f5..81616087043 100644 --- a/tensorflow_serving/resources/resource_values.h +++ b/tensorflow_serving/resources/resource_values.h @@ -48,9 +48,6 @@ extern const char* const kRamBytes; // Fraction of a processing unit's cycles, in thousandths. extern const char* const kProcessingMillis; -// 90th percentile request processing latency, measured in microseconds. -extern const char* const kP90LatencyMicros; - } // namespace resource_kinds } // namespace serving From 34081dc5bd6f92b23fd6a5ec768f86df13535b2a Mon Sep 17 00:00:00 2001 From: Skye Wanderman-Milne Date: Fri, 7 Jul 2017 13:11:40 -0800 Subject: [PATCH 0285/8554] Change for internal compatibility. --- tensorflow_serving/servables/tensorflow/predict_impl.cc | 1 + .../servables/tensorflow/saved_model_bundle_factory.cc | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index d2afe49c91b..a8bd44d292d 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/cc/saved_model/signature_constants.h" #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/contrib/session_bundle/signature.h" +#include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/protobuf/named_tensor.pb.h" #include "tensorflow_serving/core/servable_handle.h" diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc index d0979c734fe..97d58f5b28d 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -17,6 +17,7 @@ limitations under the License. #include "tensorflow/cc/saved_model/tag_constants.h" #include "tensorflow/contrib/session_bundle/bundle_shim.h" +#include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/protobuf/config.pb.h" #include "tensorflow/core/protobuf/named_tensor.pb.h" From ea8c60ce6daefe8f1d37024bd9e183c113a10bde Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 7 Jul 2017 15:53:35 -0700 Subject: [PATCH 0286/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 4c0052dc4b7..bef67104a0d 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 4c0052dc4b7c49a876166113b49960a57f7db939 +Subproject commit bef67104a0da34b6cac0d801168f2e42a1ab7921 diff --git a/tf_models b/tf_models index 7ad450b8430..3d97b007cdb 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 7ad450b84309b82bccb0e8f2e40e9559f33cd258 +Subproject commit 3d97b007cdb2c73f23daf9141437796095436c22 From 8f7a35dcc2004a1ceb63e8a7480f29863041c413 Mon Sep 17 00:00:00 2001 From: cheyang Date: Sun, 9 Jul 2017 22:27:44 +0800 Subject: [PATCH 0287/8554] Add build for CPU (#502) --- tensorflow_serving/tools/docker/Dockerfile.devel | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 7b8402e6e4f..3f5328b2083 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -44,4 +44,17 @@ RUN mkdir /bazel && \ cd / && \ rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh +# Download TensorFlow Serving +RUN git clone --recurse-submodules https://github.com/tensorflow/serving && \ + cd serving && \ + git checkout + +WORKDIR /serving/tensorflow +RUN tensorflow/tools/ci_build/builds/configured CPU + +WORKDIR /serving +RUN bazel build -c opt tensorflow_serving/... && \ + cp bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server /usr/local/bin/ && \ + bazel clean --expunge + CMD ["/bin/bash"] From 96c98c53bfa646cf71af383e901e7cbf1e114c37 Mon Sep 17 00:00:00 2001 From: Pankaj Gupta Date: Sun, 9 Jul 2017 10:30:52 -0400 Subject: [PATCH 0288/8554] add wget to list of apps to be installed (#506) The imagenet training script uses wget to download datasets and fails since it cannot find wget command in the docker container. Hence adding wget to the list would solve it. --- tensorflow_serving/tools/docker/Dockerfile.devel | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 3f5328b2083..de1cd5ff7ed 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -21,6 +21,7 @@ RUN apt-get update && apt-get install -y \ libcurl3-dev \ openjdk-8-jdk\ openjdk-8-jre-headless \ + wget \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* From 45aad04fd5f9e3d3f8932243b14cba8deb9fb463 Mon Sep 17 00:00:00 2001 From: Xingyuan Bu Date: Mon, 10 Jul 2017 21:34:24 +0800 Subject: [PATCH 0289/8554] Add srcs_version = "PY2AND3" in BUILD files (#504) --- tensorflow_serving/apis/BUILD | 1 + tensorflow_serving/example/BUILD | 7 +++++++ tensorflow_serving/model_servers/BUILD | 1 + tensorflow_serving/servables/tensorflow/testdata/BUILD | 2 ++ 4 files changed, 11 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 62cd8392865..0e884e84dd7 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -126,6 +126,7 @@ serving_proto_library( py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], + srcs_version = "PY2AND3", deps = [ ":classification_proto_py_pb2", ":get_model_metadata_proto_py_pb2", diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 19150a86301..2d496c5f352 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -22,6 +22,7 @@ filegroup( py_library( name = "mnist_input_data", + srcs_version = "PY2AND3", srcs = ["mnist_input_data.py"], ) @@ -32,6 +33,7 @@ py_binary( srcs = [ "mnist_export.py", ], + srcs_version = "PY2AND3", deps = [ ":mnist_input_data", "@org_tensorflow//tensorflow:tensorflow_py", @@ -44,6 +46,7 @@ py_binary( srcs = [ "mnist_saved_model.py", ], + srcs_version = "PY2AND3", deps = [ ":mnist_input_data", "@org_tensorflow//tensorflow:tensorflow_py", @@ -55,6 +58,7 @@ py_binary( srcs = [ "mnist_client.py", ], + srcs_version = "PY2AND3", deps = [ ":mnist_input_data", "//tensorflow_serving/apis:predict_proto_py_pb2", @@ -70,6 +74,7 @@ py_binary( srcs = [ "inception_export.py", ], + srcs_version = "PY2AND3", deps = [ "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", @@ -82,6 +87,7 @@ py_binary( srcs = [ "inception_saved_model.py", ], + srcs_version = "PY2AND3", deps = [ "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", @@ -93,6 +99,7 @@ py_binary( srcs = [ "inception_client.py", ], + srcs_version = "PY2AND3", deps = [ "//tensorflow_serving/apis:predict_proto_py_pb2", "//tensorflow_serving/apis:prediction_service_proto_py_pb2", diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 313e35f9113..69f1c307348 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -169,6 +169,7 @@ py_test( py_binary( name = "tensorflow_model_server_test_client", srcs = ["tensorflow_model_server_test_client.py"], + srcs_version = "PY2AND3", deps = [ "//tensorflow_serving/apis:prediction_service_proto_py_pb2", "@org_tensorflow//tensorflow:tensorflow_py", diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 5cfbac11907..49d6079f89b 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -25,6 +25,7 @@ filegroup( py_binary( name = "export_half_plus_two", + srcs_version = "PY2AND3", srcs = [ "export_half_plus_two.py", ], @@ -36,6 +37,7 @@ py_binary( py_binary( name = "export_bad_half_plus_two", + srcs_version = "PY2AND3", srcs = [ "export_bad_half_plus_two.py", ], From c0fe4830f1fb64547ed073205346ffbc93d4aaf7 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 11 Jul 2017 09:41:36 -0800 Subject: [PATCH 0290/8554] Add deb package build targets. Change: 161548730 --- tensorflow_serving/model_servers/BUILD | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 313e35f9113..651e7bddd82 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -174,3 +174,23 @@ py_binary( "@org_tensorflow//tensorflow:tensorflow_py", ], ) + +load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_deb", "pkg_tar") + +pkg_tar( + name = "tensorflow_model_server_tar", + files = [ + ":tensorflow_model_server", + ], + package_dir = "/usr/bin", +) + +pkg_deb( + name = "tensorflow_model_server_deb", + data = ":tensorflow_model_server_tar", + description = "TensorFlow Serving ModelServer", + homepage = "https://github.com/tensorflow/serving", + maintainer = "TensorFlow Serving team", + package = "tensorflow-model-server", + version = "1.0.0", +) From a09e494c3e792165573273913fc178a4b6f6fd7a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 11 Jul 2017 11:32:01 -0800 Subject: [PATCH 0291/8554] Small performance-related improvements: * Preallocate known vector capacities * Move rather than copy objects or use references rather than copies whenever possible Change: 161565802 --- tensorflow_serving/batching/batching_session.cc | 1 + tensorflow_serving/batching/streaming_batch_scheduler.cc | 6 ++++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index cff81f9a955..e0047fac47f 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -406,6 +406,7 @@ Status BatchingSession::SplitOutputTensors( } std::vector task_sizes_plus_optional_padding; + task_sizes_plus_optional_padding.reserve(batch->num_tasks()); for (int i = 0; i < batch->num_tasks(); ++i) { task_sizes_plus_optional_padding.push_back(batch->task(i).zeroth_dim_size); } diff --git a/tensorflow_serving/batching/streaming_batch_scheduler.cc b/tensorflow_serving/batching/streaming_batch_scheduler.cc index 8cb0a45e843..b33b4d4a453 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler.cc +++ b/tensorflow_serving/batching/streaming_batch_scheduler.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow_serving/batching/streaming_batch_scheduler.h" +#include + namespace tensorflow { namespace serving { @@ -25,7 +27,7 @@ namespace internal { SingleTaskScheduler::SingleTaskScheduler(Env* env, string thread_name, uint64 no_tasks_wait_time_micros) : env_(env), - thread_name_(thread_name), + thread_name_(std::move(thread_name)), no_tasks_wait_time_micros_(no_tasks_wait_time_micros) {} SingleTaskScheduler::~SingleTaskScheduler() { stop_.Notify(); } @@ -37,7 +39,7 @@ void SingleTaskScheduler::Schedule(uint64 time_micros, { mutex_lock l(mu_); - updated_task_ = {time_micros, closure}; + updated_task_ = {time_micros, std::move(closure)}; } if (thread_ == nullptr) { From acec5255b57ec85ecfd08040d0fdd2abdad813fd Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 11 Jul 2017 12:52:36 -0800 Subject: [PATCH 0292/8554] Merge changes from github. Change: 161576012 --- tensorflow_serving/apis/BUILD | 1 + tensorflow_serving/example/BUILD | 7 +++ tensorflow_serving/g3doc/leftnav_files | 9 ++++ tensorflow_serving/model_servers/BUILD | 1 + .../servables/tensorflow/testdata/BUILD | 2 + .../tools/docker/Dockerfile.devel | 49 ++++++++----------- 6 files changed, 40 insertions(+), 29 deletions(-) create mode 100644 tensorflow_serving/g3doc/leftnav_files diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 62cd8392865..0e884e84dd7 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -126,6 +126,7 @@ serving_proto_library( py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], + srcs_version = "PY2AND3", deps = [ ":classification_proto_py_pb2", ":get_model_metadata_proto_py_pb2", diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 19150a86301..af19bbc222f 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -23,6 +23,7 @@ filegroup( py_library( name = "mnist_input_data", srcs = ["mnist_input_data.py"], + srcs_version = "PY2AND3", ) # TODO(b/32628014): remove mnist_export after we no longer support @@ -32,6 +33,7 @@ py_binary( srcs = [ "mnist_export.py", ], + srcs_version = "PY2AND3", deps = [ ":mnist_input_data", "@org_tensorflow//tensorflow:tensorflow_py", @@ -44,6 +46,7 @@ py_binary( srcs = [ "mnist_saved_model.py", ], + srcs_version = "PY2AND3", deps = [ ":mnist_input_data", "@org_tensorflow//tensorflow:tensorflow_py", @@ -55,6 +58,7 @@ py_binary( srcs = [ "mnist_client.py", ], + srcs_version = "PY2AND3", deps = [ ":mnist_input_data", "//tensorflow_serving/apis:predict_proto_py_pb2", @@ -70,6 +74,7 @@ py_binary( srcs = [ "inception_export.py", ], + srcs_version = "PY2AND3", deps = [ "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", @@ -82,6 +87,7 @@ py_binary( srcs = [ "inception_saved_model.py", ], + srcs_version = "PY2AND3", deps = [ "@inception_model//inception", "@org_tensorflow//tensorflow:tensorflow_py", @@ -93,6 +99,7 @@ py_binary( srcs = [ "inception_client.py", ], + srcs_version = "PY2AND3", deps = [ "//tensorflow_serving/apis:predict_proto_py_pb2", "//tensorflow_serving/apis:prediction_service_proto_py_pb2", diff --git a/tensorflow_serving/g3doc/leftnav_files b/tensorflow_serving/g3doc/leftnav_files new file mode 100644 index 00000000000..146e204d6a1 --- /dev/null +++ b/tensorflow_serving/g3doc/leftnav_files @@ -0,0 +1,9 @@ +### TensorFlow Serving +architecture_overview.md +setup.md +serving_basic.md +serving_advanced.md +serving_inception.md +custom_servable.md +custom_source.md +docker.md \ No newline at end of file diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 651e7bddd82..3d02efd6a06 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -169,6 +169,7 @@ py_test( py_binary( name = "tensorflow_model_server_test_client", srcs = ["tensorflow_model_server_test_client.py"], + srcs_version = "PY2AND3", deps = [ "//tensorflow_serving/apis:prediction_service_proto_py_pb2", "@org_tensorflow//tensorflow:tensorflow_py", diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 5cfbac11907..842ce699782 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -28,6 +28,7 @@ py_binary( srcs = [ "export_half_plus_two.py", ], + srcs_version = "PY2AND3", deps = [ "@org_tensorflow//tensorflow:tensorflow_py", "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", @@ -39,6 +40,7 @@ py_binary( srcs = [ "export_bad_half_plus_two.py", ], + srcs_version = "PY2AND3", deps = [ "@org_tensorflow//tensorflow:tensorflow_py", ], diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index 8ea5fc08c71..de1cd5ff7ed 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -1,4 +1,4 @@ -FROM ubuntu:14.04 +FROM ubuntu:16.04 MAINTAINER Jeremiah Harmsen @@ -9,6 +9,7 @@ RUN apt-get update && apt-get install -y \ libfreetype6-dev \ libpng12-dev \ libzmq3-dev \ + mlocate \ pkg-config \ python-dev \ python-numpy \ @@ -18,45 +19,22 @@ RUN apt-get update && apt-get install -y \ zip \ zlib1g-dev \ libcurl3-dev \ + openjdk-8-jdk\ + openjdk-8-jre-headless \ + wget \ && \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py && \ - rm get-pip.py - # Set up grpc -RUN pip install enum34 futures mock six && \ - pip install --pre 'protobuf>=3.0.0a3' && \ - pip install 'grpcio>=1.1.3' +RUN pip install mock grpcio # Set up Bazel. -# We need to add a custom PPA to pick up JDK8, since trusty doesn't -# have an openjdk8 backport. openjdk-r is maintained by a reliable contributor: -# Matthias Klose (https://launchpad.net/~doko). It will do until -# we either update the base image beyond 14.04 or openjdk-8 is -# finally backported to trusty; see e.g. -# https://bugs.launchpad.net/trusty-backports/+bug/1368094 -RUN add-apt-repository -y ppa:openjdk-r/ppa && \ - apt-get update && \ - apt-get install -y openjdk-8-jdk openjdk-8-jre-headless && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -# Running bazel inside a `docker build` command causes trouble, cf: -# https://github.com/bazelbuild/bazel/issues/134 -# The easiest solution is to set up a bazelrc file forcing --batch. -RUN echo "startup --batch" >>/root/.bazelrc -# Similarly, we need to workaround sandboxing issues: -# https://github.com/bazelbuild/bazel/issues/418 -RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ - >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.4.5 +ENV BAZEL_VERSION 0.5.1 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ @@ -67,4 +45,17 @@ RUN mkdir /bazel && \ cd / && \ rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh +# Download TensorFlow Serving +RUN git clone --recurse-submodules https://github.com/tensorflow/serving && \ + cd serving && \ + git checkout + +WORKDIR /serving/tensorflow +RUN tensorflow/tools/ci_build/builds/configured CPU + +WORKDIR /serving +RUN bazel build -c opt tensorflow_serving/... && \ + cp bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server /usr/local/bin/ && \ + bazel clean --expunge + CMD ["/bin/bash"] From 89f31a36f22000d4d519b7e594dace2807a7f38d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 12 Jul 2017 12:09:58 -0800 Subject: [PATCH 0293/8554] Adding a test to validate the renaming a version directory removes the corresponding model version from the list of aspired versions and from serving. Change: 161706005 --- .../file_system_storage_path_source_test.cc | 69 +++++++++++++++++++ 1 file changed, 69 insertions(+) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index 3fa36304244..350e0801a85 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -375,6 +375,75 @@ TEST(FileSystemStoragePathSourceTest, ParseTimestampedVersion) { .PollFileSystemAndInvokeCallback()); } +TEST(FileSystemStoragePathSourceTest, ModelVersionDirRenamed) { + // Create one servable that is set up to serve two latest version. + const string base_path_prefix = + io::JoinPath(testing::TmpDir(), "ModelVersionDirRenamed_"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path_prefix)); + for (const string& version : {"1", "2", "3", "5", "8"}) { + TF_ASSERT_OK( + Env::Default()->CreateDir(io::JoinPath(base_path_prefix, version))); + } + + auto config = test_util::CreateProto( + strings::Printf("servables: { " + " version_policy: ALL_VERSIONS " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path_prefix.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 1}, + io::JoinPath(base_path_prefix, "1")), + ServableData({"test_servable_name", 2}, + io::JoinPath(base_path_prefix, "2")), + ServableData({"test_servable_name", 3}, + io::JoinPath(base_path_prefix, "3")), + ServableData({"test_servable_name", 5}, + io::JoinPath(base_path_prefix, "5")), + ServableData({"test_servable_name", 8}, + io::JoinPath(base_path_prefix, "8"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); + + // Blacklist version 2 and 5 by renaming corresponding directories. + // Blacklist version 8 by removing the directory alltogether. + TF_ASSERT_OK(Env::Default()->RenameFile( + io::JoinPath(base_path_prefix, "2"), + io::JoinPath(base_path_prefix, "2.blacklisted"))); + TF_ASSERT_OK(Env::Default()->RenameFile( + io::JoinPath(base_path_prefix, "5"), + io::JoinPath(base_path_prefix, "5.blacklisted"))); + TF_ASSERT_OK(Env::Default()->DeleteDir(io::JoinPath(base_path_prefix, "8"))); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 1}, + io::JoinPath(base_path_prefix, "1")), + ServableData({"test_servable_name", 3}, + io::JoinPath(base_path_prefix, "3"))))); + + TF_ASSERT_OK(source->UpdateConfig(config)); + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + } // namespace } // namespace serving } // namespace tensorflow From 2c75f81b1e571a4f347589d2ac9f93de975c5ab7 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 13 Jul 2017 11:25:39 -0800 Subject: [PATCH 0294/8554] Correct the terminology model->servable in the new storage-path-source test. Change: 161846440 --- .../storage_path/file_system_storage_path_source_test.cc | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index 350e0801a85..07ce7955678 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -375,10 +375,10 @@ TEST(FileSystemStoragePathSourceTest, ParseTimestampedVersion) { .PollFileSystemAndInvokeCallback()); } -TEST(FileSystemStoragePathSourceTest, ModelVersionDirRenamed) { - // Create one servable that is set up to serve two latest version. +TEST(FileSystemStoragePathSourceTest, ServableVersionDirRenamed) { + // Create one servable that is set up to serve the two latest versions. const string base_path_prefix = - io::JoinPath(testing::TmpDir(), "ModelVersionDirRenamed_"); + io::JoinPath(testing::TmpDir(), "ServableVersionDirRenamed_"); TF_ASSERT_OK(Env::Default()->CreateDir(base_path_prefix)); for (const string& version : {"1", "2", "3", "5", "8"}) { TF_ASSERT_OK( From fd90ee6165c03b5a81fff16e60e0e5db00a68f81 Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Fri, 14 Jul 2017 05:53:21 -0700 Subject: [PATCH 0295/8554] Remove duplicate PY2AND3 lines from BUILD files. --- tensorflow_serving/example/BUILD | 1 - tensorflow_serving/servables/tensorflow/testdata/BUILD | 2 -- 2 files changed, 3 deletions(-) diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index 88951544563..af19bbc222f 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -22,7 +22,6 @@ filegroup( py_library( name = "mnist_input_data", - srcs_version = "PY2AND3", srcs = ["mnist_input_data.py"], srcs_version = "PY2AND3", ) diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 729cfb6edc7..842ce699782 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -25,7 +25,6 @@ filegroup( py_binary( name = "export_half_plus_two", - srcs_version = "PY2AND3", srcs = [ "export_half_plus_two.py", ], @@ -38,7 +37,6 @@ py_binary( py_binary( name = "export_bad_half_plus_two", - srcs_version = "PY2AND3", srcs = [ "export_bad_half_plus_two.py", ], From 1f54d1c1abe5f70c29262085ed3b4128d6e9a51d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 17 Jul 2017 14:30:07 -0800 Subject: [PATCH 0296/8554] Implement new model version serving policies for the ModelServer deployment. The new model version policies allow users to serve N latest models or a list of specific versions in addition to existing "serve single latest version" or "serve all available versions" policies. Change: 162279912 --- .../config/model_server_config.proto | 9 +- tensorflow_serving/model_servers/main.cc | 41 +-- .../model_servers/server_core.cc | 19 +- .../file_system_storage_path_source.cc | 113 +++++--- .../file_system_storage_path_source.proto | 39 ++- .../file_system_storage_path_source_test.cc | 251 +++++++++++++++++- 6 files changed, 402 insertions(+), 70 deletions(-) diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index cc172297ddc..e69928fcdf1 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -38,12 +38,19 @@ message ModelConfig { // (This cannot be changed once a model is in serving.) string model_platform = 4; + // DEPRECATED: This field is deprecated. For now it's still obeyed as long as + // 'model_version_policy' is not set. If 'model_version_policy' is set, then + // the value of this field is ignored. + FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5 + [deprecated = true]; + // Version policy for the model indicating how many versions of the model to // be served at the same time. // The default option is to serve only the latest version of the model. // // (This can be changed once a model is in serving.) - FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5; + FileSystemStoragePathSourceConfig.ServableVersionPolicy model_version_policy = + 7; // Configures logging requests and responses, to the model. // diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 7e33f10bc96..a582afc33b9 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -143,22 +143,18 @@ tensorflow::Status LoadCustomModelConfig( << "ModelServer does not yet support custom model config."; } -ModelServerConfig BuildSingleModelConfig( - const string& model_name, const string& model_base_path, - const FileSystemStoragePathSourceConfig_VersionPolicy& - model_version_policy) { +ModelServerConfig BuildSingleModelConfig(const string& model_name, + const string& model_base_path) { ModelServerConfig config; LOG(INFO) << "Building single TensorFlow model file config: " << " model_name: " << model_name - << " model_base_path: " << model_base_path - << " model_version_policy: " << model_version_policy; + << " model_base_path: " << model_base_path; tensorflow::serving::ModelConfig* single_model = config.mutable_model_config_list()->add_config(); single_model->set_name(model_name); single_model->set_base_path(model_base_path); single_model->set_model_platform( tensorflow::serving::kTensorFlowModelPlatform); - single_model->set_version_policy(model_version_policy); return config; } @@ -318,9 +314,6 @@ int main(int argc, char** argv) { tensorflow::int64 tensorflow_session_parallelism = 0; string platform_config_file = ""; string model_config_file; - tensorflow::string model_version_policy = - FileSystemStoragePathSourceConfig_VersionPolicy_Name( - FileSystemStoragePathSourceConfig::LATEST_VERSION); std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), @@ -331,24 +324,16 @@ int main(int argc, char** argv) { tensorflow::Flag("model_config_file", &model_config_file, "If non-empty, read an ascii ModelServerConfig " "protobuf from the supplied file name, and serve the " - "models in that file. (If used, --model_name, " - "--model_base_path and --model_version_policy " - "are ignored.)"), + "models in that file. This config file can be used to " + "specify multiple models to serve and other advanced " + "parameters including non-default version policy. (If " + "used, --model_name, --model_base_path are ignored.)"), tensorflow::Flag("model_name", &model_name, "name of model (ignored " "if --model_config_file flag is set"), tensorflow::Flag("model_base_path", &model_base_path, "path to export (ignored if --model_config_file flag " "is set, otherwise required)"), - tensorflow::Flag( - "model_version_policy", &model_version_policy, - "The version policy which determines the number of model " - "versions to be served at the same time. The default " - "value is LATEST_VERSION, which will serve only the " - "latest version. " - "See file_system_storage_path_source.proto for " - "the list of possible VersionPolicy. (Ignored if " - "--model_config_file flag is set)"), tensorflow::Flag("file_system_poll_wait_seconds", &file_system_poll_wait_seconds, "interval in seconds between each poll of the file " @@ -381,22 +366,14 @@ int main(int argc, char** argv) { std::cout << "unknown argument: " << argv[1] << "\n" << usage; } - FileSystemStoragePathSourceConfig_VersionPolicy parsed_version_policy; - bool valid_policy = FileSystemStoragePathSourceConfig_VersionPolicy_Parse( - model_version_policy, &parsed_version_policy); - QCHECK(valid_policy) // Crash ok. - << "Invalid model_version_policy input argument: " << model_version_policy - << "\n" - << usage; - // For ServerCore Options, we leave servable_state_monitor_creator unspecified // so the default servable_state_monitor_creator will be used. ServerCore::Options options; // model server config if (model_config_file.empty()) { - options.model_server_config = BuildSingleModelConfig( - model_name, model_base_path, parsed_version_policy); + options.model_server_config = + BuildSingleModelConfig(model_name, model_base_path); } else { options.model_server_config = ReadProtoFromFile(model_config_file); diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 8eb49da68a3..d813005486c 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -423,7 +423,24 @@ FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( source_config.add_servables(); servable->set_servable_name(model.name()); servable->set_base_path(model.base_path()); - servable->set_version_policy(model.version_policy()); + // TODO(akhorlin): remove this logic once the corresponding deprecated + // field is removed (b/62834753). + if (!model.has_model_version_policy()) { + switch (model.version_policy()) { + case FileSystemStoragePathSourceConfig::LATEST_VERSION: + servable->mutable_servable_version_policy()->mutable_latest(); + break; + case FileSystemStoragePathSourceConfig::ALL_VERSIONS: + servable->mutable_servable_version_policy()->mutable_all(); + break; + default: + LOG(FATAL) << "Unknown version policy: " // Crash ok. + << model.version_policy(); + } + } else { + *servable->mutable_servable_version_policy() = + model.model_version_policy(); + } } return source_config; } diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 16797ed985f..7c39052a592 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include #include #include "tensorflow/core/lib/core/errors.h" @@ -76,7 +77,7 @@ std::set GetDeletedServables( return deleted_servables; } -// Adds a new ServableData for the model version to the vector of versions to +// Adds a new ServableData for the servable version to the vector of versions to // aspire. void AspireVersion( const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, @@ -94,8 +95,8 @@ bool ParseVersionNumber(const string& version_path, int64* version_number) { return strings::safe_strto64(version_path.c_str(), version_number); } -// Update the servable data to include all the model versions found in the base -// path as aspired versions. +// Update the servable data to include all the servable versions found in the +// base path as aspired versions. // The argument 'children' represents a list of base-path children from the file // system. // Returns true if one or more valid servable version paths are found, otherwise @@ -119,41 +120,79 @@ bool AspireAllVersions( return at_least_one_version_found; } -// Update the servable data to include the latest version found in the base path -// as aspired version. -// The argument 'children' represents a list of base-path children from the file -// system. -// Returns true if a valid servable path is found, otherwise returns false. -bool AspireLatestVersion( - const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, - const std::vector& children, - std::vector>* versions) { - // Identify the latest version among children that can be interpreted as - // version numbers. - int latest_version_child_index; - int64 latest_version_number; - bool at_least_one_version_found = false; +// Helper that indexes a list of the given "children" (where child is the +// name of the directory corresponding to a servable version). Note that strings +// that cannot be parsed as a number are skipped (no error is returned). +std::map +IndexChildrenByVersion(const std::vector& children) { + std::map children_by_version; for (int i = 0; i < children.size(); ++i) { int64 version_number; if (!ParseVersionNumber(children[i], &version_number)) { continue; } - // Check if this is the largest version number. - if (!at_least_one_version_found || latest_version_number < version_number) { - latest_version_child_index = i; - latest_version_number = version_number; + children_by_version[version_number] = children[i]; + } + return children_by_version; +} + +// Aspire versions for a servable configured with the "latest" version policy. +// +// 'children' represents a list of base-path children from the file system. +// +// Returns true iff it winds up aspiring at least one version. +bool AspireLatestVersions( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const std::map& children_by_version, + std::vector>* versions) { + const int32 num_servable_versions_to_serve = + std::max(servable.servable_version_policy().latest().num_versions(), 1U); + // Identify 'num_servable_versions_to_serve' latest version(s) among children + // that can be interpreted as version numbers and emit as aspired versions. + int num_versions_emitted = 0; + for (auto rit = children_by_version.rbegin(); + rit != children_by_version.rend(); ++rit) { + if (num_versions_emitted == num_servable_versions_to_serve) { + break; } - at_least_one_version_found = true; + const int64 version = rit->first; + const string& child = rit->second; + AspireVersion(servable, child, version, versions); + num_versions_emitted++; } - // Emit the latest aspired version. - if (at_least_one_version_found) { - AspireVersion(servable, children[latest_version_child_index], - latest_version_number, versions); + return !children_by_version.empty(); +} + +// Aspire versions for a servable configured with the "specific" version policy. +// +// 'children' represents a list of base-path children from the file system. +// +// Returns true iff it winds up aspiring at least one version. +bool AspireSpecificVersions( + const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, + const std::map& children_by_version, + std::vector>* versions) { + const std::unordered_set versions_to_serve( + servable.servable_version_policy().specific().versions().begin(), + servable.servable_version_policy().specific().versions().end()); + // Identify specific version to serve (as specified by 'versions_to_serve') + // among children that can be interpreted as version numbers and emit as + // aspired versions. + bool at_least_one_version_emitted = false; + for (auto it = children_by_version.begin(); it != children_by_version.end(); + ++it) { + const int64 version = it->first; + if (versions_to_serve.count(version) == 0) { + continue; // Current version is not specified by policy for serving. + } + const string& child = it->second; + AspireVersion(servable, child, version, versions); + at_least_one_version_emitted = true; } - return at_least_one_version_found; + return at_least_one_version_emitted; } // Like PollFileSystemForConfig(), but for a single servable. @@ -184,20 +223,30 @@ Status PollFileSystemForServable( } children.clear(); children.insert(children.begin(), real_children.begin(), real_children.end()); + const std::map children_by_version = + IndexChildrenByVersion(children); bool at_least_one_version_found = false; - switch (servable.version_policy()) { - case FileSystemStoragePathSourceConfig::LATEST_VERSION: + switch (servable.servable_version_policy().policy_choice_case()) { + case FileSystemStoragePathSourceConfig::ServableVersionPolicy:: + POLICY_CHOICE_NOT_SET: + FALLTHROUGH_INTENDED; // Default policy is kLatest. + case FileSystemStoragePathSourceConfig::ServableVersionPolicy::kLatest: at_least_one_version_found = - AspireLatestVersion(servable, children, versions); + AspireLatestVersions(servable, children_by_version, versions); break; - case FileSystemStoragePathSourceConfig::ALL_VERSIONS: + case FileSystemStoragePathSourceConfig::ServableVersionPolicy::kAll: at_least_one_version_found = AspireAllVersions(servable, children, versions); break; + case FileSystemStoragePathSourceConfig::ServableVersionPolicy::kSpecific: { + at_least_one_version_found = + AspireSpecificVersions(servable, children_by_version, versions); + break; + } default: return errors::Internal("Unhandled servable version_policy: ", - servable.version_policy()); + servable.servable_version_policy().DebugString()); } if (!at_least_one_version_found) { diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto index 5f067c45221..8b11ce8237a 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto @@ -4,6 +4,40 @@ package tensorflow.serving; // Config proto for FileSystemStoragePathSource. message FileSystemStoragePathSourceConfig { + // A policy that dictates which version(s) of a servable should be served. + message ServableVersionPolicy { + // Serve the latest versions (i.e. the ones with the highest version + // numbers), among those found on disk. + // + // This is the default policy, with the default number of versions as 1. + message Latest { + // Number of latest versions to serve. (The default is 1.) + uint32 num_versions = 1; + } + + // Serve all versions found on disk. + message All { + } + + // Serve a specific version (or set of versions). + // + // This policy is useful for rolling back to a specific version, or for + // canarying a specific version while still serving a separate stable + // version. + message Specific { + // The version numbers to serve. + repeated int64 versions = 1; + } + + oneof policy_choice { + Latest latest = 100; + All all = 101; + Specific specific = 102; + } + } + + // DEPRECATED: Please use VersionPolicy message definition instead. The + // enum will be removed once noone depends on it any longer. // The policy to define how many versions of the servable should be // served at the same time. enum VersionPolicy { @@ -25,7 +59,10 @@ message FileSystemStoragePathSourceConfig { // The policy to determines the number of versions of the servable to be // served at the same time. - VersionPolicy version_policy = 3; + tensorflow.serving.FileSystemStoragePathSourceConfig.ServableVersionPolicy + servable_version_policy = 4; + + reserved 3; // Legacy version_policy definition. }; // The servables to monitor for new versions, and aspire. diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index 07ce7955678..2e52fb3436d 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow/core/platform/env.h" @@ -181,7 +182,10 @@ TEST(FileSystemStoragePathSourceTest, MultipleVersionsAtTheSameTime) { auto config = test_util::CreateProto( strings::Printf("servables: { " - " version_policy: ALL_VERSIONS " + " servable_version_policy { " + " all { " + " } " + " } " " servable_name: 'test_servable_name' " " base_path: '%s' " "} " @@ -208,6 +212,166 @@ TEST(FileSystemStoragePathSourceTest, MultipleVersionsAtTheSameTime) { .PollFileSystemAndInvokeCallback()); } +TEST(FileSystemStoragePathSourceTest, NLatestVersions) { + const string base_path = + io::JoinPath(testing::TmpDir(), "NLatestVersionsAtTheSameTime"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + for (const string& version : + {"non_numerical_child", "42", "33", "30", "21", "17"}) { + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, version))); + } + + const FileSystemStoragePathSourceConfig config = + test_util::CreateProto( + strings::Printf("servables: { " + " servable_version_policy { " + " latest { " + " num_versions: 3" + " } " + " } " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 42}, + io::JoinPath(base_path, "42")), + ServableData({"test_servable_name", 33}, + io::JoinPath(base_path, "33")), + ServableData({"test_servable_name", 30}, + io::JoinPath(base_path, "30"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + +TEST(FileSystemStoragePathSourceTest, SpecificVersions) { + const string base_path = io::JoinPath(testing::TmpDir(), "SpecificVersions"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + for (const string& version : + {"non_numerical_child", "42", "33", "30", "21", "17"}) { + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, version))); + } + + const FileSystemStoragePathSourceConfig config = + test_util::CreateProto( + strings::Printf("servables: { " + " servable_version_policy { " + " specific { " + " versions: 17" + " versions: 30" + " } " + " } " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 17}, + io::JoinPath(base_path, "17")), + ServableData({"test_servable_name", 30}, + io::JoinPath(base_path, "30"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + +TEST(FileSystemStoragePathSourceTest, DefaultVersionPolicy) { + // Validate that default version policy is to serve the latest servable + // version. + const string base_path = + io::JoinPath(testing::TmpDir(), "DefaultVersionPolicy"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + for (const string& version : {"non_numerical_child", "42", "33", "30"}) { + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, version))); + } + + const FileSystemStoragePathSourceConfig config = + test_util::CreateProto( + strings::Printf("servables: { " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL(*target, SetAspiredVersions(Eq("test_servable_name"), + ElementsAre(ServableData( + {"test_servable_name", 42}, + io::JoinPath(base_path, "42"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + +TEST(FileSystemStoragePathSourceTest, DefaultNumLatestVersions) { + // Validate that if num_versions in latest servable version policy is not + // specified, the default is 1. + const string base_path = + io::JoinPath(testing::TmpDir(), "DefaultNumLatestVersions"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + for (const string& version : {"non_numerical_child", "42", "33", "30"}) { + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, version))); + } + + const FileSystemStoragePathSourceConfig config = + test_util::CreateProto( + strings::Printf("servables: { " + " servable_version_policy { " + " latest { " + " } " + " } " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL(*target, SetAspiredVersions(Eq("test_servable_name"), + ElementsAre(ServableData( + {"test_servable_name", 42}, + io::JoinPath(base_path, "42"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + TEST(FileSystemStoragePathSourceTest, MultipleServables) { FileSystemStoragePathSourceConfig config; config.set_fail_if_zero_versions_at_startup(false); @@ -328,6 +492,81 @@ TEST(FileSystemStoragePathSourceTest, ChangeSetOfServables) { .PollFileSystemAndInvokeCallback()); } +TEST(FileSystemStoragePathSourceTest, ChangeVersionPolicy) { + // Create one servable and configure the source to serve the two latest + // versions. + const string base_path_prefix = + io::JoinPath(testing::TmpDir(), "ChangeVersionPolicy_"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path_prefix)); + for (const string& version : {"1", "2", "3", "5", "8", "13"}) { + TF_ASSERT_OK( + Env::Default()->CreateDir(io::JoinPath(base_path_prefix, version))); + } + + FileSystemStoragePathSourceConfig config = + test_util::CreateProto( + strings::Printf("servables: { " + " servable_version_policy { " + " latest { " + " num_versions: 2" + " } " + " } " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path_prefix.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 13}, + io::JoinPath(base_path_prefix, "13")), + ServableData({"test_servable_name", 8}, + io::JoinPath(base_path_prefix, "8"))))); + + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); + + // Reconfigure the source to have serve specific versions (2 and 5). + config = test_util::CreateProto( + strings::Printf("servables: { " + " servable_version_policy { " + " specific { " + " versions: 2" + " versions: 5" + " } " + " } " + " servable_name: 'test_servable_name' " + " base_path: '%s' " + "} " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path_prefix.c_str())); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + ElementsAre( + ServableData({"test_servable_name", 2}, + io::JoinPath(base_path_prefix, "2")), + ServableData({"test_servable_name", 5}, + io::JoinPath(base_path_prefix, "5"))))); + + TF_ASSERT_OK(source->UpdateConfig(config)); + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + TEST(FileSystemStoragePathSourceTest, AttemptToChangePollingPeriod) { FileSystemStoragePathSourceConfig config; config.set_file_system_poll_wait_seconds(1); @@ -352,7 +591,10 @@ TEST(FileSystemStoragePathSourceTest, ParseTimestampedVersion) { Env::Default()->CreateDir(io::JoinPath(base_path, "20170111173521"))); auto config = test_util::CreateProto( strings::Printf("servables: { " - " version_policy: ALL_VERSIONS " + " servable_version_policy { " + " all { " + " } " + " } " " servable_name: 'test_servable_name' " " base_path: '%s' " "} " @@ -387,7 +629,10 @@ TEST(FileSystemStoragePathSourceTest, ServableVersionDirRenamed) { auto config = test_util::CreateProto( strings::Printf("servables: { " - " version_policy: ALL_VERSIONS " + " servable_version_policy { " + " all { " + " } " + " } " " servable_name: 'test_servable_name' " " base_path: '%s' " "} " From 9db98bab2c61b0690b6422206245e918b59878d2 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 21 Jul 2017 15:01:03 -0800 Subject: [PATCH 0297/8554] Add more Doxygen documentation to some classes. Change: 162796197 --- tensorflow_serving/apis/classifier.h | 15 ++++- tensorflow_serving/apis/regressor.h | 15 ++++- tensorflow_serving/core/loader_harness.h | 6 ++ .../core/servable_state_monitor.h | 3 + tensorflow_serving/core/source_adapter.h | 4 ++ .../tensorflow/saved_model_bundle_factory.h | 57 ++++++++++++------- 6 files changed, 75 insertions(+), 25 deletions(-) diff --git a/tensorflow_serving/apis/classifier.h b/tensorflow_serving/apis/classifier.h index 50fb1466006..fbd5ef573f0 100644 --- a/tensorflow_serving/apis/classifier.h +++ b/tensorflow_serving/apis/classifier.h @@ -12,8 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// -// Interface for performing classification using classification messages. #ifndef TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ #define TENSORFLOW_SERVING_APIS_CLASSIFIER_H_ @@ -24,8 +22,21 @@ limitations under the License. namespace tensorflow { namespace serving { +/// Model-type agnostic interface for performing classification. +/// +/// Specific implementations will exist for different model types +/// (e.g. TensorFlow SavedModel) that can convert the request into a model +/// specific input and know how to convert the output into a generic +/// ClassificationResult. class ClassifierInterface { public: + /// Given a ClassificationRequest, populates the ClassificationResult with the + /// result. + /// + /// @param request Input request specifying the model/signature to query + /// along with the data payload. + /// @param result The output classifications that will get populated. + /// @return A status object indicating success or failure. virtual Status Classify(const ClassificationRequest& request, ClassificationResult* result) = 0; diff --git a/tensorflow_serving/apis/regressor.h b/tensorflow_serving/apis/regressor.h index 95aadc873eb..2f635825701 100644 --- a/tensorflow_serving/apis/regressor.h +++ b/tensorflow_serving/apis/regressor.h @@ -13,8 +13,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -// -// Interface for performing regression using regression messages. #ifndef TENSORFLOW_SERVING_APIS_REGRESSOR_H #define TENSORFLOW_SERVING_APIS_REGRESSOR_H @@ -25,8 +23,21 @@ limitations under the License. namespace tensorflow { namespace serving { +/// Model agnostic interface for performing regression. +/// +/// Specific implementations will exist for different model types +/// (e.g. TensorFlow SavedModel) that can convert the request into a model +/// specific input and know how to convert the output into a generic +/// RegressionResult. class RegressorInterface { public: + /// Given a RegressionRequest, populates the RegressionResult with the + /// result. + /// + /// @param request Input request specifying the model/signature to query + /// along with the data payload. + /// @param result The output regression results that will get populated. + /// @return A status object indicating success or failure. virtual Status Regress(const RegressionRequest& request, RegressionResult* result) = 0; diff --git a/tensorflow_serving/core/loader_harness.h b/tensorflow_serving/core/loader_harness.h index 34180354cf2..992fd0529c3 100644 --- a/tensorflow_serving/core/loader_harness.h +++ b/tensorflow_serving/core/loader_harness.h @@ -30,9 +30,11 @@ limitations under the License. namespace tensorflow { namespace serving { +// START_SKIP_DOXYGEN // Forward-declaration for the use in LoaderHarness. template struct ServableStateSnapshot; +// END_SKIP_DOXYGEN /// LoaderHarness is a widget the Manager uses to hold on to and talk to a /// Loader while it owns it. It tracks the overall state of a Servable such that @@ -243,6 +245,8 @@ class LoaderHarness final { TF_DISALLOW_COPY_AND_ASSIGN(LoaderHarness); }; +// START_SKIP_DOXYGEN + // A snapshot of a servable's state and aspiredness, from the LoaderHarness's // perspective. template @@ -294,6 +298,8 @@ LoaderHarness::loader_state_snapshot() const { return {id_, state_, {}}; } +// END_SKIP_DOXYGEN + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/servable_state_monitor.h b/tensorflow_serving/core/servable_state_monitor.h index bd9e256c743..86451d777fe 100644 --- a/tensorflow_serving/core/servable_state_monitor.h +++ b/tensorflow_serving/core/servable_state_monitor.h @@ -44,6 +44,7 @@ namespace serving { /// published on the event bus, e.g. giving the event bus to a Manager. class ServableStateMonitor { public: + // START_SKIP_DOXYGEN struct ServableStateAndTime { ServableStateAndTime() = default; ServableStateAndTime(ServableState servable_state, const uint64 event_time) @@ -73,6 +74,8 @@ class ServableStateMonitor { /// to 0, logging is disabled. uint64 max_count_log_events = 0; }; + // END_SKIP_DOXYGEN + using BoundedLog = std::deque; explicit ServableStateMonitor(EventBus* bus, diff --git a/tensorflow_serving/core/source_adapter.h b/tensorflow_serving/core/source_adapter.h index 30cbcc470a1..6383ace58b8 100644 --- a/tensorflow_serving/core/source_adapter.h +++ b/tensorflow_serving/core/source_adapter.h @@ -89,6 +89,8 @@ class SourceAdapter : public TargetBase, public Source { Notification outgoing_callback_set_; }; +// START_SKIP_DOXYGEN + // Define a SourceAdapter registry for the common case of adapting from a // storage path to a loader. using StoragePathSourceAdapter = @@ -256,6 +258,8 @@ ErrorInjectingSourceAdapter::Adapt( return adapted_versions; } +// END_SKIP_DOXYGEN + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h index 253499ab311..1dcb8529900 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h @@ -27,35 +27,50 @@ limitations under the License. namespace tensorflow { namespace serving { -// A factory that creates SavedModelBundles from SavedModel or SessionBundle -// export paths. -// -// The emitted sessions only support Run(), and although not enforced it is -// expected that the client will only make non-mutating Run() calls. (If this -// restriction, which we've added as a safety measure, is problematic for your -// use-case please contact the TensorFlow Serving team to discuss disabling it.) -// -// If the config calls for batching, the emitted sessions automatically batch -// Run() calls behind the scenes, using a SharedBatchScheduler owned by the -// factory. The 'config.num_batch_threads' threads are shared across all session -// instances created by this factory. However, each session has its own -// dedicated queue of size 'config.max_enqueued_batches'. -// -// The factory can also estimate the resource (e.g. RAM) requirements of a -// SavedModelBundle based on the SavedModel (i.e. prior to loading the session). -// -// This class is thread-safe. +/// A factory that creates SavedModelBundles from SavedModel or SessionBundle +/// export paths. +/// +/// The emitted sessions only support Run(), and although not enforced it is +/// expected that the client will only make non-mutating Run() calls. (If this +/// restriction, which we've added as a safety measure, is problematic for your +/// use-case please contact the TensorFlow Serving team to discuss disabling +/// it.) +/// +/// If the config calls for batching, the emitted sessions automatically batch +/// Run() calls behind the scenes, using a SharedBatchScheduler owned by the +/// factory. The 'config.num_batch_threads' threads are shared across all +/// session instances created by this factory. However, each session has its own +/// dedicated queue of size 'config.max_enqueued_batches'. +/// +/// The factory can also estimate the resource (e.g. RAM) requirements of a +/// SavedModelBundle based on the SavedModel (i.e. prior to loading the +/// session). +/// +/// This class is thread-safe. class SavedModelBundleFactory { public: + /// Instantiates a SavedModelBundleFactory using a config. + /// + /// @param config Config with initialization options. + /// @param factory Newly created factory if the returned Status is OK. static Status Create(const SessionBundleConfig& config, std::unique_ptr* factory); - // Instantiates a bundle from a given export or SavedModel path. + /// Instantiates a bundle from a given export or SavedModel path. + /// + /// @param path Path to the model. + /// @param bundle Newly created SavedModelBundle if the returned Status is + /// OK. Status CreateSavedModelBundle(const string& path, std::unique_ptr* bundle); - // Estimates the resources a SavedModel bundle will use once loaded, from its - // export path. + /// Estimates the resources a SavedModel bundle will use once loaded, from its + /// export path. + /// + /// @param path Path to the model. + /// @param estimate Output resource usage estimates. Different kinds of + /// resources (e.g. CPU, RAM, etc.) may get populated. + // // TODO(b/33078719): remove this method after we switch all the callers to // the following one. Status EstimateResourceRequirement(const string& path, From bfa2805b2bfa7c434286820030564a965021c467 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Fri, 21 Jul 2017 16:49:40 -0800 Subject: [PATCH 0298/8554] Adds a pre_load_hook, which is called just before a servable is to be loaded. Change: 162806165 --- .../core/aspired_versions_manager.cc | 1 + .../core/aspired_versions_manager.h | 6 ++++ tensorflow_serving/core/basic_manager.cc | 13 +++++++-- tensorflow_serving/core/basic_manager.h | 12 +++++++- tensorflow_serving/core/basic_manager_test.cc | 29 +++++++++++++++++++ .../model_servers/server_core.cc | 1 + .../model_servers/server_core.h | 6 ++++ .../model_servers/server_core_test.cc | 14 +++++++++ 8 files changed, 78 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index c867b03d089..3894a2b0012 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -157,6 +157,7 @@ Status AspiredVersionsManager::Create( basic_manager_options.load_retry_interval_micros = options.load_retry_interval_micros; basic_manager_options.servable_event_bus = options.servable_event_bus; + basic_manager_options.pre_load_hook = std::move(options.pre_load_hook); std::unique_ptr basic_manager; TF_RETURN_IF_ERROR( BasicManager::Create(std::move(basic_manager_options), &basic_manager)); diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 704710ddb8f..2bb59912b50 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -80,6 +80,8 @@ class AspiredVersionsManagerTestAccess; class AspiredVersionsManager : public Manager, public Target> { public: + using PreLoadHook = BasicManager::PreLoadHook; + /// Config options and pluggable objects that will be used by the /// AspiredVersionsManager. struct Options { @@ -123,6 +125,10 @@ class AspiredVersionsManager : public Manager, /// The environment to use for starting threads in the thread-pool or for /// sleeping. Env* env = Env::Default(); + + /// Callback to be called just before a servable is to be loaded. This will + /// called on the same manager load thread which starts the load. + PreLoadHook pre_load_hook; }; static Status Create(Options options, std::unique_ptr* manager); diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index 777ea9a0376..e6c1d86e904 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -205,7 +205,8 @@ Status BasicManager::Create(Options options, manager->reset(new BasicManager( options.env, options.num_load_threads, options.num_unload_threads, options.max_num_load_retries, options.load_retry_interval_micros, - std::move(options.resource_tracker), options.servable_event_bus)); + std::move(options.resource_tracker), options.servable_event_bus, + std::move(options.pre_load_hook))); return Status::OK(); } @@ -214,10 +215,12 @@ BasicManager::BasicManager(Env* const env, const uint32 num_load_threads, uint32 max_num_load_retries, int64 load_retry_interval_micros, std::unique_ptr resource_tracker, - EventBus* servable_event_bus) + EventBus* servable_event_bus, + std::function pre_load_hook) : servable_event_bus_(servable_event_bus), env_(env), - num_load_threads_(num_load_threads) { + num_load_threads_(num_load_threads), + pre_load_hook_(std::move(pre_load_hook)) { harness_options_.max_num_load_retries = max_num_load_retries; harness_options_.load_retry_interval_micros = load_retry_interval_micros; harness_options_.error_callback = [this](const ServableId& id, @@ -454,6 +457,10 @@ Status BasicManager::ExecuteLoad(LoaderHarness* harness) { // thread that called StopManagingServable().) const ServableId id = harness->id(); + if (pre_load_hook_) { + pre_load_hook_(id); + } + // We don't hold the lock while calling Load() as it may block. TF_RETURN_IF_ERROR(harness->Load()); diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index d60b33fc748..3bd185abd9f 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -104,6 +104,9 @@ class BasicManagerTestAccess; /// TF_CHECK_OK(manager.StopManagingServable(id)); class BasicManager : public Manager { public: + // Type of the callback to be called just before a servable is to be loaded. + using PreLoadHook = std::function; + /// Config options and pluggable objects that will be used by the /// BasicManager. struct Options { @@ -138,6 +141,10 @@ class BasicManager : public Manager { // The environment to use for starting threads in the thread-pool. Env* env = Env::Default(); + + // Callback to be called just before a servable is to be loaded. This will + // called on the same manager load thread which starts the load. + PreLoadHook pre_load_hook; }; static Status Create(Options options, std::unique_ptr* manager); @@ -257,7 +264,8 @@ class BasicManager : public Manager { BasicManager(Env* env, uint32 num_load_threads, uint32 num_unload_threads, uint32 max_num_load_retries, int64 load_retry_interval_micros, std::unique_ptr resource_tracker, - EventBus* servable_event_bus); + EventBus* servable_event_bus, + PreLoadHook pre_load_hook); // Starts managing the servable. // @@ -491,6 +499,8 @@ class BasicManager : public Manager { // decrease. condition_variable num_ongoing_load_unload_executions_cv_; + PreLoadHook pre_load_hook_; + TF_DISALLOW_COPY_AND_ASSIGN(BasicManager); }; diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index b0880fd07ea..6b55c4587b2 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -44,6 +44,7 @@ using ::testing::HasSubstr; using ::testing::InSequence; using ::testing::Invoke; using ::testing::InvokeWithoutArgs; +using ::testing::MockFunction; using ::testing::NiceMock; using ::testing::Return; using ::testing::UnorderedElementsAre; @@ -1109,6 +1110,34 @@ TEST_P(BasicManagerTest, LoadAfterCancelledLoad) { id, [](const Status& status) { EXPECT_FALSE(status.ok()) << status; }); } +TEST(NonParameterizedBasicManagerTest, PreLoadHook) { + BasicManager::Options options; + // Single threaded execution. + options.num_load_threads = 0; + // No event bus. + options.servable_event_bus = nullptr; + MockFunction mock_pre_load_hook; + options.pre_load_hook = mock_pre_load_hook.AsStdFunction(); + std::unique_ptr manager; + TF_ASSERT_OK(BasicManager::Create(std::move(options), &manager)); + + const ServableId id = {kServableName, 7}; + test_util::MockLoader* loader = new NiceMock(); + TF_CHECK_OK(manager->ManageServable({id, std::unique_ptr(loader)})); + + bool pre_load_hook_called = false; + EXPECT_CALL(mock_pre_load_hook, Call(id)).WillOnce(InvokeWithoutArgs([&]() { + pre_load_hook_called = true; + })); + EXPECT_CALL(*loader, Load()).WillOnce(InvokeWithoutArgs([&]() { + EXPECT_TRUE(pre_load_hook_called); + return Status::OK(); + })); + manager->LoadServable(id, [](const Status& status) { TF_ASSERT_OK(status); }); + manager->UnloadServable(id, + [](const Status& status) { TF_ASSERT_OK(status); }); +} + // Creates a ResourceAllocation proto with 'quantity' units of RAM. ResourceAllocation CreateResourceQuantity(const int quantity) { ResourceAllocation allocation; diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index d813005486c..762f69394de 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -583,6 +583,7 @@ Status ServerCore::CreateAspiredVersionsManager( manager_options.num_load_threads = options_.num_load_threads; manager_options.num_unload_threads = options_.num_unload_threads; manager_options.max_num_load_retries = options_.max_num_load_retries; + manager_options.pre_load_hook = std::move(options_.pre_load_hook); const tensorflow::Status status = AspiredVersionsManager::Create(std::move(manager_options), manager); if (!status.ok()) { diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index d5ea3933084..341c512d9e6 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -64,6 +64,8 @@ class ServerCoreTestAccess; /// ServerCore. class ServerCore : public Manager { public: + using PreLoadHook = AspiredVersionsManager::PreLoadHook; + using ServableStateMonitorCreator = std::function* event_bus, std::unique_ptr* monitor)>; @@ -128,6 +130,10 @@ class ServerCore : public Manager { // If set, we use this function to update the server_request_logger. ServerRequestLoggerUpdater server_request_logger_updater; + + // Callback to be called just before a servable is to be loaded. This will + // called on the same manager load thread which starts the load. + PreLoadHook pre_load_hook; }; virtual ~ServerCore() = default; diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 7b1f8efdbc5..5583287d3dd 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -38,9 +38,23 @@ namespace { using ::testing::_; using ::testing::Invoke; +using ::testing::MockFunction; using ::testing::NiceMock; using test_util::ServerCoreTest; +TEST_P(ServerCoreTest, PreLoadHook) { + std::unique_ptr server_core; + ServerCore::Options options = GetDefaultOptions(); + MockFunction mock_pre_load_hook; + options.pre_load_hook = mock_pre_load_hook.AsStdFunction(); + options.model_server_config = GetTestModelServerConfigForFakePlatform(); + + const ServableId expected_id = {test_util::kTestModelName, + test_util::kTestModelVersion}; + EXPECT_CALL(mock_pre_load_hook, Call(expected_id)); + TF_ASSERT_OK(ServerCore::Create(std::move(options), &server_core)); +} + TEST_P(ServerCoreTest, CreateWaitsTillModelsAvailable) { std::unique_ptr server_core; TF_ASSERT_OK(CreateServerCore(GetTestModelServerConfigForFakePlatform(), From 3f303f1459811de4ea9e84efc6ebc0033cb33211 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 24 Jul 2017 10:47:44 -0700 Subject: [PATCH 0299/8554] Prepend TF_ to FALLTHROUGH_INTENDED in file_system_storage_path_source.cc --- .../sources/storage_path/file_system_storage_path_source.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 7c39052a592..ebea569692e 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -230,7 +230,7 @@ Status PollFileSystemForServable( switch (servable.servable_version_policy().policy_choice_case()) { case FileSystemStoragePathSourceConfig::ServableVersionPolicy:: POLICY_CHOICE_NOT_SET: - FALLTHROUGH_INTENDED; // Default policy is kLatest. + TF_FALLTHROUGH_INTENDED; // Default policy is kLatest. case FileSystemStoragePathSourceConfig::ServableVersionPolicy::kLatest: at_least_one_version_found = AspireLatestVersions(servable, children_by_version, versions); From 4d9d213ac0a15ce658e566276d34e8f7af8f57bd Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Mon, 24 Jul 2017 10:53:17 -0800 Subject: [PATCH 0300/8554] Merge changes from github. Change: 162966769 --- .../sources/storage_path/file_system_storage_path_source.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 7c39052a592..ebea569692e 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -230,7 +230,7 @@ Status PollFileSystemForServable( switch (servable.servable_version_policy().policy_choice_case()) { case FileSystemStoragePathSourceConfig::ServableVersionPolicy:: POLICY_CHOICE_NOT_SET: - FALLTHROUGH_INTENDED; // Default policy is kLatest. + TF_FALLTHROUGH_INTENDED; // Default policy is kLatest. case FileSystemStoragePathSourceConfig::ServableVersionPolicy::kLatest: at_least_one_version_found = AspireLatestVersions(servable, children_by_version, versions); From 8eba5dbbf42f69bbb2b9a23f9d5978557b98d10c Mon Sep 17 00:00:00 2001 From: Mark Daoust Date: Mon, 24 Jul 2017 14:09:12 -0800 Subject: [PATCH 0301/8554] Replace "~~~" with "```" for TensorFlow.org Change: 162994475 --- tensorflow_serving/g3doc/custom_servable.md | 16 ++++----- tensorflow_serving/g3doc/custom_source.md | 12 +++---- tensorflow_serving/g3doc/serving_advanced.md | 36 ++++++++++---------- tensorflow_serving/g3doc/serving_basic.md | 28 +++++++-------- tensorflow_serving/g3doc/setup.md | 36 ++++++++++---------- tensorflow_serving/g3doc/signature_defs.md | 12 +++---- 6 files changed, 70 insertions(+), 70 deletions(-) diff --git a/tensorflow_serving/g3doc/custom_servable.md b/tensorflow_serving/g3doc/custom_servable.md index 658ede6bae8..28cce071d0b 100644 --- a/tensorflow_serving/g3doc/custom_servable.md +++ b/tensorflow_serving/g3doc/custom_servable.md @@ -65,20 +65,20 @@ should be more careful): First, create a manager: -~~~c++ +```c++ std::unique_ptr manager = ...; -~~~ +``` Then, create a `YourServable` source adapter and plug it into the manager: -~~~c++ +```c++ auto your_adapter = new YourServableSourceAdapter(...); ConnectSourceToTarget(your_adapter, manager.get()); -~~~ +``` Lastly, create a simple path source and plug it into your adapter: -~~~c++ +```c++ std::unique_ptr path_source; // Here are some FileSystemStoragePathSource config settings that ought to get // it working, but for details please see its documentation. @@ -89,13 +89,13 @@ config.set_base_path(FLAGS::base_path /* base path for our servable files */); config.set_file_system_poll_wait_seconds(1); TF_CHECK_OK(FileSystemStoragePathSource::Create(config, &path_source)); ConnectSourceToTarget(path_source.get(), your_adapter.get()); -~~~ +``` ## Accessing loaded `YourServable` objects Here is how to get a handle to a loaded `YourServable`, and use it: -~~~c++ +```c++ auto handle_request = serving::ServableRequest::Latest("default"); ServableHandle servable; Status status = manager->GetServableHandle(handle_request, &servable); @@ -105,7 +105,7 @@ if (!status.ok()) { } // Use the servable. (*servable)->SomeYourServableMethod(); -~~~ +``` ## Advanced: Arranging for multiple servable instances to share state diff --git a/tensorflow_serving/g3doc/custom_source.md b/tensorflow_serving/g3doc/custom_source.md index 1162875be19..fc8e802cb0c 100644 --- a/tensorflow_serving/g3doc/custom_source.md +++ b/tensorflow_serving/g3doc/custom_source.md @@ -92,26 +92,26 @@ the main code flow (with bad error handling; real code should be more careful): First, create a manager: -~~~c++ +```c++ std::unique_ptr manager = ...; -~~~ +``` Then, create a `SavedModelBundle` source adapter and plug it into the manager: -~~~c++ +```c++ std::unique_ptr bundle_adapter; SessionBundleSourceAdapterConfig config; // ... populate 'config' with TensorFlow options. TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(config, &bundle_adapter)); ConnectSourceToTarget(bundle_adapter.get(), manager.get()); -~~~ +``` Lastly, create your path source and plug it into the `SavedModelBundle` adapter: -~~~c++ +```c++ auto your_source = new YourPathSource(...); ConnectSourceToTarget(your_source, bundle_adapter.get()); -~~~ +``` The `ConnectSourceToTarget()` function (defined in `core/target.h`) merely invokes `SetAspiredVersionsCallback()` to connect a `Source` to a diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 12d49645b15..458fdee8df3 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -37,22 +37,22 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisit Clear the export directory if it already exists: -~~~shell +```shell $>rm -rf /tmp/mnist_model -~~~ +``` Train (with 100 iterations) and export the first version of model: -~~~shell +```shell $>bazel build //tensorflow_serving/example:mnist_saved_model $>bazel-bin/tensorflow_serving/example/mnist_saved_model --training_iteration=100 --model_version=1 /tmp/mnist_model -~~~ +``` Train (with 2000 iterations) and export the second version of model: -~~~shell +```shell $>bazel-bin/tensorflow_serving/example/mnist_saved_model --training_iteration=2000 --model_version=2 /tmp/mnist_model -~~~ +``` As you can see in `mnist_saved_model.py`, the training and exporting is done the same way it is in the [TensorFlow Serving basic tutorial](serving_basic.md). For @@ -63,10 +63,10 @@ expect the latter to achieve better classification accuracy due to more intensive training. You should see training data for each training run in your `mnist_model` directory: -~~~shell +```shell $>ls /tmp/mnist_model 1 2 -~~~ +``` ## ServerCore @@ -85,7 +85,7 @@ version transitions. In this tutorial, you will build your server on top of a TensorFlow Serving `ServerCore`, which internally wraps an `AspiredVersionsManager`. -~~~c++ +```c++ int main(int argc, char** argv) { ... @@ -108,7 +108,7 @@ int main(int argc, char** argv) { return 0; } -~~~ +``` `ServerCore::Create()` takes a ServerCore::Options parameter. Here are a few commonly used options: @@ -176,7 +176,7 @@ creating the `SavedModelBundleSourceAdapter`. In this case we set the by setting custom timeout, batch_size, etc. values. For details, please refer to `BatchingParameters`. -~~~c++ +```c++ SessionBundleConfig session_bundle_config; // Batching config if (enable_batching) { @@ -187,7 +187,7 @@ if (enable_batching) { } *saved_model_bundle_source_adapter_config.mutable_legacy_config() = session_bundle_config; -~~~ +``` Upon reaching full batch, inference requests are merged internally into a single large request (tensor), and `tensorflow::Session::Run()` is invoked @@ -259,12 +259,12 @@ To put all these into the context of this tutorial: Copy the first version of the export to the monitored folder and start the server. -~~~shell +```shell $>mkdir /tmp/monitored $>cp -r /tmp/mnist_model/1 /tmp/monitored $>bazel build //tensorflow_serving/model_servers:tensorflow_model_server $>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored -~~~ +``` The server will emit log messages every one second that say "Aspiring version for servable ...", which means it has found the export, and is @@ -273,22 +273,22 @@ tracking its continued existence. Run the test with `--concurrency=10`. This will send concurrent requests to the server and thus trigger your batching logic. -~~~shell +```shell $>bazel build //tensorflow_serving/example:mnist_client $>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 --concurrency=10 ... Inference error rate: 13.1% -~~~ +``` Then we copy the second version of the export to the monitored folder and re-run the test: -~~~shell +```shell $>cp -r /tmp/mnist_model/2 /tmp/monitored $>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 --concurrency=10 ... Inference error rate: 9.5% -~~~ +``` This confirms that your server automatically discovers the new version and uses it for serving! diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index d1a7e236702..ad8d4dade7c 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -41,7 +41,7 @@ From [mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/te the following is a short code snippet to illustrate the general process of saving a model to disk. -~~~python +```python from tensorflow.python.saved_model import builder as saved_model_builder ... export_path_base = sys.argv[-1] @@ -60,7 +60,7 @@ builder.add_meta_graph_and_variables( }, legacy_init_op=legacy_init_op) builder.save() -~~~ +``` `SavedModelBuilder.__init__` takes the following argument: @@ -138,11 +138,11 @@ Let's run it! Clear the export directory if it already exists: -~~~shell +```shell $>rm -rf /tmp/mnist_model -~~~ +``` -~~~shell +```shell $>bazel build //tensorflow_serving/example:mnist_saved_model $>bazel-bin/tensorflow_serving/example/mnist_saved_model /tmp/mnist_model Training model... @@ -152,23 +152,23 @@ Training model... Done training! Exporting trained model to /tmp/mnist_model Done exporting! -~~~ +``` Now let's take a look at the export directory. -~~~shell +```shell $>ls /tmp/mnist_model 1 -~~~ +``` As mentioned above, a sub-directory will be created for exporting each version of the model. `FLAGS.model_version` has the default value of 1, therefore the corresponding sub-directory `1` is created. -~~~shell +```shell $>ls /tmp/mnist_model/1 saved_model.pb variables -~~~ +``` Each version sub-directory contains the following files: @@ -182,10 +182,10 @@ With that, your TensorFlow model is exported and ready to be loaded! ## Load Exported Model With Standard TensorFlow Model Server -~~~shell +```shell $>bazel build //tensorflow_serving/model_servers:tensorflow_model_server $>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ -~~~ +``` ## Test The Server @@ -195,12 +195,12 @@ requests to the server, and calculates the inference error rate. To run it: -~~~shell +```shell $>bazel build //tensorflow_serving/example:mnist_client $>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 ... Inference error rate: 10.5% -~~~ +``` We expect around 91% accuracy for the trained Softmax model and we get 10.5% inference error rate for the first 1000 test images. This confirms that diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 10fe58994a7..dd827889f54 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -17,16 +17,16 @@ following steps: Let's say you downloaded bazel-0.4.5-installer-linux-x86_64.sh. You would execute: - ~~~shell + ```shell cd ~/Downloads chmod +x bazel-0.4.5-installer-linux-x86_64.sh ./bazel-0.4.5-installer-linux-x86_64.sh --user - ~~~ + ``` 2. Set up your environment. Put this in your ~/.bashrc. - ~~~shell + ```shell export PATH="$PATH:$HOME/bin" - ~~~ + ``` ### gRPC @@ -38,7 +38,7 @@ framework. You can find the installation instructions To install TensorFlow Serving dependencies, execute the following: -~~~shell +```shell sudo apt-get update && sudo apt-get install -y \ build-essential \ curl \ @@ -55,16 +55,16 @@ sudo apt-get update && sudo apt-get install -y \ swig \ zip \ zlib1g-dev -~~~ +``` ## Installing from source ### Clone the TensorFlow Serving repository -~~~shell +```shell git clone --recurse-submodules https://github.com/tensorflow/serving cd serving -~~~ +``` `--recurse-submodules` is required to fetch TensorFlow, gRPC, and other libraries that TensorFlow Serving depends on. Note that these instructions @@ -77,11 +77,11 @@ to the `git clone` command. Follow the Prerequisites section above to install all dependencies. To configure TensorFlow, run -~~~shell +```shell cd tensorflow ./configure cd .. -~~~ +``` Consult the [TensorFlow install instructions](https://www.tensorflow.org/install/) @@ -95,22 +95,22 @@ targets or the entire source tree. To build the entire tree, execute: -~~~shell +```shell bazel build tensorflow_serving/... -~~~ +``` Binaries are placed in the bazel-bin directory, and can be run using a command like: -~~~shell +```shell bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server -~~~ +``` To test your installation, execute: -~~~shell +```shell bazel test tensorflow_serving/... -~~~ +``` See the [basic tutorial](serving_basic.md) and [advanced tutorial](serving_advanced.md) for more in-depth examples of running TensorFlow Serving. @@ -123,11 +123,11 @@ using TensorFlow [ci_build](https://github.com/tensorflow/tensorflow/tree/master infrastructure offers you simplified development using docker. All you need is git and docker. No need to install all other dependencies manually. -~~~shell +```shell git clone --recursive https://github.com/tensorflow/serving cd serving CI_TENSORFLOW_SUBMODULE_PATH=tensorflow tensorflow/tensorflow/tools/ci_build/ci_build.sh CPU bazel test //tensorflow_serving/... -~~~ +``` Note: The `serving` directory is mapped into the container. You can develop outside the docker container (in your favourite editor) and when you run this diff --git a/tensorflow_serving/g3doc/signature_defs.md b/tensorflow_serving/g3doc/signature_defs.md index efb8b0ccf8d..69d2cb1c42d 100644 --- a/tensorflow_serving/g3doc/signature_defs.md +++ b/tensorflow_serving/g3doc/signature_defs.md @@ -75,7 +75,7 @@ Classification API. These prescribe that there must be an `inputs` Tensor, and that there are two optional output Tensors: `classes` and `scores`, at least one of which must be present. -~~~proto +```proto signature_def: { key : "my_classification_signature" value: { @@ -106,7 +106,7 @@ signature_def: { method_name: "tensorflow/serving/classify" } } -~~~ +``` ### Predict SignatureDef @@ -128,7 +128,7 @@ key below of `scores`, you also wanted to fetch a pooling layer for debugging or other purposes. In that case, you would simply add an additional Tensor with a key like `pool` and appropriate value. -~~~proto +```proto signature_def: { key : "my_prediction_signature" value: { @@ -151,7 +151,7 @@ signature_def: { method_name: "tensorflow/serving/predict" } } -~~~ +``` ### Regression SignatureDef @@ -159,7 +159,7 @@ Regression SignatureDefs support structured calls to TensorFlow Serving's Regression API. These prescribe that there must be exactly one `inputs` Tensor, and one `outputs` Tensor. -~~~proto +```proto signature_def: { key : "my_regression_signature" value: { @@ -182,4 +182,4 @@ signature_def: { method_name: "tensorflow/serving/regress" } } -~~~ +``` From b57eb58000828feb6241e90991197c8c6c437c80 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 24 Jul 2017 16:38:52 -0800 Subject: [PATCH 0302/8554] No public change Change: 163014191 --- tensorflow_serving/apis/BUILD | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 0e884e84dd7..30c45a47b02 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -21,6 +21,7 @@ filegroup( load("//tensorflow_serving:serving.bzl", "serving_proto_library") load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") load("//tensorflow_serving:serving.bzl", "serving_go_grpc_library") +load("@org_tensorflow//tensorflow/core:platform/default/build_config.bzl", "tf_pyclif_proto_library") serving_proto_library( name = "get_model_metadata_proto", @@ -66,6 +67,12 @@ serving_proto_library_py( ], ) +tf_pyclif_proto_library( + name = "input_pyclif", + proto_lib = ":input_proto", + proto_srcfile = "input.proto", +) + serving_proto_library( name = "model_proto", srcs = ["model.proto"], @@ -84,6 +91,12 @@ serving_proto_library_py( deps = [], ) +tf_pyclif_proto_library( + name = "model_pyclif", + proto_lib = ":model_proto", + proto_srcfile = "model.proto", +) + serving_proto_library( name = "predict_proto", srcs = ["predict.proto"], @@ -165,6 +178,12 @@ serving_proto_library_py( ], ) +tf_pyclif_proto_library( + name = "classification_pyclif", + proto_lib = ":classification_proto", + proto_srcfile = "classification.proto", +) + serving_proto_library( name = "inference_proto", srcs = ["inference.proto"], @@ -191,6 +210,12 @@ serving_proto_library_py( ], ) +tf_pyclif_proto_library( + name = "inference_pyclif", + proto_lib = ":inference_proto", + proto_srcfile = "inference.proto", +) + serving_proto_library( name = "regression_proto", srcs = ["regression.proto"], @@ -214,6 +239,12 @@ serving_proto_library_py( ], ) +tf_pyclif_proto_library( + name = "regression_pyclif", + proto_lib = ":regression_proto", + proto_srcfile = "regression.proto", +) + cc_library( name = "classifier", hdrs = ["classifier.h"], From 7c3a774ec51c496a5e8b972c85d7a8a4ada9cb71 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 26 Jul 2017 08:22:47 -0800 Subject: [PATCH 0303/8554] Remove stale TODO. Change: 163215318 --- .../servables/tensorflow/saved_model_bundle_factory.h | 3 --- 1 file changed, 3 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h index 1dcb8529900..9ab961a2e9f 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h @@ -70,9 +70,6 @@ class SavedModelBundleFactory { /// @param path Path to the model. /// @param estimate Output resource usage estimates. Different kinds of /// resources (e.g. CPU, RAM, etc.) may get populated. - // - // TODO(b/33078719): remove this method after we switch all the callers to - // the following one. Status EstimateResourceRequirement(const string& path, ResourceAllocation* estimate) const; From 0e3c0ea724bca025badd226027f664cdab1cd72e Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 26 Jul 2017 10:20:45 -0700 Subject: [PATCH 0304/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index bef67104a0d..16d39e94e37 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit bef67104a0da34b6cac0d801168f2e42a1ab7921 +Subproject commit 16d39e94e3724417fcaed87035434e098e892842 diff --git a/tf_models b/tf_models index 3d97b007cdb..78007443138 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 3d97b007cdb2c73f23daf9141437796095436c22 +Subproject commit 78007443138108abf5170b296b4d703b49454487 From 80746ac2904462ffc780893fd27d7ed6ae0c46b8 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 26 Jul 2017 13:08:18 -0800 Subject: [PATCH 0305/8554] Add universal debian target for a non-AVX ModelServer. Change: 163252885 --- tensorflow_serving/model_servers/BUILD | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 3d02efd6a06..704b2ecd909 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -186,6 +186,7 @@ pkg_tar( package_dir = "/usr/bin", ) +# Build with '-c opt --copt=-mavx --copt=-msse4.2' pkg_deb( name = "tensorflow_model_server_deb", data = ":tensorflow_model_server_tar", @@ -195,3 +196,14 @@ pkg_deb( package = "tensorflow-model-server", version = "1.0.0", ) + +# Build with '-c opt' +pkg_deb( + name = "tensorflow_model_server_universal_deb", + data = ":tensorflow_model_server_tar", + description = "TensorFlow Serving ModelServer", + homepage = "https://github.com/tensorflow/serving", + maintainer = "TensorFlow Serving team", + package = "tensorflow-model-server-universal", + version = "1.0.0", +) From 3a2d38b9e64241d57d7275bc92501eac13ff146e Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 26 Jul 2017 15:30:42 -0800 Subject: [PATCH 0306/8554] Add missing space to log message. Change: 163273788 --- tensorflow_serving/core/aspired_versions_manager.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 3894a2b0012..58074fbb91e 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -284,7 +284,7 @@ void AspiredVersionsManager::ProcessAspiredVersionsRequest( // if this aspired version is not already present in the map. if (std::find(additions.begin(), additions.end(), version.id().version) != additions.end()) { - VLOG(1) << "Adding " << version.id() << "to BasicManager"; + VLOG(1) << "Adding " << version.id() << " to BasicManager"; const Status manage_status = basic_manager_->ManageServableWithAdditionalState( std::move(version), std::unique_ptr(new Aspired{true})); From 48d1fe7a3839c98aa923e98e8c77574c9c611d9b Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 27 Jul 2017 13:19:29 -0800 Subject: [PATCH 0307/8554] Correct mock -> fake in InlineExecutor documentation. Change: 163386931 --- tensorflow_serving/util/inline_executor.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/util/inline_executor.h b/tensorflow_serving/util/inline_executor.h index e07632c1550..18b5314a2d5 100644 --- a/tensorflow_serving/util/inline_executor.h +++ b/tensorflow_serving/util/inline_executor.h @@ -25,7 +25,7 @@ namespace tensorflow { namespace serving { // An InlineExecutor is a trivial executor that immediately executes the closure -// given to it. It's useful as a mock, and in cases where an executor is needed, +// given to it. It's useful as a fake, and in cases where an executor is needed, // but multi-threadedness is not. class InlineExecutor : public Executor { public: From 7ee68c36cf495d8dd47fc9e9b3943a0f6335b6e4 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 28 Jul 2017 08:01:04 -0800 Subject: [PATCH 0308/8554] Undo change to Dockerfile.devel that clones and builds TensorFlow serving. The Dockerfile will just set up the environment and then the user can clone and build, or alternatively install using apt-get. Change: 163475394 --- tensorflow_serving/tools/docker/Dockerfile.devel | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index de1cd5ff7ed..dea056ebe6e 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -45,17 +45,4 @@ RUN mkdir /bazel && \ cd / && \ rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh -# Download TensorFlow Serving -RUN git clone --recurse-submodules https://github.com/tensorflow/serving && \ - cd serving && \ - git checkout - -WORKDIR /serving/tensorflow -RUN tensorflow/tools/ci_build/builds/configured CPU - -WORKDIR /serving -RUN bazel build -c opt tensorflow_serving/... && \ - cp bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server /usr/local/bin/ && \ - bazel clean --expunge - CMD ["/bin/bash"] From 6749112f6b9b0f7140350aac38d95f930bb0f0a3 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 28 Jul 2017 09:08:11 -0800 Subject: [PATCH 0309/8554] Add instructions to build optimized binaries. Change: 163483230 --- tensorflow_serving/g3doc/setup.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index dd827889f54..692752e00a5 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -96,7 +96,7 @@ targets or the entire source tree. To build the entire tree, execute: ```shell -bazel build tensorflow_serving/... +bazel build -c opt tensorflow_serving/... ``` Binaries are placed in the bazel-bin directory, and can be run using a command @@ -109,12 +109,28 @@ bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server To test your installation, execute: ```shell -bazel test tensorflow_serving/... +bazel test -c opt tensorflow_serving/... ``` See the [basic tutorial](serving_basic.md) and [advanced tutorial](serving_advanced.md) for more in-depth examples of running TensorFlow Serving. +### Optimized build + +It's possible to compile using some platform specific instruction sets (e.g. +AVX) that can significantly improve performance. Wherever you see 'bazel build' +in the documentation, you can add the flags `-c opt --copt=-msse4.1 +--copt=-msse4.2 --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-O3` (or some +subset of these flags). For example: + +```shell +bazel build -c opt --config=mkl --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-O3 tensorflow_serving/... +``` + +Note: These instruction sets are not available on all machines, especially with +older processors, so it may not work with all flags. You can try some subset of +them, or revert to just the basic '-c opt' which is guaranteed to work on all +machines. ### Continuous integration build From 2c8cbed1b50f815aa5142bcb6ee83e6f4a997b57 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 28 Jul 2017 14:49:01 -0800 Subject: [PATCH 0310/8554] Update install documentation to include installing using apt-get. Change: 163529264 --- tensorflow_serving/g3doc/setup.md | 50 ++++++++++++++++++++++++++++++- 1 file changed, 49 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 692752e00a5..2b1731b0cc4 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -4,7 +4,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. -### Bazel +### Bazel (only if compiling source code) TensorFlow Serving requires Bazel 0.4.5 or higher. You can find the Bazel installation instructions [here](http://bazel.build/docs/install.html). @@ -57,6 +57,54 @@ sudo apt-get update && sudo apt-get install -y \ zlib1g-dev ``` +## Installing using apt-get + +### Available binaries + +The TensorFlow Serving ModelServer binary is available in two variants: + +**tensorflow-model-server**: Fully optimized server that uses some platform +specific compiler optimizations like SSE4 and AVX instructions. This should be +the preferred option for most users, but may not work on some older machines. + +**tensorflow-model-server-universal**: Compiled with basic optimizations, but +doesn't include platform specific instructiob sets, so should work on most if +not all machines out there. Use this if `tensorflow-model-server` does not work +for you. Note that the binary name is the same for both packages, so if you +already installed tensorflow-model-server, you should first uninstall it using + +```shell +sudo apt-get remove tensorflow-model-server +``` + +### Installing the ModelServer + +1. Add TensorFlow Serving distribution URI as a package source (one time setup) + +```shell +echo "deb [arch=amd64] http://storage.googleapis.com/tensorflow-serving-apt stable tensorflow-model-server tensorflow-model-server-universal" | sudo tee /etc/apt/sources.list.d/tensorflow-serving.list + +curl https://storage.googleapis.com/tensorflow-serving-apt/tensorflow-serving.release.pub.gpg | sudo apt-key add - +``` + +2. Install and update TensorFlow ModelServer + +```shell +sudo apt-get update && sudo apt-get install tensorflow-model-server +``` + +Once installed, the binary can be invoked using the command `tensorflow_model_server`. + +You can upgrade to a newer version of tensorflow-model-server with: + +```shell +sudo apt-get upgrade tensorflow-model-server +``` + +Note: In the above commands, replace tensorflow-model-server with +tensorflow-model-server-universal if your processor does not support AVX +instructions. + ## Installing from source ### Clone the TensorFlow Serving repository From 6eeec2b2e0e78881c9dec975e9a2a4e4559e75ec Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 28 Jul 2017 15:13:48 -0800 Subject: [PATCH 0311/8554] Update tutorials to use either a Debian package or locally compiled ModelServer. Change: 163531970 --- tensorflow_serving/g3doc/serving_advanced.md | 29 +++++++------ tensorflow_serving/g3doc/serving_basic.md | 43 +++++++++++++------ tensorflow_serving/g3doc/serving_inception.md | 29 +++++++++---- 3 files changed, 69 insertions(+), 32 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 458fdee8df3..1ddf2dc92c5 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -1,7 +1,7 @@ -# Building Standard TensorFlow Model Server +# Building Standard TensorFlow ModelServer This tutorial shows you how to use TensorFlow Serving components to build the -standard TensorFlow model server that dynamically discovers and serves new +standard TensorFlow ModelServer that dynamically discovers and serves new versions of a trained TensorFlow model. If you just want to use the standard server to serve your models, see [TensorFlow Serving basic tutorial](serving_basic.md). @@ -20,18 +20,23 @@ The code for this tutorial consists of two parts: * A C++ file [main.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc) - which is the standard TensorFlow model server that discovers new exported + which is the standard TensorFlow ModelServer that discovers new exported models and runs a [gRPC](http://www.grpc.io) service for serving them. This tutorial steps through the following tasks: - 1. Train and export a TensorFlow model. - 2. Manage model versioning with TensorFlow Serving `ServerCore`. - 3. Configure batching using `SessionBundleSourceAdapterConfig`. - 4. Serve request with TensorFlow Serving `ServerCore`. - 5. Run and test the service. +1. Train and export a TensorFlow model. +2. Manage model versioning with TensorFlow Serving `ServerCore`. +3. Configure batching using `SessionBundleSourceAdapterConfig`. +4. Serve request with TensorFlow Serving `ServerCore`. +5. Run and test the service. -Before getting started, please complete the [prerequisites](setup.md#prerequisites). +Before getting started, please complete the +[prerequisites](setup.md#prerequisites). + +Note: All `bazel build` commands below use the standard `-c opt` flag. To +further optimize the build, refer to the [instructions +here](setup.md#optimized-build). ## Train And Export TensorFlow Model @@ -44,7 +49,7 @@ $>rm -rf /tmp/mnist_model Train (with 100 iterations) and export the first version of model: ```shell -$>bazel build //tensorflow_serving/example:mnist_saved_model +$>bazel build -c opt //tensorflow_serving/example:mnist_saved_model $>bazel-bin/tensorflow_serving/example/mnist_saved_model --training_iteration=100 --model_version=1 /tmp/mnist_model ``` @@ -262,7 +267,7 @@ server. ```shell $>mkdir /tmp/monitored $>cp -r /tmp/mnist_model/1 /tmp/monitored -$>bazel build //tensorflow_serving/model_servers:tensorflow_model_server +$>bazel build -c opt //tensorflow_serving/model_servers:tensorflow_model_server $>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --enable_batching --port=9000 --model_name=mnist --model_base_path=/tmp/monitored ``` @@ -274,7 +279,7 @@ Run the test with `--concurrency=10`. This will send concurrent requests to the server and thus trigger your batching logic. ```shell -$>bazel build //tensorflow_serving/example:mnist_client +$>bazel build -c opt //tensorflow_serving/example:mnist_client $>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 --concurrency=10 ... Inference error rate: 13.1% diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index ad8d4dade7c..e67f70ba68b 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -14,14 +14,22 @@ tutorial. The code for this tutorial consists of two parts: -* A Python file, [mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_saved_model.py), -that trains and exports the model. +* A Python file, + [mnist_saved_model.py](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_saved_model.py), + that trains and exports the model. -* A C++ file, [main.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc), -which is the standard TensorFlow model server that discovers new exported -models and runs a [gRPC](http://www.grpc.io) service for serving them. +* A ModelServer binary which can be either installed using apt-get, or + compiled from a C++ file + ([main.cc](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc)). + The TensorFlow Serving ModelServer discovers new exported models and runs a + [gRPC](http://www.grpc.io) service for serving them. -Before getting started, please complete the [prerequisites](setup.md#prerequisites). +Before getting started, please complete the +[prerequisites](setup.md#prerequisites). + +Note: All `bazel build` commands below use the standard `-c opt` flag. To +further optimize the build, refer to the [instructions +here](setup.md#optimized-build). ## Train And Export TensorFlow Model @@ -143,7 +151,7 @@ $>rm -rf /tmp/mnist_model ``` ```shell -$>bazel build //tensorflow_serving/example:mnist_saved_model +$>bazel build -c opt //tensorflow_serving/example:mnist_saved_model $>bazel-bin/tensorflow_serving/example/mnist_saved_model /tmp/mnist_model Training model... @@ -180,23 +188,34 @@ Each version sub-directory contains the following files: With that, your TensorFlow model is exported and ready to be loaded! -## Load Exported Model With Standard TensorFlow Model Server +## Load Exported Model With Standard TensorFlow ModelServer + +If you'd like to use a locally compiled ModelServer, run the following: ```shell -$>bazel build //tensorflow_serving/model_servers:tensorflow_model_server +$>bazel build -c opt //tensorflow_serving/model_servers:tensorflow_model_server $>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ ``` +If you'd prefer to skip compilation and install using apt-get, follow the +[instructions here](setup.md#installing-using-apt-get). Then run the server with +the following command: + +```shell +tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ +``` + ## Test The Server -We can use the provided [mnist_client](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_client.py) utility -to test the server. The client downloads MNIST test data, sends them as +We can use the provided +[mnist_client](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/example/mnist_client.py) +utility to test the server. The client downloads MNIST test data, sends them as requests to the server, and calculates the inference error rate. To run it: ```shell -$>bazel build //tensorflow_serving/example:mnist_client +$>bazel build -c opt //tensorflow_serving/example:mnist_client $>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 ... Inference error rate: 10.5% diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 7cf93883f31..81d89660f0a 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -35,22 +35,35 @@ $ docker run --name=inception_container -it $USER/tensorflow-serving-devel ### Clone, configure, and build TensorFlow Serving in a container -In the running container, we clone, configure and build TensorFlow Serving. -Then test run [tensorflow_model_server](https://github.com/tensorflow/serving/tree/master/tensorflow_serving/model_servers/main.cc). +Note: All `bazel build` commands below use the standard `-c opt` flag. To +further optimize the build, refer to the [instructions +here](setup.md#optimized-build). + +In the running container, we clone, configure and build TensorFlow Serving +example code. ```shell root@c97d8e820ced:/# git clone --recurse-submodules https://github.com/tensorflow/serving root@c97d8e820ced:/# cd serving/tensorflow root@c97d8e820ced:/serving/tensorflow# ./configure root@c97d8e820ced:/serving# cd .. -root@c97d8e820ced:/serving# bazel build -c opt tensorflow_serving/... -root@c97d8e820ced:/serving# ls -AUTHORS LICENSE RELEASE.md bazel-bin bazel-out bazel-testlogs tensorflow zlib.BUILD -CONTRIBUTING.md README.md WORKSPACE bazel-genfiles bazel-serving grpc tensorflow_serving -root@c97d8e820ced:/serving# bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server -Usage: model_server [--port=8500] [--enable_batching] [--model_name=my_name] --model_base_path=/path/to/export +root@c97d8e820ced:/serving# bazel build -c opt tensorflow_serving/example/... +``` + +Next we can either install a TensorFlow ModelServer with apt-get using the +[instructions here](setup.md#installing-using-apt-get), or build a ModelServer +binary using: + +```shell +root@c97d8e820ced:/serving# bazel build -c opt tensorflow_serving/model_servers:tensorflow_model_server ``` +The rest of this tutorial assumes you compiled the ModelServer locally, in which +case the command to run it is +`bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server`. If however +you installed the ModelServer using apt-get, simply replace that command with +`tensorflow_model_server`. + ### Export Inception model in container In the running container, we run From 99cff0f64e04360a9fd9bc842ab8bb8e9022a5c4 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Sun, 30 Jul 2017 22:37:09 -0800 Subject: [PATCH 0312/8554] Fix race in ServableStateMonitor ctor, which is triggered if an event arrives while the ctor is running. (In the past we never initialized a ServableStateMonitor *after* events had already started coming in, but that changed when we added ' fresh_servable_state_monitor' to ServerCore, which exposed this latent bug.) Change: 163656928 --- tensorflow_serving/core/servable_state_monitor.cc | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/core/servable_state_monitor.cc b/tensorflow_serving/core/servable_state_monitor.cc index 480d7648d52..29448086831 100644 --- a/tensorflow_serving/core/servable_state_monitor.cc +++ b/tensorflow_serving/core/servable_state_monitor.cc @@ -103,11 +103,15 @@ string ServableStateMonitor::ServableStateAndTime::DebugString() const { ServableStateMonitor::ServableStateMonitor(EventBus* bus, const Options& options) - : options_(options), - bus_subscription_(bus->Subscribe( - [this](const EventBus::EventAndTime& state_and_time) { - this->HandleEvent(state_and_time); - })) {} + : options_(options) { + // Important: We must allow the state members ('states_', 'live_states_' and + // so on) to be initialized *before* we start the bus subscription, in case an + // event comes in while we are initializing. + bus_subscription_ = bus->Subscribe( + [this](const EventBus::EventAndTime& state_and_time) { + this->HandleEvent(state_and_time); + }); +} ServableStateMonitor::~ServableStateMonitor() { // Halt event handling first, before tearing down state that event handling From f385c84c4038adbc6f7285e187f99973b106a1c7 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Mon, 31 Jul 2017 13:03:51 -0800 Subject: [PATCH 0313/8554] Removes failing request logging if duplicate prefixes are present in the config. Change: 163744747 --- tensorflow_serving/core/server_request_logger.cc | 8 +++++--- tensorflow_serving/core/server_request_logger_test.cc | 11 ----------- 2 files changed, 5 insertions(+), 14 deletions(-) diff --git a/tensorflow_serving/core/server_request_logger.cc b/tensorflow_serving/core/server_request_logger.cc index 9e5267e3239..4c135682c6e 100644 --- a/tensorflow_serving/core/server_request_logger.cc +++ b/tensorflow_serving/core/server_request_logger.cc @@ -56,9 +56,11 @@ Status ServerRequestLogger::Update( model_and_logging_config.second.log_collector_config() .filename_prefix(); if (!gtl::InsertIfNotPresent(&filename_prefixes, filename_prefix)) { - // Logs for each model is supposed to be separated from each other. - return errors::InvalidArgument( - "Duplicate LogCollectorConfig::filename_prefix(): ", filename_prefix); + // Each model's logs are supposed to be separated from each other, though + // there could be systems which can distinguish based on the model-spec + // in the logging proto, so we issue only a warning. + LOG(WARNING) << "Duplicate LogCollectorConfig::filename_prefix(): " + << filename_prefix << ". Possibly a misconfiguration."; } TF_RETURN_IF_ERROR(request_logger_creator_(model_and_logging_config.second, &request_logger)); diff --git a/tensorflow_serving/core/server_request_logger_test.cc b/tensorflow_serving/core/server_request_logger_test.cc index 6cec506207d..36821516acd 100644 --- a/tensorflow_serving/core/server_request_logger_test.cc +++ b/tensorflow_serving/core/server_request_logger_test.cc @@ -114,17 +114,6 @@ TEST_F(ServerRequestLoggerTest, AbsentModel) { EXPECT_EQ(0, log_collector_map_["/file/model0"]->collect_count()); } -TEST_F(ServerRequestLoggerTest, DuplicateFilenamePrefix) { - std::map model_logging_configs; - model_logging_configs.insert(CreateLoggingConfigForModel("model0")); - const std::pair model_and_logging_config = - CreateLoggingConfigForModel("model0"); - model_logging_configs.insert({"model1", model_and_logging_config.second}); - const auto status = server_request_logger_->Update(model_logging_configs); - EXPECT_THAT(status.error_message(), - HasSubstr("Duplicate LogCollectorConfig::filename_prefix()")); -} - TEST_F(ServerRequestLoggerTest, MultipleModels) { std::map model_logging_configs; model_logging_configs.insert(CreateLoggingConfigForModel("model0")); From 60dc989b8665f13c12af5b60af9b8da4952cb582 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Mon, 31 Jul 2017 13:45:11 -0800 Subject: [PATCH 0314/8554] Change MockServerCore to create server_request_logger by default. Change: 163750903 --- .../model_servers/test_util/BUILD | 11 +++++++- .../test_util/mock_server_core.h | 27 ++++++++++++++----- 2 files changed, 31 insertions(+), 7 deletions(-) diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index b79deeded5d..eb61182027a 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -28,15 +28,24 @@ cc_library( "//visibility:public", ], deps = [ + "//base", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:model_server_config_proto", + "//tensorflow_serving/config:platform_config_proto", + "//tensorflow_serving/core:aspired_versions_manager", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/core:servable_state", + "//tensorflow_serving/core:servable_state_monitor", + "//tensorflow_serving/core:server_request_logger", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", "//tensorflow_serving/core/test_util:fake_loader_source_adapter_proto", "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/util:event_bus", + "//tensorflow_serving/util:unique_ptr_with_deps", + "//testing/base/public:gunit_for_library", "@org_tensorflow//tensorflow/core:lib", - "@org_tensorflow//tensorflow/core:test", + "@protobuf//:cc_wkt_protos", + "@protobuf//:protobuf_lite", ], ) diff --git a/tensorflow_serving/model_servers/test_util/mock_server_core.h b/tensorflow_serving/model_servers/test_util/mock_server_core.h index 53985381393..26aa2ecba1d 100644 --- a/tensorflow_serving/model_servers/test_util/mock_server_core.h +++ b/tensorflow_serving/model_servers/test_util/mock_server_core.h @@ -17,15 +17,27 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_MOCK_SERVER_CORE_H_ #define TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_MOCK_SERVER_CORE_H_ +#include +#include + +#include "base/logging.h" +#include "google/protobuf/any.pb.h" +#include "google/protobuf/map.h" #include #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" +#include "tensorflow_serving/config/platform_config.pb.h" +#include "tensorflow_serving/core/aspired_versions_manager.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/core/servable_state.h" +#include "tensorflow_serving/core/servable_state_monitor.h" +#include "tensorflow_serving/core/server_request_logger.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.pb.h" #include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/util/event_bus.h" +#include "tensorflow_serving/util/unique_ptr_with_deps.h" namespace tensorflow { namespace serving { @@ -46,17 +58,20 @@ class MockServerCore : public ServerCore { static Options GetOptions(const PlatformConfigMap& platform_config_map) { Options options; options.platform_config_map = platform_config_map; - options.servable_state_monitor_creator = []( - EventBus* event_bus, - std::unique_ptr* monitor) -> Status { + options.servable_state_monitor_creator = + [](EventBus* event_bus, + std::unique_ptr* monitor) -> Status { monitor->reset(new ServableStateMonitor(event_bus)); return Status::OK(); }; - options.custom_model_config_loader = []( - const ::google::protobuf::Any& any, EventBus* event_bus, - UniquePtrWithDeps* manager) -> Status { + options.custom_model_config_loader = + [](const ::google::protobuf::Any& any, + EventBus* event_bus, + UniquePtrWithDeps* manager) -> Status { return Status::OK(); }; + TF_CHECK_OK( + ServerRequestLogger::Create(nullptr, &options.server_request_logger)); return options; } From 282c94abe741773f8c48c9f91dcb07eb1c1df161 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 31 Jul 2017 15:47:03 -0800 Subject: [PATCH 0315/8554] Some fixes for Doxygen rendering (making these show up, and formatting code samples) Change: 163768153 --- .../core/aspired_versions_manager.h | 14 +++--- .../core/aspired_versions_manager_builder.h | 37 ++++++++------- tensorflow_serving/core/basic_manager.h | 46 +++++++++---------- tensorflow_serving/core/loader.h | 2 +- tensorflow_serving/core/servable_handle.h | 24 +++++----- .../core/servable_state_monitor.h | 6 +-- tensorflow_serving/core/source_adapter.h | 7 +-- 7 files changed, 71 insertions(+), 65 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 2bb59912b50..3e62b6c1b49 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -60,13 +60,13 @@ namespace test_util { class AspiredVersionsManagerTestAccess; } // namespace test_util -/// A manager that implements the Target API which uses aspired-versions -/// callbacks to dictate which servable versions to load. This manager also uses -/// that API to infer which ones to unload: If a given servable version is -/// currently loaded, and is omitted from an aspired-versions callback -/// invocation pertaining to its servable stream, this manager interprets that -/// omission as an implicit instruction to unload the version. See below for -/// details. +/// A manager that implements the Target&lt;Loader> API which uses +/// aspired-versions callbacks to dictate which servable versions to load. This +/// manager also uses that API to infer which ones to unload: If a given +/// servable version is currently loaded, and is omitted from an +/// aspired-versions callback invocation pertaining to its servable stream, this +/// manager interprets that omission as an implicit instruction to unload the +/// version. See below for details. /// /// (The implicit-unload semantics facilitates stateless Source implementations, /// whereby a given iteration of the Source's logic simply decides which diff --git a/tensorflow_serving/core/aspired_versions_manager_builder.h b/tensorflow_serving/core/aspired_versions_manager_builder.h index 566128791ec..e2ab9c9cdb9 100644 --- a/tensorflow_serving/core/aspired_versions_manager_builder.h +++ b/tensorflow_serving/core/aspired_versions_manager_builder.h @@ -28,27 +28,30 @@ limitations under the License. namespace tensorflow { namespace serving { +// TODO(b/64163389): revisit the escaped HTML characters in c2devsite toolchain + /// Builds an AspiredVersionsManager with options and sources connected to it. /// It takes over the ownership of the sources and the returned manager handles -/// the destruction of itself and its dependencies. Both single sources and +/// the destruction of itself and its dependencies. Both single sources and /// source/source-adapter chains are accepted, i.e. you can use sources that -/// directly supply loaders (Source) or composites that -/// consist of Source + some chain of SourceAdapter, ..., -/// SourceAdapter<..., std::unique_ptr>. The builder connects the chain -/// for you. +/// directly supply loaders (Source&lt;std::unique_ptr&lt;Loader>>) or +/// composites that consist of Source&lt;S> + some chain of +/// SourceAdapter&lt;S, ...>, ..., SourceAdapter&lt;..., +/// std::unique_ptr&lt;Loader>>. The builder connects the chain for you. /// /// Usage: -/// ... -/// AspiredVersionsManagerBuilder::Options options = ManagerOptions(); -/// std::unique_ptr builder; -/// TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( -/// std::move(options), &builder)); -/// builder->AddSource(std::move(some_source)); -/// builder->AddSourceChain( -/// std::move(source), std::move(source_adapter1), -/// std::move(source_adapter2)); -/// std::unique_ptr manager = builder->Build(); -/// ... +/// +/// ... +/// AspiredVersionsManagerBuilder::Options options = ManagerOptions(); +/// std::unique_ptr<AspiredVersionsManagerBuilder> builder; +/// TF_CHECK_OK(AspiredVersionsManagerBuilder::Create( +/// std::move(options), &builder)); +/// builder->AddSource(std::move(some_source)); +/// builder->AddSourceChain( +/// std::move(source), std::move(source_adapter1), +/// std::move(source_adapter2)); +/// std::unique_ptr<Manager> manager = builder->Build(); +/// ... /// /// NOTE: A builder can only be used to build a single AspiredVersionsManager. /// @@ -65,7 +68,7 @@ class AspiredVersionsManagerBuilder { /// over its ownership. /// /// REQUIRES: Template type S be convertible to - /// Source>. + /// Source&lt;std::unique_ptr&lt;Loader>>. template void AddSource(std::unique_ptr source); diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index 3bd185abd9f..72178973269 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -52,9 +52,9 @@ class BasicManagerTestAccess; /// unloading them. The manager accepts servables in the form of Loaders. /// /// We start managing a servable through one of the ManageServable* methods. You -/// can go on to load the servable after this by calling LoadServable. Loading +/// can go on to load the servable after this by calling LoadServable(). Loading /// will also make the servable available to serve. Once you decide to unload -/// it, you can call UnloadServable on it, which will make it unavailable to +/// it, you can call UnloadServable() on it, which will make it unavailable to /// serve, then unload the servable. /// /// Servables are retained until StopManagingServable() is called. This allows a @@ -65,8 +65,8 @@ class BasicManagerTestAccess; /// only allows loading new servables that fit within the overall resource pool. /// /// BasicManager can be configured to use a thread-pool to do it's load and -/// unloads. This makes the {Load,Unload}Servable() methods schedule the -/// load/unloads rather than executing them synchronously. If there are more +/// unloads. This makes the LoadServable() and UnloadServable() methods schedule +/// the load/unloads rather than executing them synchronously. If there are more /// pending load/unloads than threads in the thread pool, they are processed in /// FIFO order. /// @@ -78,7 +78,7 @@ class BasicManagerTestAccess; /// /// REQUIRES: /// 1. Order of method calls - -/// ManageServable*() -> LoadServable() -> UnloadServable() -> +/// ManageServable() (and variants) -> LoadServable() -> UnloadServable() -> /// StopManagingServable(). /// 2. Do not schedule concurrent load and unloads of the same servable. /// 3. Do not call load or unload multiple times on the same servable. @@ -87,21 +87,21 @@ class BasicManagerTestAccess; /// /// Example usage: /// -/// const ServableId id = {kServableName, 0}; -/// std::unique_ptr loader = ...; -/// ... -/// BasicManager manager; -/// TF_CHECK_OK(manager.ManageServable( -/// CreateServableData(id, std::move(loader)))); -/// TF_CHECK_OK(manager.LoadServable(id)); +/// const ServableId id = {kServableName, 0}; +/// std::unique_ptr<Loader> loader = ...; +/// ... +/// BasicManager manager; +/// TF_CHECK_OK(manager.ManageServable( +/// CreateServableData(id, std::move(loader)))); +/// TF_CHECK_OK(manager.LoadServable(id)); /// -/// ... -/// TF_CHECK_OK(manager.GetServableHandle( -/// ServableRequest::Latest(kServableName), &handle)); -/// ... +/// ... +/// TF_CHECK_OK(manager.GetServableHandle( +/// ServableRequest::Latest(kServableName), &handle)); +/// ... /// -/// TF_CHECK_OK(manager.UnloadServable(id)); -/// TF_CHECK_OK(manager.StopManagingServable(id)); +/// TF_CHECK_OK(manager.UnloadServable(id)); +/// TF_CHECK_OK(manager.StopManagingServable(id)); class BasicManager : public Manager { public: // Type of the callback to be called just before a servable is to be loaded. @@ -165,7 +165,7 @@ class BasicManager : public Manager { /// /// Returns an error if given a servable that is already being managed. /// - /// If 'servable' is in an error state, this method does *not* return an + /// If *servable* is in an error state, this method does **not** return an /// error. Instead, the manager accepts the servable, puts it in state kError /// (with a notification sent to the event bus), and then immediately stops /// managing it. This behavior facilitates uniform handling of errors that @@ -222,7 +222,7 @@ class BasicManager : public Manager { using DoneCallback = std::function; /// Loads the servable with this id, and updates the serving map too. Calls - /// 'done_callback' with ok iff the servable was loaded successfully, else + /// *done_callback* with ok iff the servable was loaded successfully, else /// returns an error status. /// /// If using a thread-pool, this method transitions the servable harness to @@ -230,7 +230,7 @@ class BasicManager : public Manager { /// completes the load before returning. /// /// REQUIRES: This manager should have been managing this servable already, - /// for it to be loaded, else we call 'done_callback' with an error status. Do + /// for it to be loaded, else we call *done_callback* with an error status. Do /// not call this multiple times on the same servable. Only one of those will /// succeed and the rest will fail with an error status. void LoadServable(const ServableId& id, DoneCallback done_callback); @@ -244,7 +244,7 @@ class BasicManager : public Manager { void CancelLoadServableRetry(const ServableId& id); /// Unloads the servable with this id, and updates the serving map too. Calls - /// 'done_callback' with ok iff the servable was unloaded successfully, else + /// *done_callback* with ok iff the servable was unloaded successfully, else /// returns an error status. /// /// If using a thread-pool, this method transitions the servable harness to @@ -252,7 +252,7 @@ class BasicManager : public Manager { /// the unload before returning. /// /// REQUIRES: This manager should have loaded and made this servable - /// available, for it to be unloaded, else calls 'done_callback' with an error + /// available, for it to be unloaded, else calls *done_callback* with an error /// status. Do not call this multiple times on the same servable. Only one of /// those will succeed and the rest will fail with an error status. void UnloadServable(const ServableId& id, DoneCallback done_callback); diff --git a/tensorflow_serving/core/loader.h b/tensorflow_serving/core/loader.h index 0e379932b30..88d9fb631bc 100644 --- a/tensorflow_serving/core/loader.h +++ b/tensorflow_serving/core/loader.h @@ -110,7 +110,7 @@ class Loader { /// /// Serving user request: /// - /// ServableHandle handle = ... + /// ServableHandle<CustomServable> handle = ... /// CustomServable* servable = handle.get(); /// servable->... /// diff --git a/tensorflow_serving/core/servable_handle.h b/tensorflow_serving/core/servable_handle.h index 27bab01ce97..1067f6ac53e 100644 --- a/tensorflow_serving/core/servable_handle.h +++ b/tensorflow_serving/core/servable_handle.h @@ -57,19 +57,21 @@ class UntypedServableHandle { /// Using a pointer type for T will fail to compile, since it would be a mistake /// to do so in most situations. /// -/// Example Use: -/// // Define or use an existing servable: -/// class MyServable { -/// public: -/// void MyMethod(); -/// }; +/// Example use: /// -/// // Get your handle from a manager. -/// ServableHandle handle; -/// TF_RETURN_IF_ERROR(manager->GetServableHandle(id, &handle)); +/// // Define or use an existing servable: +/// class MyServable { +/// public: +/// void MyMethod(); +/// }; +/// +/// // Get your handle from a manager. +/// ServableHandle<MyServable> handle; +/// TF_RETURN_IF_ERROR(manager->GetServableHandle(id, &handle)); +/// +/// // Use your handle as a smart-pointer: +/// handle->MyMethod(); /// -/// // Use your handle as a smart-pointer: -/// handle->MyMethod(); template class ServableHandle { public: diff --git a/tensorflow_serving/core/servable_state_monitor.h b/tensorflow_serving/core/servable_state_monitor.h index 86451d777fe..f2abc56be16 100644 --- a/tensorflow_serving/core/servable_state_monitor.h +++ b/tensorflow_serving/core/servable_state_monitor.h @@ -33,9 +33,9 @@ limitations under the License. namespace tensorflow { namespace serving { -/// A utility that listens to an EventBus, and keeps track of the -/// state of each servable mentioned on the bus. The intended use case is to -/// track the states of servables in a Manager. +/// A utility that listens to an EventBus&lt;ServableState>, and keeps track +/// of the state of each servable mentioned on the bus. The intended use case is +/// to track the states of servables in a Manager. /// /// Offers an interface for querying the servable states. It may be useful as /// the basis for dashboards, as well as for testing a manager. diff --git a/tensorflow_serving/core/source_adapter.h b/tensorflow_serving/core/source_adapter.h index 6383ace58b8..0eb6c34d36e 100644 --- a/tensorflow_serving/core/source_adapter.h +++ b/tensorflow_serving/core/source_adapter.h @@ -39,9 +39,10 @@ namespace serving { /// data of type InputType and converts them into calls with data of type /// OutputType. /// -/// A common example uses InputType=StoragePath, OutputType=unique_ptr, -/// in which case the module "converts" each incoming storage path into a loader -/// capable of loading a (particular type of) servable based on the path. +/// A common example uses InputType=StoragePath, +/// OutputType=unique_ptr&lt;Loader>, in which case the module "converts" +/// each incoming storage path into a loader capable of loading a (particular +/// type of) servable based on the path. /// /// SourceAdapters are typically stateless. However, as with all Sources they /// can house state that is shared among multiple emitted servables. See the From 4a01690fa4aeb2643a05fbb1be69097a3df43f42 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 1 Aug 2017 10:04:43 -0800 Subject: [PATCH 0316/8554] Internal change. Change: 163855121 --- tensorflow_serving/tools/pip_package/BUILD | 21 +++++++ .../tools/pip_package/build_pip_package.sh | 61 +++++++++++++++++++ tensorflow_serving/tools/pip_package/setup.py | 38 ++++++++++++ 3 files changed, 120 insertions(+) create mode 100644 tensorflow_serving/tools/pip_package/BUILD create mode 100755 tensorflow_serving/tools/pip_package/build_pip_package.sh create mode 100644 tensorflow_serving/tools/pip_package/setup.py diff --git a/tensorflow_serving/tools/pip_package/BUILD b/tensorflow_serving/tools/pip_package/BUILD new file mode 100644 index 00000000000..d2313b95663 --- /dev/null +++ b/tensorflow_serving/tools/pip_package/BUILD @@ -0,0 +1,21 @@ +# Description: Tensorflow Serving pip package. + +licenses(["notice"]) # Apache 2.0 + +sh_binary( + name = "build_pip_package", + srcs = ["build_pip_package.sh"], + data = [ + "setup.py", + + # Scripts. + "//tensorflow_serving/apis:get_model_metadata_proto_py_pb2", + "//tensorflow_serving/apis:input_proto_py_pb2", + "//tensorflow_serving/apis:model_proto_py_pb2", + "//tensorflow_serving/apis:predict_proto_py_pb2", + "//tensorflow_serving/apis:prediction_service_proto_py_pb2", + "//tensorflow_serving/apis:classification_proto_py_pb2", + "//tensorflow_serving/apis:inference_proto_py_pb2", + "//tensorflow_serving/apis:regression_proto_py_pb2", + ], +) diff --git a/tensorflow_serving/tools/pip_package/build_pip_package.sh b/tensorflow_serving/tools/pip_package/build_pip_package.sh new file mode 100755 index 00000000000..4a5f929854c --- /dev/null +++ b/tensorflow_serving/tools/pip_package/build_pip_package.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +set -e + +function main() { + if [[ $# -lt 1 ]] ; then + echo "No destination dir provided." + echo "Example usage: $0 /tmp/tfs_api_pkg" + exit 1 + fi + + if [[ ! -d "bazel-bin/tensorflow_serving" ]]; then + echo "Could not find bazel-bin. Did you run from the root of the build "\ + "tree?" + exit 1 + fi + + DEST="$1" + TMPDIR="$(mktemp -d)" + local PIP_SRC_DIR="tensorflow_serving/tools/pip_package" + + echo $(date) : "=== Using tmpdir: ${TMPDIR}" + mkdir -p ${TMPDIR}/tensorflow_serving/apis + + echo "Adding python files" + cp bazel-genfiles/tensorflow_serving/apis/*_pb2.py \ + "${TMPDIR}/tensorflow_serving/apis" + + cp bazel-serving/tensorflow_serving/apis/prediction_service_pb2.py \ + "${TMPDIR}/tensorflow_serving/apis" + + touch "${TMPDIR}/tensorflow_serving/apis/__init__.py" + touch "${TMPDIR}/tensorflow_serving/__init__.py" + + echo "Adding package setup files" + cp ${PIP_SRC_DIR}/setup.py "${TMPDIR}" + + pushd "${TMPDIR}" + echo $(date) : "=== Building wheel" + python setup.py bdist_wheel # >/dev/null + mkdir -p "${DEST}" + cp dist/* "${DEST}" + popd + rm -rf "${TMPDIR}" + echo $(date) : "=== Output wheel file is in: ${DEST}" +} + +main "$@" diff --git a/tensorflow_serving/tools/pip_package/setup.py b/tensorflow_serving/tools/pip_package/setup.py new file mode 100644 index 00000000000..018803b36ed --- /dev/null +++ b/tensorflow_serving/tools/pip_package/setup.py @@ -0,0 +1,38 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Package Setup script for the TensorFlow Serving Python API. +""" +from setuptools import find_packages +from setuptools import setup + + +_VERSION = '1.0.0.rc0' + +REQUIRED_PACKAGES = [ + 'tensorflow>=1.2.0', + 'grpcio>=1.0', +] + +setup( + name='tensorflow-serving-api', + version=_VERSION, + author='Google Inc.', + author_email='opensource@google.com', + packages=find_packages(), + description='TensorFlow Serving API libraries', + license='Apache 2.0', + url='http://tensorflow.org/deploy/tfserve', + keywords='tensorflow serving machine learning api libraries', + install_requires=REQUIRED_PACKAGES, +) From 4eb2fcb09a81ba01234f0a8f20115af5809613a3 Mon Sep 17 00:00:00 2001 From: Wolff Date: Tue, 1 Aug 2017 13:48:55 -0700 Subject: [PATCH 0317/8554] Add index to leftnav --- tensorflow_serving/g3doc/leftnav_files | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/g3doc/leftnav_files b/tensorflow_serving/g3doc/leftnav_files index 146e204d6a1..d7d658c8677 100644 --- a/tensorflow_serving/g3doc/leftnav_files +++ b/tensorflow_serving/g3doc/leftnav_files @@ -1,4 +1,5 @@ ### TensorFlow Serving +index.md architecture_overview.md setup.md serving_basic.md From 54e6747fae55207b43684633c544a3b2a76ba07c Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 1 Aug 2017 12:44:55 -0800 Subject: [PATCH 0318/8554] Fix broken OSS build. Change: 163879728 --- tensorflow_serving/model_servers/test_util/BUILD | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index eb61182027a..caee7eefaf0 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -28,7 +28,7 @@ cc_library( "//visibility:public", ], deps = [ - "//base", + "//external:gtest", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/config:platform_config_proto", @@ -42,7 +42,6 @@ cc_library( "//tensorflow_serving/model_servers:server_core", "//tensorflow_serving/util:event_bus", "//tensorflow_serving/util:unique_ptr_with_deps", - "//testing/base/public:gunit_for_library", "@org_tensorflow//tensorflow/core:lib", "@protobuf//:cc_wkt_protos", "@protobuf//:protobuf_lite", From 6848ba2650b6369982eef693793d774cef96f782 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 1 Aug 2017 12:55:24 -0800 Subject: [PATCH 0319/8554] Update author_email in setup.py Change: 163881359 --- tensorflow_serving/tools/pip_package/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/tools/pip_package/setup.py b/tensorflow_serving/tools/pip_package/setup.py index 018803b36ed..00e64635c23 100644 --- a/tensorflow_serving/tools/pip_package/setup.py +++ b/tensorflow_serving/tools/pip_package/setup.py @@ -17,7 +17,7 @@ from setuptools import setup -_VERSION = '1.0.0.rc0' +_VERSION = '1.0.0rc0' REQUIRED_PACKAGES = [ 'tensorflow>=1.2.0', @@ -28,7 +28,7 @@ name='tensorflow-serving-api', version=_VERSION, author='Google Inc.', - author_email='opensource@google.com', + author_email='tensorflow-serving-dev@googlegroups.com', packages=find_packages(), description='TensorFlow Serving API libraries', license='Apache 2.0', From 5b1bd50c20d2592710c646c9e06ccb4f328c7a1f Mon Sep 17 00:00:00 2001 From: Wolff Dobson Date: Tue, 1 Aug 2017 14:31:17 -0700 Subject: [PATCH 0320/8554] Add title to pick up TOC (#546) --- tensorflow_serving/g3doc/index.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_serving/g3doc/index.md b/tensorflow_serving/g3doc/index.md index d1869e0072f..18d0e28c42d 100644 --- a/tensorflow_serving/g3doc/index.md +++ b/tensorflow_serving/g3doc/index.md @@ -1,3 +1,5 @@ +# Introduction + TensorFlow Serving is a flexible, high-performance serving system for machine learning models, designed for production environments. TensorFlow Serving makes it easy to deploy new algorithms and experiments, while keeping the same From 26c64be127965974401f6082b80372ebb5954521 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 1 Aug 2017 15:07:52 -0800 Subject: [PATCH 0321/8554] Deflake tensorflow_model_server_test. Change: 163901492 --- .../model_servers/tensorflow_model_server_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 56e038ff7fe..96406e4327c 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -48,7 +48,7 @@ def PickUnusedPort(): s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM) - s.bind(('localhost', 0)) + s.bind(('', 0)) port = s.getsockname()[1] s.close() return port From 25f1501d8e8305a2a08e2d2c61be09ac82b47602 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 2 Aug 2017 10:35:07 -0800 Subject: [PATCH 0322/8554] Make mnist tutorials work with both Bazel and locally installed tensorflow-serving-api PIP package. Change: 164014022 --- tensorflow_serving/example/mnist_client.py | 2 +- tensorflow_serving/example/mnist_export.py | 2 +- tensorflow_serving/example/mnist_saved_model.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/example/mnist_client.py b/tensorflow_serving/example/mnist_client.py index 843696ac33b..947f7c49b03 100644 --- a/tensorflow_serving/example/mnist_client.py +++ b/tensorflow_serving/example/mnist_client.py @@ -38,7 +38,7 @@ from tensorflow_serving.apis import predict_pb2 from tensorflow_serving.apis import prediction_service_pb2 -from tensorflow_serving.example import mnist_input_data +import mnist_input_data tf.app.flags.DEFINE_integer('concurrency', 1, diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py index 7c2c1879ff0..a6c26db098a 100644 --- a/tensorflow_serving/example/mnist_export.py +++ b/tensorflow_serving/example/mnist_export.py @@ -37,7 +37,7 @@ import tensorflow as tf from tensorflow.contrib.session_bundle import exporter -from tensorflow_serving.example import mnist_input_data +import mnist_input_data tf.app.flags.DEFINE_integer('training_iteration', 1000, 'number of training iterations.') diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py index 795b3865e9b..7fc4dabed8b 100644 --- a/tensorflow_serving/example/mnist_saved_model.py +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -31,7 +31,7 @@ import tensorflow as tf -from tensorflow_serving.example import mnist_input_data +import mnist_input_data tf.app.flags.DEFINE_integer('training_iteration', 1000, 'number of training iterations.') From bef2c6a239d070d8d95e9839ce3559958ff886c2 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 2 Aug 2017 10:51:04 -0800 Subject: [PATCH 0323/8554] Add documentation to install the new TensorFlow Serving API PIP package and update the simple tutorial to use it. Change: 164016451 --- tensorflow_serving/g3doc/serving_basic.md | 24 ++++++++++++++++++++++- tensorflow_serving/g3doc/setup.md | 9 +++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index e67f70ba68b..e3249f4ed8e 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -150,6 +150,16 @@ Clear the export directory if it already exists: $>rm -rf /tmp/mnist_model ``` +If you would like to install the `tensorflow` and `tensorflow-serving-api` PIP +packages, you can run all Python code (export and client) using a simple +`python` command. To install the PIP package, follow the [instructions +here](setup.md#tensorflow-serving-python-api-pip-package). It's also possible to +use Bazel to build the necessary dependencies and run all code without +installing those packages. The rest of the codelab will have instructions for +both the Bazel and PIP options. + +Bazel: + ```shell $>bazel build -c opt //tensorflow_serving/example:mnist_saved_model $>bazel-bin/tensorflow_serving/example/mnist_saved_model /tmp/mnist_model @@ -162,6 +172,12 @@ Exporting trained model to /tmp/mnist_model Done exporting! ``` +Or if you have `tensorflow-serving-api` installed, you can run: + +```shell +python tensorflow_serving/example/mnist_saved_model.py /tmp/mnist_model +``` + Now let's take a look at the export directory. ```shell @@ -212,7 +228,7 @@ We can use the provided utility to test the server. The client downloads MNIST test data, sends them as requests to the server, and calculates the inference error rate. -To run it: +To run it with Bazel: ```shell $>bazel build -c opt //tensorflow_serving/example:mnist_client @@ -221,6 +237,12 @@ $>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=lo Inference error rate: 10.5% ``` +Alternatively if you installed the PIP package, run: + +```shell +python tensorflow_serving/example/mnist_client.py --num_tests=1000 --server=localhost:9000 +``` + We expect around 91% accuracy for the trained Softmax model and we get 10.5% inference error rate for the first 1000 test images. This confirms that the server loads and runs the trained model successfully! diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 2b1731b0cc4..e664c107435 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -57,6 +57,15 @@ sudo apt-get update && sudo apt-get install -y \ zlib1g-dev ``` +### TensorFlow Serving Python API PIP package + +To run Python client code without the need to install Bazel, you can install +the `tensorflow-serving-api` PIP package using: + +```shell +pip install tensorflow-serving-api +``` + ## Installing using apt-get ### Available binaries From 6855c039840c4a4dd8137d4e80cada86fd168d48 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 2 Aug 2017 14:37:56 -0800 Subject: [PATCH 0324/8554] Delete deprecated SessionBundle example models keeping only the SavedModel variants (only supported TF format). Change: 164049143 --- tensorflow_serving/example/BUILD | 30 --- .../example/inception_export.py | 172 ------------------ tensorflow_serving/example/mnist_export.py | 117 ------------ .../example/mnist_saved_model.py | 5 +- 4 files changed, 3 insertions(+), 321 deletions(-) delete mode 100644 tensorflow_serving/example/inception_export.py delete mode 100644 tensorflow_serving/example/mnist_export.py diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index af19bbc222f..d86b2677709 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -26,21 +26,6 @@ py_library( srcs_version = "PY2AND3", ) -# TODO(b/32628014): remove mnist_export after we no longer support -# SessionBundle. -py_binary( - name = "mnist_export", - srcs = [ - "mnist_export.py", - ], - srcs_version = "PY2AND3", - deps = [ - ":mnist_input_data", - "@org_tensorflow//tensorflow:tensorflow_py", - "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", - ], -) - py_binary( name = "mnist_saved_model", srcs = [ @@ -67,21 +52,6 @@ py_binary( ], ) -# TODO(b/32628014): remove inception_export after we no longer support -# SessionBundle. -py_binary( - name = "inception_export", - srcs = [ - "inception_export.py", - ], - srcs_version = "PY2AND3", - deps = [ - "@inception_model//inception", - "@org_tensorflow//tensorflow:tensorflow_py", - "@org_tensorflow//tensorflow/contrib/session_bundle:exporter", - ], -) - py_binary( name = "inception_saved_model", srcs = [ diff --git a/tensorflow_serving/example/inception_export.py b/tensorflow_serving/example/inception_export.py deleted file mode 100644 index 5db25de18dd..00000000000 --- a/tensorflow_serving/example/inception_export.py +++ /dev/null @@ -1,172 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -#!/usr/bin/env python2.7 -"""Export inception model given existing training checkpoints. - -Note the export format for this example is SessionBundle, which is deprecated. -Please use SavedModel instead. Specifically, see: `inception_saved_model.py`. - -The model is exported with proper signatures that can be loaded by standard -tensorflow_model_server. -""" - -from __future__ import print_function - -import os.path - -# This is a placeholder for a Google-internal import. - -import tensorflow as tf - -from tensorflow.contrib.session_bundle import exporter -from inception import inception_model - - -tf.app.flags.DEFINE_string('checkpoint_dir', '/tmp/inception_train', - """Directory where to read training checkpoints.""") -tf.app.flags.DEFINE_string('export_dir', '/tmp/inception_export', - """Directory where to export inference model.""") -tf.app.flags.DEFINE_integer('image_size', 299, - """Needs to provide same value as in training.""") -FLAGS = tf.app.flags.FLAGS - - -NUM_CLASSES = 1000 -NUM_TOP_CLASSES = 5 - -WORKING_DIR = os.path.dirname(os.path.realpath(__file__)) -SYNSET_FILE = os.path.join(WORKING_DIR, 'imagenet_lsvrc_2015_synsets.txt') -METADATA_FILE = os.path.join(WORKING_DIR, 'imagenet_metadata.txt') - - -def export(): - # Create index->synset mapping - synsets = [] - with open(SYNSET_FILE) as f: - synsets = f.read().splitlines() - # Create synset->metadata mapping - texts = {} - with open(METADATA_FILE) as f: - for line in f.read().splitlines(): - parts = line.split('\t') - assert len(parts) == 2 - texts[parts[0]] = parts[1] - - with tf.Graph().as_default(): - # Build inference model. - # Please refer to Tensorflow inception model for details. - - # Input transformation. - serialized_tf_example = tf.placeholder(tf.string, name='tf_example') - feature_configs = { - 'image/encoded': tf.FixedLenFeature(shape=[], dtype=tf.string), - } - tf_example = tf.parse_example(serialized_tf_example, feature_configs) - jpegs = tf_example['image/encoded'] - images = tf.map_fn(preprocess_image, jpegs, dtype=tf.float32) - - # Run inference. - logits, _ = inception_model.inference(images, NUM_CLASSES + 1) - - # Transform output to topK result. - values, indices = tf.nn.top_k(logits, NUM_TOP_CLASSES) - - # Create a constant string Tensor where the i'th element is - # the human readable class description for the i'th index. - # Note that the 0th index is an unused background class - # (see inception model definition code). - class_descriptions = ['unused background'] - for s in synsets: - class_descriptions.append(texts[s]) - class_tensor = tf.constant(class_descriptions) - - table = tf.contrib.lookup.index_to_string_table_from_tensor(class_tensor) - classes = table.lookup(tf.to_int64(indices)) - - # Restore variables from training checkpoint. - variable_averages = tf.train.ExponentialMovingAverage( - inception_model.MOVING_AVERAGE_DECAY) - variables_to_restore = variable_averages.variables_to_restore() - saver = tf.train.Saver(variables_to_restore) - with tf.Session() as sess: - # Restore variables from training checkpoints. - ckpt = tf.train.get_checkpoint_state(FLAGS.checkpoint_dir) - if ckpt and ckpt.model_checkpoint_path: - saver.restore(sess, ckpt.model_checkpoint_path) - # Assuming model_checkpoint_path looks something like: - # /my-favorite-path/imagenet_train/model.ckpt-0, - # extract global_step from it. - global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] - print('Successfully loaded model from %s at step=%s.' % - (ckpt.model_checkpoint_path, global_step)) - else: - print('No checkpoint file found at %s' % FLAGS.checkpoint_dir) - return - - # Export inference model. - init_op = tf.group(tf.tables_initializer(), name='init_op') - classification_signature = exporter.classification_signature( - input_tensor=serialized_tf_example, - classes_tensor=classes, - scores_tensor=values) - named_graph_signature = { - 'inputs': exporter.generic_signature({'images': jpegs}), - 'outputs': exporter.generic_signature({ - 'classes': classes, - 'scores': values - })} - model_exporter = exporter.Exporter(saver) - model_exporter.init( - init_op=init_op, - default_graph_signature=classification_signature, - named_graph_signatures=named_graph_signature) - model_exporter.export(FLAGS.export_dir, tf.constant(global_step), sess) - print('Successfully exported model to %s' % FLAGS.export_dir) - - -def preprocess_image(image_buffer): - """Preprocess JPEG encoded bytes to 3D float Tensor.""" - - # Decode the string as an RGB JPEG. - # Note that the resulting image contains an unknown height and width - # that is set dynamically by decode_jpeg. In other words, the height - # and width of image is unknown at compile-time. - image = tf.image.decode_jpeg(image_buffer, channels=3) - # After this point, all image pixels reside in [0,1) - # until the very end, when they're rescaled to (-1, 1). The various - # adjust_* ops all require this range for dtype float. - image = tf.image.convert_image_dtype(image, dtype=tf.float32) - # Crop the central region of the image with an area containing 87.5% of - # the original image. - image = tf.image.central_crop(image, central_fraction=0.875) - # Resize the image to the original height and width. - image = tf.expand_dims(image, 0) - image = tf.image.resize_bilinear(image, - [FLAGS.image_size, FLAGS.image_size], - align_corners=False) - image = tf.squeeze(image, [0]) - # Finally, rescale to [-1,1] instead of [0, 1) - image = tf.subtract(image, 0.5) - image = tf.multiply(image, 2.0) - return image - - -def main(unused_argv=None): - export() - - -if __name__ == '__main__': - tf.app.run() diff --git a/tensorflow_serving/example/mnist_export.py b/tensorflow_serving/example/mnist_export.py deleted file mode 100644 index a6c26db098a..00000000000 --- a/tensorflow_serving/example/mnist_export.py +++ /dev/null @@ -1,117 +0,0 @@ -# Copyright 2016 Google Inc. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -#!/usr/bin/env python2.7 - -"""Train and export a simple Softmax Regression TensorFlow model. - -Note the export format for this example is SessionBundle, which is deprecated. -Please use SavedModel instead. Specifically, see: `mnist_saved_model.py`. - -The model is from the TensorFlow "MNIST For ML Beginner" tutorial. This program -simply follows all its training instructions, and uses TensorFlow Serving -exporter to export the trained model with proper signatures that can be -loaded by standard tensorflow_model_server. - -Usage: mnist_export.py [--training_iteration=x] [--export_version=y] export_dir -""" - -from __future__ import print_function - -import sys - -# This is a placeholder for a Google-internal import. - -import tensorflow as tf - -from tensorflow.contrib.session_bundle import exporter -import mnist_input_data - -tf.app.flags.DEFINE_integer('training_iteration', 1000, - 'number of training iterations.') -tf.app.flags.DEFINE_integer('export_version', 1, 'version number of the model.') -tf.app.flags.DEFINE_string('work_dir', '/tmp', 'Working directory.') -FLAGS = tf.app.flags.FLAGS - - -def main(_): - if len(sys.argv) < 2 or sys.argv[-1].startswith('-'): - print('Usage: mnist_export.py [--training_iteration=x] ' - '[--export_version=y] export_dir') - sys.exit(-1) - if FLAGS.training_iteration <= 0: - print('Please specify a positive value for training iteration.') - sys.exit(-1) - if FLAGS.export_version <= 0: - print('Please specify a positive value for version number.') - sys.exit(-1) - - # Train model - print('Training model...') - mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) - sess = tf.InteractiveSession() - serialized_tf_example = tf.placeholder(tf.string, name='tf_example') - feature_configs = { - 'x': tf.FixedLenFeature(shape=[784], dtype=tf.float32), - } - tf_example = tf.parse_example(serialized_tf_example, feature_configs) - x = tf.identity(tf_example['x'], name='x') # use tf.identity() to assign name - y_ = tf.placeholder('float', shape=[None, 10]) - w = tf.Variable(tf.zeros([784, 10])) - b = tf.Variable(tf.zeros([10])) - sess.run(tf.global_variables_initializer()) - y = tf.nn.softmax(tf.matmul(x, w) + b, name='y') - cross_entropy = -tf.reduce_sum(y_ * tf.log(y)) - train_step = tf.train.GradientDescentOptimizer(0.01).minimize(cross_entropy) - values, indices = tf.nn.top_k(y, 10) - table = tf.contrib.lookup.index_to_string_table_from_tensor( - tf.constant([str(i) for i in range(10)])) - prediction_classes = table.lookup(tf.to_int64(indices)) - for _ in range(FLAGS.training_iteration): - batch = mnist.train.next_batch(50) - train_step.run(feed_dict={x: batch[0], y_: batch[1]}) - correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) - accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) - print('training accuracy %g' % - sess.run(accuracy, - feed_dict={x: mnist.test.images, - y_: mnist.test.labels})) - print('Done training!') - - # Export model - # WARNING(break-tutorial-inline-code): The following code snippet is - # in-lined in tutorials, please update tutorial documents accordingly - # whenever code changes. - export_path = sys.argv[-1] - print('Exporting trained model to %s' % export_path) - init_op = tf.group(tf.tables_initializer(), name='init_op') - saver = tf.train.Saver(sharded=True) - model_exporter = exporter.Exporter(saver) - model_exporter.init( - sess.graph.as_graph_def(), - init_op=init_op, - default_graph_signature=exporter.classification_signature( - input_tensor=serialized_tf_example, - classes_tensor=prediction_classes, - scores_tensor=values), - named_graph_signatures={ - 'inputs': exporter.generic_signature({'images': x}), - 'outputs': exporter.generic_signature({'scores': y})}) - model_exporter.export(export_path, tf.constant(FLAGS.export_version), sess) - print('Done exporting!') - - -if __name__ == '__main__': - tf.app.run() diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py index 7fc4dabed8b..196cf9af69c 100644 --- a/tensorflow_serving/example/mnist_saved_model.py +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -14,14 +14,15 @@ # ============================================================================== #!/usr/bin/env python2.7 -"""Train and export a simple Softmax Regression TensorFlow model. +r"""Train and export a simple Softmax Regression TensorFlow model. The model is from the TensorFlow "MNIST For ML Beginner" tutorial. This program simply follows all its training instructions, and uses TensorFlow SavedModel to export the trained model with proper signatures that can be loaded by standard tensorflow_model_server. -Usage: mnist_export.py [--training_iteration=x] [--model_version=y] export_dir +Usage: mnist_saved_model.py [--training_iteration=x] [--model_version=y] \ + export_dir """ import os From c70c1d4af85ced17b2f65ab6793835faf20ba565 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 3 Aug 2017 08:37:44 -0800 Subject: [PATCH 0325/8554] Remove use_saved_model flag in the ModelServer. Change: 164134747 --- tensorflow_serving/model_servers/main.cc | 10 +---- .../tensorflow_model_server_test.py | 42 +++++-------------- 2 files changed, 12 insertions(+), 40 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index a582afc33b9..f097d24423a 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -308,7 +308,7 @@ int main(int argc, char** argv) { tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; tensorflow::string model_base_path; - bool use_saved_model = true; + const bool use_saved_model = true; // Tensorflow session parallelism of zero means that both inter and intra op // thread pools will be auto configured. tensorflow::int64 tensorflow_session_parallelism = 0; @@ -338,11 +338,6 @@ int main(int argc, char** argv) { &file_system_poll_wait_seconds, "interval in seconds between each poll of the file " "system for new model version"), - tensorflow::Flag("use_saved_model", &use_saved_model, - "If true, use SavedModel in the server; otherwise, use " - "SessionBundle. It is used by tensorflow serving team " - "to control the rollout of SavedModel and is not " - "expected to be set by users directly."), tensorflow::Flag("tensorflow_session_parallelism", &tensorflow_session_parallelism, "Number of threads to use for running a " @@ -353,8 +348,7 @@ int main(int argc, char** argv) { "If non-empty, read an ascii PlatformConfigMap protobuf " "from the supplied file name, and use that platform " "config instead of the Tensorflow platform. (If used, " - "--enable_batching and --use_saved_model are " - "ignored.)")}; + "--enable_batching is ignored.)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || (model_base_path.empty() && model_config_file.empty())) { diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 96406e4327c..45145a3cb66 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -117,7 +117,6 @@ def RunServer(self, port, model_name, model_path, - use_saved_model, batching_parameters_file='', wait_for_server_ready=True): """Run tensorflow_model_server using test config.""" @@ -126,7 +125,6 @@ def RunServer(self, command += ' --port=' + str(port) command += ' --model_name=' + model_name command += ' --model_base_path=' + model_path - command += ' --use_saved_model=' + str(use_saved_model).lower() if batching_parameters_file: command += ' --enable_batching' command += ' --batching_parameters_file=' + batching_parameters_file @@ -140,7 +138,6 @@ def RunServer(self, def RunServerWithModelConfigFile(self, port, model_config_file, - use_saved_model, pipe=None, wait_for_server_ready=True): """Run tensorflow_model_server using test config.""" @@ -148,7 +145,6 @@ def RunServerWithModelConfigFile(self, command = os.path.join(self.binary_dir, 'tensorflow_model_server') command += ' --port=' + str(port) command += ' --model_config_file=' + model_config_file - command += ' --use_saved_model=' + str(use_saved_model).lower() print command self.server_proc = subprocess.Popen(shlex.split(command), stderr=pipe) @@ -217,11 +213,10 @@ def _GetBatchingParametersFile(self): def testClassify(self): """Test PredictionService.Classify implementation.""" model_path = self._GetSavedModelBundlePath() - use_saved_model = True atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', - model_path, use_saved_model) + model_path) print 'Sending Classify request...' # Prepare request @@ -247,11 +242,10 @@ def testClassify(self): def testRegress(self): """Test PredictionService.Regress implementation.""" model_path = self._GetSavedModelBundlePath() - use_saved_model = True atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', - model_path, use_saved_model) + model_path) print 'Sending Regress request...' # Prepare request @@ -275,12 +269,11 @@ def testRegress(self): def testMultiInference(self): """Test PredictionService.MultiInference implementation.""" model_path = self._GetSavedModelBundlePath() - use_saved_model = True enable_batching = False atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', - model_path, use_saved_model, + model_path, enable_batching) print 'Sending MultiInference request...' @@ -312,47 +305,39 @@ def testMultiInference(self): def _TestPredict(self, model_path, - use_saved_model, batching_parameters_file=''): """Helper method to test prediction. Args: model_path: Path to the model on disk. - use_saved_model: Whether the model server should use SavedModel. batching_parameters_file: Batching parameters file to use (if left empty, batching is not enabled). """ atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', - model_path, use_saved_model, - batching_parameters_file) + model_path, batching_parameters_file) self.VerifyPredictRequest(model_server_address, expected_output=3.0) self.VerifyPredictRequest( model_server_address, expected_output=3.0, specify_output=False) - def testPredictSessionBundle(self): - """Test PredictionService.Predict implementation with SessionBundle.""" - self._TestPredict(self._GetSessionBundlePath(), use_saved_model=False) - - def testPredictBatchingSessionBundle(self): + def testPredictBatching(self): """Test PredictionService.Predict implementation with SessionBundle.""" self._TestPredict( self._GetSessionBundlePath(), - use_saved_model=False, batching_parameters_file=self._GetBatchingParametersFile()) def testPredictSavedModel(self): """Test PredictionService.Predict implementation with SavedModel.""" - self._TestPredict(self._GetSavedModelBundlePath(), use_saved_model=True) + self._TestPredict(self._GetSavedModelBundlePath()) def testPredictUpconvertedSavedModel(self): """Test PredictionService.Predict implementation. Using a SessionBundle converted to a SavedModel. """ - self._TestPredict(self._GetSessionBundlePath(), use_saved_model=True) + self._TestPredict(self._GetSessionBundlePath()) - def _TestBadModel(self, use_saved_model): + def _TestBadModel(self): """Helper method to test against a bad model export.""" atexit.register(self.TerminateProcs) # Both SessionBundle and SavedModel use the same bad model path, but in the @@ -363,7 +348,6 @@ def _TestBadModel(self, use_saved_model): PickUnusedPort(), 'default', os.path.join(self.testdata_dir, 'bad_half_plus_two'), - use_saved_model, wait_for_server_ready=False) with self.assertRaises(face.AbortionError) as error: self.VerifyPredictRequest(model_server_address, expected_output=3.0) @@ -372,18 +356,13 @@ def _TestBadModel(self, use_saved_model): def _TestBadModelUpconvertedSavedModel(self): """Test Predict against a bad upconverted SavedModel model export.""" - self._TestBadModel(use_saved_model=True) - - def _TestBadModelSessionBundle(self): - """Test Predict against a bad SessionBundle model export.""" - self._TestBadModel(use_saved_model=False) + self._TestBadModel() def testGoodModelConfig(self): """Test server configuration from file works with valid configuration.""" atexit.register(self.TerminateProcs) model_server_address = self.RunServerWithModelConfigFile( - PickUnusedPort(), self._GetGoodModelConfigFile(), - True) # use_saved_model + PickUnusedPort(), self._GetGoodModelConfigFile()) self.VerifyPredictRequest( model_server_address, model_name='half_plus_two', expected_output=3.0) @@ -407,7 +386,6 @@ def testBadModelConfig(self): self.RunServerWithModelConfigFile( PickUnusedPort(), self._GetBadModelConfigFile(), - True, # use_saved_model pipe=subprocess.PIPE) error_message = ( From a35d47239f6de954892f5fdf6ac5172ef2d5a790 Mon Sep 17 00:00:00 2001 From: Wolff Dobson Date: Thu, 3 Aug 2017 16:09:59 -0700 Subject: [PATCH 0326/8554] Fixes linkebreaks in links (#550) * Add title to pick up TOC * Fix linebreaks in links --- tensorflow_serving/g3doc/custom_source.md | 11 ++++++----- tensorflow_serving/g3doc/serving_advanced.md | 4 ++-- tensorflow_serving/g3doc/serving_basic.md | 9 +++++---- tensorflow_serving/g3doc/serving_inception.md | 4 ++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/tensorflow_serving/g3doc/custom_source.md b/tensorflow_serving/g3doc/custom_source.md index fc8e802cb0c..fcef94d3408 100644 --- a/tensorflow_serving/g3doc/custom_source.md +++ b/tensorflow_serving/g3doc/custom_source.md @@ -23,11 +23,12 @@ loaders directly. Of course, whatever kind of data your source emits (whether it is POSIX paths, Google Cloud Storage paths, or RPC handles), there needs to be accompanying module(s) that are able to load servables based on that. Such modules are called -`SourceAdapters`. Creating a custom one is described in the [Custom -Servable](custom_servable.md) document. TensorFlow Serving comes with one for -instantiating TensorFlow sessions based on paths in file systems that TensorFlow -supports. One can add support for additional file systems to TensorFlow by -extending the `RandomAccessFile` abstraction (`tensorflow/core/public/env.h`). +`SourceAdapters`. Creating a custom one is described in the +[Custom Servable](custom_servable.md) document. TensorFlow Serving +comes with one for instantiating TensorFlow sessions based on paths +in file systems that TensorFlow supports. One can add support for +additional file systems to TensorFlow by extending the `RandomAccessFile` +abstraction (`tensorflow/core/public/env.h`). This document focuses on creating a source that emits paths in a TensorFlow-supported file system. It ends with a walk-through of how to use your diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 1ddf2dc92c5..bb9c862b02c 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -35,8 +35,8 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisites). Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the [instructions -here](setup.md#optimized-build). +further optimize the build, refer to the +[instructions here](setup.md#optimized-build). ## Train And Export TensorFlow Model diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index e3249f4ed8e..ed45aef1fd5 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -28,8 +28,8 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisites). Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the [instructions -here](setup.md#optimized-build). +further optimize the build, refer to the +[instructions here](setup.md#optimized-build). ## Train And Export TensorFlow Model @@ -152,8 +152,9 @@ $>rm -rf /tmp/mnist_model If you would like to install the `tensorflow` and `tensorflow-serving-api` PIP packages, you can run all Python code (export and client) using a simple -`python` command. To install the PIP package, follow the [instructions -here](setup.md#tensorflow-serving-python-api-pip-package). It's also possible to +`python` command. To install the PIP package, follow the +[instructions here](setup.md#tensorflow-serving-python-api-pip-package). +It's also possible to use Bazel to build the necessary dependencies and run all code without installing those packages. The rest of the codelab will have instructions for both the Bazel and PIP options. diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 81d89660f0a..661a8ce687b 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -36,8 +36,8 @@ $ docker run --name=inception_container -it $USER/tensorflow-serving-devel ### Clone, configure, and build TensorFlow Serving in a container Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the [instructions -here](setup.md#optimized-build). +further optimize the build, refer to the +[instructions here](setup.md#optimized-build). In the running container, we clone, configure and build TensorFlow Serving example code. From 70709cb00407af8b429d976842427022e4432b9d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 3 Aug 2017 15:22:08 -0800 Subject: [PATCH 0327/8554] Change the default release version numbers to a dummy value. These version numbers only have meaning inside a release branch. Change: 164199528 --- tensorflow_serving/model_servers/BUILD | 4 ++-- tensorflow_serving/tools/pip_package/setup.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 704b2ecd909..5ee703c34ca 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -194,7 +194,7 @@ pkg_deb( homepage = "https://github.com/tensorflow/serving", maintainer = "TensorFlow Serving team", package = "tensorflow-model-server", - version = "1.0.0", + version = "undefined", # Set when releasing a new version of TensorFlow Serving (e.g. 1.0.0). ) # Build with '-c opt' @@ -205,5 +205,5 @@ pkg_deb( homepage = "https://github.com/tensorflow/serving", maintainer = "TensorFlow Serving team", package = "tensorflow-model-server-universal", - version = "1.0.0", + version = "undefined", # Set when releasing a new version of TensorFlow Serving (e.g. 1.0.0). ) diff --git a/tensorflow_serving/tools/pip_package/setup.py b/tensorflow_serving/tools/pip_package/setup.py index 00e64635c23..fad0cff50ad 100644 --- a/tensorflow_serving/tools/pip_package/setup.py +++ b/tensorflow_serving/tools/pip_package/setup.py @@ -16,8 +16,8 @@ from setuptools import find_packages from setuptools import setup - -_VERSION = '1.0.0rc0' +# Set when releasing a new version of TensorFlow Serving (e.g. 1.0.0). +_VERSION = 'undefined' REQUIRED_PACKAGES = [ 'tensorflow>=1.2.0', From c62410615515fd182e8f77e346a4712c4e704897 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 3 Aug 2017 16:31:58 -0800 Subject: [PATCH 0328/8554] Minor documentation fixes. Change: 164207603 --- tensorflow_serving/g3doc/serving_basic.md | 2 +- tensorflow_serving/g3doc/setup.md | 17 +++++++++-------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index e3249f4ed8e..337e8de58ee 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -90,7 +90,7 @@ You can add meta graph and variables to the builder using * `tags` is the set of tags with which to save the meta graph. In this case, since we intend to use the graph in serving, we use the `serve` tag from predefined SavedModel tag constants. For more details, see [tag_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) - and [related TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/tag_constants). + and [related TensorFlow API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/tag_constants). * `signature_def_map` specifies the map of user-supplied key for a **signature** to a tensorflow::SignatureDef to add to the meta graph. diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index e664c107435..b59a61b5eec 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -22,6 +22,7 @@ following steps: chmod +x bazel-0.4.5-installer-linux-x86_64.sh ./bazel-0.4.5-installer-linux-x86_64.sh --user ``` + 2. Set up your environment. Put this in your ~/.bashrc. ```shell @@ -77,7 +78,7 @@ specific compiler optimizations like SSE4 and AVX instructions. This should be the preferred option for most users, but may not work on some older machines. **tensorflow-model-server-universal**: Compiled with basic optimizations, but -doesn't include platform specific instructiob sets, so should work on most if +doesn't include platform specific instruction sets, so should work on most if not all machines out there. Use this if `tensorflow-model-server` does not work for you. Note that the binary name is the same for both packages, so if you already installed tensorflow-model-server, you should first uninstall it using @@ -90,17 +91,17 @@ sudo apt-get remove tensorflow-model-server 1. Add TensorFlow Serving distribution URI as a package source (one time setup) -```shell -echo "deb [arch=amd64] http://storage.googleapis.com/tensorflow-serving-apt stable tensorflow-model-server tensorflow-model-server-universal" | sudo tee /etc/apt/sources.list.d/tensorflow-serving.list + ```shell + echo "deb [arch=amd64] http://storage.googleapis.com/tensorflow-serving-apt stable tensorflow-model-server tensorflow-model-server-universal" | sudo tee /etc/apt/sources.list.d/tensorflow-serving.list -curl https://storage.googleapis.com/tensorflow-serving-apt/tensorflow-serving.release.pub.gpg | sudo apt-key add - -``` + curl https://storage.googleapis.com/tensorflow-serving-apt/tensorflow-serving.release.pub.gpg | sudo apt-key add - + ``` 2. Install and update TensorFlow ModelServer -```shell -sudo apt-get update && sudo apt-get install tensorflow-model-server -``` + ```shell + sudo apt-get update && sudo apt-get install tensorflow-model-server + ``` Once installed, the binary can be invoked using the command `tensorflow_model_server`. From 90b49a5a16d5d242b900a184575fff6e13f4d5f1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 4 Aug 2017 06:47:46 -0800 Subject: [PATCH 0329/8554] Print the warning message when servable version is requested for serving but it's not found in the source directory. Change: 164261601 --- .../file_system_storage_path_source.cc | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index ebea569692e..95dc15aef4d 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -174,13 +174,13 @@ bool AspireSpecificVersions( const FileSystemStoragePathSourceConfig::ServableToMonitor& servable, const std::map& children_by_version, std::vector>* versions) { - const std::unordered_set versions_to_serve( + const std::unordered_set versions_to_serve( servable.servable_version_policy().specific().versions().begin(), servable.servable_version_policy().specific().versions().end()); // Identify specific version to serve (as specified by 'versions_to_serve') // among children that can be interpreted as version numbers and emit as // aspired versions. - bool at_least_one_version_emitted = false; + std::unordered_set aspired_versions; for (auto it = children_by_version.begin(); it != children_by_version.end(); ++it) { const int64 version = it->first; @@ -189,10 +189,19 @@ bool AspireSpecificVersions( } const string& child = it->second; AspireVersion(servable, child, version, versions); - at_least_one_version_emitted = true; + aspired_versions.insert(version); + } + for (const int64 version : versions_to_serve) { + if (aspired_versions.count(version) == 0) { + LOG(WARNING) + << "Version " << version << " of servable " + << servable.servable_name() << ", which was requested to be served " + << "as a 'specific' version in the servable's version policy, was " + << "not found in the file system"; + } } - return at_least_one_version_emitted; + return !aspired_versions.empty(); } // Like PollFileSystemForConfig(), but for a single servable. From 5f5ab368c869c9bdf9f292848cde31627160d6fb Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 4 Aug 2017 07:27:49 -0800 Subject: [PATCH 0330/8554] Merge changes from github. Change: 164264922 --- tensorflow_serving/g3doc/custom_source.md | 11 ++++++----- tensorflow_serving/g3doc/index.md | 2 ++ tensorflow_serving/g3doc/leftnav_files | 1 + tensorflow_serving/g3doc/serving_advanced.md | 4 ++-- tensorflow_serving/g3doc/serving_basic.md | 9 +++++---- tensorflow_serving/g3doc/serving_inception.md | 4 ++-- 6 files changed, 18 insertions(+), 13 deletions(-) diff --git a/tensorflow_serving/g3doc/custom_source.md b/tensorflow_serving/g3doc/custom_source.md index fc8e802cb0c..fcef94d3408 100644 --- a/tensorflow_serving/g3doc/custom_source.md +++ b/tensorflow_serving/g3doc/custom_source.md @@ -23,11 +23,12 @@ loaders directly. Of course, whatever kind of data your source emits (whether it is POSIX paths, Google Cloud Storage paths, or RPC handles), there needs to be accompanying module(s) that are able to load servables based on that. Such modules are called -`SourceAdapters`. Creating a custom one is described in the [Custom -Servable](custom_servable.md) document. TensorFlow Serving comes with one for -instantiating TensorFlow sessions based on paths in file systems that TensorFlow -supports. One can add support for additional file systems to TensorFlow by -extending the `RandomAccessFile` abstraction (`tensorflow/core/public/env.h`). +`SourceAdapters`. Creating a custom one is described in the +[Custom Servable](custom_servable.md) document. TensorFlow Serving +comes with one for instantiating TensorFlow sessions based on paths +in file systems that TensorFlow supports. One can add support for +additional file systems to TensorFlow by extending the `RandomAccessFile` +abstraction (`tensorflow/core/public/env.h`). This document focuses on creating a source that emits paths in a TensorFlow-supported file system. It ends with a walk-through of how to use your diff --git a/tensorflow_serving/g3doc/index.md b/tensorflow_serving/g3doc/index.md index d1869e0072f..18d0e28c42d 100644 --- a/tensorflow_serving/g3doc/index.md +++ b/tensorflow_serving/g3doc/index.md @@ -1,3 +1,5 @@ +# Introduction + TensorFlow Serving is a flexible, high-performance serving system for machine learning models, designed for production environments. TensorFlow Serving makes it easy to deploy new algorithms and experiments, while keeping the same diff --git a/tensorflow_serving/g3doc/leftnav_files b/tensorflow_serving/g3doc/leftnav_files index 146e204d6a1..d7d658c8677 100644 --- a/tensorflow_serving/g3doc/leftnav_files +++ b/tensorflow_serving/g3doc/leftnav_files @@ -1,4 +1,5 @@ ### TensorFlow Serving +index.md architecture_overview.md setup.md serving_basic.md diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 1ddf2dc92c5..bb9c862b02c 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -35,8 +35,8 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisites). Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the [instructions -here](setup.md#optimized-build). +further optimize the build, refer to the +[instructions here](setup.md#optimized-build). ## Train And Export TensorFlow Model diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 337e8de58ee..cbad3ae9047 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -28,8 +28,8 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisites). Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the [instructions -here](setup.md#optimized-build). +further optimize the build, refer to the +[instructions here](setup.md#optimized-build). ## Train And Export TensorFlow Model @@ -152,8 +152,9 @@ $>rm -rf /tmp/mnist_model If you would like to install the `tensorflow` and `tensorflow-serving-api` PIP packages, you can run all Python code (export and client) using a simple -`python` command. To install the PIP package, follow the [instructions -here](setup.md#tensorflow-serving-python-api-pip-package). It's also possible to +`python` command. To install the PIP package, follow the +[instructions here](setup.md#tensorflow-serving-python-api-pip-package). +It's also possible to use Bazel to build the necessary dependencies and run all code without installing those packages. The rest of the codelab will have instructions for both the Bazel and PIP options. diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 81d89660f0a..661a8ce687b 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -36,8 +36,8 @@ $ docker run --name=inception_container -it $USER/tensorflow-serving-devel ### Clone, configure, and build TensorFlow Serving in a container Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the [instructions -here](setup.md#optimized-build). +further optimize the build, refer to the +[instructions here](setup.md#optimized-build). In the running container, we clone, configure and build TensorFlow Serving example code. From d1f5e7aff0f0d0c70e02a9d77c3cf5f1cf558a31 Mon Sep 17 00:00:00 2001 From: Wolff Dobson Date: Fri, 4 Aug 2017 09:34:47 -0700 Subject: [PATCH 0331/8554] More formatting issues (#551) * Add title to pick up TOC * Fix linebreaks in links * Fix some formatting issues; slight reordering --- tensorflow_serving/g3doc/leftnav_files | 1 + tensorflow_serving/g3doc/serving_basic.md | 59 ++++++++++++----------- 2 files changed, 31 insertions(+), 29 deletions(-) diff --git a/tensorflow_serving/g3doc/leftnav_files b/tensorflow_serving/g3doc/leftnav_files index d7d658c8677..272cbf0eba0 100644 --- a/tensorflow_serving/g3doc/leftnav_files +++ b/tensorflow_serving/g3doc/leftnav_files @@ -7,4 +7,5 @@ serving_advanced.md serving_inception.md custom_servable.md custom_source.md +signature_defs.md docker.md \ No newline at end of file diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index cbad3ae9047..026de84ee82 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -112,35 +112,36 @@ You can add meta graph and variables to the builder using As an example for how `predict_signature` is defined, the util takes the following arguments: - * `inputs={'images': tensor_info_x}` specifies the input tensor info. - - * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. - - Note that `tensor_info_x` and `tensor_info_y` have the structure of - `tensorflow::TensorInfo` protocol buffer defined [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto). - To easily build tensor infos, the TensorFlow SavedModel API also provides - [utils.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py), - with [related TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/utils). - - Also, note that `images` and `scores` are tensor alias names. They can be - whatever unique strings you want, and they will become the logical names - of tensor `x` and `y` that you refer to for tensor binding when sending - prediction requests later. - - For instance, if `x` refers to the tensor with name 'long_tensor_name_foo' - and `y` refers to the tensor with name 'generated_tensor_name_bar', - `builder` will store tensor logical name to real name mapping - ('images' -> 'long_tensor_name_foo') and ('scores' -> 'generated_tensor_name_bar'). - This allows the user to refer to these tensors with their logical names - when running inference. - - * `method_name` is the method used for the inference. For Prediction - requests, it should be set to `tensorflow/serving/predict`. For other - method names, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) - and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). - - In addition to the description above, documentation related to signature def - structure and how to set up them up can be found [here](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/signature_defs.md). + * `inputs={'images': tensor_info_x}` specifies the input tensor info. + + * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. + + * `method_name` is the method used for the inference. For Prediction + requests, it should be set to `tensorflow/serving/predict`. For other + method names, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) + and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). + + +Note that `tensor_info_x` and `tensor_info_y` have the structure of +`tensorflow::TensorInfo` protocol buffer defined [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto). +To easily build tensor infos, the TensorFlow SavedModel API also provides +[utils.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py), +with [related TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/utils). + +Also, note that `images` and `scores` are tensor alias names. They can be +whatever unique strings you want, and they will become the logical names +of tensor `x` and `y` that you refer to for tensor binding when sending +prediction requests later. + +For instance, if `x` refers to the tensor with name 'long_tensor_name_foo' and +`y` refers to the tensor with name 'generated_tensor_name_bar', `builder` will +store tensor logical name to real name mapping ('images' -> +'long_tensor_name_foo') and ('scores' -> 'generated_tensor_name_bar'). This +allows the user to refer to these tensors with their logical names when +running inference. + +Note: In addition to the description above, documentation related to signature +def structure and how to set up them up can be found [here](signature_defs.md). Let's run it! From 293d8d82b884d7fc9410e52be375d3cc01c18744 Mon Sep 17 00:00:00 2001 From: Mark Hamilton Date: Fri, 4 Aug 2017 15:45:42 -0400 Subject: [PATCH 0332/8554] update gpu docker file (#541) --- tensorflow_serving/tools/docker/Dockerfile.devel-gpu | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu index 4c80705492c..caf9e35c972 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -1,4 +1,4 @@ -FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 +FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04 RUN apt-get update && apt-get install -y \ build-essential \ @@ -83,8 +83,16 @@ RUN mkdir /usr/lib/x86_64-linux-gnu/include/ && \ ln -s /usr/lib/x86_64-linux-gnu/include/cudnn.h /usr/lib/x86_64-linux-gnu/include/cudnn.h && \ ln -s /usr/include/cudnn.h /usr/local/cuda/include/cudnn.h && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/local/cuda/lib64/libcudnn.so && \ - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/local/cuda/lib64/libcudnn.so.5 + ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.6 /usr/local/cuda/lib64/libcudnn.so.6 +# Fix from https://github.com/tensorflow/serving/issues/327#issuecomment-282207825 +WORKDIR / +RUN git clone https://github.com/NVIDIA/nccl.git && \ + cd nccl/ && \ + make CUDA_HOME=/usr/local/cuda && \ + make install && \ + mkdir -p /usr/local/include/external/nccl_archive/src && \ + ln -s /usr/local/include/nccl.h /usr/local/include/external/nccl_archive/src/nccl.h # Configure Tensorflow to use the GPU WORKDIR /serving/tensorflow From 9a0e768b66061e53ac24004d6a3f63738d7d2f4b Mon Sep 17 00:00:00 2001 From: Wolff Dobson Date: Fri, 4 Aug 2017 16:24:48 -0700 Subject: [PATCH 0333/8554] Fix rendering of shell commands in numbered list. (#553) * Add title to pick up TOC * Fix linebreaks in links * Fix some formatting issues; slight reordering * Fix rendering of numbered lists with shell commands --- tensorflow_serving/g3doc/setup.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index b59a61b5eec..cffd48103ed 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -17,17 +17,17 @@ following steps: Let's say you downloaded bazel-0.4.5-installer-linux-x86_64.sh. You would execute: - ```shell + 
     cd ~/Downloads
     chmod +x bazel-0.4.5-installer-linux-x86_64.sh
     ./bazel-0.4.5-installer-linux-x86_64.sh --user
-    ```
+
2. Set up your environment. Put this in your ~/.bashrc. - ```shell + 
     export PATH="$PATH:$HOME/bin"
-    ```
+
### gRPC @@ -91,17 +91,17 @@ sudo apt-get remove tensorflow-model-server 1. Add TensorFlow Serving distribution URI as a package source (one time setup) - ```shell + 
     echo "deb [arch=amd64] http://storage.googleapis.com/tensorflow-serving-apt stable tensorflow-model-server tensorflow-model-server-universal" | sudo tee /etc/apt/sources.list.d/tensorflow-serving.list
 
     curl https://storage.googleapis.com/tensorflow-serving-apt/tensorflow-serving.release.pub.gpg | sudo apt-key add -
-    ```
+
2. Install and update TensorFlow ModelServer - ```shell + 
     sudo apt-get update && sudo apt-get install tensorflow-model-server
-    ```
+
Once installed, the binary can be invoked using the command `tensorflow_model_server`. From 7690c8af4d5a1f9c021aaae774b39a7d13db362f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 4 Aug 2017 07:54:05 -0800 Subject: [PATCH 0334/8554] Change link for SignatureDef utils from utils.py to signature_def_utils.py. utils.py contains generic SavedModel utilities, whereas signature_def_utils.py contains the SignatureDef specific utilities the doc mentions. Change: 164267409 --- tensorflow_serving/g3doc/signature_defs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/signature_defs.md b/tensorflow_serving/g3doc/signature_defs.md index 69d2cb1c42d..a68f192f7de 100644 --- a/tensorflow_serving/g3doc/signature_defs.md +++ b/tensorflow_serving/g3doc/signature_defs.md @@ -54,7 +54,7 @@ constants. Specifically: C++](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/signature_constants.h). In addition, SavedModel provides a -[util](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py) +[util](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_def_utils.py) to help build a signature-def. ## Sample structures From d8db35f02e4fc620b5c7f371aaa64246f417475c Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Fri, 4 Aug 2017 12:06:12 -0800 Subject: [PATCH 0335/8554] Internal. Public no-op. Change: 164299280 --- tensorflow_serving/g3doc/METADATA | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/METADATA b/tensorflow_serving/g3doc/METADATA index 230bbb59486..819e6aaf439 100644 --- a/tensorflow_serving/g3doc/METADATA +++ b/tensorflow_serving/g3doc/METADATA @@ -1,6 +1,6 @@ name: "TensorFlow Serving" g3doc: { include: "/learning/serving/g3doc/METADATA" - sitemap_file: "/learning/serving/g3doc/users/sitemap.md" + sitemap_file: "/learning/serving/g3doc/sitemap.md" } From 913e8fb1f3862550fe453f1264830fefb8981b21 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 7 Aug 2017 12:45:43 -0800 Subject: [PATCH 0336/8554] Remove mkl for optimized build options. Change: 164505061 --- tensorflow_serving/g3doc/setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index b59a61b5eec..e9b0bde1a7a 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -182,7 +182,7 @@ in the documentation, you can add the flags `-c opt --copt=-msse4.1 subset of these flags). For example: ```shell -bazel build -c opt --config=mkl --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-O3 tensorflow_serving/... +bazel build -c opt --copt=-msse4.1 --copt=-msse4.2 --copt=-mavx --copt=-mavx2 --copt=-mfma --copt=-O3 tensorflow_serving/... ``` Note: These instruction sets are not available on all machines, especially with From 1b4351e76f26f5189982385db326b4490e48e3c5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 10 Aug 2017 00:20:49 -0800 Subject: [PATCH 0337/8554] Removing deprecated VersionPolicy enum Change: 164826520 --- .../config/model_server_config.proto | 6 +----- tensorflow_serving/model_servers/main.cc | 2 -- .../model_servers/server_core.cc | 19 +------------------ .../file_system_storage_path_source.proto | 12 ------------ 4 files changed, 2 insertions(+), 37 deletions(-) diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index e69928fcdf1..864afdc4466 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -38,11 +38,7 @@ message ModelConfig { // (This cannot be changed once a model is in serving.) string model_platform = 4; - // DEPRECATED: This field is deprecated. For now it's still obeyed as long as - // 'model_version_policy' is not set. If 'model_version_policy' is set, then - // the value of this field is ignored. - FileSystemStoragePathSourceConfig.VersionPolicy version_policy = 5 - [deprecated = true]; + reserved 5; // Version policy for the model indicating how many versions of the model to // be served at the same time. diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index f097d24423a..6cad8011c24 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -87,8 +87,6 @@ using tensorflow::serving::AvailabilityPreservingPolicy; using tensorflow::serving::BatchingParameters; using tensorflow::serving::EventBus; using tensorflow::serving::FileSystemStoragePathSourceConfig; -using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy; -using tensorflow::serving::FileSystemStoragePathSourceConfig_VersionPolicy_Name; using tensorflow::serving::GetModelMetadataImpl; using tensorflow::serving::ModelServerConfig; using tensorflow::serving::ServableState; diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 762f69394de..f74ea397baa 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -423,24 +423,7 @@ FileSystemStoragePathSourceConfig ServerCore::CreateStoragePathSourceConfig( source_config.add_servables(); servable->set_servable_name(model.name()); servable->set_base_path(model.base_path()); - // TODO(akhorlin): remove this logic once the corresponding deprecated - // field is removed (b/62834753). - if (!model.has_model_version_policy()) { - switch (model.version_policy()) { - case FileSystemStoragePathSourceConfig::LATEST_VERSION: - servable->mutable_servable_version_policy()->mutable_latest(); - break; - case FileSystemStoragePathSourceConfig::ALL_VERSIONS: - servable->mutable_servable_version_policy()->mutable_all(); - break; - default: - LOG(FATAL) << "Unknown version policy: " // Crash ok. - << model.version_policy(); - } - } else { - *servable->mutable_servable_version_policy() = - model.model_version_policy(); - } + *servable->mutable_servable_version_policy() = model.model_version_policy(); } return source_config; } diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto index 8b11ce8237a..8fa7be5e55a 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.proto @@ -36,18 +36,6 @@ message FileSystemStoragePathSourceConfig { } } - // DEPRECATED: Please use VersionPolicy message definition instead. The - // enum will be removed once noone depends on it any longer. - // The policy to define how many versions of the servable should be - // served at the same time. - enum VersionPolicy { - // Only serve the latest version that exists in the base path. - // This is the default behavior. - LATEST_VERSION = 0; - // Serves all the versions that exist in the base path. - ALL_VERSIONS = 1; - } - // A servable name and base path to look for versions of the servable. message ServableToMonitor { // The servable name to supply in aspired-versions callback calls. Child From a7de0286fc2ae2559b47dabf26c055ca7a46d9bc Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 11 Aug 2017 01:08:15 -0800 Subject: [PATCH 0338/8554] Adding a warning when a single version maps to multiple different directories. Change: 164956623 --- .../file_system_storage_path_source.cc | 6 +++ .../file_system_storage_path_source_test.cc | 38 +++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc index 95dc15aef4d..582503a3ec9 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source.cc @@ -132,6 +132,12 @@ IndexChildrenByVersion(const std::vector& children) { continue; } + if (children_by_version.count(version_number) > 0) { + LOG(WARNING) << "Duplicate version directories detected. Version " + << version_number << " will be loaded from " << children[i] + << ", " << children_by_version[version_number] + << " will be ignored."; + } children_by_version[version_number] = children[i]; } return children_by_version; diff --git a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc index 2e52fb3436d..cc18864a67b 100644 --- a/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc +++ b/tensorflow_serving/sources/storage_path/file_system_storage_path_source_test.cc @@ -33,6 +33,7 @@ limitations under the License. #include "tensorflow_serving/sources/storage_path/file_system_storage_path_source.pb.h" #include "tensorflow_serving/test_util/test_util.h" +using ::testing::AnyOf; using ::testing::ElementsAre; using ::testing::Eq; using ::testing::IsEmpty; @@ -689,6 +690,43 @@ TEST(FileSystemStoragePathSourceTest, ServableVersionDirRenamed) { .PollFileSystemAndInvokeCallback()); } +TEST(FileSystemStoragePathSourceTest, DuplicateVersions) { + const string base_path = io::JoinPath(testing::TmpDir(), "DuplicateVersions"); + TF_ASSERT_OK(Env::Default()->CreateDir(base_path)); + TF_ASSERT_OK(Env::Default()->CreateDir( + io::JoinPath(base_path, "non_numerical_child"))); + for (const string& version : {"0001", "001", "00001"}) { + TF_ASSERT_OK(Env::Default()->CreateDir(io::JoinPath(base_path, version))); + } + + auto config = test_util::CreateProto( + strings::Printf("servable_name: 'test_servable_name' " + "base_path: '%s' " + // Disable the polling thread. + "file_system_poll_wait_seconds: -1 ", + base_path.c_str())); + std::unique_ptr source; + TF_ASSERT_OK(FileSystemStoragePathSource::Create(config, &source)); + std::unique_ptr target( + new StrictMock); + ConnectSourceToTarget(source.get(), target.get()); + + EXPECT_CALL( + *target, + SetAspiredVersions( + Eq("test_servable_name"), + AnyOf( + ElementsAre(ServableData( + {"test_servable_name", 1}, io::JoinPath(base_path, "001"))), + ElementsAre(ServableData( + {"test_servable_name", 1}, io::JoinPath(base_path, "0001"))), + ElementsAre(ServableData( + {"test_servable_name", 1}, + io::JoinPath(base_path, "00001")))))); + TF_ASSERT_OK(internal::FileSystemStoragePathSourceTestAccess(source.get()) + .PollFileSystemAndInvokeCallback()); +} + } // namespace } // namespace serving } // namespace tensorflow From 1a88c25b64ea8dd777347bded205c4d1a102f0b2 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 14 Aug 2017 16:40:27 -0700 Subject: [PATCH 0339/8554] Remove unnecessary dependency servable_handle.h (#563) https://github.com/tensorflow/serving/issues/561 --- tensorflow_serving/core/servable_handle.h | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/core/servable_handle.h b/tensorflow_serving/core/servable_handle.h index 1067f6ac53e..8f4d5b2c19a 100644 --- a/tensorflow_serving/core/servable_handle.h +++ b/tensorflow_serving/core/servable_handle.h @@ -22,7 +22,6 @@ limitations under the License. #include #include "tensorflow_serving/core/loader.h" -#include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/util/any_ptr.h" namespace tensorflow { From 89c408d68ed59a24c114986dc51b4655e9e33a70 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 11 Aug 2017 10:38:00 -0800 Subject: [PATCH 0340/8554] Add test for checking raw classification scores. Change: 165003878 --- .../servables/tensorflow/classifier_test.cc | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc index 582ea12dd75..173543ab677 100644 --- a/tensorflow_serving/servables/tensorflow/classifier_test.cc +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -493,6 +493,31 @@ TEST_P(ClassifierTest, ScoresOnly) { " } ")); } +TEST_P(ClassifierTest, ZeroScoresArePresent) { + tensorflow::serving::Signatures signatures; + auto signature = signatures.mutable_default_signature() + ->mutable_classification_signature(); + signature->mutable_input()->set_tensor_name(kInputTensor); + // No classes Tensor. + signature->mutable_scores()->set_tensor_name(kScoreTensor); + TF_ASSERT_OK(tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); + TF_ASSERT_OK(Create()); + auto* examples = + request_.mutable_input()->mutable_example_list()->mutable_examples(); + *examples->Add() = example({{"minus", -1}, {"zero", 0}, {"one", 1}}); + const std::vector expected_outputs = {-1, 0, 1}; + + TF_ASSERT_OK(classifier_->Classify(request_, &result_)); + // Parse the protos and compare the results with expected scores. + ASSERT_EQ(result_.classifications_size(), 1); + auto& classification = result_.classifications(0); + ASSERT_EQ(classification.classes_size(), 3); + + for (int i = 0; i < 3; ++i) { + EXPECT_NEAR(classification.classes(i).score(), expected_outputs[i], 1e-7); + } +} + TEST_P(ClassifierTest, ValidNamedSignature) { TF_ASSERT_OK(Create()); request_.mutable_model_spec()->set_signature_name(kOutputPlusOneSignature); From 5f1ddb5b609df6888e95a54ca6ddf52bd4c50499 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 11 Aug 2017 11:17:59 -0800 Subject: [PATCH 0341/8554] Add a parameter that accounts for additional, transient memory used during servable load that is released when the loading phase is complete. This parameter is a bit of a hack, among other things because it doesn't permit per-model differentiation. The long-term plan is to move resource estimation to an earlier stage than tensorflow-serving and store the estimates in the SavedModels (we'll have two estimates there: one for the during-load memory usage and one for the post-load usage). At that time we can remove this parameter. Change: 165009205 --- tensorflow_serving/core/BUILD | 1 + tensorflow_serving/core/loader.h | 5 +- tensorflow_serving/core/simple_loader.h | 92 ++++++++++++++++--- tensorflow_serving/core/simple_loader_test.cc | 58 +++++++++++- tensorflow_serving/servables/tensorflow/BUILD | 2 + .../tensorflow/saved_model_bundle_factory.h | 2 + .../saved_model_bundle_source_adapter.cc | 29 +++++- .../saved_model_bundle_source_adapter_test.cc | 32 ++++++- .../tensorflow/session_bundle_config.proto | 10 ++ 9 files changed, 214 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index e1ef7755b8c..13bd550cbf9 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -87,6 +87,7 @@ cc_library( deps = [ ":loader", ":source_adapter", + "//tensorflow_serving/resources:resource_util", "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/util:any_ptr", "//tensorflow_serving/util:optional", diff --git a/tensorflow_serving/core/loader.h b/tensorflow_serving/core/loader.h index 88d9fb631bc..b76c16cee34 100644 --- a/tensorflow_serving/core/loader.h +++ b/tensorflow_serving/core/loader.h @@ -71,7 +71,10 @@ class Loader { /// the estimate must specify the instance to which each resource is /// bound. /// 4. The estimate must be monotonically non-increasing, i.e. it cannot - /// increase over time. + /// increase over time. Reasons to have it potentially decrease over time + // include: (a) replace conservative estimate with actual measurement + // once loaded in memory; (b) load process consumes extra transient + // memory that is not used in steady-state after the load completes. /// /// @return an estimate of the resources the servable will consume once /// loaded. If the servable has already been loaded, returns an estimate of diff --git a/tensorflow_serving/core/simple_loader.h b/tensorflow_serving/core/simple_loader.h index 62452a36043..63c041681ff 100644 --- a/tensorflow_serving/core/simple_loader.h +++ b/tensorflow_serving/core/simple_loader.h @@ -26,6 +26,7 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/source_adapter.h" +#include "tensorflow_serving/resources/resource_util.h" #include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/util/any_ptr.h" #include "tensorflow_serving/util/optional.h" @@ -62,6 +63,9 @@ namespace serving { // }; // std::unique_ptr loader(new SimpleLoader( // servable_creator, resource_estimator)); +// +// This class is not thread-safe. Synchronization is assumed to be done by the +// caller. template class SimpleLoader : public Loader { public: @@ -80,7 +84,19 @@ class SimpleLoader : public Loader { // and hence the serving system cannot enforce resource safety. static ResourceEstimator EstimateNoResources(); + // Constructor that takes a single resource estimator, to use for estimating + // the resources needed during load as well as post-load. SimpleLoader(Creator creator, ResourceEstimator resource_estimator); + + // Constructor that takes two resource estimators: one to use for estimating + // the resources needed during load, as well as a second one that gives a + // different estimate after loading has finished. See the documentation on + // Loader::EstimateResources() for (a) potential reasons the estimate might + // decrease, and (b) correctness constraints on how the estimate is allowed to + // change over time. + SimpleLoader(Creator creator, ResourceEstimator resource_estimator, + ResourceEstimator post_load_resource_estimator); + ~SimpleLoader() override = default; Status EstimateResources(ResourceAllocation* estimate) const override; @@ -94,11 +110,20 @@ class SimpleLoader : public Loader { private: Creator creator_; + // A function that estimates the resources needed to load the servable. ResourceEstimator resource_estimator_; - // The memoized estimated resource requirement of the session bundle servable. + // An optional function that estimates the resources needed for the servable + // after it has been loaded. (If omitted, 'resource_estimator_' should be used + // for all estimates, i.e. before, during and after load.) + optional post_load_resource_estimator_; + + // The memoized estimated resource requirement of the servable. mutable optional memoized_resource_estimate_; + std::unique_ptr resource_util_; + Resource ram_resource_; + std::unique_ptr servable_; TF_DISALLOW_COPY_AND_ASSIGN(SimpleLoader); @@ -180,7 +205,23 @@ SimpleLoader::EstimateNoResources() { template SimpleLoader::SimpleLoader(Creator creator, ResourceEstimator resource_estimator) - : creator_(creator), resource_estimator_(resource_estimator) {} + : creator_(creator), resource_estimator_(resource_estimator) { + ResourceUtil::Options resource_util_options; + resource_util_options.devices = {{device_types::kMain, 1}}; + resource_util_ = + std::unique_ptr(new ResourceUtil(resource_util_options)); + + ram_resource_ = resource_util_->CreateBoundResource( + device_types::kMain, resource_kinds::kRamBytes); +} + +template +SimpleLoader::SimpleLoader( + Creator creator, ResourceEstimator resource_estimator, + ResourceEstimator post_load_resource_estimator) + : SimpleLoader(creator, resource_estimator) { + post_load_resource_estimator_ = post_load_resource_estimator; +} template Status SimpleLoader::EstimateResources( @@ -198,8 +239,36 @@ Status SimpleLoader::EstimateResources( template Status SimpleLoader::Load() { - const Status status = creator_(&servable_); - return status; + TF_RETURN_IF_ERROR(creator_(&servable_)); + + if (post_load_resource_estimator_) { + // Save the during-load estimate (may be able to use the memoized value). + ResourceAllocation during_load_resource_estimate; + TF_RETURN_IF_ERROR(EstimateResources(&during_load_resource_estimate)); + + // Obtain the post-load estimate, and store it as the memoized value. + ResourceAllocation post_load_resource_estimate; + TF_RETURN_IF_ERROR( + (*post_load_resource_estimator_)(&post_load_resource_estimate)); + memoized_resource_estimate_ = post_load_resource_estimate; + + // Release any transient memory used only during load to the OS. + const uint64 during_load_ram_estimate = resource_util_->GetQuantity( + ram_resource_, during_load_resource_estimate); + const uint64 post_load_ram_estimate = + resource_util_->GetQuantity(ram_resource_, post_load_resource_estimate); + if (post_load_ram_estimate < during_load_ram_estimate) { + const uint64 transient_ram_estimate = + during_load_ram_estimate - post_load_ram_estimate; + LOG(INFO) << "Calling MallocExtension_ReleaseToSystem() after servable " + "load with " + << transient_ram_estimate; + ::tensorflow::port::MallocExtension_ReleaseToSystem( + transient_ram_estimate); + } + } + + return Status::OK(); } template @@ -219,14 +288,13 @@ void SimpleLoader::Unload() { // If we have a main-memory footprint estimate, release that amount of memory // to the OS. - for (const ResourceAllocation::Entry& entry : - resource_estimate.resource_quantities()) { - if (entry.resource().device() == device_types::kMain && - entry.resource().kind() == resource_kinds::kRamBytes) { - LOG(INFO) << "Calling MallocExtension_ReleaseToSystem() with " - << entry.quantity(); - ::tensorflow::port::MallocExtension_ReleaseToSystem(entry.quantity()); - } + const uint64 memory_estimate = + resource_util_->GetQuantity(ram_resource_, resource_estimate); + if (memory_estimate > 0) { + LOG(INFO) << "Calling MallocExtension_ReleaseToSystem() after servable " + "unload with " + << memory_estimate; + ::tensorflow::port::MallocExtension_ReleaseToSystem(memory_estimate); } } diff --git a/tensorflow_serving/core/simple_loader_test.cc b/tensorflow_serving/core/simple_loader_test.cc index 186b98cf922..da3d5b5632e 100644 --- a/tensorflow_serving/core/simple_loader_test.cc +++ b/tensorflow_serving/core/simple_loader_test.cc @@ -96,13 +96,69 @@ TEST(SimpleLoaderTest, ResourceEstimation) { *estimate = want; return Status::OK(); })); - for (int i = 0; i < 2; ++i) { + + { + ResourceAllocation got; + TF_ASSERT_OK(loader->EstimateResources(&got)); + EXPECT_THAT(got, EqualsProto(want)); + } + + // The estimate should remain the same after load. + TF_ASSERT_OK(loader->Load()); + { ResourceAllocation got; TF_ASSERT_OK(loader->EstimateResources(&got)); EXPECT_THAT(got, EqualsProto(want)); } } +TEST(SimpleLoaderTest, ResourceEstimationWithPostLoadRelease) { + const auto pre_load_resources = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'processing' " + " } " + " quantity: 42 " + "} "); + const auto post_load_resources = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'processing' " + " } " + " quantity: 17 " + "} "); + std::unique_ptr loader(new SimpleLoader( + [](std::unique_ptr* servable) { + servable->reset(new int); + return Status::OK(); + }, + [&pre_load_resources](ResourceAllocation* estimate) { + *estimate = pre_load_resources; + return Status::OK(); + }, + [&post_load_resources](ResourceAllocation* estimate) { + *estimate = post_load_resources; + return Status::OK(); + })); + + // Run it twice, to exercise memoization. + for (int i = 0; i < 2; ++i) { + ResourceAllocation got; + TF_ASSERT_OK(loader->EstimateResources(&got)); + EXPECT_THAT(got, EqualsProto(pre_load_resources)); + } + + // The estimate should switch to the post-load one after load. + TF_ASSERT_OK(loader->Load()); + { + ResourceAllocation got; + TF_ASSERT_OK(loader->EstimateResources(&got)); + EXPECT_THAT(got, EqualsProto(post_load_resources)); + } +} + // Verify that the error returned by the Creator is propagates back through // Load. TEST(SimpleLoaderTest, LoadError) { diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 93f3366308c..9648d188ec5 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -264,6 +264,8 @@ cc_library( "//tensorflow_serving/core:simple_loader", "//tensorflow_serving/core:source_adapter", "//tensorflow_serving/core:storage_path", + "//tensorflow_serving/resources:resource_util", + "//tensorflow_serving/resources:resource_values", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/cc/saved_model:loader", diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h index 9ab961a2e9f..df3c3419b20 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.h @@ -73,6 +73,8 @@ class SavedModelBundleFactory { Status EstimateResourceRequirement(const string& path, ResourceAllocation* estimate) const; + const SessionBundleConfig& config() const { return config_; } + private: using Batcher = SharedBatchScheduler; diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc index 8b4bbb2fff9..505219cdc58 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc @@ -21,6 +21,8 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/core/simple_loader.h" +#include "tensorflow_serving/resources/resource_util.h" +#include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/util/optional.h" @@ -52,10 +54,33 @@ Status SavedModelBundleSourceAdapter::Convert(const StoragePath& path, }; auto resource_estimator = [bundle_factory, path](ResourceAllocation* estimate) { + TF_RETURN_IF_ERROR( + bundle_factory->EstimateResourceRequirement(path, estimate)); + + // Add experimental_transient_ram_bytes_during_load. + // TODO(b/38376838): Remove once resource estimates are moved inside + // SavedModel. + ResourceUtil::Options resource_util_options; + resource_util_options.devices = {{device_types::kMain, 1}}; + std::unique_ptr resource_util = + std::unique_ptr(new ResourceUtil(resource_util_options)); + const Resource ram_resource = resource_util->CreateBoundResource( + device_types::kMain, resource_kinds::kRamBytes); + resource_util->SetQuantity( + ram_resource, + resource_util->GetQuantity(ram_resource, *estimate) + + bundle_factory->config() + .experimental_transient_ram_bytes_during_load(), + estimate); + + return Status::OK(); + }; + auto post_load_resource_estimator = [bundle_factory, + path](ResourceAllocation* estimate) { return bundle_factory->EstimateResourceRequirement(path, estimate); }; - loader->reset( - new SimpleLoader(servable_creator, resource_estimator)); + loader->reset(new SimpleLoader( + servable_creator, resource_estimator, post_load_resource_estimator)); return Status::OK(); } diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc index 8dc6087d7f6..261b78e4eda 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc @@ -27,6 +27,8 @@ limitations under the License. #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/servable_data.h" +#include "tensorflow_serving/resources/resource_util.h" +#include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/resources/resources.pb.h" #include "tensorflow_serving/servables/tensorflow/bundle_factory_test_util.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" @@ -41,6 +43,16 @@ using test_util::EqualsProto; class SavedModelBundleSourceAdapterTest : public ::testing::Test { protected: + SavedModelBundleSourceAdapterTest() { + ResourceUtil::Options resource_util_options; + resource_util_options.devices = {{device_types::kMain, 1}}; + resource_util_ = + std::unique_ptr(new ResourceUtil(resource_util_options)); + + ram_resource_ = resource_util_->CreateBoundResource( + device_types::kMain, resource_kinds::kRamBytes); + } + void TestSavedModelBundleSourceAdapter( const SessionBundleSourceAdapterConfig& config, const string& export_dir) const { @@ -69,15 +81,33 @@ class SavedModelBundleSourceAdapterTest : public ::testing::Test { TF_ASSERT_OK(loader->Load()); + // We should get a new (lower) resource estimate post-load. + ResourceAllocation expected_post_load_resource_estimate = + first_resource_estimate; + resource_util_->SetQuantity( + ram_resource_, + resource_util_->GetQuantity(ram_resource_, first_resource_estimate) - + config.config().experimental_transient_ram_bytes_during_load(), + &expected_post_load_resource_estimate); + ResourceAllocation actual_post_load_resource_estimate; + TF_ASSERT_OK( + loader->EstimateResources(&actual_post_load_resource_estimate)); + EXPECT_THAT(actual_post_load_resource_estimate, + EqualsProto(expected_post_load_resource_estimate)); + const SavedModelBundle* bundle = loader->servable().get(); test_util::TestSingleRequest(bundle->session.get()); loader->Unload(); } + + std::unique_ptr resource_util_; + Resource ram_resource_; }; TEST_F(SavedModelBundleSourceAdapterTest, Basic) { - const SessionBundleSourceAdapterConfig config; + SessionBundleSourceAdapterConfig config; + config.mutable_config()->set_experimental_transient_ram_bytes_during_load(42); TestSavedModelBundleSourceAdapter(config, test_util::GetTestSavedModelPath()); } diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto index 602819c88e0..69a3934f2b3 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto +++ b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto @@ -41,6 +41,16 @@ message SessionBundleConfig { // part of `session_config.session_inter_op_thread_pool`. google.protobuf.Int32Value session_run_load_threadpool_index = 4; + // EXPERIMENTAL. THIS FIELD MAY CHANGE OR GO AWAY. USE WITH CAUTION. + // + // Transient memory used while loading a model, which is released once the + // loading phase has completed. (This is on top of the memory used in steady- + // state while the model is in memory after it has finished loading.) + // + // TODO(b/38376838): This is a temporary hack, and it applies to all models. + // Remove it once resource estimates are moved inside SavedModel. + uint64 experimental_transient_ram_bytes_during_load = 5; + // EXPERIMENTAL. THIS FIELD MAY CHANGE OR GO AWAY. USE WITH CAUTION. // // Input tensors to append to every Session::Run() call. From f8cc9fd0d36ab6830340875d26aa7870369afe9e Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 17 Aug 2017 13:49:56 -0800 Subject: [PATCH 0342/8554] Enable relative paths in ModelConfig for ServerCore, setting the absolute prefix as a new ServerCore::Options::model_config_list_root_dir field. By default, the model_config_list_root_dir is absent. This preserves the old behavior for existing ModelConfig's that are well-formed. In this case, all paths must be absolute: this condition has always been expected, but now it will be explicitly validated. If the model_config_list_root_dir is present, it must be an absolute path. In this case, all paths in an input ModelConfig must be relative paths. These relative paths will be prepended with model_config_list_root_dir when the ModelConfig is reloaded, and the result must be an absolute path. Change: 165632548 --- tensorflow_serving/model_servers/BUILD | 5 + .../model_servers/server_core.cc | 68 ++++++++++++- .../model_servers/server_core.h | 3 + .../model_servers/server_core_test.cc | 98 +++++++++++++++++++ .../model_servers/test_util/BUILD | 1 + .../test_util/server_core_test_util.cc | 13 ++- .../test_util/server_core_test_util.h | 15 ++- 7 files changed, 197 insertions(+), 6 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 5ee703c34ca..d29d96789d0 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -54,6 +54,7 @@ cc_library( deps = [ ":model_platform_types", "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/config:logging_config_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/config:platform_config_proto", "//tensorflow_serving/core:aspired_versions_manager", @@ -71,6 +72,7 @@ cc_library( "//tensorflow_serving/sources/storage_path:file_system_storage_path_source", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", "//tensorflow_serving/util:event_bus", + "//tensorflow_serving/util:optional", "//tensorflow_serving/util:unique_ptr_with_deps", "@org_tensorflow//tensorflow/core:lib", "@protobuf//:cc_wkt_protos", @@ -82,6 +84,7 @@ cc_test( size = "medium", srcs = ["server_core_test.cc"], deps = [ + ":model_platform_types", ":server_core", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/apis:predict_proto", @@ -96,6 +99,8 @@ cc_test( "//tensorflow_serving/model_servers/test_util:storage_path_error_injecting_source_adapter", "//tensorflow_serving/model_servers/test_util:storage_path_error_injecting_source_adapter_proto", "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", "@protobuf//:cc_wkt_protos", ], diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index f74ea397baa..db07a923d28 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -19,6 +19,7 @@ limitations under the License. #include "google/protobuf/any.pb.h" #include "google/protobuf/wrappers.pb.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow_serving/core/load_servables_fast.h" #include "tensorflow_serving/model_servers/model_platform_types.h" @@ -69,7 +70,9 @@ Status GetPlatform(const ModelConfig& model_config, string* platform) { // Returns an error if 'config_list' is invalid in some way, e.g. a model name // appearing multiple times. -Status ValidateModelConfigList(const ModelConfigList& config_list) { +Status ValidateModelConfigList(const ModelConfigList& config_list, + const ServerCore::Options& options) { + // Unique model-names. std::set model_names; for (const ModelConfig& config : config_list.config()) { if (model_names.find(config.name()) != model_names.end()) { @@ -79,6 +82,35 @@ Status ValidateModelConfigList(const ModelConfigList& config_list) { } model_names.insert(config.name()); } + + // Base-paths are either all relative, or all absolute. + using ::tensorflow::io::IsAbsolutePath; + using ::tensorflow::io::JoinPath; + if (options.model_config_list_root_dir) { + // All paths must be relative. + if (!IsAbsolutePath(*options.model_config_list_root_dir)) { + return errors::InvalidArgument(strings::StrCat( + "Expected non-empty absolute path; got model_config_list_root_dir=", + *options.model_config_list_root_dir)); + } + for (const ModelConfig& config : config_list.config()) { + if (IsAbsolutePath(config.base_path())) { + return errors::InvalidArgument(strings::StrCat( + "Expected model ", config.name(), + " to have a relative path; got base_path()=", config.base_path())); + } + } + } else { + // All paths must be absolute. + for (const ModelConfig& config : config_list.config()) { + if (!IsAbsolutePath(config.base_path())) { + return errors::InvalidArgument(strings::StrCat( + "Expected model ", config.name(), + " to have an absolute path; got base_path()=", config.base_path())); + } + } + } + return Status::OK(); } @@ -151,6 +183,32 @@ std::set NewModelNamesInSourceConfig( return new_models; } +// Updates the base_path fields in each ModelConfig, prepending an +// absolute model_config_list_root_dir. +// It is assumed that initially, all the base_path fields are relative. +Status UpdateModelConfigListRelativePaths( + const string& model_config_list_root_dir, ModelConfigList* config_list) { + using ::tensorflow::io::IsAbsolutePath; + using ::tensorflow::io::JoinPath; + std::vector updated_paths; + updated_paths.reserve(config_list->config_size()); + for (const ModelConfig& config : config_list->config()) { + updated_paths.emplace_back( + JoinPath(model_config_list_root_dir, config.base_path())); + if (!IsAbsolutePath(updated_paths.back())) { + return errors::InvalidArgument(strings::StrCat( + "Expected model ", config.name(), + " with updated base_path = JoinPath(", model_config_list_root_dir, + ", ", config.base_path(), ") to have an absolute path; got ", + updated_paths.back())); + } + } + for (int ii = 0; ii < updated_paths.size(); ++ii) { + config_list->mutable_config(ii)->set_base_path(updated_paths[ii]); + } + return Status::OK(); +} + } // namespace // ************************************************************************ @@ -363,7 +421,8 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { return Status::OK(); } if (new_config.config_case() == ModelServerConfig::kModelConfigList) { - TF_RETURN_IF_ERROR(ValidateModelConfigList(new_config.model_config_list())); + TF_RETURN_IF_ERROR( + ValidateModelConfigList(new_config.model_config_list(), options_)); } if (new_config.config_case() == ModelServerConfig::kModelConfigList && config_.config_case() == ModelServerConfig::kModelConfigList) { @@ -375,6 +434,11 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { LOG(INFO) << "Adding/updating models."; switch (config_.config_case()) { case ModelServerConfig::kModelConfigList: { + if (options_.model_config_list_root_dir) { + TF_RETURN_IF_ERROR(UpdateModelConfigListRelativePaths( + *options_.model_config_list_root_dir, + config_.mutable_model_config_list())); + } TF_RETURN_IF_ERROR(AddModelsViaModelConfigList()); break; } diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 341c512d9e6..53d941fc078 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -87,6 +87,9 @@ class ServerCore : public Manager { struct Options { // ModelServer configuration. ModelServerConfig model_server_config; + // Relative (non-absolute) base-paths in model_server_config will + // be prepended with model_config_list_root_dir. + optional model_config_list_root_dir; // The AspiredVersionPolicy to use for the manager. Must be non-null. std::unique_ptr aspired_version_policy; diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 5583287d3dd..18312ec59a9 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -16,8 +16,10 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" #include "google/protobuf/any.pb.h" +#include "tensorflow/core/lib/core/error_codes.proto.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/apis/predict.pb.h" #include "tensorflow_serving/core/servable_handle.h" @@ -179,6 +181,98 @@ TEST_P(ServerCoreTest, ReloadConfigChangeModelBasePath) { available_servables.at(0).version != test_util::kTestModelVersion); } +class RelativePathsServerCoreTest : public ServerCoreTest { + protected: + // Creates a ModelServerConfig instance where the directory name has + // been stripped off the ModelConfig::base_path. Instead that prefix + // can be supplied via the Options::model_config_list_root_dir. + ModelServerConfig GetTestModelServerConfigWithRelativePath( + string* optional_config_list_root_dir = nullptr) { + using ::tensorflow::io::Basename; + using ::tensorflow::io::Dirname; + using ::tensorflow::io::IsAbsolutePath; + + ModelServerConfig result = GetTestModelServerConfigForFakePlatform(); + const string model_name = result.model_config_list().config(0).name(); + + ModelConfig& relative = + *result.mutable_model_config_list()->mutable_config(0); + relative.set_name(strings::StrCat(model_name, "_relative")); + CHECK(IsAbsolutePath(relative.base_path())) + << "relative.base_path()=" << relative.base_path() + << " must start as an absolute path for this unit-test to work."; + const string dirname = Dirname(relative.base_path()).ToString(); + const string basename = Basename(relative.base_path()).ToString(); + CHECK(!dirname.empty()); + CHECK(!basename.empty()); + relative.set_base_path(basename); + + if (optional_config_list_root_dir) { + *optional_config_list_root_dir = dirname; + } + + return result; + } +}; + +TEST_P(RelativePathsServerCoreTest, AbsolutePathSucceeds) { + std::unique_ptr server_core; + ModelServerConfig absolute = GetTestModelServerConfigForFakePlatform(); + TF_ASSERT_OK(CreateServerCore(absolute, &server_core)); +} + +TEST_P(RelativePathsServerCoreTest, RelativePathFails) { + std::unique_ptr server_core; + ModelServerConfig relative = GetTestModelServerConfigWithRelativePath(); + EXPECT_EQ(error::INVALID_ARGUMENT, + CreateServerCore(relative, &server_core).code()); +} + +TEST_P(RelativePathsServerCoreTest, AbsolutePathWithOptionsFails) { + std::unique_ptr server_core; + ModelServerConfig absolute = GetTestModelServerConfigForFakePlatform(); + ServerCore::Options options = GetDefaultOptions(); + { + string model_config_list_root_dir; + ModelServerConfig relative = + GetTestModelServerConfigWithRelativePath(&model_config_list_root_dir); + options.model_config_list_root_dir = std::move(model_config_list_root_dir); + } + EXPECT_EQ( + error::INVALID_ARGUMENT, + CreateServerCore(absolute, std::move(options), &server_core).code()); +} + +TEST_P(RelativePathsServerCoreTest, AbsolutePathWithEmptyPathFails) { + std::unique_ptr server_core; + ModelServerConfig absolute = GetTestModelServerConfigForFakePlatform(); + ServerCore::Options options = GetDefaultOptions(); + options.model_config_list_root_dir = ""; // This should fail IsAbsolutePath. + EXPECT_EQ( + error::INVALID_ARGUMENT, + CreateServerCore(absolute, std::move(options), &server_core).code()); +} + +TEST_P(RelativePathsServerCoreTest, RelativePathWithOptionsSucceeds) { + std::unique_ptr server_core; + ServerCore::Options options = GetDefaultOptions(); + string model_config_list_root_dir; + ModelServerConfig relative = + GetTestModelServerConfigWithRelativePath(&model_config_list_root_dir); + options.model_config_list_root_dir = std::move(model_config_list_root_dir); + TF_ASSERT_OK(CreateServerCore(relative, std::move(options), &server_core)); +} + +TEST_P(RelativePathsServerCoreTest, MixedAbsoluteRelativeFails) { + std::unique_ptr server_core; + ModelServerConfig mixed = GetTestModelServerConfigForFakePlatform(); + const ModelServerConfig relative = GetTestModelServerConfigWithRelativePath(); + *mixed.mutable_model_config_list()->add_config() = + relative.model_config_list().config(0); + EXPECT_EQ(error::INVALID_ARGUMENT, + CreateServerCore(mixed, &server_core).code()); +} + TEST_P(ServerCoreTest, ErroringModel) { ServerCore::Options options = GetDefaultOptions(); test_util::StoragePathErrorInjectingSourceAdapterConfig source_adapter_config; @@ -521,6 +615,10 @@ INSTANTIATE_TEST_CASE_P( TestType, ServerCoreTest, ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES))); +INSTANTIATE_TEST_CASE_P( + TestType, RelativePathsServerCoreTest, + ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES))); + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index caee7eefaf0..050d5f8790a 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -79,6 +79,7 @@ cc_library( "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", "//tensorflow_serving/test_util", + "//testing/base/public:gunit_for_library", "@org_tensorflow//tensorflow/core:test", ], ) diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index 1805bf8677c..de523763537 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -70,12 +70,18 @@ ServerCore::Options GetDefaultOptions(const bool use_saved_model) { } // namespace Status CreateServerCore(const ModelServerConfig& config, + ServerCore::Options options, std::unique_ptr* server_core) { - ServerCore::Options options = GetDefaultOptions(true /*use_saved_model */); options.model_server_config = config; return ServerCore::Create(std::move(options), server_core); } +Status CreateServerCore(const ModelServerConfig& config, + std::unique_ptr* server_core) { + return CreateServerCore(config, GetDefaultOptions(true /*use_saved_model */), + server_core); +} + ModelServerConfig ServerCoreTest::GetTestModelServerConfigForFakePlatform() { ModelServerConfig config = GetTestModelServerConfigForTensorflowPlatform(); ModelConfig* model_config = @@ -117,8 +123,9 @@ ServerCore::Options ServerCoreTest::GetDefaultOptions() { } Status ServerCoreTest::CreateServerCore( - const ModelServerConfig& config, std::unique_ptr* server_core) { - return test_util::CreateServerCore(config, server_core); + const ModelServerConfig& config, ServerCore::Options options, + std::unique_ptr* server_core) { + return test_util::CreateServerCore(config, std::move(options), server_core); } } // namespace test_util diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index f8ffe135863..c936386ec9e 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -63,14 +63,27 @@ class ServerCoreTest : public ::testing::TestWithParam { ServerCore::Options GetDefaultOptions(); // Creates a ServerCore object configured with both a fake platform and the - // tensorflow platform, using GetDefaultOptions(). + // tensorflow platform, using the supplied options. Status CreateServerCore(const ModelServerConfig& config, + ServerCore::Options options, std::unique_ptr* server_core); + // Creates a ServerCore object configured with both a fake platform and the + // tensorflow platform, using GetDefaultOptions(). + Status CreateServerCore(const ModelServerConfig& config, + std::unique_ptr* server_core) { + return CreateServerCore(config, GetDefaultOptions(), server_core); + } + // Returns test type. This is the parameter of this test. TestType GetTestType() { return static_cast(GetParam()); } }; +// Creates a ServerCore object with the supplied options. +Status CreateServerCore(const ModelServerConfig& config, + ServerCore::Options options, + std::unique_ptr* server_core); + // Creates a ServerCore object with sane defaults. Status CreateServerCore(const ModelServerConfig& config, std::unique_ptr* server_core); From 59c19b047246017e5132c508c18abfc5943337d0 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 18 Aug 2017 13:08:18 -0800 Subject: [PATCH 0343/8554] Fix build issue. Change: 165750141 --- tensorflow_serving/model_servers/server_core_test.cc | 2 +- tensorflow_serving/model_servers/test_util/BUILD | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 18312ec59a9..0941594652d 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -16,7 +16,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" #include "google/protobuf/any.pb.h" -#include "tensorflow/core/lib/core/error_codes.proto.h" +#include "tensorflow/core/lib/core/error_codes.pb.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 050d5f8790a..94624d94ac7 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -69,6 +69,7 @@ cc_library( "//visibility:public", ], deps = [ + "//external:gtest", "//tensorflow_serving/core:availability_preserving_policy", "//tensorflow_serving/core:servable_id", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", @@ -79,7 +80,6 @@ cc_library( "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", "//tensorflow_serving/test_util", - "//testing/base/public:gunit_for_library", "@org_tensorflow//tensorflow/core:test", ], ) From a8f8488afbefb7a3af34a2f389ed207234a84492 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 23 Aug 2017 11:31:04 -0800 Subject: [PATCH 0344/8554] Add go_api_version tag to model_server_config_proto and file_system_storage_path_source_proto. Change: 166237993 --- tensorflow_serving/config/BUILD | 1 + tensorflow_serving/sources/storage_path/BUILD | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index c7122d52122..7215a33cecd 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -24,6 +24,7 @@ serving_proto_library( name = "model_server_config_proto", srcs = ["model_server_config.proto"], cc_api_version = 2, + go_api_version = 2, deps = [ ":logging_config_proto", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", diff --git a/tensorflow_serving/sources/storage_path/BUILD b/tensorflow_serving/sources/storage_path/BUILD index 44da570369d..82731055afd 100644 --- a/tensorflow_serving/sources/storage_path/BUILD +++ b/tensorflow_serving/sources/storage_path/BUILD @@ -84,6 +84,7 @@ serving_proto_library( name = "file_system_storage_path_source_proto", srcs = ["file_system_storage_path_source.proto"], cc_api_version = 2, + go_api_version = 2, visibility = ["//visibility:public"], ) From 13ee900c113985747f2fba0ebda0b7fe131d5d4e Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 23 Aug 2017 16:31:41 -0800 Subject: [PATCH 0345/8554] Re-enable tensorflow_model_server_test on Jenkins now that the gRPC dependencies are included. Change: 166280710 --- tensorflow_serving/model_servers/BUILD | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index d29d96789d0..f26e41e8f6b 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -163,8 +163,6 @@ py_test( "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_three/00000123/variables/variables.index", "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], - # TODO(b/36008812): Remove this tag. - tags = ["manual"], deps = [ "//tensorflow_serving/apis:prediction_service_proto_py_pb2", "@org_tensorflow//tensorflow:tensorflow_py", From 0e8e7fa92310334cf4a68571778fd8a2d02873ab Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 24 Aug 2017 13:23:33 -0800 Subject: [PATCH 0346/8554] Makes the meta_graph_def_ in TensorFlowMultiInferenceRunner const. Change: 166394194 --- tensorflow_serving/servables/tensorflow/multi_inference.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.h b/tensorflow_serving/servables/tensorflow/multi_inference.h index 2d22a256fb9..708db9cb7b8 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.h +++ b/tensorflow_serving/servables/tensorflow/multi_inference.h @@ -28,7 +28,8 @@ namespace serving { // Only supports Models in the SavedModel format. class TensorFlowMultiInferenceRunner { public: - TensorFlowMultiInferenceRunner(Session* session, MetaGraphDef* meta_graph_def) + TensorFlowMultiInferenceRunner(Session* session, + const MetaGraphDef* meta_graph_def) : session_(session), meta_graph_def_(meta_graph_def) {} // Run inference and return the inference results in the same order as the @@ -41,7 +42,7 @@ class TensorFlowMultiInferenceRunner { private: Session* const session_; - MetaGraphDef* const meta_graph_def_; + const MetaGraphDef* const meta_graph_def_; }; Status RunMultiInference(const RunOptions& run_options, ServerCore* core, From dce201d79e3c638c49fbb33c53ed5eb725b61e92 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 24 Aug 2017 23:21:05 -0800 Subject: [PATCH 0347/8554] Limit the rate of a warning LOG message to once every 2 minutes. Change: 166445523 --- tensorflow_serving/batching/batching_session.cc | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index e0047fac47f..9f5aa1aace3 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -244,9 +244,15 @@ Status BatchingSession::Run( if (batch_scheduler_it == batch_schedulers_.end()) { // We have a Run() call that doesn't match one of our batching signatures. // Run it in-line. - LOG(WARNING) << "Request doesn't match any declared signature. Bypassing " - "batcher. Request signature is: " - << TensorSignatureDebugString(signature); + static uint64 last_log_message_secs = 0; + uint64 now_secs = Env::Default()->NowSeconds(); + // The time check is not strictly thread safe, but it doesn't matter. + if (now_secs - last_log_message_secs >= 120) { + LOG(WARNING) << "Request doesn't match any declared signature. Bypassing " + "batcher. Request signature is: " + << TensorSignatureDebugString(signature); + last_log_message_secs = now_secs; + } return wrapped_->Run(run_options, inputs, output_tensor_names, target_node_names, outputs, run_metadata); } From f8c779f6525749d5d7835b4fb8dcb6a8d80af035 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 28 Aug 2017 13:55:21 -0800 Subject: [PATCH 0348/8554] Necessary Bazel related changes to allow us to sync the TensorFlow submodule to head. Change: 166761676 --- WORKSPACE | 8 +++---- tensorflow_serving/apis/BUILD | 6 ++--- tensorflow_serving/config/BUILD | 4 ++-- tensorflow_serving/core/BUILD | 8 +++---- tensorflow_serving/core/test_util/BUILD | 22 +++++++++---------- tensorflow_serving/example/BUILD | 4 ++-- tensorflow_serving/model_servers/BUILD | 8 +++---- .../model_servers/test_util/BUILD | 8 +++---- tensorflow_serving/resources/BUILD | 2 +- tensorflow_serving/servables/tensorflow/BUILD | 22 +++++++++---------- tensorflow_serving/serving.bzl | 16 +++++++------- tensorflow_serving/test_util/BUILD | 4 ++-- tensorflow_serving/util/BUILD | 4 ++-- tensorflow_serving/util/test_util/BUILD | 2 +- 14 files changed, 59 insertions(+), 59 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 7a510f9440c..fcf6cdb503a 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -9,11 +9,11 @@ local_repository( # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. http_archive( name = "io_bazel_rules_closure", - sha256 = "4be8a887f6f38f883236e77bb25c2da10d506f2bf1a8e5d785c0f35574c74ca4", - strip_prefix = "rules_closure-aac19edc557aec9b603cd7ffe359401264ceff0d", + sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846", + strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz", # 2017-05-10 - "https://github.com/bazelbuild/rules_closure/archive/aac19edc557aec9b603cd7ffe359401264ceff0d.tar.gz", + "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02 + "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", ], ) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 30c45a47b02..772380a8469 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -32,7 +32,7 @@ serving_proto_library( deps = [ ":model_proto", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -54,7 +54,7 @@ serving_proto_library( java_api_version = 2, deps = [ "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -80,7 +80,7 @@ serving_proto_library( go_api_version = 2, java_api_version = 2, deps = [ - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 7215a33cecd..486e20e49fb 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -28,7 +28,7 @@ serving_proto_library( deps = [ ":logging_config_proto", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -37,7 +37,7 @@ serving_proto_library( srcs = ["platform_config.proto"], cc_api_version = 2, deps = [ - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 13bd550cbf9..fff97f00c86 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -754,7 +754,7 @@ cc_library( ":logging_proto", "//tensorflow_serving/config:logging_config_proto", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) @@ -774,7 +774,7 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) @@ -792,7 +792,7 @@ cc_library( "//tensorflow_serving/core:logging_proto", "//tensorflow_serving/util:fast_read_dynamic_ptr", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) @@ -812,6 +812,6 @@ cc_test( "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 70b02df045a..44c5a71790f 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -27,7 +27,7 @@ cc_library( srcs = ["test_main.cc"], linkopts = ["-lm"], deps = [ - "//external:gtest", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:testlib", ], ) @@ -37,9 +37,9 @@ cc_library( testonly = 1, hdrs = ["mock_loader.h"], deps = [ - "//external:gtest", "//tensorflow_serving/core:loader", "//tensorflow_serving/util:any_ptr", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -100,9 +100,9 @@ cc_library( testonly = 1, hdrs = ["mock_storage_path_target.h"], deps = [ - "//external:gtest", "//tensorflow_serving/core:storage_path", "//tensorflow_serving/core:target", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -114,8 +114,8 @@ cc_library( hdrs = ["availability_test_util.h"], visibility = ["//visibility:public"], deps = [ - "//external:gtest", "//tensorflow_serving/core:servable_state_monitor", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", ], ) @@ -150,10 +150,10 @@ cc_library( hdrs = ["mock_log_collector.h"], visibility = ["//visibility:public"], deps = [ - "//external:gtest", "//tensorflow_serving/core:log_collector", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) @@ -162,10 +162,10 @@ cc_library( testonly = 1, hdrs = ["fake_log_collector.h"], deps = [ - "//external:gtest", "//tensorflow_serving/core:log_collector", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) @@ -174,11 +174,11 @@ cc_library( testonly = 1, hdrs = ["mock_request_logger.h"], deps = [ - "//external:gtest", "//tensorflow_serving/core:logging_proto", "//tensorflow_serving/core:request_logger", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) @@ -187,7 +187,7 @@ cc_library( testonly = 1, hdrs = ["mock_session.h"], deps = [ - "//external:gtest", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", ], diff --git a/tensorflow_serving/example/BUILD b/tensorflow_serving/example/BUILD index d86b2677709..1beb14fd976 100644 --- a/tensorflow_serving/example/BUILD +++ b/tensorflow_serving/example/BUILD @@ -84,8 +84,8 @@ cc_binary( ], deps = [ "//tensorflow_serving/apis:prediction_service_proto", - "@grpc//:grpc++", + "@grpc//:grpc++_unsecure", "@org_tensorflow//tensorflow/core:framework", - "@protobuf//:protobuf_lite", + "@protobuf_archive//:protobuf_lite", ], ) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index f26e41e8f6b..8044df9d1a4 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -75,7 +75,7 @@ cc_library( "//tensorflow_serving/util:optional", "//tensorflow_serving/util:unique_ptr_with_deps", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -102,7 +102,7 @@ cc_test( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -132,14 +132,14 @@ cc_binary( ":model_platform_types", ":platform_config_util", ":server_core", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "@org_tensorflow//tensorflow/core/platform/hadoop:hadoop_file_system", "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:availability_preserving_policy", - "@grpc//:grpc++", + "@grpc//:grpc++_unsecure", ] + TENSORFLOW_DEPS + SUPPORTED_TENSORFLOW_OPS, ) diff --git a/tensorflow_serving/model_servers/test_util/BUILD b/tensorflow_serving/model_servers/test_util/BUILD index 94624d94ac7..3bd55c5f3bb 100644 --- a/tensorflow_serving/model_servers/test_util/BUILD +++ b/tensorflow_serving/model_servers/test_util/BUILD @@ -28,7 +28,6 @@ cc_library( "//visibility:public", ], deps = [ - "//external:gtest", "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/config:platform_config_proto", @@ -42,9 +41,10 @@ cc_library( "//tensorflow_serving/model_servers:server_core", "//tensorflow_serving/util:event_bus", "//tensorflow_serving/util:unique_ptr_with_deps", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:cc_wkt_protos", - "@protobuf//:protobuf_lite", + "@protobuf_archive//:cc_wkt_protos", + "@protobuf_archive//:protobuf_lite", ], ) @@ -69,7 +69,6 @@ cc_library( "//visibility:public", ], deps = [ - "//external:gtest", "//tensorflow_serving/core:availability_preserving_policy", "//tensorflow_serving/core:servable_id", "//tensorflow_serving/core/test_util:fake_loader_source_adapter", @@ -80,6 +79,7 @@ cc_library( "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", "//tensorflow_serving/test_util", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:test", ], ) diff --git a/tensorflow_serving/resources/BUILD b/tensorflow_serving/resources/BUILD index d94b1b0206d..3f7dbe2507b 100644 --- a/tensorflow_serving/resources/BUILD +++ b/tensorflow_serving/resources/BUILD @@ -26,7 +26,7 @@ serving_proto_library( srcs = ["resources.proto"], cc_api_version = 2, visibility = ["//visibility:public"], - deps = ["@protobuf//:cc_wkt_protos"], + deps = ["@protobuf_archive//:cc_wkt_protos"], ) cc_library( diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 9648d188ec5..cab925e163a 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -31,7 +31,7 @@ serving_proto_library( ], deps = [ "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -54,7 +54,7 @@ cc_library( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -79,7 +79,7 @@ cc_test( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -113,7 +113,7 @@ cc_library( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -155,7 +155,7 @@ cc_test( "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -200,7 +200,7 @@ cc_test( "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -478,8 +478,8 @@ cc_test( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:cc_wkt_protos", - "@protobuf//:protobuf", + "@protobuf_archive//:cc_wkt_protos", + "@protobuf_archive//:protobuf", ], ) @@ -555,7 +555,7 @@ cc_test( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:protobuf_lite", + "@protobuf_archive//:protobuf_lite", ], ) @@ -639,7 +639,7 @@ cc_test( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:protobuf_lite", + "@protobuf_archive//:protobuf_lite", ], ) @@ -662,7 +662,7 @@ cc_library( "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) diff --git a/tensorflow_serving/serving.bzl b/tensorflow_serving/serving.bzl index c55e12567c8..b9306d1d366 100644 --- a/tensorflow_serving/serving.bzl +++ b/tensorflow_serving/serving.bzl @@ -1,5 +1,5 @@ -load("@protobuf//:protobuf.bzl", "cc_proto_library") -load("@protobuf//:protobuf.bzl", "py_proto_library") +load("@protobuf_archive//:protobuf.bzl", "cc_proto_library") +load("@protobuf_archive//:protobuf.bzl", "py_proto_library") def serving_proto_library(name, srcs=[], has_services=False, deps=[], visibility=None, testonly=0, # pylint: disable=unused-argument @@ -17,9 +17,9 @@ def serving_proto_library(name, srcs=[], has_services=False, cc_proto_library(name=name, srcs=srcs, deps=deps, - cc_libs = ["@protobuf//:protobuf"], - protoc="@protobuf//:protoc", - default_runtime="@protobuf//:protobuf", + cc_libs = ["@protobuf_archive//:protobuf"], + protoc="@protobuf_archive//:protoc", + default_runtime="@protobuf_archive//:protobuf", use_grpc_plugin=use_grpc_plugin, testonly=testonly, visibility=visibility,) @@ -32,8 +32,8 @@ def serving_proto_library_py(name, proto_library, srcs=[], deps=[], visibility=N py_proto_library(name=name, srcs=srcs, srcs_version = "PY2AND3", - deps=["@protobuf//:protobuf_python"] + deps, - default_runtime="@protobuf//:protobuf_python", - protoc="@protobuf//:protoc", + deps=["@protobuf_archive//:protobuf_python"] + deps, + default_runtime="@protobuf_archive//:protobuf_python", + protoc="@protobuf_archive//:protoc", visibility=visibility, testonly=testonly,) diff --git a/tensorflow_serving/test_util/BUILD b/tensorflow_serving/test_util/BUILD index 8ab767118a6..698f0d29328 100644 --- a/tensorflow_serving/test_util/BUILD +++ b/tensorflow_serving/test_util/BUILD @@ -23,10 +23,10 @@ cc_library( srcs = ["test_util.cc"], hdrs = ["test_util.h"], deps = [ - "//external:gtest", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", - "@protobuf//:protobuf", + "@protobuf_archive//:protobuf", ], ) diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index da565b3a282..069e39c6efa 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -49,7 +49,7 @@ cc_library( deps = [ ":class_registration_util", "@org_tensorflow//tensorflow/core:lib", - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -305,6 +305,6 @@ serving_proto_library( srcs = ["class_registration_test.proto"], cc_api_version = 2, deps = [ - "@protobuf//:cc_wkt_protos", + "@protobuf_archive//:cc_wkt_protos", ], ) diff --git a/tensorflow_serving/util/test_util/BUILD b/tensorflow_serving/util/test_util/BUILD index 676554a200b..07f7acce80c 100644 --- a/tensorflow_serving/util/test_util/BUILD +++ b/tensorflow_serving/util/test_util/BUILD @@ -14,8 +14,8 @@ cc_library( testonly = 1, hdrs = ["mock_file_probing_env.h"], deps = [ - "//external:gtest", "//tensorflow_serving/util:file_probing_env", + "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:lib", ], ) From 827aea0d180dc2cdb7d01349302772587415212e Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 28 Aug 2017 15:01:05 -0700 Subject: [PATCH 0349/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 16d39e94e37..668db64a5d6 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 16d39e94e3724417fcaed87035434e098e892842 +Subproject commit 668db64a5d612d5f96b5d87772ce6ff6531fc035 diff --git a/tf_models b/tf_models index 78007443138..3bf85a4eddb 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 78007443138108abf5170b296b4d703b49454487 +Subproject commit 3bf85a4eddb9c56a28cc266ee4aa5604fb4d8334 From 3989d7e0ff74bb178178b35e8e2323814a156cf3 Mon Sep 17 00:00:00 2001 From: Nastya Torunova Date: Tue, 5 Sep 2017 20:40:09 +0300 Subject: [PATCH 0350/8554] Add padding for batching session (#535) * Add padding for batching session * add definitions in .h file and comments for padding code, also add parameter to toggle padding * switch to 2 spaces indentation and fix line lengths * add tests for padding * move padding code to separate library, add matrix_half_plus_two_model for padding tests, add end-to-end padding test for batching session * Add docs about padding values and supported datatypes. Change signatures of padding functions, so that they return Status and remove PaddingResult. Move internal padding functions to batching_util.cc. Make MergeInputTensors return error when tensors don't have equal shapes and padding is turned off. * fix optional and eliminate int64 narrowing warniings * fix code style --- tensorflow_serving/batching/BUILD | 27 +++ .../batching/batching_session.cc | 65 +++++- .../batching/batching_session.h | 31 +++ .../batching/batching_session_test.cc | 85 +++++++ tensorflow_serving/batching/batching_util.cc | 209 ++++++++++++++++++ tensorflow_serving/batching/batching_util.h | 68 ++++++ .../batching/batching_util_test.cc | 88 ++++++++ tensorflow_serving/batching/test_util/BUILD | 8 + .../matrix_half_plus_two_saved_model.py | 55 +++++ tensorflow_serving/batching/testdata/BUILD | 14 ++ .../matrix_half_plus_two/1/saved_model.pb | Bin 0 -> 856 bytes .../tensorflow/bundle_factory_util.cc | 2 + .../tensorflow/session_bundle_config.proto | 3 + 13 files changed, 651 insertions(+), 4 deletions(-) create mode 100644 tensorflow_serving/batching/batching_util.cc create mode 100644 tensorflow_serving/batching/batching_util.h create mode 100644 tensorflow_serving/batching/batching_util_test.cc create mode 100644 tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py create mode 100644 tensorflow_serving/batching/testdata/BUILD create mode 100644 tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index 7dcceede9a7..0c736d0ecf6 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -18,6 +18,7 @@ filegroup( ), ) + cc_library( name = "streaming_batch_scheduler", srcs = ["streaming_batch_scheduler.cc"], @@ -57,6 +58,8 @@ cc_library( "//tensorflow_serving/servables/tensorflow:serving_session", "//tensorflow_serving/util:cleanup", "//tensorflow_serving/util:hash", + "//tensorflow_serving/batching:batching_util", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/contrib/batching:basic_batch_scheduler", "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", "@org_tensorflow//tensorflow/core:core_cpu", @@ -73,6 +76,7 @@ cc_test( ], data = [ "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + "//tensorflow_serving/batching/testdata:matrix_half_plus_two" ], deps = [ ":batching_session", @@ -114,3 +118,26 @@ cc_test( "@org_tensorflow//tensorflow/core:test", ], ) + +cc_library( + name = "batching_util", + srcs = ["batching_util.cc"], + hdrs = ["batching_util.h"], + deps = [ + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "batching_util_test", + srcs = [ + "batching_util_test.cc" + ], + deps = [ + ":batching_util", + "//tensorflow_serving/core/test_util:test_main", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + ], +) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index 9f5aa1aace3..1730c11fc67 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -29,6 +29,8 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/serving_session.h" #include "tensorflow_serving/util/cleanup.h" #include "tensorflow_serving/util/hash.h" +#include "tensorflow_serving/batching/batching_util.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -79,6 +81,34 @@ TensorSignature TensorSignatureFromRunArgs( return signature; } +// Constructs vector of all task inputs from Batch of BatchingSessionTasks. +// Input for each task is a vector of pairs (tensor_name, tensor_value). +std::vector>> GetTaskInputsVector( + const Batch& batch) { + std::vector>> all_task_inputs; + for (int i = 0; i < batch.num_tasks(); ++i) { + const std::vector>& task_inputs = + *batch.task(i).inputs; + all_task_inputs.push_back(task_inputs); + } + return all_task_inputs; +} +// Returns true iff all dims of shape1 are equal to dims of shape2 starting with +// the first (not zeroth) dimension. +// For example, for shapes [1, 2, 3] and [4, 2, 3] the result is true. +bool AreShapesEqualExceptZeroDim(const TensorShape& shape1, + const TensorShape& shape2) { + if (shape1.dims() != shape2.dims()) { + return false; + } + for (int i = 1; i < shape1.dims(); ++i) { + if (shape1.dim_size(i) != shape2.dim_size(i)) { + return false; + } + } + return true; +} + } // namespace TensorSignature TensorSignatureFromSignatureDef( @@ -346,7 +376,14 @@ Status BatchingSession::MergeInputTensors( // For each input tensor name, a vector of tensors from the individual tasks. std::map> tensors_to_merge; - + // For each input tensor name a vector of maximum dimension sizes + // among tensors from individual tasks. + optional>> max_dim_sizes; + if (options_.pad_variable_length_inputs) { + std::vector>> all_task_inputs = + GetTaskInputsVector(batch); + max_dim_sizes = CalculateMaxDimSizes(all_task_inputs); + } // Populate 'tensors_to_merge'. for (int i = 0; i < batch.num_tasks(); ++i) { const std::vector>& task_inputs = @@ -356,8 +393,28 @@ Status BatchingSession::MergeInputTensors( const Tensor& tensor = entry.second; std::vector& tensor_vec = tensors_to_merge[tensor_name]; - tensor_vec.push_back(tensor); - + Tensor optionally_padded_tensor; + if (options_.pad_variable_length_inputs) { + TF_RETURN_IF_ERROR(AddPadding(tensor, (*max_dim_sizes)[tensor_name], + &optionally_padded_tensor)); + } else { + optionally_padded_tensor = tensor; + // Check whether tensors with the same name have equal dims + // (except zeroth dim) when padding is turned off. + if (i > 0) { // added at least one task to tensors_to_merge + TensorShape reference_shape = + tensors_to_merge[tensor_name][0].shape(); + if (!AreShapesEqualExceptZeroDim(tensor.shape(), reference_shape)) { + return errors::FailedPrecondition( + "Tensors with name '" + tensor_name + "' from different tasks" + + " have different shapes and padding is turned off." + + "Set pad_variable_length_inputs to true, or ensure that " + + "all tensors with the same name" + + "have equal dimensions starting with the first dim."); + } + } + } + tensor_vec.push_back(optionally_padded_tensor); if (i == batch.num_tasks() - 1 && padding_size > 0) { // This is the last task. Insert padding. // @@ -367,7 +424,7 @@ Status BatchingSession::MergeInputTensors( // // Slice() operates on the 0th dimension, which is the batch dimension. // It avoids a deep copy, which is a nice efficiency bonus. - const Tensor padding_tensor = tensor.Slice(0, 1); + const Tensor padding_tensor = optionally_padded_tensor.Slice(0, 1); for (int i = 0; i < padding_size; ++i) { tensor_vec.push_back(padding_tensor); } diff --git a/tensorflow_serving/batching/batching_session.h b/tensorflow_serving/batching/batching_session.h index 4d2e0ddf2d8..0ce8bf34939 100644 --- a/tensorflow_serving/batching/batching_session.h +++ b/tensorflow_serving/batching/batching_session.h @@ -99,6 +99,37 @@ struct BatchingSessionOptions { // // If left empty, no rounding/padding is performed. std::vector allowed_batch_sizes; + + // If set to true, padding is performed for tensors of the same name + // but with unequal dimensions (modulo zeroth dimension), so that + // all tensors of the same name have equal dim sizes. + // For each tensor its first element is used as padding value. + // + // For example: + // given input tensors of shapes [1, 500, 101], [2, 300, 101], [1, 400, 101] + // they will be padded to shapes [1, 500, 101], [2, 500, 101], [1, 500, 101]. + // Padding is not performed in zeroth dimension. + // + // Supported tensor datatypes: + // DT_FLOAT, DT_DOUBLE, DT_INT8, DT_UINT8, DT_INT16, + // DT_UINT16, DT_INT32, DT_INT64, DT_COMPLEX64, DT_COMPLEX128, + // DT_STRING, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, + // DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE. + // + // Supported ranks: from 1 to 6. + // + // This option is useful when using recurrent models(like LSTMs) with serving. + // These models typically accept variable-length inputs and when + // training them typical strategy is to save sequence lengths for decoding + // and pad those variable-length dims to maximum in batch. + // So, this option is used to achieve similar behavior + // when serving with batching, it is assumed that sequence lengths + // have already been saved. + // + // If tensors with the same name have different shapes + // (modulo zeroth dimension) and this option is set to false, + // then error Status will be returned. + bool pad_variable_length_inputs = false; }; // Wraps a session in a new session that automatically batches Run() calls. diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index a173aead2d8..02e5b69ed84 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -92,6 +92,17 @@ std::unique_ptr CreateHalfPlusTwoSession() { return std::move(bundle.session); } +std::unique_ptr CreateMatrixHalfPlusTwoSession() { + tensorflow::SessionOptions session_options; + tensorflow::RunOptions run_options; + const string export_dir = test_util::TestSrcDirPath( + "batching/testdata/matrix_half_plus_two/1"); + SavedModelBundle bundle; + TF_CHECK_OK(LoadSavedModel(session_options, run_options, export_dir, + {kSavedModelTagServe}, &bundle)); + return std::move(bundle.session); +} + // Test that a session handles a single request for the half-plus-two model // properly. The request has two input floats (size=2 for batching purposes). void TestSingleRequest(float input_0, float input_1, Session* session) { @@ -107,6 +118,18 @@ void TestSingleRequest(float input_0, float input_1, Session* session) { test::ExpectTensorEqual(expected_output, outputs[0]); } +void TestRequestToMatrixHalfPlusTwo(const std::vector& x_values, + TensorShape x_shape, const std::vector& y_values, + TensorShape y_shape, Session* session) { + Tensor input = test::AsTensor(x_values, x_shape); + Tensor expected_output = test::AsTensor(y_values, y_shape); + std::vector output; + TF_ASSERT_OK(session->Run({{"x", input}}, {"y"}, {}, &output)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); +} + + // Invoke Run() with the supplied arguments, and expect a particular error. void ExpectError(const string& error_message, const std::vector>& inputs, @@ -181,6 +204,68 @@ TEST(BatchingSessionTest, Basic) { })); } +TEST(BatchingSessionTest, BatchingWithPadding) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 1; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + batching_session_options.pad_variable_length_inputs = true; + TF_ASSERT_OK(CreateBasicBatchingSession(schedule_options, + batching_session_options, {{"x"}, {"y"}}, + CreateMatrixHalfPlusTwoSession(), &batching_session)); + // two requests form a batch and first input gets padded with zeros to match + // [1, 3, 3] shape that is accepted by the model. + // if padding doesn't work, test will fail. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session] { + TestRequestToMatrixHalfPlusTwo({1, 2, 3, 4}, {1, 2, 2}, + {2.5, 3, 2.5, 3.5, 4, 2.5, 2.5, 2.5, 2.5}, {1, 3, 3}, + batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + TestRequestToMatrixHalfPlusTwo({5, 6, 7, 8, 9, 10, 11, 12, 13}, + {1, 3, 3}, {4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5}, {1, 3, 3}, + batching_session.get()); + })); +} + +TEST(BatchingSessionTest, UnequalTensorShapesWithPaddingTurnedOff) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 1; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + batching_session_options.pad_variable_length_inputs = false; + TF_ASSERT_OK(CreateBasicBatchingSession(schedule_options, + batching_session_options, {{"x"}, {"y"}}, + CreateMatrixHalfPlusTwoSession(), &batching_session)); + string expected_error_msg = "Tensors with name 'x' from different tasks" + " have different shapes and padding is turned off." + "Set pad_variable_length_inputs to true, or ensure that " + "all tensors with the same name" + "have equal dimensions starting with the first dim."; + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session, + &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor({1, 2, 3, 4}, {1, 2, 2})}}, + {"y"}, batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session, + &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor({5, 6, 7, 8, 9, 10, 11, 12, 13}, + {1, 3, 3})}}, + {"y"}, batching_session.get()); + })); +} + + TEST(BatchingSessionTest, SingletonBatch) { BasicBatchScheduler::Options schedule_options; schedule_options.max_batch_size = 4; // fits two 2-unit tasks diff --git a/tensorflow_serving/batching/batching_util.cc b/tensorflow_serving/batching/batching_util.cc new file mode 100644 index 00000000000..eefc1e1a3b6 --- /dev/null +++ b/tensorflow_serving/batching/batching_util.cc @@ -0,0 +1,209 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/batching_util.h" + +#include + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/lib/core/errors.h" + + +namespace tensorflow { +namespace serving { + +// Padding for one dimension of a tensor. +// pad_before is a number of values to add before elements in one dimension, +// pad_after is number of objects to add after. +// NOTE: fields are named this way because of Eigen::TensorMap::pad method. +// It requires padding to be an array of elements that have fields +// "first" and "second". +struct OneDimPadding { + int64 first; // pad before + int64 second; // pad after +}; + +// Constructs array of paddings, where: +// paddings[i].first - number of objects to add before elements in dimension i +// of given tensor, +// paddings[i].second - number of objects to add after elements in dimension i. +// This padding signature is used when performing internal padding with Eigen. +// +// When building paddings it is assumed that tensor needs to be padded +// after each dimension, so its shape matches max_dim_sizes, +// First entry of max_dim_sizes, which is maximum size of zeroth dimension, +// is ignored, because we don't perform padding in that dimension. +template +std::array CreatePadding(Tensor tensor, + const std::vector& max_dim_sizes) { + std::array padding; + for (unsigned int i = 0; i < max_dim_sizes.size(); ++i) { + if (i > 0 && max_dim_sizes[i] - tensor.dim_size(i) > 0) { + padding[i] = {0, max_dim_sizes[i] - tensor.dim_size(i)}; + } else { + padding[i] = {0, 0}; + } + } + return padding; +} + +// Functor, which performs padding of given input tensor +// using specified padding signature. +// For example, given tensor of shape [1, 2, 3] and padding signature +// [[0, 0], [0, 2], [2, 2]] +// functor produces padded_tensor of shape [1, 4, 7]. +template +struct PadTensor { + Status operator()(Tensor input, + const std::array& padding, + Tensor* output) { + TensorShape output_shape; + for (int d = 0; d < num_dims; ++d) { + // Pad before existing elements. + const int32 before_d = padding[d].first; + // Pad after existing elements. + const int32 after_d = padding[d].second; + output_shape.AddDim(before_d + input.dim_size(d) + after_d); + } + if (output_shape.num_elements() == input.NumElements()) { + bool result = output->CopyFrom(input, output_shape); + if (!result) { + return errors::Internal("Couldn't create output."); + } + return Status::OK(); + } + if (input.NumElements() < 1) { + return errors::InvalidArgument( + "Got empty tensor in batch of non-empty tensors."); + } + *output = Tensor(input.dtype(), output_shape); + typename TTypes::Tensor inputs = input.tensor(); + T pad_value(input.flat()(0)); // using existing values in padding + output->tensor() = inputs.pad(padding, pad_value); + return Status::OK(); + } +}; + +// Invokes padding procedure for specific tensor ranks. +// Only ranks from 1 to 6 are supported (like in PadOp). +template +Status PadTensorOfSpecificType(const Tensor& tensor, + const std::vector& max_dim_sizes, + Tensor* output_tensor) { + int num_dims = tensor.dims(); + switch (num_dims) { + case 1: { + std::array padding; + padding = CreatePadding<1>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 2: { + std::array padding; + padding = CreatePadding<2>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 3: { + std::array padding; + padding = CreatePadding<3>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 4: { + std::array padding; + padding = CreatePadding<4>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 5: { + std::array padding; + padding = CreatePadding<5>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 6: { + std::array padding; + padding = CreatePadding<6>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + default: + // only ranks from 1 to 6 are supported + // (like in tensorflow/core/kernels/pad_op.cc) + return errors::InvalidArgument( + "Only tensors with rank from 1 to 6 can be padded."); + } +} + +std::map> CalculateMaxDimSizes( + const std::vector>>& batch) { + std::map> max_dim_sizes; + // Populate 'max_dim_sizes' + // init + const std::vector>& task_inputs = batch[0]; + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; + max_dim_sizes[tensor_name] = std::vector(tensor.dims(), 0); + } + // fill + for (int i = 0; i < batch.size(); ++i) { + const std::vector>& task_inputs = batch[i]; + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; + + std::vector& max_dim_sizes_for_one_input = + max_dim_sizes[tensor_name]; + for (int j = 0; j < tensor.dims(); ++j) { + const int old_max_size = max_dim_sizes_for_one_input[j]; + if (tensor.shape().dim_size(j) > old_max_size) { + max_dim_sizes_for_one_input[j] = tensor.shape().dim_size(j); + } + } + } + } + return max_dim_sizes; +} + +Status AddPadding(const Tensor& tensor, + const std::vector& max_dim_sizes, + Tensor* padded_tensor) { + const DataType input_dtype = tensor.dtype(); + Status padding_status; +#define CASE(type) \ + case DataTypeToEnum::value: { \ + padding_status = PadTensorOfSpecificType(tensor, \ + max_dim_sizes, \ + padded_tensor); \ + break; \ + } + switch (input_dtype) { + TF_CALL_ALL_TYPES(CASE); + TF_CALL_QUANTIZED_TYPES(CASE); + // quantized types macro doesn't include these types + TF_CALL_quint16(CASE); + TF_CALL_qint16(CASE); + default: + padding_status = errors::InvalidArgument("Unsupported type"); + } +#undef CASE + return padding_status; +} +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/batching_util.h b/tensorflow_serving/batching/batching_util.h new file mode 100644 index 00000000000..6cb574d4611 --- /dev/null +++ b/tensorflow_serving/batching/batching_util.h @@ -0,0 +1,68 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ +#define TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ + +#include +#include +#include + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" + + + +namespace tensorflow { +namespace serving { + +// For batch of inputs calculates maximum dim sizes across all tensors +// with the same name. +// These dim sizes are used later to calculate padding amount for each tensor. +// For example, for batch containing three tasks with the following inputs +// (instead of tensors there are their shapes): +// +// task1: {'tensor_a': [100, 500, 300], 'tensor_b': [100]} +// task2: {'tensor_a': [100, 200, 123], 'tensor_b': [100]} +// task3: {'tensor_a': [200, 100, 400], 'tensor_b': [200]} +// +// the following map will be generated: +// {'tensor_a': [200, 500, 400], 'tensor_b': [200]} +std::map> CalculateMaxDimSizes( + const std::vector>>& batch); + +// Pads tensor so that its shape becomes as specified in max_dim_sizes, +// except for zeroth dimension, which is left as is. +// First entry in max_dim_sizes is ignored. +// First element of a tensor is used as padding value. +// If tensor is empty, an error will be returned. +// +// For example given input tensor with shape [1, 2, 3, 4] and max_dim_sizes +// [1, 2, 5, 8] function produces padded_tensor of shape +// [1, 2, 5, 8], padded with tensor[0][0][0][0] element. +// +// Supported tensor datatypes: +// DT_FLOAT, DT_DOUBLE, DT_INT8, DT_UINT8, DT_INT16, +// DT_UINT16, DT_INT32, DT_INT64, DT_COMPLEX64, DT_COMPLEX128, +// DT_STRING, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, +// DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE. +// +// Supported tensor ranks: from 1 to 6. + +Status AddPadding(const Tensor& tensor, + const std::vector& max_dim_sizes, Tensor* padded_tensor); +} // namespace serving +} // namespace tensorflow +#endif // TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ diff --git a/tensorflow_serving/batching/batching_util_test.cc b/tensorflow_serving/batching/batching_util_test.cc new file mode 100644 index 00000000000..b4ed316a235 --- /dev/null +++ b/tensorflow_serving/batching/batching_util_test.cc @@ -0,0 +1,88 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/batching_util.h" + + + +#include +#include +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::UnorderedElementsAre; +using ::testing::ElementsAre; +using ::testing::Pair; + +// Creates vector of pairs (tensor_name, tensor_value), where tensors +// have shapes as specified in shapes. +// Tensor with shape shapes[i] has tensor_name "x" + std::to_string(i). +std::vector> CreateInputsWithTensorShapes( + const std::vector& shapes) { + std::vector> inputs; + for (int i = 0; i < shapes.size(); ++i) { + inputs.push_back({"x" + std::to_string(i), Tensor(DT_FLOAT, shapes[i])}); + } + return inputs; +} + + +TEST(BatchingUtilTest, CalculateMaxDimSizes) { + const std::vector shapes1 {{10, 20, 30}, {10, 100}}; + std::vector> inputs1 = + CreateInputsWithTensorShapes(shapes1); + const std::vector shapes2 {{20, 50, 15}, {20, 101}}; + std::vector> inputs2 = + CreateInputsWithTensorShapes(shapes2); + std::vector>> batch {inputs1, inputs2}; + std::map> max_dim_sizes = + CalculateMaxDimSizes(batch); + EXPECT_THAT(max_dim_sizes, + UnorderedElementsAre(Pair("x0", ElementsAre(20, 50, 30)), + Pair("x1", ElementsAre(20, 101)))); +} + +TEST(BatchingUtilTest, AddPadding) { + const std::vector max_dim_sizes {20, 100, 200}; + const std::vector types {DT_FLOAT, DT_DOUBLE, DT_INT32, DT_UINT8, + DT_INT16, DT_UINT16, DT_INT8, DT_STRING, DT_COMPLEX64, DT_COMPLEX128, + DT_INT64, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, + DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE}; + Status padding_status; + for (DataType type : types) { + Tensor tensor(type, {10, 20, 30}); + Tensor padded_tensor; + padding_status = AddPadding(tensor, max_dim_sizes, &padded_tensor); + ASSERT_EQ(Status::OK(), padding_status); + EXPECT_EQ(TensorShape({10, 100, 200}), padded_tensor.shape()); + } +} + +TEST(BatchingUtilTest, AddPaddingTensorWithUnsupportedRank) { + const std::vector max_dim_sizes {20, 100, 200, 300, 400, 500, 600}; + const Tensor tensor(DT_FLOAT, {10, 20, 30, 40, 50, 60, 70}); + Tensor padded_tensor; + ASSERT_EQ(errors::InvalidArgument( + "Only tensors with rank from 1 to 6 can be padded."), + AddPadding(tensor, max_dim_sizes, &padded_tensor)); +} +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/test_util/BUILD b/tensorflow_serving/batching/test_util/BUILD index 9bf13315bd6..5673ad23d06 100644 --- a/tensorflow_serving/batching/test_util/BUILD +++ b/tensorflow_serving/batching/test_util/BUILD @@ -41,3 +41,11 @@ cc_test( "@org_tensorflow//tensorflow/core:test", ], ) + +# script that generates saved_model for matrix_half_plus_two model. +py_binary( + name = "matrix_half_plus_two_saved_model", + srcs = ["matrix_half_plus_two_saved_model.py"], + srcs_version = "PY2AND3", + deps = ["@org_tensorflow//tensorflow:tensorflow_py"], +) diff --git a/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py b/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py new file mode 100644 index 00000000000..c1282a09d98 --- /dev/null +++ b/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py @@ -0,0 +1,55 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import tensorflow as tf + + +def _generate_saved_model_for_matrix_half_plus_two(export_dir): + """Creates SavedModel for half plus two model that accepts batches of + 3*3 matrices. + The model divides all elements in each matrix by 2 and adds 2 to them. + So, for one input matrix [[1, 2, 3], [4, 5, 6], [7, 8, 9]] + the result will be [[2.5, 3, 3.5], [4, 4.5, 5], [5.5, 6, 6.5]]. + Args: + export_dir: The directory where to write SavedModel files. + """ + builder = tf.saved_model.builder.SavedModelBuilder(export_dir) + with tf.Session() as session: + x = tf.placeholder(tf.float32, shape=[None, 3, 3], name="x") + a = tf.constant(0.5) + b = tf.constant(2.0) + y = tf.add(tf.multiply(a, x), b, name="y") + predict_signature_def = ( + tf.saved_model.signature_def_utils.predict_signature_def({"x": x}, + {"y": y})) + signature_def_map = { + tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + predict_signature_def} + session.run(tf.global_variables_initializer()) + builder.add_meta_graph_and_variables( + session, + [tf.saved_model.tag_constants.SERVING], + signature_def_map=signature_def_map) + builder.save() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--output_dir", type=str, + default="/tmp/matrix_half_plus_two/1", + help="The directory where to write SavedModel files.") + args = parser.parse_args() + _generate_saved_model_for_matrix_half_plus_two(args.output_dir) diff --git a/tensorflow_serving/batching/testdata/BUILD b/tensorflow_serving/batching/testdata/BUILD new file mode 100644 index 00000000000..92b1b770b61 --- /dev/null +++ b/tensorflow_serving/batching/testdata/BUILD @@ -0,0 +1,14 @@ +# Description: Tensorflow Serving batching test data. + +package( + default_visibility = ["//tensorflow_serving:internal"], + features = ["-layering_check"], +) + +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "matrix_half_plus_two", + srcs = glob( + ["matrix_half_plus_two/**/*"]), +) diff --git a/tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb b/tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..8651e45a0443bb54b19e70518515820a3cc4f089 GIT binary patch literal 856 zcmbVK%TB^j6zuJbazN3eDBguJbyp*=l{O~6b_5>__k_@k5}UT^1E?!M!7uPD{1ZP& zds~`F5TiC}CTS-#_s)z)@Lh#h5uZgk2I<_gPz5s8$sNZyL7L!Gf;xrdqa4N_Gb^yt zE^QD|Ru!qJVF8XnxpbW%q!m!zDD~wObSjwx{B<- znXadYJbh4}Fhe22?i18qt-2`D11jTlIFm zdFT(??Ly={I_|{5Hwo+{7X0dBl=7R1O=9fjlG|n#OWH^N5EM0uzs3Qq_Ofhxsd<1q zu#60yS)iEZI`37@Gc8|$Mo=(hK@d)jtpx)aL{oq+6w-2ft>3GiEmiAeh4I$ttaLIp z3Va+qhdATWspbCD8zIUx~SER7%IHek#pa(*svejAr4`m7r6`4 i)NHO1`h8iLkYY}Xe;`qsO?qg%lg7-a;rYxO4Z=^a-s7_X literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc index 46f0b8e192e..302bc2bbfba 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc @@ -177,6 +177,8 @@ Status WrapSessionForBatching(const BatchingParameters& batching_config, batching_session_options.allowed_batch_sizes.push_back(allowed_batch_size); } + batching_session_options.pad_variable_length_inputs = batching_config.pad_variable_length_inputs(); + auto create_queue = [batch_scheduler, queue_options]( std::function>)> process_batch_callback, diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto index 69a3934f2b3..8f63adcbb1f 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto +++ b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto @@ -96,4 +96,7 @@ message BatchingParameters { // - The entries must be in increasing order. // - The final entry must equal 'max_batch_size'. repeated int64 allowed_batch_sizes = 6; + + // Whether to pad variable-length inputs when a batch is formed. + bool pad_variable_length_inputs = 7; } From e7fa547b50a128a44c55a93f3ca12e8760df1e52 Mon Sep 17 00:00:00 2001 From: Vladimir Moskva Date: Wed, 6 Sep 2017 01:39:23 +0300 Subject: [PATCH 0351/8554] Update rules_closure (#580) * Update rules_closure * Updated submodules --- WORKSPACE | 8 ++++---- tensorflow | 2 +- tf_models | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index fcf6cdb503a..b54eefe1aba 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -9,11 +9,11 @@ local_repository( # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. http_archive( name = "io_bazel_rules_closure", - sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846", - strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f", + sha256 = "25f5399f18d8bf9ce435f85c6bbf671ec4820bc4396b3022cc5dc4bc66303609", + strip_prefix = "rules_closure-0.4.2", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02 - "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", + "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/0.4.2.tar.gz", # 2017-08-30 + "https://github.com/bazelbuild/rules_closure/archive/0.4.2.tar.gz", ], ) diff --git a/tensorflow b/tensorflow index 668db64a5d6..be4e5eff77b 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 668db64a5d612d5f96b5d87772ce6ff6531fc035 +Subproject commit be4e5eff77ba08ca230897cf7c8399c9d0f1884f diff --git a/tf_models b/tf_models index 3bf85a4eddb..42f507f51bd 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 3bf85a4eddb9c56a28cc266ee4aa5604fb4d8334 +Subproject commit 42f507f51bd612a9d5dda01672d9460a68b4914f From a8a8ac542c9e0fadd7eb12ef67c43ba2ae508e62 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 30 Aug 2017 13:15:11 -0800 Subject: [PATCH 0352/8554] Remove logging for resource subtraction overflow, since it creates log spam. (Some code tries subtraction just to see if it would work.) Change: 167047856 --- tensorflow_serving/resources/resource_util.cc | 4 ---- 1 file changed, 4 deletions(-) diff --git a/tensorflow_serving/resources/resource_util.cc b/tensorflow_serving/resources/resource_util.cc index f5a34c17ba8..45ef2bc2db9 100644 --- a/tensorflow_serving/resources/resource_util.cc +++ b/tensorflow_serving/resources/resource_util.cc @@ -335,10 +335,6 @@ bool ResourceUtil::SubtractNormalized(const ResourceAllocation& to_subtract, FindMutableEntry(to_subtract_entry.resource(), base); if (base_entry == nullptr || base_entry->quantity() < to_subtract_entry.quantity()) { - LOG(ERROR) << "Subtracting\n" - << to_subtract.DebugString() << "from\n" - << base->DebugString() - << "would result in a negative quantity"; return false; } const uint64 new_quantity = From ef0cd2a301cfc50dba1e0d85313305dda1f3abd6 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 31 Aug 2017 14:11:48 -0800 Subject: [PATCH 0353/8554] Internal only; CL is a no-op. Change: 167198316 --- tensorflow_serving/servables/tensorflow/BUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index cab925e163a..034b500a075 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -610,10 +610,10 @@ cc_library( "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/apis:regression_proto", "//tensorflow_serving/apis:regressor", - "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:loader_lite", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", - "@org_tensorflow//tensorflow/contrib/session_bundle", - "@org_tensorflow//tensorflow/contrib/session_bundle:signature", + "@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_lite", + "@org_tensorflow//tensorflow/contrib/session_bundle:signature_lite", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", From eea4ccc3dbd9ff41a568cd87a91ade96ccca969f Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 6 Sep 2017 12:50:33 -0800 Subject: [PATCH 0354/8554] Fix ServerCore deadlock involving multiple platforms. Change: 167767693 --- .../core/aspired_versions_manager.h | 10 +++--- .../core/load_servables_fast.cc | 31 ++++++++++++++----- tensorflow_serving/core/load_servables_fast.h | 13 ++++++-- .../model_servers/server_core.cc | 28 +++++++---------- .../model_servers/server_core_test.cc | 22 +++++++++++-- 5 files changed, 72 insertions(+), 32 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 3e62b6c1b49..d8a0e8c9f63 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -50,8 +50,9 @@ namespace internal { class AspiredVersionsManagerTargetImpl; -Status ConnectSourceWithFastInitialLoad( - AspiredVersionsManager* manager, Source>* source, +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, const std::function& wait_until_loaded_fn, uint32 num_threads); } // namespace internal @@ -197,8 +198,9 @@ class AspiredVersionsManager : public Manager, friend class internal::AspiredVersionsManagerTargetImpl; friend class test_util::AspiredVersionsManagerTestAccess; friend class ServerCore; - friend Status internal::ConnectSourceWithFastInitialLoad( - AspiredVersionsManager* manager, Source>* source, + friend Status internal::ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, const std::function& wait_until_loaded_fn, uint32 num_threads); AspiredVersionsManager( diff --git a/tensorflow_serving/core/load_servables_fast.cc b/tensorflow_serving/core/load_servables_fast.cc index 653d908b225..9a7eb45fb8b 100644 --- a/tensorflow_serving/core/load_servables_fast.cc +++ b/tensorflow_serving/core/load_servables_fast.cc @@ -28,13 +28,16 @@ namespace serving { namespace internal { -Status ConnectSourceWithFastInitialLoad( - AspiredVersionsManager* manager, Source>* source, +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, const std::function& wait_until_loaded_fn, const uint32 num_threads) { const uint32 prev_num_load_threads = manager->num_load_threads(); manager->SetNumLoadThreads(num_threads); - ConnectSourceToTarget(source, manager); + for (Source>* source : sources) { + ConnectSourceToTarget(source, manager); + } const Status status = wait_until_loaded_fn(); manager->SetNumLoadThreads(prev_num_load_threads); return status; @@ -45,17 +48,29 @@ Status ConnectSourceWithFastInitialLoad( Status ConnectSourceWithFastInitialLoad( AspiredVersionsManager* manager, Source>* source, ServableStateMonitor* servable_state_monitor, - const std::vector& servables, const uint32 num_threads) { - return internal::ConnectSourceWithFastInitialLoad( - manager, source, + const std::vector& initial_servables, + const uint32 num_threads) { + return ConnectSourcesWithFastInitialLoad(manager, {source}, + servable_state_monitor, + initial_servables, num_threads); +} + +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, + ServableStateMonitor* servable_state_monitor, + const std::vector& initial_servables, + const uint32 num_threads) { + return internal::ConnectSourcesWithFastInitialLoad( + manager, sources, [&]() { std::map states_reached; const bool all_servables_available = servable_state_monitor->WaitUntilServablesReachState( - servables, ServableState::ManagerState::kAvailable, + initial_servables, ServableState::ManagerState::kAvailable, &states_reached); if (!all_servables_available) { - string message = "Some models did not become available: {"; + string message = "Some servables did not become available: {"; for (const auto& id_and_state : states_reached) { if (id_and_state.second != ServableState::ManagerState::kAvailable) { diff --git a/tensorflow_serving/core/load_servables_fast.h b/tensorflow_serving/core/load_servables_fast.h index 8a5ca0df617..bdbfde2d441 100644 --- a/tensorflow_serving/core/load_servables_fast.h +++ b/tensorflow_serving/core/load_servables_fast.h @@ -38,14 +38,23 @@ Status ConnectSourceWithFastInitialLoad( const std::vector& initial_servables, uint32 num_threads = 4 * port::NumSchedulableCPUs()); +// Like ConnectSourceWithFastInitialLoad(), but with multiple sources. +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, + ServableStateMonitor* servable_state_monitor, + const std::vector& initial_servables, + uint32 num_threads = 4 * port::NumSchedulableCPUs()); + //// // Implementation detail. API readers may skip. /// namespace internal { -Status ConnectSourceWithFastInitialLoad( - AspiredVersionsManager* manager, Source>* source, +Status ConnectSourcesWithFastInitialLoad( + AspiredVersionsManager* manager, + std::vector>*> sources, const std::function& wait_until_loaded_fn, uint32 num_threads); } // namespace internal diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index db07a923d28..6101870a71b 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -571,28 +571,24 @@ Status ServerCore::CreateAdapters(SourceAdapters* adapters) const { Status ServerCore::ConnectAdaptersToManagerAndAwaitModelLoads( SourceAdapters* adapters) { - std::map> models_by_platform; + std::vector models_to_await; for (const ModelConfig& model_config : config_.model_config_list().config()) { - string platform; - TF_RETURN_IF_ERROR(GetPlatform(model_config, &platform)); - models_by_platform[platform].push_back( - ServableRequest::Latest(model_config.name())); + models_to_await.push_back(ServableRequest::Latest(model_config.name())); } + std::vector>*> adapter_list; for (auto& entry : adapters->platform_adapters) { - const string& platform = entry.first; - StoragePathSourceAdapter* adapter = entry.second.get(); + adapter_list.push_back(entry.second.get()); + } + adapter_list.push_back(adapters->error_adapter.get()); - const Status status = ConnectSourceWithFastInitialLoad( - manager_.get(), adapter, servable_state_monitor_.get(), - models_by_platform[platform], options_.num_initial_load_threads); - if (!status.ok()) { - VLOG(1) << "Unable to ConnectSourceWithFastInitialLoad due to: " - << status; - return status; - } + const Status status = ConnectSourcesWithFastInitialLoad( + manager_.get(), adapter_list, servable_state_monitor_.get(), + models_to_await, options_.num_initial_load_threads); + if (!status.ok()) { + VLOG(1) << "Unable to ConnectSourcesWithFastInitialLoad due to: " << status; + return status; } - ConnectSourceToTarget(adapters->error_adapter.get(), manager_.get()); return Status::OK(); } diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 0941594652d..63611efe288 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/random/random.h" +#include "tensorflow/core/lib/strings/stringprintf.h" #include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/apis/predict.pb.h" #include "tensorflow_serving/core/servable_handle.h" @@ -287,7 +289,7 @@ TEST_P(ServerCoreTest, ErroringModel) { Status status = ServerCore::Create(std::move(options), &server_core); EXPECT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), - ::testing::HasSubstr("Some models did not become available")); + ::testing::HasSubstr("Some servables did not become available")); } TEST_P(ServerCoreTest, IllegalReconfigurationToCustomConfig) { @@ -364,8 +366,24 @@ TEST_P(ServerCoreTest, UnknownModelPlatform) { // Creates a model name that incorporates 'platform'. Useful for tests that have // one model for a given platform. +// +// The model names contain a random element, to vary the model name sort order +// independently from the platform name order. This is to get regression +// coverage of b/65363800. If called twice for a given platform, always returns +// the same model name. string ModelNameForPlatform(const string& platform) { - return strings::StrCat("model_for_", platform); + static std::map* platform_to_model_map = [] { + return new std::map(); + }(); + auto it = platform_to_model_map->find(platform); + if (it != platform_to_model_map->end()) { + return it->second; + } + const string random = strings::Printf("%llu", random::New64()); + const string model_name = + strings::StrCat("model_", random, "_for_", platform); + (*platform_to_model_map)[platform] = model_name; + return model_name; } // Builds a ModelSpec with a model named 'ModelNameForPlatform(platform)' and From d2c5b658f5a1b00692be43bed25cd726222c1695 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 6 Sep 2017 15:55:24 -0800 Subject: [PATCH 0355/8554] Ensure using "path" as a URI will keep working. Change: 167793848 --- .../model_servers/server_core.cc | 37 +++++++++++-------- .../model_servers/server_core_test.cc | 23 ++++++------ .../test_util/server_core_test_util.cc | 7 ++++ .../test_util/server_core_test_util.h | 34 +++++++++++++++-- 4 files changed, 72 insertions(+), 29 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 6101870a71b..d821923d9a4 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -19,6 +19,7 @@ limitations under the License. #include "google/protobuf/any.pb.h" #include "google/protobuf/wrappers.pb.h" +#include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/io/path.h" #include "tensorflow/core/platform/logging.h" #include "tensorflow_serving/core/load_servables_fast.h" @@ -68,6 +69,13 @@ Status GetPlatform(const ModelConfig& model_config, string* platform) { return Status::OK(); } +// Determines whether a URI is just a relative path. +bool UriIsRelativePath(StringPiece uri) { + StringPiece scheme, host, path; + io::ParseURI(uri, &scheme, &host, &path); + return scheme.empty() && host.empty() && !io::IsAbsolutePath(path); +} + // Returns an error if 'config_list' is invalid in some way, e.g. a model name // appearing multiple times. Status ValidateModelConfigList(const ModelConfigList& config_list, @@ -84,29 +92,30 @@ Status ValidateModelConfigList(const ModelConfigList& config_list, } // Base-paths are either all relative, or all absolute. - using ::tensorflow::io::IsAbsolutePath; - using ::tensorflow::io::JoinPath; + // WARNING: abuse of terminology! These "paths" may be URIs :-( if (options.model_config_list_root_dir) { - // All paths must be relative. - if (!IsAbsolutePath(*options.model_config_list_root_dir)) { - return errors::InvalidArgument(strings::StrCat( - "Expected non-empty absolute path; got model_config_list_root_dir=", - *options.model_config_list_root_dir)); + // All base-paths must be relative. + if (UriIsRelativePath(*options.model_config_list_root_dir)) { + return errors::InvalidArgument( + strings::StrCat("Expected non-empty absolute path or URI; got " + "model_config_list_root_dir=", + *options.model_config_list_root_dir)); } for (const ModelConfig& config : config_list.config()) { - if (IsAbsolutePath(config.base_path())) { + if (!UriIsRelativePath(config.base_path())) { return errors::InvalidArgument(strings::StrCat( "Expected model ", config.name(), " to have a relative path; got base_path()=", config.base_path())); } } } else { - // All paths must be absolute. + // All base-paths must be absolute. for (const ModelConfig& config : config_list.config()) { - if (!IsAbsolutePath(config.base_path())) { + if (UriIsRelativePath(config.base_path())) { return errors::InvalidArgument(strings::StrCat( "Expected model ", config.name(), - " to have an absolute path; got base_path()=", config.base_path())); + " to have an absolute path or URI; got base_path()=", + config.base_path())); } } } @@ -188,14 +197,12 @@ std::set NewModelNamesInSourceConfig( // It is assumed that initially, all the base_path fields are relative. Status UpdateModelConfigListRelativePaths( const string& model_config_list_root_dir, ModelConfigList* config_list) { - using ::tensorflow::io::IsAbsolutePath; - using ::tensorflow::io::JoinPath; std::vector updated_paths; updated_paths.reserve(config_list->config_size()); for (const ModelConfig& config : config_list->config()) { updated_paths.emplace_back( - JoinPath(model_config_list_root_dir, config.base_path())); - if (!IsAbsolutePath(updated_paths.back())) { + io::JoinPath(model_config_list_root_dir, config.base_path())); + if (UriIsRelativePath(updated_paths.back())) { return errors::InvalidArgument(strings::StrCat( "Expected model ", config.name(), " with updated base_path = JoinPath(", model_config_list_root_dir, diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index 63611efe288..c251529f021 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -192,7 +192,6 @@ class RelativePathsServerCoreTest : public ServerCoreTest { string* optional_config_list_root_dir = nullptr) { using ::tensorflow::io::Basename; using ::tensorflow::io::Dirname; - using ::tensorflow::io::IsAbsolutePath; ModelServerConfig result = GetTestModelServerConfigForFakePlatform(); const string model_name = result.model_config_list().config(0).name(); @@ -200,9 +199,6 @@ class RelativePathsServerCoreTest : public ServerCoreTest { ModelConfig& relative = *result.mutable_model_config_list()->mutable_config(0); relative.set_name(strings::StrCat(model_name, "_relative")); - CHECK(IsAbsolutePath(relative.base_path())) - << "relative.base_path()=" << relative.base_path() - << " must start as an absolute path for this unit-test to work."; const string dirname = Dirname(relative.base_path()).ToString(); const string basename = Basename(relative.base_path()).ToString(); CHECK(!dirname.empty()); @@ -436,8 +432,9 @@ string ServableDataForPlatform(const string& root_path, const string& platform, } TEST_P(ServerCoreTest, MultiplePlatforms) { - const string root_path = io::JoinPath( - testing::TmpDir(), strings::StrCat("MultiplePlatforms_", GetTestType())); + const string root_path = + io::JoinPath(testing::TmpDir(), + strings::StrCat("MultiplePlatforms_", GetNameForTestCase())); TF_ASSERT_OK(Env::Default()->CreateDir(root_path)); // Create a ServerCore with two platforms, and one model for each platform. @@ -470,8 +467,8 @@ TEST_P(ServerCoreTest, MultiplePlatforms) { TEST_P(ServerCoreTest, MultiplePlatformsWithConfigChange) { const string root_path = io::JoinPath( - testing::TmpDir(), - strings::StrCat("MultiplePlatformsWithConfigChange_", GetTestType())); + testing::TmpDir(), strings::StrCat("MultiplePlatformsWithConfigChange_", + GetNameForTestCase())); TF_ASSERT_OK(Env::Default()->CreateDir(root_path)); // Create config for three platforms, and one model per platform. @@ -533,7 +530,7 @@ TEST_P(ServerCoreTest, MultiplePlatformsWithConfigChange) { TEST_P(ServerCoreTest, IllegalToChangeModelPlatform) { const string root_path = io::JoinPath( testing::TmpDir(), - strings::StrCat("IllegalToChangeModelPlatform_", GetTestType())); + strings::StrCat("IllegalToChangeModelPlatform_", GetNameForTestCase())); TF_ASSERT_OK(Env::Default()->CreateDir(root_path)); ServerCore::Options options = GetDefaultOptions(); @@ -631,11 +628,15 @@ TEST_P(ServerCoreTest, RequestLoggingOn) { INSTANTIATE_TEST_CASE_P( TestType, ServerCoreTest, - ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES))); + ::testing::Combine( + ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES)), + ::testing::Bool())); INSTANTIATE_TEST_CASE_P( TestType, RelativePathsServerCoreTest, - ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES))); + ::testing::Combine( + ::testing::Range(0, static_cast(ServerCoreTest::NUM_TEST_TYPES)), + ::testing::Bool())); } // namespace } // namespace serving diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc index de523763537..464b5e02ba5 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.cc +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/test_util/server_core_test_util.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" #include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/core/test_util/fake_loader_source_adapter.h" #include "tensorflow_serving/model_servers/model_platform_types.h" @@ -102,6 +103,9 @@ ServerCoreTest::GetTestModelServerConfigForTensorflowPlatform() { model->set_base_path(test_util::TestSrcDirPath( "/servables/tensorflow/testdata/half_plus_two")); } + if (PrefixPathsWithURIScheme()) { + model->set_base_path(io::CreateURI("file", "", model->base_path())); + } model->set_model_platform(kTensorFlowModelPlatform); return config; } @@ -112,6 +116,9 @@ void ServerCoreTest::SwitchToHalfPlusTwoWith2Versions( auto model = config->mutable_model_config_list()->mutable_config(0); model->set_base_path(test_util::TestSrcDirPath( "/servables/tensorflow/testdata/half_plus_two_2_versions")); + if (PrefixPathsWithURIScheme()) { + model->set_base_path(io::CreateURI("file", "", model->base_path())); + } } ServerCore::Options ServerCoreTest::GetDefaultOptions() { diff --git a/tensorflow_serving/model_servers/test_util/server_core_test_util.h b/tensorflow_serving/model_servers/test_util/server_core_test_util.h index c936386ec9e..5150f59ce0a 100644 --- a/tensorflow_serving/model_servers/test_util/server_core_test_util.h +++ b/tensorflow_serving/model_servers/test_util/server_core_test_util.h @@ -16,6 +16,8 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ #define TENSORFLOW_SERVING_MODEL_SERVERS_TEST_UTIL_SERVER_CORE_TEST_UTIL_H_ +#include + #include #include "tensorflow_serving/core/servable_id.h" #include "tensorflow_serving/model_servers/server_core.h" @@ -33,7 +35,7 @@ constexpr char kFakePlatform[] = "fake_servable"; // ServerCoreTest is parameterized based on the TestType enum defined below. // TODO(b/32248363): remove the parameter and TestType after we switch Model // Server to Saved Model. -class ServerCoreTest : public ::testing::TestWithParam { +class ServerCoreTest : public ::testing::TestWithParam> { public: // The parameter of this test. enum TestType { @@ -47,6 +49,19 @@ class ServerCoreTest : public ::testing::TestWithParam { NUM_TEST_TYPES, }; + static string GetNameOfTestType(int test_type) { + switch (static_cast(test_type)) { + case SESSION_BUNDLE: + return "SESSION_BUNDLE"; + case SAVED_MODEL_BACKWARD_COMPATIBILITY: + return "SAVED_MODEL_BACKWARD_COMPATIBILITY"; + case SAVED_MODEL: + return "SAVED_MODEL"; + default: + return "unknown"; + } + } + protected: // Returns ModelServerConfig that contains test model for the fake platform. ModelServerConfig GetTestModelServerConfigForFakePlatform(); @@ -75,8 +90,21 @@ class ServerCoreTest : public ::testing::TestWithParam { return CreateServerCore(config, GetDefaultOptions(), server_core); } - // Returns test type. This is the parameter of this test. - TestType GetTestType() { return static_cast(GetParam()); } + // Returns test type. + // This is the first parameter of this test. + TestType GetTestType() { + return static_cast(std::get<0>(GetParam())); + } + + // Returns whether to assume paths are URIs. + // This is the second parameter of this test. + bool PrefixPathsWithURIScheme() { return std::get<1>(GetParam()); } + + // Returns a string corresponding to the parameterized test-case. + string GetNameForTestCase() { + return GetNameOfTestType(GetTestType()) + "_" + + (PrefixPathsWithURIScheme() ? "URI" : "Path"); + } }; // Creates a ServerCore object with the supplied options. From af6acd44d47a9077e7bee5d8e2f1a56d7d97d4df Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 7 Sep 2017 15:00:59 -0800 Subject: [PATCH 0356/8554] Update inception codelab export model_base_path now that relative paths are disallowed. Change: 167925408 --- tensorflow_serving/g3doc/serving_inception.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 661a8ce687b..c2546507d73 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -79,10 +79,10 @@ root@c97d8e820ced:/serving# curl -O http://download.tensorflow.org/models/image/ root@c97d8e820ced:/serving# tar xzf inception-v3-2016-03-01.tar.gz root@c97d8e820ced:/serving# ls inception-v3 README.txt checkpoint model.ckpt-157585 -root@c97d8e820ced:/serving# bazel-bin/tensorflow_serving/example/inception_saved_model --checkpoint_dir=inception-v3 --output_dir=inception-export +root@c97d8e820ced:/serving# bazel-bin/tensorflow_serving/example/inception_saved_model --checkpoint_dir=inception-v3 --output_dir=/tmp/inception-export Successfully loaded model from inception-v3/model.ckpt-157585 at step=157585. -Successfully exported model to inception-export -root@c97d8e820ced:/serving# ls inception-export +Successfully exported model to /tmp/inception-export +root@c97d8e820ced:/serving# ls /tmp/inception-export 1 root@c97d8e820ced:/serving# [Ctrl-p] + [Ctrl-q] ``` @@ -112,7 +112,7 @@ Run the [gRPC]( http://www.grpc.io/) `tensorflow_model_server` in the container. ```shell root@f07eec53fd95:/# cd serving -root@f07eec53fd95:/serving# bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=inception --model_base_path=inception-export &> inception_log & +root@f07eec53fd95:/serving# bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=inception --model_base_path=/tmp/inception-export &> inception_log & [1] 45 ``` From c16040c98470cf56a87ed797f19136963bad3327 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 7 Sep 2017 15:30:07 -0800 Subject: [PATCH 0357/8554] Update setup documentation to include a link to TF's instructions which contain additional packages that may need to be installed. Change: 167929189 --- tensorflow_serving/g3doc/setup.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index e9b0bde1a7a..3c97d89091b 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -58,6 +58,12 @@ sudo apt-get update && sudo apt-get install -y \ zlib1g-dev ``` +The list of packages needed to build TensorFlow changes over time, so if you +encounter any issues, refer TensorFlow's [build +instructions](https://www.tensorflow.org/install/install_sources). Pay +particular attention to `apt-get install` and `pip install` commands which you +may need to run. + ### TensorFlow Serving Python API PIP package To run Python client code without the need to install Bazel, you can install From afde4287ee68976e44c792a7b317849434ee1b3c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 11 Sep 2017 07:42:50 -0800 Subject: [PATCH 0358/8554] Change for internal compatibility. --- tensorflow_serving/model_servers/BUILD | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 8044df9d1a4..fb6686550aa 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -132,14 +132,14 @@ cc_binary( ":model_platform_types", ":platform_config_util", ":server_core", - "@protobuf_archive//:cc_wkt_protos", + "@protobuf_archive//:wrappers", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "@org_tensorflow//tensorflow/core/platform/hadoop:hadoop_file_system", "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:availability_preserving_policy", - "@grpc//:grpc++_unsecure", + "//net/grpc:grpc++", ] + TENSORFLOW_DEPS + SUPPORTED_TENSORFLOW_OPS, ) From 52ea250f2927f0a6eca9918916c59db4f14c88ba Mon Sep 17 00:00:00 2001 From: Li Lao Date: Tue, 12 Sep 2017 10:55:57 -0700 Subject: [PATCH 0359/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 668db64a5d6..1e96d54d9f9 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 668db64a5d612d5f96b5d87772ce6ff6531fc035 +Subproject commit 1e96d54d9f928c4ea4bf0564ef9900f6bd03acd5 diff --git a/tf_models b/tf_models index 3bf85a4eddb..78bb025c6d6 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 3bf85a4eddb9c56a28cc266ee4aa5604fb4d8334 +Subproject commit 78bb025c6d68fc24867ecd1dc478543abecdb197 From 18392ec63616c4fb374038aa9113fe0980517f4c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 12 Sep 2017 16:41:43 -0800 Subject: [PATCH 0360/8554] Fix warning from bazel. Change: 168475927 --- tensorflow_serving/model_servers/BUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index fb6686550aa..77b9adf37ec 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -132,14 +132,14 @@ cc_binary( ":model_platform_types", ":platform_config_util", ":server_core", - "@protobuf_archive//:wrappers", + "@protobuf_archive//:cc_wkt_protos", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "@org_tensorflow//tensorflow/core/platform/hadoop:hadoop_file_system", "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:availability_preserving_policy", - "//net/grpc:grpc++", + "@grpc//:grpc++_unsecure", ] + TENSORFLOW_DEPS + SUPPORTED_TENSORFLOW_OPS, ) @@ -183,7 +183,7 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_deb", "pkg_tar") pkg_tar( name = "tensorflow_model_server_tar", - files = [ + srcs = [ ":tensorflow_model_server", ], package_dir = "/usr/bin", From 3a57b1040e3510dfd8b9b4a48336363f5771a119 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 18 Sep 2017 13:47:31 -0800 Subject: [PATCH 0361/8554] Internal change. Change: 169150073 --- tensorflow_serving/apis/BUILD | 30 ++++++++++++++++++++++++++++++ tensorflow_serving/config/BUILD | 5 +++++ 2 files changed, 35 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 772380a8469..fcc26ed7cdf 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -58,6 +58,11 @@ serving_proto_library( ], ) +go_proto_library( + name = "input_go_proto", + deps = [":input_proto"], +) + serving_proto_library_py( name = "input_proto_py_pb2", srcs = ["input.proto"], @@ -84,6 +89,11 @@ serving_proto_library( ], ) +go_proto_library( + name = "model_go_proto", + deps = [":model_proto"], +) + serving_proto_library_py( name = "model_proto_py_pb2", srcs = ["model.proto"], @@ -109,6 +119,11 @@ serving_proto_library( ], ) +go_proto_library( + name = "predict_go_proto", + deps = [":predict_proto"], +) + serving_proto_library_py( name = "predict_proto_py_pb2", srcs = ["predict.proto"], @@ -136,6 +151,11 @@ serving_proto_library( ], ) +go_proto_library( + name = "prediction_service_go_proto", + deps = [":prediction_service_proto"], +) + py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], @@ -167,6 +187,11 @@ serving_proto_library( ], ) +go_proto_library( + name = "classification_go_proto", + deps = [":classification_proto"], +) + serving_proto_library_py( name = "classification_proto_py_pb2", srcs = ["classification.proto"], @@ -228,6 +253,11 @@ serving_proto_library( ], ) +go_proto_library( + name = "regression_go_proto", + deps = [":regression_proto"], +) + serving_proto_library_py( name = "regression_proto_py_pb2", srcs = ["regression.proto"], diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 486e20e49fb..ab936766a7e 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -32,6 +32,11 @@ serving_proto_library( ], ) +go_proto_library( + name = "model_server_config_go_proto", + deps = [":model_server_config_proto"], +) + serving_proto_library( name = "platform_config_proto", srcs = ["platform_config.proto"], From 72858daa1de33097c4bb835578745e40663026d5 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 20 Sep 2017 09:46:33 -0800 Subject: [PATCH 0362/8554] Made the error message clearer when a predict signature could not be found. Change: 169415489 --- tensorflow_serving/servables/tensorflow/predict_impl.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index a8bd44d292d..c4e52d499ee 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -254,8 +254,8 @@ Status SavedModelPredict(const RunOptions& run_options, ServerCore* core, : request.model_spec().signature_name(); auto iter = bundle->meta_graph_def.signature_def().find(signature_name); if (iter == bundle->meta_graph_def.signature_def().end()) { - return errors::FailedPrecondition( - "Default serving signature key not found."); + return errors::FailedPrecondition(strings::StrCat( + "Serving signature key \"", signature_name, "\" not found.")); } SignatureDef signature = iter->second; From dab115c4975695137b03a2bd696fe2e99558a626 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 20 Sep 2017 17:17:34 -0800 Subject: [PATCH 0363/8554] Internal Change. Change: 169482861 --- tensorflow_serving/model_servers/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 6cad8011c24..413025fa17e 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -137,7 +137,7 @@ tensorflow::Status LoadCustomModelConfig( const ::google::protobuf::Any& any, EventBus* servable_event_bus, UniquePtrWithDeps* manager) { - CHECK(false) // Crash ok + LOG(FATAL) // Crash ok << "ModelServer does not yet support custom model config."; } @@ -385,7 +385,7 @@ int main(int argc, char** argv) { ReadProtoFromFile(batching_parameters_file); } } else if (!batching_parameters_file.empty()) { - CHECK(false) // Crash ok + LOG(FATAL) // Crash ok << "You supplied --batching_parameters_file without " "--enable_batching"; } From ec1a555d84fc75e43fd84acb94543600da67904a Mon Sep 17 00:00:00 2001 From: Neal Wu Date: Fri, 22 Sep 2017 17:18:29 -0800 Subject: [PATCH 0364/8554] Fix broken GitHub links in tensorflow and tensorflow_models resulting from The Great Models Move (a.k.a. the research subfolder) Change: 169763373 --- tensorflow_serving/g3doc/serving_inception.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index c2546507d73..83e6c134237 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -9,7 +9,7 @@ To learn more about TensorFlow Serving, we recommend [TensorFlow Serving advanced tutorial](serving_advanced.md). To learn more about TensorFlow Inception model, we recommend -[Inception in TensorFlow](https://github.com/tensorflow/models/tree/master/inception). +[Inception in TensorFlow](https://github.com/tensorflow/models/tree/master/research/inception). - [Part 0](#part_0_create_a_docker_image) shows how to create a TensorFlow Serving Docker image for deployment From b5b9c95df5a87bb71ce55cb8297576395f0dbd18 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Mon, 25 Sep 2017 18:14:15 -0700 Subject: [PATCH 0365/8554] Internal Change PiperOrigin-RevId: 169991644 --- tensorflow_serving/apis/BUILD | 30 ------------------------------ tensorflow_serving/config/BUILD | 5 ----- 2 files changed, 35 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index fcc26ed7cdf..772380a8469 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -58,11 +58,6 @@ serving_proto_library( ], ) -go_proto_library( - name = "input_go_proto", - deps = [":input_proto"], -) - serving_proto_library_py( name = "input_proto_py_pb2", srcs = ["input.proto"], @@ -89,11 +84,6 @@ serving_proto_library( ], ) -go_proto_library( - name = "model_go_proto", - deps = [":model_proto"], -) - serving_proto_library_py( name = "model_proto_py_pb2", srcs = ["model.proto"], @@ -119,11 +109,6 @@ serving_proto_library( ], ) -go_proto_library( - name = "predict_go_proto", - deps = [":predict_proto"], -) - serving_proto_library_py( name = "predict_proto_py_pb2", srcs = ["predict.proto"], @@ -151,11 +136,6 @@ serving_proto_library( ], ) -go_proto_library( - name = "prediction_service_go_proto", - deps = [":prediction_service_proto"], -) - py_library( name = "prediction_service_proto_py_pb2", srcs = ["prediction_service_pb2.py"], @@ -187,11 +167,6 @@ serving_proto_library( ], ) -go_proto_library( - name = "classification_go_proto", - deps = [":classification_proto"], -) - serving_proto_library_py( name = "classification_proto_py_pb2", srcs = ["classification.proto"], @@ -253,11 +228,6 @@ serving_proto_library( ], ) -go_proto_library( - name = "regression_go_proto", - deps = [":regression_proto"], -) - serving_proto_library_py( name = "regression_proto_py_pb2", srcs = ["regression.proto"], diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index ab936766a7e..486e20e49fb 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -32,11 +32,6 @@ serving_proto_library( ], ) -go_proto_library( - name = "model_server_config_go_proto", - deps = [":model_server_config_proto"], -) - serving_proto_library( name = "platform_config_proto", srcs = ["platform_config.proto"], From 902b7635f0c8e4c11b1f836a88aa68d4528f7975 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 27 Sep 2017 13:22:24 -0800 Subject: [PATCH 0366/8554] Add a resource multiplication utility. Change: 170251639 --- tensorflow_serving/resources/resource_util.cc | 14 +++++ tensorflow_serving/resources/resource_util.h | 8 +++ .../resources/resource_util_test.cc | 57 +++++++++++++++++++ 3 files changed, 79 insertions(+) diff --git a/tensorflow_serving/resources/resource_util.cc b/tensorflow_serving/resources/resource_util.cc index 45ef2bc2db9..4390849ce2e 100644 --- a/tensorflow_serving/resources/resource_util.cc +++ b/tensorflow_serving/resources/resource_util.cc @@ -194,6 +194,11 @@ bool ResourceUtil::Subtract(const ResourceAllocation& to_subtract, return SubtractNormalized(Normalize(to_subtract), base); } +void ResourceUtil::Multiply(uint64 multiplier, ResourceAllocation* base) const { + *base = Normalize(*base); + return MultiplyNormalized(multiplier, base); +} + bool ResourceUtil::Equal(const ResourceAllocation& lhs, const ResourceAllocation& rhs) const { return EqualNormalized(Normalize(lhs), Normalize(rhs)); @@ -350,6 +355,15 @@ bool ResourceUtil::SubtractNormalized(const ResourceAllocation& to_subtract, return true; } +void ResourceUtil::MultiplyNormalized(uint64 multiplier, + ResourceAllocation* base) const { + DCHECK(IsNormalized(*base)); + for (int i = 0; i < base->resource_quantities().size(); ++i) { + ResourceAllocation::Entry* entry = base->mutable_resource_quantities(i); + entry->set_quantity(entry->quantity() * multiplier); + } +} + bool ResourceUtil::EqualNormalized(const ResourceAllocation& lhs, const ResourceAllocation& rhs) const { if (!VerifyValidityInternal(lhs, DCHECKFailOption::kDoDCHECKFail).ok() || diff --git a/tensorflow_serving/resources/resource_util.h b/tensorflow_serving/resources/resource_util.h index b90c880759f..167dfc5b052 100644 --- a/tensorflow_serving/resources/resource_util.h +++ b/tensorflow_serving/resources/resource_util.h @@ -99,6 +99,10 @@ class ResourceUtil { bool Subtract(const ResourceAllocation& to_subtract, ResourceAllocation* base) const; + // Multiplies every resource quantity in 'base' by 'multiplier'. Keeps bound + // and unbound entries separate. + void Multiply(uint64 multiplier, ResourceAllocation* base) const; + // Determines whether two ResourceAllocation objects are identical (modulo // normalization). bool Equal(const ResourceAllocation& lhs, @@ -164,6 +168,10 @@ class ResourceUtil { bool SubtractNormalized(const ResourceAllocation& to_subtract, ResourceAllocation* base) const; + // Like Multiply(), but assumes the input is normalized and produces + // normalized output. + void MultiplyNormalized(uint64 multiplier, ResourceAllocation* base) const; + // Like Equal(), but assumes the input is normalized. bool EqualNormalized(const ResourceAllocation& lhs, const ResourceAllocation& rhs) const; diff --git a/tensorflow_serving/resources/resource_util_test.cc b/tensorflow_serving/resources/resource_util_test.cc index dd78ad62fe3..73d901fd426 100644 --- a/tensorflow_serving/resources/resource_util_test.cc +++ b/tensorflow_serving/resources/resource_util_test.cc @@ -887,6 +887,63 @@ TEST_F(ResourceUtilTest, SubtractBoundAndUnbound) { "} ")); } +TEST_F(ResourceUtilTest, MultiplyEmpty) { + auto base = CreateProto(""); + util_.Multiply(2, &base); + EXPECT_THAT(base, EqualsProto("")); +} + +TEST_F(ResourceUtilTest, MultiplyBasic) { + auto base = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 300 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 8 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " kind: 'ram' " + " } " + " quantity: 16 " + "} "); + util_.Multiply(2, &base); + EXPECT_THAT(base, EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 600 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 16 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " kind: 'ram' " + " } " + " quantity: 32 " + "} ")); +} + TEST_F(ResourceUtilTest, Equal) { const std::vector values = { CreateProto(""), From 5ee9a6adcb23689fe206bab9c471324fea1a33b3 Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Thu, 28 Sep 2017 15:28:10 -0700 Subject: [PATCH 0367/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 1e96d54d9f9..4270c4a00bb 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 1e96d54d9f928c4ea4bf0564ef9900f6bd03acd5 +Subproject commit 4270c4a00bb90d97418df5b0e9b6b3e148a72a1e diff --git a/tf_models b/tf_models index 78bb025c6d6..d43c736eae2 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 78bb025c6d68fc24867ecd1dc478543abecdb197 +Subproject commit d43c736eae267f98061b2cc9aa4f1936e10e2c4e From 55b8dd7f31676e1f0c56188358e0046b52169349 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 28 Sep 2017 16:18:29 -0700 Subject: [PATCH 0368/8554] Update BUILD --- tensorflow_serving/model_servers/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 77b9adf37ec..8044df9d1a4 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -183,7 +183,7 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_deb", "pkg_tar") pkg_tar( name = "tensorflow_model_server_tar", - srcs = [ + files = [ ":tensorflow_model_server", ], package_dir = "/usr/bin", From 169c28d3349b4c874ee2a022f2c32a7c269ceaa7 Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Thu, 28 Sep 2017 18:07:34 -0700 Subject: [PATCH 0369/8554] Sync submodules. --- tensorflow | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow b/tensorflow index 4270c4a00bb..0b0d3c12ace 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 4270c4a00bb90d97418df5b0e9b6b3e148a72a1e +Subproject commit 0b0d3c12ace80381f4a44365d30275a9a262609b From 5283acb26525c4d7aa3765b02438df1dac6f81c5 Mon Sep 17 00:00:00 2001 From: Justine Tunney Date: Tue, 3 Oct 2017 14:50:58 -0800 Subject: [PATCH 0370/8554] Update Closure Rules dependency This should hopefully avoid any potential build break due to breaking changes made in Bazel 0.6.0. Change: 170927986 --- WORKSPACE | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index fcf6cdb503a..4b5ab706213 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -9,18 +9,20 @@ local_repository( # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. http_archive( name = "io_bazel_rules_closure", - sha256 = "bc41b80486413aaa551860fc37471dbc0666e1dbb5236fb6177cb83b0c105846", - strip_prefix = "rules_closure-dec425a4ff3faf09a56c85d082e4eed05d8ce38f", + sha256 = "110fe68753413777944b473c25eed6368c4a0487cee23a7bac1b13cc49d3e257", + strip_prefix = "rules_closure-4af89ef1db659eb41f110df189b67d4cf14073e1", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", # 2017-06-02 - "https://github.com/bazelbuild/rules_closure/archive/dec425a4ff3faf09a56c85d082e4eed05d8ce38f.tar.gz", + "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", # 2017-08-28 ], ) # Please add all new TensorFlow Serving dependencies in workspace.bzl. -load('//tensorflow_serving:workspace.bzl', 'tf_serving_workspace') +load("//tensorflow_serving:workspace.bzl", "tf_serving_workspace") + tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:workspace.bzl", "check_version") + check_version("0.4.5") From 2bf74c59e2943f991da5de49a1142d85b77e6eb8 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 10 Oct 2017 11:41:17 -0800 Subject: [PATCH 0371/8554] Update Predict.proto comments to remove references to deprecated generic signatures. Change: 171719029 --- tensorflow_serving/apis/BUILD | 2 +- tensorflow_serving/apis/predict.proto | 10 ++++------ 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 772380a8469..24391f23949 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -152,7 +152,7 @@ py_library( serving_go_grpc_library( name = "prediction_service_grpc", srcs = [":prediction_service_proto"], - deps = [":prediction_service_proto"], + deps = [":prediction_service_go_proto"], ) serving_proto_library( diff --git a/tensorflow_serving/apis/predict.proto b/tensorflow_serving/apis/predict.proto index 97011de793d..4d3f4a765a2 100644 --- a/tensorflow_serving/apis/predict.proto +++ b/tensorflow_serving/apis/predict.proto @@ -15,16 +15,14 @@ message PredictRequest { // Input tensors. // Names of input tensor are alias names. The mapping from aliases to real - // input tensor names is expected to be stored as named generic signature - // under the key "inputs" in the model export. - // Each alias listed in a generic signature named "inputs" should be provided - // exactly once in order to run the prediction. + // input tensor names is stored in the SavedModel export as a prediction + // SignatureDef under the 'inputs' field. map inputs = 2; // Output filter. // Names specified are alias names. The mapping from aliases to real output - // tensor names is expected to be stored as named generic signature under - // the key "outputs" in the model export. + // tensor names is stored in the SavedModel export as a prediction + // SignatureDef under the 'outputs' field. // Only tensors specified here will be run/fetched and returned, with the // exception that when none is specified, all tensors specified in the // named signature will be run/fetched and returned. From 75e95416b0c649bf2b4f5652d886f3d20d13999a Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 11 Oct 2017 15:17:31 -0800 Subject: [PATCH 0372/8554] Speed up integration test from ~70s to ~8s. Change: 171891229 --- .../model_servers/tensorflow_model_server_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 45145a3cb66..bfff5de79ad 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -386,7 +386,8 @@ def testBadModelConfig(self): self.RunServerWithModelConfigFile( PickUnusedPort(), self._GetBadModelConfigFile(), - pipe=subprocess.PIPE) + pipe=subprocess.PIPE, + wait_for_server_ready=False) error_message = ( 'Invalid protobuf file: \'%s\'') % self._GetBadModelConfigFile() From 25bad1e486cb0dfd7af9d83c583d52bc3b171095 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 11 Oct 2017 16:35:04 -0700 Subject: [PATCH 0373/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 0b0d3c12ace..1ad5e692e2f 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 0b0d3c12ace80381f4a44365d30275a9a262609b +Subproject commit 1ad5e692e2fc218ca0b2a9a461c19762fdc9674b diff --git a/tf_models b/tf_models index d43c736eae2..b8e5a44bec0 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit d43c736eae267f98061b2cc9aa4f1936e10e2c4e +Subproject commit b8e5a44bec01d25cec6e5a9b3c49a54d6bb9897f From 41d366140d07e6b7723bacb6e61469c111b6ca61 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 11 Oct 2017 18:42:52 -0800 Subject: [PATCH 0374/8554] Update inception model path Change: 171910161 --- tensorflow_serving/workspace.bzl | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index a465eb7f137..6719ae6d92d 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -8,17 +8,17 @@ load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') # as a submodule, it'll likely be '__workspace_dir__ + "/serving"' def tf_serving_workspace(): native.new_local_repository( - name = "inception_model", - path = "tf_models/inception", - build_file = "tf_models/inception/inception/BUILD", + name = "inception_model", + path = "tf_models/research/inception", + build_file = "tf_models/research/inception/inception/BUILD", ) tf_workspace(path_prefix = "", tf_repo_name = "org_tensorflow") # ===== gRPC dependencies ===== native.bind( - name = "libssl", - actual = "@boringssl//:ssl", + name = "libssl", + actual = "@boringssl//:ssl", ) native.bind( From 3f1bfb1675e82c00c326024ffe3db3dc2b25e10f Mon Sep 17 00:00:00 2001 From: Johnny Chan Date: Wed, 18 Oct 2017 17:18:52 +0100 Subject: [PATCH 0375/8554] use bazel 0.5.4 in Dockerfile (#620) * use bazel 0.5.4 in Dockerfile * use bazel 0.5.4 * use bazel 0.5.4 * use bazel 0.5.4 --- WORKSPACE | 2 +- tensorflow_serving/g3doc/setup.md | 8 ++++---- tensorflow_serving/tools/docker/Dockerfile.devel | 2 +- tensorflow_serving/tools/docker/Dockerfile.devel-gpu | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 4b5ab706213..431829ff22f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -25,4 +25,4 @@ tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:workspace.bzl", "check_version") -check_version("0.4.5") +check_version("0.5.4") diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index e786db4ebf9..404fbaf7833 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -6,7 +6,7 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel (only if compiling source code) -TensorFlow Serving requires Bazel 0.4.5 or higher. You can find the Bazel +TensorFlow Serving requires Bazel 0.5.4 or higher. You can find the Bazel installation instructions [here](http://bazel.build/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the @@ -14,13 +14,13 @@ following steps: 1. Download the relevant binary from [here](https://github.com/bazelbuild/bazel/releases). - Let's say you downloaded bazel-0.4.5-installer-linux-x86_64.sh. You would + Let's say you downloaded bazel-0.5.4-installer-linux-x86_64.sh. You would execute: 
     cd ~/Downloads
-    chmod +x bazel-0.4.5-installer-linux-x86_64.sh
-    ./bazel-0.4.5-installer-linux-x86_64.sh --user
+    chmod +x bazel-0.5.4-installer-linux-x86_64.sh
+    ./bazel-0.5.4-installer-linux-x86_64.sh --user
2. Set up your environment. Put this in your ~/.bashrc. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index dea056ebe6e..eed44caa601 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -34,7 +34,7 @@ RUN pip install mock grpcio ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.5.1 +ENV BAZEL_VERSION 0.5.4 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu index caf9e35c972..9b351de876c 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -54,7 +54,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.4.5 +ENV BAZEL_VERSION 0.5.4 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ From e8930f060957fc8f5f7efdeadd26ce67ef1861d9 Mon Sep 17 00:00:00 2001 From: Vitaly Li Date: Sat, 21 Oct 2017 05:36:14 -0700 Subject: [PATCH 0376/8554] Working GPU docker: Nvidia driver 384.81;Cuda8 CuDNN6 bazel 5.4 (#619) --- .../tools/docker/Dockerfile.devel-gpu | 18 ++++++------------ 1 file changed, 6 insertions(+), 12 deletions(-) diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu index 9b351de876c..ed1781fa631 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -20,9 +20,11 @@ RUN apt-get update && apt-get install -y \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py && \ - rm get-pip.py +RUN apt-get install python-pip python-dev build-essential + +#RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ +# python get-pip.py && \ +# rm get-pip.py # Set up grpc @@ -85,14 +87,6 @@ RUN mkdir /usr/lib/x86_64-linux-gnu/include/ && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/local/cuda/lib64/libcudnn.so && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.6 /usr/local/cuda/lib64/libcudnn.so.6 -# Fix from https://github.com/tensorflow/serving/issues/327#issuecomment-282207825 -WORKDIR / -RUN git clone https://github.com/NVIDIA/nccl.git && \ - cd nccl/ && \ - make CUDA_HOME=/usr/local/cuda && \ - make install && \ - mkdir -p /usr/local/include/external/nccl_archive/src && \ - ln -s /usr/local/include/nccl.h /usr/local/include/external/nccl_archive/src/nccl.h # Configure Tensorflow to use the GPU WORKDIR /serving/tensorflow @@ -106,4 +100,4 @@ RUN bazel build -c opt --config=cuda \ cp bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server /usr/local/bin/ && \ bazel clean --expunge -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] From 28b8c75540d1dc3c80afb31380f9e0445b8de629 Mon Sep 17 00:00:00 2001 From: Ales Tamchyna Date: Mon, 23 Oct 2017 17:00:01 +0200 Subject: [PATCH 0377/8554] check for UnpackTo return value when parsing platform_config file (#614) * check for UnpackTo return value when parsing platform_config file * include DebugString in error message * more general error message to be future-proof --- tensorflow_serving/util/class_registration.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h index 8ddd78c6926..67640568ad8 100644 --- a/tensorflow_serving/util/class_registration.h +++ b/tensorflow_serving/util/class_registration.h @@ -266,7 +266,10 @@ class ClassRegistry { protobuf::MessageFactory::generated_factory() ->GetPrototype(descriptor) ->New()); - any_config.UnpackTo(config.get()); + if (!any_config.UnpackTo(config.get())) { + return errors::InvalidArgument("Malformed content of Any: ", + any_config.DebugString()); + } return Create(*config, std::forward(args)..., result); } From 8ab0e9aeaff33d44798d6bc429195012483d48cb Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 27 Oct 2017 11:48:03 -0800 Subject: [PATCH 0378/8554] Merge changes from github. Change: 173706180 --- WORKSPACE | 2 +- tensorflow_serving/core/servable_handle.h | 1 - tensorflow_serving/g3doc/leftnav_files | 1 + tensorflow_serving/g3doc/serving_basic.md | 123 ++++++++++-------- tensorflow_serving/g3doc/setup.md | 27 ++-- tensorflow_serving/model_servers/BUILD | 2 +- .../tools/docker/Dockerfile.devel | 2 +- .../tools/docker/Dockerfile.devel-gpu | 16 ++- tensorflow_serving/util/class_registration.h | 5 +- 9 files changed, 96 insertions(+), 83 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 4b5ab706213..431829ff22f 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -25,4 +25,4 @@ tf_serving_workspace() # Specify the minimum required bazel version. load("@org_tensorflow//tensorflow:workspace.bzl", "check_version") -check_version("0.4.5") +check_version("0.5.4") diff --git a/tensorflow_serving/core/servable_handle.h b/tensorflow_serving/core/servable_handle.h index 1067f6ac53e..8f4d5b2c19a 100644 --- a/tensorflow_serving/core/servable_handle.h +++ b/tensorflow_serving/core/servable_handle.h @@ -22,7 +22,6 @@ limitations under the License. #include #include "tensorflow_serving/core/loader.h" -#include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/util/any_ptr.h" namespace tensorflow { diff --git a/tensorflow_serving/g3doc/leftnav_files b/tensorflow_serving/g3doc/leftnav_files index d7d658c8677..272cbf0eba0 100644 --- a/tensorflow_serving/g3doc/leftnav_files +++ b/tensorflow_serving/g3doc/leftnav_files @@ -7,4 +7,5 @@ serving_advanced.md serving_inception.md custom_servable.md custom_source.md +signature_defs.md docker.md \ No newline at end of file diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index cbad3ae9047..e35cbc59da1 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -84,63 +84,72 @@ sub-directory under the given path. You can add meta graph and variables to the builder using `SavedModelBuilder.add_meta_graph_and_variables()` with the following arguments: -* `sess` is the TensorFlow session that holds the trained model you are - exporting. - -* `tags` is the set of tags with which to save the meta graph. In this case, - since we intend to use the graph in serving, we use the `serve` tag from - predefined SavedModel tag constants. For more details, see [tag_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) - and [related TensorFlow API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/tag_constants). - -* `signature_def_map` specifies the map of user-supplied key for a - **signature** to a tensorflow::SignatureDef to add to the meta graph. - Signature specifies what type of model is being exported, and the - input/output tensors to bind to when running inference. - - The special signature key `serving_default` specifies the default serving - signature. The default serving signature def key, along with other constants - related to signatures, are defined as part of SavedModel signature constants. - For more details, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) - and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). - - Further, to help build signature defs easily, the SavedModel API provides - [signature def utils](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_def_utils). - Specifically, in the `mnist_saved_model.py` code snippet above, we use - `signature_def_utils.build_signature_def()` to build `predict_signature` and - `classification_signature`. - - As an example for how `predict_signature` is defined, the util takes the - following arguments: - - * `inputs={'images': tensor_info_x}` specifies the input tensor info. - - * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. - - Note that `tensor_info_x` and `tensor_info_y` have the structure of - `tensorflow::TensorInfo` protocol buffer defined [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto). - To easily build tensor infos, the TensorFlow SavedModel API also provides - [utils.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py), - with [related TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/utils). - - Also, note that `images` and `scores` are tensor alias names. They can be - whatever unique strings you want, and they will become the logical names - of tensor `x` and `y` that you refer to for tensor binding when sending - prediction requests later. - - For instance, if `x` refers to the tensor with name 'long_tensor_name_foo' - and `y` refers to the tensor with name 'generated_tensor_name_bar', - `builder` will store tensor logical name to real name mapping - ('images' -> 'long_tensor_name_foo') and ('scores' -> 'generated_tensor_name_bar'). - This allows the user to refer to these tensors with their logical names - when running inference. - - * `method_name` is the method used for the inference. For Prediction - requests, it should be set to `tensorflow/serving/predict`. For other - method names, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) - and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). - - In addition to the description above, documentation related to signature def - structure and how to set up them up can be found [here](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/g3doc/signature_defs.md). +* `sess` is the TensorFlow session that holds the trained model you are + exporting. + +* `tags` is the set of tags with which to save the meta graph. In this case, + since we intend to use the graph in serving, we use the `serve` tag from + predefined SavedModel tag constants. For more details, see + [tag_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) + and [related TensorFlow API + documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/tag_constants). + +* `signature_def_map` specifies the map of user-supplied key for a + **signature** to a tensorflow::SignatureDef to add to the meta graph. + Signature specifies what type of model is being exported, and the + input/output tensors to bind to when running inference. + + The special signature key `serving_default` specifies the default serving + signature. The default serving signature def key, along with other constants + related to signatures, are defined as part of SavedModel signature + constants. For more details, see + [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) + and related [TensorFlow 1.0 API + documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). + + Further, to help build signature defs easily, the SavedModel API provides + [signature def + utils](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_def_utils). + Specifically, in the `mnist_saved_model.py` code snippet above, we use + `signature_def_utils.build_signature_def()` to build `predict_signature` and + `classification_signature`. + + As an example for how `predict_signature` is defined, the util takes the + following arguments: + + * `inputs={'images': tensor_info_x}` specifies the input tensor info. + + * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. + + * `method_name` is the method used for the inference. For Prediction + requests, it should be set to `tensorflow/serving/predict`. For other + method names, see + [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) + and related [TensorFlow 1.0 API + documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). + +Note that `tensor_info_x` and `tensor_info_y` have the structure of +`tensorflow::TensorInfo` protocol buffer defined +[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto). +To easily build tensor infos, the TensorFlow SavedModel API also provides +[utils.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py), +with [related TensorFlow 1.0 API +documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/utils). + +Also, note that `images` and `scores` are tensor alias names. They can be +whatever unique strings you want, and they will become the logical names of +tensor `x` and `y` that you refer to for tensor binding when sending prediction +requests later. + +For instance, if `x` refers to the tensor with name 'long_tensor_name_foo' and +`y` refers to the tensor with name 'generated_tensor_name_bar', `builder` will +store tensor logical name to real name mapping ('images' -> +'long_tensor_name_foo') and ('scores' -> 'generated_tensor_name_bar'). This +allows the user to refer to these tensors with their logical names when running +inference. + +Note: In addition to the description above, documentation related to signature +def structure and how to set up them up can be found [here](signature_defs.md). Let's run it! diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 3c97d89091b..ba049a47548 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -6,28 +6,27 @@ To compile and use TensorFlow Serving, you need to set up some prerequisites. ### Bazel (only if compiling source code) -TensorFlow Serving requires Bazel 0.4.5 or higher. You can find the Bazel +TensorFlow Serving requires Bazel 0.5.4 or higher. You can find the Bazel installation instructions [here](http://bazel.build/docs/install.html). If you have the prerequisites for Bazel, those instructions consist of the following steps: 1. Download the relevant binary from - [here](https://github.com/bazelbuild/bazel/releases). - Let's say you downloaded bazel-0.4.5-installer-linux-x86_64.sh. You would - execute: + [here](https://github.com/bazelbuild/bazel/releases). Let's say you + downloaded bazel-0.5.4-installer-linux-x86_64.sh. You would execute: - ```shell + 
     cd ~/Downloads
-    chmod +x bazel-0.4.5-installer-linux-x86_64.sh
-    ./bazel-0.4.5-installer-linux-x86_64.sh --user
-    ```
+    chmod +x bazel-0.5.4-installer-linux-x86_64.sh
+    ./bazel-0.5.4-installer-linux-x86_64.sh --user
+
2. Set up your environment. Put this in your ~/.bashrc. - ```shell + 
     export PATH="$PATH:$HOME/bin"
-    ```
+
### gRPC @@ -97,17 +96,17 @@ sudo apt-get remove tensorflow-model-server 1. Add TensorFlow Serving distribution URI as a package source (one time setup) - ```shell + 
     echo "deb [arch=amd64] http://storage.googleapis.com/tensorflow-serving-apt stable tensorflow-model-server tensorflow-model-server-universal" | sudo tee /etc/apt/sources.list.d/tensorflow-serving.list
 
     curl https://storage.googleapis.com/tensorflow-serving-apt/tensorflow-serving.release.pub.gpg | sudo apt-key add -
-    ```
+
2. Install and update TensorFlow ModelServer - ```shell + 
     sudo apt-get update && sudo apt-get install tensorflow-model-server
-    ```
+
Once installed, the binary can be invoked using the command `tensorflow_model_server`. diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 77b9adf37ec..8044df9d1a4 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -183,7 +183,7 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_deb", "pkg_tar") pkg_tar( name = "tensorflow_model_server_tar", - srcs = [ + files = [ ":tensorflow_model_server", ], package_dir = "/usr/bin", diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel b/tensorflow_serving/tools/docker/Dockerfile.devel index dea056ebe6e..eed44caa601 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel +++ b/tensorflow_serving/tools/docker/Dockerfile.devel @@ -34,7 +34,7 @@ RUN pip install mock grpcio ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.5.1 +ENV BAZEL_VERSION 0.5.4 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu index 4c80705492c..6cae9d52286 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -1,4 +1,4 @@ -FROM nvidia/cuda:8.0-cudnn5-devel-ubuntu16.04 +FROM nvidia/cuda:8.0-cudnn6-devel-ubuntu16.04 RUN apt-get update && apt-get install -y \ build-essential \ @@ -20,9 +20,11 @@ RUN apt-get update && apt-get install -y \ apt-get clean && \ rm -rf /var/lib/apt/lists/* -RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ - python get-pip.py && \ - rm get-pip.py +RUN apt-get install python-pip python-dev build-essential + +#RUN curl -fSsL -O https://bootstrap.pypa.io/get-pip.py && \ +# python get-pip.py && \ +# rm get-pip.py # Set up grpc @@ -54,7 +56,7 @@ RUN echo "build --spawn_strategy=standalone --genrule_strategy=standalone" \ >>/root/.bazelrc ENV BAZELRC /root/.bazelrc # Install the most recent bazel release. -ENV BAZEL_VERSION 0.4.5 +ENV BAZEL_VERSION 0.5.4 WORKDIR / RUN mkdir /bazel && \ cd /bazel && \ @@ -83,7 +85,7 @@ RUN mkdir /usr/lib/x86_64-linux-gnu/include/ && \ ln -s /usr/lib/x86_64-linux-gnu/include/cudnn.h /usr/lib/x86_64-linux-gnu/include/cudnn.h && \ ln -s /usr/include/cudnn.h /usr/local/cuda/include/cudnn.h && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/local/cuda/lib64/libcudnn.so && \ - ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.5 /usr/local/cuda/lib64/libcudnn.so.5 + ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.6 /usr/local/cuda/lib64/libcudnn.so.6 # Configure Tensorflow to use the GPU @@ -98,4 +100,4 @@ RUN bazel build -c opt --config=cuda \ cp bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server /usr/local/bin/ && \ bazel clean --expunge -CMD ["/bin/bash"] \ No newline at end of file +CMD ["/bin/bash"] diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h index 8ddd78c6926..30ef145b6af 100644 --- a/tensorflow_serving/util/class_registration.h +++ b/tensorflow_serving/util/class_registration.h @@ -266,7 +266,10 @@ class ClassRegistry { protobuf::MessageFactory::generated_factory() ->GetPrototype(descriptor) ->New()); - any_config.UnpackTo(config.get()); + if (!any_config.UnpackTo(config.get())) { + return errors::InvalidArgument("Malformed content of Any: ", + any_config.DebugString()); + } return Create(*config, std::forward(args)..., result); } From 2191b205a28c871a51ab8d07d2381c9b60e3c793 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 27 Oct 2017 13:15:28 -0800 Subject: [PATCH 0379/8554] Add support to read models from S3 in TensorFlow Serving. Change: 173716945 --- tensorflow_serving/model_servers/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 8044df9d1a4..aac64ef15ed 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -136,6 +136,7 @@ cc_binary( "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", "@org_tensorflow//tensorflow/core/platform/hadoop:hadoop_file_system", + "@org_tensorflow//tensorflow/core/platform/s3:s3_file_system", "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:availability_preserving_policy", From a619a88577f4f628faae6655048948b1389bceca Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 27 Oct 2017 14:47:15 -0700 Subject: [PATCH 0380/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 1ad5e692e2f..cb7cb40a57f 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 1ad5e692e2fc218ca0b2a9a461c19762fdc9674b +Subproject commit cb7cb40a57fde5cfd4731bc551e82a1e2fef43a5 diff --git a/tf_models b/tf_models index b8e5a44bec0..edcd29f2dbb 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit b8e5a44bec01d25cec6e5a9b3c49a54d6bb9897f +Subproject commit edcd29f2dbb4b3eaed387fe17cb5270f867aec42 From da0d6e0dcafe71e57df29bf5892dbcd280197738 Mon Sep 17 00:00:00 2001 From: Sarah Maddox Date: Wed, 1 Nov 2017 04:08:33 +1100 Subject: [PATCH 0381/8554] Adds a link to the referenced tutorial (#632) Adds a link to the MNIST For ML Beginners tutorial, which is currently referred to with text alone. --- tensorflow_serving/g3doc/serving_basic.md | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 026de84ee82..ff06b10343d 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -34,9 +34,10 @@ further optimize the build, refer to the ## Train And Export TensorFlow Model As you can see in `mnist_saved_model.py`, the training is done the same way it -is in the MNIST For ML Beginners tutorial. The TensorFlow graph is launched in -TensorFlow session `sess`, with the input tensor (image) as `x` and output -tensor (Softmax score) as `y`. +is in the +[MNIST For ML Beginners tutorial](https://www.tensorflow.org/get_started/mnist/beginners). +The TensorFlow graph is launched in TensorFlow session `sess`, with the input +tensor (image) as `x` and output tensor (Softmax score) as `y`. Then we use TensorFlow's [SavedModelBuilder module](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/builder.py) to export the model. `SavedModelBuilder` saves a "snapshot" of the trained model From 42c3274e6f89cd519bee3a45faae36577542d71f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 1 Nov 2017 17:07:23 -0800 Subject: [PATCH 0382/8554] No-op. Change: 174266189 --- tensorflow_serving/apis/BUILD | 1 + tensorflow_serving/serving.bzl | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 24391f23949..2c8487441bd 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -79,6 +79,7 @@ serving_proto_library( cc_api_version = 2, go_api_version = 2, java_api_version = 2, + js_api_version = 2, deps = [ "@protobuf_archive//:cc_wkt_protos", ], diff --git a/tensorflow_serving/serving.bzl b/tensorflow_serving/serving.bzl index b9306d1d366..a56ea39498b 100644 --- a/tensorflow_serving/serving.bzl +++ b/tensorflow_serving/serving.bzl @@ -5,7 +5,7 @@ def serving_proto_library(name, srcs=[], has_services=False, deps=[], visibility=None, testonly=0, # pylint: disable=unused-argument cc_grpc_version = None, cc_api_version=2, go_api_version=2, - java_api_version=2, + java_api_version=2, js_api_version=2, py_api_version=2): native.filegroup(name=name + "_proto_srcs", srcs=srcs, From 30f821017f0044306fcf0aa1de615ae732242a85 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 2 Nov 2017 09:57:51 -0800 Subject: [PATCH 0383/8554] Clean up markdown and add TOC Change: 174349627 --- tensorflow_serving/batching/README.md | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/batching/README.md b/tensorflow_serving/batching/README.md index 03869e2fa44..f92e47e04e6 100644 --- a/tensorflow_serving/batching/README.md +++ b/tensorflow_serving/batching/README.md @@ -1,10 +1,13 @@ ---- ---- - - # TensorFlow Serving Batching Guide -[TOC] +* [Introduction](#introduction) +* [Simple Batching](#simple-batching) + * [BatchingSession](#batchingsession) + * [BasicBatchScheduler](#basicbatchscheduler) +* [Batch Scheduling Parameters and Tuning](#batch-scheduling-parameters-and-tuning) + * [Performance Tuning](#performance-tuning) +* [Servers with Multiple Models, Model Versions or Subtasks](#servers-with-multiple-models-model-versions-or-subtasks) +* [Mixed CPU/GPU/IO Workloads](#mixed-cpugpuio-workloads) ## Introduction From a899b2d3f555573934cc8fd4e943d79955d7bba5 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 2 Nov 2017 14:53:28 -0800 Subject: [PATCH 0384/8554] Adds a sampler to monitor the number of examples per model-request. Change: 174393370 --- .../servables/tensorflow/classifier.cc | 1 + .../servables/tensorflow/multi_inference.cc | 1 + .../servables/tensorflow/regressor.cc | 1 + .../servables/tensorflow/util.cc | 18 ++++++++++++++++++ tensorflow_serving/servables/tensorflow/util.h | 12 ++++++++++++ .../servables/tensorflow/util_test.cc | 15 +++++++++++++++ 6 files changed, 48 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/classifier.cc b/tensorflow_serving/servables/tensorflow/classifier.cc index 7c1d490e970..17fd7d4fdf1 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.cc +++ b/tensorflow_serving/servables/tensorflow/classifier.cc @@ -63,6 +63,7 @@ class TensorFlowClassifier : public ClassifierInterface { if (num_examples == 0) { return errors::InvalidArgument("ClassificationRequest::input is empty."); } + RecordRequestExampleCount(request.model_spec().name(), num_examples); TRACELITERAL("RunClassification"); // Support serving models that return classes, scores or both. diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.cc b/tensorflow_serving/servables/tensorflow/multi_inference.cc index 1068240c876..36adbe37f60 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference.cc @@ -97,6 +97,7 @@ Status TensorFlowMultiInferenceRunner::Infer( TF_RETURN_IF_ERROR(PerformOneShotTensorComputation( run_options, request.input(), input_tensor_name, output_tensor_names, session_, &outputs, &num_examples)); + RecordRequestExampleCount(model_name, num_examples); TRACELITERAL("PostProcessResults"); for (const auto& task : request.tasks()) { diff --git a/tensorflow_serving/servables/tensorflow/regressor.cc b/tensorflow_serving/servables/tensorflow/regressor.cc index a6ccf1d9f1a..b7e0493a376 100644 --- a/tensorflow_serving/servables/tensorflow/regressor.cc +++ b/tensorflow_serving/servables/tensorflow/regressor.cc @@ -65,6 +65,7 @@ class TensorFlowRegressor : public RegressorInterface { if (num_examples == 0) { return errors::InvalidArgument("RegressionRequest::input is empty."); } + RecordRequestExampleCount(request.model_spec().name(), num_examples); TRACELITERAL("RunRegression"); Tensor output; diff --git a/tensorflow_serving/servables/tensorflow/util.cc b/tensorflow_serving/servables/tensorflow/util.cc index 3831df0a48d..f4badcf8c90 100644 --- a/tensorflow_serving/servables/tensorflow/util.cc +++ b/tensorflow_serving/servables/tensorflow/util.cc @@ -27,6 +27,13 @@ namespace tensorflow { namespace serving { namespace { +auto* example_counts = monitoring::Sampler<1>::New( + {"/tensorflow/serving/request_example_counts", + "The number of tensorflow.Examples per request.", "model"}, + // It's 15 buckets with the last bucket being 2^14 to DBL_MAX; + // so the limits are [1, 2, 4, 8, ..., 16 * 1024, DBL_MAX]. + monitoring::Buckets::Exponential(1, 2, 15)); + // Returns the number of examples in the Input. int NumInputExamples(const internal::SerializedInput& input) { switch (input.kind_case()) { @@ -39,8 +46,19 @@ int NumInputExamples(const internal::SerializedInput& input) { } return 0; } + } // namespace +namespace internal { + +monitoring::Sampler<1>* GetExampleCounts() { return example_counts; } + +} // namespace internal + +void RecordRequestExampleCount(const string& model_name, size_t count) { + example_counts->GetCell(model_name)->Add(count); +} + Status InputToSerializedExampleTensor(const Input& input, Tensor* examples) { const string serialized_input_str = input.SerializeAsString(); internal::SerializedInput serialized_input; diff --git a/tensorflow_serving/servables/tensorflow/util.h b/tensorflow_serving/servables/tensorflow/util.h index 832ced2544b..cf6590bb38a 100644 --- a/tensorflow_serving/servables/tensorflow/util.h +++ b/tensorflow_serving/servables/tensorflow/util.h @@ -18,12 +18,24 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/monitoring/sampler.h" #include "tensorflow/core/public/session.h" #include "tensorflow_serving/apis/input.pb.h" namespace tensorflow { namespace serving { +// Implementation details mainly used for testing; please don't depend on it. +namespace internal { + +monitoring::Sampler<1>* GetExampleCounts(); + +} // namespace internal + +// Records the example count of this request with the metric tracking the +// histogram of number of examples per request. +void RecordRequestExampleCount(const string& model_name, size_t count); + // InputToSerializedExampleTensor populates a string Tensor of serialized // Examples. // If input has n Examples returns a string Tensor with shape {n}. diff --git a/tensorflow_serving/servables/tensorflow/util_test.cc b/tensorflow_serving/servables/tensorflow/util_test.cc index 89ce28ca965..64be84c55ff 100644 --- a/tensorflow_serving/servables/tensorflow/util_test.cc +++ b/tensorflow_serving/servables/tensorflow/util_test.cc @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor_shape.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/histogram/histogram.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { @@ -201,6 +202,20 @@ TEST_F(InputUtilTest, RequestNumExamplesStreamz) { EXPECT_EQ(1, tensor_2.NumElements()); } +TEST(ExampleCountsTest, Simple) { + using histogram::Histogram; + + const HistogramProto before_histogram = + internal::GetExampleCounts()->GetCell("model-name")->value(); + RecordRequestExampleCount("model-name", 3); + const HistogramProto after_histogram = + internal::GetExampleCounts()->GetCell("model-name")->value(); + + ASSERT_GE(before_histogram.bucket().size(), 3); + ASSERT_GE(after_histogram.bucket().size(), 3); + EXPECT_EQ(1, after_histogram.bucket(2) - before_histogram.bucket(2)); +} + } // namespace } // namespace serving } // namespace tensorflow From cbb475da672e3385e9adf452588c20ead604ccae Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Fri, 3 Nov 2017 16:51:49 -0700 Subject: [PATCH 0385/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index cb7cb40a57f..d752244fbaa 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit cb7cb40a57fde5cfd4731bc551e82a1e2fef43a5 +Subproject commit d752244fbaad5e4268243355046d30990f59418f diff --git a/tf_models b/tf_models index edcd29f2dbb..9557674c73e 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit edcd29f2dbb4b3eaed387fe17cb5270f867aec42 +Subproject commit 9557674c73eb57540672a74366afcab359cb9fd1 From 6b5d30e5e09f7c9033912aa35d28f16144d8dc01 Mon Sep 17 00:00:00 2001 From: Li Lao Date: Thu, 9 Nov 2017 14:09:18 -0800 Subject: [PATCH 0386/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index d752244fbaa..8af1964d7b0 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit d752244fbaad5e4268243355046d30990f59418f +Subproject commit 8af1964d7b07fedcbfcd43d2ba13c97aad51daa8 diff --git a/tf_models b/tf_models index 9557674c73e..18a4e59fd72 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 9557674c73eb57540672a74366afcab359cb9fd1 +Subproject commit 18a4e59fd7209422a9fa23c9c950876299ce534d From a82e7d060dad08b911be945fa1732a2a71a73771 Mon Sep 17 00:00:00 2001 From: Fred Reiss Date: Mon, 13 Nov 2017 10:26:46 -0800 Subject: [PATCH 0387/8554] Sync code example in docs with original (#654) --- tensorflow_serving/g3doc/serving_basic.md | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index ff06b10343d..ddcdab36030 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -51,14 +51,12 @@ the following is a short code snippet to illustrate the general process of saving a model to disk. ```python -from tensorflow.python.saved_model import builder as saved_model_builder -... export_path_base = sys.argv[-1] export_path = os.path.join( compat.as_bytes(export_path_base), compat.as_bytes(str(FLAGS.model_version))) print 'Exporting trained model to', export_path -builder = saved_model_builder.SavedModelBuilder(export_path) +builder = tf.saved_model.builder.SavedModelBuilder(export_path) builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ From f3c65d2bb01c748bfdb6eccb45d7a92079085017 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 15 Nov 2017 15:32:18 -0800 Subject: [PATCH 0388/8554] Add a python proto library for model_server_config.proto. Change: 175891437 --- tensorflow_serving/config/BUILD | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 486e20e49fb..411611dc0d0 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -19,6 +19,7 @@ filegroup( ) load("//tensorflow_serving:serving.bzl", "serving_proto_library") +load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") serving_proto_library( name = "model_server_config_proto", @@ -32,6 +33,12 @@ serving_proto_library( ], ) +serving_proto_library_py( + name = "model_server_config_proto_py_pb2", + srcs = ["model_server_config.proto"], + proto_library = "model_server_config_proto", +) + serving_proto_library( name = "platform_config_proto", srcs = ["platform_config.proto"], From 30e5bc4c5dce237d3299187c8403f0c32e08d098 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Wed, 15 Nov 2017 16:36:58 -0800 Subject: [PATCH 0389/8554] Public no-op. Change: 175900655 --- tensorflow_serving/config/BUILD | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index 411611dc0d0..b09d8b2e2c8 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -53,6 +53,7 @@ serving_proto_library( srcs = ["log_collector_config.proto"], cc_api_version = 2, go_api_version = 2, + java_api_version = 2, deps = [ ], ) @@ -62,6 +63,7 @@ serving_proto_library( srcs = ["logging_config.proto"], cc_api_version = 2, go_api_version = 2, + java_api_version = 2, deps = [ ":log_collector_config_proto", ], From be1b001d1f17a4e70b234179e143207438a5714f Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 20 Nov 2017 08:54:47 -0800 Subject: [PATCH 0390/8554] Change to allow server core to read in both relative and absolute paths in its configs when a root directory is specified. Change: 176367198 --- tensorflow_serving/model_servers/server_core.cc | 12 +++++------- .../model_servers/server_core_test.cc | 17 +++++++++++++---- 2 files changed, 18 insertions(+), 11 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index d821923d9a4..224f7c68286 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -101,13 +101,6 @@ Status ValidateModelConfigList(const ModelConfigList& config_list, "model_config_list_root_dir=", *options.model_config_list_root_dir)); } - for (const ModelConfig& config : config_list.config()) { - if (!UriIsRelativePath(config.base_path())) { - return errors::InvalidArgument(strings::StrCat( - "Expected model ", config.name(), - " to have a relative path; got base_path()=", config.base_path())); - } - } } else { // All base-paths must be absolute. for (const ModelConfig& config : config_list.config()) { @@ -200,6 +193,11 @@ Status UpdateModelConfigListRelativePaths( std::vector updated_paths; updated_paths.reserve(config_list->config_size()); for (const ModelConfig& config : config_list->config()) { + // Don't modify absolute paths. + if (!UriIsRelativePath(config.base_path())) { + updated_paths.push_back(config.base_path()); + continue; + } updated_paths.emplace_back( io::JoinPath(model_config_list_root_dir, config.base_path())); if (UriIsRelativePath(updated_paths.back())) { diff --git a/tensorflow_serving/model_servers/server_core_test.cc b/tensorflow_serving/model_servers/server_core_test.cc index c251529f021..961305c7fb9 100644 --- a/tensorflow_serving/model_servers/server_core_test.cc +++ b/tensorflow_serving/model_servers/server_core_test.cc @@ -226,19 +226,28 @@ TEST_P(RelativePathsServerCoreTest, RelativePathFails) { CreateServerCore(relative, &server_core).code()); } -TEST_P(RelativePathsServerCoreTest, AbsolutePathWithOptionsFails) { +TEST_P(RelativePathsServerCoreTest, + AbsoluteAndRelativePathsWithOptionsSucceed) { std::unique_ptr server_core; + ModelServerConfig absolute_and_relative; ModelServerConfig absolute = GetTestModelServerConfigForFakePlatform(); ServerCore::Options options = GetDefaultOptions(); { string model_config_list_root_dir; ModelServerConfig relative = GetTestModelServerConfigWithRelativePath(&model_config_list_root_dir); + // Add the absolute and relative config to absolute. We'll check that both + // paths can be interleaved in the same model config list. + CHECK_GT(relative.model_config_list().config_size(), 0); + CHECK_GT(absolute.model_config_list().config_size(), 0); + *absolute_and_relative.mutable_model_config_list()->add_config() = + absolute.model_config_list().config(0); + *absolute_and_relative.mutable_model_config_list()->add_config() = + relative.model_config_list().config(0); options.model_config_list_root_dir = std::move(model_config_list_root_dir); } - EXPECT_EQ( - error::INVALID_ARGUMENT, - CreateServerCore(absolute, std::move(options), &server_core).code()); + TF_ASSERT_OK(CreateServerCore(absolute_and_relative, std::move(options), + &server_core)); } TEST_P(RelativePathsServerCoreTest, AbsolutePathWithEmptyPathFails) { From eeea3b01553a2147ff03542f1f96a62bd44d0614 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Mon, 20 Nov 2017 13:01:51 -0800 Subject: [PATCH 0391/8554] Adds a MockServerRequestLogger. Change: 176403560 --- .../core/server_request_logger.h | 13 +++--- tensorflow_serving/core/test_util/BUILD | 11 +++++ .../test_util/mock_server_request_logger.h | 45 +++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) create mode 100644 tensorflow_serving/core/test_util/mock_server_request_logger.h diff --git a/tensorflow_serving/core/server_request_logger.h b/tensorflow_serving/core/server_request_logger.h index b730643fcec..e337978ef30 100644 --- a/tensorflow_serving/core/server_request_logger.h +++ b/tensorflow_serving/core/server_request_logger.h @@ -49,25 +49,28 @@ class ServerRequestLogger { request_logger_creator, std::unique_ptr* server_request_logger); - ~ServerRequestLogger() = default; + virtual ~ServerRequestLogger() = default; // Updates the logger with the new 'logging_config_map'. // // If the ServerRequestLogger was created using an empty // request_logger_creator, this will return an error if a non-empty // logging_config_map is passed in. - Status Update(const std::map& logging_config_map); + virtual Status Update( + const std::map& logging_config_map); // Similar to RequestLogger::Log(). - Status Log(const google::protobuf::Message& request, const google::protobuf::Message& response, - const LogMetadata& log_metadata); + virtual Status Log(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata); - private: + protected: explicit ServerRequestLogger( const std::function*)>& request_logger_creator); + private: // A map from model_name to its corresponding RequestLogger. using RequestLoggerMap = std::unordered_map>; diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 44c5a71790f..082dd7f14ce 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -182,6 +182,17 @@ cc_library( ], ) +cc_library( + name = "mock_server_request_logger", + testonly = 1, + hdrs = ["mock_server_request_logger.h"], + visibility = ["//visibility:public"], + deps = [ + "//tensorflow_serving/core:server_request_logger", + "@com_google_googletest//:gtest", + ], +) + cc_library( name = "mock_session", testonly = 1, diff --git a/tensorflow_serving/core/test_util/mock_server_request_logger.h b/tensorflow_serving/core/test_util/mock_server_request_logger.h new file mode 100644 index 00000000000..7460c39c059 --- /dev/null +++ b/tensorflow_serving/core/test_util/mock_server_request_logger.h @@ -0,0 +1,45 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SERVER_REQUEST_LOGGER_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SERVER_REQUEST_LOGGER_H_ + +#include + +#include +#include "tensorflow_serving/core/server_request_logger.h" + +namespace tensorflow { +namespace serving { +namespace test_util { + +class MockServerRequestLogger : public ServerRequestLogger { + public: + MockServerRequestLogger() : ServerRequestLogger({}) {} + + MOCK_METHOD1( + Update, + Status(const std::map& logging_config_map)); + + MOCK_METHOD3(Log, Status(const google::protobuf::Message& request, + const google::protobuf::Message& response, + const LogMetadata& log_metadata)); +}; + +} // namespace test_util +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SERVER_REQUEST_LOGGER_H_ From d5e10a163d1f886d255d28a81c9ea75e2eb9bedd Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 21 Nov 2017 10:47:02 -0800 Subject: [PATCH 0392/8554] Bazel workaround to compile gRPC with the new 'cares' package which we don't use (grpc_no_ares=true in bazel.rc). Same workaround as TF added. Change: 176530560 --- tensorflow_serving/workspace.bzl | 7 +++++++ tools/bazel.rc | 2 ++ 2 files changed, 9 insertions(+) diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index 6719ae6d92d..8035841245c 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -25,3 +25,10 @@ def tf_serving_workspace(): name = "zlib", actual = "@zlib_archive//:zlib", ) + + # gRPC wants the existence of a cares dependence but its contents are not + # actually important since we have set GRPC_ARES=0 in tools/bazel.rc + native.bind( + name = "cares", + actual = "@grpc//third_party/nanopb:nanopb", + ) diff --git a/tools/bazel.rc b/tools/bazel.rc index 9397f97379e..9ecd306107d 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -13,3 +13,5 @@ run --define PYTHON_BIN_PATH=/usr/bin/python build --spawn_strategy=standalone --genrule_strategy=standalone test --spawn_strategy=standalone --genrule_strategy=standalone run --spawn_strategy=standalone --genrule_strategy=standalone + +build --define=grpc_no_ares=true From ddcce34c527b7366bdcb97d908dac1cbd1af599a Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Wed, 22 Nov 2017 14:59:04 -0800 Subject: [PATCH 0393/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 8af1964d7b0..79422ab39b5 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 8af1964d7b07fedcbfcd43d2ba13c97aad51daa8 +Subproject commit 79422ab39b5fe0e1491abb8deabc7ecb5fd9f3a2 diff --git a/tf_models b/tf_models index 18a4e59fd72..220772b56f5 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 18a4e59fd7209422a9fa23c9c950876299ce534d +Subproject commit 220772b56f5afcd8996a9aed4ca9c1e45240f898 From 5ca1f8af1d0de83e73762f3d17859754d15a0450 Mon Sep 17 00:00:00 2001 From: Sancho McCann Date: Wed, 22 Nov 2017 16:26:15 -0800 Subject: [PATCH 0394/8554] Rewrote (simplified) "sources" presentation. (#666) --- .../g3doc/architecture_overview.md | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/tensorflow_serving/g3doc/architecture_overview.md b/tensorflow_serving/g3doc/architecture_overview.md index 6da0df270bd..42b81fd6c9d 100644 --- a/tensorflow_serving/g3doc/architecture_overview.md +++ b/tensorflow_serving/g3doc/architecture_overview.md @@ -74,20 +74,18 @@ unloading a servable. ### Sources -**Sources** are plugin modules that originate servables; each Source -originates zero or more servable streams. For each stream, a Source supplies -one Loader instance for each version it wants to have loaded. (To be precise, -a Source is actually chained together with zero or more SourceAdapters, and -the last item in the chain emits Loaders.) - -TensorFlow Serving’s interface for Sources is simple and narrow, so it lets -you use arbitrary storage systems to discover servables to load. Sources may -access other mechanisms such as RPC. TensorFlow Serving includes common -reference Source implementations. For example, TensorFlow Serving can poll a -file system. - -Sources can house state that is shared across multiple servables or versions, -for special cases such as models that efficiently accept delta updates. +**Sources** are plugin modules that find and provide servables. Each Source +provides zero or more servable streams. For each servable stream, a Source supplies +one Loader instance for each version it makes available to be loaded. (A Source is +actually chained together with zero or more SourceAdapters, and the last item in +the chain emits the Loaders.) + +TensorFlow Serving’s interface for Sources can discover servables from arbitrary +storage systems. TensorFlow Serving includes common reference Source implementations. +For example, Sources may access mechanisms such as RPC and can poll a file system. + +Sources can maintain state that is shared across multiple servables or versions. This +is useful for servables that use delta (diff) updates between versions. #### Aspired Versions From 7466c93a0f223bcb3341152e0b48db51fb0ea689 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Mon, 27 Nov 2017 16:21:50 -0800 Subject: [PATCH 0395/8554] Add required build --copt for TensorFlow Some recently added TF targets currently require --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK to build. It seems that tensorflow_serving depends on these targets and will not build without this option. This copt was added to TF default build as seen here... https://github.com/tensorflow/tensorflow/blob/0d3a49a/configure.py#L489 Change: 177088048 --- tools/bazel.rc | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tools/bazel.rc b/tools/bazel.rc index 9ecd306107d..52a6241d651 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -15,3 +15,6 @@ test --spawn_strategy=standalone --genrule_strategy=standalone run --spawn_strategy=standalone --genrule_strategy=standalone build --define=grpc_no_ares=true + +# TODO(b/69809703): Remove once no longer required for TensorFlow to build. +build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK From 6cb902c7ef0cffca9f66e7a55c3b1a9f67d0a6be Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Tue, 28 Nov 2017 15:05:44 -0800 Subject: [PATCH 0396/8554] Adds an API to subscribe to servable-state change notifications on the ServableStateMonitor. Change: 177224154 --- .../core/servable_state_monitor.cc | 27 +++++++++++++++---- .../core/servable_state_monitor.h | 23 +++++++++++++--- .../core/servable_state_monitor_test.cc | 7 +++++ .../model_servers/server_core.h | 2 +- tensorflow_serving/util/BUILD | 17 ++++++------ 5 files changed, 59 insertions(+), 17 deletions(-) diff --git a/tensorflow_serving/core/servable_state_monitor.cc b/tensorflow_serving/core/servable_state_monitor.cc index 29448086831..9c425c7c711 100644 --- a/tensorflow_serving/core/servable_state_monitor.cc +++ b/tensorflow_serving/core/servable_state_monitor.cc @@ -16,6 +16,7 @@ limitations under the License. #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow/core/lib/core/notification.h" +#include "tensorflow/core/lib/gtl/cleanup.h" namespace tensorflow { namespace serving { @@ -184,7 +185,12 @@ void ServableStateMonitor::NotifyWhenServablesReachState( mutex_lock l(mu_); servable_state_notification_requests_.push_back( {servables, goal_state, notifier_fn}); - MaybeSendNotifications(); + MaybeSendStateReachedNotifications(); +} + +void ServableStateMonitor::Notify(const NotifyFn& notify_fn) { + mutex_lock l(notify_mu_); + notify_fns_.push_back(notify_fn); } bool ServableStateMonitor::WaitUntilServablesReachState( @@ -215,13 +221,16 @@ void ServableStateMonitor::HandleEvent( const EventBus::EventAndTime& event_and_time) { PreHandleEvent(event_and_time); + auto cleanup = + gtl::MakeCleanup([&]() { SendNotifications(event_and_time.event); }); + mutex_lock l(mu_); const ServableStateAndTime state_and_time = { event_and_time.event, event_and_time.event_time_micros}; states_[state_and_time.state.id.name][state_and_time.state.id.version] = state_and_time; UpdateLiveStates(state_and_time, &live_states_); - MaybeSendNotifications(); + MaybeSendStateReachedNotifications(); if (options_.max_count_log_events == 0) { return; @@ -233,7 +242,7 @@ void ServableStateMonitor::HandleEvent( } optional>> -ServableStateMonitor::ShouldSendNotification( +ServableStateMonitor::ShouldSendStateReachedNotification( const ServableStateNotificationRequest& notification_request) { bool reached_goal_state = true; std::map states_reached; @@ -270,14 +279,14 @@ ServableStateMonitor::ShouldSendNotification( return {{reached_goal_state, states_reached}}; } -void ServableStateMonitor::MaybeSendNotifications() { +void ServableStateMonitor::MaybeSendStateReachedNotifications() { for (auto iter = servable_state_notification_requests_.begin(); iter != servable_state_notification_requests_.end();) { const ServableStateNotificationRequest& notification_request = *iter; const optional< std::pair>> opt_state_and_states_reached = - ShouldSendNotification(notification_request); + ShouldSendStateReachedNotification(notification_request); if (opt_state_and_states_reached) { notification_request.notifier_fn(opt_state_and_states_reached->first, opt_state_and_states_reached->second); @@ -288,5 +297,13 @@ void ServableStateMonitor::MaybeSendNotifications() { } } +void ServableStateMonitor::SendNotifications( + const ServableState& servable_state) { + mutex_lock l(notify_mu_); + for (const auto& notify_fn : notify_fns_) { + notify_fn(servable_state); + } +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/core/servable_state_monitor.h b/tensorflow_serving/core/servable_state_monitor.h index f2abc56be16..b5687305f2b 100644 --- a/tensorflow_serving/core/servable_state_monitor.h +++ b/tensorflow_serving/core/servable_state_monitor.h @@ -150,6 +150,11 @@ class ServableStateMonitor { std::map* states_reached = nullptr) LOCKS_EXCLUDED(mu_) TF_MUST_USE_RESULT; + // Subscribes to all servable state changes hitting this monitor. This is + // called after the monitor updates its own state based on the event. + using NotifyFn = std::function; + void Notify(const NotifyFn& notify_fn) LOCKS_EXCLUDED(notify_mu_); + private: optional GetStateAndTimeInternal( const ServableId& servable_id) const EXCLUSIVE_LOCKS_REQUIRED(mu_); @@ -166,13 +171,18 @@ class ServableStateMonitor { // If so, returns the 'reached_goal_state' bool and the 'states_reached' by // each servable. Oterwise returns nullopt. optional>> - ShouldSendNotification( + ShouldSendStateReachedNotification( const ServableStateNotificationRequest& notification_request) EXCLUSIVE_LOCKS_REQUIRED(mu_); // Goes through the notification requests and tries to see if any of them can // be sent. If a notification is sent, the corresponding request is removed. - void MaybeSendNotifications() EXCLUSIVE_LOCKS_REQUIRED(mu_); + void MaybeSendStateReachedNotifications() EXCLUSIVE_LOCKS_REQUIRED(mu_); + + // Goes through the notify_fns list and calls each one with the currently + // received ServableState. + void SendNotifications(const ServableState& servable_state) + LOCKS_EXCLUDED(notify_mu_); // This method is called when an event comes in, but before we update our // state with the contents of the event. Subclasses may override this method @@ -183,7 +193,7 @@ class ServableStateMonitor { // Handles a bus event. void HandleEvent(const EventBus::EventAndTime& state_and_time) - LOCKS_EXCLUDED(mu_); + LOCKS_EXCLUDED(mu_, notify_mu_); const Options options_; @@ -207,6 +217,13 @@ class ServableStateMonitor { std::vector servable_state_notification_requests_ GUARDED_BY(mu_); + // Separate mutex to protect the notify_fns_ so that they can be updated + // independently. This also allows these notify_fns_ to call other methods + // in ServableStateMonitor which don't depend on this mutex without being + // deadlocked. + mutable mutex notify_mu_; + std::vector notify_fns_ GUARDED_BY(notify_mu_); + TF_DISALLOW_COPY_AND_ASSIGN(ServableStateMonitor); }; diff --git a/tensorflow_serving/core/servable_state_monitor_test.cc b/tensorflow_serving/core/servable_state_monitor_test.cc index e59a796551c..35f4bf6f8ef 100644 --- a/tensorflow_serving/core/servable_state_monitor_test.cc +++ b/tensorflow_serving/core/servable_state_monitor_test.cc @@ -41,6 +41,10 @@ TEST(ServableStateMonitorTest, AddingStates) { monitor_options.max_count_log_events = 4; ServableStateMonitor monitor(bus.get(), monitor_options); + ServableState notified_state; + monitor.Notify([&](const ServableState& servable_state) { + notified_state = servable_state; + }); EXPECT_FALSE(monitor.GetState(ServableId{"foo", 42})); EXPECT_TRUE(monitor.GetVersionStates("foo").empty()); EXPECT_TRUE(monitor.GetAllServableStates().empty()); @@ -54,6 +58,7 @@ TEST(ServableStateMonitorTest, AddingStates) { bus->Publish(state_0); ASSERT_TRUE(monitor.GetState(ServableId{"foo", 42})); EXPECT_EQ(state_0, *monitor.GetState(ServableId{"foo", 42})); + EXPECT_EQ(state_0, notified_state); EXPECT_FALSE(monitor.GetState(ServableId{"foo", 99})); EXPECT_FALSE(monitor.GetState(ServableId{"bar", 42})); EXPECT_THAT(monitor.GetVersionStates("foo"), @@ -75,6 +80,7 @@ TEST(ServableStateMonitorTest, AddingStates) { EXPECT_EQ(state_0, *monitor.GetState(ServableId{"foo", 42})); ASSERT_TRUE(monitor.GetState(ServableId{"foo", 43})); EXPECT_EQ(state_1, *monitor.GetState(ServableId{"foo", 43})); + EXPECT_EQ(state_1, notified_state); EXPECT_FALSE(monitor.GetState(ServableId{"foo", 99})); EXPECT_FALSE(monitor.GetState(ServableId{"bar", 42})); EXPECT_THAT( @@ -101,6 +107,7 @@ TEST(ServableStateMonitorTest, AddingStates) { EXPECT_EQ(state_1, *monitor.GetState(ServableId{"foo", 43})); ASSERT_TRUE(monitor.GetState(ServableId{"bar", 7})); EXPECT_EQ(state_2, *monitor.GetState(ServableId{"bar", 7})); + EXPECT_EQ(state_2, notified_state); EXPECT_FALSE(monitor.GetState(ServableId{"bar", 42})); EXPECT_THAT( monitor.GetVersionStates("foo"), diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 53d941fc078..98450d29885 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -166,7 +166,7 @@ class ServerCore : public Manager { LOCKS_EXCLUDED(config_mu_); /// Returns ServableStateMonitor that can be used to query servable states. - virtual const ServableStateMonitor* servable_state_monitor() const { + virtual ServableStateMonitor* servable_state_monitor() const { return servable_state_monitor_.get(); } diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 069e39c6efa..5470b7cd529 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -53,6 +53,15 @@ cc_library( ], ) +cc_library( + name = "observer", + hdrs = ["observer.h"], + visibility = ["//visibility:public"], + deps = [ + "@org_tensorflow//tensorflow/core:lib", + ], +) + ############################################################################### # Internal targets ############################################################################### @@ -66,14 +75,6 @@ cc_library( ], ) -cc_library( - name = "observer", - hdrs = ["observer.h"], - deps = [ - "@org_tensorflow//tensorflow/core:lib", - ], -) - cc_test( name = "observer_test", size = "small", From fbd9ae2666328aaa8130c48c644ad60c47ce5cd5 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 29 Nov 2017 13:58:55 -0800 Subject: [PATCH 0397/8554] Add a way to query a batch scheduler to determine the max task size. A layer on top of the batcher could use this interface to pre-split large tasks that exceed the max batch size. Change: 177359263 --- tensorflow_serving/batching/batch_scheduler_retrier.h | 2 ++ tensorflow_serving/batching/batch_scheduler_retrier_test.cc | 5 +++++ tensorflow_serving/batching/streaming_batch_scheduler.h | 2 ++ .../batching/streaming_batch_scheduler_test.cc | 2 ++ .../batching/test_util/puppet_batch_scheduler.h | 4 ++++ 5 files changed, 15 insertions(+) diff --git a/tensorflow_serving/batching/batch_scheduler_retrier.h b/tensorflow_serving/batching/batch_scheduler_retrier.h index f62f31a9272..b65d5960b86 100644 --- a/tensorflow_serving/batching/batch_scheduler_retrier.h +++ b/tensorflow_serving/batching/batch_scheduler_retrier.h @@ -57,6 +57,8 @@ class BatchSchedulerRetrier : public BatchScheduler { size_t NumEnqueuedTasks() const override; size_t SchedulingCapacity() const override; + size_t max_task_size() const override { return wrapped_->max_task_size(); } + private: BatchSchedulerRetrier(const Options& options, std::unique_ptr> wrapped); diff --git a/tensorflow_serving/batching/batch_scheduler_retrier_test.cc b/tensorflow_serving/batching/batch_scheduler_retrier_test.cc index acaf7f68bd3..1bbe388993f 100644 --- a/tensorflow_serving/batching/batch_scheduler_retrier_test.cc +++ b/tensorflow_serving/batching/batch_scheduler_retrier_test.cc @@ -58,6 +58,8 @@ class BrokenScheduler : public BatchScheduler { int num_submit_calls() const { return num_submit_calls_; } + size_t max_task_size() const override { return 1000; } + private: int num_submit_calls_ = 0; @@ -90,6 +92,8 @@ class StubbornScheduler : public BatchScheduler { return std::numeric_limits::max(); } + size_t max_task_size() const override { return 1000; } + int num_attempts() const { return num_attempts_; } private: @@ -105,6 +109,7 @@ TEST(BatchSchedulerRetrierTest, ConstMethodsForwardToWrappedScheduler) { std::unique_ptr> retrier; TF_CHECK_OK(BatchSchedulerRetrier::Create( options, std::move(broken_scheduler), &retrier)); + EXPECT_EQ(1000, retrier->max_task_size()); EXPECT_EQ(7, retrier->NumEnqueuedTasks()); EXPECT_EQ(42, retrier->SchedulingCapacity()); } diff --git a/tensorflow_serving/batching/streaming_batch_scheduler.h b/tensorflow_serving/batching/streaming_batch_scheduler.h index 4341f10a21c..a2b3369541f 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler.h +++ b/tensorflow_serving/batching/streaming_batch_scheduler.h @@ -171,6 +171,8 @@ class StreamingBatchScheduler : public BatchScheduler { // immediately (there is no queueing). size_t SchedulingCapacity() const override; + size_t max_task_size() const override { return options_.max_batch_size; } + private: StreamingBatchScheduler(const Options& options, std::function>)> diff --git a/tensorflow_serving/batching/streaming_batch_scheduler_test.cc b/tensorflow_serving/batching/streaming_batch_scheduler_test.cc index 3f7ac4470d3..a7949e948c4 100644 --- a/tensorflow_serving/batching/streaming_batch_scheduler_test.cc +++ b/tensorflow_serving/batching/streaming_batch_scheduler_test.cc @@ -277,6 +277,8 @@ TEST(StreamingBatchSchedulerTest, ConstMethods) { TF_ASSERT_OK(StreamingBatchScheduler::Create(options, callback, &scheduler)); + EXPECT_EQ(2, scheduler->max_task_size()); + // Submit 'num_threads' full batches, to make the scheduling threads "full". // (At all times, the queue length should show as 0, since // StreamingBatchScheduler never enqueues tasks.) diff --git a/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h b/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h index 4410a82ca3b..84df13926e2 100644 --- a/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h +++ b/tensorflow_serving/batching/test_util/puppet_batch_scheduler.h @@ -59,6 +59,10 @@ class PuppetBatchScheduler : public BatchScheduler { // Processes all enqueued tasks. void ProcessAllTasks(); + size_t max_task_size() const override { + return std::numeric_limits::max(); + } + private: std::function>)> process_batch_callback_; From f22af43960e4f11cc0b8fd98dd684eee422ef00f Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Tue, 5 Dec 2017 13:01:10 -0800 Subject: [PATCH 0398/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 79422ab39b5..efbdc15b280 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 79422ab39b5fe0e1491abb8deabc7ecb5fd9f3a2 +Subproject commit efbdc15b280374607895ab0ada467de4a0512e0c diff --git a/tf_models b/tf_models index 220772b56f5..9b51944bfba 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 220772b56f5afcd8996a9aed4ca9c1e45240f898 +Subproject commit 9b51944bfbae5c971ea9a60226e081a578bbfc3b From b78f7803a4f242de6261a826f78ba9892a0882bb Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 8 Dec 2017 15:22:37 -0800 Subject: [PATCH 0399/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index efbdc15b280..e9020250b50 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit efbdc15b280374607895ab0ada467de4a0512e0c +Subproject commit e9020250b50922d0449da44ae5ea7d62ad899b0e diff --git a/tf_models b/tf_models index 9b51944bfba..5a5d330539d 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 9b51944bfbae5c971ea9a60226e081a578bbfc3b +Subproject commit 5a5d330539dff11eef79ca2e716fb477baf13cf9 From 8ae7bcc4affe2f567590b2211bc9c77187ce2e05 Mon Sep 17 00:00:00 2001 From: Keiji Ariyama Date: Fri, 15 Dec 2017 03:23:17 +0900 Subject: [PATCH 0400/8554] Add note about the TensorFlow Serving Python API not supporting Python 3. (#685) * Add note about the TensorFlow Serving Python API for Python 3.5. * Update setup.md --- tensorflow_serving/g3doc/setup.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index ba049a47548..06607f7aec5 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -72,6 +72,8 @@ the `tensorflow-serving-api` PIP package using: pip install tensorflow-serving-api ``` +Note: The TensorFlow Serving Python API only supports Python 2.7. It does not support Python 3. + ## Installing using apt-get ### Available binaries From d02aa686c51ebbd551adc617e6274c6e5870279a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 6 Dec 2017 09:40:20 -0800 Subject: [PATCH 0401/8554] CL is a no-op. Change: 178109250 --- tensorflow_serving/apis/BUILD | 63 +++++++++++++++++++++++++++++------ 1 file changed, 52 insertions(+), 11 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 2c8487441bd..4e90d5b5637 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -31,8 +31,8 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf_archive//:cc_wkt_protos", + "@protobuf_archive//:any", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -42,7 +42,7 @@ serving_proto_library_py( proto_library = "get_model_metadata_proto", deps = [ ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow/core:protos_all_py_pb2", ], ) @@ -50,11 +50,12 @@ serving_proto_library( name = "input_proto", srcs = ["input.proto"], cc_api_version = 2, + compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ - "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf_archive//:cc_wkt_protos", + "@protobuf_archive//:wrappers", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -63,7 +64,7 @@ serving_proto_library_py( srcs = ["input.proto"], proto_library = "input_proto", deps = [ - "@org_tensorflow//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow/core:protos_all_py_pb2", ], ) @@ -77,11 +78,12 @@ serving_proto_library( name = "model_proto", srcs = ["model.proto"], cc_api_version = 2, + compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, js_api_version = 2, deps = [ - "@protobuf_archive//:cc_wkt_protos", + "@protobuf_archive//:wrappers", ], ) @@ -106,7 +108,7 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:protos_all", ], ) @@ -116,7 +118,7 @@ serving_proto_library_py( proto_library = "predict_proto", deps = [ ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow/core:protos_all_py_pb2", ], ) @@ -147,6 +149,7 @@ py_library( ":inference_proto_py_pb2", ":predict_proto_py_pb2", ":regression_proto_py_pb2", + "//net/grpc/python:grpc", ], ) @@ -160,6 +163,7 @@ serving_proto_library( name = "classification_proto", srcs = ["classification.proto"], cc_api_version = 2, + compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ @@ -175,7 +179,7 @@ serving_proto_library_py( deps = [ ":input_proto_py_pb2", ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow/core:protos_all_py_pb2", ], ) @@ -189,6 +193,7 @@ serving_proto_library( name = "inference_proto", srcs = ["inference.proto"], cc_api_version = 2, + compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ @@ -221,6 +226,7 @@ serving_proto_library( name = "regression_proto", srcs = ["regression.proto"], cc_api_version = 2, + compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ @@ -236,7 +242,7 @@ serving_proto_library_py( deps = [ ":input_proto_py_pb2", ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py", + "@org_tensorflow//tensorflow/core:protos_all_py_pb2", ], ) @@ -263,3 +269,38 @@ cc_library( "@org_tensorflow//tensorflow/core:lib", ], ) + +go_proto_library( + name = "classification_go_proto", + deps = [":classification_proto"], +) + +go_proto_library( + name = "inference_go_proto", + deps = [":inference_proto"], +) + +go_proto_library( + name = "input_go_proto", + deps = [":input_proto"], +) + +go_proto_library( + name = "model_go_proto", + deps = [":model_proto"], +) + +go_proto_library( + name = "predict_go_proto", + deps = [":predict_proto"], +) + +go_proto_library( + name = "prediction_service_go_proto", + deps = [":prediction_service_proto"], +) + +go_proto_library( + name = "regression_go_proto", + deps = [":regression_proto"], +) From 6adac8991027f02b9691371e2597eb5a6914e74b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 11 Dec 2017 12:31:12 -0800 Subject: [PATCH 0402/8554] Public no-op. Change: 178656028 --- tensorflow_serving/apis/BUILD | 63 ++++++----------------------------- 1 file changed, 11 insertions(+), 52 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 4e90d5b5637..2c8487441bd 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -31,8 +31,8 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@protobuf_archive//:any", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -42,7 +42,7 @@ serving_proto_library_py( proto_library = "get_model_metadata_proto", deps = [ ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", ], ) @@ -50,12 +50,11 @@ serving_proto_library( name = "input_proto", srcs = ["input.proto"], cc_api_version = 2, - compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ - "@protobuf_archive//:wrappers", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -64,7 +63,7 @@ serving_proto_library_py( srcs = ["input.proto"], proto_library = "input_proto", deps = [ - "@org_tensorflow//tensorflow/core:protos_all_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", ], ) @@ -78,12 +77,11 @@ serving_proto_library( name = "model_proto", srcs = ["model.proto"], cc_api_version = 2, - compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, js_api_version = 2, deps = [ - "@protobuf_archive//:wrappers", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -108,7 +106,7 @@ serving_proto_library( java_api_version = 2, deps = [ ":model_proto", - "@org_tensorflow//tensorflow/core:protos_all", + "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) @@ -118,7 +116,7 @@ serving_proto_library_py( proto_library = "predict_proto", deps = [ ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", ], ) @@ -149,7 +147,6 @@ py_library( ":inference_proto_py_pb2", ":predict_proto_py_pb2", ":regression_proto_py_pb2", - "//net/grpc/python:grpc", ], ) @@ -163,7 +160,6 @@ serving_proto_library( name = "classification_proto", srcs = ["classification.proto"], cc_api_version = 2, - compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ @@ -179,7 +175,7 @@ serving_proto_library_py( deps = [ ":input_proto_py_pb2", ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", ], ) @@ -193,7 +189,6 @@ serving_proto_library( name = "inference_proto", srcs = ["inference.proto"], cc_api_version = 2, - compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ @@ -226,7 +221,6 @@ serving_proto_library( name = "regression_proto", srcs = ["regression.proto"], cc_api_version = 2, - compatible_with = ["//buildenv/target:appengine"], go_api_version = 2, java_api_version = 2, deps = [ @@ -242,7 +236,7 @@ serving_proto_library_py( deps = [ ":input_proto_py_pb2", ":model_proto_py_pb2", - "@org_tensorflow//tensorflow/core:protos_all_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", ], ) @@ -269,38 +263,3 @@ cc_library( "@org_tensorflow//tensorflow/core:lib", ], ) - -go_proto_library( - name = "classification_go_proto", - deps = [":classification_proto"], -) - -go_proto_library( - name = "inference_go_proto", - deps = [":inference_proto"], -) - -go_proto_library( - name = "input_go_proto", - deps = [":input_proto"], -) - -go_proto_library( - name = "model_go_proto", - deps = [":model_proto"], -) - -go_proto_library( - name = "predict_go_proto", - deps = [":predict_proto"], -) - -go_proto_library( - name = "prediction_service_go_proto", - deps = [":prediction_service_proto"], -) - -go_proto_library( - name = "regression_go_proto", - deps = [":regression_proto"], -) From 2b98cd7511862f9f31f66dc312af6a6af1ab8152 Mon Sep 17 00:00:00 2001 From: Chris Olston Date: Thu, 14 Dec 2017 10:29:57 -0800 Subject: [PATCH 0403/8554] Sync submodules. --- tf_models | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tf_models b/tf_models index 5a5d330539d..6db9f0282e2 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 5a5d330539dff11eef79ca2e716fb477baf13cf9 +Subproject commit 6db9f0282e2ab12795628de6200670892a8ad6ba From da24ed8e07d1e4e969e1ef10c2af39ed8d9ef8c1 Mon Sep 17 00:00:00 2001 From: John Zhou Date: Fri, 15 Dec 2017 21:30:15 +0800 Subject: [PATCH 0404/8554] add flag to config per_process_gpu_memory_fraction (#694) --- tensorflow_serving/model_servers/main.cc | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 413025fa17e..c20ada13bc9 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -76,6 +76,7 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/multi_inference.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" #include "tensorflow_serving/servables/tensorflow/regression_service.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" namespace grpc { class ServerCompletionQueue; @@ -302,6 +303,7 @@ tensorflow::serving::PlatformConfigMap ParsePlatformConfigMap( int main(int argc, char** argv) { tensorflow::int32 port = 8500; bool enable_batching = false; + float per_process_gpu_memory_fraction = 0.; tensorflow::string batching_parameters_file; tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; @@ -346,7 +348,13 @@ int main(int argc, char** argv) { "If non-empty, read an ascii PlatformConfigMap protobuf " "from the supplied file name, and use that platform " "config instead of the Tensorflow platform. (If used, " - "--enable_batching is ignored.)")}; + "--enable_batching is ignored.)"), + tensorflow::Flag("per_process_gpu_memory_fraction", &per_process_gpu_memory_fraction, + "Fraction that each process occupies gpu memory space " + "the value is between 0.0 and 1.0 (with 0.0 as the default) " + "If 1.0, the server will allocate all the memory when the server starts, " + "If 0.0, Tensorflow will automatically select a value.")}; + string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || (model_base_path.empty() && model_config_file.empty())) { @@ -390,6 +398,9 @@ int main(int argc, char** argv) { "--enable_batching"; } + session_bundle_config.mutable_session_config() + ->mutable_gpu_options() + ->set_per_process_gpu_memory_fraction(per_process_gpu_memory_fraction); session_bundle_config.mutable_session_config() ->set_intra_op_parallelism_threads(tensorflow_session_parallelism); session_bundle_config.mutable_session_config() From 1653f9b3adf00e202c76de46c49ef6e69bf2cdbe Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Thu, 28 Dec 2017 11:07:34 -0800 Subject: [PATCH 0405/8554] Clarify VersionPolicy wording. Change: 180282026 --- tensorflow_serving/config/model_server_config.proto | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/config/model_server_config.proto b/tensorflow_serving/config/model_server_config.proto index 864afdc4466..c2c9ed5dd4e 100644 --- a/tensorflow_serving/config/model_server_config.proto +++ b/tensorflow_serving/config/model_server_config.proto @@ -40,8 +40,8 @@ message ModelConfig { reserved 5; - // Version policy for the model indicating how many versions of the model to - // be served at the same time. + // Version policy for the model indicating which version(s) of the model to + // load and make available for serving simultaneously. // The default option is to serve only the latest version of the model. // // (This can be changed once a model is in serving.) From cfdc4456394d5b5babe81547741d3b0743e79b93 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Wed, 3 Jan 2018 10:07:12 -0800 Subject: [PATCH 0406/8554] Merge changes from github. Change: 180684365 --- .../g3doc/architecture_overview.md | 27 +++-- tensorflow_serving/g3doc/serving_basic.md | 109 ++++++++---------- tensorflow_serving/g3doc/setup.md | 2 + tensorflow_serving/model_servers/main.cc | 20 +++- 4 files changed, 81 insertions(+), 77 deletions(-) diff --git a/tensorflow_serving/g3doc/architecture_overview.md b/tensorflow_serving/g3doc/architecture_overview.md index 6da0df270bd..06a4c08e419 100644 --- a/tensorflow_serving/g3doc/architecture_overview.md +++ b/tensorflow_serving/g3doc/architecture_overview.md @@ -74,20 +74,19 @@ unloading a servable. ### Sources -**Sources** are plugin modules that originate servables; each Source -originates zero or more servable streams. For each stream, a Source supplies -one Loader instance for each version it wants to have loaded. (To be precise, -a Source is actually chained together with zero or more SourceAdapters, and -the last item in the chain emits Loaders.) - -TensorFlow Serving’s interface for Sources is simple and narrow, so it lets -you use arbitrary storage systems to discover servables to load. Sources may -access other mechanisms such as RPC. TensorFlow Serving includes common -reference Source implementations. For example, TensorFlow Serving can poll a -file system. - -Sources can house state that is shared across multiple servables or versions, -for special cases such as models that efficiently accept delta updates. +**Sources** are plugin modules that find and provide servables. Each Source +provides zero or more servable streams. For each servable stream, a Source +supplies one Loader instance for each version it makes available to be loaded. +(A Source is actually chained together with zero or more SourceAdapters, and the +last item in the chain emits the Loaders.) + +TensorFlow Serving’s interface for Sources can discover servables from arbitrary +storage systems. TensorFlow Serving includes common reference Source +implementations. For example, Sources may access mechanisms such as RPC and can +poll a file system. + +Sources can maintain state that is shared across multiple servables or versions. +This is useful for servables that use delta (diff) updates between versions. #### Aspired Versions diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index e35cbc59da1..121bdee6eaa 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -34,9 +34,10 @@ further optimize the build, refer to the ## Train And Export TensorFlow Model As you can see in `mnist_saved_model.py`, the training is done the same way it -is in the MNIST For ML Beginners tutorial. The TensorFlow graph is launched in -TensorFlow session `sess`, with the input tensor (image) as `x` and output -tensor (Softmax score) as `y`. +is in the +[MNIST For ML Beginners tutorial](https://www.tensorflow.org/get_started/mnist/beginners). +The TensorFlow graph is launched in TensorFlow session `sess`, with the input +tensor (image) as `x` and output tensor (Softmax score) as `y`. Then we use TensorFlow's [SavedModelBuilder module](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/builder.py) to export the model. `SavedModelBuilder` saves a "snapshot" of the trained model @@ -50,14 +51,12 @@ the following is a short code snippet to illustrate the general process of saving a model to disk. ```python -from tensorflow.python.saved_model import builder as saved_model_builder -... export_path_base = sys.argv[-1] export_path = os.path.join( compat.as_bytes(export_path_base), compat.as_bytes(str(FLAGS.model_version))) print 'Exporting trained model to', export_path -builder = saved_model_builder.SavedModelBuilder(export_path) +builder = tf.saved_model.builder.SavedModelBuilder(export_path) builder.add_meta_graph_and_variables( sess, [tag_constants.SERVING], signature_def_map={ @@ -84,69 +83,61 @@ sub-directory under the given path. You can add meta graph and variables to the builder using `SavedModelBuilder.add_meta_graph_and_variables()` with the following arguments: -* `sess` is the TensorFlow session that holds the trained model you are - exporting. - -* `tags` is the set of tags with which to save the meta graph. In this case, - since we intend to use the graph in serving, we use the `serve` tag from - predefined SavedModel tag constants. For more details, see - [tag_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) - and [related TensorFlow API - documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/tag_constants). - -* `signature_def_map` specifies the map of user-supplied key for a - **signature** to a tensorflow::SignatureDef to add to the meta graph. - Signature specifies what type of model is being exported, and the - input/output tensors to bind to when running inference. - - The special signature key `serving_default` specifies the default serving - signature. The default serving signature def key, along with other constants - related to signatures, are defined as part of SavedModel signature - constants. For more details, see - [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) - and related [TensorFlow 1.0 API - documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). - - Further, to help build signature defs easily, the SavedModel API provides - [signature def - utils](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_def_utils). - Specifically, in the `mnist_saved_model.py` code snippet above, we use - `signature_def_utils.build_signature_def()` to build `predict_signature` and - `classification_signature`. - - As an example for how `predict_signature` is defined, the util takes the - following arguments: - - * `inputs={'images': tensor_info_x}` specifies the input tensor info. - - * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. - - * `method_name` is the method used for the inference. For Prediction - requests, it should be set to `tensorflow/serving/predict`. For other - method names, see - [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) - and related [TensorFlow 1.0 API - documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). +* `sess` is the TensorFlow session that holds the trained model you are + exporting. + +* `tags` is the set of tags with which to save the meta graph. In this case, + since we intend to use the graph in serving, we use the `serve` tag from + predefined SavedModel tag constants. For more details, see [tag_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/tag_constants.py) + and [related TensorFlow API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/tag_constants). + +* `signature_def_map` specifies the map of user-supplied key for a + **signature** to a tensorflow::SignatureDef to add to the meta graph. + Signature specifies what type of model is being exported, and the + input/output tensors to bind to when running inference. + + The special signature key `serving_default` specifies the default serving + signature. The default serving signature def key, along with other constants + related to signatures, are defined as part of SavedModel signature constants. + For more details, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) + and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). + + Further, to help build signature defs easily, the SavedModel API provides + [signature def utils](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_def_utils). + Specifically, in the `mnist_saved_model.py` code snippet above, we use + `signature_def_utils.build_signature_def()` to build `predict_signature` and + `classification_signature`. + + As an example for how `predict_signature` is defined, the util takes the + following arguments: + + * `inputs={'images': tensor_info_x}` specifies the input tensor info. + + * `outputs={'scores': tensor_info_y}` specifies the scores tensor info. + + * `method_name` is the method used for the inference. For Prediction + requests, it should be set to `tensorflow/serving/predict`. For other + method names, see [signature_constants.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/signature_constants.py) + and related [TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/signature_constants). + Note that `tensor_info_x` and `tensor_info_y` have the structure of -`tensorflow::TensorInfo` protocol buffer defined -[here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto). +`tensorflow::TensorInfo` protocol buffer defined [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/core/protobuf/meta_graph.proto). To easily build tensor infos, the TensorFlow SavedModel API also provides [utils.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/saved_model/utils.py), -with [related TensorFlow 1.0 API -documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/utils). +with [related TensorFlow 1.0 API documentation](https://www.tensorflow.org/api_docs/python/tf/saved_model/utils). Also, note that `images` and `scores` are tensor alias names. They can be -whatever unique strings you want, and they will become the logical names of -tensor `x` and `y` that you refer to for tensor binding when sending prediction -requests later. +whatever unique strings you want, and they will become the logical names +of tensor `x` and `y` that you refer to for tensor binding when sending +prediction requests later. For instance, if `x` refers to the tensor with name 'long_tensor_name_foo' and `y` refers to the tensor with name 'generated_tensor_name_bar', `builder` will store tensor logical name to real name mapping ('images' -> -'long_tensor_name_foo') and ('scores' -> 'generated_tensor_name_bar'). This -allows the user to refer to these tensors with their logical names when running -inference. +'long_tensor_name_foo') and ('scores' -> 'generated_tensor_name_bar'). This +allows the user to refer to these tensors with their logical names when +running inference. Note: In addition to the description above, documentation related to signature def structure and how to set up them up can be found [here](signature_defs.md). diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index ba049a47548..06607f7aec5 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -72,6 +72,8 @@ the `tensorflow-serving-api` PIP package using: pip install tensorflow-serving-api ``` +Note: The TensorFlow Serving Python API only supports Python 2.7. It does not support Python 3. + ## Installing using apt-get ### Available binaries diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 413025fa17e..32c43294ad5 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -76,13 +76,15 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/multi_inference.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" #include "tensorflow_serving/servables/tensorflow/regression_service.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" namespace grpc { class ServerCompletionQueue; } // namespace grpc -using tensorflow::serving::AspiredVersionsManager; +using tensorflow::string; using tensorflow::serving::AspiredVersionPolicy; +using tensorflow::serving::AspiredVersionsManager; using tensorflow::serving::AvailabilityPreservingPolicy; using tensorflow::serving::BatchingParameters; using tensorflow::serving::EventBus; @@ -93,10 +95,9 @@ using tensorflow::serving::ServableState; using tensorflow::serving::ServerCore; using tensorflow::serving::SessionBundleConfig; using tensorflow::serving::TensorflowClassificationServiceImpl; -using tensorflow::serving::TensorflowRegressionServiceImpl; using tensorflow::serving::TensorflowPredictor; +using tensorflow::serving::TensorflowRegressionServiceImpl; using tensorflow::serving::UniquePtrWithDeps; -using tensorflow::string; using grpc::InsecureServerCredentials; using grpc::Server; @@ -302,6 +303,7 @@ tensorflow::serving::PlatformConfigMap ParsePlatformConfigMap( int main(int argc, char** argv) { tensorflow::int32 port = 8500; bool enable_batching = false; + float per_process_gpu_memory_fraction = 0; tensorflow::string batching_parameters_file; tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; @@ -346,7 +348,14 @@ int main(int argc, char** argv) { "If non-empty, read an ascii PlatformConfigMap protobuf " "from the supplied file name, and use that platform " "config instead of the Tensorflow platform. (If used, " - "--enable_batching is ignored.)")}; + "--enable_batching is ignored.)"), + tensorflow::Flag( + "per_process_gpu_memory_fraction", &per_process_gpu_memory_fraction, + "Fraction that each process occupies of the GPU memory space " + "the value is between 0.0 and 1.0 (with 0.0 as the default) " + "If 1.0, the server will allocate all the memory when the server " + "starts, If 0.0, Tensorflow will automatically select a value.")}; + string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); if (!parse_result || (model_base_path.empty() && model_config_file.empty())) { @@ -390,6 +399,9 @@ int main(int argc, char** argv) { "--enable_batching"; } + session_bundle_config.mutable_session_config() + ->mutable_gpu_options() + ->set_per_process_gpu_memory_fraction(per_process_gpu_memory_fraction); session_bundle_config.mutable_session_config() ->set_intra_op_parallelism_threads(tensorflow_session_parallelism); session_bundle_config.mutable_session_config() From 7e73cc5fe85e7830e87d574727a9b69d9eef41ee Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 9 Jan 2018 09:32:29 -0800 Subject: [PATCH 0407/8554] Add a --flush_filesystem_caches flag (default false) that, if set, will flush filesystem caches for all registered filesystems after servables are loaded. Change: 181330086 --- .../core/aspired_versions_manager.cc | 2 + .../core/aspired_versions_manager.h | 6 ++ tensorflow_serving/core/basic_manager.cc | 35 ++++++---- tensorflow_serving/core/basic_manager.h | 26 ++++--- tensorflow_serving/core/basic_manager_test.cc | 67 +++++++++++++++++++ tensorflow_serving/model_servers/main.cc | 10 +++ .../model_servers/server_core.cc | 5 ++ .../model_servers/server_core.h | 11 +++ 8 files changed, 142 insertions(+), 20 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 58074fbb91e..9404913621d 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -156,6 +156,8 @@ Status AspiredVersionsManager::Create( basic_manager_options.max_num_load_retries = options.max_num_load_retries; basic_manager_options.load_retry_interval_micros = options.load_retry_interval_micros; + basic_manager_options.flush_filesystem_caches = + options.flush_filesystem_caches; basic_manager_options.servable_event_bus = options.servable_event_bus; basic_manager_options.pre_load_hook = std::move(options.pre_load_hook); std::unique_ptr basic_manager; diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index d8a0e8c9f63..e080184374c 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -123,6 +123,12 @@ class AspiredVersionsManager : public Manager, /// Default: 1 minute. int64 load_retry_interval_micros = 1LL * 60 * 1000 * 1000; + // If true, and there are not multiple load threads, filesystem caches will + // be flushed after each servable is loaded. (Cache flush is skipped when + // multiple load threads are active, in order to avoid setting back a + // concurrent load on another thread.) + bool flush_filesystem_caches = false; + /// The environment to use for starting threads in the thread-pool or for /// sleeping. Env* env = Env::Default(); diff --git a/tensorflow_serving/core/basic_manager.cc b/tensorflow_serving/core/basic_manager.cc index e6c1d86e904..13f0214593a 100644 --- a/tensorflow_serving/core/basic_manager.cc +++ b/tensorflow_serving/core/basic_manager.cc @@ -205,8 +205,8 @@ Status BasicManager::Create(Options options, manager->reset(new BasicManager( options.env, options.num_load_threads, options.num_unload_threads, options.max_num_load_retries, options.load_retry_interval_micros, - std::move(options.resource_tracker), options.servable_event_bus, - std::move(options.pre_load_hook))); + options.flush_filesystem_caches, std::move(options.resource_tracker), + options.servable_event_bus, std::move(options.pre_load_hook))); return Status::OK(); } @@ -214,12 +214,14 @@ BasicManager::BasicManager(Env* const env, const uint32 num_load_threads, const uint32 num_unload_threads, uint32 max_num_load_retries, int64 load_retry_interval_micros, + bool flush_filesystem_caches, std::unique_ptr resource_tracker, EventBus* servable_event_bus, std::function pre_load_hook) : servable_event_bus_(servable_event_bus), env_(env), num_load_threads_(num_load_threads), + flush_filesystem_caches_(flush_filesystem_caches), pre_load_hook_(std::move(pre_load_hook)) { harness_options_.max_num_load_retries = max_num_load_retries; harness_options_.load_retry_interval_micros = load_retry_interval_micros; @@ -229,7 +231,7 @@ BasicManager::BasicManager(Env* const env, const uint32 num_load_threads, }; { - mutex_lock l(num_load_threads_mu_); + mutex_lock l(load_executor_mu_); load_executor_ = CreateExecutor(env_, num_load_threads, "BasicManager_Load_ThreadPool"); } @@ -241,7 +243,7 @@ BasicManager::BasicManager(Env* const env, const uint32 num_load_threads, BasicManager::~BasicManager() { // Reset the executors first to finish all pending loads/unloads. { - mutex_lock l(num_load_threads_mu_); + mutex_lock l(load_executor_mu_); load_executor_.reset(); } unload_executor_.reset(); @@ -462,7 +464,18 @@ Status BasicManager::ExecuteLoad(LoaderHarness* harness) { } // We don't hold the lock while calling Load() as it may block. - TF_RETURN_IF_ERROR(harness->Load()); + const Status status = harness->Load(); + + // Whether the load succeeded or failed, flush filesystem caches if there is + // only one load thread. + if (flush_filesystem_caches_ && num_load_threads() <= 1) { + const Status flush_status = Env::Default()->FlushFileSystemCaches(); + if (!flush_status.ok()) { + LOG(WARNING) << "flushing filesystem caches failed: " << flush_status; + } + } + + TF_RETURN_IF_ERROR(status); { mutex_lock l(mu_); @@ -546,18 +559,16 @@ Status BasicManager::ExecuteLoadOrUnload(const LoadOrUnloadRequest& request, } void BasicManager::SetNumLoadThreads(const uint32 num_load_threads) { - mutex_lock l(num_load_threads_mu_); + mutex_lock l(load_executor_mu_); load_executor_.reset(); - num_load_threads_ = num_load_threads; + num_load_threads_.store(num_load_threads); load_executor_ = - CreateExecutor(env_, num_load_threads_, "BasicManager_Load_ThreadPool"); + CreateExecutor(env_, num_load_threads, "BasicManager_Load_ThreadPool"); } uint32 BasicManager::num_load_threads() const { - mutex_lock l(num_load_threads_mu_); - - return num_load_threads_; + return num_load_threads_.load(); } void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, @@ -585,7 +596,7 @@ void BasicManager::LoadOrUnloadServable(const LoadOrUnloadRequest& request, switch (request.kind) { case LoadOrUnloadRequest::Kind::kLoad: { - mutex_lock l(num_load_threads_mu_); + mutex_lock l(load_executor_mu_); load_executor_->Schedule([this, request, done_callback]() { HandleLoadOrUnloadRequest(request, done_callback); }); diff --git a/tensorflow_serving/core/basic_manager.h b/tensorflow_serving/core/basic_manager.h index 72178973269..eb9eee55c90 100644 --- a/tensorflow_serving/core/basic_manager.h +++ b/tensorflow_serving/core/basic_manager.h @@ -16,6 +16,7 @@ limitations under the License. #ifndef TENSORFLOW_SERVING_CORE_BASIC_MANAGER_H_ #define TENSORFLOW_SERVING_CORE_BASIC_MANAGER_H_ +#include #include #include #include @@ -139,6 +140,12 @@ class BasicManager : public Manager { // Default: 1 minute. int64 load_retry_interval_micros = 1LL * 60 * 1000 * 1000; + // If true, and there are not multiple load threads, filesystem caches will + // be flushed after each servable is loaded. (Cache flush is skipped when + // multiple load threads are active, in order to avoid setting back a + // concurrent load on another thread.) + bool flush_filesystem_caches = false; + // The environment to use for starting threads in the thread-pool. Env* env = Env::Default(); @@ -263,6 +270,7 @@ class BasicManager : public Manager { BasicManager(Env* env, uint32 num_load_threads, uint32 num_unload_threads, uint32 max_num_load_retries, int64 load_retry_interval_micros, + bool flush_filesystem_caches, std::unique_ptr resource_tracker, EventBus* servable_event_bus, PreLoadHook pre_load_hook); @@ -380,8 +388,8 @@ class BasicManager : public Manager { // the old thread pool blocks until all threads are done, so it could block // for a long time. void SetNumLoadThreads(uint32 num_load_threads) - LOCKS_EXCLUDED(num_load_threads_mu_); - uint32 num_load_threads() const LOCKS_EXCLUDED(num_load_threads_mu_); + LOCKS_EXCLUDED(load_executor_mu_); + uint32 num_load_threads() const; // Keys are the servable names. // Values are the harnesses for each servable version. The values when @@ -474,12 +482,14 @@ class BasicManager : public Manager { Env* const env_; - // The number of load threads and the associated executor. They can be changed - // after instantiation of the manager via SetNumLoadThreads(). - mutable mutex num_load_threads_mu_; - uint32 num_load_threads_ GUARDED_BY(num_load_threads_mu_); - // The executor used for executing loads of servables. - std::unique_ptr load_executor_ GUARDED_BY(num_load_threads_mu_); + // The number of load threads. Can be changed after instantiation of the + // manager via SetNumLoadThreads(). + std::atomic num_load_threads_; + // Whether to flush filesystem caches (if num_load_threads_ == 1) + const bool flush_filesystem_caches_ = false; + // The executor (and associated mutex) used for executing loads of servables. + mutable mutex load_executor_mu_; + std::unique_ptr load_executor_ GUARDED_BY(load_executor_mu_); // The executor used for executing unloads of servables. (Unlike for loads, // the unload executor is fixed for the lifetime of the manager.) diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 6b55c4587b2..4e21d6df51d 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -938,6 +938,73 @@ TEST_F(SetNumLoadThreadsBasicManagerTest, FastLoad) { } } +// This filesystem detects a call to FlushCaches(), which is triggered by the +// BasicManager's call to Env::Default()->FlushFileSystemCaches() after loading +// a servable. +class FlushDetectingFileSystem : public NullFileSystem { + public: + void FlushCaches() override { flushed = true; } + static std::atomic flushed; +}; + +std::atomic FlushDetectingFileSystem::flushed; + +REGISTER_FILE_SYSTEM("flush", FlushDetectingFileSystem); + +// This test loads servables with BasicManager::Options::flush_filesystem_caches +// true or false, and verifies that filesystem caches were flushed (or not +// flushed) as expected. +class FlushFileSystemCachesTest : public ::testing::TestWithParam { + protected: + FlushFileSystemCachesTest() : flush_filesystem_caches_(GetParam()) { + BasicManager::Options options; + options.flush_filesystem_caches = flush_filesystem_caches_; + TF_CHECK_OK(BasicManager::Create(std::move(options), &basic_manager_)); + } + + std::unique_ptr basic_manager_; + bool flush_filesystem_caches_; +}; + +TEST_P(FlushFileSystemCachesTest, Load) { + test_util::BasicManagerTestAccess manager_test_access(basic_manager_.get()); + // The number of load threads is initially zero, so filesystems should be + // flushed if flush_filesystem_caches_ is true. + FlushDetectingFileSystem::flushed.store(false); + const ServableId id0 = {kServableName3, 0}; + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id0))); + basic_manager_->LoadServable(id0, [&](const Status& status) { + TF_ASSERT_OK(status); + EXPECT_EQ(flush_filesystem_caches_, + FlushDetectingFileSystem::flushed.load()); + }); + // Load another servable with two load threads. Filesystem caches should not + // be flushed. + manager_test_access.SetNumLoadThreads(2); + FlushDetectingFileSystem::flushed.store(false); + const ServableId id1 = {kServableName3, 1}; + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id1))); + basic_manager_->LoadServable(id1, [&](const Status& status) { + TF_ASSERT_OK(status); + EXPECT_FALSE(FlushDetectingFileSystem::flushed.load()); + }); + // Now move to a single load thread and load a third servable. Filesystem + // caches should once again be flushed if flush_filesystem_caches_ is true. + manager_test_access.SetNumLoadThreads(1); + FlushDetectingFileSystem::flushed.store(false); + const ServableId id2 = {kServableName3, 2}; + TF_CHECK_OK(basic_manager_->ManageServable(CreateServable(id2))); + basic_manager_->LoadServable(id2, [&](const Status& status) { + TF_ASSERT_OK(status); + EXPECT_EQ(flush_filesystem_caches_, + FlushDetectingFileSystem::flushed.load()); + }); + basic_manager_.reset(); +} + +INSTANTIATE_TEST_CASE_P(WithOrWithoutFlush, FlushFileSystemCachesTest, + ::testing::Bool()); + TEST_P(BasicManagerTest, ConcurrentLoadsOnlyOneSucceeds) { const ServableId id = {kServableName3, 0}; mutex status_mu; diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 32c43294ad5..3dbb130030c 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -307,6 +307,7 @@ int main(int argc, char** argv) { tensorflow::string batching_parameters_file; tensorflow::string model_name = "default"; tensorflow::int32 file_system_poll_wait_seconds = 1; + bool flush_filesystem_caches = true; tensorflow::string model_base_path; const bool use_saved_model = true; // Tensorflow session parallelism of zero means that both inter and intra op @@ -338,6 +339,14 @@ int main(int argc, char** argv) { &file_system_poll_wait_seconds, "interval in seconds between each poll of the file " "system for new model version"), + tensorflow::Flag("flush_filesystem_caches", &flush_filesystem_caches, + "If true (the default), filesystem caches will be " + "flushed after the initial load of all servables, and " + "after each subsequent individual servable reload (if " + "the number of load threads is 1). This reduces memory " + "consumption of the model server, at the potential cost " + "of cache misses if model files are accessed after " + "servables are loaded."), tensorflow::Flag("tensorflow_session_parallelism", &tensorflow_session_parallelism, "Number of threads to use for running a " @@ -417,6 +426,7 @@ int main(int argc, char** argv) { options.aspired_version_policy = std::unique_ptr(new AvailabilityPreservingPolicy); options.file_system_poll_wait_seconds = file_system_poll_wait_seconds; + options.flush_filesystem_caches = flush_filesystem_caches; std::unique_ptr core; TF_CHECK_OK(ServerCore::Create(std::move(options), &core)); diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 224f7c68286..9fa4708ebd6 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -459,6 +459,10 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { } TF_RETURN_IF_ERROR(MaybeUpdateServerRequestLogger()); + if (options_.flush_filesystem_caches) { + return Env::Default()->FlushFileSystemCaches(); + } + return Status::OK(); } @@ -632,6 +636,7 @@ Status ServerCore::CreateAspiredVersionsManager( manager_options.num_unload_threads = options_.num_unload_threads; manager_options.max_num_load_retries = options_.max_num_load_retries; manager_options.pre_load_hook = std::move(options_.pre_load_hook); + manager_options.flush_filesystem_caches = options_.flush_filesystem_caches; const tensorflow::Status status = AspiredVersionsManager::Create(std::move(manager_options), manager); if (!status.ok()) { diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 98450d29885..00c01faa690 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -117,6 +117,17 @@ class ServerCore : public Manager { // Time interval between file-system polls, in seconds. int32 file_system_poll_wait_seconds = 30; + // If true, filesystem caches are flushed in the following cases: + // + // 1) After the initial models are loaded. + // 2) After a new config is supplied and a changed set of models are loaded. + // 3) After each new model version is loaded, if num_load_threads == 1. + // + // In the common scenario where the number of load threads is set to 1 after + // the initial load, this will take care of flushing the cache once after + // the initial load, and after every subsequent load of every model version. + bool flush_filesystem_caches = false; + // Configuration for the supported platforms. PlatformConfigMap platform_config_map; From 8d0c2c36f01c95bce945779d015ad65c5324a229 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 9 Jan 2018 11:00:59 -0800 Subject: [PATCH 0408/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index e9020250b50..4cb0c13c777 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit e9020250b50922d0449da44ae5ea7d62ad899b0e +Subproject commit 4cb0c13c7779da536cac6c682180c5757611b384 diff --git a/tf_models b/tf_models index 6db9f0282e2..aeb8cfcba85 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit 6db9f0282e2ab12795628de6200670892a8ad6ba +Subproject commit aeb8cfcba8512c0a3ca3e50b057b7981d35f512f From c1ec43508ee57a5d6269116aba82d2a16d383c8a Mon Sep 17 00:00:00 2001 From: Jesse Kinkead Date: Tue, 9 Jan 2018 17:40:55 -0800 Subject: [PATCH 0409/8554] Clarify that the API works in Python 3. (#721) * Clarify that the API works in Python 3. Also add a passive-aggressive link to the closed issue. * Clean up based on PR feedback. --- tensorflow_serving/g3doc/setup.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 06607f7aec5..55985262b79 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -72,7 +72,12 @@ the `tensorflow-serving-api` PIP package using: pip install tensorflow-serving-api ``` -Note: The TensorFlow Serving Python API only supports Python 2.7. It does not support Python 3. +Note: The TensorFlow Serving Python API +[is only published for Python 2](https://pypi.python.org/pypi/tensorflow-serving-api), +but will work for Python 3 if you either build it yourself, or download the +Python 2 version, unzip it, and copy it into your Python 3 path. There is a +[feature request](https://github.com/tensorflow/serving/issues/700) to publish +the Python 3 package as well. ## Installing using apt-get From 557d6d72e9448df4b6b8ecc9861f3b146a46ca75 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 11 Jan 2018 17:25:05 -0800 Subject: [PATCH 0410/8554] Explicitly asks for the config_case in updating the request logger from ServerCore. Change: 181685462 --- .../model_servers/server_core.cc | 21 ++++++++++++------- .../model_servers/server_core.h | 4 +++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/tensorflow_serving/model_servers/server_core.cc b/tensorflow_serving/model_servers/server_core.cc index 9fa4708ebd6..f58dd2a74de 100644 --- a/tensorflow_serving/model_servers/server_core.cc +++ b/tensorflow_serving/model_servers/server_core.cc @@ -388,20 +388,25 @@ Status ServerCore::AddModelsViaCustomModelConfig() { config_.custom_model_config(), servable_event_bus_.get(), &manager_); } -Status ServerCore::MaybeUpdateServerRequestLogger() { +Status ServerCore::MaybeUpdateServerRequestLogger( + const ModelServerConfig::ConfigCase config_case) { if (options_.server_request_logger_updater) { return options_.server_request_logger_updater( config_, options_.server_request_logger.get()); } - std::map logging_config_map; - for (const auto& model_config : config_.model_config_list().config()) { - if (model_config.has_logging_config()) { - logging_config_map.insert( - {model_config.name(), model_config.logging_config()}); + if (config_case == ModelServerConfig::kModelConfigList) { + std::map logging_config_map; + for (const auto& model_config : config_.model_config_list().config()) { + if (model_config.has_logging_config()) { + logging_config_map.insert( + {model_config.name(), model_config.logging_config()}); + } } + return options_.server_request_logger->Update(logging_config_map); } - return options_.server_request_logger->Update(logging_config_map); + + return Status::OK(); } Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { @@ -457,7 +462,7 @@ Status ServerCore::ReloadConfig(const ModelServerConfig& new_config) { default: return errors::InvalidArgument("Invalid ServerModelConfig"); } - TF_RETURN_IF_ERROR(MaybeUpdateServerRequestLogger()); + TF_RETURN_IF_ERROR(MaybeUpdateServerRequestLogger(config_.config_case())); if (options_.flush_filesystem_caches) { return Env::Default()->FlushFileSystemCaches(); diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index 00c01faa690..aff6705789f 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -312,7 +312,9 @@ class ServerCore : public Manager { Status AddModelsViaCustomModelConfig() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); // Updates the ServerRequestLogger based on the ModelConfigList. - Status MaybeUpdateServerRequestLogger() EXCLUSIVE_LOCKS_REQUIRED(config_mu_); + Status MaybeUpdateServerRequestLogger( + ModelServerConfig::ConfigCase config_case) + EXCLUSIVE_LOCKS_REQUIRED(config_mu_); // ************************************************************************ // Request Processing. From 89b740f9d68a5f5478dffd4e99025ca4bb4b785c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 16 Jan 2018 16:38:00 -0800 Subject: [PATCH 0411/8554] Replace: s/Google Container Engine/Google Kubernetes Engine/ Change: 182130630 --- tensorflow_serving/g3doc/serving_inception.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 83e6c134237..93a41196182 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -188,8 +188,9 @@ $ gcloud auth login --project tensorflow-serving ### Create a container cluster -First we create a [Google Container Engine](https://cloud.google.com/container-engine/) -cluster for service deployment. +First we create a +[Google Kubernetes Engine](https://cloud.google.com/container-engine/) cluster +for service deployment. ```shell $ gcloud container clusters create inception-serving-cluster --num-nodes 5 From a42a23aacb9768e441c6a0dfbb5737336491c1c7 Mon Sep 17 00:00:00 2001 From: Sukriti Ramesh Date: Wed, 17 Jan 2018 10:49:06 -0800 Subject: [PATCH 0412/8554] Support specification of SavedModel tag-sets in TensorFlow Serving. Change: 182237383 --- tensorflow_serving/model_servers/main.cc | 13 ++++++++++++- tensorflow_serving/servables/tensorflow/BUILD | 2 ++ .../tensorflow/saved_model_bundle_factory.cc | 9 ++++++++- .../tensorflow/saved_model_bundle_factory_test.cc | 2 ++ .../tensorflow/session_bundle_config.proto | 4 ++++ 5 files changed, 28 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index 3dbb130030c..d770420167b 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -57,7 +57,9 @@ limitations under the License. #include "grpc++/support/status.h" #include "grpc++/support/status_code_enum.h" #include "grpc/grpc.h" +#include "tensorflow/cc/saved_model/tag_constants.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/init_main.h" #include "tensorflow/core/platform/protobuf.h" @@ -310,6 +312,7 @@ int main(int argc, char** argv) { bool flush_filesystem_caches = true; tensorflow::string model_base_path; const bool use_saved_model = true; + tensorflow::string saved_model_tags = tensorflow::kSavedModelTagServe; // Tensorflow session parallelism of zero means that both inter and intra op // thread pools will be auto configured. tensorflow::int64 tensorflow_session_parallelism = 0; @@ -363,7 +366,10 @@ int main(int argc, char** argv) { "Fraction that each process occupies of the GPU memory space " "the value is between 0.0 and 1.0 (with 0.0 as the default) " "If 1.0, the server will allocate all the memory when the server " - "starts, If 0.0, Tensorflow will automatically select a value.")}; + "starts, If 0.0, Tensorflow will automatically select a value."), + tensorflow::Flag("saved_model_tags", &saved_model_tags, + "Comma-separated set of tags corresponding to the meta " + "graph def to load from SavedModel.")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); @@ -415,6 +421,11 @@ int main(int argc, char** argv) { ->set_intra_op_parallelism_threads(tensorflow_session_parallelism); session_bundle_config.mutable_session_config() ->set_inter_op_parallelism_threads(tensorflow_session_parallelism); + const std::vector tags = + tensorflow::str_util::Split(saved_model_tags, ","); + for (const string& tag : tags) { + *session_bundle_config.add_saved_model_tags() = tag; + } options.platform_config_map = CreateTensorFlowPlatformConfigMap( session_bundle_config, use_saved_model); } else { diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 034b500a075..2211d320158 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -110,6 +110,7 @@ cc_library( ":session_bundle_config_proto", "//tensorflow_serving/resources:resources_proto", "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", @@ -197,6 +198,7 @@ cc_test( ":session_bundle_config_proto", "//tensorflow_serving/core/test_util:test_main", "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:tag_constants", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:test", diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc index 97d58f5b28d..e04b6f25f6b 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory.cc @@ -78,9 +78,16 @@ Status SavedModelBundleFactory::EstimateResourceRequirement( Status SavedModelBundleFactory::CreateSavedModelBundle( const string& path, std::unique_ptr* bundle) { bundle->reset(new SavedModelBundle); + std::unordered_set saved_model_tags( + config_.saved_model_tags().begin(), config_.saved_model_tags().end()); + // Defaults to loading the meta graph def corresponding to the `serve` tag if + // no `saved_model_tags` are specified. + if (saved_model_tags.empty()) { + saved_model_tags.insert(kSavedModelTagServe); + } TF_RETURN_IF_ERROR(LoadSessionBundleOrSavedModelBundle( GetSessionOptions(config_), GetRunOptions(config_), path, - {kSavedModelTagServe}, bundle->get())); + saved_model_tags, bundle->get())); if (!config_.experimental_fixed_input_tensors().empty()) { LOG(INFO) << "Wrapping session to inject fixed input tensors"; std::vector> fixed_input_tensors; diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc index 1d2267f4c4b..eb66e9d9430 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_factory_test.cc @@ -24,6 +24,7 @@ limitations under the License. #include #include #include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/tag_constants.h" #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/core/status_test_util.h" @@ -74,6 +75,7 @@ TEST_F(SavedModelBundleFactoryTest, FixedInputTensors) { fixed_input.AsProtoField(fixed_input_proto.mutable_tensor()); SessionBundleConfig config; + *config.add_saved_model_tags() = kSavedModelTagServe; *config.add_experimental_fixed_input_tensors() = fixed_input_proto; std::unique_ptr session; TF_ASSERT_OK(CreateSession(config, &session)); diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto index 69a3934f2b3..0114c6f7a27 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto +++ b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto @@ -51,6 +51,10 @@ message SessionBundleConfig { // Remove it once resource estimates are moved inside SavedModel. uint64 experimental_transient_ram_bytes_during_load = 5; + // Set of SavedModel tags identifying the specific meta graph def to be + // loaded. + repeated string saved_model_tags = 6; + // EXPERIMENTAL. THIS FIELD MAY CHANGE OR GO AWAY. USE WITH CAUTION. // // Input tensors to append to every Session::Run() call. From 318e22c2c6c847aa1ec220e1247adc652a1e4e23 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 22 Jan 2018 11:04:11 -0800 Subject: [PATCH 0413/8554] Replace check_version with check_bazel_version_at_least (the new name). Change: 182804819 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 431829ff22f..af75a0cae27 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -23,6 +23,6 @@ load("//tensorflow_serving:workspace.bzl", "tf_serving_workspace") tf_serving_workspace() # Specify the minimum required bazel version. -load("@org_tensorflow//tensorflow:workspace.bzl", "check_version") +load("@org_tensorflow//tensorflow:workspace.bzl", "check_bazel_version_at_least") -check_version("0.5.4") +check_bazel_version_at_least("0.5.4") From 99e0661c62d6a10b81fb1734a8f141d46109b7cf Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 22 Jan 2018 13:28:13 -0800 Subject: [PATCH 0414/8554] Sync submodules. --- tensorflow | 2 +- tf_models | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow b/tensorflow index 4cb0c13c777..57b32eabca4 160000 --- a/tensorflow +++ b/tensorflow @@ -1 +1 @@ -Subproject commit 4cb0c13c7779da536cac6c682180c5757611b384 +Subproject commit 57b32eabca4597241120cb4aba8308a431853c30 diff --git a/tf_models b/tf_models index aeb8cfcba85..6fc65ee60ac 160000 --- a/tf_models +++ b/tf_models @@ -1 +1 @@ -Subproject commit aeb8cfcba8512c0a3ca3e50b057b7981d35f512f +Subproject commit 6fc65ee60ac39be0445e5a311b40dc7ccce214d0 From 03df9f3078d01ca820eb0d077ddd2814a331c1dc Mon Sep 17 00:00:00 2001 From: Shintaro Murakami Date: Fri, 26 Jan 2018 01:44:39 +0900 Subject: [PATCH 0415/8554] Fix serving_advanced.md (#740) --- tensorflow_serving/g3doc/serving_advanced.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index bb9c862b02c..8cd51592e41 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -165,8 +165,8 @@ that monitors cloud storage instead of local storage, or you could build a version policy plugin that does version transition in a different way -- in fact, you could even build a custom model plugin that serves non-TensorFlow models. These topics are out of scope for this tutorial. However, you can refer -to the [custom source](custom_source.md) and [custom servable] -(custom_servable.md) tutorials for more information. +to the [custom source](custom_source.md) and +[custom servable](custom_servable.md) tutorials for more information. ## Batching @@ -250,7 +250,7 @@ To put all these into the context of this tutorial: servables that can be loaded. * `AspiredVersionsManager` monitors the export stream, and manages lifecycle - of all SavedModelBundle` servables dynamically. + of all `SavedModelBundle` servables dynamically. `TensorflowPredictImpl::Predict` then just: From 416e5d2d85302d590756101c38a14452b881310c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 23 Jan 2018 11:12:56 -0800 Subject: [PATCH 0416/8554] model_server: Add ModelSpec to Classify/Regress/Predict response. Also fixed the MultiInference handler to populate the ModelSpec in the InferenceResults. Change: 182962504 --- tensorflow_serving/apis/classification.proto | 3 + tensorflow_serving/apis/predict.proto | 3 + tensorflow_serving/apis/regression.proto | 3 + .../tensorflow_model_server_test.py | 103 +++++++++-- tensorflow_serving/servables/tensorflow/BUILD | 51 ++++++ .../tensorflow/classification_service.cc | 6 + .../tensorflow/classification_service_test.cc | 172 ++++++++++++++++++ .../servables/tensorflow/multi_inference.cc | 8 +- .../servables/tensorflow/multi_inference.h | 14 +- .../tensorflow/multi_inference_test.cc | 45 +++-- .../servables/tensorflow/predict_impl.cc | 7 + .../servables/tensorflow/predict_impl_test.cc | 8 + .../tensorflow/regression_service.cc | 6 + .../tensorflow/regression_service_test.cc | 172 ++++++++++++++++++ .../servables/tensorflow/util.cc | 19 ++ .../servables/tensorflow/util.h | 10 + .../servables/tensorflow/util_test.cc | 63 ++++++- 17 files changed, 649 insertions(+), 44 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/classification_service_test.cc create mode 100644 tensorflow_serving/servables/tensorflow/regression_service_test.cc diff --git a/tensorflow_serving/apis/classification.proto b/tensorflow_serving/apis/classification.proto index 2161b3ac880..95ae342a8a5 100644 --- a/tensorflow_serving/apis/classification.proto +++ b/tensorflow_serving/apis/classification.proto @@ -38,6 +38,9 @@ message ClassificationRequest { } message ClassificationResponse { + // Effective Model Specification used for classification. + ModelSpec model_spec = 2; + // Result of the classification. ClassificationResult result = 1; } diff --git a/tensorflow_serving/apis/predict.proto b/tensorflow_serving/apis/predict.proto index 4d3f4a765a2..4a703383564 100644 --- a/tensorflow_serving/apis/predict.proto +++ b/tensorflow_serving/apis/predict.proto @@ -31,6 +31,9 @@ message PredictRequest { // Response for PredictRequest on successful run. message PredictResponse { + // Effective Model Specification used to process PredictRequest. + ModelSpec model_spec = 2; + // Output tensors. map outputs = 1; } diff --git a/tensorflow_serving/apis/regression.proto b/tensorflow_serving/apis/regression.proto index 8864300cd8f..84dc381ab22 100644 --- a/tensorflow_serving/apis/regression.proto +++ b/tensorflow_serving/apis/regression.proto @@ -29,5 +29,8 @@ message RegressionRequest { } message RegressionResponse { + // Effective Model Specification used for regression. + ModelSpec model_spec = 2; + RegressionResult result = 1; } diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index bfff5de79ad..2e66b64f67c 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -38,6 +38,7 @@ from tensorflow_serving.apis import prediction_service_pb2 from tensorflow_serving.apis import regression_pb2 from tensorflow_serving.apis import inference_pb2 +from tensorflow.python.saved_model import signature_constants FLAGS = flags.FLAGS @@ -156,8 +157,12 @@ def RunServerWithModelConfigFile(self, def VerifyPredictRequest(self, model_server_address, expected_output, + expected_version, model_name='default', - specify_output=True): + specify_output=True, + signature_name= + signature_constants. + DEFAULT_SERVING_SIGNATURE_DEF_KEY): """Send PredictionService.Predict request and verify output.""" print 'Sending Predict request...' # Prepare request @@ -180,12 +185,28 @@ def VerifyPredictRequest(self, self.assertIs(types_pb2.DT_FLOAT, result.outputs['y'].dtype) self.assertEquals(1, len(result.outputs['y'].float_val)) self.assertEquals(expected_output, result.outputs['y'].float_val[0]) + self._VerifyModelSpec(result.model_spec, request.model_spec.name, + signature_name, expected_version) def _GetSavedModelBundlePath(self): """Returns a path to a model in SavedModel format.""" return os.path.join(os.environ['TEST_SRCDIR'], 'tf_serving/external/org_tensorflow/tensorflow/', 'cc/saved_model/testdata/half_plus_two') + def _GetModelVersion(self, model_path): + """Returns version of SavedModel/SessionBundle in given path. + + This method assumes there is exactly one directory with an 'int' valued + directory name under `model_path`. + + Args: + model_path: A string representing path to the SavedModel/SessionBundle. + + Returns: + version of SavedModel/SessionBundle in given path. + """ + return int(os.listdir(model_path)[0]) + def _GetSavedModelHalfPlusThreePath(self): """Returns a path to a half_plus_three model in SavedModel format.""" return os.path.join(self.testdata_dir, 'saved_model_half_plus_three') @@ -210,6 +231,26 @@ def _GetBatchingParametersFile(self): """Returns a path to a batching configuration file.""" return os.path.join(self.testdata_dir, 'batching_config.txt') + def _VerifyModelSpec(self, + actual_model_spec, + exp_model_name, + exp_signature_name, + exp_version): + """Verifies model_spec matches expected model name, signature, version. + + Args: + actual_model_spec: An instance of ModelSpec proto. + exp_model_name: A string that represents expected model name. + exp_signature_name: A string that represents expected signature. + exp_version: An integer that represents expected version. + + Returns: + None. + """ + self.assertEquals(actual_model_spec.name, exp_model_name) + self.assertEquals(actual_model_spec.signature_name, exp_signature_name) + self.assertEquals(actual_model_spec.version.value, exp_version) + def testClassify(self): """Test PredictionService.Classify implementation.""" model_path = self._GetSavedModelBundlePath() @@ -238,6 +279,9 @@ def testClassify(self): expected_output = 3.0 self.assertEquals(expected_output, result.result.classifications[0].classes[0].score) + self._VerifyModelSpec(result.model_spec, request.model_spec.name, + request.model_spec.signature_name, + self._GetModelVersion(model_path)) def testRegress(self): """Test PredictionService.Regress implementation.""" @@ -265,6 +309,9 @@ def testRegress(self): self.assertEquals(1, len(result.result.regressions)) expected_output = 3.0 self.assertEquals(expected_output, result.result.regressions[0].value) + self._VerifyModelSpec(result.model_spec, request.model_spec.name, + request.model_spec.signature_name, + self._GetModelVersion(model_path)) def testMultiInference(self): """Test PredictionService.MultiInference implementation.""" @@ -302,23 +349,35 @@ def testMultiInference(self): result.results[0].regression_result.regressions[0].value) self.assertEquals(expected_output, result.results[ 1].classification_result.classifications[0].classes[0].score) + for i in xrange(2): + self._VerifyModelSpec(result.results[i].model_spec, + request.tasks[i].model_spec.name, + request.tasks[i].model_spec.signature_name, + self._GetModelVersion(model_path)) def _TestPredict(self, model_path, - batching_parameters_file=''): + batching_parameters_file='', + signature_name= + signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY): """Helper method to test prediction. Args: model_path: Path to the model on disk. batching_parameters_file: Batching parameters file to use (if left empty, batching is not enabled). + signature_name: Signature name to expect in the PredictResponse. """ atexit.register(self.TerminateProcs) model_server_address = self.RunServer(PickUnusedPort(), 'default', model_path, batching_parameters_file) - self.VerifyPredictRequest(model_server_address, expected_output=3.0) + expected_version = self._GetModelVersion(model_path) + self.VerifyPredictRequest(model_server_address, expected_output=3.0, + expected_version=expected_version, + signature_name=signature_name) self.VerifyPredictRequest( - model_server_address, expected_output=3.0, specify_output=False) + model_server_address, expected_output=3.0, specify_output=False, + expected_version=expected_version, signature_name=signature_name) def testPredictBatching(self): """Test PredictionService.Predict implementation with SessionBundle.""" @@ -344,13 +403,15 @@ def _TestBadModel(self): # case of SavedModel, the export will get up-converted to a SavedModel. # As the bad model will prevent the server from becoming ready, we set the # wait_for_server_ready param to False to avoid blocking/timing out. - model_server_address = self.RunServer( - PickUnusedPort(), - 'default', - os.path.join(self.testdata_dir, 'bad_half_plus_two'), - wait_for_server_ready=False) + model_path = os.path.join(self.testdata_dir, 'bad_half_plus_two'), + model_server_address = self.RunServer(PickUnusedPort(), 'default', + model_path, + wait_for_server_ready=False) with self.assertRaises(face.AbortionError) as error: - self.VerifyPredictRequest(model_server_address, expected_output=3.0) + self.VerifyPredictRequest( + model_server_address, expected_output=3.0, + expected_version=self._GetModelVersion(model_path), + signature_name='') self.assertIs(beta_interfaces.StatusCode.FAILED_PRECONDITION, error.exception.code) @@ -365,20 +426,22 @@ def testGoodModelConfig(self): PickUnusedPort(), self._GetGoodModelConfigFile()) self.VerifyPredictRequest( - model_server_address, model_name='half_plus_two', expected_output=3.0) + model_server_address, model_name='half_plus_two', expected_output=3.0, + expected_version=self._GetModelVersion(self._GetSavedModelBundlePath())) self.VerifyPredictRequest( - model_server_address, - model_name='half_plus_two', - expected_output=3.0, - specify_output=False) + model_server_address, model_name='half_plus_two', + expected_output=3.0, specify_output=False, + expected_version=self._GetModelVersion(self._GetSavedModelBundlePath())) self.VerifyPredictRequest( - model_server_address, model_name='half_plus_three', expected_output=4.0) + model_server_address, model_name='half_plus_three', expected_output=4.0, + expected_version=self._GetModelVersion( + self._GetSavedModelHalfPlusThreePath())) self.VerifyPredictRequest( - model_server_address, - model_name='half_plus_three', - expected_output=4.0, - specify_output=False) + model_server_address, model_name='half_plus_three', expected_output=4.0, + specify_output=False, + expected_version=self._GetModelVersion( + self._GetSavedModelHalfPlusThreePath())) def testBadModelConfig(self): """Test server model configuration from file fails for invalid file.""" diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 2211d320158..570ff8afa7f 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -422,6 +422,7 @@ cc_library( "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/servables/tensorflow:util", "@org_tensorflow//tensorflow/cc/saved_model:loader", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle", @@ -574,12 +575,35 @@ cc_library( "//tensorflow_serving/apis:classifier", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/servables/tensorflow:util", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:signature", "@org_tensorflow//tensorflow/core:lib", ], ) +cc_test( + name = "classification_service_test", + srcs = ["classification_service_test.cc"], + data = [ + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + ], + deps = [ + ":classification_service", + ":session_bundle_config_proto", + "//tensorflow_serving/config:model_server_config_proto", + "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", + "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:test", + ], +) + cc_library( name = "regression_service", srcs = ["regression_service.cc"], @@ -593,12 +617,35 @@ cc_library( "//tensorflow_serving/apis:regressor", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/servables/tensorflow:util", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:signature", "@org_tensorflow//tensorflow/core:lib", ], ) +cc_test( + name = "regression_service_test", + srcs = ["regression_service_test.cc"], + data = [ + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + ], + deps = [ + ":regression_service", + ":session_bundle_config_proto", + "//tensorflow_serving/config:model_server_config_proto", + "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", + "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:test", + ], +) + cc_library( name = "regressor", srcs = ["regressor.cc"], @@ -703,7 +750,10 @@ cc_library( ], deps = [ "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/apis/internal:serialized_input_proto", + "//tensorflow_serving/util:optional", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", @@ -719,6 +769,7 @@ cc_test( ":util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", diff --git a/tensorflow_serving/servables/tensorflow/classification_service.cc b/tensorflow_serving/servables/tensorflow/classification_service.cc index 4116fca2763..9f27897c08e 100644 --- a/tensorflow_serving/servables/tensorflow/classification_service.cc +++ b/tensorflow_serving/servables/tensorflow/classification_service.cc @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow_serving/apis/classifier.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/servables/tensorflow/classifier.h" +#include "tensorflow_serving/servables/tensorflow/util.h" namespace tensorflow { namespace serving { @@ -49,6 +50,11 @@ Status TensorflowClassificationServiceImpl::Classify( TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( run_options, saved_model_bundle->session.get(), &signature, &classifier_interface)); + + MakeModelSpec( + request.model_spec().name(), request.model_spec().signature_name(), + saved_model_bundle.id().version, response->mutable_model_spec()); + // Run classification. return classifier_interface->Classify(request, response->mutable_result()); } diff --git a/tensorflow_serving/servables/tensorflow/classification_service_test.cc b/tensorflow_serving/servables/tensorflow/classification_service_test.cc new file mode 100644 index 00000000000..159f73b1738 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/classification_service_test.cc @@ -0,0 +1,172 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/classification_service.h" + +#include +#include + +#include +#include +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow_serving/config/model_server_config.pb.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; + +// Test fixture for ClassificationService related tests sets up a ServerCore +// pointing to the half_plus_two SavedModel. +class ClassificationServiceTest : public ::testing::Test { + public: + static void SetUpTestCase() { + ModelServerConfig config; + auto model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two")); + model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + options.model_server_config = config; + options.platform_config_map = + CreateTensorFlowPlatformConfigMap(SessionBundleConfig(), true); + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + TF_ASSERT_OK(ServerCore::Create(std::move(options), &server_core_)); + } + + static void TearDownTestCase() { server_core_ = nullptr; } + + protected: + static std::unique_ptr server_core_; +}; + +std::unique_ptr ClassificationServiceTest::server_core_; + +// Verifies that Classify() returns an error for different cases of an invalid +// ClassificationRequest.model_spec. +TEST_F(ClassificationServiceTest, InvalidModelSpec) { + ClassificationRequest request; + ClassificationResponse response; + + // No model_spec specified. + EXPECT_EQ(TensorflowClassificationServiceImpl::Classify( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::INVALID_ARGUMENT); + + // No model name specified. + auto* model_spec = request.mutable_model_spec(); + EXPECT_EQ(TensorflowClassificationServiceImpl::Classify( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::INVALID_ARGUMENT); + + // No servable found for model name "foo". + model_spec->set_name("foo"); + EXPECT_EQ(TensorflowClassificationServiceImpl::Classify( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::NOT_FOUND); +} + +// Verifies that Classify() returns an error for an invalid signature_name in +// ClassificationRequests's model_spec. +TEST_F(ClassificationServiceTest, InvalidSignature) { + auto request = test_util::CreateProto(R"( + model_spec { + name: "test_model" + signature_name: "invalid_signature_name" + })"); + ClassificationResponse response; + EXPECT_EQ(TensorflowClassificationServiceImpl::Classify( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::INVALID_ARGUMENT); +} + +// Verifies that Classify() returns the correct score for a valid +// ClassificationRequest against the half_plus_two SavedModel's classify_x_to_y +// signature. +TEST_F(ClassificationServiceTest, ClassificationSuccess) { + auto request = test_util::CreateProto(R"( + model_spec { + name: "test_model" + signature_name: "classify_x_to_y" + } + input { + example_list { + examples { + features { + feature: { + key : "x" + value: { + float_list: { + value: [ 80.0 ] + } + } + } + feature: { + key : "locale" + value: { + bytes_list: { + value: [ "pt_BR" ] + } + } + } + feature: { + key : "age" + value: { + float_list: { + value: [ 19.0 ] + } + } + } + } + } + } + })"); + ClassificationResponse response; + TF_EXPECT_OK(TensorflowClassificationServiceImpl::Classify( + RunOptions(), server_core_.get(), request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(R"( + result { classifications { classes { score: 42 } } } + model_spec { + name: "test_model" + signature_name: "classify_x_to_y" + version { value: 123 } + })")); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.cc b/tensorflow_serving/servables/tensorflow/multi_inference.cc index 36adbe37f60..1a21c151072 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference.cc @@ -121,6 +121,10 @@ Status TensorFlowMultiInferenceRunner::Infer( return errors::InvalidArgument("Unrecognized signature method_name: ", task.method_name()); } + MakeModelSpec(task.model_spec().name(), task.model_spec().signature_name(), + servable_version_, + response->mutable_results(response->results_size() - 1) + ->mutable_model_spec()); } return Status::OK(); } @@ -144,8 +148,8 @@ Status RunMultiInference(const RunOptions& run_options, ServerCore* core, TF_RETURN_IF_ERROR( core->GetServableHandle(GetModelSpecFromRequest(request), &bundle)); - TensorFlowMultiInferenceRunner inference_runner(bundle->session.get(), - &bundle->meta_graph_def); + TensorFlowMultiInferenceRunner inference_runner( + bundle->session.get(), &bundle->meta_graph_def, bundle.id().version); return inference_runner.Infer(run_options, request, response); } diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.h b/tensorflow_serving/servables/tensorflow/multi_inference.h index 708db9cb7b8..c8a42ea8df3 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.h +++ b/tensorflow_serving/servables/tensorflow/multi_inference.h @@ -20,6 +20,7 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/apis/inference.pb.h" #include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -30,7 +31,15 @@ class TensorFlowMultiInferenceRunner { public: TensorFlowMultiInferenceRunner(Session* session, const MetaGraphDef* meta_graph_def) - : session_(session), meta_graph_def_(meta_graph_def) {} + : TensorFlowMultiInferenceRunner(session, meta_graph_def, + /*servable_version*/ {}) {} + + TensorFlowMultiInferenceRunner(Session* session, + const MetaGraphDef* meta_graph_def, + optional servable_version) + : session_(session), + meta_graph_def_(meta_graph_def), + servable_version_(servable_version) {} // Run inference and return the inference results in the same order as the // InferenceTasks in the request. @@ -43,6 +52,9 @@ class TensorFlowMultiInferenceRunner { private: Session* const session_; const MetaGraphDef* const meta_graph_def_; + // If available, servable_version is used to set the ModelSpec version in the + // InferenceResults of the MultiInferenceResponse. + const optional servable_version_; }; Status RunMultiInference(const RunOptions& run_options, ServerCore* core, diff --git a/tensorflow_serving/servables/tensorflow/multi_inference_test.cc b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc index 0a1a33999b2..00123d1b586 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference_test.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc @@ -22,9 +22,9 @@ limitations under the License. #include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/example/feature.pb.h" #include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/classification.pb.h" #include "tensorflow_serving/apis/input.pb.h" #include "tensorflow_serving/apis/regression.pb.h" -#include "tensorflow_serving/apis/classification.pb.h" #include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" @@ -79,10 +79,12 @@ class MultiInferenceTest : public ::testing::Test { TF_RETURN_IF_ERROR(GetServerCore()->GetServableHandle(model_spec, &bundle)); inference_runner->reset(new TensorFlowMultiInferenceRunner( - bundle->session.get(), &bundle->meta_graph_def)); + bundle->session.get(), &bundle->meta_graph_def, {servable_version_})); return Status::OK(); } + const int64 servable_version_ = 1; + private: static std::unique_ptr server_core_; }; @@ -224,8 +226,11 @@ TEST_F(MultiInferenceTest, ValidSingleSignatureTest) { PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); MultiInferenceResponse expected_response; - auto* regression_result = - expected_response.add_results()->mutable_regression_result(); + auto* inference_result = expected_response.add_results(); + auto* model_spec = inference_result->mutable_model_spec(); + *model_spec = request.tasks(0).model_spec(); + model_spec->mutable_version()->set_value(servable_version_); + auto* regression_result = inference_result->mutable_regression_result(); regression_result->add_regressions()->set_value(3.0); MultiInferenceResponse response; @@ -242,14 +247,22 @@ TEST_F(MultiInferenceTest, MultipleValidRegressSignaturesTest) { PopulateTask("regress_x_to_y", kRegressMethodName, request.add_tasks()); PopulateTask("regress_x_to_y2", kRegressMethodName, request.add_tasks()); - // regress_x_to_y is y = 0.5x + 2. MultiInferenceResponse expected_response; - auto* regression_result_1 = - expected_response.add_results()->mutable_regression_result(); + + // regress_x_to_y is y = 0.5x + 2. + auto* inference_result_1 = expected_response.add_results(); + auto* model_spec_1 = inference_result_1->mutable_model_spec(); + *model_spec_1 = request.tasks(0).model_spec(); + model_spec_1->mutable_version()->set_value(servable_version_); + auto* regression_result_1 = inference_result_1->mutable_regression_result(); regression_result_1->add_regressions()->set_value(3.0); + // regress_x_to_y2 is y2 = 0.5x + 3. - auto* regression_result_2 = - expected_response.add_results()->mutable_regression_result(); + auto* inference_result_2 = expected_response.add_results(); + auto* model_spec_2 = inference_result_2->mutable_model_spec(); + *model_spec_2 = request.tasks(1).model_spec(); + model_spec_2->mutable_version()->set_value(servable_version_); + auto* regression_result_2 = inference_result_2->mutable_regression_result(); regression_result_2->add_regressions()->set_value(4.0); MultiInferenceResponse response; @@ -267,11 +280,19 @@ TEST_F(MultiInferenceTest, RegressAndClassifySignaturesTest) { PopulateTask("classify_x_to_y", kClassifyMethodName, request.add_tasks()); MultiInferenceResponse expected_response; - auto* regression_result = - expected_response.add_results()->mutable_regression_result(); + auto* inference_result_1 = expected_response.add_results(); + auto* model_spec_1 = inference_result_1->mutable_model_spec(); + *model_spec_1 = request.tasks(0).model_spec(); + model_spec_1->mutable_version()->set_value(servable_version_); + auto* regression_result = inference_result_1->mutable_regression_result(); regression_result->add_regressions()->set_value(3.0); + + auto* inference_result_2 = expected_response.add_results(); + auto* model_spec_2 = inference_result_2->mutable_model_spec(); + *model_spec_2 = request.tasks(1).model_spec(); + model_spec_2->mutable_version()->set_value(servable_version_); auto* classification_result = - expected_response.add_results()->mutable_classification_result(); + inference_result_2->mutable_classification_result(); classification_result->add_classifications()->add_classes()->set_score(3.0); MultiInferenceResponse response; diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index c4e52d499ee..058ebd2c086 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -29,6 +29,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/protobuf/named_tensor.pb.h" #include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/servables/tensorflow/util.h" namespace tensorflow { namespace serving { @@ -111,6 +112,9 @@ Status SessionBundlePredict(const RunOptions& run_options, ServerCore* core, } } + MakeModelSpec(request.model_spec().name(), /*signature_name=*/{}, + bundle.id().version, response->mutable_model_spec()); + // Run session. std::vector outputs; RunMetadata run_metadata; @@ -259,6 +263,9 @@ Status SavedModelPredict(const RunOptions& run_options, ServerCore* core, } SignatureDef signature = iter->second; + MakeModelSpec(request.model_spec().name(), signature_name, + bundle.id().version, response->mutable_model_spec()); + std::vector> input_tensors; std::vector output_tensor_names; std::vector output_tensor_aliases; diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 46d276a0e1c..9bb40588e2e 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -278,6 +278,12 @@ TEST_P(PredictImplTest, PredictionSuccess) { output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); output_tensor_proto.mutable_tensor_shape(); PredictResponse expected_response; + *expected_response.mutable_model_spec() = *model_spec; + // signature_name in ModelSpec is populated only for SavedModels. + if (GetParam()) { + expected_response.mutable_model_spec()->set_signature_name( + kDefaultServingSignatureDefKey); + } (*expected_response.mutable_outputs())[kOutputTensorKey] = output_tensor_proto; EXPECT_THAT(response, test_util::EqualsProto(expected_response)); @@ -315,6 +321,7 @@ TEST_P(PredictImplTest, PredictionWithNamedRegressionSignature) { output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); output_tensor_proto.mutable_tensor_shape(); PredictResponse expected_response; + *expected_response.mutable_model_spec() = *model_spec; (*expected_response.mutable_outputs())[kRegressOutputs] = output_tensor_proto; EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } @@ -353,6 +360,7 @@ TEST_P(PredictImplTest, PredictionWithNamedClassificationSignature) { output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); output_tensor_proto.mutable_tensor_shape(); PredictResponse expected_response; + *expected_response.mutable_model_spec() = *model_spec; (*expected_response.mutable_outputs())[kClassifyOutputScores] = output_tensor_proto; EXPECT_THAT(response, test_util::EqualsProto(expected_response)); diff --git a/tensorflow_serving/servables/tensorflow/regression_service.cc b/tensorflow_serving/servables/tensorflow/regression_service.cc index 1c616914041..9241317fc88 100644 --- a/tensorflow_serving/servables/tensorflow/regression_service.cc +++ b/tensorflow_serving/servables/tensorflow/regression_service.cc @@ -22,6 +22,7 @@ limitations under the License. #include "tensorflow_serving/apis/regressor.h" #include "tensorflow_serving/core/servable_handle.h" #include "tensorflow_serving/servables/tensorflow/regressor.h" +#include "tensorflow_serving/servables/tensorflow/util.h" namespace tensorflow { namespace serving { @@ -47,6 +48,11 @@ Status TensorflowRegressionServiceImpl::Regress( TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( run_options, saved_model_bundle->session.get(), &signature, ®ressor_interface)); + + MakeModelSpec( + request.model_spec().name(), request.model_spec().signature_name(), + saved_model_bundle.id().version, response->mutable_model_spec()); + // Run regression return regressor_interface->Regress(request, response->mutable_result()); } diff --git a/tensorflow_serving/servables/tensorflow/regression_service_test.cc b/tensorflow_serving/servables/tensorflow/regression_service_test.cc new file mode 100644 index 00000000000..401744e7684 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/regression_service_test.cc @@ -0,0 +1,172 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/regression_service.h" + +#include +#include + +#include +#include +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow_serving/config/model_server_config.pb.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; + +// Test fixture for RegressionService related tests sets up a ServerCore +// pointing to the half_plus_two SavedModel. +class RegressionServiceTest : public ::testing::Test { + public: + static void SetUpTestCase() { + ModelServerConfig config; + auto model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two")); + model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + options.model_server_config = config; + options.platform_config_map = + CreateTensorFlowPlatformConfigMap(SessionBundleConfig(), true); + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + TF_ASSERT_OK(ServerCore::Create(std::move(options), &server_core_)); + } + + static void TearDownTestCase() { server_core_ = nullptr; } + + protected: + static std::unique_ptr server_core_; +}; + +std::unique_ptr RegressionServiceTest::server_core_; + +// Verifies that Regress() returns an error for different cases of an invalid +// RegressionRequest.model_spec. +TEST_F(RegressionServiceTest, InvalidModelSpec) { + RegressionRequest request; + RegressionResponse response; + + // No model_spec specified. + EXPECT_EQ(TensorflowRegressionServiceImpl::Regress( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::INVALID_ARGUMENT); + + // No model name specified. + auto* model_spec = request.mutable_model_spec(); + EXPECT_EQ(TensorflowRegressionServiceImpl::Regress( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::INVALID_ARGUMENT); + + // No servable found for model name "foo". + model_spec->set_name("foo"); + EXPECT_EQ(TensorflowRegressionServiceImpl::Regress( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::NOT_FOUND); +} + +// Verifies that Regress() returns an error for an invalid signature_name in +// RegressionRequests's model_spec. +TEST_F(RegressionServiceTest, InvalidSignature) { + auto request = test_util::CreateProto(R"( + model_spec { + name: "test_model" + signature_name: "invalid_signature_name" + })"); + RegressionResponse response; + EXPECT_EQ(TensorflowRegressionServiceImpl::Regress( + RunOptions(), server_core_.get(), request, &response) + .code(), + tensorflow::error::INVALID_ARGUMENT); +} + +// Verifies that Regress() returns the correct value for a valid +// RegressionRequest against the half_plus_two SavedModel's regress_x_to_y +// signature. +TEST_F(RegressionServiceTest, RegressionSuccess) { + auto request = test_util::CreateProto(R"( + model_spec { + name: "test_model" + signature_name: "regress_x_to_y" + } + input { + example_list { + examples { + features { + feature: { + key : "x" + value: { + float_list: { + value: [ 80.0 ] + } + } + } + feature: { + key : "locale" + value: { + bytes_list: { + value: [ "pt_BR" ] + } + } + } + feature: { + key : "age" + value: { + float_list: { + value: [ 19.0 ] + } + } + } + } + } + } + })"); + RegressionResponse response; + TF_EXPECT_OK(TensorflowRegressionServiceImpl::Regress( + RunOptions(), server_core_.get(), request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(R"( + result { regressions { value: 42 } } + model_spec { + name: "test_model" + signature_name: "regress_x_to_y" + version { value: 123 } + })")); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/util.cc b/tensorflow_serving/servables/tensorflow/util.cc index f4badcf8c90..fc14965680b 100644 --- a/tensorflow_serving/servables/tensorflow/util.cc +++ b/tensorflow_serving/servables/tensorflow/util.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/util.h" +#include "google/protobuf/wrappers.pb.h" +#include "tensorflow/cc/saved_model/signature_constants.h" #include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" @@ -22,6 +24,8 @@ limitations under the License. #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/apis/input.pb.h" #include "tensorflow_serving/apis/internal/serialized_input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -119,5 +123,20 @@ Status PerformOneShotTensorComputation( output_tensor_names, {}, outputs, &run_metadata); } +void MakeModelSpec(const string& model_name, + const optional& signature_name, + const optional& version, ModelSpec* model_spec) { + model_spec->Clear(); + model_spec->set_name(model_name); + if (signature_name) { + model_spec->set_signature_name(signature_name->empty() + ? kDefaultServingSignatureDefKey + : *signature_name); + } + if (version) { + model_spec->mutable_version()->set_value(*version); + } +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/util.h b/tensorflow_serving/servables/tensorflow/util.h index cf6590bb38a..404e21048da 100644 --- a/tensorflow_serving/servables/tensorflow/util.h +++ b/tensorflow_serving/servables/tensorflow/util.h @@ -21,6 +21,8 @@ limitations under the License. #include "tensorflow/core/lib/monitoring/sampler.h" #include "tensorflow/core/public/session.h" #include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -59,6 +61,14 @@ Status PerformOneShotTensorComputation( const std::vector& output_tensor_names, Session* session, std::vector* outputs, int* num_input_examples); +// Populates given model_spec based on the model name and optional +// signature/version information. +// If signature_name has a value and is empty, model_spec's signature_name is +// set to tensorflow::kDefaultServingSignatureDefKey. +void MakeModelSpec(const string& model_name, + const optional& signature_name, + const optional& version, ModelSpec* model_spec); + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/util_test.cc b/tensorflow_serving/servables/tensorflow/util_test.cc index 64be84c55ff..e79f984d3a0 100644 --- a/tensorflow_serving/servables/tensorflow/util_test.cc +++ b/tensorflow_serving/servables/tensorflow/util_test.cc @@ -15,6 +15,8 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/util.h" +#include "google/protobuf/wrappers.pb.h" +#include "tensorflow/cc/saved_model/signature_constants.h" #include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/example/feature.pb.h" #include "tensorflow/core/framework/tensor_shape.h" @@ -27,8 +29,9 @@ namespace tensorflow { namespace serving { namespace { -using ::testing::HasSubstr; using test_util::EqualsProto; +using ::testing::Eq; +using ::testing::HasSubstr; class InputUtilTest : public ::testing::Test { protected: @@ -112,18 +115,18 @@ TEST_F(InputUtilTest, ExampleListWithContext) { { Example serialized_example; ASSERT_TRUE(serialized_example.ParseFromString(vec(0))); - EXPECT_THAT(serialized_example.features().feature().at("c"), EqualsProto( - example_C().features().feature().at("c"))); - EXPECT_THAT(serialized_example.features().feature().at("a"), EqualsProto( - example_A().features().feature().at("a"))); + EXPECT_THAT(serialized_example.features().feature().at("c"), + EqualsProto(example_C().features().feature().at("c"))); + EXPECT_THAT(serialized_example.features().feature().at("a"), + EqualsProto(example_A().features().feature().at("a"))); } { Example serialized_example; ASSERT_TRUE(serialized_example.ParseFromString(vec(1))); - EXPECT_THAT(serialized_example.features().feature().at("c"), EqualsProto( - example_C().features().feature().at("c"))); - EXPECT_THAT(serialized_example.features().feature().at("b"), EqualsProto( - example_B().features().feature().at("b"))); + EXPECT_THAT(serialized_example.features().feature().at("c"), + EqualsProto(example_C().features().feature().at("c"))); + EXPECT_THAT(serialized_example.features().feature().at("b"), + EqualsProto(example_B().features().feature().at("b"))); } } @@ -216,6 +219,48 @@ TEST(ExampleCountsTest, Simple) { EXPECT_EQ(1, after_histogram.bucket(2) - before_histogram.bucket(2)); } +TEST(ModelSpecTest, NoOptional) { + ModelSpec model_spec; + MakeModelSpec("foo", /*signature_name=*/{}, /*version=*/{}, &model_spec); + EXPECT_THAT(model_spec.name(), Eq("foo")); + EXPECT_THAT(model_spec.signature_name(), ::testing::IsEmpty()); + EXPECT_FALSE(model_spec.has_version()); +} + +TEST(ModelSpecTest, OptionalSignature) { + ModelSpec model_spec; + MakeModelSpec("foo", /*signature_name=*/{"classify"}, /*version=*/{}, + &model_spec); + EXPECT_THAT(model_spec.name(), Eq("foo")); + EXPECT_THAT(model_spec.signature_name(), Eq("classify")); + EXPECT_FALSE(model_spec.has_version()); +} + +TEST(ModelSpecTest, EmptySignature) { + ModelSpec model_spec; + MakeModelSpec("foo", /*signature_name=*/{""}, /*version=*/{1}, &model_spec); + EXPECT_THAT(model_spec.name(), Eq("foo")); + EXPECT_THAT(model_spec.signature_name(), Eq(kDefaultServingSignatureDefKey)); + EXPECT_THAT(model_spec.version().value(), Eq(1)); +} + +TEST(ModelSpecTest, OptionalVersion) { + ModelSpec model_spec; + MakeModelSpec("foo", /*signature_name=*/{}, /*version=*/{1}, &model_spec); + EXPECT_THAT(model_spec.name(), Eq("foo")); + EXPECT_THAT(model_spec.signature_name(), ::testing::IsEmpty()); + EXPECT_THAT(model_spec.version().value(), Eq(1)); +} + +TEST(ModelSpecTest, AllOptionalSet) { + ModelSpec model_spec; + MakeModelSpec("foo", /*signature_name=*/{"classify"}, /*version=*/{1}, + &model_spec); + EXPECT_THAT(model_spec.name(), Eq("foo")); + EXPECT_THAT(model_spec.signature_name(), Eq("classify")); + EXPECT_THAT(model_spec.version().value(), Eq(1)); +} + } // namespace } // namespace serving } // namespace tensorflow From 58c7e415364e5c5d8c63a16fd86d00bdf3959602 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 23 Jan 2018 13:48:05 -0800 Subject: [PATCH 0417/8554] Renaming ":tensorflow_model_server" dependency for the pkg_tar with name = "tensorflow_model_server_tar" to 'srcs' instead of 'files' grouping. Change: 182986029 --- tensorflow_serving/model_servers/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index aac64ef15ed..140cc26029b 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -184,7 +184,7 @@ load("@bazel_tools//tools/build_defs/pkg:pkg.bzl", "pkg_deb", "pkg_tar") pkg_tar( name = "tensorflow_model_server_tar", - files = [ + srcs = [ ":tensorflow_model_server", ], package_dir = "/usr/bin", From 7fbfa1c0606e5fd067fe8109dec4ffbcdebe7bee Mon Sep 17 00:00:00 2001 From: Michael Case Date: Tue, 23 Jan 2018 14:21:16 -0800 Subject: [PATCH 0418/8554] Pull in TensorFlow and inception model as Bazel deps in serving. Currently, these are pulled in as gitsubmodules. This complicates the open-sourcing workflow and the build. Would like to change these to Bazel deps instead. Change: 182991873 --- WORKSPACE | 24 ++++++++++++++++++------ tensorflow_serving/workspace.bzl | 16 ++++++++++------ 2 files changed, 28 insertions(+), 12 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index af75a0cae27..c4aeac89006 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,19 +1,31 @@ workspace(name = "tf_serving") -local_repository( +# To update TensorFlow to a new revision. +# 1. Update the 'urls' and 'strip_prefix' args below to include the new git hash. +# 2. Get the sha256 hash of the archive with a command such as... +# curl -L https://github.com/tensorflow/tensorflow/archive/.tar.gz | sha256sum +# and update the 'sha256' arg with the result. +# 3. Request the new archive to be mirrored on mirror.bazel.build for more +# reliable downloads. +http_archive( name = "org_tensorflow", - path = "tensorflow", + sha256 = "9c3db796b2c6bc24ebff94d689c442f2cf1d8955f122a277e4ad893c0ce96aac", + strip_prefix = "tensorflow-57b32eabca4597241120cb4aba8308a431853c30", + urls = [ + "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/57b32eabca4597241120cb4aba8308a431853c30.tar.gz", + "https://github.com/tensorflow/tensorflow/archive/57b32eabca4597241120cb4aba8308a431853c30.tar.gz", + ], ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. http_archive( name = "io_bazel_rules_closure", - sha256 = "110fe68753413777944b473c25eed6368c4a0487cee23a7bac1b13cc49d3e257", - strip_prefix = "rules_closure-4af89ef1db659eb41f110df189b67d4cf14073e1", + sha256 = "dbe0da2cca88194d13dc5a7125a25dd7b80e1daec7839f33223de654d7a1bcc8", + strip_prefix = "rules_closure-ba3e07cb88be04a2d4af7009caa0ff3671a79d06", urls = [ - "http://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", - "https://github.com/bazelbuild/rules_closure/archive/4af89ef1db659eb41f110df189b67d4cf14073e1.tar.gz", # 2017-08-28 + "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/ba3e07cb88be04a2d4af7009caa0ff3671a79d06.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/ba3e07cb88be04a2d4af7009caa0ff3671a79d06.tar.gz", # 2017-10-31 ], ) diff --git a/tensorflow_serving/workspace.bzl b/tensorflow_serving/workspace.bzl index 8035841245c..657649b1cd4 100644 --- a/tensorflow_serving/workspace.bzl +++ b/tensorflow_serving/workspace.bzl @@ -3,14 +3,18 @@ load('@org_tensorflow//tensorflow:workspace.bzl', 'tf_workspace') -# All TensorFlow Serving external dependencies. -# workspace_dir is the absolute path to the TensorFlow Serving repo. If linked -# as a submodule, it'll likely be '__workspace_dir__ + "/serving"' def tf_serving_workspace(): - native.new_local_repository( + '''All TensorFlow Serving external dependencies.''' + # The inception model's BUILD file is written as if it is the root BUILD + # file. We use strip_prefix to make the inception model directory the root. + native.http_archive( name = "inception_model", - path = "tf_models/research/inception", - build_file = "tf_models/research/inception/inception/BUILD", + urls = [ + "https://mirror.bazel.build/github.com/tensorflow/models/archive/6fc65ee60ac39be0445e5a311b40dc7ccce214d0.tar.gz", + "https://github.com/tensorflow/models/archive/6fc65ee60ac39be0445e5a311b40dc7ccce214d0.tar.gz", + ], + sha256 = "7a908017d60fca54c80405527576f08dbf8d130efe6a53791639ff3b26afffbc", + strip_prefix = "models-6fc65ee60ac39be0445e5a311b40dc7ccce214d0/research/inception", ) tf_workspace(path_prefix = "", tf_repo_name = "org_tensorflow") From c6bb916518d72c385815fb84c21073cce60b591a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 29 Jan 2018 11:42:49 -0800 Subject: [PATCH 0419/8554] Add implementation of GetModelStatus for ModelService without actually using ModelService in model server yet. This is a first step towards implementing the ModelService API. Change: 183700298 --- tensorflow_serving/apis/BUILD | 25 +++ tensorflow_serving/apis/classification.proto | 3 +- .../apis/get_model_metadata.proto | 1 + .../apis/get_model_status.proto | 67 +++++++ tensorflow_serving/apis/inference.proto | 2 + tensorflow_serving/apis/model.proto | 5 +- tensorflow_serving/apis/model_service.proto | 17 ++ tensorflow_serving/apis/predict.proto | 3 +- tensorflow_serving/apis/regression.proto | 3 +- tensorflow_serving/model_servers/BUILD | 51 +++++ .../model_servers/get_model_status_impl.cc | 104 ++++++++++ .../model_servers/get_model_status_impl.h | 40 ++++ .../get_model_status_impl_test.cc | 177 ++++++++++++++++++ .../00000123/assets/foo.txt | 1 + .../00000123/saved_model.pb | Bin 0 -> 9935 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 12 bytes .../00000123/variables/variables.index | Bin 0 -> 151 bytes .../00000124/assets/foo.txt | 1 + .../00000124/saved_model.pb | Bin 0 -> 9935 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 12 bytes .../00000124/variables/variables.index | Bin 0 -> 151 bytes tensorflow_serving/util/BUILD | 31 +++ tensorflow_serving/util/status.proto | 17 ++ tensorflow_serving/util/status_util.cc | 37 ++++ tensorflow_serving/util/status_util.h | 33 ++++ tensorflow_serving/util/status_util_test.cc | 58 ++++++ 26 files changed, 669 insertions(+), 7 deletions(-) create mode 100644 tensorflow_serving/apis/get_model_status.proto create mode 100644 tensorflow_serving/apis/model_service.proto create mode 100644 tensorflow_serving/model_servers/get_model_status_impl.cc create mode 100644 tensorflow_serving/model_servers/get_model_status_impl.h create mode 100644 tensorflow_serving/model_servers/get_model_status_impl_test.cc create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/assets/foo.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/saved_model.pb create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/variables/variables.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/variables/variables.index create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/assets/foo.txt create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/saved_model.pb create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/variables/variables.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/variables/variables.index create mode 100644 tensorflow_serving/util/status.proto create mode 100644 tensorflow_serving/util/status_util.cc create mode 100644 tensorflow_serving/util/status_util.h create mode 100644 tensorflow_serving/util/status_util_test.cc diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 2c8487441bd..1a47cab655d 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -156,6 +156,31 @@ serving_go_grpc_library( deps = [":prediction_service_go_proto"], ) +serving_proto_library( + name = "get_model_status_proto", + srcs = ["get_model_status.proto"], + cc_api_version = 2, + go_api_version = 2, + java_api_version = 2, + deps = [ + ":model_proto", + "//tensorflow_serving/util:status_proto", + ], +) + +serving_proto_library( + name = "model_service_proto", + srcs = ["model_service.proto"], + has_services = 1, + cc_api_version = 2, + cc_grpc_version = 1, + go_api_version = 2, + java_api_version = 2, + deps = [ + ":get_model_status_proto", + ], +) + serving_proto_library( name = "classification_proto", srcs = ["classification.proto"], diff --git a/tensorflow_serving/apis/classification.proto b/tensorflow_serving/apis/classification.proto index 95ae342a8a5..aa81bd8c19b 100644 --- a/tensorflow_serving/apis/classification.proto +++ b/tensorflow_serving/apis/classification.proto @@ -30,7 +30,8 @@ message ClassificationResult { // RPC Interfaces message ClassificationRequest { - // Model Specification. + // Model Specification. If version is not specified, will use the latest + // (numerical) version. ModelSpec model_spec = 1; // Input data. diff --git a/tensorflow_serving/apis/get_model_metadata.proto b/tensorflow_serving/apis/get_model_metadata.proto index d9773e9ae3a..5d765d8e39b 100644 --- a/tensorflow_serving/apis/get_model_metadata.proto +++ b/tensorflow_serving/apis/get_model_metadata.proto @@ -14,6 +14,7 @@ message SignatureDefMap { message GetModelMetadataRequest { // Model Specification indicating which model we are querying for metadata. + // If version is not specified, will use the latest (numerical) version. ModelSpec model_spec = 1; // Metadata fields to get. Currently supported: "signature_def". repeated string metadata_field = 2; diff --git a/tensorflow_serving/apis/get_model_status.proto b/tensorflow_serving/apis/get_model_status.proto new file mode 100644 index 00000000000..f683abfa7b8 --- /dev/null +++ b/tensorflow_serving/apis/get_model_status.proto @@ -0,0 +1,67 @@ +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/model.proto"; +import "tensorflow_serving/util/status.proto"; + +package tensorflow.serving; + +// GetModelStatusRequest contains a ModelSpec indicating the model for which +// to get status. +message GetModelStatusRequest { + // Model Specification. If version is not specified, information about all + // versions of the model will be returned. If a version is specified, the + // status of only that version will be returned. + ModelSpec model_spec = 1; +} + +// Version number, state, and status for a single version of a model. +message ModelVersionStatus { + // Model version. + int64 version = 1; + + // States that map to ManagerState enum in + // tensorflow_serving/core/servable_state.h + enum State { + // Default value. + UNKNOWN = 0; + + // The manager is tracking this servable, but has not initiated any action + // pertaining to it. + START = 10; + + // The manager has decided to load this servable. In particular, checks + // around resource availability and other aspects have passed, and the + // manager is about to invoke the loader's Load() method. + LOADING = 20; + + // The manager has successfully loaded this servable and made it available + // for serving (i.e. GetServableHandle(id) will succeed). To avoid races, + // this state is not reported until *after* the servable is made + // available. + AVAILABLE = 30; + + // The manager has decided to make this servable unavailable, and unload + // it. To avoid races, this state is reported *before* the servable is + // made unavailable. + UNLOADING = 40; + + // This servable has reached the end of its journey in the manager. Either + // it loaded and ultimately unloaded successfully, or it hit an error at + // some point in its lifecycle. + END = 50; + } + + // Model state. + State state = 2; + + // Model status. + StatusProto status = 3; +} + +// Response for ModelStatusRequest on successful run. +message GetModelStatusResponse { + // Version number and status information for applicable model version(s). + repeated ModelVersionStatus model_version_status = 1; +} diff --git a/tensorflow_serving/apis/inference.proto b/tensorflow_serving/apis/inference.proto index 1eac7f9743f..b9bf59ea5a1 100644 --- a/tensorflow_serving/apis/inference.proto +++ b/tensorflow_serving/apis/inference.proto @@ -21,6 +21,8 @@ package tensorflow.serving; // Inference request such as classification, regression, etc... message InferenceTask { + // Model Specification. If version is not specified, will use the latest + // (numerical) version. ModelSpec model_spec = 1; // Signature's method_name. Should be one of the method names defined in diff --git a/tensorflow_serving/apis/model.proto b/tensorflow_serving/apis/model.proto index 14ec3f0f48e..9a0d50e7809 100644 --- a/tensorflow_serving/apis/model.proto +++ b/tensorflow_serving/apis/model.proto @@ -10,10 +10,7 @@ message ModelSpec { // Required servable name. string name = 1; - // Optional version. If unspecified, will use the latest (numerical) version. - // Typically not needed unless coordinating across multiple models that were - // co-trained and/or have inter-dependencies on the versions used at inference - // time. + // Optional version. google.protobuf.Int64Value version = 2; // A named signature to evaluate. If unspecified, the default signature will diff --git a/tensorflow_serving/apis/model_service.proto b/tensorflow_serving/apis/model_service.proto new file mode 100644 index 00000000000..f3f0b8c8833 --- /dev/null +++ b/tensorflow_serving/apis/model_service.proto @@ -0,0 +1,17 @@ +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/get_model_status.proto"; + +package tensorflow.serving; + +// ModelService provides access to information about model versions +// that have been handled by the model server. +service ModelService { + // Gets status of model. If the ModelSpec in the request does not specify + // version, information about all versions of the model will be returned. If + // the ModelSpec in the request does specify a version, the status of only + // that version will be returned. + rpc GetModelStatus(GetModelStatusRequest) returns (GetModelStatusResponse); +} diff --git a/tensorflow_serving/apis/predict.proto b/tensorflow_serving/apis/predict.proto index 4a703383564..0e9ebf39cab 100644 --- a/tensorflow_serving/apis/predict.proto +++ b/tensorflow_serving/apis/predict.proto @@ -10,7 +10,8 @@ import "tensorflow_serving/apis/model.proto"; // how inputs are mapped to tensors and how outputs are filtered before // returning to user. message PredictRequest { - // Model Specification. + // Model Specification. If version is not specified, will use the latest + // (numerical) version. ModelSpec model_spec = 1; // Input tensors. diff --git a/tensorflow_serving/apis/regression.proto b/tensorflow_serving/apis/regression.proto index 84dc381ab22..e0ac300e846 100644 --- a/tensorflow_serving/apis/regression.proto +++ b/tensorflow_serving/apis/regression.proto @@ -21,7 +21,8 @@ message RegressionResult { // RPC interfaces. message RegressionRequest { - // Model Specification. + // Model Specification. If version is not specified, will use the latest + // (numerical) version. ModelSpec model_spec = 1; // Input data. diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 140cc26029b..70fd8d9cf73 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -106,6 +106,57 @@ cc_test( ], ) +cc_library( + name = "get_model_status_impl", + srcs = ["get_model_status_impl.cc"], + hdrs = ["get_model_status_impl.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":server_core", + "//tensorflow_serving/apis:get_model_status_proto", + "//tensorflow_serving/core:servable_state", + "//tensorflow_serving/core:servable_state_monitor", + "//tensorflow_serving/util:status_proto", + "//tensorflow_serving/util:status_util", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "get_model_status_impl_test", + size = "medium", + srcs = ["get_model_status_impl_test.cc"], + data = [ + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/assets/foo.txt", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/saved_model.pb", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/variables/variables.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/variables/variables.index", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/assets/foo.txt", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/saved_model.pb", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/variables/variables.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/variables/variables.index", + ], + deps = [ + ":get_model_status_impl", + ":model_platform_types", + ":platform_config_util", + ":server_core", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/servables/tensorflow:saved_model_bundle_source_adapter_proto", + "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", + "//tensorflow_serving/servables/tensorflow:session_bundle_source_adapter_proto", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/core:test", + ], +) + SUPPORTED_TENSORFLOW_OPS = [ "@org_tensorflow//tensorflow/contrib:contrib_kernels", "@org_tensorflow//tensorflow/contrib:contrib_ops_op_lib", diff --git a/tensorflow_serving/model_servers/get_model_status_impl.cc b/tensorflow_serving/model_servers/get_model_status_impl.cc new file mode 100644 index 00000000000..8080e709223 --- /dev/null +++ b/tensorflow_serving/model_servers/get_model_status_impl.cc @@ -0,0 +1,104 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/model_servers/get_model_status_impl.h" + +#include + +#include "tensorflow_serving/core/servable_state.h" +#include "tensorflow_serving/core/servable_state_monitor.h" +#include "tensorflow_serving/util/status.proto.h" +#include "tensorflow_serving/util/status_util.h" + +namespace tensorflow { +namespace serving { + +namespace { + +// Converts ManagerState to enum State in GetModelStatusResponse. +ModelVersionStatus_State ManagerStateToStateProtoEnum( + const ServableState::ManagerState& manager_state) { + switch (manager_state) { + case ServableState::ManagerState::kStart: { + return ModelVersionStatus_State_START; + } + case ServableState::ManagerState::kLoading: { + return ModelVersionStatus_State_LOADING; + } + case ServableState::ManagerState::kAvailable: { + return ModelVersionStatus_State_AVAILABLE; + } + case ServableState::ManagerState::kUnloading: { + return ModelVersionStatus_State_UNLOADING; + } + case ServableState::ManagerState::kEnd: { + return ModelVersionStatus_State_END; + } + } +} + +// Adds ModelVersionStatus to GetModelStatusResponse +void AddModelVersionStatusToResponse(GetModelStatusResponse* response, + const int64& version, + const ServableState& servable_state) { + ModelVersionStatus* version_status = response->add_model_version_status(); + version_status->set_version(version); + version_status->set_state( + ManagerStateToStateProtoEnum(servable_state.manager_state)); + *version_status->mutable_status() = ToStatusProto(servable_state.health); +} + +} // namespace + +Status GetModelStatusImpl::GetModelStatus(ServerCore* core, + const GetModelStatusRequest& request, + GetModelStatusResponse* response) { + if (!request.has_model_spec()) { + return tensorflow::errors::InvalidArgument("Missing ModelSpec"); + } + + const string& model_name = request.model_spec().name(); + const ServableStateMonitor& monitor = *core->servable_state_monitor(); + + if (request.model_spec().has_version()) { + // Only gets status for specified version of specified model. + const int64 version = request.model_spec().version().value(); + const ServableId id = {model_name, version}; + const optional opt_servable_state = monitor.GetState(id); + if (!opt_servable_state) { + return tensorflow::errors::NotFound("Could not find version ", version, + " of model ", model_name); + } + AddModelVersionStatusToResponse(response, version, + opt_servable_state.value()); + } else { + // Gets status for all versions of specified model. + const ServableStateMonitor::VersionMap versions_and_states = + monitor.GetVersionStates(model_name); + if (versions_and_states.empty()) { + return tensorflow::errors::NotFound( + "Could not find any versions of model ", model_name); + } + for (const auto& version_and_state : versions_and_states) { + const int64 version = version_and_state.first; + const ServableState& servable_state = version_and_state.second.state; + AddModelVersionStatusToResponse(response, version, servable_state); + } + } + return tensorflow::Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/model_servers/get_model_status_impl.h b/tensorflow_serving/model_servers/get_model_status_impl.h new file mode 100644 index 00000000000..0a85e46aecf --- /dev/null +++ b/tensorflow_serving/model_servers/get_model_status_impl.h @@ -0,0 +1,40 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_GET_MODEL_STATUS_IMPL_H_ +#define TENSORFLOW_SERVING_MODEL_SERVERS_GET_MODEL_STATUS_IMPL_H_ + +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/get_model_status.proto.h" +#include "tensorflow_serving/model_servers/server_core.h" + +namespace tensorflow { +namespace serving { + +// Returns response with status information for model. If the request +// specifies a model version, information about only that version will be +// returned. If no version is specified, information about all versions of the +// model will be returned. +class GetModelStatusImpl { + public: + static Status GetModelStatus(ServerCore* core, + const GetModelStatusRequest& request, + GetModelStatusResponse* response); +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_MODEL_SERVERS_GET_MODEL_STATUS_IMPL_H_ diff --git a/tensorflow_serving/model_servers/get_model_status_impl_test.cc b/tensorflow_serving/model_servers/get_model_status_impl_test.cc new file mode 100644 index 00000000000..82c785a1b78 --- /dev/null +++ b/tensorflow_serving/model_servers/get_model_status_impl_test.cc @@ -0,0 +1,177 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/model_servers/get_model_status_impl.h" + +#include +#include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/model.proto.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.proto.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" +#include "tensorflow_serving/test_util/test_util.h" +#include "tensorflow_serving/util/status.proto.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelBasePath[] = + "/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions"; +constexpr char kTestModelName[] = "saved_model_half_plus_two_2_versions"; +constexpr char kNonexistentModelName[] = "nonexistent_model"; +constexpr int kTestModelVersion1 = 123; +constexpr int kTestModelVersion2 = 124; +constexpr int kNonexistentModelVersion = 125; + +class GetModelStatusImplTest : public ::testing::Test { + public: + static void SetUpTestCase() { TF_ASSERT_OK(CreateServerCore(&server_core_)); } + + static void TearDownTestCase() { server_core_.reset(); } + + protected: + static Status CreateServerCore(std::unique_ptr* server_core) { + ModelServerConfig config; + auto* model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(test_util::TestSrcDirPath(kTestModelBasePath)); + auto* specific_versions = + model_config->mutable_model_version_policy()->mutable_specific(); + specific_versions->add_versions(kTestModelVersion1); + specific_versions->add_versions(kTestModelVersion2); + + model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + options.model_server_config = config; + + options.platform_config_map = CreateTensorFlowPlatformConfigMap( + SessionBundleConfig(), true /* use_saved_model */); + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + return ServerCore::Create(std::move(options), server_core); + } + + ServerCore* GetServerCore() { return server_core_.get(); } + + private: + static std::unique_ptr server_core_; +}; + +std::unique_ptr GetModelStatusImplTest::server_core_; + +TEST_F(GetModelStatusImplTest, MissingOrEmptyModelSpecFailure) { + GetModelStatusRequest request; + GetModelStatusResponse response; + + // Empty request is invalid. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + GetModelStatusImpl::GetModelStatus(GetServerCore(), request, &response) + .code()); +} + +TEST_F(GetModelStatusImplTest, InvalidModelNameFailure) { + GetModelStatusRequest request; + GetModelStatusResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kNonexistentModelName); + + // If no versions of model are managed by ServerCore, response is + // NOT_FOUND error. + EXPECT_EQ( + tensorflow::error::NOT_FOUND, + GetModelStatusImpl::GetModelStatus(GetServerCore(), request, &response) + .code()); + EXPECT_EQ(0, response.model_version_status_size()); +} + +TEST_F(GetModelStatusImplTest, InvalidModelVersionFailure) { + GetModelStatusRequest request; + GetModelStatusResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kNonexistentModelVersion); + + // If model version not managed by ServerCore, response is NOT_FOUND error. + EXPECT_EQ( + tensorflow::error::NOT_FOUND, + GetModelStatusImpl::GetModelStatus(GetServerCore(), request, &response) + .code()); + EXPECT_EQ(0, response.model_version_status_size()); +} + +TEST_F(GetModelStatusImplTest, AllVersionsSuccess) { + GetModelStatusRequest request; + GetModelStatusResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + + // If two versions of model are managed by ServerCore, succesfully get model + // status for both versions of the model. + TF_EXPECT_OK( + GetModelStatusImpl::GetModelStatus(GetServerCore(), request, &response)); + EXPECT_EQ(2, response.model_version_status_size()); + std::set expected_versions = {kTestModelVersion1, kTestModelVersion2}; + std::set actual_versions = { + response.model_version_status(0).version(), + response.model_version_status(1).version()}; + EXPECT_EQ(expected_versions, actual_versions); + EXPECT_EQ(tensorflow::error::OK, + response.model_version_status(0).status().error_code()); + EXPECT_EQ("", response.model_version_status(0).status().error_message()); + EXPECT_EQ(tensorflow::error::OK, + response.model_version_status(1).status().error_code()); + EXPECT_EQ("", response.model_version_status(1).status().error_message()); +} + +TEST_F(GetModelStatusImplTest, SingleVersionSuccess) { + GetModelStatusRequest request; + GetModelStatusResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion1); + + // If model version is managed by ServerCore, succesfully get model status. + TF_EXPECT_OK( + GetModelStatusImpl::GetModelStatus(GetServerCore(), request, &response)); + EXPECT_EQ(1, response.model_version_status_size()); + EXPECT_EQ(kTestModelVersion1, response.model_version_status(0).version()); + EXPECT_EQ(tensorflow::error::OK, + response.model_version_status(0).status().error_code()); + EXPECT_EQ("", response.model_version_status(0).status().error_message()); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/assets/foo.txt b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/assets/foo.txt new file mode 100644 index 00000000000..f9ff0366880 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/assets/foo.txt @@ -0,0 +1 @@ +asset-file-contents \ No newline at end of file diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/saved_model.pb b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..4a4fd025d9de0da7bd8e70eb0f017b851f686f28 GIT binary patch literal 9935 zcmcf{+in}jl|xFr98tHCV$&l#PTAUZ*04y?c54#`>d=jo7`5WmZXVjko8|6EW=wHe z?#iYN1PRb0$U`4epeXv%pXdkl1Nt-lg!atL&T=nYnpEmofcG-9=YGzd3(JK5`yu?Z zO8;7g+faDVacB|9xK5sk>j_>NUSHw0&H!I7`6ol(UWe7z$`;8l6_*Q@GA+Rvitl+| z?{Gj@pfKVGxc_THD8TXytAES|7<_P#{o%21hU@i82oj;M;ntpG`P}rLSwl{jx~|)2 z`75M;7FLcuZuVXKd2etyS@=64a0{02xdYE<>rhni>&uQHvc4W5r$wI+Ja<%I1+}7> zfL$nm6Dr zZ_K?~iJk%KaL?}zEWhUtf|XjSz60wH?{j3+?K`H^8+j9k;0_cX9`_eh!KT!~cPNV@ zEbY4=58)v!J+kcQbPbA9{2sDpko`DkRUrGcfIM6wO%mY~<+fyvd!Afl`C1PClZWaf zYvl3w$JWuX&k-(oeAGjAz2XkCT7l((btJj2!HPE&d(G$k1jS&K_SG}W15nzNh2p+^ zq%~Mm?;U<%9rt~&ReT~}upL+nVD$!0&qnx_5FEPztHGjTu~iKoh3-AZ!ewbpu}{xH zRbdRi4Di)AAS71F1x2B$3|?G<7#a%wp66c?DBrz8+9g-0tu&i80UD#4PUnB54DE*qN_h{0A51leRqWJfZkJ7hL5(e0ab;{)F!bF8xoIc z4II=TE-tkufgTheTQ5$DiY|oeLs)z^!2_*A zxodgyUKu;GGjeJL;Rl1`BN47fj#n!%+;|XtR4c(gz+-=e0qX%q3c3nK8DWG^pev%t zRJ6i~5ckBhQ1LjrkcGmfMNdLtd71+c;8P2|uG>dJpiRg)t0Ef4+WK1D2oli_V`#;8 z2fo!C@EF+oIrLa$fICuQQH!}wOMP9TlWs~U-Ko0nW+^trEd0ctu9Ip0a5padEbu}n z_79{JJEaqwC0PpDUBFN51r#ZCVyARsN71JJ#Lkr7#{m9;$%B+8dfpm%ysGdb57MY3 zlnhK)5`hhmM0wu+252-(YRI%y$HPq>_sX3j#=Rhj$%9XZmJF~Mrap(Q7+B1!%yCP@ zJ*VMrbk-2=63v&1jH!vZi0O5L>@pPFayvM{ViVREu!v)^*NL0W$3b(>eS2(*6SAp>C(I|PLGrb{(N1~D6}a5qq$}nB z>VN3BWisUz>Ly+mQKr)*qtm`YqPsk&GZ)`o>RF zq`w7Q)9BQh41A#KOi^sIAo*Zqd@4ZXgo?HSQ?S&bc?v)u!7>tK?$IPon&!D-!UM>k zv=mVaCJWAuxj{_g#basUOJjk2 zNgHt~BOj}WbxRG-&}~__B#4C zmZ9;-an)hXWUq6lwwE)F{Yha>lRXgYj|^|fgI zvC>n;AUwhz9kxv3S?C$B0zC{v<#Sqx(m97tdG?HbXZpl0p7HkPf< zCK`(u$^3=DSxJHD<&E6FaD6_&n!-Mb zBq`JBsXmR+j`@WymM_2#Jtw2TT8~>}mVzg-8l&n}`DJ=K(wH(Dp>j5nB{M~1n!6bq zBL)N$3(T4YS;B+KguX8Wk)_F_QwBP05niIv23wn{!4CG9;1y0Hgw2Grrb&}87OS#2 zXEn8wCja%S;jkldhpqalewa8FWK#hBa9$52=<62~XtJCz*6vciMLMy-HOn^%d`EC5 zB%~Q~L??5Is5aOv#@Xx0#qh635zA0$VeOXDpTRN9wJ<{V5>ersOk$JtLQ zKTY)aQGRQpd^@H5)@d}lP;2}{pwdT1Y9^~jl0iujocqd3*_t{{w9O`-TW~G#HVw(@ z_X9Xv>Px*Kv($RNe3sq8?w4>-+3;>RsY)pIdDm_R^3We5#CMjjkej)?=kMRj*Wbn& zqG%hKqR<>#{xc7MLsEfK0ImK7Z22P`TZ*|S&KT7I)T58-CvXl57UB!1cnt5v9If3H zOI@)P)=hXfS+d2FuA>;xKV7!kr|EuD61iOc>d)VjJ zu;=aC?or^v2YvVDF6xZ?_fbl}A@z4)tU{gQf z$V6hkPM^c25cLi$jawNkQQQ_#fZI;4XKZ6LNq7nCBYucncmd}v>b#cq%TQ9IbB_wl rSp-g%d4SD0hP622IYfSGf_G6>YqVd%Mu>J{fhYLRl=w7T3$XqVKAtaQ literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/variables/variables.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..15b75d6ef6bffc336d138d923badb3928b8c4c13 GIT binary patch literal 12 RcmZQzV6bOkU~m8;2LJ>^0RR91 literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/variables/variables.index b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000123/variables/variables.index new file mode 100644 index 0000000000000000000000000000000000000000..7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326 GIT binary patch literal 151 zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5d=jo7`5WmZXVjko8|6EW=wHe z?#iYN1PRb0$U`4epeXv%pXdkl1Nt-lg!atL&T=nYnpEmofcG-9=YGzd3(JK5`yu?Z zO8;7g+faDVacB|9xK5sk>j_>NUSHw0&H!I7`6ol(UWe7z$`;8l6_*Q@GA+Rvitl+| z?{Gj@pfKVGxc_THD8TXytAES|7<_P#{o%21hU@i82oj;M;ntpG`P}rLSwl{jx~|)2 z`75M;7FLcuZuVXKd2etyS@=64a0{02xdYE<>rhni>&uQHvc4W5r$wI+Ja<%I1+}7> zfL$nm6Dr zZ_K?~iJk%KaL?}zEWhUtf|XjSz60wH?{j3+?K`H^8+j9k;0_cX9`_eh!KT!~cPNV@ zEbY4=58)v!J+kcQbPbA9{2sDpko`DkRUrGcfIM6wO%mY~<+fyvd!Afl`C1PClZWaf zYvl3w$JWuX&k-(oeAGjAz2XkCT7l((btJj2!HPE&d(G$k1jS&K_SG}W15nzNh2p+^ zq%~Mm?;U<%9rt~&ReT~}upL+nVD$!0&qnx_5FEPztHGjTu~iKoh3-AZ!ewbpu}{xH zRbdRi4Di)AAS71F1x2B$3|?G<7#a%wp66c?DBrz8+9g-0tu&i80UD#4PUnB54DE*qN_h{0A51leRqWJfZkJ7hL5(e0ab;{)F!bF8xoIc z4II=TE-tkufgTheTQ5$DiY|oeLs)z^!2_*A zxodgyUKu;GGjeJL;Rl1`BN47fj#n!%+;|XtR4c(gz+-=e0qX%q3c3nK8DWG^pev%t zRJ6i~5ckBhQ1LjrkcGmfMNdLtd71+c;8P2|uG>dJpiRg)t0Ef4+WK1D2oli_V`#;8 z2fo!C@EF+oIrLa$fICuQQH!}wOMP9TlWs~U-Ko0nW+^trEd0ctu9Ip0a5padEbu}n z_79{JJEaqwC0PpDUBFN51r#ZCVyARsN71JJ#Lkr7#{m9;$%B+8dfpm%ysGdb57MY3 zlnhK)5`hhmM0wu+252-(YRI%y$HPq>_sX3j#=Rhj$%9XZmJF~Mrap(Q7+B1!%yCP@ zJ*VMrbk-2=63v&1jH!vZi0O5L>@pPFayvM{ViVREu!v)^*NL0W$3b(>eS2(*6SAp>C(I|PLGrb{(N1~D6}a5qq$}nB z>VN3BWisUz>Ly+mQKr)*qtm`YqPsk&GZ)`o>RF zq`w7Q)9BQh41A#KOi^sIAo*Zqd@4ZXgo?HSQ?S&bc?v)u!7>tK?$IPon&!D-!UM>k zv=mVaCJWAuxj{_g#basUOJjk2 zNgHt~BOj}WbxRG-&}~__B#4C zmZ9;-an)hXWUq6lwwE)F{Yha>lRXgYj|^|fgI zvC>n;AUwhz9kxv3S?C$B0zC{v<#Sqx(m97tdG?HbXZpl0p7HkPf< zCK`(u$^3=DSxJHD<&E6FaD6_&n!-Mb zBq`JBsXmR+j`@WymM_2#Jtw2TT8~>}mVzg-8l&n}`DJ=K(wH(Dp>j5nB{M~1n!6bq zBL)N$3(T4YS;B+KguX8Wk)_F_QwBP05niIv23wn{!4CG9;1y0Hgw2Grrb&}87OS#2 zXEn8wCja%S;jkldhpqalewa8FWK#hBa9$52=<62~XtJCz*6vciMLMy-HOn^%d`EC5 zB%~Q~L??5Is5aOv#@Xx0#qh635zA0$VeOXDpTRN9wJ<{V5>ersOk$JtLQ zKTY)aQGRQpd^@H5)@d}lP;2}{pwdT1Y9^~jl0iujocqd3*_t{{w9O`-TW~G#HVw(@ z_X9Xv>Px*Kv($RNe3sq8?w4>-+3;>RsY)pIdDm_R^3We5#CMjjkej)?=kMRj*Wbn& zqG%hKqR<>#{xc7MLsEfK0ImK7Z22P`TZ*|S&KT7I)T58-CvXl57UB!1cnt5v9If3H zOI@)P)=hXfS+d2FuA>;xKV7!kr|EuD61iOc>d)VjJ zu;=aC?or^v2YvVDF6xZ?_fbl}A@z4)tU{gQf z$V6hkPM^c25cLi$jawNkQQQ_#fZI;4XKZ6LNq7nCBYucncmd}v>b#cq%TQ9IbB_wl rSp-g%d4SD0hP622IYfSGf_G6>YqVd%Mu>J{fhYLRl=w7T3$XqVKAtaQ literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/variables/variables.data-00000-of-00001 b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/variables/variables.data-00000-of-00001 new file mode 100644 index 0000000000000000000000000000000000000000..15b75d6ef6bffc336d138d923badb3928b8c4c13 GIT binary patch literal 12 RcmZQzV6bOkU~m8;2LJ>^0RR91 literal 0 HcmV?d00001 diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/variables/variables.index b/tensorflow_serving/servables/tensorflow/testdata/saved_model_half_plus_two_2_versions/00000124/variables/variables.index new file mode 100644 index 0000000000000000000000000000000000000000..7ec9fb4fe2dd21d0a6c324aecd7658fc37cf2326 GIT binary patch literal 151 zcmZQzVB=tvV&Y(AVB}8ZU=(7|U@>L0P?u+5 + +namespace tensorflow { +namespace serving { +namespace { + +TEST(StatusUtilTest, ConvertsErrorStatusToStatusProto) { + Status status = Status(tensorflow::error::ABORTED, "aborted error message"); + StatusProto status_proto = ToStatusProto(status); + EXPECT_EQ(tensorflow::error::ABORTED, status_proto.error_code()); + EXPECT_EQ("aborted error message", status_proto.error_message()); +} + +TEST(StatusUtilTest, ConvertsOkStatusToStatusProto) { + Status status; + StatusProto status_proto = ToStatusProto(status); + EXPECT_EQ(tensorflow::error::OK, status_proto.error_code()); + EXPECT_EQ("", status_proto.error_message()); +} + +TEST(StatusUtilTest, ConvertsErrorStatusProtoToStatus) { + StatusProto status_proto; + status_proto.set_error_code(tensorflow::error::ALREADY_EXISTS); + status_proto.set_error_message("already exists error message"); + Status status = FromStatusProto(status_proto); + EXPECT_EQ(tensorflow::error::ALREADY_EXISTS, status.code()); + EXPECT_EQ("already exists error message", status.error_message()); +} + +TEST(StatusUtilTest, ConvertsOkStatusProtoToStatus) { + StatusProto status_proto; + status_proto.set_error_code(tensorflow::error::OK); + Status status = FromStatusProto(status_proto); + EXPECT_EQ(tensorflow::error::OK, status.code()); + EXPECT_EQ("", status.error_message()); +} + +} // namespace + +} // namespace serving +} // namespace tensorflow From 767f30db104a56940b0fdc7c85766f73576ba502 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Mon, 29 Jan 2018 19:19:20 -0800 Subject: [PATCH 0420/8554] Fix the open source build. Change: 183769574 --- .../model_servers/get_model_status_impl.cc | 2 +- .../model_servers/get_model_status_impl.h | 2 +- .../get_model_status_impl_test.cc | 10 +- .../tensorflow/classification_service_test.cc | 99 ++++++++++--------- .../tensorflow/regression_service_test.cc | 98 +++++++++--------- 5 files changed, 106 insertions(+), 105 deletions(-) diff --git a/tensorflow_serving/model_servers/get_model_status_impl.cc b/tensorflow_serving/model_servers/get_model_status_impl.cc index 8080e709223..4580bd96f37 100644 --- a/tensorflow_serving/model_servers/get_model_status_impl.cc +++ b/tensorflow_serving/model_servers/get_model_status_impl.cc @@ -19,7 +19,7 @@ limitations under the License. #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/servable_state_monitor.h" -#include "tensorflow_serving/util/status.proto.h" +#include "tensorflow_serving/util/status.pb.h" #include "tensorflow_serving/util/status_util.h" namespace tensorflow { diff --git a/tensorflow_serving/model_servers/get_model_status_impl.h b/tensorflow_serving/model_servers/get_model_status_impl.h index 0a85e46aecf..af4743a0727 100644 --- a/tensorflow_serving/model_servers/get_model_status_impl.h +++ b/tensorflow_serving/model_servers/get_model_status_impl.h @@ -17,7 +17,7 @@ limitations under the License. #define TENSORFLOW_SERVING_MODEL_SERVERS_GET_MODEL_STATUS_IMPL_H_ #include "tensorflow/core/lib/core/status.h" -#include "tensorflow_serving/apis/get_model_status.proto.h" +#include "tensorflow_serving/apis/get_model_status.pb.h" #include "tensorflow_serving/model_servers/server_core.h" namespace tensorflow { diff --git a/tensorflow_serving/model_servers/get_model_status_impl_test.cc b/tensorflow_serving/model_servers/get_model_status_impl_test.cc index 82c785a1b78..f36cb152c51 100644 --- a/tensorflow_serving/model_servers/get_model_status_impl_test.cc +++ b/tensorflow_serving/model_servers/get_model_status_impl_test.cc @@ -21,16 +21,16 @@ limitations under the License. #include "tensorflow/cc/saved_model/signature_constants.h" #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status_test_util.h" -#include "tensorflow_serving/apis/model.proto.h" +#include "tensorflow_serving/apis/model.pb.h" #include "tensorflow_serving/core/availability_preserving_policy.h" #include "tensorflow_serving/model_servers/model_platform_types.h" #include "tensorflow_serving/model_servers/platform_config_util.h" #include "tensorflow_serving/model_servers/server_core.h" -#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.proto.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_config.proto.h" -#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.proto.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" #include "tensorflow_serving/test_util/test_util.h" -#include "tensorflow_serving/util/status.proto.h" +#include "tensorflow_serving/util/status.pb.h" namespace tensorflow { namespace serving { diff --git a/tensorflow_serving/servables/tensorflow/classification_service_test.cc b/tensorflow_serving/servables/tensorflow/classification_service_test.cc index 159f73b1738..03b13e02786 100644 --- a/tensorflow_serving/servables/tensorflow/classification_service_test.cc +++ b/tensorflow_serving/servables/tensorflow/classification_service_test.cc @@ -102,11 +102,11 @@ TEST_F(ClassificationServiceTest, InvalidModelSpec) { // Verifies that Classify() returns an error for an invalid signature_name in // ClassificationRequests's model_spec. TEST_F(ClassificationServiceTest, InvalidSignature) { - auto request = test_util::CreateProto(R"( - model_spec { - name: "test_model" - signature_name: "invalid_signature_name" - })"); + auto request = test_util::CreateProto( + "model_spec {" + " name: \"test_model\"" + " signature_name: \"invalid_signature_name\"" + "}"); ClassificationResponse response; EXPECT_EQ(TensorflowClassificationServiceImpl::Classify( RunOptions(), server_core_.get(), request, &response) @@ -118,53 +118,54 @@ TEST_F(ClassificationServiceTest, InvalidSignature) { // ClassificationRequest against the half_plus_two SavedModel's classify_x_to_y // signature. TEST_F(ClassificationServiceTest, ClassificationSuccess) { - auto request = test_util::CreateProto(R"( - model_spec { - name: "test_model" - signature_name: "classify_x_to_y" - } - input { - example_list { - examples { - features { - feature: { - key : "x" - value: { - float_list: { - value: [ 80.0 ] - } - } - } - feature: { - key : "locale" - value: { - bytes_list: { - value: [ "pt_BR" ] - } - } - } - feature: { - key : "age" - value: { - float_list: { - value: [ 19.0 ] - } - } - } - } - } - } - })"); + auto request = test_util::CreateProto( + "model_spec {" + " name: \"test_model\"" + " signature_name: \"classify_x_to_y\"" + "}" + "input {" + " example_list {" + " examples {" + " features {" + " feature: {" + " key : \"x\"" + " value: {" + " float_list: {" + " value: [ 80.0 ]" + " }" + " }" + " }" + " feature: {" + " key : \"locale\"" + " value: {" + " bytes_list: {" + " value: [ \"pt_BR\" ]" + " }" + " }" + " }" + " feature: {" + " key : \"age\"" + " value: {" + " float_list: {" + " value: [ 19.0 ]" + " }" + " }" + " }" + " }" + " }" + " }" + "}"); ClassificationResponse response; TF_EXPECT_OK(TensorflowClassificationServiceImpl::Classify( RunOptions(), server_core_.get(), request, &response)); - EXPECT_THAT(response, test_util::EqualsProto(R"( - result { classifications { classes { score: 42 } } } - model_spec { - name: "test_model" - signature_name: "classify_x_to_y" - version { value: 123 } - })")); + EXPECT_THAT(response, + test_util::EqualsProto( + "result { classifications { classes { score: 42 } } }" + "model_spec {" + " name: \"test_model\"" + " signature_name: \"classify_x_to_y\"" + " version { value: 123 }" + "}")); } } // namespace diff --git a/tensorflow_serving/servables/tensorflow/regression_service_test.cc b/tensorflow_serving/servables/tensorflow/regression_service_test.cc index 401744e7684..f6bdc8a6fc4 100644 --- a/tensorflow_serving/servables/tensorflow/regression_service_test.cc +++ b/tensorflow_serving/servables/tensorflow/regression_service_test.cc @@ -102,11 +102,11 @@ TEST_F(RegressionServiceTest, InvalidModelSpec) { // Verifies that Regress() returns an error for an invalid signature_name in // RegressionRequests's model_spec. TEST_F(RegressionServiceTest, InvalidSignature) { - auto request = test_util::CreateProto(R"( - model_spec { - name: "test_model" - signature_name: "invalid_signature_name" - })"); + auto request = test_util::CreateProto( + "model_spec {" + " name: \"test_model\"" + " signature_name: \"invalid_signature_name\"" + "}"); RegressionResponse response; EXPECT_EQ(TensorflowRegressionServiceImpl::Regress( RunOptions(), server_core_.get(), request, &response) @@ -118,53 +118,53 @@ TEST_F(RegressionServiceTest, InvalidSignature) { // RegressionRequest against the half_plus_two SavedModel's regress_x_to_y // signature. TEST_F(RegressionServiceTest, RegressionSuccess) { - auto request = test_util::CreateProto(R"( - model_spec { - name: "test_model" - signature_name: "regress_x_to_y" - } - input { - example_list { - examples { - features { - feature: { - key : "x" - value: { - float_list: { - value: [ 80.0 ] - } - } - } - feature: { - key : "locale" - value: { - bytes_list: { - value: [ "pt_BR" ] - } - } - } - feature: { - key : "age" - value: { - float_list: { - value: [ 19.0 ] - } - } - } - } - } - } - })"); + auto request = test_util::CreateProto( + "model_spec {" + " name: \"test_model\"" + " signature_name: \"regress_x_to_y\"" + "}" + "input {" + " example_list {" + " examples {" + " features {" + " feature: {" + " key : \"x\"" + " value: {" + " float_list: {" + " value: [ 80.0 ]" + " }" + " }" + " }" + " feature: {" + " key : \"locale\"" + " value: {" + " bytes_list: {" + " value: [ \"pt_BR\" ]" + " }" + " }" + " }" + " feature: {" + " key : \"age\"" + " value: {" + " float_list: {" + " value: [ 19.0 ]" + " }" + " }" + " }" + " }" + " }" + " }" + "}"); RegressionResponse response; TF_EXPECT_OK(TensorflowRegressionServiceImpl::Regress( RunOptions(), server_core_.get(), request, &response)); - EXPECT_THAT(response, test_util::EqualsProto(R"( - result { regressions { value: 42 } } - model_spec { - name: "test_model" - signature_name: "regress_x_to_y" - version { value: 123 } - })")); + EXPECT_THAT(response, + test_util::EqualsProto("result { regressions { value: 42 } }" + "model_spec {" + " name: \"test_model\"" + " signature_name: \"regress_x_to_y\"" + " version { value: 123 }" + "}")); } } // namespace From 39478bc20dcab2ebd2effcdca1302d3cdcb11bae Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 30 Jan 2018 10:26:59 -0800 Subject: [PATCH 0421/8554] Merge changes from github. Change: 183852526 --- tensorflow_serving/g3doc/serving_advanced.md | 6 +++--- tensorflow_serving/g3doc/setup.md | 7 ++++++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index bb9c862b02c..8cd51592e41 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -165,8 +165,8 @@ that monitors cloud storage instead of local storage, or you could build a version policy plugin that does version transition in a different way -- in fact, you could even build a custom model plugin that serves non-TensorFlow models. These topics are out of scope for this tutorial. However, you can refer -to the [custom source](custom_source.md) and [custom servable] -(custom_servable.md) tutorials for more information. +to the [custom source](custom_source.md) and +[custom servable](custom_servable.md) tutorials for more information. ## Batching @@ -250,7 +250,7 @@ To put all these into the context of this tutorial: servables that can be loaded. * `AspiredVersionsManager` monitors the export stream, and manages lifecycle - of all SavedModelBundle` servables dynamically. + of all `SavedModelBundle` servables dynamically. `TensorflowPredictImpl::Predict` then just: diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 06607f7aec5..55985262b79 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -72,7 +72,12 @@ the `tensorflow-serving-api` PIP package using: pip install tensorflow-serving-api ``` -Note: The TensorFlow Serving Python API only supports Python 2.7. It does not support Python 3. +Note: The TensorFlow Serving Python API +[is only published for Python 2](https://pypi.python.org/pypi/tensorflow-serving-api), +but will work for Python 3 if you either build it yourself, or download the +Python 2 version, unzip it, and copy it into your Python 3 path. There is a +[feature request](https://github.com/tensorflow/serving/issues/700) to publish +the Python 3 package as well. ## Installing using apt-get From f9e602c753ef82ff96b28429dd07e900f10eb007 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 31 Jan 2018 11:38:57 -0800 Subject: [PATCH 0422/8554] Make the TF git revision easily overridable for TF Serving builds. bazel build --action_env TF_REVISION="{git hash}" //tensorflow_serving/... Change: 184022677 --- WORKSPACE | 12 +++++------ tensorflow_serving/repo.bzl | 43 +++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+), 7 deletions(-) create mode 100644 tensorflow_serving/repo.bzl diff --git a/WORKSPACE b/WORKSPACE index c4aeac89006..c6fa8cb1e41 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,20 +1,18 @@ workspace(name = "tf_serving") # To update TensorFlow to a new revision. -# 1. Update the 'urls' and 'strip_prefix' args below to include the new git hash. +# 1. Update the 'git_commit' args below to include the new git hash. # 2. Get the sha256 hash of the archive with a command such as... # curl -L https://github.com/tensorflow/tensorflow/archive/.tar.gz | sha256sum # and update the 'sha256' arg with the result. # 3. Request the new archive to be mirrored on mirror.bazel.build for more # reliable downloads. -http_archive( +load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") + +tensorflow_http_archive( name = "org_tensorflow", sha256 = "9c3db796b2c6bc24ebff94d689c442f2cf1d8955f122a277e4ad893c0ce96aac", - strip_prefix = "tensorflow-57b32eabca4597241120cb4aba8308a431853c30", - urls = [ - "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/57b32eabca4597241120cb4aba8308a431853c30.tar.gz", - "https://github.com/tensorflow/tensorflow/archive/57b32eabca4597241120cb4aba8308a431853c30.tar.gz", - ], + git_commit = "57b32eabca4597241120cb4aba8308a431853c30", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. diff --git a/tensorflow_serving/repo.bzl b/tensorflow_serving/repo.bzl new file mode 100644 index 00000000000..25830cd20a3 --- /dev/null +++ b/tensorflow_serving/repo.bzl @@ -0,0 +1,43 @@ +""" TensorFlow Http Archive + +Modified http_arhive that allows us to override the TensorFlow commit that is +downloaded by setting an environment variable. This override is to be used for +testing purposes. + +Add the following to your Bazel build command in order to override the +TensorFlow revision. + +build: --action_env TF_REVISION="" + + * `TF_REVISION`: tensorflow revision override (git commit hash) +""" + +_TF_REVISION = "TF_REVISION" + +def _tensorflow_http_archive(ctx): + git_commit = ctx.attr.git_commit + sha256 = ctx.attr.sha256 + + override_git_commit = ctx.os.environ.get(_TF_REVISION) + if override_git_commit: + sha256 = "" + git_commit = override_git_commit + + strip_prefix = "tensorflow-%s" % git_commit + urls = [ + "https://mirror.bazel.build/github.com/tensorflow/tensorflow/archive/%s.tar.gz" % git_commit, + "https://github.com/tensorflow/tensorflow/archive/%s.tar.gz" % git_commit, + ] + ctx.download_and_extract( + urls, + "", + sha256, + "", + strip_prefix) + +tensorflow_http_archive = repository_rule( + implementation=_tensorflow_http_archive, + attrs={ + "git_commit": attr.string(mandatory=True), + "sha256": attr.string(mandatory=True), + }) From 5125b4d53f6e9d35123cd457344ddef0f3e62de0 Mon Sep 17 00:00:00 2001 From: Shohini Ghosh Date: Wed, 31 Jan 2018 14:56:47 -0800 Subject: [PATCH 0423/8554] Add ModelService with GetModelStatus method to model server. Change: 184052314 --- tensorflow_serving/apis/BUILD | 29 +++++++++ tensorflow_serving/model_servers/BUILD | 32 ++++++++++ .../model_servers/grpc_status_util.cc | 37 +++++++++++ .../model_servers/grpc_status_util.h | 31 +++++++++ tensorflow_serving/model_servers/main.cc | 64 ++++++++----------- .../model_servers/model_service_impl.cc | 36 +++++++++++ .../model_servers/model_service_impl.h | 43 +++++++++++++ .../tensorflow_model_server_test.py | 24 +++++++ tensorflow_serving/util/BUILD | 10 +++ 9 files changed, 268 insertions(+), 38 deletions(-) create mode 100644 tensorflow_serving/model_servers/grpc_status_util.cc create mode 100644 tensorflow_serving/model_servers/grpc_status_util.h create mode 100644 tensorflow_serving/model_servers/model_service_impl.cc create mode 100644 tensorflow_serving/model_servers/model_service_impl.h diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 1a47cab655d..7c7ebf79b9b 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -168,6 +168,16 @@ serving_proto_library( ], ) +serving_proto_library_py( + name = "get_model_status_proto_py_pb2", + srcs = ["get_model_status.proto"], + proto_library = "get_model_status_proto", + deps = [ + ":model_proto_py_pb2", + "//tensorflow_serving/util:status_proto_py_pb2", + ], +) + serving_proto_library( name = "model_service_proto", srcs = ["model_service.proto"], @@ -181,6 +191,25 @@ serving_proto_library( ], ) +py_library( + name = "model_service_proto_py_pb2", + srcs = [ + "model_service_pb2.py", + "model_service_pb2_grpc.py", + ], + srcs_version = "PY2AND3", + deps = [ + ":get_model_status_proto_py_pb2", + "//net/grpc/python:grpc", + ], +) + +serving_go_grpc_library( + name = "model_service_grpc", + srcs = [":model_service_proto"], + deps = [":model_service_go_proto"], +) + serving_proto_library( name = "classification_proto", srcs = ["classification.proto"], diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 70fd8d9cf73..0c4ca0ab2fc 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -157,6 +157,35 @@ cc_test( ], ) +cc_library( + name = "model_service_impl", + srcs = ["model_service_impl.cc"], + hdrs = ["model_service_impl.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":get_model_status_impl", + ":grpc_status_util", + ":server_core", + "//tensorflow_serving/apis:model_service_proto", + "@grpc//:grpc++_unsecure", + ], +) + +cc_library( + name = "grpc_status_util", + srcs = ["grpc_status_util.cc"], + hdrs = ["grpc_status_util.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + "@grpc//:grpc++_unsecure", + "@org_tensorflow//tensorflow/core:lib", + ], +) + SUPPORTED_TENSORFLOW_OPS = [ "@org_tensorflow//tensorflow/contrib:contrib_kernels", "@org_tensorflow//tensorflow/contrib:contrib_ops_op_lib", @@ -183,6 +212,8 @@ cc_binary( ":model_platform_types", ":platform_config_util", ":server_core", + ":grpc_status_util", + ":model_service_impl", "@protobuf_archive//:cc_wkt_protos", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core/platform/cloud:gcs_file_system", @@ -216,6 +247,7 @@ py_test( "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], deps = [ + "//tensorflow_serving/apis:model_service_proto_py_pb2", "//tensorflow_serving/apis:prediction_service_proto_py_pb2", "@org_tensorflow//tensorflow:tensorflow_py", ], diff --git a/tensorflow_serving/model_servers/grpc_status_util.cc b/tensorflow_serving/model_servers/grpc_status_util.cc new file mode 100644 index 00000000000..5cf15060ab4 --- /dev/null +++ b/tensorflow_serving/model_servers/grpc_status_util.cc @@ -0,0 +1,37 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/model_servers/grpc_status_util.h" + +#include "grpc++/support/status_code_enum.h" + +namespace tensorflow { +namespace serving { + +::grpc::Status ToGRPCStatus(const ::tensorflow::Status& status) { + const int kErrorMessageLimit = 1024; + string error_message; + if (status.error_message().length() > kErrorMessageLimit) { + error_message = + status.error_message().substr(0, kErrorMessageLimit) + "...TRUNCATED"; + } else { + error_message = status.error_message(); + } + return ::grpc::Status(static_cast(status.code()), + error_message); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/model_servers/grpc_status_util.h b/tensorflow_serving/model_servers/grpc_status_util.h new file mode 100644 index 00000000000..5e9245224f4 --- /dev/null +++ b/tensorflow_serving/model_servers/grpc_status_util.h @@ -0,0 +1,31 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_GRPC_STATUS_UTIL_H_ +#define TENSORFLOW_SERVING_MODEL_SERVERS_GRPC_STATUS_UTIL_H_ + +#include "grpc++/support/status.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace serving { + +// Converts from tensorflow Status to GRPC Status. +::grpc::Status ToGRPCStatus(const ::tensorflow::Status& status); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_MODEL_SERVERS_GRPC_STATUS_UTIL_H_ diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index d770420167b..ef1573aa1fc 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -55,7 +55,6 @@ limitations under the License. #include "grpc++/server_builder.h" #include "grpc++/server_context.h" #include "grpc++/support/status.h" -#include "grpc++/support/status_code_enum.h" #include "grpc/grpc.h" #include "tensorflow/cc/saved_model/tag_constants.h" #include "tensorflow/core/lib/core/status.h" @@ -70,7 +69,9 @@ limitations under the License. #include "tensorflow_serving/apis/prediction_service.pb.h" #include "tensorflow_serving/config/model_server_config.pb.h" #include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/model_servers/grpc_status_util.h" #include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/model_service_impl.h" #include "tensorflow_serving/model_servers/platform_config_util.h" #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/classification_service.h" @@ -172,24 +173,10 @@ int DeadlineToTimeoutMillis(const gpr_timespec deadline) { gpr_now(GPR_CLOCK_MONOTONIC))); } -grpc::Status ToGRPCStatus(const tensorflow::Status& status) { - const int kErrorMessageLimit = 1024; - string error_message; - if (status.error_message().length() > kErrorMessageLimit) { - error_message = - status.error_message().substr(0, kErrorMessageLimit) + "...TRUNCATED"; - } else { - error_message = status.error_message(); - } - return grpc::Status(static_cast(status.code()), - error_message); -} - class PredictionServiceImpl final : public PredictionService::Service { public: - explicit PredictionServiceImpl(std::unique_ptr core, - bool use_saved_model) - : core_(std::move(core)), + explicit PredictionServiceImpl(ServerCore* core, bool use_saved_model) + : core_(core), predictor_(new TensorflowPredictor(use_saved_model)), use_saved_model_(use_saved_model) {} @@ -199,8 +186,8 @@ class PredictionServiceImpl final : public PredictionService::Service { // By default, this is infinite which is the same default as RunOptions. run_options.set_timeout_in_ms( DeadlineToTimeoutMillis(context->raw_deadline())); - const grpc::Status status = ToGRPCStatus( - predictor_->Predict(run_options, core_.get(), *request, response)); + const grpc::Status status = tensorflow::serving::ToGRPCStatus( + predictor_->Predict(run_options, core_, *request, response)); if (!status.ok()) { VLOG(1) << "Predict failed: " << status.error_message(); } @@ -211,13 +198,13 @@ class PredictionServiceImpl final : public PredictionService::Service { const GetModelMetadataRequest* request, GetModelMetadataResponse* response) override { if (!use_saved_model_) { - return ToGRPCStatus(tensorflow::errors::InvalidArgument( - "GetModelMetadata API is only available when use_saved_model is " - "set to true")); + return tensorflow::serving::ToGRPCStatus( + tensorflow::errors::InvalidArgument( + "GetModelMetadata API is only available when use_saved_model is " + "set to true")); } - const grpc::Status status = - ToGRPCStatus(GetModelMetadataImpl::GetModelMetadata( - core_.get(), *request, response)); + const grpc::Status status = tensorflow::serving::ToGRPCStatus( + GetModelMetadataImpl::GetModelMetadata(core_, *request, response)); if (!status.ok()) { VLOG(1) << "GetModelMetadata failed: " << status.error_message(); } @@ -231,25 +218,24 @@ class PredictionServiceImpl final : public PredictionService::Service { // By default, this is infinite which is the same default as RunOptions. run_options.set_timeout_in_ms( DeadlineToTimeoutMillis(context->raw_deadline())); - const grpc::Status status = - ToGRPCStatus(TensorflowClassificationServiceImpl::Classify( - run_options, core_.get(), *request, response)); + const grpc::Status status = tensorflow::serving::ToGRPCStatus( + TensorflowClassificationServiceImpl::Classify(run_options, core_, + *request, response)); if (!status.ok()) { VLOG(1) << "Classify request failed: " << status.error_message(); } return status; } - grpc::Status Regress(ServerContext* context, - const RegressionRequest* request, + grpc::Status Regress(ServerContext* context, const RegressionRequest* request, RegressionResponse* response) override { tensorflow::RunOptions run_options = tensorflow::RunOptions(); // By default, this is infinite which is the same default as RunOptions. run_options.set_timeout_in_ms( DeadlineToTimeoutMillis(context->raw_deadline())); - const grpc::Status status = - ToGRPCStatus(TensorflowRegressionServiceImpl::Regress( - run_options, core_.get(), *request, response)); + const grpc::Status status = tensorflow::serving::ToGRPCStatus( + TensorflowRegressionServiceImpl::Regress(run_options, core_, *request, + response)); if (!status.ok()) { VLOG(1) << "Regress request failed: " << status.error_message(); } @@ -263,8 +249,8 @@ class PredictionServiceImpl final : public PredictionService::Service { // By default, this is infinite which is the same default as RunOptions. run_options.set_timeout_in_ms( DeadlineToTimeoutMillis(context->raw_deadline())); - const grpc::Status status = ToGRPCStatus( - RunMultiInference(run_options, core_.get(), *request, response)); + const grpc::Status status = tensorflow::serving::ToGRPCStatus( + RunMultiInference(run_options, core_, *request, response)); if (!status.ok()) { VLOG(1) << "MultiInference request failed: " << status.error_message(); } @@ -272,7 +258,7 @@ class PredictionServiceImpl final : public PredictionService::Service { } private: - std::unique_ptr core_; + ServerCore* core_; std::unique_ptr predictor_; bool use_saved_model_; }; @@ -281,11 +267,13 @@ void RunServer(int port, std::unique_ptr core, bool use_saved_model) { // "0.0.0.0" is the way to listen on localhost in gRPC. const string server_address = "0.0.0.0:" + std::to_string(port); - PredictionServiceImpl service(std::move(core), use_saved_model); + tensorflow::serving::ModelServiceImpl model_service(core.get()); + PredictionServiceImpl prediction_service(core.get(), use_saved_model); ServerBuilder builder; std::shared_ptr creds = InsecureServerCredentials(); builder.AddListeningPort(server_address, creds); - builder.RegisterService(&service); + builder.RegisterService(&model_service); + builder.RegisterService(&prediction_service); builder.SetMaxMessageSize(tensorflow::kint32max); std::unique_ptr server(builder.BuildAndStart()); LOG(INFO) << "Running ModelServer at " << server_address << " ..."; diff --git a/tensorflow_serving/model_servers/model_service_impl.cc b/tensorflow_serving/model_servers/model_service_impl.cc new file mode 100644 index 00000000000..b239ba71bb2 --- /dev/null +++ b/tensorflow_serving/model_servers/model_service_impl.cc @@ -0,0 +1,36 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/model_servers/model_service_impl.h" + +#include "tensorflow_serving/model_servers/get_model_status_impl.h" +#include "tensorflow_serving/model_servers/grpc_status_util.h" + +namespace tensorflow { +namespace serving { + +::grpc::Status ModelServiceImpl::GetModelStatus( + ::grpc::ServerContext* context, const GetModelStatusRequest* request, + GetModelStatusResponse* response) { + const ::grpc::Status status = tensorflow::serving::ToGRPCStatus( + GetModelStatusImpl::GetModelStatus(core_, *request, response)); + if (!status.ok()) { + VLOG(1) << "GetModelStatus failed: " << status.error_message(); + } + return status; +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/model_servers/model_service_impl.h b/tensorflow_serving/model_servers/model_service_impl.h new file mode 100644 index 00000000000..b0937daa867 --- /dev/null +++ b/tensorflow_serving/model_servers/model_service_impl.h @@ -0,0 +1,43 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_MODEL_SERVICE_IMPL_H_ +#define TENSORFLOW_SERVING_MODEL_SERVERS_MODEL_SERVICE_IMPL_H_ + +#include "grpc++/server_context.h" +#include "grpc++/support/status.h" +#include "tensorflow_serving/apis/model_service.grpc.pb.h" +#include "tensorflow_serving/apis/model_service.pb.h" +#include "tensorflow_serving/model_servers/server_core.h" + +namespace tensorflow { +namespace serving { + +class ModelServiceImpl final : public grpc::ModelService::Service { + public: + explicit ModelServiceImpl(ServerCore* core) : core_(core) {} + + ::grpc::Status GetModelStatus(::grpc::ServerContext* context, + const GetModelStatusRequest* request, + GetModelStatusResponse* response) override; + + private: + ServerCore* core_; +}; + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_MODEL_SERVERS_MODEL_SERVICE_IMPL_H_ diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 2e66b64f67c..96e1de1211e 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -26,6 +26,7 @@ # This is a placeholder for a Google-internal import. +import grpc from grpc.beta import implementations from grpc.beta import interfaces as beta_interfaces from grpc.framework.interfaces.face import face @@ -34,6 +35,8 @@ from tensorflow.core.framework import types_pb2 from tensorflow.python.platform import flags from tensorflow_serving.apis import classification_pb2 +from tensorflow_serving.apis import get_model_status_pb2 +from tensorflow_serving.apis import model_service_pb2_grpc from tensorflow_serving.apis import predict_pb2 from tensorflow_serving.apis import prediction_service_pb2 from tensorflow_serving.apis import regression_pb2 @@ -251,6 +254,27 @@ def _VerifyModelSpec(self, self.assertEquals(actual_model_spec.signature_name, exp_signature_name) self.assertEquals(actual_model_spec.version.value, exp_version) + def testGetModelStatus(self): + """Test ModelService.GetModelStatus implementation.""" + model_path = self._GetSavedModelBundlePath() + + atexit.register(self.TerminateProcs) + model_server_address = self.RunServer(PickUnusedPort(), 'default', + model_path) + + print 'Sending GetModelStatus request...' + # Send request + request = get_model_status_pb2.GetModelStatusRequest() + request.model_spec.name = 'default' + channel = grpc.insecure_channel(model_server_address) + stub = model_service_pb2_grpc.ModelServiceStub(channel) + result = stub.GetModelStatus(request, RPC_TIMEOUT) # 5 secs timeout + # Verify response + self.assertEquals(1, len(result.model_version_status)) + self.assertEquals(123, result.model_version_status[0].version) + # OK error code (0) indicates no error occurred + self.assertEquals(0, result.model_version_status[0].status.error_code) + def testClassify(self): """Test PredictionService.Classify implementation.""" model_path = self._GetSavedModelBundlePath() diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 21560df484a..27f202a1dca 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -318,6 +318,7 @@ cc_test( ) load("//tensorflow_serving:serving.bzl", "serving_proto_library") +load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") serving_proto_library( name = "class_registration_test_proto", @@ -340,3 +341,12 @@ serving_proto_library( "@org_tensorflow//tensorflow/core:protos_all_cc", ], ) + +serving_proto_library_py( + name = "status_proto_py_pb2", + srcs = ["status.proto"], + proto_library = "status_proto", + deps = [ + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) From c668c586afb2c52dc992ce0db6d98c9b48f57025 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 1 Feb 2018 07:11:03 -0800 Subject: [PATCH 0424/8554] In Predict service, relax checks of inputs and outputs to encompass more general signatures allowing empty. Change: 184135564 --- tensorflow_serving/servables/tensorflow/BUILD | 3 + .../servables/tensorflow/predict_impl.cc | 8 -- .../servables/tensorflow/predict_impl_test.cc | 118 ++++++++++++++++++ .../servables/tensorflow/testdata/BUILD | 14 +++ .../tensorflow/testdata/export_counter.py | 106 ++++++++++++++++ .../00000123/saved_model.pb | Bin 0 -> 5289 bytes .../variables/variables.data-00000-of-00001 | Bin 0 -> 4 bytes .../00000123/variables/variables.index | Bin 0 -> 123 bytes 8 files changed, 241 insertions(+), 8 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/testdata/export_counter.py create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_counter/00000123/saved_model.pb create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_counter/00000123/variables/variables.data-00000-of-00001 create mode 100644 tensorflow_serving/servables/tensorflow/testdata/saved_model_counter/00000123/variables/variables.index diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 570ff8afa7f..466e2c44d60 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -496,6 +496,9 @@ cc_test( "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_counter/00000123/saved_model.pb", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_counter/00000123/variables/variables.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_counter/00000123/variables/variables.index", "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], deps = [ diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index 058ebd2c086..4d161ef1191 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -164,14 +164,6 @@ Status PreProcessPrediction(const SignatureDef& signature, kPredictMethodName, ", ", kClassifyMethodName, ", ", kRegressMethodName, "}. Was: ", signature.method_name())); } - if (signature.inputs().empty()) { - return errors::Internal(strings::StrCat( - "Expected at least one input Tensor in prediction signature.")); - } - if (signature.outputs().empty()) { - return errors::Internal(strings::StrCat( - "Expected at least one output Tensor in prediction signature.")); - } // Verify and prepare input. if (request.inputs().size() != signature.inputs().size()) { diff --git a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc index 9bb40588e2e..6a5f330f361 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl_test.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl_test.cc @@ -57,6 +57,10 @@ class PredictImplTest : public ::testing::TestWithParam { true, &saved_model_server_core_)); TF_ASSERT_OK(CreateServerCore(bad_half_plus_two_path, true, &saved_model_server_core_bad_model_)); + TF_ASSERT_OK(CreateServerCore( + test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/saved_model_counter"), + true, &saved_model_server_core_counter_model_)); } static void TearDownTestCase() { @@ -64,6 +68,7 @@ class PredictImplTest : public ::testing::TestWithParam { server_core_bad_model_.reset(); saved_model_server_core_.reset(); saved_model_server_core_bad_model_.reset(); + saved_model_server_core_counter_model_.reset(); } protected: @@ -103,6 +108,10 @@ class PredictImplTest : public ::testing::TestWithParam { return server_core_bad_model_.get(); } + ServerCore* GetServerCoreWithCounterModel() { + return saved_model_server_core_counter_model_.get(); + } + RunOptions GetRunOptions() { return RunOptions(); } private: @@ -110,12 +119,15 @@ class PredictImplTest : public ::testing::TestWithParam { static std::unique_ptr server_core_bad_model_; static std::unique_ptr saved_model_server_core_; static std::unique_ptr saved_model_server_core_bad_model_; + static std::unique_ptr saved_model_server_core_counter_model_; }; std::unique_ptr PredictImplTest::server_core_; std::unique_ptr PredictImplTest::server_core_bad_model_; std::unique_ptr PredictImplTest::saved_model_server_core_; std::unique_ptr PredictImplTest::saved_model_server_core_bad_model_; +std::unique_ptr + PredictImplTest::saved_model_server_core_counter_model_; TEST_P(PredictImplTest, MissingOrEmptyModelSpec) { PredictRequest request; @@ -366,6 +378,112 @@ TEST_P(PredictImplTest, PredictionWithNamedClassificationSignature) { EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } +// Test querying a counter model with signatures. Predict calls work with +// customized signatures. It calls get_counter, incr_counter, +// reset_counter, incr_counter, and incr_counter_by(3) in order. +// +// *Notes*: These signatures are stateful and over-simplied only to demonstrate +// Predict calls with only inputs or outputs. State is not supported in +// TensorFlow Serving on most scalable or production hosting environments. +TEST_P(PredictImplTest, PredictionWithCustomizedSignatures) { + PredictRequest request; + PredictResponse response; + TensorflowPredictor predictor(GetParam()); + + // Call get_counter. Expected result 0. + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + model_spec->set_signature_name("get_counter"); + + // This request is expected to work with SavedModel, but not SessionBundle. + const bool using_session_bundle = !GetParam(); + if (using_session_bundle) { + ASSERT_EQ(tensorflow::error::INVALID_ARGUMENT, + predictor + .Predict(GetRunOptions(), GetServerCoreWithCounterModel(), + request, &response) + .code()); + return; + } + + TF_ASSERT_OK(predictor.Predict( + GetRunOptions(), GetServerCoreWithCounterModel(), request, &response)); + + PredictResponse expected_get_counter; + *expected_get_counter.mutable_model_spec() = *model_spec; + TensorProto output_get_counter; + output_get_counter.add_float_val(0); + output_get_counter.set_dtype(tensorflow::DT_FLOAT); + output_get_counter.mutable_tensor_shape(); + (*expected_get_counter.mutable_outputs())["output"] = output_get_counter; + EXPECT_THAT(response, test_util::EqualsProto(expected_get_counter)); + + // Call incr_counter. Expect: 1. + model_spec->set_signature_name("incr_counter"); + TF_ASSERT_OK(predictor.Predict( + GetRunOptions(), GetServerCoreWithCounterModel(), request, &response)); + + PredictResponse expected_incr_counter; + *expected_incr_counter.mutable_model_spec() = *model_spec; + TensorProto output_incr_counter; + output_incr_counter.add_float_val(1); + output_incr_counter.set_dtype(tensorflow::DT_FLOAT); + output_incr_counter.mutable_tensor_shape(); + (*expected_incr_counter.mutable_outputs())["output"] = output_incr_counter; + EXPECT_THAT(response, test_util::EqualsProto(expected_incr_counter)); + + // Call reset_counter. Expect: 0. + model_spec->set_signature_name("reset_counter"); + TF_ASSERT_OK(predictor.Predict( + GetRunOptions(), GetServerCoreWithCounterModel(), request, &response)); + + PredictResponse expected_reset_counter; + *expected_reset_counter.mutable_model_spec() = *model_spec; + TensorProto output_reset_counter; + output_reset_counter.add_float_val(0); + output_reset_counter.set_dtype(tensorflow::DT_FLOAT); + output_reset_counter.mutable_tensor_shape(); + (*expected_reset_counter.mutable_outputs())["output"] = output_reset_counter; + EXPECT_THAT(response, test_util::EqualsProto(expected_reset_counter)); + + // Call incr_counter. Expect: 1. + model_spec->set_signature_name("incr_counter"); + request.add_output_filter("output"); + TF_ASSERT_OK(predictor.Predict( + GetRunOptions(), GetServerCoreWithCounterModel(), request, &response)); + request.clear_output_filter(); + + PredictResponse expected_incr_counter2; + *expected_incr_counter2.mutable_model_spec() = *model_spec; + TensorProto output_incr_counter2; + output_incr_counter2.add_float_val(1); + output_incr_counter2.set_dtype(tensorflow::DT_FLOAT); + output_incr_counter2.mutable_tensor_shape(); + (*expected_incr_counter2.mutable_outputs())["output"] = output_incr_counter2; + EXPECT_THAT(response, test_util::EqualsProto(expected_incr_counter2)); + + // Call incr_counter_by. Expect: 4. + model_spec->set_signature_name("incr_counter_by"); + TensorProto tensor_proto; + tensor_proto.add_float_val(3); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())["delta"] = tensor_proto; + + TF_ASSERT_OK(predictor.Predict( + GetRunOptions(), GetServerCoreWithCounterModel(), request, &response)); + + PredictResponse expected_incr_counter_by; + *expected_incr_counter_by.mutable_model_spec() = *model_spec; + TensorProto output_incr_counter_by; + output_incr_counter_by.add_float_val(4); + output_incr_counter_by.set_dtype(tensorflow::DT_FLOAT); + output_incr_counter_by.mutable_tensor_shape(); + (*expected_incr_counter_by.mutable_outputs())["output"] = + output_incr_counter_by; + EXPECT_THAT(response, test_util::EqualsProto(expected_incr_counter_by)); +} + // Test all PredictImplTest test cases with both SessionBundle and SavedModel. INSTANTIATE_TEST_CASE_P(UseSavedModel, PredictImplTest, ::testing::Bool()); diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index 842ce699782..a31bc204010 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -46,6 +46,17 @@ py_binary( ], ) +py_binary( + name = "export_counter", + srcs = [ + "export_counter.py", + ], + srcs_version = "PY2AND3", + deps = [ + "@org_tensorflow//tensorflow:tensorflow_py", + ], +) + # Note: re-generate these files with :export_half_plus_two whenever model # changes. exports_files([ @@ -58,6 +69,9 @@ exports_files([ "saved_model_half_plus_three/00000123/variables/variables.index", "bad_half_plus_two/00000123/export", "bad_half_plus_two/00000123/export.meta", + "saved_model_counter/00000123/saved_model.pb", + "saved_model_counter/00000123/variables/variables.data-00000-of-00001", + "saved_model_counter/00000123/variables/variables.index", "good_model_config.txt", "bad_model_config.txt", "batching_config.txt", diff --git a/tensorflow_serving/servables/tensorflow/testdata/export_counter.py b/tensorflow_serving/servables/tensorflow/testdata/export_counter.py new file mode 100644 index 00000000000..ee1b73914b8 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/testdata/export_counter.py @@ -0,0 +1,106 @@ +# Copyright 2016 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Exports a counter model. + +It contains 4 signatures: get_counter incr_counter, incr_counter_by, and +reset_counter, to test Predict service. +""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +# This is a placeholder for a Google-internal import. +import tensorflow as tf + + +def save_model(sess, signature_def_map, output_dir): + """Saves the model with given signature def map.""" + builder = tf.saved_model.builder.SavedModelBuilder(output_dir) + builder.add_meta_graph_and_variables( + sess, [tf.saved_model.tag_constants.SERVING], + signature_def_map=signature_def_map) + builder.save() + + +def build_signature_def_from_tensors(inputs, outputs, method_name): + """Builds signature def with inputs, outputs, and method_name.""" + return tf.saved_model.signature_def_utils.build_signature_def( + inputs={ + key: tf.saved_model.utils.build_tensor_info(tensor) + for key, tensor in inputs.iteritems() + }, + outputs={ + key: tf.saved_model.utils.build_tensor_info(tensor) + for key, tensor in outputs.iteritems() + }, + method_name=method_name) + + +def export_model(output_dir): + """Exports the counter model. + + Create three signatures: incr_counter, incr_counter_by, reset_counter. + + *Notes*: These signatures are stateful and over-simplied only to demonstrate + Predict calls with only inputs or outputs. State is not supported in + TensorFlow Serving on most scalable or production hosting environments. + + Args: + output_dir: string, output directory for the model. + """ + tf.logging.info("Exporting the counter model to %s.", output_dir) + method_name = tf.saved_model.signature_constants.PREDICT_METHOD_NAME + + graph = tf.Graph() + with graph.as_default(), tf.Session() as sess: + counter = tf.Variable(0.0, dtype=tf.float32, name="counter") + + with tf.name_scope("incr_counter_op", values=[counter]): + incr_counter = counter.assign_add(1.0) + + delta = tf.placeholder(dtype=tf.float32, name="delta") + with tf.name_scope("incr_counter_by_op", values=[counter, delta]): + incr_counter_by = counter.assign_add(delta) + + with tf.name_scope("reset_counter_op", values=[counter]): + reset_counter = counter.assign(0.0) + + sess.run(tf.global_variables_initializer()) + + signature_def_map = { + "get_counter": + build_signature_def_from_tensors({}, {"output": counter}, + method_name), + "incr_counter": + build_signature_def_from_tensors({}, {"output": incr_counter}, + method_name), + "incr_counter_by": + build_signature_def_from_tensors({ + "delta": delta + }, {"output": incr_counter_by}, method_name), + "reset_counter": + build_signature_def_from_tensors({}, {"output": reset_counter}, + method_name) + } + save_model(sess, signature_def_map, output_dir) + + +def main(unused_argv): + export_model("/tmp/saved_model_counter/00000123") + + +if __name__ == "__main__": + tf.app.run() diff --git a/tensorflow_serving/servables/tensorflow/testdata/saved_model_counter/00000123/saved_model.pb b/tensorflow_serving/servables/tensorflow/testdata/saved_model_counter/00000123/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..ede52f609e68fdb79d074414116a8841c844396f GIT binary patch literal 5289 zcmb_gOLN;)6!!Jo$4;Ewyqueq77a6`>NIxJh6Yb(C<#;A&NKx27+_J2V%;Q~D3V8# z(`MHN3z%WSf(0vv1xsf58w|gMpTW`9!;)+{4J4abzE|h*o#*E&p}*gRf2#EFD!hU6 zr;ei^nY0cCn;%N#`CCFLKbZ&7eOK%Rgp|oq6j?0=*_MMVSnsB2TcudPKHyz8Crocb(-Gp_hKGwMD>h8IR zj&33#9y;|%5P2Sh1vt>B+&?wP0@7Q8KU?-qF9FFp7li^o3Xq!y-lIAsl z?9l=mvPL>2W~|hNqWVU6#2L%CaW7Nv+QM8iaNb^wgL+}4%#csYuz=GOW z=#yn52#8tPcNd+VfrbyVVydUY{dt%X0uC$s_81Jx0q(aoAb#nxH{3=5cp zQ?EX3&1n@FUOWt5YE@tl;F)V#4)3eBiV)MX zBP6gq&0%T5OV!rZ0|T+Z;EBTavl)w9nodS5mFZYKwVq|Vs&4XxSZNvRo&#JHVT)@- z@THQ&?K3XdHrBV+JGbrL#%5(;o|x8alm4{=-@-!A8kjD(+q$W{x@ssQPpMyz@24$sZsE&hWF-;rTE4vQ@1)iz# z{^?!M070&JP0=&(6*NNoY_4iFYFm+g6IEU)UxVwq*|U{EiDLEJQ6Fz7VwxptT&&-I zl880@MqoKp4INI7$SHP_Rij|$E(CKDQt$+GlN+u|K0&giF-=z$5RHr3YEwJ)YK7)@N~ad4Ogt$nS>c24 z2(*OA;b_3tiCc%GY=jE#hYIfOYzPIn6qld&m3w#ZbhNGB-R+O>bnb3-HnunT=C-!6 zt@iF|2Z!4qwQ*{=?`FmTnkg%2Y*?|F*DTZAEFZnl?a_QI@Acf_DDy_xmx0>h)C_%& zjf6s`U>X8b+o@CwG}p!L-~^cONpfhny_u#3(xs`k#?Ns5=n=yv2$If#exRk9OKxMb=e z14zx~DwMn84RB%Ahrw4YT4QX<%TPW+VscXm4NF9&;-O!UsinbM$i{{w%-d0D*hJf- z(UT4WatnW3r6nkAV&>s6TN37YdzEb&ipKlgL^jbokB#T?#x-8W97${XFxnli!3td# zyMuo{nd6yt5^N0>_=CVL@#ZLLX^&wnRIrUKJbjZ#olToRDa;U15@aJ`n;>DU7XkJW z9W|GZ;a%)9zQuc5+Tw*#I)+Ob;P{>r=tUH9xOF8uWEL{4<)4hYleQqE!iUhDz=|*> zqF&>wUB067K8EWlWf>LzE@g|`R(2!#3HR=#DmrAB@sP1e?z4-`#YvPb-U&gL`wN8 zWA~vUUJ-jvw?qBu6{dZ+!$u_DD6NsL+=JDJ+j;2))XoB8>d;5@A$;a>EkO%WR*F}# z=-J~j-OlDxxN{{KWVa(JFz4FHS3Sauw9;q3SExXK2=$}kLQL<`<&cz9!4?A#W;n|+ z<`36H!+PEJC^6FdHvaOV={;9Ihh}Qar-#rAc|;Q`f>3ljGX#HyA#^>`@&UY;dNDx{ v@QsU9e*=xU1p1iXiln~}EAg|-ivAj=Fi~Rod`@phvaiE(;&3SZVx#{B$b1SE#hDlV$fhQT~;k<#Q+41AR>N&Krx(i;M2kl42(=E c3~F#8rHzTq+8h>qAOS)k{C7jQN~!y80QJ8R3;+NC literal 0 HcmV?d00001 From 2dca5c8c999a61dafbf772b88ae315ef190bf4f5 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 1 Feb 2018 15:44:25 -0800 Subject: [PATCH 0425/8554] Add model_service_pb2.py and model_service_pb2_grpc.py to open source. PiperOrigin-RevId: 184207084 --- tensorflow_serving/apis/BUILD | 5 +- tensorflow_serving/apis/model_service_pb2.py | 182 ++++++++++++++++++ .../apis/model_service_pb2_grpc.py | 71 +++++++ .../model_servers/model_service_impl.h | 2 +- 4 files changed, 255 insertions(+), 5 deletions(-) create mode 100644 tensorflow_serving/apis/model_service_pb2.py create mode 100644 tensorflow_serving/apis/model_service_pb2_grpc.py diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 7c7ebf79b9b..aed87f2ad14 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -198,10 +198,7 @@ py_library( "model_service_pb2_grpc.py", ], srcs_version = "PY2AND3", - deps = [ - ":get_model_status_proto_py_pb2", - "//net/grpc/python:grpc", - ], + deps = [":get_model_status_proto_py_pb2"], ) serving_go_grpc_library( diff --git a/tensorflow_serving/apis/model_service_pb2.py b/tensorflow_serving/apis/model_service_pb2.py new file mode 100644 index 00000000000..5c8787abd70 --- /dev/null +++ b/tensorflow_serving/apis/model_service_pb2.py @@ -0,0 +1,182 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorflow_serving/apis/model_service.proto +# To regenerate run +# python -m grpc.tools.protoc --python_out=. --grpc_python_out=. -I. tensorflow_serving/apis/model_service.proto + +import sys +_b=sys.version_info[0]<3 and (lambda x:x) or (lambda x:x.encode('latin1')) +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +from google.protobuf import descriptor_pb2 +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + +from tensorflow_serving.apis import get_model_status_pb2 as tensorflow__serving_dot_apis_dot_get__model__status__pb2 + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='tensorflow_serving/apis/model_service.proto', + package='tensorflow.serving', + syntax='proto3', + serialized_pb=_b('\n+tensorflow_serving/apis/model_service.proto\x12\x12tensorflow.serving\x1a.tensorflow_serving/apis/get_model_status.proto2w\n\x0cModelService\x12g\n\x0eGetModelStatus\x12).tensorflow.serving.GetModelStatusRequest\x1a*.tensorflow.serving.GetModelStatusResponseB\x03\xf8\x01\x01\x62\x06proto3') + , + dependencies=[tensorflow__serving_dot_apis_dot_get__model__status__pb2.DESCRIPTOR,]) +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + + + + + +DESCRIPTOR.has_options = True +DESCRIPTOR._options = _descriptor._ParseOptions(descriptor_pb2.FileOptions(), _b('\370\001\001')) +try: + # THESE ELEMENTS WILL BE DEPRECATED. + # Please use the generated *_pb2_grpc.py files instead. + import grpc + from grpc.framework.common import cardinality + from grpc.framework.interfaces.face import utilities as face_utilities + from grpc.beta import implementations as beta_implementations + from grpc.beta import interfaces as beta_interfaces + + + class ModelServiceStub(object): + """ModelService provides access to information about model versions + that have been handled by the model server. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetModelStatus = channel.unary_unary( + '/tensorflow.serving.ModelService/GetModelStatus', + request_serializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusResponse.FromString, + ) + + + class ModelServiceServicer(object): + """ModelService provides access to information about model versions + that have been handled by the model server. + """ + + def GetModelStatus(self, request, context): + """Gets status of model. If the ModelSpec in the request does not specify + version, information about all versions of the model will be returned. If + the ModelSpec in the request does specify a version, the status of only + that version will be returned. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + + def add_ModelServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetModelStatus': grpc.unary_unary_rpc_method_handler( + servicer.GetModelStatus, + request_deserializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'tensorflow.serving.ModelService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) + + + class BetaModelServiceServicer(object): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This class was generated + only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" + """ModelService provides access to information about model versions + that have been handled by the model server. + """ + def GetModelStatus(self, request, context): + """Gets status of model. If the ModelSpec in the request does not specify + version, information about all versions of the model will be returned. If + the ModelSpec in the request does specify a version, the status of only + that version will be returned. + """ + context.code(beta_interfaces.StatusCode.UNIMPLEMENTED) + + + class BetaModelServiceStub(object): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This class was generated + only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0.""" + """ModelService provides access to information about model versions + that have been handled by the model server. + """ + def GetModelStatus(self, request, timeout, metadata=None, with_call=False, protocol_options=None): + """Gets status of model. If the ModelSpec in the request does not specify + version, information about all versions of the model will be returned. If + the ModelSpec in the request does specify a version, the status of only + that version will be returned. + """ + raise NotImplementedError() + GetModelStatus.future = None + + + def beta_create_ModelService_server(servicer, pool=None, pool_size=None, default_timeout=None, maximum_timeout=None): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This function was + generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" + request_deserializers = { + ('tensorflow.serving.ModelService', 'GetModelStatus'): tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusRequest.FromString, + } + response_serializers = { + ('tensorflow.serving.ModelService', 'GetModelStatus'): tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusResponse.SerializeToString, + } + method_implementations = { + ('tensorflow.serving.ModelService', 'GetModelStatus'): face_utilities.unary_unary_inline(servicer.GetModelStatus), + } + server_options = beta_implementations.server_options(request_deserializers=request_deserializers, response_serializers=response_serializers, thread_pool=pool, thread_pool_size=pool_size, default_timeout=default_timeout, maximum_timeout=maximum_timeout) + return beta_implementations.server(method_implementations, options=server_options) + + + def beta_create_ModelService_stub(channel, host=None, metadata_transformer=None, pool=None, pool_size=None): + """The Beta API is deprecated for 0.15.0 and later. + + It is recommended to use the GA API (classes and functions in this + file not marked beta) for all further purposes. This function was + generated only to ease transition from grpcio<0.15.0 to grpcio>=0.15.0""" + request_serializers = { + ('tensorflow.serving.ModelService', 'GetModelStatus'): tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusRequest.SerializeToString, + } + response_deserializers = { + ('tensorflow.serving.ModelService', 'GetModelStatus'): tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusResponse.FromString, + } + cardinalities = { + 'GetModelStatus': cardinality.Cardinality.UNARY_UNARY, + } + stub_options = beta_implementations.stub_options(host=host, metadata_transformer=metadata_transformer, request_serializers=request_serializers, response_deserializers=response_deserializers, thread_pool=pool, thread_pool_size=pool_size) + return beta_implementations.dynamic_stub(channel, 'tensorflow.serving.ModelService', cardinalities, options=stub_options) +except ImportError: + pass +# @@protoc_insertion_point(module_scope) diff --git a/tensorflow_serving/apis/model_service_pb2_grpc.py b/tensorflow_serving/apis/model_service_pb2_grpc.py new file mode 100644 index 00000000000..a44c7c7f53a --- /dev/null +++ b/tensorflow_serving/apis/model_service_pb2_grpc.py @@ -0,0 +1,71 @@ +# Copyright 2018 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: tensorflow_serving/apis/model_service.proto +# To regenerate run +# python -m grpc.tools.protoc --python_out=. --grpc_python_out=. -I. tensorflow_serving/apis/model_service.proto + +import grpc +from grpc.framework.common import cardinality +from grpc.framework.interfaces.face import utilities as face_utilities + +import tensorflow_serving.apis.get_model_status_pb2 as tensorflow__serving_dot_apis_dot_get__model__status__pb2 + + +class ModelServiceStub(object): + """ModelService provides access to information about model versions + that have been handled by the model server. + """ + + def __init__(self, channel): + """Constructor. + + Args: + channel: A grpc.Channel. + """ + self.GetModelStatus = channel.unary_unary( + '/tensorflow.serving.ModelService/GetModelStatus', + request_serializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusRequest.SerializeToString, + response_deserializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusResponse.FromString, + ) + + +class ModelServiceServicer(object): + """ModelService provides access to information about model versions + that have been handled by the model server. + """ + + def GetModelStatus(self, request, context): + """Gets status of model. If the ModelSpec in the request does not specify + version, information about all versions of the model will be returned. If + the ModelSpec in the request does specify a version, the status of only + that version will be returned. + """ + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + +def add_ModelServiceServicer_to_server(servicer, server): + rpc_method_handlers = { + 'GetModelStatus': grpc.unary_unary_rpc_method_handler( + servicer.GetModelStatus, + request_deserializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusRequest.FromString, + response_serializer=tensorflow__serving_dot_apis_dot_get__model__status__pb2.GetModelStatusResponse.SerializeToString, + ), + } + generic_handler = grpc.method_handlers_generic_handler( + 'tensorflow.serving.ModelService', rpc_method_handlers) + server.add_generic_rpc_handlers((generic_handler,)) diff --git a/tensorflow_serving/model_servers/model_service_impl.h b/tensorflow_serving/model_servers/model_service_impl.h index b0937daa867..0c4b00cf887 100644 --- a/tensorflow_serving/model_servers/model_service_impl.h +++ b/tensorflow_serving/model_servers/model_service_impl.h @@ -25,7 +25,7 @@ limitations under the License. namespace tensorflow { namespace serving { -class ModelServiceImpl final : public grpc::ModelService::Service { +class ModelServiceImpl final : public ModelService::Service { public: explicit ModelServiceImpl(ServerCore* core) : core_(core) {} From 63595f8ca369991df355b75e2cf2aa79bb08197e Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 2 Feb 2018 14:22:21 -0800 Subject: [PATCH 0426/8554] Sync the TF commit we use to latest nightly green. PiperOrigin-RevId: 184333954 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index c6fa8cb1e41..3806a55aca3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "9c3db796b2c6bc24ebff94d689c442f2cf1d8955f122a277e4ad893c0ce96aac", - git_commit = "57b32eabca4597241120cb4aba8308a431853c30", + sha256 = "21d6ac553adcfc9d089925f6d6793fee6a67264a0ce717bc998636662df4ca7e", + git_commit = "bc69c4ceed6544c109be5693eb40ddcf3a4eb95d", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. From 2c450d295765fe5f1871ecc2db65c4ffa9ca8c73 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 2 Feb 2018 15:27:04 -0800 Subject: [PATCH 0427/8554] Remove tensorflow and tf_models as submodules now that they're linked as bazel rules. --- .gitmodules | 6 ------ tensorflow | 1 - tf_models | 1 - 3 files changed, 8 deletions(-) delete mode 160000 tensorflow delete mode 160000 tf_models diff --git a/.gitmodules b/.gitmodules index 8f873f6f781..e69de29bb2d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +0,0 @@ -[submodule "tensorflow"] - path = tensorflow - url = https://github.com/tensorflow/tensorflow.git -[submodule "tf_models"] - path = tf_models - url = https://github.com/tensorflow/models.git diff --git a/tensorflow b/tensorflow deleted file mode 160000 index 57b32eabca4..00000000000 --- a/tensorflow +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 57b32eabca4597241120cb4aba8308a431853c30 diff --git a/tf_models b/tf_models deleted file mode 160000 index 6fc65ee60ac..00000000000 --- a/tf_models +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6fc65ee60ac39be0445e5a311b40dc7ccce214d0 From e03c0ae3b8f69dde1b4e20310c090020663c6707 Mon Sep 17 00:00:00 2001 From: Jesse Kinkead Date: Mon, 5 Feb 2018 11:14:55 -0800 Subject: [PATCH 0428/8554] Use print function for python3 compatibility. (#707) --- .../example/inception_saved_model.py | 12 +++++++----- .../example/mnist_saved_model.py | 18 ++++++++++-------- 2 files changed, 17 insertions(+), 13 deletions(-) diff --git a/tensorflow_serving/example/inception_saved_model.py b/tensorflow_serving/example/inception_saved_model.py index 86b20c460d0..8f3182e7421 100644 --- a/tensorflow_serving/example/inception_saved_model.py +++ b/tensorflow_serving/example/inception_saved_model.py @@ -20,6 +20,8 @@ standard tensorflow_model_server. """ +from __future__ import print_function + import os.path # This is a placeholder for a Google-internal import. @@ -105,17 +107,17 @@ def export(): # /my-favorite-path/imagenet_train/model.ckpt-0, # extract global_step from it. global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] - print 'Successfully loaded model from %s at step=%s.' % ( - ckpt.model_checkpoint_path, global_step) + print('Successfully loaded model from %s at step=%s.' % ( + ckpt.model_checkpoint_path, global_step)) else: - print 'No checkpoint file found at %s' % FLAGS.checkpoint_dir + print('No checkpoint file found at %s' % FLAGS.checkpoint_dir) return # Export inference model. output_path = os.path.join( tf.compat.as_bytes(FLAGS.output_dir), tf.compat.as_bytes(str(FLAGS.model_version))) - print 'Exporting trained model to', output_path + print('Exporting trained model to', output_path) builder = tf.saved_model.builder.SavedModelBuilder(output_path) # Build the signature_def_map. @@ -165,7 +167,7 @@ def export(): legacy_init_op=legacy_init_op) builder.save() - print 'Successfully exported model to %s' % FLAGS.output_dir + print('Successfully exported model to %s' % FLAGS.output_dir) def preprocess_image(image_buffer): diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py index 196cf9af69c..fa527547a8b 100644 --- a/tensorflow_serving/example/mnist_saved_model.py +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -25,6 +25,8 @@ export_dir """ +from __future__ import print_function + import os import sys @@ -47,14 +49,14 @@ def main(_): '[--model_version=y] export_dir') sys.exit(-1) if FLAGS.training_iteration <= 0: - print 'Please specify a positive value for training iteration.' + print('Please specify a positive value for training iteration.') sys.exit(-1) if FLAGS.model_version <= 0: - print 'Please specify a positive value for version number.' + print('Please specify a positive value for version number.') sys.exit(-1) # Train model - print 'Training model...' + print('Training model...') mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) sess = tf.InteractiveSession() serialized_tf_example = tf.placeholder(tf.string, name='tf_example') @@ -77,10 +79,10 @@ def main(_): train_step.run(feed_dict={x: batch[0], y_: batch[1]}) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) - print 'training accuracy %g' % sess.run( + print('training accuracy %g' % sess.run( accuracy, feed_dict={x: mnist.test.images, - y_: mnist.test.labels}) - print 'Done training!' + y_: mnist.test.labels})) + print('Done training!') # Export model # WARNING(break-tutorial-inline-code): The following code snippet is @@ -90,7 +92,7 @@ def main(_): export_path = os.path.join( tf.compat.as_bytes(export_path_base), tf.compat.as_bytes(str(FLAGS.model_version))) - print 'Exporting trained model to', export_path + print('Exporting trained model to', export_path) builder = tf.saved_model.builder.SavedModelBuilder(export_path) # Build the signature_def_map. @@ -136,7 +138,7 @@ def main(_): builder.save() - print 'Done exporting!' + print('Done exporting!') if __name__ == '__main__': From c8cc43b054f0c1e3609bd1f59067c910420054ed Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 6 Feb 2018 16:32:12 -0500 Subject: [PATCH 0429/8554] Update install documentations in light of new change to use TF as a bazel rule. PiperOrigin-RevId: 184730821 --- tensorflow_serving/g3doc/setup.md | 11 +---------- tensorflow_serving/tools/docker/Dockerfile.devel-gpu | 5 ----- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 55985262b79..935f752a79e 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -144,16 +144,7 @@ to the `git clone` command. ### Install prerequisites -Follow the Prerequisites section above to install all dependencies. -To configure TensorFlow, run - -```shell -cd tensorflow -./configure -cd .. -``` - -Consult the +Follow the Prerequisites section above to install all dependencies. Consult the [TensorFlow install instructions](https://www.tensorflow.org/install/) if you encounter any issues with setting up TensorFlow or its dependencies. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu index 6cae9d52286..9e8d680411d 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -87,11 +87,6 @@ RUN mkdir /usr/lib/x86_64-linux-gnu/include/ && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/local/cuda/lib64/libcudnn.so && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.6 /usr/local/cuda/lib64/libcudnn.so.6 - -# Configure Tensorflow to use the GPU -WORKDIR /serving/tensorflow -RUN tensorflow/tools/ci_build/builds/configured GPU - # Build TensorFlow Serving and Install it in /usr/local/bin WORKDIR /serving RUN bazel build -c opt --config=cuda \ From 420636758aa717876fff0ca6e2caf7bba34f4ea5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20S=C3=B6rhus?= Date: Thu, 8 Feb 2018 16:56:00 +0100 Subject: [PATCH 0430/8554] Add possibility to specify grpc channel args (#756) * Add possibility to specify grpc channel args * Address review comments. * Use struct instead of std::pair * Update comment * More explicit naming of GrpcChannelArguments --- tensorflow_serving/model_servers/main.cc | 47 +++++++++++++++++-- .../tensorflow_model_server_test.py | 16 +++++++ 2 files changed, 60 insertions(+), 3 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index c103cd272a9..0db38ea01a6 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -59,6 +59,7 @@ limitations under the License. #include "tensorflow/cc/saved_model/tag_constants.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/init_main.h" #include "tensorflow/core/platform/protobuf.h" @@ -264,8 +265,29 @@ class PredictionServiceImpl final : public PredictionService::Service { bool use_saved_model_; }; +// gRPC Channel Arguments to be passed from command line to gRPC ServerBuilder. +struct GrpcChannelArgument { + string key; + string value; +}; + +// Parses a comma separated list of gRPC channel arguments into list of +// ChannelArgument. +std::vector parseGrpcChannelArgs( + const string& channel_arguments_str) { + const std::vector channel_arguments = + tensorflow::str_util::Split(channel_arguments_str, ","); + std::vector result; + for (const string& channel_argument : channel_arguments) { + const std::vector key_val = + tensorflow::str_util::Split(channel_argument, "="); + result.push_back({key_val[0], key_val[1]}); + } + return result; +} + void RunServer(int port, std::unique_ptr core, - bool use_saved_model) { + bool use_saved_model, const string& grpc_channel_arguments) { // "0.0.0.0" is the way to listen on localhost in gRPC. const string server_address = "0.0.0.0:" + std::to_string(port); tensorflow::serving::ModelServiceImpl model_service(core.get()); @@ -276,6 +298,19 @@ void RunServer(int port, std::unique_ptr core, builder.RegisterService(&model_service); builder.RegisterService(&prediction_service); builder.SetMaxMessageSize(tensorflow::kint32max); + const std::vector channel_arguments = + parseGrpcChannelArgs(grpc_channel_arguments); + for (GrpcChannelArgument channel_argument : channel_arguments) { + // gRPC accept arguments of two types, int and string. We will attempt to + // parse each arg as int and pass it on as such if successful. Otherwise we + // will pass it as a string. gRPC will log arguments that were not accepted. + int value; + if(tensorflow::strings::safe_strto32(channel_argument.key, &value)) { + builder.AddChannelArgument(channel_argument.key, value); + } else { + builder.AddChannelArgument(channel_argument.key, channel_argument.value); + } + } std::unique_ptr server(builder.BuildAndStart()); LOG(INFO) << "Running ModelServer at " << server_address << " ..."; server->Wait(); @@ -307,6 +342,7 @@ int main(int argc, char** argv) { tensorflow::int64 tensorflow_session_parallelism = 0; string platform_config_file = ""; string model_config_file; + string grpc_channel_arguments = ""; std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), @@ -358,7 +394,12 @@ int main(int argc, char** argv) { "starts, If 0.0, Tensorflow will automatically select a value."), tensorflow::Flag("saved_model_tags", &saved_model_tags, "Comma-separated set of tags corresponding to the meta " - "graph def to load from SavedModel.")}; + "graph def to load from SavedModel."), + tensorflow::Flag( + "grpc_channel_arguments", &grpc_channel_arguments, + "A comma separated list of arguments to be passed to " + "the grpc server. (e.g. " + "grpc.max_connection_age_ms=2000)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); @@ -430,7 +471,7 @@ int main(int argc, char** argv) { std::unique_ptr core; TF_CHECK_OK(ServerCore::Create(std::move(options), &core)); - RunServer(port, std::move(core), use_saved_model); + RunServer(port, std::move(core), use_saved_model, grpc_channel_arguments); return 0; } diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 96e1de1211e..d85c2c27737 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -122,6 +122,7 @@ def RunServer(self, model_name, model_path, batching_parameters_file='', + grpc_channel_arguments='', wait_for_server_ready=True): """Run tensorflow_model_server using test config.""" print 'Starting test server...' @@ -132,6 +133,8 @@ def RunServer(self, if batching_parameters_file: command += ' --enable_batching' command += ' --batching_parameters_file=' + batching_parameters_file + if grpc_channel_arguments: + command += ' --grpc_channel_arguments=' + grpc_channel_arguments print command self.server_proc = subprocess.Popen(shlex.split(command)) print 'Server started' @@ -481,6 +484,19 @@ def testBadModelConfig(self): self.assertNotEqual(self.server_proc.stderr, None) self.assertGreater(self.server_proc.stderr.read().find(error_message), -1) + def testGoodGrpcChannelArgs(self): + """Test server starts with grpc_channel_arguments specified.""" + atexit.register(self.TerminateProcs) + model_server_address = self.RunServer( + PickUnusedPort(), + 'default', + self._GetSavedModelBundlePath(), + grpc_channel_arguments= + 'grpc.max_connection_age_ms=2000,grpc.lb_policy_name=grpclb' + ) + self.VerifyPredictRequest(model_server_address, expected_output=3.0, + specify_output=False, expected_version=self._GetModelVersion( + self._GetSavedModelHalfPlusThreePath())) if __name__ == '__main__': tf.test.main() From 16ae0b16437d79209fecf747a9aced5b175b9895 Mon Sep 17 00:00:00 2001 From: Peng Yu Date: Mon, 12 Feb 2018 16:38:14 -0500 Subject: [PATCH 0431/8554] add proto files to pip package --- tensorflow_serving/tools/pip_package/build_pip_package.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/tools/pip_package/build_pip_package.sh b/tensorflow_serving/tools/pip_package/build_pip_package.sh index 4a5f929854c..90b904176d4 100755 --- a/tensorflow_serving/tools/pip_package/build_pip_package.sh +++ b/tensorflow_serving/tools/pip_package/build_pip_package.sh @@ -39,7 +39,10 @@ function main() { cp bazel-genfiles/tensorflow_serving/apis/*_pb2.py \ "${TMPDIR}/tensorflow_serving/apis" - cp bazel-serving/tensorflow_serving/apis/prediction_service_pb2.py \ + cp bazel-serving/tensorflow_serving/apis/*_pb2.py \ + "${TMPDIR}/tensorflow_serving/apis" + + cp bazel-serving/tensorflow_serving/apis/*_grpc.py \ "${TMPDIR}/tensorflow_serving/apis" touch "${TMPDIR}/tensorflow_serving/apis/__init__.py" From 31885da11eb40763635ad97851a1ff92e2fb616c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 8 Feb 2018 12:02:14 -0800 Subject: [PATCH 0432/8554] Add a function to convert ServableState::ManagerState enum elements to strings. PiperOrigin-RevId: 185029861 --- tensorflow_serving/core/servable_state.h | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/core/servable_state.h b/tensorflow_serving/core/servable_state.h index bfe06135194..978b71d26c4 100644 --- a/tensorflow_serving/core/servable_state.h +++ b/tensorflow_serving/core/servable_state.h @@ -62,6 +62,22 @@ struct ServableState { // some point in its lifecycle. kEnd, }; + + static string ManagerStateString(ManagerState state) { + switch (state) { + case ManagerState::kStart: + return "Start"; + case ManagerState::kLoading: + return "Loading"; + case ManagerState::kAvailable: + return "Available"; + case ManagerState::kUnloading: + return "Unloading"; + case ManagerState::kEnd: + return "End"; + } + } + ManagerState manager_state; // Whether anything has gone wrong with this servable. If not OK, the error @@ -73,8 +89,8 @@ struct ServableState { // Returns a string representation of this object. Useful in logging. string DebugString() const { return strings::StrCat("id: ", id.DebugString(), " manager_state: ", - static_cast(manager_state), " health: ", - health.ToString()); + ManagerStateString(manager_state), + " health: ", health.ToString()); } }; From d3742ee12f1f5ca86cf5d15d220b58e352e32a08 Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Thu, 1 Feb 2018 15:44:25 -0800 Subject: [PATCH 0433/8554] Add model_service_pb2.py and model_service_pb2_grpc.py to open source. Change: 184207084 --- WORKSPACE | 4 ++-- tensorflow_serving/core/servable_state.h | 20 ++----------------- tensorflow_serving/g3doc/setup.md | 11 +++++++++- .../tools/docker/Dockerfile.devel-gpu | 5 +++++ 4 files changed, 19 insertions(+), 21 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 3806a55aca3..c6fa8cb1e41 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "21d6ac553adcfc9d089925f6d6793fee6a67264a0ce717bc998636662df4ca7e", - git_commit = "bc69c4ceed6544c109be5693eb40ddcf3a4eb95d", + sha256 = "9c3db796b2c6bc24ebff94d689c442f2cf1d8955f122a277e4ad893c0ce96aac", + git_commit = "57b32eabca4597241120cb4aba8308a431853c30", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. diff --git a/tensorflow_serving/core/servable_state.h b/tensorflow_serving/core/servable_state.h index 978b71d26c4..bfe06135194 100644 --- a/tensorflow_serving/core/servable_state.h +++ b/tensorflow_serving/core/servable_state.h @@ -62,22 +62,6 @@ struct ServableState { // some point in its lifecycle. kEnd, }; - - static string ManagerStateString(ManagerState state) { - switch (state) { - case ManagerState::kStart: - return "Start"; - case ManagerState::kLoading: - return "Loading"; - case ManagerState::kAvailable: - return "Available"; - case ManagerState::kUnloading: - return "Unloading"; - case ManagerState::kEnd: - return "End"; - } - } - ManagerState manager_state; // Whether anything has gone wrong with this servable. If not OK, the error @@ -89,8 +73,8 @@ struct ServableState { // Returns a string representation of this object. Useful in logging. string DebugString() const { return strings::StrCat("id: ", id.DebugString(), " manager_state: ", - ManagerStateString(manager_state), - " health: ", health.ToString()); + static_cast(manager_state), " health: ", + health.ToString()); } }; diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 935f752a79e..55985262b79 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -144,7 +144,16 @@ to the `git clone` command. ### Install prerequisites -Follow the Prerequisites section above to install all dependencies. Consult the +Follow the Prerequisites section above to install all dependencies. +To configure TensorFlow, run + +```shell +cd tensorflow +./configure +cd .. +``` + +Consult the [TensorFlow install instructions](https://www.tensorflow.org/install/) if you encounter any issues with setting up TensorFlow or its dependencies. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu index 9e8d680411d..6cae9d52286 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -87,6 +87,11 @@ RUN mkdir /usr/lib/x86_64-linux-gnu/include/ && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/local/cuda/lib64/libcudnn.so && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.6 /usr/local/cuda/lib64/libcudnn.so.6 + +# Configure Tensorflow to use the GPU +WORKDIR /serving/tensorflow +RUN tensorflow/tools/ci_build/builds/configured GPU + # Build TensorFlow Serving and Install it in /usr/local/bin WORKDIR /serving RUN bazel build -c opt --config=cuda \ From 7f442ec14fe348adef19b27c82caad6cd798c7de Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Fri, 2 Feb 2018 14:22:21 -0800 Subject: [PATCH 0434/8554] Sync the TF commit we use to latest nightly green. Change: 184333954 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index c6fa8cb1e41..3806a55aca3 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "9c3db796b2c6bc24ebff94d689c442f2cf1d8955f122a277e4ad893c0ce96aac", - git_commit = "57b32eabca4597241120cb4aba8308a431853c30", + sha256 = "21d6ac553adcfc9d089925f6d6793fee6a67264a0ce717bc998636662df4ca7e", + git_commit = "bc69c4ceed6544c109be5693eb40ddcf3a4eb95d", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. From 1910478051db38fefc219f9dde4f0ad4322cf6ba Mon Sep 17 00:00:00 2001 From: Kiril Gorovoy Date: Tue, 6 Feb 2018 13:32:12 -0800 Subject: [PATCH 0435/8554] Update install documentations in light of new change to use TF as a bazel rule. Change: 184730821 --- tensorflow_serving/g3doc/setup.md | 11 +---------- tensorflow_serving/tools/docker/Dockerfile.devel-gpu | 5 ----- 2 files changed, 1 insertion(+), 15 deletions(-) diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 55985262b79..935f752a79e 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -144,16 +144,7 @@ to the `git clone` command. ### Install prerequisites -Follow the Prerequisites section above to install all dependencies. -To configure TensorFlow, run - -```shell -cd tensorflow -./configure -cd .. -``` - -Consult the +Follow the Prerequisites section above to install all dependencies. Consult the [TensorFlow install instructions](https://www.tensorflow.org/install/) if you encounter any issues with setting up TensorFlow or its dependencies. diff --git a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu index 6cae9d52286..9e8d680411d 100644 --- a/tensorflow_serving/tools/docker/Dockerfile.devel-gpu +++ b/tensorflow_serving/tools/docker/Dockerfile.devel-gpu @@ -87,11 +87,6 @@ RUN mkdir /usr/lib/x86_64-linux-gnu/include/ && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so /usr/local/cuda/lib64/libcudnn.so && \ ln -s /usr/lib/x86_64-linux-gnu/libcudnn.so.6 /usr/local/cuda/lib64/libcudnn.so.6 - -# Configure Tensorflow to use the GPU -WORKDIR /serving/tensorflow -RUN tensorflow/tools/ci_build/builds/configured GPU - # Build TensorFlow Serving and Install it in /usr/local/bin WORKDIR /serving RUN bazel build -c opt --config=cuda \ From 41dde270d27849729cd7354da9f6d5313501ac6b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 8 Feb 2018 12:02:14 -0800 Subject: [PATCH 0436/8554] Add a function to convert ServableState::ManagerState enum elements to strings. Change: 185029861 --- tensorflow_serving/core/servable_state.h | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/core/servable_state.h b/tensorflow_serving/core/servable_state.h index bfe06135194..978b71d26c4 100644 --- a/tensorflow_serving/core/servable_state.h +++ b/tensorflow_serving/core/servable_state.h @@ -62,6 +62,22 @@ struct ServableState { // some point in its lifecycle. kEnd, }; + + static string ManagerStateString(ManagerState state) { + switch (state) { + case ManagerState::kStart: + return "Start"; + case ManagerState::kLoading: + return "Loading"; + case ManagerState::kAvailable: + return "Available"; + case ManagerState::kUnloading: + return "Unloading"; + case ManagerState::kEnd: + return "End"; + } + } + ManagerState manager_state; // Whether anything has gone wrong with this servable. If not OK, the error @@ -73,8 +89,8 @@ struct ServableState { // Returns a string representation of this object. Useful in logging. string DebugString() const { return strings::StrCat("id: ", id.DebugString(), " manager_state: ", - static_cast(manager_state), " health: ", - health.ToString()); + ManagerStateString(manager_state), + " health: ", health.ToString()); } }; From a753696de940ea445673f09f65f38a2a65a46702 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 14 Feb 2018 15:39:39 -0800 Subject: [PATCH 0437/8554] Update API documentation Change: 185756268 --- tensorflow_serving/apis/inference.proto | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/apis/inference.proto b/tensorflow_serving/apis/inference.proto index b9bf59ea5a1..16e85cee5fb 100644 --- a/tensorflow_serving/apis/inference.proto +++ b/tensorflow_serving/apis/inference.proto @@ -23,6 +23,7 @@ package tensorflow.serving; message InferenceTask { // Model Specification. If version is not specified, will use the latest // (numerical) version. + // All ModelSpecs in a MultiInferenceRequest must access the same model name. ModelSpec model_spec = 1; // Signature's method_name. Should be one of the method names defined in From b1de20d0ea652c58dbcded9a837cd9960c6ae97b Mon Sep 17 00:00:00 2001 From: Michael Case Date: Thu, 22 Feb 2018 09:43:18 -0800 Subject: [PATCH 0438/8554] Remove redundant .bazelrc configs PiperOrigin-RevId: 186629739 --- tools/bazel.rc | 2 -- 1 file changed, 2 deletions(-) diff --git a/tools/bazel.rc b/tools/bazel.rc index 52a6241d651..0fee21a1ae9 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -7,8 +7,6 @@ build --python2_path=/usr/bin/python build --action_env PYTHON_BIN_PATH="/usr/bin/python" build --define PYTHON_BIN_PATH=/usr/bin/python -test --define PYTHON_BIN_PATH=/usr/bin/python -run --define PYTHON_BIN_PATH=/usr/bin/python build --spawn_strategy=standalone --genrule_strategy=standalone test --spawn_strategy=standalone --genrule_strategy=standalone From 7c5338972397eefccb640f1f727266a916584cfc Mon Sep 17 00:00:00 2001 From: Vamsi Sripathi Date: Fri, 23 Feb 2018 13:26:36 -0800 Subject: [PATCH 0439/8554] Enable TensorFlow to build with Intel MKL for optimal performance on CPU --- tools/bazel.rc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tools/bazel.rc b/tools/bazel.rc index 52a6241d651..137426c324b 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -18,3 +18,5 @@ build --define=grpc_no_ares=true # TODO(b/69809703): Remove once no longer required for TensorFlow to build. build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK + +build:mkl --define=using_mkl=true From 7b59d396dc061d726dc9d3204187fe4d8cc5fdc1 Mon Sep 17 00:00:00 2001 From: Gautam Vasudevan Date: Mon, 26 Feb 2018 19:46:55 -0800 Subject: [PATCH 0440/8554] Merge changes from github. PiperOrigin-RevId: 187120499 --- tensorflow_serving/batching/BUILD | 27 +++ .../batching/batching_session.cc | 67 +++++- .../batching/batching_session.h | 31 +++ .../batching/batching_session_test.cc | 86 ++++++++ tensorflow_serving/batching/batching_util.cc | 206 ++++++++++++++++++ tensorflow_serving/batching/batching_util.h | 66 ++++++ .../batching/batching_util_test.cc | 97 +++++++++ tensorflow_serving/batching/test_util/BUILD | 8 + .../matrix_half_plus_two_saved_model.py | 58 +++++ tensorflow_serving/batching/testdata/BUILD | 15 ++ .../matrix_half_plus_two/1/saved_model.pb | Bin 0 -> 856 bytes .../example/inception_saved_model.py | 12 +- .../example/mnist_saved_model.py | 22 +- tensorflow_serving/model_servers/main.cc | 48 +++- .../tensorflow_model_server_test.py | 19 ++ .../tensorflow/bundle_factory_util.cc | 3 + .../tensorflow/session_bundle_config.proto | 3 + 17 files changed, 745 insertions(+), 23 deletions(-) create mode 100644 tensorflow_serving/batching/batching_util.cc create mode 100644 tensorflow_serving/batching/batching_util.h create mode 100644 tensorflow_serving/batching/batching_util_test.cc create mode 100644 tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py create mode 100644 tensorflow_serving/batching/testdata/BUILD create mode 100644 tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb diff --git a/tensorflow_serving/batching/BUILD b/tensorflow_serving/batching/BUILD index 7dcceede9a7..73e7212e97b 100644 --- a/tensorflow_serving/batching/BUILD +++ b/tensorflow_serving/batching/BUILD @@ -54,9 +54,11 @@ cc_library( "//visibility:public", ], deps = [ + "//tensorflow_serving/batching:batching_util", "//tensorflow_serving/servables/tensorflow:serving_session", "//tensorflow_serving/util:cleanup", "//tensorflow_serving/util:hash", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/contrib/batching:basic_batch_scheduler", "@org_tensorflow//tensorflow/contrib/batching:batch_scheduler", "@org_tensorflow//tensorflow/core:core_cpu", @@ -72,6 +74,7 @@ cc_test( "batching_session_test.cc", ], data = [ + "//tensorflow_serving/batching/testdata:matrix_half_plus_two", "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", ], deps = [ @@ -114,3 +117,27 @@ cc_test( "@org_tensorflow//tensorflow/core:test", ], ) + +cc_library( + name = "batching_util", + srcs = ["batching_util.cc"], + hdrs = ["batching_util.h"], + deps = [ + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + ], +) + +cc_test( + name = "batching_util_test", + srcs = [ + "batching_util_test.cc", + ], + deps = [ + ":batching_util", + "//tensorflow_serving/core/test_util:test_main", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + ], +) diff --git a/tensorflow_serving/batching/batching_session.cc b/tensorflow_serving/batching/batching_session.cc index 9f5aa1aace3..275519ae304 100644 --- a/tensorflow_serving/batching/batching_session.cc +++ b/tensorflow_serving/batching/batching_session.cc @@ -26,9 +26,11 @@ limitations under the License. #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" +#include "tensorflow_serving/batching/batching_util.h" #include "tensorflow_serving/servables/tensorflow/serving_session.h" #include "tensorflow_serving/util/cleanup.h" #include "tensorflow_serving/util/hash.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -79,6 +81,34 @@ TensorSignature TensorSignatureFromRunArgs( return signature; } +// Constructs vector of all task inputs from Batch of BatchingSessionTasks. +// Input for each task is a vector of pairs (tensor_name, tensor_value). +std::vector>> GetTaskInputsVector( + const Batch& batch) { + std::vector>> all_task_inputs; + for (int i = 0; i < batch.num_tasks(); ++i) { + const std::vector>& task_inputs = + *batch.task(i).inputs; + all_task_inputs.push_back(task_inputs); + } + return all_task_inputs; +} +// Returns true iff all dims of shape1 are equal to dims of shape2 starting with +// the first (not zeroth) dimension. +// For example, for shapes [1, 2, 3] and [4, 2, 3] the result is true. +bool AreShapesEqualExceptZeroDim(const TensorShape& shape1, + const TensorShape& shape2) { + if (shape1.dims() != shape2.dims()) { + return false; + } + for (int i = 1; i < shape1.dims(); ++i) { + if (shape1.dim_size(i) != shape2.dim_size(i)) { + return false; + } + } + return true; +} + } // namespace TensorSignature TensorSignatureFromSignatureDef( @@ -288,7 +318,7 @@ BatchingSession::BatchingSession(const BatchingSessionOptions& options) Status BatchingSession::ComputeInputSize( const std::vector>& inputs, size_t* size) const { - if (inputs.size() == 0) { + if (inputs.empty()) { return errors::InvalidArgument( "Batching session Run() must have at least one input tensor"); } @@ -346,7 +376,14 @@ Status BatchingSession::MergeInputTensors( // For each input tensor name, a vector of tensors from the individual tasks. std::map> tensors_to_merge; - + // For each input tensor name a vector of maximum dimension sizes + // among tensors from individual tasks. + optional>> max_dim_sizes; + if (options_.pad_variable_length_inputs) { + std::vector>> all_task_inputs = + GetTaskInputsVector(batch); + max_dim_sizes = CalculateMaxDimSizes(all_task_inputs); + } // Populate 'tensors_to_merge'. for (int i = 0; i < batch.num_tasks(); ++i) { const std::vector>& task_inputs = @@ -356,8 +393,28 @@ Status BatchingSession::MergeInputTensors( const Tensor& tensor = entry.second; std::vector& tensor_vec = tensors_to_merge[tensor_name]; - tensor_vec.push_back(tensor); - + Tensor optionally_padded_tensor; + if (options_.pad_variable_length_inputs) { + TF_RETURN_IF_ERROR(AddPadding(tensor, (*max_dim_sizes)[tensor_name], + &optionally_padded_tensor)); + } else { + optionally_padded_tensor = tensor; + // Check whether tensors with the same name have equal dims + // (except zeroth dim) when padding is turned off. + if (i > 0) { // added at least one task to tensors_to_merge + TensorShape reference_shape = + tensors_to_merge[tensor_name][0].shape(); + if (!AreShapesEqualExceptZeroDim(tensor.shape(), reference_shape)) { + return errors::FailedPrecondition( + "Tensors with name '" + tensor_name + "' from different tasks" + + " have different shapes and padding is turned off." + + "Set pad_variable_length_inputs to true, or ensure that " + + "all tensors with the same name" + + "have equal dimensions starting with the first dim."); + } + } + } + tensor_vec.push_back(optionally_padded_tensor); if (i == batch.num_tasks() - 1 && padding_size > 0) { // This is the last task. Insert padding. // @@ -367,7 +424,7 @@ Status BatchingSession::MergeInputTensors( // // Slice() operates on the 0th dimension, which is the batch dimension. // It avoids a deep copy, which is a nice efficiency bonus. - const Tensor padding_tensor = tensor.Slice(0, 1); + const Tensor padding_tensor = optionally_padded_tensor.Slice(0, 1); for (int i = 0; i < padding_size; ++i) { tensor_vec.push_back(padding_tensor); } diff --git a/tensorflow_serving/batching/batching_session.h b/tensorflow_serving/batching/batching_session.h index 4d2e0ddf2d8..0ce8bf34939 100644 --- a/tensorflow_serving/batching/batching_session.h +++ b/tensorflow_serving/batching/batching_session.h @@ -99,6 +99,37 @@ struct BatchingSessionOptions { // // If left empty, no rounding/padding is performed. std::vector allowed_batch_sizes; + + // If set to true, padding is performed for tensors of the same name + // but with unequal dimensions (modulo zeroth dimension), so that + // all tensors of the same name have equal dim sizes. + // For each tensor its first element is used as padding value. + // + // For example: + // given input tensors of shapes [1, 500, 101], [2, 300, 101], [1, 400, 101] + // they will be padded to shapes [1, 500, 101], [2, 500, 101], [1, 500, 101]. + // Padding is not performed in zeroth dimension. + // + // Supported tensor datatypes: + // DT_FLOAT, DT_DOUBLE, DT_INT8, DT_UINT8, DT_INT16, + // DT_UINT16, DT_INT32, DT_INT64, DT_COMPLEX64, DT_COMPLEX128, + // DT_STRING, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, + // DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE. + // + // Supported ranks: from 1 to 6. + // + // This option is useful when using recurrent models(like LSTMs) with serving. + // These models typically accept variable-length inputs and when + // training them typical strategy is to save sequence lengths for decoding + // and pad those variable-length dims to maximum in batch. + // So, this option is used to achieve similar behavior + // when serving with batching, it is assumed that sequence lengths + // have already been saved. + // + // If tensors with the same name have different shapes + // (modulo zeroth dimension) and this option is set to false, + // then error Status will be returned. + bool pad_variable_length_inputs = false; }; // Wraps a session in a new session that automatically batches Run() calls. diff --git a/tensorflow_serving/batching/batching_session_test.cc b/tensorflow_serving/batching/batching_session_test.cc index a173aead2d8..afa2dbd85b3 100644 --- a/tensorflow_serving/batching/batching_session_test.cc +++ b/tensorflow_serving/batching/batching_session_test.cc @@ -92,6 +92,17 @@ std::unique_ptr CreateHalfPlusTwoSession() { return std::move(bundle.session); } +std::unique_ptr CreateMatrixHalfPlusTwoSession() { + tensorflow::SessionOptions session_options; + tensorflow::RunOptions run_options; + const string export_dir = + test_util::TestSrcDirPath("batching/testdata/matrix_half_plus_two/1"); + SavedModelBundle bundle; + TF_CHECK_OK(LoadSavedModel(session_options, run_options, export_dir, + {kSavedModelTagServe}, &bundle)); + return std::move(bundle.session); +} + // Test that a session handles a single request for the half-plus-two model // properly. The request has two input floats (size=2 for batching purposes). void TestSingleRequest(float input_0, float input_1, Session* session) { @@ -107,6 +118,18 @@ void TestSingleRequest(float input_0, float input_1, Session* session) { test::ExpectTensorEqual(expected_output, outputs[0]); } +void TestRequestToMatrixHalfPlusTwo(const std::vector& x_values, + TensorShape x_shape, + const std::vector& y_values, + TensorShape y_shape, Session* session) { + Tensor input = test::AsTensor(x_values, x_shape); + Tensor expected_output = test::AsTensor(y_values, y_shape); + std::vector output; + TF_ASSERT_OK(session->Run({{"x", input}}, {"y"}, {}, &output)); + ASSERT_EQ(1, output.size()); + test::ExpectTensorEqual(expected_output, output[0]); +} + // Invoke Run() with the supplied arguments, and expect a particular error. void ExpectError(const string& error_message, const std::vector>& inputs, @@ -181,6 +204,69 @@ TEST(BatchingSessionTest, Basic) { })); } +TEST(BatchingSessionTest, BatchingWithPadding) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 1; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + batching_session_options.pad_variable_length_inputs = true; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateMatrixHalfPlusTwoSession(), &batching_session)); + // two requests form a batch and first input gets padded with zeros to match + // [1, 3, 3] shape that is accepted by the model. + // if padding doesn't work, test will fail. + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", [&batching_session] { + TestRequestToMatrixHalfPlusTwo( + {1, 2, 3, 4}, {1, 2, 2}, {2.5, 3, 2.5, 3.5, 4, 2.5, 2.5, 2.5, 2.5}, + {1, 3, 3}, batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "second_request", [&batching_session] { + TestRequestToMatrixHalfPlusTwo({5, 6, 7, 8, 9, 10, 11, 12, 13}, + {1, 3, 3}, + {4.5, 5, 5.5, 6, 6.5, 7, 7.5, 8, 8.5}, + {1, 3, 3}, batching_session.get()); + })); +} + +TEST(BatchingSessionTest, UnequalTensorShapesWithPaddingTurnedOff) { + BasicBatchScheduler::Options schedule_options; + schedule_options.max_batch_size = 2; + schedule_options.batch_timeout_micros = 1e6; + schedule_options.num_batch_threads = 1; + std::unique_ptr batching_session; + BatchingSessionOptions batching_session_options; + batching_session_options.pad_variable_length_inputs = false; + TF_ASSERT_OK(CreateBasicBatchingSession( + schedule_options, batching_session_options, {{"x"}, {"y"}}, + CreateMatrixHalfPlusTwoSession(), &batching_session)); + string expected_error_msg = + "Tensors with name 'x' from different tasks" + " have different shapes and padding is turned off." + "Set pad_variable_length_inputs to true, or ensure that " + "all tensors with the same name" + "have equal dimensions starting with the first dim."; + std::unique_ptr first_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", + [&batching_session, &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor({1, 2, 3, 4}, {1, 2, 2})}}, + {"y"}, batching_session.get()); + })); + std::unique_ptr second_request_thread(Env::Default()->StartThread( + ThreadOptions(), "first_request", + [&batching_session, &expected_error_msg] { + ExpectError(expected_error_msg, + {{"x", test::AsTensor( + {5, 6, 7, 8, 9, 10, 11, 12, 13}, {1, 3, 3})}}, + {"y"}, batching_session.get()); + })); +} + TEST(BatchingSessionTest, SingletonBatch) { BasicBatchScheduler::Options schedule_options; schedule_options.max_batch_size = 4; // fits two 2-unit tasks diff --git a/tensorflow_serving/batching/batching_util.cc b/tensorflow_serving/batching/batching_util.cc new file mode 100644 index 00000000000..4daf1193b45 --- /dev/null +++ b/tensorflow_serving/batching/batching_util.cc @@ -0,0 +1,206 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/batching_util.h" + +#include + +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/framework/types.h" +#include "tensorflow/core/lib/core/errors.h" + +namespace tensorflow { +namespace serving { + +// Padding for one dimension of a tensor. +// pad_before is a number of values to add before elements in one dimension, +// pad_after is number of objects to add after. +// NOTE: fields are named this way because of Eigen::TensorMap::pad method. +// It requires padding to be an array of elements that have fields +// "first" and "second". +struct OneDimPadding { + int64 first; // pad before + int64 second; // pad after +}; + +// Constructs array of paddings, where: +// paddings[i].first - number of objects to add before elements in dimension i +// of given tensor, +// paddings[i].second - number of objects to add after elements in dimension i. +// This padding signature is used when performing internal padding with Eigen. +// +// When building paddings it is assumed that tensor needs to be padded +// after each dimension, so its shape matches max_dim_sizes, +// First entry of max_dim_sizes, which is maximum size of zeroth dimension, +// is ignored, because we don't perform padding in that dimension. +template +Eigen::array CreatePadding( + Tensor tensor, const std::vector& max_dim_sizes) { + Eigen::array padding; + for (unsigned int i = 0; i < max_dim_sizes.size(); ++i) { + if (i > 0 && max_dim_sizes[i] - tensor.dim_size(i) > 0) { + padding[i] = {0, max_dim_sizes[i] - tensor.dim_size(i)}; + } else { + padding[i] = {0, 0}; + } + } + return padding; +} + +// Functor, which performs padding of given input tensor +// using specified padding signature. +// For example, given tensor of shape [1, 2, 3] and padding signature +// [[0, 0], [0, 2], [2, 2]] +// functor produces padded_tensor of shape [1, 4, 7]. +template +struct PadTensor { + Status operator()(Tensor input, + const Eigen::array& padding, + Tensor* output) { + TensorShape output_shape; + for (int d = 0; d < num_dims; ++d) { + // Pad before existing elements. + const int32 before_d = padding[d].first; + // Pad after existing elements. + const int32 after_d = padding[d].second; + output_shape.AddDim(before_d + input.dim_size(d) + after_d); + } + if (output_shape.num_elements() == input.NumElements()) { + bool result = output->CopyFrom(input, output_shape); + if (!result) { + return errors::Internal("Couldn't create output."); + } + return Status::OK(); + } + if (input.NumElements() < 1) { + return errors::InvalidArgument( + "Got empty tensor in batch of non-empty tensors."); + } + *output = Tensor(input.dtype(), output_shape); + typename TTypes::Tensor inputs = input.tensor(); + T pad_value(input.flat()(0)); // using existing values in padding + output->tensor() = inputs.pad(padding, pad_value); + return Status::OK(); + } +}; + +// Invokes padding procedure for specific tensor ranks. +// Only ranks from 1 to 6 are supported (like in PadOp). +template +Status PadTensorOfSpecificType(const Tensor& tensor, + const std::vector& max_dim_sizes, + Tensor* output_tensor) { + int num_dims = tensor.dims(); + switch (num_dims) { + case 1: { + Eigen::array padding; + padding = CreatePadding<1>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 2: { + Eigen::array padding; + padding = CreatePadding<2>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 3: { + Eigen::array padding; + padding = CreatePadding<3>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 4: { + Eigen::array padding; + padding = CreatePadding<4>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 5: { + Eigen::array padding; + padding = CreatePadding<5>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + case 6: { + Eigen::array padding; + padding = CreatePadding<6>(tensor, max_dim_sizes); + PadTensor padding_functor = PadTensor(); + return padding_functor(tensor, padding, output_tensor); + } + default: + // only ranks from 1 to 6 are supported + // (like in tensorflow/core/kernels/pad_op.cc) + return errors::InvalidArgument( + "Only tensors with rank from 1 to 6 can be padded."); + } +} + +std::map> CalculateMaxDimSizes( + const std::vector>>& batch) { + std::map> max_dim_sizes; + // Populate 'max_dim_sizes' + // init + const std::vector>& task_inputs = batch[0]; + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; + max_dim_sizes[tensor_name] = std::vector(tensor.dims(), 0); + } + // fill + for (int i = 0; i < batch.size(); ++i) { + const std::vector>& task_inputs = batch[i]; + for (const auto& entry : task_inputs) { + const string& tensor_name = entry.first; + const Tensor& tensor = entry.second; + + std::vector& max_dim_sizes_for_one_input = + max_dim_sizes[tensor_name]; + for (int j = 0; j < tensor.dims(); ++j) { + const int old_max_size = max_dim_sizes_for_one_input[j]; + if (tensor.shape().dim_size(j) > old_max_size) { + max_dim_sizes_for_one_input[j] = tensor.shape().dim_size(j); + } + } + } + } + return max_dim_sizes; +} + +Status AddPadding(const Tensor& tensor, const std::vector& max_dim_sizes, + Tensor* padded_tensor) { + const DataType input_dtype = tensor.dtype(); + Status padding_status; +#define CASE(type) \ + case DataTypeToEnum::value: { \ + padding_status = \ + PadTensorOfSpecificType(tensor, max_dim_sizes, padded_tensor); \ + break; \ + } + switch (input_dtype) { + TF_CALL_ALL_TYPES(CASE); + TF_CALL_QUANTIZED_TYPES(CASE); + // quantized types macro doesn't include these types + TF_CALL_quint16(CASE); + TF_CALL_qint16(CASE); + default: + padding_status = errors::InvalidArgument("Unsupported type"); + } +#undef CASE + return padding_status; +} +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/batching_util.h b/tensorflow_serving/batching/batching_util.h new file mode 100644 index 00000000000..c5ee217685d --- /dev/null +++ b/tensorflow_serving/batching/batching_util.h @@ -0,0 +1,66 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ +#define TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ + +#include +#include +#include + +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" + +namespace tensorflow { +namespace serving { + +// For batch of inputs calculates maximum dim sizes across all tensors +// with the same name. +// These dim sizes are used later to calculate padding amount for each tensor. +// For example, for batch containing three tasks with the following inputs +// (instead of tensors there are their shapes): +// +// task1: {'tensor_a': [100, 500, 300], 'tensor_b': [100]} +// task2: {'tensor_a': [100, 200, 123], 'tensor_b': [100]} +// task3: {'tensor_a': [200, 100, 400], 'tensor_b': [200]} +// +// the following map will be generated: +// {'tensor_a': [200, 500, 400], 'tensor_b': [200]} +std::map> CalculateMaxDimSizes( + const std::vector>>& batch); + +// Pads tensor so that its shape becomes as specified in max_dim_sizes, +// except for zeroth dimension, which is left as is. +// First entry in max_dim_sizes is ignored. +// First element of a tensor is used as padding value. +// If tensor is empty, an error will be returned. +// +// For example given input tensor with shape [1, 2, 3, 4] and max_dim_sizes +// [1, 2, 5, 8] function produces padded_tensor of shape +// [1, 2, 5, 8], padded with tensor[0][0][0][0] element. +// +// Supported tensor datatypes: +// DT_FLOAT, DT_DOUBLE, DT_INT8, DT_UINT8, DT_INT16, +// DT_UINT16, DT_INT32, DT_INT64, DT_COMPLEX64, DT_COMPLEX128, +// DT_STRING, DT_BOOL, DT_QINT8, DT_QUINT8, DT_QINT16, +// DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE. +// +// Supported tensor ranks: from 1 to 6. + +Status AddPadding(const Tensor& tensor, const std::vector& max_dim_sizes, + Tensor* padded_tensor); +} // namespace serving +} // namespace tensorflow +#endif // TENSORFLOW_SERVING_BATCHING_BATCHING_UTIL_H_ diff --git a/tensorflow_serving/batching/batching_util_test.cc b/tensorflow_serving/batching/batching_util_test.cc new file mode 100644 index 00000000000..1bac10e9aae --- /dev/null +++ b/tensorflow_serving/batching/batching_util_test.cc @@ -0,0 +1,97 @@ +/* Copyright 2017 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/batching/batching_util.h" + +#include +#include +#include "tensorflow/core/framework/register_types.h" +#include "tensorflow/core/framework/tensor.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/types.h" + +namespace tensorflow { +namespace serving { +namespace { + +using ::testing::ElementsAre; +using ::testing::Pair; +using ::testing::UnorderedElementsAre; + +// Creates vector of pairs (tensor_name, tensor_value), where tensors +// have shapes as specified in shapes. +// Tensor with shape shapes[i] has tensor_name "x" + std::to_string(i). +std::vector> CreateInputsWithTensorShapes( + const std::vector& shapes) { + std::vector> inputs; + for (int i = 0; i < shapes.size(); ++i) { + inputs.push_back({"x" + std::to_string(i), Tensor(DT_FLOAT, shapes[i])}); + } + return inputs; +} + +TEST(BatchingUtilTest, CalculateMaxDimSizes) { + const std::vector shapes1{{10, 20, 30}, {10, 100}}; + std::vector> inputs1 = + CreateInputsWithTensorShapes(shapes1); + const std::vector shapes2{{20, 50, 15}, {20, 101}}; + std::vector> inputs2 = + CreateInputsWithTensorShapes(shapes2); + std::vector>> batch{inputs1, inputs2}; + std::map> max_dim_sizes = + CalculateMaxDimSizes(batch); + EXPECT_THAT(max_dim_sizes, + UnorderedElementsAre(Pair("x0", ElementsAre(20, 50, 30)), + Pair("x1", ElementsAre(20, 101)))); +} + +TEST(BatchingUtilTest, AddPadding) { + const std::vector max_dim_sizes{20, 100, 200}; + const std::vector types{ + DT_FLOAT, DT_DOUBLE, DT_INT32, DT_UINT8, DT_INT16, + DT_UINT16, DT_INT8, DT_STRING, DT_BOOL, DT_COMPLEX64, + DT_COMPLEX128, DT_INT64, DT_QINT8, DT_QUINT8, DT_QINT16, + DT_QUINT16, DT_QINT32, DT_HALF, DT_RESOURCE}; + Status padding_status; + for (DataType type : types) { + Tensor tensor(type, {10, 20, 30}); +#define INIT_TYPE(T) \ + if (type == DataTypeToEnum::value) { \ + tensor.flat().setConstant(T()); \ + } + TF_CALL_ALL_TYPES(INIT_TYPE); + TF_CALL_QUANTIZED_TYPES(INIT_TYPE); + // quantized types macro doesn't include these types + TF_CALL_quint16(INIT_TYPE); + TF_CALL_qint16(INIT_TYPE); +#undef INIT_TYPE + Tensor padded_tensor; + padding_status = AddPadding(tensor, max_dim_sizes, &padded_tensor); + ASSERT_EQ(Status::OK(), padding_status); + EXPECT_EQ(TensorShape({10, 100, 200}), padded_tensor.shape()); + } +} + +TEST(BatchingUtilTest, AddPaddingTensorWithUnsupportedRank) { + const std::vector max_dim_sizes{1, 1, 1, 1, 1, 1, 1}; + const Tensor tensor(DT_FLOAT, {1, 1, 1, 1, 1, 1, 1}); + Tensor padded_tensor; + ASSERT_EQ(errors::InvalidArgument( + "Only tensors with rank from 1 to 6 can be padded."), + AddPadding(tensor, max_dim_sizes, &padded_tensor)); +} +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/batching/test_util/BUILD b/tensorflow_serving/batching/test_util/BUILD index 9bf13315bd6..5673ad23d06 100644 --- a/tensorflow_serving/batching/test_util/BUILD +++ b/tensorflow_serving/batching/test_util/BUILD @@ -41,3 +41,11 @@ cc_test( "@org_tensorflow//tensorflow/core:test", ], ) + +# script that generates saved_model for matrix_half_plus_two model. +py_binary( + name = "matrix_half_plus_two_saved_model", + srcs = ["matrix_half_plus_two_saved_model.py"], + srcs_version = "PY2AND3", + deps = ["@org_tensorflow//tensorflow:tensorflow_py"], +) diff --git a/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py b/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py new file mode 100644 index 00000000000..bbc8a891ffe --- /dev/null +++ b/tensorflow_serving/batching/test_util/matrix_half_plus_two_saved_model.py @@ -0,0 +1,58 @@ +# Copyright 2017 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +import argparse +import tensorflow as tf + + +def _generate_saved_model_for_matrix_half_plus_two(export_dir): + """Creates SavedModel for half plus two model that accepts batches of + 3*3 matrices. + The model divides all elements in each matrix by 2 and adds 2 to them. + So, for one input matrix [[1, 2, 3], [4, 5, 6], [7, 8, 9]] + the result will be [[2.5, 3, 3.5], [4, 4.5, 5], [5.5, 6, 6.5]]. + Args: + export_dir: The directory where to write SavedModel files. + """ + builder = tf.saved_model.builder.SavedModelBuilder(export_dir) + with tf.Session() as session: + x = tf.placeholder(tf.float32, shape=[None, 3, 3], name="x") + a = tf.constant(0.5) + b = tf.constant(2.0) + y = tf.add(tf.multiply(a, x), b, name="y") + predict_signature_def = ( + tf.saved_model.signature_def_utils.predict_signature_def({ + "x": x + }, {"y": y})) + signature_def_map = { + tf.saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY: + predict_signature_def + } + session.run(tf.global_variables_initializer()) + builder.add_meta_graph_and_variables( + session, [tf.saved_model.tag_constants.SERVING], + signature_def_map=signature_def_map) + builder.save() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument( + "--output_dir", + type=str, + default="/tmp/matrix_half_plus_two/1", + help="The directory where to write SavedModel files.") + args = parser.parse_args() + _generate_saved_model_for_matrix_half_plus_two(args.output_dir) diff --git a/tensorflow_serving/batching/testdata/BUILD b/tensorflow_serving/batching/testdata/BUILD new file mode 100644 index 00000000000..197dd6f2fab --- /dev/null +++ b/tensorflow_serving/batching/testdata/BUILD @@ -0,0 +1,15 @@ +# Description: Tensorflow Serving batching test data. + +package( + default_visibility = ["//tensorflow_serving:internal"], + features = ["layering_check"], +) + +licenses(["notice"]) # Apache 2.0 + +filegroup( + name = "matrix_half_plus_two", + srcs = glob( + ["matrix_half_plus_two/**/*"], + ), +) diff --git a/tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb b/tensorflow_serving/batching/testdata/matrix_half_plus_two/1/saved_model.pb new file mode 100644 index 0000000000000000000000000000000000000000..8651e45a0443bb54b19e70518515820a3cc4f089 GIT binary patch literal 856 zcmbVK%TB^j6zuJbazN3eDBguJbyp*=l{O~6b_5>__k_@k5}UT^1E?!M!7uPD{1ZP& zds~`F5TiC}CTS-#_s)z)@Lh#h5uZgk2I<_gPz5s8$sNZyL7L!Gf;xrdqa4N_Gb^yt zE^QD|Ru!qJVF8XnxpbW%q!m!zDD~wObSjwx{B<- znXadYJbh4}Fhe22?i18qt-2`D11jTlIFm zdFT(??Ly={I_|{5Hwo+{7X0dBl=7R1O=9fjlG|n#OWH^N5EM0uzs3Qq_Ofhxsd<1q zu#60yS)iEZI`37@Gc8|$Mo=(hK@d)jtpx)aL{oq+6w-2ft>3GiEmiAeh4I$ttaLIp z3Va+qhdATWspbCD8zIUx~SER7%IHek#pa(*svejAr4`m7r6`4 i)NHO1`h8iLkYY}Xe;`qsO?qg%lg7-a;rYxO4Z=^a-s7_X literal 0 HcmV?d00001 diff --git a/tensorflow_serving/example/inception_saved_model.py b/tensorflow_serving/example/inception_saved_model.py index 86b20c460d0..5224d368baf 100644 --- a/tensorflow_serving/example/inception_saved_model.py +++ b/tensorflow_serving/example/inception_saved_model.py @@ -20,6 +20,8 @@ standard tensorflow_model_server. """ +from __future__ import print_function + import os.path # This is a placeholder for a Google-internal import. @@ -105,17 +107,17 @@ def export(): # /my-favorite-path/imagenet_train/model.ckpt-0, # extract global_step from it. global_step = ckpt.model_checkpoint_path.split('/')[-1].split('-')[-1] - print 'Successfully loaded model from %s at step=%s.' % ( - ckpt.model_checkpoint_path, global_step) + print('Successfully loaded model from %s at step=%s.' % + (ckpt.model_checkpoint_path, global_step)) else: - print 'No checkpoint file found at %s' % FLAGS.checkpoint_dir + print('No checkpoint file found at %s' % FLAGS.checkpoint_dir) return # Export inference model. output_path = os.path.join( tf.compat.as_bytes(FLAGS.output_dir), tf.compat.as_bytes(str(FLAGS.model_version))) - print 'Exporting trained model to', output_path + print('Exporting trained model to', output_path) builder = tf.saved_model.builder.SavedModelBuilder(output_path) # Build the signature_def_map. @@ -165,7 +167,7 @@ def export(): legacy_init_op=legacy_init_op) builder.save() - print 'Successfully exported model to %s' % FLAGS.output_dir + print('Successfully exported model to %s' % FLAGS.output_dir) def preprocess_image(image_buffer): diff --git a/tensorflow_serving/example/mnist_saved_model.py b/tensorflow_serving/example/mnist_saved_model.py index 196cf9af69c..579410e14c1 100644 --- a/tensorflow_serving/example/mnist_saved_model.py +++ b/tensorflow_serving/example/mnist_saved_model.py @@ -25,6 +25,8 @@ export_dir """ +from __future__ import print_function + import os import sys @@ -47,14 +49,14 @@ def main(_): '[--model_version=y] export_dir') sys.exit(-1) if FLAGS.training_iteration <= 0: - print 'Please specify a positive value for training iteration.' + print('Please specify a positive value for training iteration.') sys.exit(-1) if FLAGS.model_version <= 0: - print 'Please specify a positive value for version number.' + print('Please specify a positive value for version number.') sys.exit(-1) # Train model - print 'Training model...' + print('Training model...') mnist = mnist_input_data.read_data_sets(FLAGS.work_dir, one_hot=True) sess = tf.InteractiveSession() serialized_tf_example = tf.placeholder(tf.string, name='tf_example') @@ -77,10 +79,12 @@ def main(_): train_step.run(feed_dict={x: batch[0], y_: batch[1]}) correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1)) accuracy = tf.reduce_mean(tf.cast(correct_prediction, 'float')) - print 'training accuracy %g' % sess.run( - accuracy, feed_dict={x: mnist.test.images, - y_: mnist.test.labels}) - print 'Done training!' + print('training accuracy %g' % sess.run( + accuracy, feed_dict={ + x: mnist.test.images, + y_: mnist.test.labels + })) + print('Done training!') # Export model # WARNING(break-tutorial-inline-code): The following code snippet is @@ -90,7 +94,7 @@ def main(_): export_path = os.path.join( tf.compat.as_bytes(export_path_base), tf.compat.as_bytes(str(FLAGS.model_version))) - print 'Exporting trained model to', export_path + print('Exporting trained model to', export_path) builder = tf.saved_model.builder.SavedModelBuilder(export_path) # Build the signature_def_map. @@ -136,7 +140,7 @@ def main(_): builder.save() - print 'Done exporting!' + print('Done exporting!') if __name__ == '__main__': diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index ef1573aa1fc..f04248d11a0 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -58,6 +58,7 @@ limitations under the License. #include "grpc/grpc.h" #include "tensorflow/cc/saved_model/tag_constants.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/init_main.h" @@ -263,8 +264,29 @@ class PredictionServiceImpl final : public PredictionService::Service { bool use_saved_model_; }; -void RunServer(int port, std::unique_ptr core, - bool use_saved_model) { +// gRPC Channel Arguments to be passed from command line to gRPC ServerBuilder. +struct GrpcChannelArgument { + string key; + string value; +}; + +// Parses a comma separated list of gRPC channel arguments into list of +// ChannelArgument. +std::vector parseGrpcChannelArgs( + const string& channel_arguments_str) { + const std::vector channel_arguments = + tensorflow::str_util::Split(channel_arguments_str, ","); + std::vector result; + for (const string& channel_argument : channel_arguments) { + const std::vector key_val = + tensorflow::str_util::Split(channel_argument, "="); + result.push_back({key_val[0], key_val[1]}); + } + return result; +} + +void RunServer(int port, std::unique_ptr core, bool use_saved_model, + const string& grpc_channel_arguments) { // "0.0.0.0" is the way to listen on localhost in gRPC. const string server_address = "0.0.0.0:" + std::to_string(port); tensorflow::serving::ModelServiceImpl model_service(core.get()); @@ -275,6 +297,19 @@ void RunServer(int port, std::unique_ptr core, builder.RegisterService(&model_service); builder.RegisterService(&prediction_service); builder.SetMaxMessageSize(tensorflow::kint32max); + const std::vector channel_arguments = + parseGrpcChannelArgs(grpc_channel_arguments); + for (GrpcChannelArgument channel_argument : channel_arguments) { + // gRPC accept arguments of two types, int and string. We will attempt to + // parse each arg as int and pass it on as such if successful. Otherwise we + // will pass it as a string. gRPC will log arguments that were not accepted. + int value; + if (tensorflow::strings::safe_strto32(channel_argument.key, &value)) { + builder.AddChannelArgument(channel_argument.key, value); + } else { + builder.AddChannelArgument(channel_argument.key, channel_argument.value); + } + } std::unique_ptr server(builder.BuildAndStart()); LOG(INFO) << "Running ModelServer at " << server_address << " ..."; server->Wait(); @@ -306,6 +341,7 @@ int main(int argc, char** argv) { tensorflow::int64 tensorflow_session_parallelism = 0; string platform_config_file = ""; string model_config_file; + string grpc_channel_arguments = ""; std::vector flag_list = { tensorflow::Flag("port", &port, "port to listen on"), tensorflow::Flag("enable_batching", &enable_batching, "enable batching"), @@ -357,7 +393,11 @@ int main(int argc, char** argv) { "starts, If 0.0, Tensorflow will automatically select a value."), tensorflow::Flag("saved_model_tags", &saved_model_tags, "Comma-separated set of tags corresponding to the meta " - "graph def to load from SavedModel.")}; + "graph def to load from SavedModel."), + tensorflow::Flag("grpc_channel_arguments", &grpc_channel_arguments, + "A comma separated list of arguments to be passed to " + "the grpc server. (e.g. " + "grpc.max_connection_age_ms=2000)")}; string usage = tensorflow::Flags::Usage(argv[0], flag_list); const bool parse_result = tensorflow::Flags::Parse(&argc, argv, flag_list); @@ -429,7 +469,7 @@ int main(int argc, char** argv) { std::unique_ptr core; TF_CHECK_OK(ServerCore::Create(std::move(options), &core)); - RunServer(port, std::move(core), use_saved_model); + RunServer(port, std::move(core), use_saved_model, grpc_channel_arguments); return 0; } diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 96e1de1211e..8634881fd78 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -122,6 +122,7 @@ def RunServer(self, model_name, model_path, batching_parameters_file='', + grpc_channel_arguments='', wait_for_server_ready=True): """Run tensorflow_model_server using test config.""" print 'Starting test server...' @@ -132,6 +133,8 @@ def RunServer(self, if batching_parameters_file: command += ' --enable_batching' command += ' --batching_parameters_file=' + batching_parameters_file + if grpc_channel_arguments: + command += ' --grpc_channel_arguments=' + grpc_channel_arguments print command self.server_proc = subprocess.Popen(shlex.split(command)) print 'Server started' @@ -481,6 +484,22 @@ def testBadModelConfig(self): self.assertNotEqual(self.server_proc.stderr, None) self.assertGreater(self.server_proc.stderr.read().find(error_message), -1) + def testGoodGrpcChannelArgs(self): + """Test server starts with grpc_channel_arguments specified.""" + atexit.register(self.TerminateProcs) + model_server_address = self.RunServer( + PickUnusedPort(), + 'default', + self._GetSavedModelBundlePath(), + grpc_channel_arguments= + 'grpc.max_connection_age_ms=2000,grpc.lb_policy_name=grpclb') + self.VerifyPredictRequest( + model_server_address, + expected_output=3.0, + specify_output=False, + expected_version=self._GetModelVersion( + self._GetSavedModelHalfPlusThreePath())) + if __name__ == '__main__': tf.test.main() diff --git a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc index 46f0b8e192e..82dd9701a1c 100644 --- a/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc +++ b/tensorflow_serving/servables/tensorflow/bundle_factory_util.cc @@ -177,6 +177,9 @@ Status WrapSessionForBatching(const BatchingParameters& batching_config, batching_session_options.allowed_batch_sizes.push_back(allowed_batch_size); } + batching_session_options.pad_variable_length_inputs = + batching_config.pad_variable_length_inputs(); + auto create_queue = [batch_scheduler, queue_options]( std::function>)> process_batch_callback, diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto index 0114c6f7a27..8f6cf515b8b 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto +++ b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto @@ -100,4 +100,7 @@ message BatchingParameters { // - The entries must be in increasing order. // - The final entry must equal 'max_batch_size'. repeated int64 allowed_batch_sizes = 6; + + // Whether to pad variable-length inputs when a batch is formed. + bool pad_variable_length_inputs = 7; } From 79d3354cd8a408fb981670512d661c2b7e89f5bf Mon Sep 17 00:00:00 2001 From: Oleksii Date: Wed, 7 Mar 2018 07:33:13 +0200 Subject: [PATCH 0441/8554] Runtime model config reload gRPC API (#774) * Added model config reload gRPC API * Update model_service_impl.cc * Update model_service.proto --- tensorflow_serving/apis/BUILD | 13 ++++ .../apis/model_management.proto | 15 +++++ tensorflow_serving/apis/model_service.proto | 9 ++- .../model_servers/model_service_impl.cc | 62 ++++++++++++++----- .../model_servers/model_service_impl.h | 27 ++++---- 5 files changed, 99 insertions(+), 27 deletions(-) create mode 100644 tensorflow_serving/apis/model_management.proto diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index aed87f2ad14..f2682fd1c4f 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -156,6 +156,18 @@ serving_go_grpc_library( deps = [":prediction_service_go_proto"], ) +serving_proto_library( + name = "model_management_proto", + srcs = ["model_management.proto"], + cc_api_version = 2, + go_api_version = 2, + java_api_version = 2, + deps = [ + "//tensorflow_serving/config:model_server_config_proto", + "//tensorflow_serving/util:status_proto", + ], +) + serving_proto_library( name = "get_model_status_proto", srcs = ["get_model_status.proto"], @@ -188,6 +200,7 @@ serving_proto_library( java_api_version = 2, deps = [ ":get_model_status_proto", + ":model_management_proto", ], ) diff --git a/tensorflow_serving/apis/model_management.proto b/tensorflow_serving/apis/model_management.proto new file mode 100644 index 00000000000..d41885374b5 --- /dev/null +++ b/tensorflow_serving/apis/model_management.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +import "tensorflow_serving/config/model_server_config.proto"; +import "tensorflow_serving/util/status.proto"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +message ReloadConfigRequest { + ModelServerConfig config = 1; +} + +message ReloadConfigResponse { + StatusProto status = 1; +} \ No newline at end of file diff --git a/tensorflow_serving/apis/model_service.proto b/tensorflow_serving/apis/model_service.proto index f3f0b8c8833..7b9dbaefb81 100644 --- a/tensorflow_serving/apis/model_service.proto +++ b/tensorflow_serving/apis/model_service.proto @@ -3,15 +3,20 @@ syntax = "proto3"; option cc_enable_arenas = true; import "tensorflow_serving/apis/get_model_status.proto"; +import "tensorflow_serving/apis/model_management.proto"; package tensorflow.serving; -// ModelService provides access to information about model versions -// that have been handled by the model server. +// ModelService provides methods to query and update the state of the server, e.g. +// which models/versions are being served. service ModelService { // Gets status of model. If the ModelSpec in the request does not specify // version, information about all versions of the model will be returned. If // the ModelSpec in the request does specify a version, the status of only // that version will be returned. rpc GetModelStatus(GetModelStatusRequest) returns (GetModelStatusResponse); + + // Reloads the set of served models. The new config supersedes the old one, so if a + // model is omitted from the new config it will be unloaded and no longer served. + rpc HandleReloadConfigRequest(ReloadConfigRequest) returns (ReloadConfigResponse); } diff --git a/tensorflow_serving/model_servers/model_service_impl.cc b/tensorflow_serving/model_servers/model_service_impl.cc index b239ba71bb2..eda3cd01fd6 100644 --- a/tensorflow_serving/model_servers/model_service_impl.cc +++ b/tensorflow_serving/model_servers/model_service_impl.cc @@ -17,20 +17,54 @@ limitations under the License. #include "tensorflow_serving/model_servers/get_model_status_impl.h" #include "tensorflow_serving/model_servers/grpc_status_util.h" +#include "tensorflow_serving/util/status_util.h" namespace tensorflow { -namespace serving { - -::grpc::Status ModelServiceImpl::GetModelStatus( - ::grpc::ServerContext* context, const GetModelStatusRequest* request, - GetModelStatusResponse* response) { - const ::grpc::Status status = tensorflow::serving::ToGRPCStatus( - GetModelStatusImpl::GetModelStatus(core_, *request, response)); - if (!status.ok()) { - VLOG(1) << "GetModelStatus failed: " << status.error_message(); - } - return status; -} - -} // namespace serving + namespace serving { + + ::grpc::Status ModelServiceImpl::GetModelStatus(::grpc::ServerContext *context, + const GetModelStatusRequest *request, + GetModelStatusResponse *response) { + const ::grpc::Status status = tensorflow::serving::ToGRPCStatus( + GetModelStatusImpl::GetModelStatus(core_, *request, response)); + if (!status.ok()) { + VLOG(1) << "GetModelStatus failed: " << status.error_message(); + } + return status; + } + + ::grpc::Status ModelServiceImpl::HandleReloadConfigRequest(::grpc::ServerContext *context, + const ReloadConfigRequest *request, + ReloadConfigResponse *response) { + ModelServerConfig server_config = request->config(); + Status status; + switch (server_config.config_case()) { + case ModelServerConfig::kModelConfigList: { + const ModelConfigList list = server_config.model_config_list(); + + for (int index = 0; index < list.config_size(); index++) { + const ModelConfig config = list.config(index); + LOG(INFO) << "\nConfig entry" + << "\n\tindex : " << index + << "\n\tpath : " << config.base_path() + << "\n\tname : " << config.name() + << "\n\tplatform : " << config.model_platform(); + } + status = core_->ReloadConfig(server_config); + break; + } + default: + status = errors::InvalidArgument("ServerModelConfig type not supported by HandleReloadConfigRequest. Only ModelConfigList is currently supported"); + } + + if (!status.ok()) { + LOG(ERROR) << "ReloadConfig failed: " << status.error_message(); + } + + const StatusProto status_proto = ToStatusProto(status); + *response->mutable_status() = status_proto; + return ToGRPCStatus(status); + } + + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/model_servers/model_service_impl.h b/tensorflow_serving/model_servers/model_service_impl.h index 0c4b00cf887..29b972ee33d 100644 --- a/tensorflow_serving/model_servers/model_service_impl.h +++ b/tensorflow_serving/model_servers/model_service_impl.h @@ -21,23 +21,28 @@ limitations under the License. #include "tensorflow_serving/apis/model_service.grpc.pb.h" #include "tensorflow_serving/apis/model_service.pb.h" #include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/apis/model_management.pb.h" namespace tensorflow { -namespace serving { + namespace serving { -class ModelServiceImpl final : public ModelService::Service { - public: - explicit ModelServiceImpl(ServerCore* core) : core_(core) {} + class ModelServiceImpl final : public ModelService::Service { + public: + explicit ModelServiceImpl(ServerCore *core) : core_(core) {} - ::grpc::Status GetModelStatus(::grpc::ServerContext* context, - const GetModelStatusRequest* request, - GetModelStatusResponse* response) override; + ::grpc::Status GetModelStatus(::grpc::ServerContext *context, + const GetModelStatusRequest *request, + GetModelStatusResponse *response) override; - private: - ServerCore* core_; -}; + ::grpc::Status HandleReloadConfigRequest(::grpc::ServerContext *context, + const ReloadConfigRequest *request, + ReloadConfigResponse *response); -} // namespace serving + private: + ServerCore *core_; + }; + + } // namespace serving } // namespace tensorflow #endif // TENSORFLOW_SERVING_MODEL_SERVERS_MODEL_SERVICE_IMPL_H_ From 2b8334dfec8c7b1481fad4b9292411cab71d1a56 Mon Sep 17 00:00:00 2001 From: Gautam Vasudevan Date: Thu, 1 Mar 2018 16:50:45 -0800 Subject: [PATCH 0442/8554] Merge changes from github. PiperOrigin-RevId: 187555302 --- tensorflow_serving/batching/testdata/BUILD | 2 +- tensorflow_serving/tools/pip_package/build_pip_package.sh | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/batching/testdata/BUILD b/tensorflow_serving/batching/testdata/BUILD index 197dd6f2fab..a33d0e732da 100644 --- a/tensorflow_serving/batching/testdata/BUILD +++ b/tensorflow_serving/batching/testdata/BUILD @@ -2,7 +2,7 @@ package( default_visibility = ["//tensorflow_serving:internal"], - features = ["layering_check"], + features = ["-layering_check"], ) licenses(["notice"]) # Apache 2.0 diff --git a/tensorflow_serving/tools/pip_package/build_pip_package.sh b/tensorflow_serving/tools/pip_package/build_pip_package.sh index 4a5f929854c..90b904176d4 100755 --- a/tensorflow_serving/tools/pip_package/build_pip_package.sh +++ b/tensorflow_serving/tools/pip_package/build_pip_package.sh @@ -39,7 +39,10 @@ function main() { cp bazel-genfiles/tensorflow_serving/apis/*_pb2.py \ "${TMPDIR}/tensorflow_serving/apis" - cp bazel-serving/tensorflow_serving/apis/prediction_service_pb2.py \ + cp bazel-serving/tensorflow_serving/apis/*_pb2.py \ + "${TMPDIR}/tensorflow_serving/apis" + + cp bazel-serving/tensorflow_serving/apis/*_grpc.py \ "${TMPDIR}/tensorflow_serving/apis" touch "${TMPDIR}/tensorflow_serving/apis/__init__.py" From 376850b4304a28a997a33e01e1a32b27b87e2d38 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 7 Mar 2018 18:30:38 -0800 Subject: [PATCH 0443/8554] Merged commit includes the following changes: 188267957 by A. Unique TensorFlower: Internal change 188168655 by mikecase: Internal Change. -- 187687045 by mikecase: Internal Change -- PiperOrigin-RevId: 188267957 --- tensorflow_serving/apis/BUILD | 10 ---------- tensorflow_serving/config/BUILD | 3 --- tensorflow_serving/core/BUILD | 1 - tensorflow_serving/sources/storage_path/BUILD | 1 - tensorflow_serving/util/BUILD | 1 - 5 files changed, 16 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index aed87f2ad14..a0472a9792c 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -27,7 +27,6 @@ serving_proto_library( name = "get_model_metadata_proto", srcs = ["get_model_metadata.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ":model_proto", @@ -50,7 +49,6 @@ serving_proto_library( name = "input_proto", srcs = ["input.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ "@org_tensorflow//tensorflow/core:protos_all_cc", @@ -77,7 +75,6 @@ serving_proto_library( name = "model_proto", srcs = ["model.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, js_api_version = 2, deps = [ @@ -102,7 +99,6 @@ serving_proto_library( name = "predict_proto", srcs = ["predict.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ":model_proto", @@ -126,7 +122,6 @@ serving_proto_library( has_services = 1, cc_api_version = 2, cc_grpc_version = 1, - go_api_version = 2, java_api_version = 2, deps = [ ":classification_proto", @@ -160,7 +155,6 @@ serving_proto_library( name = "get_model_status_proto", srcs = ["get_model_status.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ":model_proto", @@ -184,7 +178,6 @@ serving_proto_library( has_services = 1, cc_api_version = 2, cc_grpc_version = 1, - go_api_version = 2, java_api_version = 2, deps = [ ":get_model_status_proto", @@ -211,7 +204,6 @@ serving_proto_library( name = "classification_proto", srcs = ["classification.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ":input_proto", @@ -240,7 +232,6 @@ serving_proto_library( name = "inference_proto", srcs = ["inference.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ":classification_proto", @@ -272,7 +263,6 @@ serving_proto_library( name = "regression_proto", srcs = ["regression.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ":input_proto", diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index b09d8b2e2c8..bc97fb8df40 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -25,7 +25,6 @@ serving_proto_library( name = "model_server_config_proto", srcs = ["model_server_config.proto"], cc_api_version = 2, - go_api_version = 2, deps = [ ":logging_config_proto", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", @@ -52,7 +51,6 @@ serving_proto_library( name = "log_collector_config_proto", srcs = ["log_collector_config.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ], @@ -62,7 +60,6 @@ serving_proto_library( name = "logging_config_proto", srcs = ["logging_config.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, deps = [ ":log_collector_config_proto", diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index fff97f00c86..e7284abbbb6 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -732,7 +732,6 @@ serving_proto_library( name = "logging_proto", srcs = ["logging.proto"], cc_api_version = 2, - go_api_version = 2, visibility = [ "//visibility:public", ], diff --git a/tensorflow_serving/sources/storage_path/BUILD b/tensorflow_serving/sources/storage_path/BUILD index 82731055afd..44da570369d 100644 --- a/tensorflow_serving/sources/storage_path/BUILD +++ b/tensorflow_serving/sources/storage_path/BUILD @@ -84,7 +84,6 @@ serving_proto_library( name = "file_system_storage_path_source_proto", srcs = ["file_system_storage_path_source.proto"], cc_api_version = 2, - go_api_version = 2, visibility = ["//visibility:public"], ) diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 27f202a1dca..762fd610e8a 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -334,7 +334,6 @@ serving_proto_library( name = "status_proto", srcs = ["status.proto"], cc_api_version = 2, - go_api_version = 2, java_api_version = 2, visibility = ["//visibility:public"], deps = [ From cf0011500452c897a7206dcb07f33da87968113c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 13 Mar 2018 15:18:43 -0700 Subject: [PATCH 0444/8554] Merged commit includes the following changes: 188938744 by olston: Public no-op. -- 188481447 by mikecase: Internal Change. -- PiperOrigin-RevId: 188938744 --- tensorflow_serving/servables/tensorflow/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 466e2c44d60..11ea428684a 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -375,6 +375,9 @@ cc_library( name = "serving_session", srcs = ["serving_session.cc"], hdrs = ["serving_session.h"], + visibility = [ + "//visibility:public", + ], deps = [ "@org_tensorflow//tensorflow/core:core_cpu", "@org_tensorflow//tensorflow/core:lib", From 2285c7a8eccfe1cda34a4fecb9790895b908b3af Mon Sep 17 00:00:00 2001 From: Alex Glikson Date: Wed, 14 Mar 2018 13:16:03 -0400 Subject: [PATCH 0445/8554] Repo cloning and shell prompt in example readme - Added repo cloning instructions - Updated shell prompt to "$ ", so that syntax highlighting shows it correctly, and unified across the entire document --- tensorflow_serving/g3doc/serving_basic.md | 32 ++++++++++++++--------- 1 file changed, 20 insertions(+), 12 deletions(-) diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 121bdee6eaa..4a4175a2f10 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -144,10 +144,18 @@ def structure and how to set up them up can be found [here](signature_defs.md). Let's run it! +First, if you haven't done so yet, clone this repository to your local machine: +```shell +$ git clone https://github.com/tensorflow/serving.git +Cloning into 'serving'... +[...] +$ cd serving +``` + Clear the export directory if it already exists: ```shell -$>rm -rf /tmp/mnist_model +$ rm -rf /tmp/mnist_model ``` If you would like to install the `tensorflow` and `tensorflow-serving-api` PIP @@ -162,8 +170,8 @@ both the Bazel and PIP options. Bazel: ```shell -$>bazel build -c opt //tensorflow_serving/example:mnist_saved_model -$>bazel-bin/tensorflow_serving/example/mnist_saved_model /tmp/mnist_model +$ bazel build -c opt //tensorflow_serving/example:mnist_saved_model +$ bazel-bin/tensorflow_serving/example/mnist_saved_model /tmp/mnist_model Training model... ... @@ -176,13 +184,13 @@ Done exporting! Or if you have `tensorflow-serving-api` installed, you can run: ```shell -python tensorflow_serving/example/mnist_saved_model.py /tmp/mnist_model +$ python tensorflow_serving/example/mnist_saved_model.py /tmp/mnist_model ``` Now let's take a look at the export directory. ```shell -$>ls /tmp/mnist_model +$ ls /tmp/mnist_model 1 ``` @@ -191,7 +199,7 @@ of the model. `FLAGS.model_version` has the default value of 1, therefore the corresponding sub-directory `1` is created. ```shell -$>ls /tmp/mnist_model/1 +$ ls /tmp/mnist_model/1 saved_model.pb variables ``` @@ -210,8 +218,8 @@ With that, your TensorFlow model is exported and ready to be loaded! If you'd like to use a locally compiled ModelServer, run the following: ```shell -$>bazel build -c opt //tensorflow_serving/model_servers:tensorflow_model_server -$>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ +$ bazel build -c opt //tensorflow_serving/model_servers:tensorflow_model_server +$ bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ ``` If you'd prefer to skip compilation and install using apt-get, follow the @@ -219,7 +227,7 @@ If you'd prefer to skip compilation and install using apt-get, follow the the following command: ```shell -tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ +$ tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ ``` ## Test The Server @@ -232,8 +240,8 @@ requests to the server, and calculates the inference error rate. To run it with Bazel: ```shell -$>bazel build -c opt //tensorflow_serving/example:mnist_client -$>bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 +$ bazel build -c opt //tensorflow_serving/example:mnist_client +$ bazel-bin/tensorflow_serving/example/mnist_client --num_tests=1000 --server=localhost:9000 ... Inference error rate: 10.5% ``` @@ -241,7 +249,7 @@ Inference error rate: 10.5% Alternatively if you installed the PIP package, run: ```shell -python tensorflow_serving/example/mnist_client.py --num_tests=1000 --server=localhost:9000 +$ python tensorflow_serving/example/mnist_client.py --num_tests=1000 --server=localhost:9000 ``` We expect around 91% accuracy for the trained Softmax model and we get From b9b85bf13af09714178c5a62c3a0136b62e10a59 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 28 Mar 2018 01:37:48 -0400 Subject: [PATCH 0446/8554] Merged commit includes the following changes: 190725818 by mikecase: Internal Change. -- 190683775 by A. Unique TensorFlower: Documentation fix so tf-serving users can export inception-v3 checkpoints as serving models. -- 190671960 by olston: Public no-op. -- 190625265 by mikecase: Fix import of version check skylark function. -- 190367404 by A. Unique TensorFlower: No public changes. -- 190249020 by A. Unique TensorFlower: Move NullFileSystem to its own file. -- 190176889 by christis: Internal change. -- 190091955 by A. Unique TensorFlower: No public changes. -- 189971143 by gvasudevan: Update links to anchors to use specific targets -- 189951980 by nfiedel: Add a counter for the total number of examples processed by TensorFlow Serving's Example oriented APIs, e.g. Regress, Classify, and MultiInference. -- 189617327 by christis: Merge changes from github. -- 189593522 by A. Unique TensorFlower: Remove a few unused #includes -- 189110492 by A. Unique TensorFlower: No public changes. -- PiperOrigin-RevId: 190725818 --- WORKSPACE | 2 +- tensorflow_serving/apis/BUILD | 44 +++++++++++++++++ .../apis/model_management.proto | 15 ++++++ tensorflow_serving/apis/model_service.proto | 11 ++++- tensorflow_serving/apis/prediction_log.proto | 49 +++++++++++++++++++ tensorflow_serving/apis/session_service.proto | 48 ++++++++++++++++++ tensorflow_serving/config/BUILD | 1 + tensorflow_serving/core/BUILD | 5 ++ tensorflow_serving/core/basic_manager_test.cc | 1 + tensorflow_serving/core/load_servables_fast.h | 1 + tensorflow_serving/g3doc/serving_advanced.md | 4 +- tensorflow_serving/g3doc/serving_basic.md | 18 +++---- tensorflow_serving/g3doc/serving_inception.md | 7 ++- tensorflow_serving/g3doc/setup.md | 6 +-- tensorflow_serving/model_servers/BUILD | 11 ++--- .../model_servers/model_service_impl.cc | 40 ++++++++++++++- .../model_servers/model_service_impl.h | 15 ++++-- .../model_servers/server_core.h | 1 + .../tensorflow_model_server_test.py | 1 - tensorflow_serving/servables/tensorflow/BUILD | 7 ++- .../servables/tensorflow/testdata/BUILD | 11 +++++ .../servables/tensorflow/util.cc | 9 ++++ .../servables/tensorflow/util.h | 3 ++ .../servables/tensorflow/util_test.cc | 7 +++ tensorflow_serving/sources/storage_path/BUILD | 3 +- 25 files changed, 279 insertions(+), 41 deletions(-) create mode 100644 tensorflow_serving/apis/model_management.proto create mode 100644 tensorflow_serving/apis/prediction_log.proto create mode 100644 tensorflow_serving/apis/session_service.proto diff --git a/WORKSPACE b/WORKSPACE index 3806a55aca3..70cabb04034 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -33,6 +33,6 @@ load("//tensorflow_serving:workspace.bzl", "tf_serving_workspace") tf_serving_workspace() # Specify the minimum required bazel version. -load("@org_tensorflow//tensorflow:workspace.bzl", "check_bazel_version_at_least") +load("@org_tensorflow//tensorflow:version_check.bzl", "check_bazel_version_at_least") check_bazel_version_at_least("0.5.4") diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index a0472a9792c..f3837f774b6 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -116,6 +116,20 @@ serving_proto_library_py( ], ) +serving_proto_library( + name = "prediction_log_proto", + srcs = ["prediction_log.proto"], + cc_api_version = 2, + deps = [ + ":classification_proto", + ":inference_proto", + ":predict_proto", + ":regression_proto", + ":session_service_proto", + "//tensorflow_serving/core:logging_proto", + ], +) + serving_proto_library( name = "prediction_service_proto", srcs = ["prediction_service.proto"], @@ -151,6 +165,17 @@ serving_go_grpc_library( deps = [":prediction_service_go_proto"], ) +serving_proto_library( + name = "model_management_proto", + srcs = ["model_management.proto"], + cc_api_version = 2, + java_api_version = 2, + deps = [ + "//tensorflow_serving/config:model_server_config_proto", + "//tensorflow_serving/util:status_proto", + ], +) + serving_proto_library( name = "get_model_status_proto", srcs = ["get_model_status.proto"], @@ -181,6 +206,7 @@ serving_proto_library( java_api_version = 2, deps = [ ":get_model_status_proto", + ":model_management_proto", ], ) @@ -281,6 +307,24 @@ serving_proto_library_py( ], ) +serving_proto_library( + name = "session_service_proto", + srcs = ["session_service.proto"], + has_services = 1, + cc_api_version = 2, + java_api_version = 2, + deps = [ + ":model_proto", + "@org_tensorflow//tensorflow/core:protos_all_cc", + ], +) + +java_proto_library( + name = "session_service_java_proto", + strict_deps = 0, + deps = [":session_service_proto"], +) + tf_pyclif_proto_library( name = "regression_pyclif", proto_lib = ":regression_proto", diff --git a/tensorflow_serving/apis/model_management.proto b/tensorflow_serving/apis/model_management.proto new file mode 100644 index 00000000000..09f2628ae86 --- /dev/null +++ b/tensorflow_serving/apis/model_management.proto @@ -0,0 +1,15 @@ +syntax = "proto3"; + +import "tensorflow_serving/config/model_server_config.proto"; +import "tensorflow_serving/util/status.proto"; + +package tensorflow.serving; +option cc_enable_arenas = true; + +message ReloadConfigRequest { + ModelServerConfig config = 1; +} + +message ReloadConfigResponse { + StatusProto status = 1; +} \ No newline at end of file diff --git a/tensorflow_serving/apis/model_service.proto b/tensorflow_serving/apis/model_service.proto index f3f0b8c8833..29a3b077512 100644 --- a/tensorflow_serving/apis/model_service.proto +++ b/tensorflow_serving/apis/model_service.proto @@ -3,15 +3,22 @@ syntax = "proto3"; option cc_enable_arenas = true; import "tensorflow_serving/apis/get_model_status.proto"; +import "tensorflow_serving/apis/model_management.proto"; package tensorflow.serving; -// ModelService provides access to information about model versions -// that have been handled by the model server. +// ModelService provides methods to query and update the state of the server, +// e.g. which models/versions are being served. service ModelService { // Gets status of model. If the ModelSpec in the request does not specify // version, information about all versions of the model will be returned. If // the ModelSpec in the request does specify a version, the status of only // that version will be returned. rpc GetModelStatus(GetModelStatusRequest) returns (GetModelStatusResponse); + + // Reloads the set of served models. The new config supersedes the old one, + // so if a model is omitted from the new config it will be unloaded and no + // longer served. + rpc HandleReloadConfigRequest(ReloadConfigRequest) + returns (ReloadConfigResponse); } diff --git a/tensorflow_serving/apis/prediction_log.proto b/tensorflow_serving/apis/prediction_log.proto new file mode 100644 index 00000000000..474d6bb37e6 --- /dev/null +++ b/tensorflow_serving/apis/prediction_log.proto @@ -0,0 +1,49 @@ +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/classification.proto"; +import "tensorflow_serving/apis/inference.proto"; +import "tensorflow_serving/apis/predict.proto"; +import "tensorflow_serving/apis/regression.proto"; +import "tensorflow_serving/apis/session_service.proto"; +import "tensorflow_serving/core/logging.proto"; + +package tensorflow.serving; + +message ClassifyLog { + ClassificationRequest request = 1; + ClassificationResponse response = 2; +} + +message RegressLog { + RegressionRequest request = 1; + RegressionResponse response = 2; +} + +message PredictLog { + PredictRequest request = 1; + PredictResponse response = 2; +} + +message MultiInferenceLog { + MultiInferenceRequest request = 1; + MultiInferenceResponse response = 2; +} + +message SessionRunLog { + SessionRunRequest request = 1; + SessionRunResponse response = 2; +} + +// Logged model inference request. +message PredictionLog { + LogMetadata log_metadata = 1; + oneof log_type { + ClassifyLog classify_log = 2; + RegressLog regress_log = 3; + PredictLog predict_log = 6; + MultiInferenceLog multi_inference_log = 4; + SessionRunLog session_run_log = 5; + } +} diff --git a/tensorflow_serving/apis/session_service.proto b/tensorflow_serving/apis/session_service.proto new file mode 100644 index 00000000000..0c42131cf58 --- /dev/null +++ b/tensorflow_serving/apis/session_service.proto @@ -0,0 +1,48 @@ +syntax = "proto3"; + +option cc_enable_arenas = true; + +import "tensorflow_serving/apis/model.proto"; +import "tensorflow/core/protobuf/config.proto"; +import "tensorflow/core/protobuf/named_tensor.proto"; + +package tensorflow.serving; + +message SessionRunRequest { + // Model Specification. If version is not specified, will use the latest + // (numerical) version. + ModelSpec model_spec = 1; + + // Tensors to be fed in the step. Each feed is a named tensor. + repeated NamedTensorProto feed = 2; + + // Fetches. A list of tensor names. The caller expects a tensor to + // be returned for each fetch[i] (see RunResponse.tensor). The + // order of specified fetches does not change the execution order. + repeated string fetch = 3; + + // Target Nodes. A list of node names. The named nodes will be run + // to but their outputs will not be fetched. + repeated string target = 4; + + // Options for the run call. **Currently ignored.** + RunOptions options = 5; +} + +message SessionRunResponse { + // NOTE: The order of the returned tensors may or may not match + // the fetch order specified in RunRequest. + repeated NamedTensorProto tensor = 1; + + // Returned metadata if requested in the options. + RunMetadata metadata = 2; +} + +// SessionService defines a service with which a client can interact to execute +// Tensorflow model inference. The SessionService::SessionRun method is similar +// to MasterService::RunStep of Tensorflow, except that all sessions are ready +// to run, and you request a specific model/session with ModelSpec. +service SessionService { + // Runs inference of a given model. + rpc SessionRun(SessionRunRequest) returns (SessionRunResponse); +} diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index bc97fb8df40..cc33650d4be 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -25,6 +25,7 @@ serving_proto_library( name = "model_server_config_proto", srcs = ["model_server_config.proto"], cc_api_version = 2, + java_api_version = 2, deps = [ ":logging_config_proto", "//tensorflow_serving/sources/storage_path:file_system_storage_path_source_proto", diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index e7284abbbb6..40f95843900 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -699,7 +699,12 @@ cc_library( ], deps = [ ":aspired_versions_manager", + ":loader", + ":manager", + ":servable_state", ":servable_state_monitor", + ":source", + ":target", "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/core/basic_manager_test.cc b/tensorflow_serving/core/basic_manager_test.cc index 4e21d6df51d..5e8817851fe 100644 --- a/tensorflow_serving/core/basic_manager_test.cc +++ b/tensorflow_serving/core/basic_manager_test.cc @@ -25,6 +25,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/platform/null_file_system.h" #include "tensorflow_serving/core/servable_state_monitor.h" #include "tensorflow_serving/core/test_util/availability_test_util.h" #include "tensorflow_serving/core/test_util/fake_loader.h" diff --git a/tensorflow_serving/core/load_servables_fast.h b/tensorflow_serving/core/load_servables_fast.h index bdbfde2d441..b775cb59f6f 100644 --- a/tensorflow_serving/core/load_servables_fast.h +++ b/tensorflow_serving/core/load_servables_fast.h @@ -20,6 +20,7 @@ limitations under the License. #include #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow_serving/core/aspired_versions_manager.h" #include "tensorflow_serving/core/loader.h" #include "tensorflow_serving/core/manager.h" diff --git a/tensorflow_serving/g3doc/serving_advanced.md b/tensorflow_serving/g3doc/serving_advanced.md index 8cd51592e41..c76c366d970 100644 --- a/tensorflow_serving/g3doc/serving_advanced.md +++ b/tensorflow_serving/g3doc/serving_advanced.md @@ -35,8 +35,8 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisites). Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the -[instructions here](setup.md#optimized-build). +further optimize the build, refer to the [instructions +here](setup.md#optimized). ## Train And Export TensorFlow Model diff --git a/tensorflow_serving/g3doc/serving_basic.md b/tensorflow_serving/g3doc/serving_basic.md index 121bdee6eaa..246aa04259c 100644 --- a/tensorflow_serving/g3doc/serving_basic.md +++ b/tensorflow_serving/g3doc/serving_basic.md @@ -28,8 +28,8 @@ Before getting started, please complete the [prerequisites](setup.md#prerequisites). Note: All `bazel build` commands below use the standard `-c opt` flag. To -further optimize the build, refer to the -[instructions here](setup.md#optimized-build). +further optimize the build, refer to the [instructions +here](setup.md#optimized). ## Train And Export TensorFlow Model @@ -152,12 +152,10 @@ $>rm -rf /tmp/mnist_model If you would like to install the `tensorflow` and `tensorflow-serving-api` PIP packages, you can run all Python code (export and client) using a simple -`python` command. To install the PIP package, follow the -[instructions here](setup.md#tensorflow-serving-python-api-pip-package). -It's also possible to -use Bazel to build the necessary dependencies and run all code without -installing those packages. The rest of the codelab will have instructions for -both the Bazel and PIP options. +`python` command. To install the PIP package, follow the [instructions +here](setup.md#pip). It's also possible to use Bazel to build the necessary +dependencies and run all code without installing those packages. The rest of the +codelab will have instructions for both the Bazel and PIP options. Bazel: @@ -215,8 +213,8 @@ $>bazel-bin/tensorflow_serving/model_servers/tensorflow_model_server --port=9000 ``` If you'd prefer to skip compilation and install using apt-get, follow the -[instructions here](setup.md#installing-using-apt-get). Then run the server with -the following command: +[instructions here](setup.md#aptget). Then run the server with the following +command: ```shell tensorflow_model_server --port=9000 --model_name=mnist --model_base_path=/tmp/mnist_model/ diff --git a/tensorflow_serving/g3doc/serving_inception.md b/tensorflow_serving/g3doc/serving_inception.md index 93a41196182..c62918f77c8 100644 --- a/tensorflow_serving/g3doc/serving_inception.md +++ b/tensorflow_serving/g3doc/serving_inception.md @@ -37,13 +37,13 @@ $ docker run --name=inception_container -it $USER/tensorflow-serving-devel Note: All `bazel build` commands below use the standard `-c opt` flag. To further optimize the build, refer to the -[instructions here](setup.md#optimized-build). +[instructions here](setup.md#optimized). In the running container, we clone, configure and build TensorFlow Serving example code. ```shell -root@c97d8e820ced:/# git clone --recurse-submodules https://github.com/tensorflow/serving +root@c97d8e820ced:/# git clone -b r1.6 --recurse-submodules https://github.com/tensorflow/serving root@c97d8e820ced:/# cd serving/tensorflow root@c97d8e820ced:/serving/tensorflow# ./configure root@c97d8e820ced:/serving# cd .. @@ -51,8 +51,7 @@ root@c97d8e820ced:/serving# bazel build -c opt tensorflow_serving/example/... ``` Next we can either install a TensorFlow ModelServer with apt-get using the -[instructions here](setup.md#installing-using-apt-get), or build a ModelServer -binary using: +[instructions here](setup.md#aptget), or build a ModelServer binary using: ```shell root@c97d8e820ced:/serving# bazel build -c opt tensorflow_serving/model_servers:tensorflow_model_server diff --git a/tensorflow_serving/g3doc/setup.md b/tensorflow_serving/g3doc/setup.md index 935f752a79e..4f3a2ae3973 100644 --- a/tensorflow_serving/g3doc/setup.md +++ b/tensorflow_serving/g3doc/setup.md @@ -63,7 +63,7 @@ instructions](https://www.tensorflow.org/install/install_sources). Pay particular attention to `apt-get install` and `pip install` commands which you may need to run. -### TensorFlow Serving Python API PIP package +### TensorFlow Serving Python API PIP package {#pip} To run Python client code without the need to install Bazel, you can install the `tensorflow-serving-api` PIP package using: @@ -79,7 +79,7 @@ Python 2 version, unzip it, and copy it into your Python 3 path. There is a [feature request](https://github.com/tensorflow/serving/issues/700) to publish the Python 3 package as well. -## Installing using apt-get +## Installing using apt-get {#aptget} ### Available binaries @@ -176,7 +176,7 @@ bazel test -c opt tensorflow_serving/... See the [basic tutorial](serving_basic.md) and [advanced tutorial](serving_advanced.md) for more in-depth examples of running TensorFlow Serving. -### Optimized build +### Optimized build {#optimized} It's possible to compile using some platform specific instruction sets (e.g. AVX) that can significantly improve performance. Wherever you see 'bazel build' diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index 0c4ca0ab2fc..b5f9f83f996 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -129,14 +129,7 @@ cc_test( size = "medium", srcs = ["get_model_status_impl_test.cc"], data = [ - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/assets/foo.txt", - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/saved_model.pb", - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/variables/variables.data-00000-of-00001", - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000123/variables/variables.index", - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/assets/foo.txt", - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/saved_model.pb", - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/variables/variables.data-00000-of-00001", - "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions/00000124/variables/variables.index", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_half_plus_two_2_versions", ], deps = [ ":get_model_status_impl", @@ -168,7 +161,9 @@ cc_library( ":get_model_status_impl", ":grpc_status_util", ":server_core", + "//tensorflow_serving/apis:model_management_proto", "//tensorflow_serving/apis:model_service_proto", + "//tensorflow_serving/util:status_util", "@grpc//:grpc++_unsecure", ], ) diff --git a/tensorflow_serving/model_servers/model_service_impl.cc b/tensorflow_serving/model_servers/model_service_impl.cc index b239ba71bb2..655808b75cd 100644 --- a/tensorflow_serving/model_servers/model_service_impl.cc +++ b/tensorflow_serving/model_servers/model_service_impl.cc @@ -17,13 +17,14 @@ limitations under the License. #include "tensorflow_serving/model_servers/get_model_status_impl.h" #include "tensorflow_serving/model_servers/grpc_status_util.h" +#include "tensorflow_serving/util/status_util.h" namespace tensorflow { namespace serving { ::grpc::Status ModelServiceImpl::GetModelStatus( - ::grpc::ServerContext* context, const GetModelStatusRequest* request, - GetModelStatusResponse* response) { + ::grpc::ServerContext *context, const GetModelStatusRequest *request, + GetModelStatusResponse *response) { const ::grpc::Status status = tensorflow::serving::ToGRPCStatus( GetModelStatusImpl::GetModelStatus(core_, *request, response)); if (!status.ok()) { @@ -32,5 +33,40 @@ ::grpc::Status ModelServiceImpl::GetModelStatus( return status; } +::grpc::Status ModelServiceImpl::HandleReloadConfigRequest( + ::grpc::ServerContext *context, const ReloadConfigRequest *request, + ReloadConfigResponse *response) { + ModelServerConfig server_config = request->config(); + Status status; + switch (server_config.config_case()) { + case ModelServerConfig::kModelConfigList: { + const ModelConfigList list = server_config.model_config_list(); + + for (int index = 0; index < list.config_size(); index++) { + const ModelConfig config = list.config(index); + LOG(INFO) << "\nConfig entry" + << "\n\tindex : " << index + << "\n\tpath : " << config.base_path() + << "\n\tname : " << config.name() + << "\n\tplatform : " << config.model_platform(); + } + status = core_->ReloadConfig(server_config); + break; + } + default: + status = errors::InvalidArgument( + "ServerModelConfig type not supported by HandleReloadConfigRequest." + " Only ModelConfigList is currently supported"); + } + + if (!status.ok()) { + LOG(ERROR) << "ReloadConfig failed: " << status.error_message(); + } + + const StatusProto status_proto = ToStatusProto(status); + *response->mutable_status() = status_proto; + return ToGRPCStatus(status); +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/model_servers/model_service_impl.h b/tensorflow_serving/model_servers/model_service_impl.h index 0c4b00cf887..6411689cee9 100644 --- a/tensorflow_serving/model_servers/model_service_impl.h +++ b/tensorflow_serving/model_servers/model_service_impl.h @@ -18,6 +18,7 @@ limitations under the License. #include "grpc++/server_context.h" #include "grpc++/support/status.h" +#include "tensorflow_serving/apis/model_management.pb.h" #include "tensorflow_serving/apis/model_service.grpc.pb.h" #include "tensorflow_serving/apis/model_service.pb.h" #include "tensorflow_serving/model_servers/server_core.h" @@ -27,14 +28,18 @@ namespace serving { class ModelServiceImpl final : public ModelService::Service { public: - explicit ModelServiceImpl(ServerCore* core) : core_(core) {} + explicit ModelServiceImpl(ServerCore *core) : core_(core) {} - ::grpc::Status GetModelStatus(::grpc::ServerContext* context, - const GetModelStatusRequest* request, - GetModelStatusResponse* response) override; + ::grpc::Status GetModelStatus(::grpc::ServerContext *context, + const GetModelStatusRequest *request, + GetModelStatusResponse *response) override; + + ::grpc::Status HandleReloadConfigRequest(::grpc::ServerContext *context, + const ReloadConfigRequest *request, + ReloadConfigResponse *response); private: - ServerCore* core_; + ServerCore *core_; }; } // namespace serving diff --git a/tensorflow_serving/model_servers/server_core.h b/tensorflow_serving/model_servers/server_core.h index aff6705789f..419f9744a50 100644 --- a/tensorflow_serving/model_servers/server_core.h +++ b/tensorflow_serving/model_servers/server_core.h @@ -24,6 +24,7 @@ limitations under the License. #include "google/protobuf/any.pb.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/platform/cpu_info.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/mutex.h" #include "tensorflow/core/platform/types.h" diff --git a/tensorflow_serving/model_servers/tensorflow_model_server_test.py b/tensorflow_serving/model_servers/tensorflow_model_server_test.py index 8634881fd78..a81c7d9b3ac 100644 --- a/tensorflow_serving/model_servers/tensorflow_model_server_test.py +++ b/tensorflow_serving/model_servers/tensorflow_model_server_test.py @@ -500,6 +500,5 @@ def testGoodGrpcChannelArgs(self): expected_version=self._GetModelVersion( self._GetSavedModelHalfPlusThreePath())) - if __name__ == '__main__': tf.test.main() diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 11ea428684a..4585caf87db 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -764,6 +764,7 @@ cc_library( "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf_archive//:wrappers", ], ) @@ -773,12 +774,14 @@ cc_test( srcs = ["util_test.cc"], deps = [ ":util", - "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/core/test_util:test_main", # buildcleaner: keep "//tensorflow_serving/test_util", - "//tensorflow_serving/util:optional", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", "@org_tensorflow//tensorflow/core:testlib", + "@protobuf_archive//:wrappers", ], ) diff --git a/tensorflow_serving/servables/tensorflow/testdata/BUILD b/tensorflow_serving/servables/tensorflow/testdata/BUILD index a31bc204010..b2520bafea9 100644 --- a/tensorflow_serving/servables/tensorflow/testdata/BUILD +++ b/tensorflow_serving/servables/tensorflow/testdata/BUILD @@ -23,6 +23,17 @@ filegroup( ), ) +filegroup( + name = "saved_model_half_plus_two_2_versions", + srcs = glob( + ["saved_model_half_plus_two_2_versions/**/*"], + exclude = [ + "**/METADATA", + "**/OWNERS", + ], + ), +) + py_binary( name = "export_half_plus_two", srcs = [ diff --git a/tensorflow_serving/servables/tensorflow/util.cc b/tensorflow_serving/servables/tensorflow/util.cc index fc14965680b..239a06ead60 100644 --- a/tensorflow_serving/servables/tensorflow/util.cc +++ b/tensorflow_serving/servables/tensorflow/util.cc @@ -20,6 +20,8 @@ limitations under the License. #include "tensorflow/core/example/example.pb.h" #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/monitoring/counter.h" +#include "tensorflow/core/lib/monitoring/sampler.h" #include "tensorflow/core/lib/strings/strcat.h" #include "tensorflow/core/platform/types.h" #include "tensorflow_serving/apis/input.pb.h" @@ -38,6 +40,10 @@ auto* example_counts = monitoring::Sampler<1>::New( // so the limits are [1, 2, 4, 8, ..., 16 * 1024, DBL_MAX]. monitoring::Buckets::Exponential(1, 2, 15)); +auto* example_count_total = monitoring::Counter<1>::New( + "/tensorflow/serving/request_example_count_total", + "The total number of tensorflow.Examples.", "model"); + // Returns the number of examples in the Input. int NumInputExamples(const internal::SerializedInput& input) { switch (input.kind_case()) { @@ -57,10 +63,13 @@ namespace internal { monitoring::Sampler<1>* GetExampleCounts() { return example_counts; } +monitoring::Counter<1>* GetExampleCountTotal() { return example_count_total; } + } // namespace internal void RecordRequestExampleCount(const string& model_name, size_t count) { example_counts->GetCell(model_name)->Add(count); + example_count_total->GetCell(model_name)->IncrementBy(count); } Status InputToSerializedExampleTensor(const Input& input, Tensor* examples) { diff --git a/tensorflow_serving/servables/tensorflow/util.h b/tensorflow_serving/servables/tensorflow/util.h index 404e21048da..699b28f419a 100644 --- a/tensorflow_serving/servables/tensorflow/util.h +++ b/tensorflow_serving/servables/tensorflow/util.h @@ -18,6 +18,7 @@ limitations under the License. #include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/lib/core/status.h" +#include "tensorflow/core/lib/monitoring/counter.h" #include "tensorflow/core/lib/monitoring/sampler.h" #include "tensorflow/core/public/session.h" #include "tensorflow_serving/apis/input.pb.h" @@ -32,6 +33,8 @@ namespace internal { monitoring::Sampler<1>* GetExampleCounts(); +monitoring::Counter<1>* GetExampleCountTotal(); + } // namespace internal // Records the example count of this request with the metric tracking the diff --git a/tensorflow_serving/servables/tensorflow/util_test.cc b/tensorflow_serving/servables/tensorflow/util_test.cc index e79f984d3a0..076cb7754e5 100644 --- a/tensorflow_serving/servables/tensorflow/util_test.cc +++ b/tensorflow_serving/servables/tensorflow/util_test.cc @@ -23,6 +23,8 @@ limitations under the License. #include "tensorflow/core/framework/tensor_testutil.h" #include "tensorflow/core/lib/core/status_test_util.h" #include "tensorflow/core/lib/histogram/histogram.h" +#include "tensorflow/core/lib/monitoring/counter.h" +#include "tensorflow/core/lib/monitoring/sampler.h" #include "tensorflow_serving/test_util/test_util.h" namespace tensorflow { @@ -210,13 +212,18 @@ TEST(ExampleCountsTest, Simple) { const HistogramProto before_histogram = internal::GetExampleCounts()->GetCell("model-name")->value(); + const int before_count = + internal::GetExampleCountTotal()->GetCell("model-name")->value(); RecordRequestExampleCount("model-name", 3); const HistogramProto after_histogram = internal::GetExampleCounts()->GetCell("model-name")->value(); + const int after_count = + internal::GetExampleCountTotal()->GetCell("model-name")->value(); ASSERT_GE(before_histogram.bucket().size(), 3); ASSERT_GE(after_histogram.bucket().size(), 3); EXPECT_EQ(1, after_histogram.bucket(2) - before_histogram.bucket(2)); + EXPECT_EQ(3, after_count - before_count); } TEST(ModelSpecTest, NoOptional) { diff --git a/tensorflow_serving/sources/storage_path/BUILD b/tensorflow_serving/sources/storage_path/BUILD index 44da570369d..6106cbc55a2 100644 --- a/tensorflow_serving/sources/storage_path/BUILD +++ b/tensorflow_serving/sources/storage_path/BUILD @@ -74,7 +74,7 @@ cc_library( "//tensorflow_serving/core:servable_id", "//tensorflow_serving/core:source", "//tensorflow_serving/core:storage_path", - "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", + "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function_dynamic", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:tensorflow", ], @@ -84,6 +84,7 @@ serving_proto_library( name = "file_system_storage_path_source_proto", srcs = ["file_system_storage_path_source.proto"], cc_api_version = 2, + java_api_version = 2, visibility = ["//visibility:public"], ) From a650b230d6db4023b0b6d7433232c67df5791498 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 28 Mar 2018 16:41:43 -0700 Subject: [PATCH 0447/8554] Merged commit includes the following changes: 190856719 by christis: Internal change. -- PiperOrigin-RevId: 190856719 --- tensorflow_serving/apis/BUILD | 1 + tensorflow_serving/core/BUILD | 1 + 2 files changed, 2 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index f3837f774b6..155d9c62d85 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -120,6 +120,7 @@ serving_proto_library( name = "prediction_log_proto", srcs = ["prediction_log.proto"], cc_api_version = 2, + java_api_version = 2, deps = [ ":classification_proto", ":inference_proto", diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 40f95843900..64b4fc32c2d 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -737,6 +737,7 @@ serving_proto_library( name = "logging_proto", srcs = ["logging.proto"], cc_api_version = 2, + java_api_version = 2, visibility = [ "//visibility:public", ], From ea09e5c063ef805190ace52b0e28e1f70f6077a2 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 28 Mar 2018 16:50:12 -0700 Subject: [PATCH 0448/8554] Making TF Serving API dependencies semantiv-versioning hermetic. PiperOrigin-RevId: 190857880 --- tensorflow_serving/tools/pip_package/setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/tools/pip_package/setup.py b/tensorflow_serving/tools/pip_package/setup.py index fad0cff50ad..a00da3d0333 100644 --- a/tensorflow_serving/tools/pip_package/setup.py +++ b/tensorflow_serving/tools/pip_package/setup.py @@ -20,8 +20,8 @@ _VERSION = 'undefined' REQUIRED_PACKAGES = [ - 'tensorflow>=1.2.0', - 'grpcio>=1.0', + 'tensorflow>=1.2.0,<2', + 'grpcio>=1.0<2', ] setup( From d5c85abb346cf1a4f415c355e8eb1e8662d78f35 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Wed, 28 Mar 2018 17:58:57 -0700 Subject: [PATCH 0449/8554] Internal Change. PiperOrigin-RevId: 190866240 --- WORKSPACE | 4 ++-- tensorflow_serving/apis/BUILD | 6 ------ tensorflow_serving/servables/tensorflow/BUILD | 4 ++-- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 70cabb04034..a221be3e2c4 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "21d6ac553adcfc9d089925f6d6793fee6a67264a0ce717bc998636662df4ca7e", - git_commit = "bc69c4ceed6544c109be5693eb40ddcf3a4eb95d", + sha256 = "dd44550909aab50790495264d3e5c9e9373f9c0f0272047fd68df3e56c07cc78", + git_commit = "7a212edc6b3ed6200158fe51acf4694a62ca6938", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 155d9c62d85..2319cd24245 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -320,12 +320,6 @@ serving_proto_library( ], ) -java_proto_library( - name = "session_service_java_proto", - strict_deps = 0, - deps = [":session_service_proto"], -) - tf_pyclif_proto_library( name = "regression_pyclif", proto_lib = ":regression_proto", diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 4585caf87db..90166d71c58 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -764,7 +764,7 @@ cc_library( "@org_tensorflow//tensorflow/core:framework", "@org_tensorflow//tensorflow/core:lib", "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf_archive//:wrappers", + "@protobuf_archive//:cc_wkt_protos", ], ) @@ -782,6 +782,6 @@ cc_test( "@org_tensorflow//tensorflow/core:protos_all_cc", "@org_tensorflow//tensorflow/core:test", "@org_tensorflow//tensorflow/core:testlib", - "@protobuf_archive//:wrappers", + "@protobuf_archive//:cc_wkt_protos", ], ) From 815faed29cc4d3af53a637b6729222211b2e8b7b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 30 Mar 2018 14:42:55 -0700 Subject: [PATCH 0450/8554] Merged commit includes the following changes: 191119221 by olston: Fix small comment inaccuracy in class_registration.h. -- 191025795 by mikecase: Internal Change. -- PiperOrigin-RevId: 191119221 --- tensorflow_serving/util/class_registration.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h index 30ef145b6af..73b784480e8 100644 --- a/tensorflow_serving/util/class_registration.h +++ b/tensorflow_serving/util/class_registration.h @@ -99,7 +99,7 @@ limitations under the License. // // This mechanism also allows additional parameter passing into the Create() // factory method. Consider the following example in which Create() takes an -// int and a string, in addition to the config proto: +// int, a string and an int->string map, in addition to the config proto: // // class MyParameterizedBaseClass { // ... From 7534cd19e8c3705dbe662827140195399a1c868b Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Fri, 30 Mar 2018 15:45:54 -0700 Subject: [PATCH 0451/8554] Rename COMMA to TFS_COMMA to reduce the chance of collisions. PiperOrigin-RevId: 191127867 --- tensorflow_serving/util/class_registration.h | 14 +++++++------- tensorflow_serving/util/class_registration_test.cc | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h index 73b784480e8..01a3cf1e5ad 100644 --- a/tensorflow_serving/util/class_registration.h +++ b/tensorflow_serving/util/class_registration.h @@ -106,11 +106,11 @@ limitations under the License. // }; // DEFINE_CLASS_REGISTRY(MyParameterizedBaseClassRegistry, // MyParameterizedBaseClass, int, const string& -// const std::map&); +// const std::map&); // #define REGISTER_MY_BASE_CLASS(ClassCreator, ConfigProto) // REGISTER_CLASS(MyBaseClassRegistry, MyBaseClass, ClassCreator, // ConfigProto, int, const string&, -// const std::map&); +// const std::map&); // // class OneClass : public MyParameterizedBaseClass { // public: @@ -138,9 +138,9 @@ limitations under the License. // registries for the same base class, potentially with different factory // signatures. // -// Note that types containing a comma, e.g. std::map must use COMMA -// in place of ','. -// TODO(b/24472377): Eliminate the requirement to use COMMA. +// Note that types containing a comma, e.g. std::map must use +// TFS_COMMA in place of ','. +// TODO(b/24472377): Eliminate the requirement to use TFS_COMMA. #ifndef TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_H_ #define TENSORFLOW_SERVING_UTIL_CLASS_REGISTRATION_H_ @@ -330,9 +330,9 @@ class ClassRegistry { // Used to enable the following macros to work with types containing commas // (e.g. map). -// TODO(b/24472377): Eliminate the requirement to use COMMA, via some fancy +// TODO(b/24472377): Eliminate the requirement to use TFS_COMMA, via some fancy // macro gymnastics. -#define COMMA , +#define TFS_COMMA , // Given a base class BaseClass, creates a registry named RegistryName. #define DEFINE_CLASS_REGISTRY(RegistryName, BaseClass, ...) \ diff --git a/tensorflow_serving/util/class_registration_test.cc b/tensorflow_serving/util/class_registration_test.cc index ad779f7f703..5e4317d2cea 100644 --- a/tensorflow_serving/util/class_registration_test.cc +++ b/tensorflow_serving/util/class_registration_test.cc @@ -192,11 +192,11 @@ class MyParameterizedBaseClass { }; DEFINE_CLASS_REGISTRY(MyParameterizedBaseClassRegistry, MyParameterizedBaseClass, int, const string&, - const std::map&); + const std::map&); #define REGISTER_MY_PARAMETERIZED_BASE_CLASS(SubClassCreator, ConfigProto) \ REGISTER_CLASS(MyParameterizedBaseClassRegistry, MyParameterizedBaseClass, \ SubClassCreator, ConfigProto, int, const string&, \ - const std::map&); + const std::map&); // A subclass of MyParameterizedBaseClass that should be instantiated via // Config1. From 13650f2220fe9a17a3ba541ab397578bbedcb22b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 2 Apr 2018 14:46:13 -0700 Subject: [PATCH 0452/8554] Replaced calls to deprecated tensorflow::StringPiece methods with their tensorflow::str_util equivalents. This will allow the deprecated methods to be removed. PiperOrigin-RevId: 191350894 --- tensorflow_serving/core/static_source_router.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/core/static_source_router.h b/tensorflow_serving/core/static_source_router.h index b54e49954ed..6aea4805773 100644 --- a/tensorflow_serving/core/static_source_router.h +++ b/tensorflow_serving/core/static_source_router.h @@ -20,6 +20,7 @@ limitations under the License. #include #include +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/macros.h" #include "tensorflow_serving/core/source_router.h" @@ -80,7 +81,7 @@ template int StaticSourceRouter::Route(const StringPiece servable_name, const std::vector>& versions) { for (int i = 0; i < routes_except_default_.size(); ++i) { - if (servable_name.contains(routes_except_default_[i])) { + if (str_util::StrContains(servable_name, routes_except_default_[i])) { LOG(INFO) << "Routing servable(s) from stream " << servable_name << " to route " << i; return i; From 7199b6dec5036747ec9128e68d5ea3dc1e04a3c4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 3 Apr 2018 13:16:21 -0700 Subject: [PATCH 0453/8554] Merged commit includes the following changes: 191489624 by A. Unique TensorFlower: Expose AspiredVersionsManager's num_load_threads() and SetNumLoadThreads() to internal namespace. -- PiperOrigin-RevId: 191489624 --- .../core/aspired_versions_manager.h | 15 +++++++-------- tensorflow_serving/core/load_servables_fast.cc | 16 +++++++++++++--- tensorflow_serving/core/load_servables_fast.h | 4 ++++ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index e080184374c..11e86d2e9cb 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -50,10 +50,9 @@ namespace internal { class AspiredVersionsManagerTargetImpl; -Status ConnectSourcesWithFastInitialLoad( - AspiredVersionsManager* manager, - std::vector>*> sources, - const std::function& wait_until_loaded_fn, uint32 num_threads); +uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager); +void SetManagerNumLoadThreads(uint32 num_load_threads, + AspiredVersionsManager* manager); } // namespace internal @@ -204,10 +203,10 @@ class AspiredVersionsManager : public Manager, friend class internal::AspiredVersionsManagerTargetImpl; friend class test_util::AspiredVersionsManagerTestAccess; friend class ServerCore; - friend Status internal::ConnectSourcesWithFastInitialLoad( - AspiredVersionsManager* manager, - std::vector>*> sources, - const std::function& wait_until_loaded_fn, uint32 num_threads); + friend uint32 internal::GetManagerNumLoadThreads( + AspiredVersionsManager* manager); + friend void internal::SetManagerNumLoadThreads( + uint32 num_load_threads, AspiredVersionsManager* manager); AspiredVersionsManager( int64 manage_state_interval_micros, Env* env, diff --git a/tensorflow_serving/core/load_servables_fast.cc b/tensorflow_serving/core/load_servables_fast.cc index 9a7eb45fb8b..2205763c166 100644 --- a/tensorflow_serving/core/load_servables_fast.cc +++ b/tensorflow_serving/core/load_servables_fast.cc @@ -17,6 +17,7 @@ limitations under the License. #include #include +#include #include "tensorflow/core/lib/core/errors.h" #include "tensorflow_serving/core/servable_state.h" @@ -28,18 +29,27 @@ namespace serving { namespace internal { +uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager) { + return manager->num_load_threads(); +} + +void SetManagerNumLoadThreads(uint32 num_load_threads, + AspiredVersionsManager* manager) { + manager->SetNumLoadThreads(num_load_threads); +} + Status ConnectSourcesWithFastInitialLoad( AspiredVersionsManager* manager, std::vector>*> sources, const std::function& wait_until_loaded_fn, const uint32 num_threads) { - const uint32 prev_num_load_threads = manager->num_load_threads(); - manager->SetNumLoadThreads(num_threads); + const uint32 prev_num_load_threads = GetManagerNumLoadThreads(manager); + SetManagerNumLoadThreads(num_threads, manager); for (Source>* source : sources) { ConnectSourceToTarget(source, manager); } const Status status = wait_until_loaded_fn(); - manager->SetNumLoadThreads(prev_num_load_threads); + SetManagerNumLoadThreads(prev_num_load_threads, manager); return status; } diff --git a/tensorflow_serving/core/load_servables_fast.h b/tensorflow_serving/core/load_servables_fast.h index b775cb59f6f..794b21f20b4 100644 --- a/tensorflow_serving/core/load_servables_fast.h +++ b/tensorflow_serving/core/load_servables_fast.h @@ -58,6 +58,10 @@ Status ConnectSourcesWithFastInitialLoad( std::vector>*> sources, const std::function& wait_until_loaded_fn, uint32 num_threads); +uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager); +void SetManagerNumLoadThreads(uint32 num_load_threads, + AspiredVersionsManager* manager); + } // namespace internal } // namespace serving From ecccab23fe13de094b7af3cd11fbabbbeb73d51c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 5 Apr 2018 07:48:58 -0700 Subject: [PATCH 0454/8554] Adds util functions for getting max of two ResourceAllocations. PiperOrigin-RevId: 191741238 --- tensorflow_serving/resources/BUILD | 3 + tensorflow_serving/resources/resource_util.cc | 21 +++ tensorflow_serving/resources/resource_util.h | 19 ++ .../resources/resource_util_test.cc | 174 ++++++++++++++++++ 4 files changed, 217 insertions(+) diff --git a/tensorflow_serving/resources/BUILD b/tensorflow_serving/resources/BUILD index 3f7dbe2507b..8a3a4dc886c 100644 --- a/tensorflow_serving/resources/BUILD +++ b/tensorflow_serving/resources/BUILD @@ -46,6 +46,7 @@ cc_library( deps = [ ":resources_proto", "@org_tensorflow//tensorflow/core:lib", + "@protobuf_archive//:wrappers", ], ) @@ -57,6 +58,8 @@ cc_test( ":resource_util", "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:test", ], ) diff --git a/tensorflow_serving/resources/resource_util.cc b/tensorflow_serving/resources/resource_util.cc index 4390849ce2e..e17a4db2592 100644 --- a/tensorflow_serving/resources/resource_util.cc +++ b/tensorflow_serving/resources/resource_util.cc @@ -220,6 +220,11 @@ ResourceAllocation ResourceUtil::Overbind( return OverbindNormalized(Normalize(allocation)); } +ResourceAllocation ResourceUtil::Max(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const { + return MaxNormalized(Normalize(lhs), Normalize(rhs)); +} + bool ResourceUtil::IsBoundNormalized( const ResourceAllocation& allocation) const { DCHECK(IsNormalized(allocation)); @@ -487,5 +492,21 @@ ResourceAllocation ResourceUtil::OverbindNormalized( return result; } +ResourceAllocation ResourceUtil::MaxNormalized( + const ResourceAllocation& lhs, const ResourceAllocation& rhs) const { + DCHECK(IsNormalized(lhs)); + DCHECK(IsNormalized(rhs)); + + ResourceAllocation max_resource_allocation = rhs; + for (const ResourceAllocation::Entry& lhs_entry : lhs.resource_quantities()) { + ResourceAllocation::Entry* max_entry = FindOrInsertMutableEntry( + lhs_entry.resource(), &max_resource_allocation); + if (lhs_entry.quantity() >= max_entry->quantity()) { + max_entry->set_quantity(lhs_entry.quantity()); + } + } + return max_resource_allocation; +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/resources/resource_util.h b/tensorflow_serving/resources/resource_util.h index 167dfc5b052..2d6d8bccb48 100644 --- a/tensorflow_serving/resources/resource_util.h +++ b/tensorflow_serving/resources/resource_util.h @@ -136,6 +136,20 @@ class ResourceUtil { // (because it binds resources redundantly to all device instances). ResourceAllocation Overbind(const ResourceAllocation& allocation) const; + // Gets the max of the two ResourceAllocations by taking the max of every + // paired entry and keep all the unpaired entries in the max. Like the Add() + // and Subtract() methods, this keeps the bound and unbound resources + // separate. + // + // E.g. + // Max({(GPU/instance_0/RAM/8), + // (CPU/instance_0/processing_in_millicores/100)}, + // {(GPU/instance_0/RAM/16), (CPU//RAM/4)}) returns + // {(GPU/instance_0/RAM/16), (CPU/instance_0/processing_in_millicores/100), + // (CPU//RAM/4)} + ResourceAllocation Max(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const; + private: enum class DCHECKFailOption { kDoDCHECKFail, kDoNotDCHECKFail }; @@ -188,6 +202,11 @@ class ResourceUtil { ResourceAllocation OverbindNormalized( const ResourceAllocation& allocation) const; + // Like Max(), but assumes the input are normalized and produces a normalized + // result. + ResourceAllocation MaxNormalized(const ResourceAllocation& lhs, + const ResourceAllocation& rhs) const; + const std::map devices_; TF_DISALLOW_COPY_AND_ASSIGN(ResourceUtil); diff --git a/tensorflow_serving/resources/resource_util_test.cc b/tensorflow_serving/resources/resource_util_test.cc index 73d901fd426..1f1cb7a6712 100644 --- a/tensorflow_serving/resources/resource_util_test.cc +++ b/tensorflow_serving/resources/resource_util_test.cc @@ -1493,6 +1493,180 @@ TEST_F(ResourceUtilTest, Overbind) { "} ")); } +TEST_F(ResourceUtilTest, MaxEmpty) { + const auto lhs = CreateProto(""); + const auto rhs = CreateProto(""); + EXPECT_THAT(util_.Max(lhs, rhs), EqualsProto("")); +} + +TEST_F(ResourceUtilTest, MaxBound) { + const auto lhs = CreateProto(R"( + resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'processing' + } + quantity: 100 + } + resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'ram' + } + quantity: 8 + } )"); + const auto rhs = CreateProto(R"( + resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'processing' + } + quantity: 300 + } + resource_quantities { + resource { + device: 'gpu' + device_instance { value: 0 } + kind: 'ram' + } + quantity: 4 + } )"); + EXPECT_THAT(util_.Max(lhs, rhs), EqualsProto(R"(resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'processing' + } + quantity: 300 + } + resource_quantities { + resource { + device: 'gpu' + device_instance { value: 0 } + kind: 'ram' + } + quantity: 4 + } + resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'ram' + } + quantity: 8 + })")); +} + +TEST_F(ResourceUtilTest, MaxBoundAndUnbound) { + const auto lhs = CreateProto(R"( + resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'processing' + } + quantity: 100 + } + resource_quantities { + resource { + device: 'gpu' + kind: 'ram' + } + quantity: 8 + } )"); + const auto rhs = CreateProto(R"( + resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'processing' + } + quantity: 300 + } + resource_quantities { + resource { + device: 'gpu' + device_instance { value: 0 } + kind: 'ram' + } + quantity: 4 + } )"); + EXPECT_THAT(util_.Max(lhs, rhs), EqualsProto(R"(resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'processing' + } + quantity: 300 + } + resource_quantities { + resource { + device: 'gpu' + device_instance { value: 0 } + kind: 'ram' + } + quantity: 4 + } + resource_quantities { + resource { + device: 'gpu' + kind: 'ram' + } + quantity: 8 + })")); +} + +TEST_F(ResourceUtilTest, MaxUnbound) { + const auto lhs = CreateProto(R"( + resource_quantities { + resource { + device: 'main' + kind: 'processing' + } + quantity: 100 + } + resource_quantities { + resource { + device: 'main' + kind: 'ram' + } + quantity: 8 + } )"); + const auto rhs = CreateProto(R"( + resource_quantities { + resource { + device: 'main' + kind: 'processing' + } + quantity: 300 + } + resource_quantities { + resource { + device: 'main' + kind: 'ram' + } + quantity: 4 + } )"); + EXPECT_THAT(util_.Max(lhs, rhs), EqualsProto(R"(resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'processing' + } + quantity: 300 + } + resource_quantities { + resource { + device: 'main' + device_instance { value: 0 } + kind: 'ram' + } + quantity: 8 + })")); +} } // namespace } // namespace serving } // namespace tensorflow From df318417e6db94662250cd73bb25734913fbc10a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 5 Apr 2018 11:01:50 -0700 Subject: [PATCH 0455/8554] Expose the Notifier of an Observer that forwards SetNumLoadThreads(), so that we can call the Notifier without worry of the underlying manager being deleted. PiperOrigin-RevId: 191766317 --- tensorflow_serving/core/BUILD | 3 ++- .../core/aspired_versions_manager.cc | 4 ++++ .../core/aspired_versions_manager.h | 17 +++++++++++++---- tensorflow_serving/core/load_servables_fast.cc | 12 +++++++----- tensorflow_serving/core/load_servables_fast.h | 4 ++-- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 64b4fc32c2d..4ec8fefcd1e 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -501,8 +501,9 @@ cc_library( ":source", ":target", "//tensorflow_serving/util:event_bus", + "//tensorflow_serving/util:observer", "//tensorflow_serving/util:optional", - "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function", + "@org_tensorflow//tensorflow/contrib/batching/util:periodic_function_dynamic", "@org_tensorflow//tensorflow/core:lib", ], ) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 9404913621d..8252766f53f 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -177,6 +177,10 @@ AspiredVersionsManager::AspiredVersionsManager( : aspired_version_policy_(std::move(aspired_version_policy)), target_impl_(new internal::AspiredVersionsManagerTargetImpl(this)), basic_manager_(std::move(basic_manager)) { + set_num_load_threads_observer_.reset( + new Observer([this](const uint32 num_load_threads) { + this->SetNumLoadThreads(num_load_threads); + })); if (manage_state_interval_micros > 0) { PeriodicFunction::Options pf_options; pf_options.env = env; diff --git a/tensorflow_serving/core/aspired_versions_manager.h b/tensorflow_serving/core/aspired_versions_manager.h index 11e86d2e9cb..505c7a7c073 100644 --- a/tensorflow_serving/core/aspired_versions_manager.h +++ b/tensorflow_serving/core/aspired_versions_manager.h @@ -39,6 +39,7 @@ limitations under the License. #include "tensorflow_serving/core/servable_state.h" #include "tensorflow_serving/core/target.h" #include "tensorflow_serving/util/event_bus.h" +#include "tensorflow_serving/util/observer.h" #include "tensorflow_serving/util/optional.h" namespace tensorflow { @@ -51,8 +52,12 @@ namespace internal { class AspiredVersionsManagerTargetImpl; uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager); -void SetManagerNumLoadThreads(uint32 num_load_threads, - AspiredVersionsManager* manager); + +// Returns the Notifier function of the manager's Observer, which forwards +// SetNumLoadThreads(). This indirection is to prevent callers from using +// SetNumLoadThreads() on a deleted manager. +std::function SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager); } // namespace internal @@ -205,8 +210,8 @@ class AspiredVersionsManager : public Manager, friend class ServerCore; friend uint32 internal::GetManagerNumLoadThreads( AspiredVersionsManager* manager); - friend void internal::SetManagerNumLoadThreads( - uint32 num_load_threads, AspiredVersionsManager* manager); + friend std::function internal::SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager); AspiredVersionsManager( int64 manage_state_interval_micros, Env* env, @@ -306,6 +311,10 @@ class AspiredVersionsManager : public Manager, // This is where the servables "live" while they are being managed. std::unique_ptr basic_manager_; + // An observer object that forwards to SetNumLoadThreads(), if not detached. + // This is declared last here so that it is deleted before basic_manager_. + std::unique_ptr> set_num_load_threads_observer_; + TF_DISALLOW_COPY_AND_ASSIGN(AspiredVersionsManager); }; diff --git a/tensorflow_serving/core/load_servables_fast.cc b/tensorflow_serving/core/load_servables_fast.cc index 2205763c166..cbf482167cc 100644 --- a/tensorflow_serving/core/load_servables_fast.cc +++ b/tensorflow_serving/core/load_servables_fast.cc @@ -33,9 +33,9 @@ uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager) { return manager->num_load_threads(); } -void SetManagerNumLoadThreads(uint32 num_load_threads, - AspiredVersionsManager* manager) { - manager->SetNumLoadThreads(num_load_threads); +std::function SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager) { + return manager->set_num_load_threads_observer_->Notifier(); } Status ConnectSourcesWithFastInitialLoad( @@ -44,12 +44,14 @@ Status ConnectSourcesWithFastInitialLoad( const std::function& wait_until_loaded_fn, const uint32 num_threads) { const uint32 prev_num_load_threads = GetManagerNumLoadThreads(manager); - SetManagerNumLoadThreads(num_threads, manager); + std::function set_manager_num_load_threads = + SetManagerNumLoadThreadsNotifier(manager); + set_manager_num_load_threads(num_threads); for (Source>* source : sources) { ConnectSourceToTarget(source, manager); } const Status status = wait_until_loaded_fn(); - SetManagerNumLoadThreads(prev_num_load_threads, manager); + set_manager_num_load_threads(prev_num_load_threads); return status; } diff --git a/tensorflow_serving/core/load_servables_fast.h b/tensorflow_serving/core/load_servables_fast.h index 794b21f20b4..cf2c706b6fd 100644 --- a/tensorflow_serving/core/load_servables_fast.h +++ b/tensorflow_serving/core/load_servables_fast.h @@ -59,8 +59,8 @@ Status ConnectSourcesWithFastInitialLoad( const std::function& wait_until_loaded_fn, uint32 num_threads); uint32 GetManagerNumLoadThreads(AspiredVersionsManager* manager); -void SetManagerNumLoadThreads(uint32 num_load_threads, - AspiredVersionsManager* manager); +std::function SetManagerNumLoadThreadsNotifier( + AspiredVersionsManager* manager); } // namespace internal From 9642c610ebd88b0083122ca7b25c1fd9f07c6a26 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Fri, 6 Apr 2018 06:49:58 -0700 Subject: [PATCH 0456/8554] Update the Serving GitHub build badge to use Kokoro. The Jenkins builds are broken/deprecated. Updating to Kokoro. Unfortunately, we can't really link to the builds since they are internal links (currently). But we can expose the build status publicly. PiperOrigin-RevId: 191882615 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3ea0278e69d..f752191b217 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TensorFlow Serving -[![Build Status](http://ci.tensorflow.org/buildStatus/icon?job=serving-master-cpu)](http://ci.tensorflow.org/job/serving-master-cpu) +![Build Status](https://storage.cloud.google.com/tensorflow-serving-kokoro-build-badges/ubuntu.png) TensorFlow Serving is an open-source software library for serving machine learning models. It deals with the *inference* aspect of machine From 6b162481eab0b4f1a47b72ff32b226504ae2f47c Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 6 Apr 2018 09:44:56 -0700 Subject: [PATCH 0457/8554] Fix comment formatting issue. PiperOrigin-RevId: 191900004 --- tensorflow_serving/core/servable_handle.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/core/servable_handle.h b/tensorflow_serving/core/servable_handle.h index 8f4d5b2c19a..a1d66cf594f 100644 --- a/tensorflow_serving/core/servable_handle.h +++ b/tensorflow_serving/core/servable_handle.h @@ -65,7 +65,7 @@ class UntypedServableHandle { /// }; /// /// // Get your handle from a manager. -/// ServableHandle<MyServable> handle; +/// ServableHandle handle; /// TF_RETURN_IF_ERROR(manager->GetServableHandle(id, &handle)); /// /// // Use your handle as a smart-pointer: From e99b01bdb1e2b5ccb8a8ce8f48ed922cf0a9b51d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 6 Apr 2018 11:12:14 -0700 Subject: [PATCH 0458/8554] Internal change. PiperOrigin-RevId: 191912892 --- tensorflow_serving/resources/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/resources/BUILD b/tensorflow_serving/resources/BUILD index 8a3a4dc886c..9b4fea20935 100644 --- a/tensorflow_serving/resources/BUILD +++ b/tensorflow_serving/resources/BUILD @@ -46,7 +46,6 @@ cc_library( deps = [ ":resources_proto", "@org_tensorflow//tensorflow/core:lib", - "@protobuf_archive//:wrappers", ], ) From 902747e1dad43bce59c9c946ca8f9763914dde64 Mon Sep 17 00:00:00 2001 From: Michael Case Date: Fri, 6 Apr 2018 13:18:01 -0700 Subject: [PATCH 0459/8554] Remove C++11 raw string from resource_util_test The C++11 raw strings might have been causing issues in opensource build. My suspicion is that maybe the test MACROs dont work with raw strings. PiperOrigin-RevId: 191930346 --- .../resources/resource_util_test.cc | 323 +++++++++--------- 1 file changed, 163 insertions(+), 160 deletions(-) diff --git a/tensorflow_serving/resources/resource_util_test.cc b/tensorflow_serving/resources/resource_util_test.cc index 1f1cb7a6712..9e3ab9e6f7f 100644 --- a/tensorflow_serving/resources/resource_util_test.cc +++ b/tensorflow_serving/resources/resource_util_test.cc @@ -1500,172 +1500,175 @@ TEST_F(ResourceUtilTest, MaxEmpty) { } TEST_F(ResourceUtilTest, MaxBound) { - const auto lhs = CreateProto(R"( - resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'processing' - } - quantity: 100 - } - resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'ram' - } - quantity: 8 - } )"); - const auto rhs = CreateProto(R"( - resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'processing' - } - quantity: 300 - } - resource_quantities { - resource { - device: 'gpu' - device_instance { value: 0 } - kind: 'ram' - } - quantity: 4 - } )"); - EXPECT_THAT(util_.Max(lhs, rhs), EqualsProto(R"(resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'processing' - } - quantity: 300 - } - resource_quantities { - resource { - device: 'gpu' - device_instance { value: 0 } - kind: 'ram' - } - quantity: 4 - } - resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'ram' - } - quantity: 8 - })")); + const auto lhs = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 100 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 8 " + "} "); + const auto rhs = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 300 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 4 " + "} "); + EXPECT_THAT(util_.Max(lhs, rhs), + EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 300 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 4 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 8 " + "} ")); } TEST_F(ResourceUtilTest, MaxBoundAndUnbound) { - const auto lhs = CreateProto(R"( - resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'processing' - } - quantity: 100 - } - resource_quantities { - resource { - device: 'gpu' - kind: 'ram' - } - quantity: 8 - } )"); - const auto rhs = CreateProto(R"( - resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'processing' - } - quantity: 300 - } - resource_quantities { - resource { - device: 'gpu' - device_instance { value: 0 } - kind: 'ram' - } - quantity: 4 - } )"); - EXPECT_THAT(util_.Max(lhs, rhs), EqualsProto(R"(resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'processing' - } - quantity: 300 - } - resource_quantities { - resource { - device: 'gpu' - device_instance { value: 0 } - kind: 'ram' - } - quantity: 4 - } - resource_quantities { - resource { - device: 'gpu' - kind: 'ram' - } - quantity: 8 - })")); + const auto lhs = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 100 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " kind: 'ram' " + " } " + " quantity: 8 " + "} "); + const auto rhs = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 300 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 4 " + "} "); + EXPECT_THAT(util_.Max(lhs, rhs), + EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 300 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 4 " + "} " + "resource_quantities { " + " resource { " + " device: 'gpu' " + " kind: 'ram' " + " } " + " quantity: 8 " + "} ")); } TEST_F(ResourceUtilTest, MaxUnbound) { - const auto lhs = CreateProto(R"( - resource_quantities { - resource { - device: 'main' - kind: 'processing' - } - quantity: 100 - } - resource_quantities { - resource { - device: 'main' - kind: 'ram' - } - quantity: 8 - } )"); - const auto rhs = CreateProto(R"( - resource_quantities { - resource { - device: 'main' - kind: 'processing' - } - quantity: 300 - } - resource_quantities { - resource { - device: 'main' - kind: 'ram' - } - quantity: 4 - } )"); - EXPECT_THAT(util_.Max(lhs, rhs), EqualsProto(R"(resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'processing' - } - quantity: 300 - } - resource_quantities { - resource { - device: 'main' - device_instance { value: 0 } - kind: 'ram' - } - quantity: 8 - })")); + const auto lhs = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'processing' " + " } " + " quantity: 100 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'ram' " + " } " + " quantity: 8 " + "} "); + const auto rhs = CreateProto( + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'processing' " + " } " + " quantity: 300 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " kind: 'ram' " + " } " + " quantity: 4 " + "} "); + EXPECT_THAT(util_.Max(lhs, rhs), + EqualsProto("resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'processing' " + " } " + " quantity: 300 " + "} " + "resource_quantities { " + " resource { " + " device: 'main' " + " device_instance { value: 0 } " + " kind: 'ram' " + " } " + " quantity: 8 " + "} ")); } } // namespace } // namespace serving From d89ffc84bb27723641e411b15962d469a9482e58 Mon Sep 17 00:00:00 2001 From: Christina Sorokin Date: Fri, 6 Apr 2018 16:05:11 -0700 Subject: [PATCH 0460/8554] Internal change. PiperOrigin-RevId: 191954237 --- tensorflow_serving/model_servers/BUILD | 1 + tensorflow_serving/model_servers/main.cc | 7 +- tensorflow_serving/servables/tensorflow/BUILD | 95 +++- .../tensorflow/classification_service.cc | 18 +- .../servables/tensorflow/classifier.cc | 21 + .../servables/tensorflow/classifier.h | 8 + .../servables/tensorflow/classifier_test.cc | 350 +++++++++++++- .../servables/tensorflow/multi_inference.cc | 24 +- .../servables/tensorflow/multi_inference.h | 14 +- .../tensorflow/multi_inference_helper.cc | 51 +++ .../tensorflow/multi_inference_helper.h | 37 ++ .../tensorflow/multi_inference_helper_test.cc | 289 ++++++++++++ .../tensorflow/multi_inference_test.cc | 82 ++++ .../servables/tensorflow/predict_impl.cc | 182 +------- .../servables/tensorflow/predict_util.cc | 177 ++++++++ .../servables/tensorflow/predict_util.h | 38 ++ .../servables/tensorflow/predict_util_test.cc | 428 ++++++++++++++++++ .../tensorflow/regression_service.cc | 18 +- .../servables/tensorflow/regressor.cc | 21 + .../servables/tensorflow/regressor.h | 8 + .../servables/tensorflow/regressor_test.cc | 182 +++++++- 21 files changed, 1815 insertions(+), 236 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/multi_inference_helper.cc create mode 100644 tensorflow_serving/servables/tensorflow/multi_inference_helper.h create mode 100644 tensorflow_serving/servables/tensorflow/multi_inference_helper_test.cc create mode 100644 tensorflow_serving/servables/tensorflow/predict_util.cc create mode 100644 tensorflow_serving/servables/tensorflow/predict_util.h create mode 100644 tensorflow_serving/servables/tensorflow/predict_util_test.cc diff --git a/tensorflow_serving/model_servers/BUILD b/tensorflow_serving/model_servers/BUILD index b5f9f83f996..92a29e70319 100644 --- a/tensorflow_serving/model_servers/BUILD +++ b/tensorflow_serving/model_servers/BUILD @@ -217,6 +217,7 @@ cc_binary( "//tensorflow_serving/apis:prediction_service_proto", "//tensorflow_serving/config:model_server_config_proto", "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/servables/tensorflow:multi_inference_helper", "@grpc//:grpc++_unsecure", ] + TENSORFLOW_DEPS + SUPPORTED_TENSORFLOW_OPS, ) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index f04248d11a0..f45ea986624 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -77,7 +77,7 @@ limitations under the License. #include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/servables/tensorflow/classification_service.h" #include "tensorflow_serving/servables/tensorflow/get_model_metadata_impl.h" -#include "tensorflow_serving/servables/tensorflow/multi_inference.h" +#include "tensorflow_serving/servables/tensorflow/multi_inference_helper.h" #include "tensorflow_serving/servables/tensorflow/predict_impl.h" #include "tensorflow_serving/servables/tensorflow/regression_service.h" #include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" @@ -250,8 +250,9 @@ class PredictionServiceImpl final : public PredictionService::Service { // By default, this is infinite which is the same default as RunOptions. run_options.set_timeout_in_ms( DeadlineToTimeoutMillis(context->raw_deadline())); - const grpc::Status status = tensorflow::serving::ToGRPCStatus( - RunMultiInference(run_options, core_, *request, response)); + const grpc::Status status = + tensorflow::serving::ToGRPCStatus(RunMultiInferenceWithServerCore( + run_options, core_, *request, response)); if (!status.ok()) { VLOG(1) << "MultiInference request failed: " << status.error_message(); } diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 90166d71c58..d5770744daa 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -422,11 +422,23 @@ cc_library( "//visibility:public", ], deps = [ + ":predict_util", "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/core:servable_handle", "//tensorflow_serving/model_servers:server_core", "//tensorflow_serving/servables/tensorflow:util", "@org_tensorflow//tensorflow/cc/saved_model:loader", + ], +) + +cc_library( + name = "predict_util", + srcs = ["predict_util.cc"], + hdrs = ["predict_util.h"], + deps = [ + "//tensorflow_serving/apis:predict_proto", + "//tensorflow_serving/servables/tensorflow:util", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle", "@org_tensorflow//tensorflow/contrib/session_bundle:signature", @@ -435,6 +447,39 @@ cc_library( ], ) +cc_test( + name = "predict_util_test", + size = "medium", + srcs = ["predict_util_test.cc"], + data = [ + "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export", + "//tensorflow_serving/servables/tensorflow/testdata:bad_half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.index", + "//tensorflow_serving/servables/tensorflow/testdata:half_plus_two/00000123/export.meta", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_counter/00000123/saved_model.pb", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_counter/00000123/variables/variables.data-00000-of-00001", + "//tensorflow_serving/servables/tensorflow/testdata:saved_model_counter/00000123/variables/variables.index", + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + ], + deps = [ + ":predict_util", + ":saved_model_bundle_source_adapter_proto", + ":session_bundle_config_proto", + ":session_bundle_source_adapter_proto", + "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", + "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/contrib/session_bundle", + "@org_tensorflow//tensorflow/core:test", + ], +) + cc_library( name = "get_model_metadata_impl", srcs = ["get_model_metadata_impl.cc"], @@ -534,6 +579,7 @@ cc_library( "//tensorflow_serving/apis:classifier", "//tensorflow_serving/apis:input_proto", "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/cc/saved_model:loader_lite", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_lite", @@ -665,6 +711,7 @@ cc_library( "//tensorflow_serving/apis:model_proto", "//tensorflow_serving/apis:regression_proto", "//tensorflow_serving/apis:regressor", + "//tensorflow_serving/util:optional", "@org_tensorflow//tensorflow/cc/saved_model:loader_lite", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/contrib/session_bundle:session_bundle_lite", @@ -709,7 +756,6 @@ cc_library( "//tensorflow_serving/apis:inference_proto", "//tensorflow_serving/apis:input_proto", "//tensorflow_serving/apis:model_proto", - "//tensorflow_serving/model_servers:server_core", "//tensorflow_serving/servables/tensorflow:classifier", "//tensorflow_serving/servables/tensorflow:regressor", "//tensorflow_serving/servables/tensorflow:util", @@ -747,6 +793,53 @@ cc_test( ], ) +cc_library( + name = "multi_inference_helper", + srcs = ["multi_inference_helper.cc"], + hdrs = ["multi_inference_helper.h"], + visibility = [ + "//visibility:public", + ], + deps = [ + ":multi_inference", + "//tensorflow_serving/apis:inference_proto", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/model_servers:server_core", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@protobuf_archive//:protobuf", + ], +) + +cc_test( + name = "multi_inference_helper_test", + size = "medium", + srcs = ["multi_inference_helper_test.cc"], + data = [ + "@org_tensorflow//tensorflow/cc/saved_model:saved_model_half_plus_two", + ], + deps = [ + ":multi_inference_helper", + "//tensorflow_serving/apis:classification_proto", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:regression_proto", + "//tensorflow_serving/core:availability_preserving_policy", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/model_servers:model_platform_types", + "//tensorflow_serving/model_servers:platform_config_util", + "//tensorflow_serving/model_servers:server_core", + "//tensorflow_serving/servables/tensorflow:session_bundle_config_proto", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:loader", + "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:test", + ], +) + cc_library( name = "util", srcs = ["util.cc"], diff --git a/tensorflow_serving/servables/tensorflow/classification_service.cc b/tensorflow_serving/servables/tensorflow/classification_service.cc index 9f27897c08e..9d377c194d8 100644 --- a/tensorflow_serving/servables/tensorflow/classification_service.cc +++ b/tensorflow_serving/servables/tensorflow/classification_service.cc @@ -42,21 +42,9 @@ Status TensorflowClassificationServiceImpl::Classify( ServableHandle saved_model_bundle; TF_RETURN_IF_ERROR( core->GetServableHandle(request.model_spec(), &saved_model_bundle)); - SignatureDef signature; - TF_RETURN_IF_ERROR(GetClassificationSignatureDef( - request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); - - std::unique_ptr classifier_interface; - TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( - run_options, saved_model_bundle->session.get(), &signature, - &classifier_interface)); - - MakeModelSpec( - request.model_spec().name(), request.model_spec().signature_name(), - saved_model_bundle.id().version, response->mutable_model_spec()); - - // Run classification. - return classifier_interface->Classify(request, response->mutable_result()); + return RunClassify(run_options, saved_model_bundle->meta_graph_def, + saved_model_bundle.id().version, + saved_model_bundle->session.get(), request, response); } } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/classifier.cc b/tensorflow_serving/servables/tensorflow/classifier.cc index 17fd7d4fdf1..616840e07eb 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.cc +++ b/tensorflow_serving/servables/tensorflow/classifier.cc @@ -444,5 +444,26 @@ Status PostProcessClassificationResult( return Status::OK(); } +Status RunClassify(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, Session* session, + const ClassificationRequest& request, + ClassificationResponse* response) { + SignatureDef signature; + TF_RETURN_IF_ERROR(GetClassificationSignatureDef(request.model_spec(), + meta_graph_def, &signature)); + + std::unique_ptr classifier_interface; + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowClassifier( + run_options, session, &signature, &classifier_interface)); + + MakeModelSpec(request.model_spec().name(), + request.model_spec().signature_name(), servable_version, + response->mutable_model_spec()); + + // Run classification. + return classifier_interface->Classify(request, response->mutable_result()); +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/classifier.h b/tensorflow_serving/servables/tensorflow/classifier.h index 14d796daa36..14b203b7500 100644 --- a/tensorflow_serving/servables/tensorflow/classifier.h +++ b/tensorflow_serving/servables/tensorflow/classifier.h @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/apis/classifier.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -84,6 +85,13 @@ Status PostProcessClassificationResult( const std::vector& output_tensor_names, const std::vector& output_tensors, ClassificationResult* result); +// Creates SavedModelTensorflowClassifier and runs Classification on it. +Status RunClassify(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, Session* session, + const ClassificationRequest& request, + ClassificationResponse* response); + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/classifier_test.cc b/tensorflow_serving/servables/tensorflow/classifier_test.cc index 173543ab677..05f564f6d66 100644 --- a/tensorflow_serving/servables/tensorflow/classifier_test.cc +++ b/tensorflow_serving/servables/tensorflow/classifier_test.cc @@ -314,6 +314,12 @@ class ClassifierTest : public ::testing::TestWithParam { } } + RunOptions GetRunOptions() const { + RunOptions run_options; + run_options.set_timeout_in_ms(42); + return run_options; + } + // Variables used to create the classifier. tensorflow::MetaGraphDef* meta_graph_def_; FakeSession* fake_session_; @@ -325,13 +331,6 @@ class ClassifierTest : public ::testing::TestWithParam { // Convenience variables. ClassificationRequest request_; ClassificationResult result_; - - private: - RunOptions GetRunOptions() const { - RunOptions run_options; - run_options.set_timeout_in_ms(42); - return run_options; - } }; TEST_P(ClassifierTest, ExampleList) { @@ -361,6 +360,35 @@ TEST_P(ClassifierTest, ExampleList) { " score: 3 " " } " " } ")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + TF_ASSERT_OK(RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " score: 4 " + " } " + " classes { " + " label: 'tres' " + " score: 3 " + " } " + " } ")); + } } TEST_P(ClassifierTest, ExampleListWithContext) { @@ -393,6 +421,36 @@ TEST_P(ClassifierTest, ExampleListWithContext) { " score: 1 " " } " " } ")); + + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + TF_ASSERT_OK(RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " } ")); + } } TEST_P(ClassifierTest, ExampleListWithContext_DuplicateFeatures) { @@ -427,6 +485,36 @@ TEST_P(ClassifierTest, ExampleListWithContext_DuplicateFeatures) { " score: 4 " " } " " } ")); + + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + TF_ASSERT_OK(RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" classifications { " + " classes { " + " label: 'uno' " + " score: 1 " + " } " + " classes { " + " label: 'dos' " + " score: 2 " + " } " + " } " + " classifications { " + " classes { " + " label: 'tres' " + " score: 3 " + " } " + " classes { " + " label: 'cuatro' " + " score: 4 " + " } " + " } ")); + } } TEST_P(ClassifierTest, ClassesOnly) { @@ -459,6 +547,31 @@ TEST_P(ClassifierTest, ClassesOnly) { " label: 'tres' " " } " " } ")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + TF_ASSERT_OK(RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " } " + " classes { " + " label: 'uno' " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " } " + " classes { " + " label: 'tres' " + " } " + " } ")); + } } TEST_P(ClassifierTest, ScoresOnly) { @@ -491,6 +604,31 @@ TEST_P(ClassifierTest, ScoresOnly) { " score: 3 " " } " " } ")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + TF_ASSERT_OK(RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" classifications { " + " classes { " + " score: 2 " + " } " + " classes { " + " score: 1 " + " } " + " } " + " classifications { " + " classes { " + " score: 4 " + " } " + " classes { " + " score: 3 " + " } " + " } ")); + } } TEST_P(ClassifierTest, ZeroScoresArePresent) { @@ -516,6 +654,24 @@ TEST_P(ClassifierTest, ZeroScoresArePresent) { for (int i = 0; i < 3; ++i) { EXPECT_NEAR(classification.classes(i).score(), expected_outputs[i], 1e-7); } + + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + TF_ASSERT_OK(RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + // Parse the protos and compare the results with expected scores. + ASSERT_EQ(response.result().classifications_size(), 1); + auto& classification = result_.classifications(0); + ASSERT_EQ(classification.classes_size(), 3); + + for (int i = 0; i < 3; ++i) { + EXPECT_NEAR(classification.classes(i).score(), expected_outputs[i], 1e-7); + } + } } TEST_P(ClassifierTest, ValidNamedSignature) { @@ -573,6 +729,35 @@ TEST_P(ClassifierTest, ValidNamedSignature) { " } " " } ")); } + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + TF_ASSERT_OK(RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" classifications { " + " classes { " + " label: 'dos' " + " score: 3 " + " } " + " classes { " + " label: 'uno' " + " score: 2 " + " } " + " } " + " classifications { " + " classes { " + " label: 'cuatro' " + " score: 5 " + " } " + " classes { " + " label: 'tres' " + " score: 4 " + " } " + " } ")); + } } TEST_P(ClassifierTest, InvalidNamedSignature) { @@ -613,6 +798,18 @@ TEST_P(ClassifierTest, InvalidNamedSignature) { " } " " } ")); } + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } } TEST_P(ClassifierTest, MalformedScores) { @@ -633,6 +830,18 @@ TEST_P(ClassifierTest, MalformedScores) { ASSERT_FALSE(status.ok()); EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } } TEST_P(ClassifierTest, MissingClassificationSignature) { @@ -655,6 +864,18 @@ TEST_P(ClassifierTest, MissingClassificationSignature) { EXPECT_EQ(::tensorflow::error::FAILED_PRECONDITION, status.code()) << status; } + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } } TEST_P(ClassifierTest, EmptyInput) { @@ -665,6 +886,19 @@ TEST_P(ClassifierTest, EmptyInput) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid argument: Input is empty")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Invalid argument: Input is empty")); + } } TEST_P(ClassifierTest, EmptyExampleList) { @@ -675,6 +909,19 @@ TEST_P(ClassifierTest, EmptyExampleList) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid argument: Input is empty")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Invalid argument: Input is empty")); + } } TEST_P(ClassifierTest, EmptyExampleListWithContext) { @@ -687,6 +934,19 @@ TEST_P(ClassifierTest, EmptyExampleListWithContext) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid argument: Input is empty")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Invalid argument: Input is empty")); + } } TEST_P(ClassifierTest, RunsFails) { @@ -708,6 +968,18 @@ TEST_P(ClassifierTest, RunsFails) { const Status status = classifier_->Classify(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Run totally failed")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Run totally failed")); + } } TEST_P(ClassifierTest, ClassesIncorrectTensorBatchSize) { @@ -735,6 +1007,18 @@ TEST_P(ClassifierTest, ClassesIncorrectTensorBatchSize) { const Status status = classifier_->Classify(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("batch size")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("batch size")); + } } TEST_P(ClassifierTest, ClassesIncorrectTensorType) { @@ -763,6 +1047,19 @@ TEST_P(ClassifierTest, ClassesIncorrectTensorType) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Expected classes Tensor of DT_STRING")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Expected classes Tensor of DT_STRING")); + } } TEST_P(ClassifierTest, ScoresIncorrectTensorBatchSize) { @@ -790,6 +1087,18 @@ TEST_P(ClassifierTest, ScoresIncorrectTensorBatchSize) { const Status status = classifier_->Classify(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("batch size")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("batch size")); + } } TEST_P(ClassifierTest, ScoresIncorrectTensorType) { @@ -818,6 +1127,19 @@ TEST_P(ClassifierTest, ScoresIncorrectTensorType) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Expected scores Tensor of DT_FLOAT")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Expected scores Tensor of DT_FLOAT")); + } } TEST_P(ClassifierTest, MismatchedNumberOfTensorClasses) { @@ -848,6 +1170,20 @@ TEST_P(ClassifierTest, MismatchedNumberOfTensorClasses) { status.ToString(), ::testing::HasSubstr( "Tensors class and score should match in dim_size(1). Got 2 vs. 3")); + // Test RunClassify + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_ASSERT_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + ClassificationResponse response; + const Status status = + RunClassify(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Tensors class and score should match in " + "dim_size(1). Got 2 vs. 3")); + } } // Test all ClassifierTest test cases with both SessionBundle and SavedModel. diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.cc b/tensorflow_serving/servables/tensorflow/multi_inference.cc index 1a21c151072..d29d8383479 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference.cc @@ -129,27 +129,15 @@ Status TensorFlowMultiInferenceRunner::Infer( return Status::OK(); } -namespace { - -const ModelSpec& GetModelSpecFromRequest(const MultiInferenceRequest& request) { - if (request.tasks_size() > 0 && request.tasks(0).has_model_spec()) { - return request.tasks(0).model_spec(); - } - return ModelSpec::default_instance(); -} - -} // namespace - -Status RunMultiInference(const RunOptions& run_options, ServerCore* core, - const MultiInferenceRequest& request, +Status RunMultiInference(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, + Session* session, const MultiInferenceRequest& request, MultiInferenceResponse* response) { TRACELITERAL("RunMultiInference"); - ServableHandle bundle; - TF_RETURN_IF_ERROR( - core->GetServableHandle(GetModelSpecFromRequest(request), &bundle)); - TensorFlowMultiInferenceRunner inference_runner( - bundle->session.get(), &bundle->meta_graph_def, bundle.id().version); + TensorFlowMultiInferenceRunner inference_runner(session, &meta_graph_def, + servable_version); return inference_runner.Infer(run_options, request, response); } diff --git a/tensorflow_serving/servables/tensorflow/multi_inference.h b/tensorflow_serving/servables/tensorflow/multi_inference.h index c8a42ea8df3..28e11914483 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference.h +++ b/tensorflow_serving/servables/tensorflow/multi_inference.h @@ -13,13 +13,12 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef LEARNING_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ -#define LEARNING_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/apis/inference.pb.h" -#include "tensorflow_serving/model_servers/server_core.h" #include "tensorflow_serving/util/optional.h" namespace tensorflow { @@ -57,11 +56,14 @@ class TensorFlowMultiInferenceRunner { const optional servable_version_; }; -Status RunMultiInference(const RunOptions& run_options, ServerCore* core, - const MultiInferenceRequest& request, +// Creates TensorFlowMultiInferenceRunner and calls Infer on it. +Status RunMultiInference(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, + Session* session, const MultiInferenceRequest& request, MultiInferenceResponse* response); } // namespace serving } // namespace tensorflow -#endif // LEARNING_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_H_ diff --git a/tensorflow_serving/servables/tensorflow/multi_inference_helper.cc b/tensorflow_serving/servables/tensorflow/multi_inference_helper.cc new file mode 100644 index 00000000000..a2beaaa00b2 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/multi_inference_helper.cc @@ -0,0 +1,51 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/multi_inference_helper.h" + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/servables/tensorflow/multi_inference.h" + +namespace tensorflow { +namespace serving { + +namespace { + +const ModelSpec& GetModelSpecFromRequest(const MultiInferenceRequest& request) { + if (request.tasks_size() > 0 && request.tasks(0).has_model_spec()) { + return request.tasks(0).model_spec(); + } + return ModelSpec::default_instance(); +} + +} // namespace + +Status RunMultiInferenceWithServerCore(const RunOptions& run_options, + ServerCore* core, + const MultiInferenceRequest& request, + MultiInferenceResponse* response) { + ServableHandle bundle; + TF_RETURN_IF_ERROR( + core->GetServableHandle(GetModelSpecFromRequest(request), &bundle)); + + return RunMultiInference(run_options, bundle->meta_graph_def, + bundle.id().version, bundle->session.get(), + request, response); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/multi_inference_helper.h b/tensorflow_serving/servables/tensorflow/multi_inference_helper.h new file mode 100644 index 00000000000..729ca8aae9d --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/multi_inference_helper.h @@ -0,0 +1,37 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_HELPER_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_HELPER_H_ + +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/inference.pb.h" +#include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/util/optional.h" + +namespace tensorflow { +namespace serving { + +// Runs MultiInference +Status RunMultiInferenceWithServerCore(const RunOptions& run_options, + ServerCore* core, + const MultiInferenceRequest& request, + MultiInferenceResponse* response); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_MULTI_INFERENCE_HELPER_H_ diff --git a/tensorflow_serving/servables/tensorflow/multi_inference_helper_test.cc b/tensorflow_serving/servables/tensorflow/multi_inference_helper_test.cc new file mode 100644 index 00000000000..a925bc62683 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/multi_inference_helper_test.cc @@ -0,0 +1,289 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/multi_inference_helper.h" + +#include +#include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/example/feature.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/apis/classification.pb.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/regression.pb.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; + +class MultiInferenceTest : public ::testing::Test { + public: + static void SetUpTestCase() { TF_ASSERT_OK(CreateServerCore(&server_core_)); } + + static void TearDownTestCase() { server_core_.reset(); } + + protected: + static Status CreateServerCore(std::unique_ptr* server_core) { + ModelServerConfig config; + auto model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two")); + + model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + options.model_server_config = config; + options.platform_config_map = CreateTensorFlowPlatformConfigMap( + SessionBundleConfig(), true /* use_saved_model */); + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + return ServerCore::Create(std::move(options), server_core); + } + + ServerCore* GetServerCore() { return server_core_.get(); } + + const int64 servable_version_ = kTestModelVersion; + + private: + static std::unique_ptr server_core_; +}; + +std::unique_ptr MultiInferenceTest::server_core_; + +//////////////////////////////////////////////////////////////////////////////// +// Test Helpers + +void AddInput(const std::vector>& feature_kv, + MultiInferenceRequest* request) { + auto* example = + request->mutable_input()->mutable_example_list()->add_examples(); + auto* features = example->mutable_features()->mutable_feature(); + for (const auto& feature : feature_kv) { + (*features)[feature.first].mutable_float_list()->add_value(feature.second); + } +} + +void PopulateTask(const string& signature_name, const string& method_name, + int64 version, InferenceTask* task) { + ModelSpec model_spec; + model_spec.set_name(kTestModelName); + if (version > 0) { + model_spec.mutable_version()->set_value(version); + } + model_spec.set_signature_name(signature_name); + *task->mutable_model_spec() = model_spec; + task->set_method_name(method_name); +} + +void ExpectStatusError(const Status& status, + const tensorflow::error::Code expected_code, + const string& message_substring) { + ASSERT_EQ(expected_code, status.code()); + EXPECT_THAT(status.error_message(), ::testing::HasSubstr(message_substring)); +} + +//////////////////////////////////////////////////////////////////////////////// +// Tests + +TEST_F(MultiInferenceTest, MissingInputTest) { + MultiInferenceRequest request; + PopulateTask("regress_x_to_y", kRegressMethodName, -1, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(RunMultiInferenceWithServerCore(RunOptions(), + GetServerCore(), + request, &response), + tensorflow::error::INVALID_ARGUMENT, "Input is empty"); +} + +TEST_F(MultiInferenceTest, UndefinedSignatureTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("ThisSignatureDoesNotExist", kRegressMethodName, -1, + request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(RunMultiInferenceWithServerCore(RunOptions(), + GetServerCore(), + request, &response), + tensorflow::error::INVALID_ARGUMENT, "signature not found"); +} + +// Two ModelSpecs, accessing different models. +TEST_F(MultiInferenceTest, InconsistentModelSpecsInRequestTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + // Valid signature. + PopulateTask("regress_x_to_y", kRegressMethodName, -1, request.add_tasks()); + + // Add invalid Task to request. + ModelSpec model_spec; + model_spec.set_name("ModelDoesNotExist"); + model_spec.set_signature_name("regress_x_to_y"); + auto* task = request.add_tasks(); + *task->mutable_model_spec() = model_spec; + task->set_method_name(kRegressMethodName); + + MultiInferenceResponse response; + ExpectStatusError(RunMultiInferenceWithServerCore(RunOptions(), + GetServerCore(), + request, &response), + tensorflow::error::INVALID_ARGUMENT, + "must access the same model name"); +} + +TEST_F(MultiInferenceTest, EvaluateDuplicateSignaturesTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, -1, request.add_tasks()); + // Add the same task again (error). + PopulateTask("regress_x_to_y", kRegressMethodName, -1, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(RunMultiInferenceWithServerCore(RunOptions(), + GetServerCore(), + request, &response), + tensorflow::error::INVALID_ARGUMENT, + "Duplicate evaluation of signature: regress_x_to_y"); +} + +TEST_F(MultiInferenceTest, UsupportedSignatureTypeTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("serving_default", kPredictMethodName, -1, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(RunMultiInferenceWithServerCore(RunOptions(), + GetServerCore(), + request, &response), + tensorflow::error::UNIMPLEMENTED, "Unsupported signature"); +} + +TEST_F(MultiInferenceTest, SignaturesWithDifferentInputsTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}, {"x2", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, -1, request.add_tasks()); + PopulateTask("regress_x2_to_y3", kRegressMethodName, -1, request.add_tasks()); + + MultiInferenceResponse response; + ExpectStatusError(RunMultiInferenceWithServerCore(RunOptions(), + GetServerCore(), + request, &response), + tensorflow::error::INVALID_ARGUMENT, + "Input tensor must be the same"); +} + +TEST_F(MultiInferenceTest, ValidSingleSignatureTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, servable_version_, + request.add_tasks()); + + MultiInferenceResponse expected_response; + auto* inference_result = expected_response.add_results(); + auto* model_spec = inference_result->mutable_model_spec(); + *model_spec = request.tasks(0).model_spec(); + model_spec->mutable_version()->set_value(servable_version_); + auto* regression_result = inference_result->mutable_regression_result(); + regression_result->add_regressions()->set_value(3.0); + + MultiInferenceResponse response; + TF_ASSERT_OK(RunMultiInferenceWithServerCore(RunOptions(), GetServerCore(), + request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +TEST_F(MultiInferenceTest, MultipleValidRegressSignaturesTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, servable_version_, + request.add_tasks()); + PopulateTask("regress_x_to_y2", kRegressMethodName, servable_version_, + request.add_tasks()); + + MultiInferenceResponse expected_response; + + // regress_x_to_y is y = 0.5x + 2. + auto* inference_result_1 = expected_response.add_results(); + auto* model_spec_1 = inference_result_1->mutable_model_spec(); + *model_spec_1 = request.tasks(0).model_spec(); + model_spec_1->mutable_version()->set_value(servable_version_); + auto* regression_result_1 = inference_result_1->mutable_regression_result(); + regression_result_1->add_regressions()->set_value(3.0); + + // regress_x_to_y2 is y2 = 0.5x + 3. + auto* inference_result_2 = expected_response.add_results(); + auto* model_spec_2 = inference_result_2->mutable_model_spec(); + *model_spec_2 = request.tasks(1).model_spec(); + model_spec_2->mutable_version()->set_value(servable_version_); + auto* regression_result_2 = inference_result_2->mutable_regression_result(); + regression_result_2->add_regressions()->set_value(4.0); + + MultiInferenceResponse response; + TF_ASSERT_OK(RunMultiInferenceWithServerCore(RunOptions(), GetServerCore(), + request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +TEST_F(MultiInferenceTest, RegressAndClassifySignaturesTest) { + MultiInferenceRequest request; + AddInput({{"x", 2}}, &request); + PopulateTask("regress_x_to_y", kRegressMethodName, servable_version_, + request.add_tasks()); + PopulateTask("classify_x_to_y", kClassifyMethodName, servable_version_, + request.add_tasks()); + + MultiInferenceResponse expected_response; + auto* inference_result_1 = expected_response.add_results(); + auto* model_spec_1 = inference_result_1->mutable_model_spec(); + *model_spec_1 = request.tasks(0).model_spec(); + model_spec_1->mutable_version()->set_value(servable_version_); + auto* regression_result = inference_result_1->mutable_regression_result(); + regression_result->add_regressions()->set_value(3.0); + + auto* inference_result_2 = expected_response.add_results(); + auto* model_spec_2 = inference_result_2->mutable_model_spec(); + *model_spec_2 = request.tasks(1).model_spec(); + model_spec_2->mutable_version()->set_value(servable_version_); + auto* classification_result = + inference_result_2->mutable_classification_result(); + classification_result->add_classifications()->add_classes()->set_score(3.0); + + MultiInferenceResponse response; + TF_ASSERT_OK(RunMultiInferenceWithServerCore(RunOptions(), GetServerCore(), + request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/multi_inference_test.cc b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc index 00123d1b586..d4070f8e848 100644 --- a/tensorflow_serving/servables/tensorflow/multi_inference_test.cc +++ b/tensorflow_serving/servables/tensorflow/multi_inference_test.cc @@ -83,6 +83,12 @@ class MultiInferenceTest : public ::testing::Test { return Status::OK(); } + Status GetServableHandle(ServableHandle* bundle) { + ModelSpec model_spec; + model_spec.set_name(kTestModelName); + return GetServerCore()->GetServableHandle(model_spec, bundle); + } + const int64 servable_version_ = 1; private: @@ -133,6 +139,14 @@ TEST_F(MultiInferenceTest, MissingInputTest) { MultiInferenceResponse response; ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "Input is empty"); + + // MultiInference testing + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + ExpectStatusError( + RunMultiInference(RunOptions(), bundle->meta_graph_def, servable_version_, + bundle->session.get(), request, &response), + tensorflow::error::INVALID_ARGUMENT, "Input is empty"); } TEST_F(MultiInferenceTest, UndefinedSignatureTest) { @@ -147,6 +161,14 @@ TEST_F(MultiInferenceTest, UndefinedSignatureTest) { MultiInferenceResponse response; ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "signature not found"); + + // MultiInference testing + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + ExpectStatusError( + RunMultiInference(RunOptions(), bundle->meta_graph_def, servable_version_, + bundle->session.get(), request, &response), + tensorflow::error::INVALID_ARGUMENT, "signature not found"); } // Two ModelSpecs, accessing different models. @@ -171,6 +193,14 @@ TEST_F(MultiInferenceTest, InconsistentModelSpecsInRequestTest) { ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "must access the same model name"); + + // MultiInference testing + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + ExpectStatusError( + RunMultiInference(RunOptions(), bundle->meta_graph_def, servable_version_, + bundle->session.get(), request, &response), + tensorflow::error::INVALID_ARGUMENT, "must access the same model name"); } TEST_F(MultiInferenceTest, EvaluateDuplicateSignaturesTest) { @@ -187,6 +217,15 @@ TEST_F(MultiInferenceTest, EvaluateDuplicateSignaturesTest) { ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "Duplicate evaluation of signature: regress_x_to_y"); + + // MultiInference testing + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + ExpectStatusError( + RunMultiInference(RunOptions(), bundle->meta_graph_def, servable_version_, + bundle->session.get(), request, &response), + tensorflow::error::INVALID_ARGUMENT, + "Duplicate evaluation of signature: regress_x_to_y"); } TEST_F(MultiInferenceTest, UsupportedSignatureTypeTest) { @@ -200,6 +239,14 @@ TEST_F(MultiInferenceTest, UsupportedSignatureTypeTest) { MultiInferenceResponse response; ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::UNIMPLEMENTED, "Unsupported signature"); + + // MultiInference testing + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + ExpectStatusError( + RunMultiInference(RunOptions(), bundle->meta_graph_def, servable_version_, + bundle->session.get(), request, &response), + tensorflow::error::UNIMPLEMENTED, "Unsupported signature"); } TEST_F(MultiInferenceTest, SignaturesWithDifferentInputsTest) { @@ -215,6 +262,14 @@ TEST_F(MultiInferenceTest, SignaturesWithDifferentInputsTest) { ExpectStatusError(inference_runner->Infer(RunOptions(), request, &response), tensorflow::error::INVALID_ARGUMENT, "Input tensor must be the same"); + + // MultiInference testing + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + ExpectStatusError( + RunMultiInference(RunOptions(), bundle->meta_graph_def, servable_version_, + bundle->session.get(), request, &response), + tensorflow::error::INVALID_ARGUMENT, "Input tensor must be the same"); } TEST_F(MultiInferenceTest, ValidSingleSignatureTest) { @@ -236,6 +291,15 @@ TEST_F(MultiInferenceTest, ValidSingleSignatureTest) { MultiInferenceResponse response; TF_ASSERT_OK(inference_runner->Infer(RunOptions(), request, &response)); EXPECT_THAT(response, test_util::EqualsProto(expected_response)); + + // MultiInference testing + response.Clear(); + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + TF_ASSERT_OK(RunMultiInference(RunOptions(), bundle->meta_graph_def, + servable_version_, bundle->session.get(), + request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } TEST_F(MultiInferenceTest, MultipleValidRegressSignaturesTest) { @@ -268,6 +332,15 @@ TEST_F(MultiInferenceTest, MultipleValidRegressSignaturesTest) { MultiInferenceResponse response; TF_ASSERT_OK(inference_runner->Infer(RunOptions(), request, &response)); EXPECT_THAT(response, test_util::EqualsProto(expected_response)); + + // MultiInference testing + response.Clear(); + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + TF_ASSERT_OK(RunMultiInference(RunOptions(), bundle->meta_graph_def, + servable_version_, bundle->session.get(), + request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } TEST_F(MultiInferenceTest, RegressAndClassifySignaturesTest) { @@ -298,6 +371,15 @@ TEST_F(MultiInferenceTest, RegressAndClassifySignaturesTest) { MultiInferenceResponse response; TF_ASSERT_OK(inference_runner->Infer(RunOptions(), request, &response)); EXPECT_THAT(response, test_util::EqualsProto(expected_response)); + + // MultiInference testing + response.Clear(); + ServableHandle bundle; + TF_ASSERT_OK(GetServableHandle(&bundle)); + TF_ASSERT_OK(RunMultiInference(RunOptions(), bundle->meta_graph_def, + servable_version_, bundle->session.get(), + request, &response)); + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); } } // namespace diff --git a/tensorflow_serving/servables/tensorflow/predict_impl.cc b/tensorflow_serving/servables/tensorflow/predict_impl.cc index 4d161ef1191..8c6d7405a97 100644 --- a/tensorflow_serving/servables/tensorflow/predict_impl.cc +++ b/tensorflow_serving/servables/tensorflow/predict_impl.cc @@ -15,44 +15,35 @@ limitations under the License. #include "tensorflow_serving/servables/tensorflow/predict_impl.h" -#include -#include #include #include -#include #include "tensorflow/cc/saved_model/loader.h" -#include "tensorflow/cc/saved_model/signature_constants.h" -#include "tensorflow/contrib/session_bundle/session_bundle.h" -#include "tensorflow/contrib/session_bundle/signature.h" -#include "tensorflow/core/framework/tensor.pb.h" #include "tensorflow/core/lib/core/errors.h" -#include "tensorflow/core/protobuf/named_tensor.pb.h" #include "tensorflow_serving/core/servable_handle.h" +#include "tensorflow_serving/servables/tensorflow/predict_util.h" #include "tensorflow_serving/servables/tensorflow/util.h" namespace tensorflow { namespace serving { + namespace { -// Implementation of Predict using the legacy SessionBundle GenericSignature. -Status SessionBundlePredict(const RunOptions& run_options, ServerCore* core, +Status SessionBundlePredict(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, const PredictRequest& request, - PredictResponse* response) { + PredictResponse* response, Session* session) { // Validate signatures. - ServableHandle bundle; - TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); Signature signature; - TF_RETURN_IF_ERROR( - GetNamedSignature("inputs", bundle->meta_graph_def, &signature)); + TF_RETURN_IF_ERROR(GetNamedSignature("inputs", meta_graph_def, &signature)); if (!signature.has_generic_signature()) { return tensorflow::Status( tensorflow::error::INVALID_ARGUMENT, "'inputs' named signature is not a generic signature"); } GenericSignature input_signature = signature.generic_signature(); - TF_RETURN_IF_ERROR( - GetNamedSignature("outputs", bundle->meta_graph_def, &signature)); + TF_RETURN_IF_ERROR(GetNamedSignature("outputs", meta_graph_def, &signature)); if (!signature.has_generic_signature()) { return tensorflow::Status( tensorflow::error::INVALID_ARGUMENT, @@ -113,13 +104,13 @@ Status SessionBundlePredict(const RunOptions& run_options, ServerCore* core, } MakeModelSpec(request.model_spec().name(), /*signature_name=*/{}, - bundle.id().version, response->mutable_model_spec()); + servable_version, response->mutable_model_spec()); // Run session. std::vector outputs; RunMetadata run_metadata; - TF_RETURN_IF_ERROR(bundle->session->Run( - run_options, inputs, output_tensor_names, {}, &outputs, &run_metadata)); + TF_RETURN_IF_ERROR(session->Run(run_options, inputs, output_tensor_names, {}, + &outputs, &run_metadata)); // Validate and return output. if (outputs.size() != output_tensor_names.size()) { @@ -134,146 +125,6 @@ Status SessionBundlePredict(const RunOptions& run_options, ServerCore* core, return Status::OK(); } -// Returns the keys in the map as a comma delimited string. Useful for debugging -// or when returning error messages. -// e.g. returns "key1, key2, key3". -string MapKeysToString(const google::protobuf::Map& map) { - string result = ""; - for (const auto& i : map) { - if (result.empty()) { - result += i.first; - } else { - result += ", " + i.first; - } - } - return result; -} - -// Validate a SignatureDef to make sure it's compatible with prediction, and -// if so, populate the input and output tensor names. -Status PreProcessPrediction(const SignatureDef& signature, - const PredictRequest& request, - std::vector>* inputs, - std::vector* output_tensor_names, - std::vector* output_tensor_aliases) { - if (signature.method_name() != kPredictMethodName && - signature.method_name() != kClassifyMethodName && - signature.method_name() != kRegressMethodName) { - return errors::Internal(strings::StrCat( - "Expected prediction signature method_name to be one of {", - kPredictMethodName, ", ", kClassifyMethodName, ", ", kRegressMethodName, - "}. Was: ", signature.method_name())); - } - - // Verify and prepare input. - if (request.inputs().size() != signature.inputs().size()) { - return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, - "input size does not match signature"); - } - for (auto& input : request.inputs()) { - const string& alias = input.first; - auto iter = signature.inputs().find(alias); - if (iter == signature.inputs().end()) { - return tensorflow::Status( - tensorflow::error::INVALID_ARGUMENT, - strings::StrCat("input tensor alias not found in signature: ", alias, - ". Inputs expected to be in the set {", - MapKeysToString(signature.inputs()), "}.")); - } - Tensor tensor; - if (!tensor.FromProto(input.second)) { - return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, - "tensor parsing error: " + alias); - } - inputs->emplace_back(std::make_pair(iter->second.name(), tensor)); - } - - // Prepare run target. - std::set seen_outputs; - std::vector output_filter(request.output_filter().begin(), - request.output_filter().end()); - for (auto& alias : output_filter) { - auto iter = signature.outputs().find(alias); - if (iter == signature.outputs().end()) { - return tensorflow::Status( - tensorflow::error::INVALID_ARGUMENT, - strings::StrCat("output tensor alias not found in signature: ", alias, - " Outputs expected to be in the set {", - MapKeysToString(signature.outputs()), "}.")); - } - if (seen_outputs.find(alias) != seen_outputs.end()) { - return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, - "duplicate output tensor alias: " + alias); - } - seen_outputs.insert(alias); - output_tensor_names->emplace_back(iter->second.name()); - output_tensor_aliases->emplace_back(alias); - } - // When no output is specified, fetch all output tensors specified in - // the signature. - if (output_tensor_names->empty()) { - for (auto& iter : signature.outputs()) { - output_tensor_names->emplace_back(iter.second.name()); - output_tensor_aliases->emplace_back(iter.first); - } - } - return Status::OK(); -} - -// Validate results and populate a PredictResponse. -Status PostProcessPredictionResult( - const SignatureDef& signature, - const std::vector& output_tensor_aliases, - const std::vector& output_tensors, PredictResponse* response) { - // Validate and return output. - if (output_tensors.size() != output_tensor_aliases.size()) { - return tensorflow::Status(tensorflow::error::UNKNOWN, - "Predict internal error"); - } - for (int i = 0; i < output_tensors.size(); i++) { - output_tensors[i].AsProtoField( - &((*response->mutable_outputs())[output_tensor_aliases[i]])); - } - return Status::OK(); -} - -// Implementation of Predict using the SavedModel SignatureDef format. -Status SavedModelPredict(const RunOptions& run_options, ServerCore* core, - const PredictRequest& request, - PredictResponse* response) { - // Validate signatures. - ServableHandle bundle; - TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); - - const string signature_name = request.model_spec().signature_name().empty() - ? kDefaultServingSignatureDefKey - : request.model_spec().signature_name(); - auto iter = bundle->meta_graph_def.signature_def().find(signature_name); - if (iter == bundle->meta_graph_def.signature_def().end()) { - return errors::FailedPrecondition(strings::StrCat( - "Serving signature key \"", signature_name, "\" not found.")); - } - SignatureDef signature = iter->second; - - MakeModelSpec(request.model_spec().name(), signature_name, - bundle.id().version, response->mutable_model_spec()); - - std::vector> input_tensors; - std::vector output_tensor_names; - std::vector output_tensor_aliases; - TF_RETURN_IF_ERROR(PreProcessPrediction(signature, request, &input_tensors, - &output_tensor_names, - &output_tensor_aliases)); - std::vector outputs; - RunMetadata run_metadata; - TF_RETURN_IF_ERROR(bundle->session->Run(run_options, input_tensors, - output_tensor_names, {}, &outputs, - &run_metadata)); - - return PostProcessPredictionResult(signature, output_tensor_aliases, outputs, - response); -} - } // namespace Status TensorflowPredictor::Predict(const RunOptions& run_options, @@ -285,9 +136,16 @@ Status TensorflowPredictor::Predict(const RunOptions& run_options, "Missing ModelSpec"); } if (use_saved_model_) { - return SavedModelPredict(run_options, core, request, response); + ServableHandle bundle; + TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); + return RunPredict(run_options, bundle->meta_graph_def, bundle.id().version, + bundle->session.get(), request, response); } - return SessionBundlePredict(run_options, core, request, response); + ServableHandle bundle; + TF_RETURN_IF_ERROR(core->GetServableHandle(request.model_spec(), &bundle)); + return SessionBundlePredict(run_options, bundle->meta_graph_def, + bundle.id().version, request, response, + bundle->session.get()); } } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/predict_util.cc b/tensorflow_serving/servables/tensorflow/predict_util.cc new file mode 100644 index 00000000000..760c968fc7a --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/predict_util.cc @@ -0,0 +1,177 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/predict_util.h" + +#include +#include +#include +#include +#include + +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/contrib/session_bundle/signature.h" +#include "tensorflow/core/framework/tensor.pb.h" +#include "tensorflow/core/lib/core/errors.h" +#include "tensorflow/core/protobuf/named_tensor.pb.h" +#include "tensorflow_serving/servables/tensorflow/util.h" + +namespace tensorflow { +namespace serving { +namespace { + +// Returns the keys in the map as a comma delimited string. Useful for debugging +// or when returning error messages. +// e.g. returns "key1, key2, key3". +string MapKeysToString(const google::protobuf::Map& map) { + string result = ""; + for (const auto& i : map) { + if (result.empty()) { + result += i.first; + } else { + result += ", " + i.first; + } + } + return result; +} + +// Validate a SignatureDef to make sure it's compatible with prediction, and +// if so, populate the input and output tensor names. +Status PreProcessPrediction(const SignatureDef& signature, + const PredictRequest& request, + std::vector>* inputs, + std::vector* output_tensor_names, + std::vector* output_tensor_aliases) { + if (signature.method_name() != kPredictMethodName && + signature.method_name() != kClassifyMethodName && + signature.method_name() != kRegressMethodName) { + return errors::Internal(strings::StrCat( + "Expected prediction signature method_name to be one of {", + kPredictMethodName, ", ", kClassifyMethodName, ", ", kRegressMethodName, + "}. Was: ", signature.method_name())); + } + + // Verify and prepare input. + if (request.inputs().size() != signature.inputs().size()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "input size does not match signature"); + } + for (auto& input : request.inputs()) { + const string& alias = input.first; + auto iter = signature.inputs().find(alias); + if (iter == signature.inputs().end()) { + return tensorflow::Status( + tensorflow::error::INVALID_ARGUMENT, + strings::StrCat("input tensor alias not found in signature: ", alias, + ". Inputs expected to be in the set {", + MapKeysToString(signature.inputs()), "}.")); + } + Tensor tensor; + if (!tensor.FromProto(input.second)) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "tensor parsing error: " + alias); + } + inputs->emplace_back(std::make_pair(iter->second.name(), tensor)); + } + + // Prepare run target. + std::set seen_outputs; + std::vector output_filter(request.output_filter().begin(), + request.output_filter().end()); + for (auto& alias : output_filter) { + auto iter = signature.outputs().find(alias); + if (iter == signature.outputs().end()) { + return tensorflow::Status( + tensorflow::error::INVALID_ARGUMENT, + strings::StrCat("output tensor alias not found in signature: ", alias, + " Outputs expected to be in the set {", + MapKeysToString(signature.outputs()), "}.")); + } + if (seen_outputs.find(alias) != seen_outputs.end()) { + return tensorflow::Status(tensorflow::error::INVALID_ARGUMENT, + "duplicate output tensor alias: " + alias); + } + seen_outputs.insert(alias); + output_tensor_names->emplace_back(iter->second.name()); + output_tensor_aliases->emplace_back(alias); + } + // When no output is specified, fetch all output tensors specified in + // the signature. + if (output_tensor_names->empty()) { + for (auto& iter : signature.outputs()) { + output_tensor_names->emplace_back(iter.second.name()); + output_tensor_aliases->emplace_back(iter.first); + } + } + return Status::OK(); +} + +// Validate results and populate a PredictResponse. +Status PostProcessPredictionResult( + const SignatureDef& signature, + const std::vector& output_tensor_aliases, + const std::vector& output_tensors, PredictResponse* response) { + // Validate and return output. + if (output_tensors.size() != output_tensor_aliases.size()) { + return tensorflow::Status(tensorflow::error::UNKNOWN, + "Predict internal error"); + } + for (int i = 0; i < output_tensors.size(); i++) { + output_tensors[i].AsProtoField( + &((*response->mutable_outputs())[output_tensor_aliases[i]])); + } + return Status::OK(); +} + +} // namespace + +Status RunPredict(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, + Session* session, + const PredictRequest& request, + PredictResponse* response) { + // Validate signatures. + const string signature_name = request.model_spec().signature_name().empty() + ? kDefaultServingSignatureDefKey + : request.model_spec().signature_name(); + auto iter = meta_graph_def.signature_def().find(signature_name); + if (iter == meta_graph_def.signature_def().end()) { + return errors::FailedPrecondition(strings::StrCat( + "Serving signature key \"", signature_name, "\" not found.")); + } + SignatureDef signature = iter->second; + + MakeModelSpec(request.model_spec().name(), signature_name, + servable_version, response->mutable_model_spec()); + + std::vector> input_tensors; + std::vector output_tensor_names; + std::vector output_tensor_aliases; + TF_RETURN_IF_ERROR(PreProcessPrediction(signature, request, &input_tensors, + &output_tensor_names, + &output_tensor_aliases)); + std::vector outputs; + RunMetadata run_metadata; + TF_RETURN_IF_ERROR(session->Run(run_options, input_tensors, + output_tensor_names, {}, &outputs, + &run_metadata)); + + return PostProcessPredictionResult(signature, output_tensor_aliases, outputs, + response); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/predict_util.h b/tensorflow_serving/servables/tensorflow/predict_util.h new file mode 100644 index 00000000000..c23be27a3cb --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/predict_util.h @@ -0,0 +1,38 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_PREDICT_UTIL_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_PREDICT_UTIL_H_ + +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status.h" +#include "tensorflow_serving/apis/predict.pb.h" +#include "tensorflow_serving/util/optional.h" + +namespace tensorflow { +namespace serving { + +// Implementation of Predict using the SavedModel SignatureDef format. +Status RunPredict(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, + Session* session, + const PredictRequest& request, + PredictResponse* response); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_PREDICT_UTIL_H_ diff --git a/tensorflow_serving/servables/tensorflow/predict_util_test.cc b/tensorflow_serving/servables/tensorflow/predict_util_test.cc new file mode 100644 index 00000000000..6bea09f10a9 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/predict_util_test.cc @@ -0,0 +1,428 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/predict_util.h" + +#include +#include +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/contrib/session_bundle/session_bundle.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow_serving/core/availability_preserving_policy.h" +#include "tensorflow_serving/model_servers/model_platform_types.h" +#include "tensorflow_serving/model_servers/platform_config_util.h" +#include "tensorflow_serving/model_servers/server_core.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_config.pb.h" +#include "tensorflow_serving/servables/tensorflow/session_bundle_source_adapter.pb.h" +#include "tensorflow_serving/test_util/test_util.h" + +namespace tensorflow { +namespace serving { +namespace { + +constexpr char kTestModelName[] = "test_model"; +constexpr int kTestModelVersion = 123; + +const char kInputTensorKey[] = "x"; +const char kOutputTensorKey[] = "y"; + +// Parameter is 'bool use_saved_model'. +class PredictImplTest : public ::testing::Test { + public: + static void SetUpTestCase() { + const string bad_half_plus_two_path = test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/bad_half_plus_two"); + + TF_ASSERT_OK(CreateServerCore(test_util::TensorflowTestSrcDirPath( + "cc/saved_model/testdata/half_plus_two"), + true, &saved_model_server_core_)); + TF_ASSERT_OK(CreateServerCore(bad_half_plus_two_path, true, + &saved_model_server_core_bad_model_)); + TF_ASSERT_OK(CreateServerCore( + test_util::TestSrcDirPath( + "/servables/tensorflow/testdata/saved_model_counter"), + true, &saved_model_server_core_counter_model_)); + } + + static void TearDownTestCase() { + saved_model_server_core_.reset(); + saved_model_server_core_bad_model_.reset(); + saved_model_server_core_counter_model_.reset(); + } + + protected: + static Status CreateServerCore(const string& model_path, bool use_saved_model, + std::unique_ptr* server_core) { + ModelServerConfig config; + auto model_config = config.mutable_model_config_list()->add_config(); + model_config->set_name(kTestModelName); + model_config->set_base_path(model_path); + model_config->set_model_platform(kTensorFlowModelPlatform); + + // For ServerCore Options, we leave servable_state_monitor_creator + // unspecified so the default servable_state_monitor_creator will be used. + ServerCore::Options options; + options.model_server_config = config; + options.platform_config_map = CreateTensorFlowPlatformConfigMap( + SessionBundleConfig(), use_saved_model); + options.aspired_version_policy = + std::unique_ptr(new AvailabilityPreservingPolicy); + // Reduce the number of initial load threads to be num_load_threads to avoid + // timing out in tests. + options.num_initial_load_threads = options.num_load_threads; + return ServerCore::Create(std::move(options), server_core); + } + + ServerCore* GetServerCore() { + return saved_model_server_core_.get(); + } + + ServerCore* GetServerCoreWithBadModel() { + return saved_model_server_core_bad_model_.get(); + } + + ServerCore* GetServerCoreWithCounterModel() { + return saved_model_server_core_counter_model_.get(); + } + + Status GetSavedModelServableHandle(ServerCore* server_core, + ServableHandle* bundle) { + ModelSpec model_spec; + model_spec.set_name(kTestModelName); + return server_core->GetServableHandle(model_spec, bundle); + } + + Status CallPredict(ServerCore* server_core, + const PredictRequest& request, PredictResponse* response) { + ServableHandle bundle; + TF_RETURN_IF_ERROR(GetSavedModelServableHandle(server_core, &bundle)); + return RunPredict(GetRunOptions(), + bundle->meta_graph_def, + kTestModelVersion, bundle->session.get(), + request, response); + } + + RunOptions GetRunOptions() { return RunOptions(); } + + private: + static std::unique_ptr saved_model_server_core_; + static std::unique_ptr saved_model_server_core_bad_model_; + static std::unique_ptr saved_model_server_core_counter_model_; +}; + +std::unique_ptr PredictImplTest::saved_model_server_core_; +std::unique_ptr PredictImplTest::saved_model_server_core_bad_model_; +std::unique_ptr + PredictImplTest::saved_model_server_core_counter_model_; + +TEST_F(PredictImplTest, MissingOrEmptyModelSpec) { + PredictRequest request; + PredictResponse response; + + // Empty request is invalid. + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->clear_name(); + + // Model name is not specified. + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); + + // Model name is wrong. + model_spec->set_name("test"); + EXPECT_EQ(tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); +} + +TEST_F(PredictImplTest, EmptyInputList) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + // The input is empty. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); +} + +TEST_F(PredictImplTest, InputTensorsDontMatchModelSpecInputs) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + TensorProto tensor_proto; + tensor_proto.add_string_val("any_key"); + tensor_proto.set_dtype(tensorflow::DT_STRING); + tensor_proto.mutable_tensor_shape()->add_dim()->set_size(1); + + auto inputs = request.mutable_inputs(); + (*inputs)["key"] = tensor_proto; + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); +} + +TEST_F(PredictImplTest, OutputFiltersDontMatchModelSpecOutputs) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())[kInputTensorKey] = tensor_proto; + request.add_output_filter("output_filter"); + + // Output filter like this doesn't exist. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); + + request.clear_output_filter(); + request.add_output_filter(kOutputTensorKey); + TF_EXPECT_OK(CallPredict(GetServerCore(), request, &response)); + request.add_output_filter(kOutputTensorKey); + + // Duplicate output filter specified. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); +} + +TEST_F(PredictImplTest, InputTensorsHaveWrongType) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + TensorProto tensor_proto; + tensor_proto.add_string_val("any_key"); + tensor_proto.set_dtype(tensorflow::DT_STRING); + tensor_proto.mutable_tensor_shape()->add_dim()->set_size(1); + (*request.mutable_inputs())[kInputTensorKey] = tensor_proto; + request.add_output_filter(kOutputTensorKey); + + // Input tensors are all wrong. + EXPECT_EQ( + tensorflow::error::INVALID_ARGUMENT, + CallPredict(GetServerCore(), request, &response).code()); +} + +TEST_F(PredictImplTest, ModelMissingSignatures) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + // Model is missing signatures. + EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, + CallPredict(GetServerCoreWithBadModel(), + request, &response).code()); +} + +TEST_F(PredictImplTest, PredictionSuccess) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())[kInputTensorKey] = tensor_proto; + + TF_EXPECT_OK(CallPredict(GetServerCore(), request, &response)); + TensorProto output_tensor_proto; + output_tensor_proto.add_float_val(3); + output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); + output_tensor_proto.mutable_tensor_shape(); + PredictResponse expected_response; + *expected_response.mutable_model_spec() = *model_spec; + expected_response.mutable_model_spec()->set_signature_name( + kDefaultServingSignatureDefKey); + (*expected_response.mutable_outputs())[kOutputTensorKey] = + output_tensor_proto; + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +// Test querying a model with a named regression signature (not default). This +TEST_F(PredictImplTest, PredictionWithNamedRegressionSignature) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + model_spec->set_signature_name("regress_x2_to_y3"); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())[kRegressInputs] = tensor_proto; + TF_ASSERT_OK(CallPredict(GetServerCore(), request, &response)); + TensorProto output_tensor_proto; + output_tensor_proto.add_float_val(4); + output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); + output_tensor_proto.mutable_tensor_shape(); + PredictResponse expected_response; + *expected_response.mutable_model_spec() = *model_spec; + (*expected_response.mutable_outputs())[kRegressOutputs] = output_tensor_proto; + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +// Test querying a model with a classification signature. Predict calls work +// with predict, classify, and regress signatures. +TEST_F(PredictImplTest, PredictionWithNamedClassificationSignature) { + PredictRequest request; + PredictResponse response; + + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + model_spec->set_signature_name("classify_x2_to_y3"); + + TensorProto tensor_proto; + tensor_proto.add_float_val(2.0); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())[kClassifyInputs] = tensor_proto; + + TF_ASSERT_OK(CallPredict(GetServerCore(), request, &response)); + TensorProto output_tensor_proto; + output_tensor_proto.add_float_val(4); + output_tensor_proto.set_dtype(tensorflow::DT_FLOAT); + output_tensor_proto.mutable_tensor_shape(); + PredictResponse expected_response; + *expected_response.mutable_model_spec() = *model_spec; + (*expected_response.mutable_outputs())[kClassifyOutputScores] = + output_tensor_proto; + EXPECT_THAT(response, test_util::EqualsProto(expected_response)); +} + +// Test querying a counter model with signatures. Predict calls work with +// customized signatures. It calls get_counter, incr_counter, +// reset_counter, incr_counter, and incr_counter_by(3) in order. +// +// *Notes*: These signatures are stateful and over-simplied only to demonstrate +// Predict calls with only inputs or outputs. State is not supported in +// TensorFlow Serving on most scalable or production hosting environments. +TEST_F(PredictImplTest, PredictionWithCustomizedSignatures) { + PredictRequest request; + PredictResponse response; + + // Call get_counter. Expected result 0. + ModelSpec* model_spec = request.mutable_model_spec(); + model_spec->set_name(kTestModelName); + model_spec->mutable_version()->set_value(kTestModelVersion); + model_spec->set_signature_name("get_counter"); + + TF_ASSERT_OK(CallPredict(GetServerCoreWithCounterModel(), + request, &response)); + + PredictResponse expected_get_counter; + *expected_get_counter.mutable_model_spec() = *model_spec; + TensorProto output_get_counter; + output_get_counter.add_float_val(0); + output_get_counter.set_dtype(tensorflow::DT_FLOAT); + output_get_counter.mutable_tensor_shape(); + (*expected_get_counter.mutable_outputs())["output"] = output_get_counter; + EXPECT_THAT(response, test_util::EqualsProto(expected_get_counter)); + + // Call incr_counter. Expect: 1. + model_spec->set_signature_name("incr_counter"); + TF_ASSERT_OK(CallPredict(GetServerCoreWithCounterModel(), + request, &response)); + + PredictResponse expected_incr_counter; + *expected_incr_counter.mutable_model_spec() = *model_spec; + TensorProto output_incr_counter; + output_incr_counter.add_float_val(1); + output_incr_counter.set_dtype(tensorflow::DT_FLOAT); + output_incr_counter.mutable_tensor_shape(); + (*expected_incr_counter.mutable_outputs())["output"] = output_incr_counter; + EXPECT_THAT(response, test_util::EqualsProto(expected_incr_counter)); + + // Call reset_counter. Expect: 0. + model_spec->set_signature_name("reset_counter"); + TF_ASSERT_OK(CallPredict(GetServerCoreWithCounterModel(), + request, &response)); + + PredictResponse expected_reset_counter; + *expected_reset_counter.mutable_model_spec() = *model_spec; + TensorProto output_reset_counter; + output_reset_counter.add_float_val(0); + output_reset_counter.set_dtype(tensorflow::DT_FLOAT); + output_reset_counter.mutable_tensor_shape(); + (*expected_reset_counter.mutable_outputs())["output"] = output_reset_counter; + EXPECT_THAT(response, test_util::EqualsProto(expected_reset_counter)); + + // Call incr_counter. Expect: 1. + model_spec->set_signature_name("incr_counter"); + request.add_output_filter("output"); + TF_ASSERT_OK(CallPredict(GetServerCoreWithCounterModel(), + request, &response)); + request.clear_output_filter(); + + PredictResponse expected_incr_counter2; + *expected_incr_counter2.mutable_model_spec() = *model_spec; + TensorProto output_incr_counter2; + output_incr_counter2.add_float_val(1); + output_incr_counter2.set_dtype(tensorflow::DT_FLOAT); + output_incr_counter2.mutable_tensor_shape(); + (*expected_incr_counter2.mutable_outputs())["output"] = output_incr_counter2; + EXPECT_THAT(response, test_util::EqualsProto(expected_incr_counter2)); + + // Call incr_counter_by. Expect: 4. + model_spec->set_signature_name("incr_counter_by"); + TensorProto tensor_proto; + tensor_proto.add_float_val(3); + tensor_proto.set_dtype(tensorflow::DT_FLOAT); + (*request.mutable_inputs())["delta"] = tensor_proto; + + TF_ASSERT_OK(CallPredict(GetServerCoreWithCounterModel(), + request, &response)); + + PredictResponse expected_incr_counter_by; + *expected_incr_counter_by.mutable_model_spec() = *model_spec; + TensorProto output_incr_counter_by; + output_incr_counter_by.add_float_val(4); + output_incr_counter_by.set_dtype(tensorflow::DT_FLOAT); + output_incr_counter_by.mutable_tensor_shape(); + (*expected_incr_counter_by.mutable_outputs())["output"] = + output_incr_counter_by; + EXPECT_THAT(response, test_util::EqualsProto(expected_incr_counter_by)); +} + +} // namespace +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/regression_service.cc b/tensorflow_serving/servables/tensorflow/regression_service.cc index 9241317fc88..5602daeb787 100644 --- a/tensorflow_serving/servables/tensorflow/regression_service.cc +++ b/tensorflow_serving/servables/tensorflow/regression_service.cc @@ -40,21 +40,9 @@ Status TensorflowRegressionServiceImpl::Regress( ServableHandle saved_model_bundle; TF_RETURN_IF_ERROR( core->GetServableHandle(request.model_spec(), &saved_model_bundle)); - SignatureDef signature; - TF_RETURN_IF_ERROR(GetRegressionSignatureDef( - request.model_spec(), saved_model_bundle->meta_graph_def, &signature)); - - std::unique_ptr regressor_interface; - TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( - run_options, saved_model_bundle->session.get(), &signature, - ®ressor_interface)); - - MakeModelSpec( - request.model_spec().name(), request.model_spec().signature_name(), - saved_model_bundle.id().version, response->mutable_model_spec()); - - // Run regression - return regressor_interface->Regress(request, response->mutable_result()); + return RunRegress(run_options, saved_model_bundle->meta_graph_def, + saved_model_bundle.id().version, + saved_model_bundle->session.get(), request, response); } } // namespace serving diff --git a/tensorflow_serving/servables/tensorflow/regressor.cc b/tensorflow_serving/servables/tensorflow/regressor.cc index b7e0493a376..47e535ea34b 100644 --- a/tensorflow_serving/servables/tensorflow/regressor.cc +++ b/tensorflow_serving/servables/tensorflow/regressor.cc @@ -326,5 +326,26 @@ Status PostProcessRegressionResult( return Status::OK(); } +Status RunRegress(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, Session* session, + const RegressionRequest& request, + RegressionResponse* response) { + SignatureDef signature; + TF_RETURN_IF_ERROR(GetRegressionSignatureDef(request.model_spec(), + meta_graph_def, &signature)); + + std::unique_ptr regressor_interface; + TF_RETURN_IF_ERROR(CreateFlyweightTensorFlowRegressor( + run_options, session, &signature, ®ressor_interface)); + + MakeModelSpec(request.model_spec().name(), + request.model_spec().signature_name(), servable_version, + response->mutable_model_spec()); + + // Run regression + return regressor_interface->Regress(request, response->mutable_result()); +} + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/regressor.h b/tensorflow_serving/servables/tensorflow/regressor.h index a1d560778e0..fbcab3dd528 100644 --- a/tensorflow_serving/servables/tensorflow/regressor.h +++ b/tensorflow_serving/servables/tensorflow/regressor.h @@ -24,6 +24,7 @@ limitations under the License. #include "tensorflow/contrib/session_bundle/session_bundle.h" #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/apis/regressor.h" +#include "tensorflow_serving/util/optional.h" namespace tensorflow { namespace serving { @@ -83,6 +84,13 @@ Status PostProcessRegressionResult( const std::vector& output_tensor_names, const std::vector& output_tensors, RegressionResult* result); +// Creates SavedModelTensorflowRegressor and runs Regression on it. +Status RunRegress(const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, + const optional& servable_version, Session* session, + const RegressionRequest& request, + RegressionResponse* response); + } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/regressor_test.cc b/tensorflow_serving/servables/tensorflow/regressor_test.cc index d9cb5219b05..98bf2a5eb5a 100644 --- a/tensorflow_serving/servables/tensorflow/regressor_test.cc +++ b/tensorflow_serving/servables/tensorflow/regressor_test.cc @@ -239,6 +239,10 @@ class RegressorTest : public ::testing::TestWithParam { tensorflow::serving::SetSignatures(signatures, meta_graph_def_)); } + // Whether or not to use SavedModel for this test. Simply wraps GetParam() + // with a more meaningful name. + bool UseSavedModel() { return GetParam(); } + protected: // Return an example with the feature "output" = [output]. Example example_with_output(const float output) { @@ -250,7 +254,7 @@ class RegressorTest : public ::testing::TestWithParam { } Status Create() { - if (GetParam()) { + if (UseSavedModel()) { std::unique_ptr saved_model(new SavedModelBundle); TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( *bundle_, saved_model.get())); @@ -261,6 +265,12 @@ class RegressorTest : public ::testing::TestWithParam { } } + RunOptions GetRunOptions() const { + RunOptions run_options; + run_options.set_timeout_in_ms(42); + return run_options; + } + // Variables used to create the regression model tensorflow::MetaGraphDef* meta_graph_def_; FakeSession* fake_session_; @@ -272,13 +282,6 @@ class RegressorTest : public ::testing::TestWithParam { // Convenience variables. RegressionRequest request_; RegressionResult result_; - - private: - RunOptions GetRunOptions() const { - RunOptions run_options; - run_options.set_timeout_in_ms(42); - return run_options; - } }; TEST_P(RegressorTest, BasicExampleList) { @@ -294,6 +297,22 @@ TEST_P(RegressorTest, BasicExampleList) { " regressions { " " value: 3.0 " " } ")); + + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + TF_ASSERT_OK(RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" regressions { " + " value: 2.0 " + " } " + " regressions { " + " value: 3.0 " + " } ")); + } } TEST_P(RegressorTest, BasicExampleListWithContext) { @@ -312,6 +331,21 @@ TEST_P(RegressorTest, BasicExampleListWithContext) { " regressions { " " value: 3.0 " " } ")); + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + TF_ASSERT_OK(RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" regressions { " + " value: 3.0 " + " } " + " regressions { " + " value: 3.0 " + " } ")); + } } TEST_P(RegressorTest, ValidNamedSignature) { @@ -341,6 +375,21 @@ TEST_P(RegressorTest, ValidNamedSignature) { " value: 3.0 " " } ")); } + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + TF_ASSERT_OK(RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response)); + EXPECT_THAT(response.result(), EqualsProto(" regressions { " + " value: 3.0 " + " } " + " regressions { " + " value: 4.0 " + " } ")); + } } TEST_P(RegressorTest, InvalidNamedSignature) { @@ -368,6 +417,18 @@ TEST_P(RegressorTest, InvalidNamedSignature) { " value: 3.0 " " } ")); } + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } } TEST_P(RegressorTest, MalformedOutputs) { @@ -384,10 +445,19 @@ TEST_P(RegressorTest, MalformedOutputs) { request_.mutable_input()->mutable_example_list()->mutable_examples(); *examples->Add() = example_with_output(2.0); *examples->Add() = example_with_output(3.0); - const Status status = regressor_->Regress(request_, &result_); + Status status = regressor_->Regress(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + // Test RunRegress + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + status = RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; } TEST_P(RegressorTest, EmptyInput) { @@ -398,6 +468,19 @@ TEST_P(RegressorTest, EmptyInput) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid argument: Input is empty")); + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Invalid argument: Input is empty")); + } } TEST_P(RegressorTest, EmptyExampleList) { @@ -407,6 +490,19 @@ TEST_P(RegressorTest, EmptyExampleList) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid argument: Input is empty")); + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Invalid argument: Input is empty")); + } } TEST_P(RegressorTest, EmptyExampleListWithContext) { @@ -419,6 +515,19 @@ TEST_P(RegressorTest, EmptyExampleListWithContext) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Invalid argument: Input is empty")); + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Invalid argument: Input is empty")); + } } TEST_P(RegressorTest, RunsFails) { @@ -439,6 +548,18 @@ TEST_P(RegressorTest, RunsFails) { const Status status = regressor_->Regress(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Run totally failed")); + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Run totally failed")); + } } TEST_P(RegressorTest, UnexpectedOutputTensorSize) { @@ -460,6 +581,21 @@ TEST_P(RegressorTest, UnexpectedOutputTensorSize) { const Status status = regressor_->Regress(request_, &result_); ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("output batch size")); + // Test RunRegress + if (UseSavedModel()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("output batch size")); + } } TEST_P(RegressorTest, UnexpectedOutputTensorType) { @@ -483,6 +619,22 @@ TEST_P(RegressorTest, UnexpectedOutputTensorType) { ASSERT_FALSE(status.ok()); EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Expected output Tensor of DT_FLOAT")); + // Test RunRegress + if (UseSavedModel()) { + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillOnce(::testing::DoAll(::testing::SetArgPointee<4>(outputs), + ::testing::Return(Status::OK()))); + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, mock, + request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Expected output Tensor of DT_FLOAT")); + } } TEST_P(RegressorTest, MissingRegressionSignature) { @@ -508,6 +660,18 @@ TEST_P(RegressorTest, MissingRegressionSignature) { EXPECT_EQ(::tensorflow::error::FAILED_PRECONDITION, status.code()) << status; } + // Test RunRegress + if (UseSavedModel()) { + std::unique_ptr saved_model(new SavedModelBundle); + TF_CHECK_OK(internal::ConvertSessionBundleToSavedModelBundle( + *bundle_, saved_model.get())); + RegressionResponse response; + const Status status = + RunRegress(GetRunOptions(), saved_model->meta_graph_def, {}, + fake_session_, request_, &response); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + } } // Test all RegressorTest test cases with both SessionBundle and SavedModel. From a92f2922e690f5ec87cc62b5f30fa855f24d8a2c Mon Sep 17 00:00:00 2001 From: Christina Sorokin Date: Fri, 6 Apr 2018 18:59:46 -0700 Subject: [PATCH 0461/8554] Internal change. PiperOrigin-RevId: 191970161 --- tensorflow_serving/servables/tensorflow/BUILD | 45 +++ .../saved_model_bundle_source_adapter.cc | 9 +- .../saved_model_bundle_source_adapter_test.cc | 34 +- .../tensorflow/saved_model_warmup.cc | 127 ++++++ .../servables/tensorflow/saved_model_warmup.h | 45 +++ .../tensorflow/saved_model_warmup_test.cc | 361 ++++++++++++++++++ .../tensorflow/session_bundle_config.proto | 3 + 7 files changed, 610 insertions(+), 14 deletions(-) create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_warmup.cc create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_warmup.h create mode 100644 tensorflow_serving/servables/tensorflow/saved_model_warmup_test.cc diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index d5770744daa..45f1d97ba7b 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -261,6 +261,7 @@ cc_library( deps = [ ":saved_model_bundle_factory", ":saved_model_bundle_source_adapter_proto", + ":saved_model_warmup", ":session_bundle_source_adapter_proto", "//tensorflow_serving/core:loader", "//tensorflow_serving/core:simple_loader", @@ -878,3 +879,47 @@ cc_test( "@protobuf_archive//:cc_wkt_protos", ], ) + +cc_library( + name = "saved_model_warmup", + srcs = ["saved_model_warmup.cc"], + hdrs = ["saved_model_warmup.h"], + deps = [ + ":classifier", + ":multi_inference", + ":predict_util", + ":regressor", + "//tensorflow_serving/apis:classification_proto", + "//tensorflow_serving/apis:inference_proto", + "//tensorflow_serving/apis:predict_proto", + "//tensorflow_serving/apis:prediction_log_proto", + "//tensorflow_serving/apis:regression_proto", + "//tensorflow_serving/core:loader", + "@org_tensorflow//tensorflow/cc/saved_model:constants", + "@org_tensorflow//tensorflow/core:core_cpu", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + ], +) + +cc_test( + name = "saved_model_warmup_test", + size = "small", + srcs = ["saved_model_warmup_test.cc"], + deps = [ + ":saved_model_warmup", + "//tensorflow_serving/apis:input_proto", + "//tensorflow_serving/apis:model_proto", + "//tensorflow_serving/apis:prediction_log_proto", + "//tensorflow_serving/core/test_util:mock_session", + "//tensorflow_serving/core/test_util:test_main", + "//tensorflow_serving/test_util", + "@org_tensorflow//tensorflow/cc/saved_model:constants", + "@org_tensorflow//tensorflow/contrib/session_bundle:manifest_proto_cc", + "@org_tensorflow//tensorflow/core:framework", + "@org_tensorflow//tensorflow/core:lib", + "@org_tensorflow//tensorflow/core:protos_all_cc", + "@org_tensorflow//tensorflow/core:test", + "@org_tensorflow//tensorflow/core:testlib", + ], +) diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc index 505219cdc58..d64ad34719f 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter.cc @@ -24,6 +24,8 @@ limitations under the License. #include "tensorflow_serving/resources/resource_util.h" #include "tensorflow_serving/resources/resource_values.h" #include "tensorflow_serving/resources/resources.pb.h" +#include "tensorflow_serving/servables/tensorflow/bundle_factory_util.h" +#include "tensorflow_serving/servables/tensorflow/saved_model_warmup.h" #include "tensorflow_serving/util/optional.h" namespace tensorflow { @@ -50,7 +52,12 @@ Status SavedModelBundleSourceAdapter::Convert(const StoragePath& path, std::shared_ptr bundle_factory = bundle_factory_; auto servable_creator = [bundle_factory, path](std::unique_ptr* bundle) { - return bundle_factory->CreateSavedModelBundle(path, bundle); + TF_RETURN_IF_ERROR(bundle_factory->CreateSavedModelBundle(path, bundle)); + if (bundle_factory->config().enable_model_warmup()) { + return RunSavedModelWarmup(GetRunOptions(bundle_factory->config()), path, + bundle->get()); + } + return Status::OK(); }; auto resource_estimator = [bundle_factory, path](ResourceAllocation* estimate) { diff --git a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc index 261b78e4eda..7d1770b7744 100644 --- a/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc +++ b/tensorflow_serving/servables/tensorflow/saved_model_bundle_source_adapter_test.cc @@ -41,7 +41,8 @@ namespace { using test_util::EqualsProto; -class SavedModelBundleSourceAdapterTest : public ::testing::Test { +class SavedModelBundleSourceAdapterTest + : public ::testing::TestWithParam { protected: SavedModelBundleSourceAdapterTest() { ResourceUtil::Options resource_util_options; @@ -51,15 +52,14 @@ class SavedModelBundleSourceAdapterTest : public ::testing::Test { ram_resource_ = resource_util_->CreateBoundResource( device_types::kMain, resource_kinds::kRamBytes); + config_.mutable_config()->set_enable_model_warmup(EnableWarmup()); } - void TestSavedModelBundleSourceAdapter( - const SessionBundleSourceAdapterConfig& config, - const string& export_dir) const { + void TestSavedModelBundleSourceAdapter(const string& export_dir) const { std::unique_ptr loader; { std::unique_ptr adapter; - TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(config, &adapter)); + TF_CHECK_OK(SavedModelBundleSourceAdapter::Create(config_, &adapter)); ServableData> loader_data = adapter->AdaptOneVersion( ServableData({"", 0}, export_dir)); @@ -87,7 +87,7 @@ class SavedModelBundleSourceAdapterTest : public ::testing::Test { resource_util_->SetQuantity( ram_resource_, resource_util_->GetQuantity(ram_resource_, first_resource_estimate) - - config.config().experimental_transient_ram_bytes_during_load(), + config_.config().experimental_transient_ram_bytes_during_load(), &expected_post_load_resource_estimate); ResourceAllocation actual_post_load_resource_estimate; TF_ASSERT_OK( @@ -101,22 +101,30 @@ class SavedModelBundleSourceAdapterTest : public ::testing::Test { loader->Unload(); } + bool EnableWarmup() { return GetParam(); } + std::unique_ptr resource_util_; Resource ram_resource_; + SessionBundleSourceAdapterConfig config_; }; -TEST_F(SavedModelBundleSourceAdapterTest, Basic) { - SessionBundleSourceAdapterConfig config; - config.mutable_config()->set_experimental_transient_ram_bytes_during_load(42); - TestSavedModelBundleSourceAdapter(config, test_util::GetTestSavedModelPath()); +TEST_P(SavedModelBundleSourceAdapterTest, Basic) { + config_.mutable_config()->set_experimental_transient_ram_bytes_during_load( + 42); + + TestSavedModelBundleSourceAdapter(test_util::GetTestSavedModelPath()); } -TEST_F(SavedModelBundleSourceAdapterTest, BackwardCompatibility) { - const SessionBundleSourceAdapterConfig config; +TEST_P(SavedModelBundleSourceAdapterTest, BackwardCompatibility) { TestSavedModelBundleSourceAdapter( - config, test_util::GetTestSessionBundleExportPath()); + test_util::GetTestSessionBundleExportPath()); } +// Test all SavedModelBundleSourceAdapterTest test cases with +// warmup enabled/disabled. +INSTANTIATE_TEST_CASE_P(EnableWarmup, SavedModelBundleSourceAdapterTest, + ::testing::Bool()); + } // namespace } // namespace serving } // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_warmup.cc b/tensorflow_serving/servables/tensorflow/saved_model_warmup.cc new file mode 100644 index 00000000000..218868f2da4 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_warmup.cc @@ -0,0 +1,127 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/saved_model_warmup.h" + +#include "tensorflow/cc/saved_model/constants.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/io/record_reader.h" +#include "tensorflow/core/lib/strings/strcat.h" +#include "tensorflow/core/protobuf/config.pb.h" +#include "tensorflow_serving/apis/prediction_log.pb.h" +#include "tensorflow_serving/servables/tensorflow/classifier.h" +#include "tensorflow_serving/servables/tensorflow/multi_inference.h" +#include "tensorflow_serving/servables/tensorflow/predict_util.h" +#include "tensorflow_serving/servables/tensorflow/regressor.h" +#include "tensorflow_serving/servables/tensorflow/util.h" + +namespace tensorflow { +namespace serving { + +namespace { + +Status RunWarmupRequest(const PredictionLog& warmup_record, + const RunOptions& run_options, + const MetaGraphDef& meta_graph_def, Session* session) { + switch (warmup_record.log_type_case()) { + case PredictionLog::kRegressLog: { + RegressionResponse response; + TF_RETURN_IF_ERROR(RunRegress(run_options, meta_graph_def, {}, session, + warmup_record.regress_log().request(), + &response)); + } break; + case PredictionLog::kClassifyLog: { + ClassificationResponse response; + TF_RETURN_IF_ERROR(RunClassify(run_options, meta_graph_def, {}, session, + warmup_record.classify_log().request(), + &response)); + } break; + case PredictionLog::kPredictLog: { + PredictResponse response; + TF_RETURN_IF_ERROR(RunPredict(run_options, meta_graph_def, {}, session, + warmup_record.predict_log().request(), + &response)); + } break; + case PredictionLog::kMultiInferenceLog: { + MultiInferenceResponse response; + TF_RETURN_IF_ERROR(RunMultiInference( + run_options, meta_graph_def, {}, session, + warmup_record.multi_inference_log().request(), &response)); + } break; + case PredictionLog::kSessionRunLog: + return errors::Unimplemented(strings::StrCat( + "Unsupported log_type for warmup: ", warmup_record.log_type_case())); + default: + break; + } + return Status::OK(); +} + +} // namespace + +constexpr char WarmupConsts::kRequestsFileName[]; +constexpr int WarmupConsts::kMaxNumRecords; + +Status RunSavedModelWarmup(const RunOptions& run_options, + const string& export_dir, SavedModelBundle* bundle) { + const string warmup_path = + io::JoinPath(export_dir, kSavedModelAssetsExtraDirectory, + WarmupConsts::kRequestsFileName); + if (!tensorflow::Env::Default()->FilesExist({warmup_path}, nullptr)) { + LOG(INFO) << "No warmup data file found at " << warmup_path; + // Having warmup data is optional, return OK + return Status::OK(); + } + + std::unique_ptr tf_record_file; + TF_RETURN_IF_ERROR(tensorflow::Env::Default()->NewRandomAccessFile( + warmup_path, &tf_record_file)); + + std::unique_ptr tf_record_file_reader; + tf_record_file_reader.reset( + new tensorflow::io::SequentialRecordReader(tf_record_file.get())); + int num_warmup_records = 0; + string record; + Status status = tf_record_file_reader->ReadRecord(&record); + tensorflow::serving::PredictionLog prediction_log; + while (status.ok()) { + if (!prediction_log.ParseFromString(record)) { + return errors::InvalidArgument(strings::StrCat( + "Failed to parse warmup record: ", record, " from ", warmup_path)); + } + + TF_RETURN_IF_ERROR(RunWarmupRequest(prediction_log, run_options, + bundle->meta_graph_def, + bundle->session.get())); + ++num_warmup_records; + if (num_warmup_records > WarmupConsts::kMaxNumRecords) { + return errors::InvalidArgument( + "Number of warmup records exceeeds the maximum (", + WarmupConsts::kMaxNumRecords, ") at ", warmup_path); + } + status = tf_record_file_reader->ReadRecord(&record); + } + // OUT_OF_RANGE error means EOF was reached, do not return error in this case + if (!errors::IsOutOfRange(status)) { + return status; + } + + LOG(INFO) << "Finished reading warmup data for model at " << warmup_path + << ". Number of warmup records read: " << num_warmup_records << "."; + return Status::OK(); +} + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/saved_model_warmup.h b/tensorflow_serving/servables/tensorflow/saved_model_warmup.h new file mode 100644 index 00000000000..6709a19396d --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_warmup.h @@ -0,0 +1,45 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#ifndef TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_WARMUP_H_ +#define TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_WARMUP_H_ + +#include + +#include "tensorflow/cc/saved_model/loader.h" +#include "tensorflow/core/protobuf/saved_model.pb.h" +#include "tensorflow/core/public/session.h" + +namespace tensorflow { +namespace serving { + +struct WarmupConsts { + static constexpr char kRequestsFileName[] = "tf_serving_warmup_requests"; + static constexpr int kMaxNumRecords = 1000; +}; + +// Reads sample warmup requests from assets.extra/tf_serving_warmup_requests +// file (if exists) and invokes them one by one on the given saved_model_bundle, +// to trigger lazy initializations (such as TF optimizations, XLA compilations) +// at load time, and consequently improve first request latency. +// Warmup is skipped if no warmup file present. +// Supported request types: Regress, Classify, Predict, MultiInference. +Status RunSavedModelWarmup(const RunOptions& run_options, + const string& export_dir, SavedModelBundle* bundle); + +} // namespace serving +} // namespace tensorflow + +#endif // TENSORFLOW_SERVING_SERVABLES_TENSORFLOW_SAVED_MODEL_WARMUP_H_ diff --git a/tensorflow_serving/servables/tensorflow/saved_model_warmup_test.cc b/tensorflow_serving/servables/tensorflow/saved_model_warmup_test.cc new file mode 100644 index 00000000000..5b99db28f93 --- /dev/null +++ b/tensorflow_serving/servables/tensorflow/saved_model_warmup_test.cc @@ -0,0 +1,361 @@ +/* Copyright 2018 Google Inc. All Rights Reserved. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +==============================================================================*/ + +#include "tensorflow_serving/servables/tensorflow/saved_model_warmup.h" + +#include +#include + +#include "tensorflow/cc/saved_model/constants.h" +#include "tensorflow/cc/saved_model/signature_constants.h" +#include "tensorflow/contrib/session_bundle/signature.h" +#include "tensorflow/core/example/example.pb.h" +#include "tensorflow/core/example/feature.pb.h" +#include "tensorflow/core/framework/tensor_shape.pb.h" +#include "tensorflow/core/lib/core/status_test_util.h" +#include "tensorflow/core/lib/io/path.h" +#include "tensorflow/core/lib/io/record_writer.h" +#include "tensorflow/core/platform/env.h" +#include "tensorflow/core/platform/test.h" +#include "tensorflow_serving/apis/classification.pb.h" +#include "tensorflow_serving/apis/inference.pb.h" +#include "tensorflow_serving/apis/input.pb.h" +#include "tensorflow_serving/apis/model.pb.h" +#include "tensorflow_serving/apis/predict.pb.h" +#include "tensorflow_serving/apis/prediction_log.pb.h" +#include "tensorflow_serving/apis/regression.pb.h" +#include "tensorflow_serving/core/test_util/mock_session.h" + +namespace tensorflow { +namespace serving { + +namespace { + +using test_util::MockSession; +using ::testing::_; +using ::testing::DoAll; +using ::testing::Return; +using ::testing::SetArgPointee; +using ::testing::SizeIs; + +void PopulateInferenceTask(const string& model_name, + const string& signature_name, + const string& method_name, InferenceTask* task) { + ModelSpec model_spec; + model_spec.set_name(model_name); + model_spec.set_signature_name(signature_name); + *task->mutable_model_spec() = model_spec; + task->set_method_name(method_name); +} + +void PopulateMultiInferenceRequest(MultiInferenceRequest* request) { + request->mutable_input()->mutable_example_list()->add_examples(); + PopulateInferenceTask("test_model", kRegressMethodName, kRegressMethodName, + request->add_tasks()); + PopulateInferenceTask("test_model", kClassifyMethodName, kClassifyMethodName, + request->add_tasks()); +} + +void PopulatePredictRequest(PredictRequest* request) { + request->mutable_model_spec()->set_signature_name(kPredictMethodName); + TensorProto tensor_proto; + tensor_proto.add_string_val("input_value"); + tensor_proto.set_dtype(tensorflow::DT_STRING); + tensor_proto.mutable_tensor_shape()->add_dim()->set_size(1); + (*request->mutable_inputs())[kPredictInputs] = tensor_proto; +} + +void PopulateClassificationRequest(ClassificationRequest* request) { + request->mutable_input()->mutable_example_list()->add_examples(); + request->mutable_model_spec()->set_signature_name(kClassifyMethodName); +} + +void PopulateRegressionRequest(RegressionRequest* request) { + request->mutable_input()->mutable_example_list()->add_examples(); + request->mutable_model_spec()->set_signature_name(kRegressMethodName); +} + +void PopulatePredictionLog(PredictionLog* prediction_log, + PredictionLog::LogTypeCase log_type) { + switch (log_type) { + case PredictionLog::kRegressLog: { + PopulateRegressionRequest( + prediction_log->mutable_regress_log()->mutable_request()); + } break; + case PredictionLog::kClassifyLog: { + PopulateClassificationRequest( + prediction_log->mutable_classify_log()->mutable_request()); + } break; + case PredictionLog::kPredictLog: { + PopulatePredictRequest( + prediction_log->mutable_predict_log()->mutable_request()); + } break; + case PredictionLog::kMultiInferenceLog: { + PopulateMultiInferenceRequest( + prediction_log->mutable_multi_inference_log()->mutable_request()); + } break; + case PredictionLog::kSessionRunLog: + prediction_log->mutable_session_run_log(); + TF_FALLTHROUGH_INTENDED; + default: + return; + } +} + +Status WriteWarmupData(const string& fname, + const std::vector& warmup_records, + int num_warmup_records) { + Env* env = Env::Default(); + std::unique_ptr file; + TF_RETURN_IF_ERROR(env->NewWritableFile(fname, &file)); + + io::RecordWriterOptions options; + io::RecordWriter writer(file.get(), options); + for (int i = 0; i < num_warmup_records; ++i) { + for (const string& warmup_record : warmup_records) { + TF_RETURN_IF_ERROR(writer.WriteRecord(warmup_record)); + } + } + TF_RETURN_IF_ERROR(writer.Flush()); + return Status::OK(); +} + +void AddMixedWarmupData(std::vector* warmup_records) { + for (auto& log_type : + {PredictionLog::kRegressLog, PredictionLog::kClassifyLog, + PredictionLog::kPredictLog, PredictionLog::kMultiInferenceLog}) { + PredictionLog prediction_log; + PopulatePredictionLog(&prediction_log, log_type); + warmup_records->push_back(prediction_log.SerializeAsString()); + } +} + +// Creates a test SignatureDef with the given parameters +SignatureDef CreateSignatureDef(const string& method_name, + const std::vector& input_names, + const std::vector& output_names) { + SignatureDef signature_def; + signature_def.set_method_name(method_name); + for (const string& input_name : input_names) { + TensorInfo input; + input.set_name(input_name); + (*signature_def.mutable_inputs())[input_name] = input; + } + for (const string& output_name : output_names) { + TensorInfo output; + output.set_name(output_name); + (*signature_def.mutable_outputs())[output_name] = output; + } + return signature_def; +} + +void AddSignatures(MetaGraphDef* meta_graph_def) { + (*meta_graph_def->mutable_signature_def())[kRegressMethodName] = + CreateSignatureDef(kRegressMethodName, {kRegressInputs}, + {kRegressOutputs}); + (*meta_graph_def->mutable_signature_def())[kClassifyMethodName] = + CreateSignatureDef(kClassifyMethodName, {kClassifyInputs}, + {kClassifyOutputClasses, kClassifyOutputScores}); + (*meta_graph_def->mutable_signature_def())[kPredictMethodName] = + CreateSignatureDef(kPredictMethodName, {kPredictInputs}, + {kPredictOutputs}); +} + +TEST(SavedModelBundleWarmupTest, MixedWarmupData) { + string base_path = io::JoinPath(testing::TmpDir(), "MixedWarmupData"); + TF_ASSERT_OK(Env::Default()->RecursivelyCreateDir( + io::JoinPath(base_path, kSavedModelAssetsExtraDirectory))); + string fname = io::JoinPath(base_path, kSavedModelAssetsExtraDirectory, + WarmupConsts::kRequestsFileName); + + int num_warmup_records = 10; + std::vector warmup_records; + AddMixedWarmupData(&warmup_records); + TF_ASSERT_OK(WriteWarmupData(fname, warmup_records, num_warmup_records)); + SavedModelBundle saved_model_bundle; + AddSignatures(&saved_model_bundle.meta_graph_def); + MockSession* mock = new MockSession; + saved_model_bundle.session.reset(mock); + Tensor scores(DT_FLOAT, TensorShape({1, 1})); + Tensor classes(DT_STRING, TensorShape({1, 1})); + // Regress and Predict case + EXPECT_CALL(*mock, Run(_, _, SizeIs(1), _, _, _)) + .Times(num_warmup_records * 2) + .WillRepeatedly(DoAll(SetArgPointee<4>(std::vector({scores})), + Return(Status::OK()))); + // Classify case + EXPECT_CALL(*mock, Run(_, _, SizeIs(2), _, _, _)) + .Times(num_warmup_records) + .WillRepeatedly( + DoAll(SetArgPointee<4>(std::vector({classes, scores})), + Return(Status::OK()))); + // MultiInference case + EXPECT_CALL(*mock, Run(_, _, SizeIs(3), _, _, _)) + .Times(num_warmup_records) + .WillRepeatedly(DoAll( + SetArgPointee<4>(std::vector({classes, scores, scores})), + Return(Status::OK()))); + TF_EXPECT_OK( + RunSavedModelWarmup(RunOptions(), base_path, &saved_model_bundle)); +} + +TEST(SavedModelBundleWarmupTest, NoWarmupDataFile) { + string base_path = io::JoinPath(testing::TmpDir(), "NoWarmupDataFile"); + TF_ASSERT_OK(Env::Default()->RecursivelyCreateDir( + io::JoinPath(base_path, kSavedModelAssetsExtraDirectory))); + + SavedModelBundle saved_model_bundle; + AddSignatures(&saved_model_bundle.meta_graph_def); + MockSession* mock = new MockSession; + saved_model_bundle.session.reset(mock); + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)).Times(0); + TF_EXPECT_OK( + RunSavedModelWarmup(RunOptions(), base_path, &saved_model_bundle)); +} + +TEST(SavedModelBundleWarmupTest, WarmupDataFileEmpty) { + string base_path = io::JoinPath(testing::TmpDir(), "WarmupDataFileEmpty"); + TF_ASSERT_OK(Env::Default()->RecursivelyCreateDir( + io::JoinPath(base_path, kSavedModelAssetsExtraDirectory))); + string fname = io::JoinPath(base_path, kSavedModelAssetsExtraDirectory, + WarmupConsts::kRequestsFileName); + + std::vector warmup_records; + TF_ASSERT_OK(WriteWarmupData(fname, warmup_records, 0)); + SavedModelBundle saved_model_bundle; + AddSignatures(&saved_model_bundle.meta_graph_def); + MockSession* mock = new MockSession; + saved_model_bundle.session.reset(mock); + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)).Times(0); + TF_EXPECT_OK( + RunSavedModelWarmup(RunOptions(), base_path, &saved_model_bundle)); +} + +TEST(SavedModelBundleWarmupTest, UnsupportedLogType) { + string base_path = io::JoinPath(testing::TmpDir(), "UnsupportedLogType"); + TF_ASSERT_OK(Env::Default()->RecursivelyCreateDir( + io::JoinPath(base_path, kSavedModelAssetsExtraDirectory))); + string fname = io::JoinPath(base_path, kSavedModelAssetsExtraDirectory, + WarmupConsts::kRequestsFileName); + + std::vector warmup_records; + // Add unsupported log type + PredictionLog prediction_log; + PopulatePredictionLog(&prediction_log, PredictionLog::kSessionRunLog); + warmup_records.push_back(prediction_log.SerializeAsString()); + TF_ASSERT_OK(WriteWarmupData(fname, warmup_records, 10)); + SavedModelBundle saved_model_bundle; + AddSignatures(&saved_model_bundle.meta_graph_def); + MockSession* mock = new MockSession; + saved_model_bundle.session.reset(mock); + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillRepeatedly(Return(Status::OK())); + const Status status = + RunSavedModelWarmup(RunOptions(), base_path, &saved_model_bundle); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::UNIMPLEMENTED, status.code()) << status; + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Unsupported log_type for warmup")); +} + +TEST(SavedModelBundleWarmupTest, TooManyWarmupRecords) { + string base_path = io::JoinPath(testing::TmpDir(), "TooManyWarmupRecords"); + TF_ASSERT_OK(Env::Default()->RecursivelyCreateDir( + io::JoinPath(base_path, kSavedModelAssetsExtraDirectory))); + string fname = io::JoinPath(base_path, kSavedModelAssetsExtraDirectory, + WarmupConsts::kRequestsFileName); + + std::vector warmup_records; + AddMixedWarmupData(&warmup_records); + TF_ASSERT_OK( + WriteWarmupData(fname, warmup_records, WarmupConsts::kMaxNumRecords + 1)); + SavedModelBundle saved_model_bundle; + AddSignatures(&saved_model_bundle.meta_graph_def); + MockSession* mock = new MockSession; + saved_model_bundle.session.reset(mock); + Tensor scores(DT_FLOAT, TensorShape({1, 1})); + Tensor classes(DT_STRING, TensorShape({1, 1})); + // Regress and Predict case + EXPECT_CALL(*mock, Run(_, _, SizeIs(1), _, _, _)) + .WillRepeatedly(DoAll(SetArgPointee<4>(std::vector({scores})), + Return(Status::OK()))); + // Classify case + EXPECT_CALL(*mock, Run(_, _, SizeIs(2), _, _, _)) + .WillRepeatedly( + DoAll(SetArgPointee<4>(std::vector({classes, scores})), + Return(Status::OK()))); + // MultiInference case + EXPECT_CALL(*mock, Run(_, _, SizeIs(3), _, _, _)) + .WillRepeatedly(DoAll( + SetArgPointee<4>(std::vector({classes, scores, scores})), + Return(Status::OK()))); + const Status status = + RunSavedModelWarmup(RunOptions(), base_path, &saved_model_bundle); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + EXPECT_THAT( + status.ToString(), + ::testing::HasSubstr("Number of warmup records exceeeds the maximum")); +} + +TEST(SavedModelBundleWarmupTest, UnparsableRecord) { + string base_path = io::JoinPath(testing::TmpDir(), "UnparsableRecord"); + TF_ASSERT_OK(Env::Default()->RecursivelyCreateDir( + io::JoinPath(base_path, kSavedModelAssetsExtraDirectory))); + string fname = io::JoinPath(base_path, kSavedModelAssetsExtraDirectory, + WarmupConsts::kRequestsFileName); + + std::vector warmup_records = {"malformed_record"}; + TF_ASSERT_OK(WriteWarmupData(fname, warmup_records, 10)); + SavedModelBundle saved_model_bundle; + MockSession* mock = new MockSession; + saved_model_bundle.session.reset(mock); + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)).Times(0); + const Status status = + RunSavedModelWarmup(RunOptions(), base_path, &saved_model_bundle); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + EXPECT_THAT(status.ToString(), + ::testing::HasSubstr("Failed to parse warmup record")); +} + +TEST(SavedModelBundleWarmupTest, RunFailure) { + string base_path = io::JoinPath(testing::TmpDir(), "RunFailure"); + TF_ASSERT_OK(Env::Default()->RecursivelyCreateDir( + io::JoinPath(base_path, kSavedModelAssetsExtraDirectory))); + string fname = io::JoinPath(base_path, kSavedModelAssetsExtraDirectory, + WarmupConsts::kRequestsFileName); + + int num_warmup_records = 10; + std::vector warmup_records; + AddMixedWarmupData(&warmup_records); + TF_ASSERT_OK(WriteWarmupData(fname, warmup_records, num_warmup_records)); + SavedModelBundle saved_model_bundle; + AddSignatures(&saved_model_bundle.meta_graph_def); + MockSession* mock = new MockSession; + saved_model_bundle.session.reset(mock); + EXPECT_CALL(*mock, Run(_, _, _, _, _, _)) + .WillOnce(::testing::Return(errors::InvalidArgument("Run failed"))); + const Status status = + RunSavedModelWarmup(RunOptions(), base_path, &saved_model_bundle); + ASSERT_FALSE(status.ok()); + EXPECT_EQ(::tensorflow::error::INVALID_ARGUMENT, status.code()) << status; + EXPECT_THAT(status.ToString(), ::testing::HasSubstr("Run failed")); +} + +} // namespace + +} // namespace serving +} // namespace tensorflow diff --git a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto index 8f6cf515b8b..3d521fc080c 100644 --- a/tensorflow_serving/servables/tensorflow/session_bundle_config.proto +++ b/tensorflow_serving/servables/tensorflow/session_bundle_config.proto @@ -59,6 +59,9 @@ message SessionBundleConfig { // // Input tensors to append to every Session::Run() call. repeated NamedTensorProto experimental_fixed_input_tensors = 778; + + // Enables model warmup. + bool enable_model_warmup = 779; } // Batching parameters. Each individual parameter is optional. If omitted, the From b67f2a0dacb37f56f28db713be04b06b99ec9ffe Mon Sep 17 00:00:00 2001 From: Michael Case Date: Mon, 9 Apr 2018 20:26:16 -0700 Subject: [PATCH 0462/8554] Fix the GCS Kokoro build badge links. The original badges I added were redirected/authenticated links (I think). This resulted in broken images on the README. These new links should not be redirected and should just link to the badge images. PiperOrigin-RevId: 192230689 --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index f752191b217..1e628c212cf 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # TensorFlow Serving -![Build Status](https://storage.cloud.google.com/tensorflow-serving-kokoro-build-badges/ubuntu.png) +![Build Status](https://storage.googleapis.com/tensorflow-serving-kokoro-build-badges/ubuntu.png) TensorFlow Serving is an open-source software library for serving machine learning models. It deals with the *inference* aspect of machine From 0fd27c7c9186a55cfdb1be9e53448fa12a4dc392 Mon Sep 17 00:00:00 2001 From: James Wexler Date: Wed, 18 Apr 2018 12:39:56 -0400 Subject: [PATCH 0463/8554] update TF version --- WORKSPACE | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index a221be3e2c4..d624c263981 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,19 +11,19 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "dd44550909aab50790495264d3e5c9e9373f9c0f0272047fd68df3e56c07cc78", - git_commit = "7a212edc6b3ed6200158fe51acf4694a62ca6938", + sha256 = "8028d51b4a911adeb9b8afa0ba6bcb99fa00a4949881cdad3ee67a8f33c8979a", + git_commit = "3128b43eb0bf37ac3c49cb22a6e1789d8ea346e8", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. http_archive( name = "io_bazel_rules_closure", - sha256 = "dbe0da2cca88194d13dc5a7125a25dd7b80e1daec7839f33223de654d7a1bcc8", - strip_prefix = "rules_closure-ba3e07cb88be04a2d4af7009caa0ff3671a79d06", + sha256 = "a38539c5b5c358548e75b44141b4ab637bba7c4dc02b46b1f62a96d6433f56ae", + strip_prefix = "rules_closure-dbb96841cc0a5fb2664c37822803b06dab20c7d1", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/ba3e07cb88be04a2d4af7009caa0ff3671a79d06.tar.gz", - "https://github.com/bazelbuild/rules_closure/archive/ba3e07cb88be04a2d4af7009caa0ff3671a79d06.tar.gz", # 2017-10-31 + "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz", # 2018-04-13 ], ) From 2bc569ba475e0ad1ebb0b47692d139b1d4b97b35 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 11 Apr 2018 14:58:09 -0700 Subject: [PATCH 0464/8554] Merged commit includes the following changes: 192514207 by vinuraja: Remove redundant lambda in retrier.cc -- 192339542 by A. Unique TensorFlower: No public changes. -- PiperOrigin-RevId: 192514207 --- tensorflow_serving/util/retrier.cc | 43 ++++++++++++++---------------- 1 file changed, 20 insertions(+), 23 deletions(-) diff --git a/tensorflow_serving/util/retrier.cc b/tensorflow_serving/util/retrier.cc index 9f03a8d4490..0067ccb9f8a 100644 --- a/tensorflow_serving/util/retrier.cc +++ b/tensorflow_serving/util/retrier.cc @@ -25,31 +25,28 @@ Status Retry(const string& description, const uint32 max_num_retries, const int64 retry_interval_micros, const std::function& retried_fn, const std::function& is_cancelled) { - return [&]() { - Status status; - int num_tries = 0; - do { - if (num_tries > 0) { - Env::Default()->SleepForMicroseconds(retry_interval_micros); - LOG(INFO) << "Retrying of " << description << " retry: " << num_tries; - } - status = retried_fn(); - if (!status.ok()) { - LOG(ERROR) << description << " failed: " << status; - } - ++num_tries; - } while (!is_cancelled() && !status.ok() && - num_tries < max_num_retries + 1); - - if (is_cancelled()) { - LOG(INFO) << "Retrying of " << description << " was cancelled."; + Status status; + int num_tries = 0; + do { + if (num_tries > 0) { + Env::Default()->SleepForMicroseconds(retry_interval_micros); + LOG(INFO) << "Retrying of " << description << " retry: " << num_tries; } - if (num_tries == max_num_retries + 1) { - LOG(INFO) << "Retrying of " << description - << " exhausted max_num_retries: " << max_num_retries; + status = retried_fn(); + if (!status.ok()) { + LOG(ERROR) << description << " failed: " << status; } - return status; - }(); + ++num_tries; + } while (!is_cancelled() && !status.ok() && num_tries < max_num_retries + 1); + + if (is_cancelled()) { + LOG(INFO) << "Retrying of " << description << " was cancelled."; + } + if (num_tries == max_num_retries + 1) { + LOG(INFO) << "Retrying of " << description + << " exhausted max_num_retries: " << max_num_retries; + } + return status; } } // namespace serving From 0fffea3ecd752dfb98de91271494ee69f3762b68 Mon Sep 17 00:00:00 2001 From: Vinu Rajashekhar Date: Thu, 12 Apr 2018 15:39:50 -0700 Subject: [PATCH 0465/8554] Merge changes from github. PiperOrigin-RevId: 192682886 --- tensorflow_serving/apis/model_management.proto | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/apis/model_management.proto b/tensorflow_serving/apis/model_management.proto index 09f2628ae86..9140fa19aac 100644 --- a/tensorflow_serving/apis/model_management.proto +++ b/tensorflow_serving/apis/model_management.proto @@ -12,4 +12,4 @@ message ReloadConfigRequest { message ReloadConfigResponse { StatusProto status = 1; -} \ No newline at end of file +} From 6c40014f6748627a8c4e794e051db4dd07f17042 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 11:04:29 -0700 Subject: [PATCH 0466/8554] Merged commit includes the following changes: 193693449 by A. Unique TensorFlower: Internal change 193572785 by A. Unique TensorFlower: No-op. -- 193127681 by mikecase: Remove deprecated/unused python related Bazel options. Since py_runtime was introduced, Bazel ignores options such as --force_python2 and --python2_path. Deleting to clean stuff up and make sure people are not misled. -- 192981122 by A. Unique TensorFlower: Internal change -- PiperOrigin-RevId: 193693449 --- tensorflow_serving/apis/BUILD | 1 + tools/bazel.rc | 4 ---- 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 2319cd24245..ff36881be97 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -50,6 +50,7 @@ serving_proto_library( srcs = ["input.proto"], cc_api_version = 2, java_api_version = 2, + js_api_version = 2, deps = [ "@org_tensorflow//tensorflow/core:protos_all_cc", "@protobuf_archive//:cc_wkt_protos", diff --git a/tools/bazel.rc b/tools/bazel.rc index 0fee21a1ae9..0facf32eb07 100644 --- a/tools/bazel.rc +++ b/tools/bazel.rc @@ -1,11 +1,7 @@ build:cuda --crosstool_top=@org_tensorflow//third_party/gpus/crosstool build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true -build --force_python=py2 -build --python2_path=/usr/bin/python - build --action_env PYTHON_BIN_PATH="/usr/bin/python" - build --define PYTHON_BIN_PATH=/usr/bin/python build --spawn_strategy=standalone --genrule_strategy=standalone From b58f8d39d1fd8ff24dfc71cd38248d9c1f6aa940 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 18:17:07 -0400 Subject: [PATCH 0467/8554] Merged commit includes the following changes: 193730916 by A. Unique TensorFlower: Internal change. -- 193715561 by christis: Internal Change. -- PiperOrigin-RevId: 193730916 --- tensorflow_serving/apis/BUILD | 1 - 1 file changed, 1 deletion(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index ff36881be97..aec21c9e064 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -53,7 +53,6 @@ serving_proto_library( js_api_version = 2, deps = [ "@org_tensorflow//tensorflow/core:protos_all_cc", - "@protobuf_archive//:cc_wkt_protos", ], ) From 3df87e8e31b88bb4f0b90c8242ecd9f52983b2e1 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 20 Apr 2018 21:34:09 -0400 Subject: [PATCH 0468/8554] No public change PiperOrigin-RevId: 193751942 --- tensorflow_serving/apis/BUILD | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index aec21c9e064..a4e42a7db2d 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -160,6 +160,12 @@ py_library( ], ) +tf_pyclif_proto_library( + name = "predict_pyclif", + proto_lib = ":predict_proto", + proto_srcfile = "predict.proto", +) + serving_go_grpc_library( name = "prediction_service_grpc", srcs = [":prediction_service_proto"], From 902e5050de418e66fb0ed30564679a756e0934cf Mon Sep 17 00:00:00 2001 From: Christina Sorokin Date: Mon, 23 Apr 2018 11:54:27 -0400 Subject: [PATCH 0469/8554] Merge changes from github. PiperOrigin-RevId: 193931024 --- WORKSPACE | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index a221be3e2c4..d624c263981 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,19 +11,19 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "dd44550909aab50790495264d3e5c9e9373f9c0f0272047fd68df3e56c07cc78", - git_commit = "7a212edc6b3ed6200158fe51acf4694a62ca6938", + sha256 = "8028d51b4a911adeb9b8afa0ba6bcb99fa00a4949881cdad3ee67a8f33c8979a", + git_commit = "3128b43eb0bf37ac3c49cb22a6e1789d8ea346e8", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. # Needs to be kept in sync with the same target in TensorFlow's WORKSPACE file. http_archive( name = "io_bazel_rules_closure", - sha256 = "dbe0da2cca88194d13dc5a7125a25dd7b80e1daec7839f33223de654d7a1bcc8", - strip_prefix = "rules_closure-ba3e07cb88be04a2d4af7009caa0ff3671a79d06", + sha256 = "a38539c5b5c358548e75b44141b4ab637bba7c4dc02b46b1f62a96d6433f56ae", + strip_prefix = "rules_closure-dbb96841cc0a5fb2664c37822803b06dab20c7d1", urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/ba3e07cb88be04a2d4af7009caa0ff3671a79d06.tar.gz", - "https://github.com/bazelbuild/rules_closure/archive/ba3e07cb88be04a2d4af7009caa0ff3671a79d06.tar.gz", # 2017-10-31 + "https://mirror.bazel.build/github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz", + "https://github.com/bazelbuild/rules_closure/archive/dbb96841cc0a5fb2664c37822803b06dab20c7d1.tar.gz", # 2018-04-13 ], ) From fcfefeb1a321f955efce20bba2c9ba97fdbcc601 Mon Sep 17 00:00:00 2001 From: Christopher Olston Date: Tue, 1 May 2018 13:55:42 -0700 Subject: [PATCH 0470/8554] Remove duplicate include/using items from main.cc --- tensorflow_serving/model_servers/main.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index e4512407735..f45ea986624 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -60,7 +60,6 @@ limitations under the License. #include "tensorflow/core/lib/core/status.h" #include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/lib/strings/str_util.h" -#include "tensorflow/core/lib/strings/numbers.h" #include "tensorflow/core/platform/env.h" #include "tensorflow/core/platform/init_main.h" #include "tensorflow/core/platform/protobuf.h" @@ -100,7 +99,6 @@ using tensorflow::serving::ServableState; using tensorflow::serving::ServerCore; using tensorflow::serving::SessionBundleConfig; using tensorflow::serving::TensorflowClassificationServiceImpl; -using tensorflow::serving::TensorflowRegressionServiceImpl; using tensorflow::serving::TensorflowPredictor; using tensorflow::serving::TensorflowRegressionServiceImpl; using tensorflow::serving::UniquePtrWithDeps; From 9ada0871a5c4aae1a20738d7cd0045e42a3ddde5 Mon Sep 17 00:00:00 2001 From: Yuan Dingping Date: Wed, 9 May 2018 00:28:31 +0800 Subject: [PATCH 0471/8554] change config pointer address compare to full_name compare (#873) --- tensorflow_serving/util/class_registration.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h index 01a3cf1e5ad..b58e7dc899d 100644 --- a/tensorflow_serving/util/class_registration.h +++ b/tensorflow_serving/util/class_registration.h @@ -196,7 +196,7 @@ class ClassRegistrationFactory // expected type. Status Create(const protobuf::Message& config, AdditionalFactoryArgs... args, std::unique_ptr* result) const override { - if (config.GetDescriptor() != config_descriptor_) { + if (config.GetDescriptor()->full_name() != config_descriptor_->full_name()) { return errors::InvalidArgument( "Supplied config proto of type ", config.GetDescriptor()->full_name(), " does not match expected type ", config_descriptor_->full_name()); From f267d62c0032559f6fc822e744a7949129ec3cfe Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Wed, 2 May 2018 18:52:02 -0700 Subject: [PATCH 0472/8554] Merged commit includes the following changes: 195188185 by A. Unique TensorFlower: Replaced calls to tensorflow::StringPiece::ToString with std::string conversions. That is, instances of sp.ToString() are replaced with std::string(sp). This will allow tensorflow::StringPiece::ToString to be removed, which is necessary before it can be replaced with absl::string_view. -- 194958789 by awk: No public changes. -- 194171771 by A. Unique TensorFlower: No public changes. -- PiperOrigin-RevId: 195188185 --- tensorflow_serving/core/aspired_versions_manager.cc | 6 +++--- tensorflow_serving/core/aspired_versions_manager_test.cc | 2 +- tensorflow_serving/core/dynamic_source_router.h | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tensorflow_serving/core/aspired_versions_manager.cc b/tensorflow_serving/core/aspired_versions_manager.cc index 8252766f53f..f3293581876 100644 --- a/tensorflow_serving/core/aspired_versions_manager.cc +++ b/tensorflow_serving/core/aspired_versions_manager.cc @@ -240,7 +240,7 @@ void AspiredVersionsManager::EnqueueAspiredVersionsRequest( mutex_lock l(pending_aspired_versions_requests_mu_); VLOG(1) << "Enqueueing aspired versions request: " << ServableVersionsDebugString(versions); - pending_aspired_versions_requests_[servable_name.ToString()] = + pending_aspired_versions_requests_[std::string(servable_name)] = std::move(versions); } } @@ -260,7 +260,7 @@ void AspiredVersionsManager::ProcessAspiredVersionsRequest( std::set current_aspired_versions; const std::vector> state_snapshots = basic_manager_->GetManagedServableStateSnapshots( - servable_name.ToString()); + std::string(servable_name)); for (const ServableStateSnapshot state_snapshot : state_snapshots) { if (state_snapshot.additional_state->is_aspired) { current_aspired_versions.insert(state_snapshot.id.version); @@ -309,7 +309,7 @@ bool AspiredVersionsManager::ContainsAnyReaspiredVersions( const std::vector>>& versions) const { const std::vector> state_snapshots = basic_manager_->GetManagedServableStateSnapshots( - servable_name.ToString()); + std::string(servable_name)); const std::set version_numbers = GetVersionNumbers(versions); for (const ServableStateSnapshot& state_snapshot : state_snapshots) { if (!state_snapshot.additional_state->is_aspired && diff --git a/tensorflow_serving/core/aspired_versions_manager_test.cc b/tensorflow_serving/core/aspired_versions_manager_test.cc index f73a0d9ba37..95f48f5cada 100644 --- a/tensorflow_serving/core/aspired_versions_manager_test.cc +++ b/tensorflow_serving/core/aspired_versions_manager_test.cc @@ -569,7 +569,7 @@ TEST_P(AspiredVersionsManagerTest, ManagerPrefersUnloadOverLoad) { for (const auto& servable_aspired : servable_aspired_list) { std::vector>> aspired_versions; for (int i = servable_aspired.start; i <= servable_aspired.end; ++i) { - const ServableId id = {servable_aspired.name.ToString(), i}; + const ServableId id = {std::string(servable_aspired.name), i}; aspired_versions.push_back(CreateAspiredVersion(id)); } manager_->GetAspiredVersionsCallback()(servable_aspired.name, diff --git a/tensorflow_serving/core/dynamic_source_router.h b/tensorflow_serving/core/dynamic_source_router.h index c0b779c182c..aa36f0acef8 100644 --- a/tensorflow_serving/core/dynamic_source_router.h +++ b/tensorflow_serving/core/dynamic_source_router.h @@ -107,7 +107,7 @@ int DynamicSourceRouter::Route( const StringPiece servable_name, const std::vector>& versions) { mutex_lock l(routes_mu_); - auto it = routes_.find(servable_name.ToString()); + auto it = routes_.find(std::string(servable_name)); if (it == routes_.end()) { LOG(INFO) << "Routing servable(s) from stream " << servable_name << " to default output port " << num_output_ports_ - 1; From 430b26576db9ee875a09d1f947d9170fbd8de138 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 4 May 2018 11:56:53 -0700 Subject: [PATCH 0473/8554] Merged commit includes the following changes: 195449477 by awk: Update TF archive to latest green build. -- 195350340 by awk: No public changes. -- PiperOrigin-RevId: 195449477 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index d624c263981..041ebab4beb 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "8028d51b4a911adeb9b8afa0ba6bcb99fa00a4949881cdad3ee67a8f33c8979a", - git_commit = "3128b43eb0bf37ac3c49cb22a6e1789d8ea346e8", + sha256 = "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", + git_commit = "2dc7575123ffa0e6413fc3d2700968ef25f049de", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. From 70671902031b74e4b7fb7254e1391110bbd185ce Mon Sep 17 00:00:00 2001 From: Abhijit Karmarkar Date: Fri, 4 May 2018 13:46:12 -0700 Subject: [PATCH 0474/8554] Fix incorrect checksum. PiperOrigin-RevId: 195465351 --- WORKSPACE | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/WORKSPACE b/WORKSPACE index 041ebab4beb..4884785f611 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,7 +11,7 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "0019dfc4b32d63c1392aa264aed2253c1e0c2fb09216f8e2cc269bbfb8bb49b5", + sha256 = "6e93fc70bd2daa73719a49b03885d1a061fc8fe87ee5da6e19ee9e3f046b273b", git_commit = "2dc7575123ffa0e6413fc3d2700968ef25f049de", ) From 93023e66052a34dde04f31dcc8aeb50818d950da Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 10 May 2018 10:39:13 -0700 Subject: [PATCH 0475/8554] Merged commit includes the following changes: 196136844 by A. Unique TensorFlower: public no-op. -- 196045249 by A. Unique TensorFlower: Public no-op. -- 196007389 by awk: Internal change -- 195877335 by A. Unique TensorFlower: Public no-op. -- PiperOrigin-RevId: 196136844 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index 4884785f611..ba6c5bfc0c7 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "6e93fc70bd2daa73719a49b03885d1a061fc8fe87ee5da6e19ee9e3f046b273b", - git_commit = "2dc7575123ffa0e6413fc3d2700968ef25f049de", + sha256 = "9f7603c50451ed941e6fe0819061a0e50e27deb867392824b31f324b877f47ba", + git_commit = "c6a3633e1f4698eb75bdb7ee035c2ba4c527347c", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. From efbb8f8b2f652c56cd7f4a17c223d30b447c4903 Mon Sep 17 00:00:00 2001 From: Abhijit Karmarkar Date: Thu, 10 May 2018 11:29:31 -0700 Subject: [PATCH 0476/8554] Internal change. PiperOrigin-RevId: 196145669 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index ba6c5bfc0c7..cc7791a07b8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "9f7603c50451ed941e6fe0819061a0e50e27deb867392824b31f324b877f47ba", - git_commit = "c6a3633e1f4698eb75bdb7ee035c2ba4c527347c", + sha256 = "4cc63651152fd9ebd82dc41114029381fad5cb45b2fe22dc42afb9a3e0fd606f", + git_commit = "4fb125264c5394c9e4295ed437adb1d9711bd456", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. From 4bb0b1418b66e7285cb5ab17a0b15090885a3580 Mon Sep 17 00:00:00 2001 From: Abhijit Karmarkar Date: Thu, 10 May 2018 12:45:35 -0700 Subject: [PATCH 0477/8554] Internal change. PiperOrigin-RevId: 196156960 --- tensorflow_serving/core/test_util/test_main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/core/test_util/test_main.cc b/tensorflow_serving/core/test_util/test_main.cc index b2b05cc74c9..8a0741d59ec 100644 --- a/tensorflow_serving/core/test_util/test_main.cc +++ b/tensorflow_serving/core/test_util/test_main.cc @@ -25,7 +25,7 @@ limitations under the License. // main() is supplied by gunit_main #else #include "gtest/gtest.h" -#include "tensorflow/core/lib/core/stringpiece.h" +#include "tensorflow/core/lib/strings/str_util.h" #include "tensorflow/core/platform/test_benchmark.h" GTEST_API_ int main(int argc, char** argv) { @@ -33,7 +33,7 @@ GTEST_API_ int main(int argc, char** argv) { testing::InitGoogleTest(&argc, argv); for (int i = 1; i < argc; i++) { - if (tensorflow::StringPiece(argv[i]).starts_with("--benchmarks=")) { + if (tensorflow::str_util::StartsWith(argv[i], "--benchmarks=")) { const char* pattern = argv[i] + strlen("--benchmarks="); tensorflow::testing::Benchmark::Run(pattern); return 0; From 17fa0a097e24511016688c2c549754514840a958 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 10 May 2018 14:27:50 -0700 Subject: [PATCH 0478/8554] Merged commit includes the following changes: 196171565 by awk: None. -- PiperOrigin-RevId: 196171565 --- tensorflow_serving/apis/regressor.h | 4 ++-- tensorflow_serving/core/test_util/mock_session.h | 6 +++--- tensorflow_serving/model_servers/model_platform_types.h | 6 +++--- 3 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tensorflow_serving/apis/regressor.h b/tensorflow_serving/apis/regressor.h index 2f635825701..5e345df9c12 100644 --- a/tensorflow_serving/apis/regressor.h +++ b/tensorflow_serving/apis/regressor.h @@ -14,8 +14,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_SERVING_APIS_REGRESSOR_H -#define TENSORFLOW_SERVING_APIS_REGRESSOR_H +#ifndef TENSORFLOW_SERVING_APIS_REGRESSOR_H_ +#define TENSORFLOW_SERVING_APIS_REGRESSOR_H_ #include "tensorflow/core/lib/core/status.h" #include "tensorflow_serving/apis/regression.pb.h" diff --git a/tensorflow_serving/core/test_util/mock_session.h b/tensorflow_serving/core/test_util/mock_session.h index 64017439fa7..84d699fb1b6 100644 --- a/tensorflow_serving/core/test_util/mock_session.h +++ b/tensorflow_serving/core/test_util/mock_session.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_SERVING_TEST_UTIL_MOCK_SESSION_H_ -#define TENSORFLOW_SERVING_TEST_UTIL_MOCK_SESSION_H_ +#ifndef TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SESSION_H_ +#define TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SESSION_H_ #include #include "tensorflow/core/public/session.h" @@ -65,4 +65,4 @@ class MockSession : public tensorflow::Session { } // namespace serving } // namespace tensorflow -#endif // TENSORFLOW_SERVING_TEST_UTIL_MOCK_SESSION_H_ +#endif // TENSORFLOW_SERVING_CORE_TEST_UTIL_MOCK_SESSION_H_ diff --git a/tensorflow_serving/model_servers/model_platform_types.h b/tensorflow_serving/model_servers/model_platform_types.h index 9701b192c1a..8e216977bc3 100644 --- a/tensorflow_serving/model_servers/model_platform_types.h +++ b/tensorflow_serving/model_servers/model_platform_types.h @@ -13,8 +13,8 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ -#define TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ +#ifndef TENSORFLOW_SERVING_MODEL_SERVERS_MODEL_PLATFORM_TYPES_H_ +#define TENSORFLOW_SERVING_MODEL_SERVERS_MODEL_PLATFORM_TYPES_H_ namespace tensorflow { namespace serving { @@ -24,4 +24,4 @@ constexpr char kTensorFlowModelPlatform[] = "tensorflow"; } // namespace serving } // namespace tensorflow -#endif // TENSORFLOW_SERVING_MODEL_SERVERS_PLATFORM_TYPES_H_ +#endif // TENSORFLOW_SERVING_MODEL_SERVERS_MODEL_PLATFORM_TYPES_H_ From 9679eb36e6d438e33cbf20a2552e9ba347f4ca35 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Anton=20S=C3=B6rhus?= Date: Tue, 15 May 2018 20:22:02 +0200 Subject: [PATCH 0479/8554] fix channel args (#871) --- tensorflow_serving/model_servers/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index f45ea986624..7a108bc3972 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -304,8 +304,8 @@ void RunServer(int port, std::unique_ptr core, bool use_saved_model, // gRPC accept arguments of two types, int and string. We will attempt to // parse each arg as int and pass it on as such if successful. Otherwise we // will pass it as a string. gRPC will log arguments that were not accepted. - int value; - if (tensorflow::strings::safe_strto32(channel_argument.key, &value)) { + tensorflow::int32 value; + if (tensorflow::strings::safe_strto32(channel_argument.value, &value)) { builder.AddChannelArgument(channel_argument.key, value); } else { builder.AddChannelArgument(channel_argument.key, channel_argument.value); From 52162430ce3da9183dfa1b0a1fd166421449cfb4 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 11 May 2018 10:34:35 -0700 Subject: [PATCH 0480/8554] Merged commit includes the following changes: 196273719 by haiming: Merge changes from github. -- PiperOrigin-RevId: 196273719 --- tensorflow_serving/util/class_registration.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/util/class_registration.h b/tensorflow_serving/util/class_registration.h index 01a3cf1e5ad..becd5e59550 100644 --- a/tensorflow_serving/util/class_registration.h +++ b/tensorflow_serving/util/class_registration.h @@ -196,7 +196,8 @@ class ClassRegistrationFactory // expected type. Status Create(const protobuf::Message& config, AdditionalFactoryArgs... args, std::unique_ptr* result) const override { - if (config.GetDescriptor() != config_descriptor_) { + if (config.GetDescriptor()->full_name() != + config_descriptor_->full_name()) { return errors::InvalidArgument( "Supplied config proto of type ", config.GetDescriptor()->full_name(), " does not match expected type ", config_descriptor_->full_name()); From 771200b66c73d752ecd8b503d6089da0f4e1c10b Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Fri, 11 May 2018 16:46:14 -0700 Subject: [PATCH 0481/8554] Merged commit includes the following changes: 196328023 by A. Unique TensorFlower: public no-op. -- 196317117 by awk: Internal change. -- PiperOrigin-RevId: 196328023 --- tensorflow_serving/apis/BUILD | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index a4e42a7db2d..5e4ed6f888d 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -21,7 +21,7 @@ filegroup( load("//tensorflow_serving:serving.bzl", "serving_proto_library") load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") load("//tensorflow_serving:serving.bzl", "serving_go_grpc_library") -load("@org_tensorflow//tensorflow/core:platform/default/build_config.bzl", "tf_pyclif_proto_library") +load("@org_tensorflow//tensorflow/core:platform/default/build_config.bzl", "tf_jspb_proto_library", "tf_pyclif_proto_library") serving_proto_library( name = "get_model_metadata_proto", @@ -95,6 +95,11 @@ tf_pyclif_proto_library( proto_srcfile = "model.proto", ) +tf_jspb_proto_library( + name = "model_jspb_proto", + deps = [":model_proto"], +) + serving_proto_library( name = "predict_proto", srcs = ["predict.proto"], @@ -116,6 +121,11 @@ serving_proto_library_py( ], ) +tf_jspb_proto_library( + name = "predict_jspb_proto", + deps = [":predict_proto"], +) + serving_proto_library( name = "prediction_log_proto", srcs = ["prediction_log.proto"], From 86ff407c99ec92e1c7c87ffe797051cf0eb1a31a Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 14 May 2018 16:19:57 -0700 Subject: [PATCH 0482/8554] Merged commit includes the following changes: 196585085 by A. Unique TensorFlower: public no-op. -- PiperOrigin-RevId: 196585085 --- tensorflow_serving/apis/BUILD | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 5e4ed6f888d..3da7d9de710 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -71,6 +71,11 @@ tf_pyclif_proto_library( proto_srcfile = "input.proto", ) +tf_jspb_proto_library( + name = "input_jspb_proto", + deps = [":input_proto"], +) + serving_proto_library( name = "model_proto", srcs = ["model.proto"], @@ -271,6 +276,11 @@ tf_pyclif_proto_library( proto_srcfile = "classification.proto", ) +tf_jspb_proto_library( + name = "classification_jspb_proto", + deps = [":classification_proto"], +) + serving_proto_library( name = "inference_proto", srcs = ["inference.proto"], @@ -324,6 +334,11 @@ serving_proto_library_py( ], ) +tf_jspb_proto_library( + name = "regression_jspb_proto", + deps = [":regression_proto"], +) + serving_proto_library( name = "session_service_proto", srcs = ["session_service.proto"], From e1fc7d6d2e26f50dd5c9dcb589e1203574dee35d Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 14 May 2018 19:38:50 -0700 Subject: [PATCH 0483/8554] Merged commit includes the following changes: 196606472 by awk: Updating TensorFlow to latest green. -- PiperOrigin-RevId: 196606472 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index cc7791a07b8..f1eda426bfa 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "4cc63651152fd9ebd82dc41114029381fad5cb45b2fe22dc42afb9a3e0fd606f", - git_commit = "4fb125264c5394c9e4295ed437adb1d9711bd456", + sha256 = "3a598c23b1790b1041cd0bd591b5db99765ca91b5c3e68cedb5189972e46864b", + git_commit = "f27033fb1212d7031a359c913d0f59e976b14c14", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. From d770be5e44507cc93b35e8817cf0f6d8bf4b8ff0 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Tue, 15 May 2018 11:49:54 -0700 Subject: [PATCH 0484/8554] Merged commit includes the following changes: 196704987 by awk: Updating TensorFlow to latest green. -- PiperOrigin-RevId: 196704987 --- WORKSPACE | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/WORKSPACE b/WORKSPACE index f1eda426bfa..bca96f2d386 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -11,8 +11,8 @@ load("//tensorflow_serving:repo.bzl", "tensorflow_http_archive") tensorflow_http_archive( name = "org_tensorflow", - sha256 = "3a598c23b1790b1041cd0bd591b5db99765ca91b5c3e68cedb5189972e46864b", - git_commit = "f27033fb1212d7031a359c913d0f59e976b14c14", + sha256 = "cbd96914936ce3aacc39e02c2efb711f937f8ebcda888c349eab075185d7c914", + git_commit = "d8fac4cb80eb0c42d2550bcb720a80d29fc5f22d", ) # TensorFlow depends on "io_bazel_rules_closure" so we need this here. From bd373815fbfbd92c4f991853906df06b17775b11 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 21 May 2018 12:23:38 -0700 Subject: [PATCH 0485/8554] Merged commit includes the following changes: 197435743 by A. Unique TensorFlower: Internal change 197423069 by awk: Internal change. -- PiperOrigin-RevId: 197435743 --- tensorflow_serving/util/BUILD | 1 + 1 file changed, 1 insertion(+) diff --git a/tensorflow_serving/util/BUILD b/tensorflow_serving/util/BUILD index 762fd610e8a..f026be2d3eb 100644 --- a/tensorflow_serving/util/BUILD +++ b/tensorflow_serving/util/BUILD @@ -263,6 +263,7 @@ cc_library( srcs = ["class_registration_util.cc"], hdrs = ["class_registration_util.h"], deps = [ + "@com_google_absl//absl/strings", "@org_tensorflow//tensorflow/core:lib", ], ) From 299af2398e52d285ca0d937315bab04b013c3822 Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 24 May 2018 05:30:57 -0400 Subject: [PATCH 0486/8554] Merged commit includes the following changes: 197861264 by A. Unique TensorFlower: public no-op. -- PiperOrigin-RevId: 197861264 --- tensorflow_serving/apis/BUILD | 24 ++++++++++++++++++++++++ tensorflow_serving/config/BUILD | 17 +++++++++++++++++ tensorflow_serving/core/BUILD | 14 ++++++++++++++ 3 files changed, 55 insertions(+) diff --git a/tensorflow_serving/apis/BUILD b/tensorflow_serving/apis/BUILD index 3da7d9de710..fb9c83ba68d 100644 --- a/tensorflow_serving/apis/BUILD +++ b/tensorflow_serving/apis/BUILD @@ -146,6 +146,20 @@ serving_proto_library( ], ) +serving_proto_library_py( + name = "prediction_log_proto_py_pb2", + srcs = ["prediction_log.proto"], + proto_library = "prediction_log_proto", + deps = [ + ":classification_proto_py_pb2", + ":inference_proto_py_pb2", + ":predict_proto_py_pb2", + ":regression_proto_py_pb2", + ":session_service_proto_py_pb2", + "//tensorflow_serving/core:logging_proto_py_pb2", + ], +) + serving_proto_library( name = "prediction_service_proto", srcs = ["prediction_service.proto"], @@ -351,6 +365,16 @@ serving_proto_library( ], ) +serving_proto_library_py( + name = "session_service_proto_py_pb2", + srcs = ["session_service.proto"], + proto_library = "session_service_proto", + deps = [ + ":model_proto_py_pb2", + "@org_tensorflow//tensorflow/core:protos_all_py", + ], +) + tf_pyclif_proto_library( name = "regression_pyclif", proto_lib = ":regression_proto", diff --git a/tensorflow_serving/config/BUILD b/tensorflow_serving/config/BUILD index cc33650d4be..84dfe5364f6 100644 --- a/tensorflow_serving/config/BUILD +++ b/tensorflow_serving/config/BUILD @@ -57,6 +57,14 @@ serving_proto_library( ], ) +serving_proto_library_py( + name = "log_collector_config_proto_py_pb2", + srcs = ["log_collector_config.proto"], + proto_library = "log_collector_config_proto", + deps = [ + ], +) + serving_proto_library( name = "logging_config_proto", srcs = ["logging_config.proto"], @@ -66,3 +74,12 @@ serving_proto_library( ":log_collector_config_proto", ], ) + +serving_proto_library_py( + name = "logging_config_proto_py_pb2", + srcs = ["logging_config.proto"], + proto_library = "logging_config_proto", + deps = [ + ":log_collector_config_proto_py_pb2", + ], +) diff --git a/tensorflow_serving/core/BUILD b/tensorflow_serving/core/BUILD index 4ec8fefcd1e..b11ab0895a4 100644 --- a/tensorflow_serving/core/BUILD +++ b/tensorflow_serving/core/BUILD @@ -733,6 +733,7 @@ cc_test( ) load("//tensorflow_serving:serving.bzl", "serving_proto_library") +load("//tensorflow_serving:serving.bzl", "serving_proto_library_py") serving_proto_library( name = "logging_proto", @@ -748,6 +749,19 @@ serving_proto_library( ], ) +serving_proto_library_py( + name = "logging_proto_py_pb2", + srcs = ["logging.proto"], + proto_library = "logging_proto", + visibility = [ + "//visibility:public", + ], + deps = [ + "//tensorflow_serving/apis:model_proto_py_pb2", + "//tensorflow_serving/config:logging_config_proto_py_pb2", + ], +) + cc_library( name = "request_logger", srcs = ["request_logger.cc"], From f5266d1a6a17a7ecf0540e4b9aa2ed87dbae8e0e Mon Sep 17 00:00:00 2001 From: Christina Sorokin Date: Thu, 24 May 2018 10:43:49 -0400 Subject: [PATCH 0487/8554] Merge changes from github. PiperOrigin-RevId: 197890974 --- tensorflow_serving/model_servers/main.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tensorflow_serving/model_servers/main.cc b/tensorflow_serving/model_servers/main.cc index f45ea986624..7a108bc3972 100644 --- a/tensorflow_serving/model_servers/main.cc +++ b/tensorflow_serving/model_servers/main.cc @@ -304,8 +304,8 @@ void RunServer(int port, std::unique_ptr core, bool use_saved_model, // gRPC accept arguments of two types, int and string. We will attempt to // parse each arg as int and pass it on as such if successful. Otherwise we // will pass it as a string. gRPC will log arguments that were not accepted. - int value; - if (tensorflow::strings::safe_strto32(channel_argument.key, &value)) { + tensorflow::int32 value; + if (tensorflow::strings::safe_strto32(channel_argument.value, &value)) { builder.AddChannelArgument(channel_argument.key, value); } else { builder.AddChannelArgument(channel_argument.key, channel_argument.value); From e66e79acd0112006a6f91a612eff44f5fdc13dfd Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Thu, 24 May 2018 21:19:22 -0400 Subject: [PATCH 0488/8554] Merged commit includes the following changes: 197986820 by A. Unique TensorFlower: Makes the predict_util library visible to library clients. -- PiperOrigin-RevId: 197986820 --- tensorflow_serving/servables/tensorflow/BUILD | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index 45f1d97ba7b..eaed0445d15 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -436,6 +436,9 @@ cc_library( name = "predict_util", srcs = ["predict_util.cc"], hdrs = ["predict_util.h"], + visibility = [ + "//visibility:public", + ], deps = [ "//tensorflow_serving/apis:predict_proto", "//tensorflow_serving/servables/tensorflow:util", From 754e528a0902cfa4bd5c9880a162aeb459421c8e Mon Sep 17 00:00:00 2001 From: Abhijit Karmarkar Date: Fri, 25 May 2018 15:17:22 -0400 Subject: [PATCH 0489/8554] Internal change. PiperOrigin-RevId: 198084919 --- tensorflow_serving/core/test_util/BUILD | 1 + tensorflow_serving/servables/tensorflow/BUILD | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/tensorflow_serving/core/test_util/BUILD b/tensorflow_serving/core/test_util/BUILD index 082dd7f14ce..b74abc10ded 100644 --- a/tensorflow_serving/core/test_util/BUILD +++ b/tensorflow_serving/core/test_util/BUILD @@ -26,6 +26,7 @@ cc_library( testonly = 1, srcs = ["test_main.cc"], linkopts = ["-lm"], + tags = ["keep_dep"], # Tell build_cleaner to keep dependencies on this. deps = [ "@com_google_googletest//:gtest", "@org_tensorflow//tensorflow/core:testlib", diff --git a/tensorflow_serving/servables/tensorflow/BUILD b/tensorflow_serving/servables/tensorflow/BUILD index eaed0445d15..ef3e651d8ac 100644 --- a/tensorflow_serving/servables/tensorflow/BUILD +++ b/tensorflow_serving/servables/tensorflow/BUILD @@ -871,7 +871,7 @@ cc_test( srcs = ["util_test.cc"], deps = [ ":util", - "//tensorflow_serving/core/test_util:test_main", # buildcleaner: keep + "//tensorflow_serving/core/test_util:test_main", "//tensorflow_serving/test_util", "@org_tensorflow//tensorflow/cc/saved_model:signature_constants", "@org_tensorflow//tensorflow/core:framework", From 2e465ed0d39c429c4fb292184d86a70432bf56ac Mon Sep 17 00:00:00 2001 From: "A. Unique TensorFlower" Date: Mon, 28 May 2018 23:48:17 -0400 Subject: [PATCH 0490/8554] Merged commit includes the following changes: 198343369 by awk: Add following support libraries for upcoming HTTP/REST APIs. o util/json_tensor -- Convert between Tensor/Example protos and JSON objects. o util/net_http/ -- Embedded light-weight HTTP server o model_servers/http_rest_prediction_handler -- HTTP/REST API handler. Documentation API in: g3doc/api_rest.md -- 198253949 by A. Unique TensorFlower: Internal change. -- 198228226 by A. Unique TensorFlower: No public changes. -- PiperOrigin-RevId: 198343369 --- tensorflow_serving/g3doc/api_rest.md | 290 +++++ tensorflow_serving/model_servers/BUILD | 52 + .../http_rest_prediction_handler.cc | 193 +++ .../http_rest_prediction_handler.h | 114 ++ .../http_rest_prediction_handler_test.cc | 378 ++++++ tensorflow_serving/util/BUILD | 37 + tensorflow_serving/util/json_tensor.cc | 932 ++++++++++++++ tensorflow_serving/util/json_tensor.h | 238 ++++ tensorflow_serving/util/json_tensor_test.cc | 1116 +++++++++++++++++ tensorflow_serving/util/net_http/OWNERS | 2 + tensorflow_serving/util/net_http/README.md | 11 + tensorflow_serving/util/net_http/client/BUILD | 25 + .../util/net_http/client/README.md | 3 + .../util/net_http/client/evhttp_connection.cc | 201 +++ .../util/net_http/client/evhttp_connection.h | 100 ++ .../util/net_http/client/testing/BUILD | 18 + .../client/testing/evhttp_echo_client.cc | 63 + .../util/net_http/server/internal/BUILD | 61 + .../server/internal/evhttp_request.cc | 261 ++++ .../net_http/server/internal/evhttp_request.h | 117 ++ .../server/internal/evhttp_request_test.cc | 221 ++++ .../net_http/server/internal/evhttp_server.cc | 368 ++++++ .../net_http/server/internal/evhttp_server.h | 143 +++ .../server/internal/evhttp_server_test.cc | 267 ++++ .../net_http/server/internal/server_support.h | 54 + .../util/net_http/server/public/BUILD | 36 + .../util/net_http/server/public/httpserver.h | 54 + .../server/public/httpserverinterface.h | 176 +++ .../server/public/response_code_enum.h | 112 ++ .../server/public/serverrequestinterface.h | 163 +++ .../util/net_http/server/testing/BUILD | 20 + .../server/testing/evhttp_echo_server.cc | 144 +++ .../util/net_http/socket/testing/BUILD | 27 + .../socket/testing/ev_fetch_client.cc | 194 +++ .../socket/testing/ev_print_req_server.cc | 241 ++++ tensorflow_serving/workspace.bzl | 22 + third_party/libevent.BUILD | 83 ++ third_party/rapidjson.BUILD | 14 + 38 files changed, 6551 insertions(+) create mode 100644 tensorflow_serving/g3doc/api_rest.md create mode 100644 tensorflow_serving/model_servers/http_rest_prediction_handler.cc create mode 100644 tensorflow_serving/model_servers/http_rest_prediction_handler.h create mode 100644 tensorflow_serving/model_servers/http_rest_prediction_handler_test.cc create mode 100644 tensorflow_serving/util/json_tensor.cc create mode 100644 tensorflow_serving/util/json_tensor.h create mode 100644 tensorflow_serving/util/json_tensor_test.cc create mode 100644 tensorflow_serving/util/net_http/OWNERS create mode 100644 tensorflow_serving/util/net_http/README.md create mode 100644 tensorflow_serving/util/net_http/client/BUILD create mode 100644 tensorflow_serving/util/net_http/client/README.md create mode 100644 tensorflow_serving/util/net_http/client/evhttp_connection.cc create mode 100644 tensorflow_serving/util/net_http/client/evhttp_connection.h create mode 100644 tensorflow_serving/util/net_http/client/testing/BUILD create mode 100644 tensorflow_serving/util/net_http/client/testing/evhttp_echo_client.cc create mode 100644 tensorflow_serving/util/net_http/server/internal/BUILD create mode 100644 tensorflow_serving/util/net_http/server/internal/evhttp_request.cc create mode 100644 tensorflow_serving/util/net_http/server/internal/evhttp_request.h create mode 100644 tensorflow_serving/util/net_http/server/internal/evhttp_request_test.cc create mode 100644 tensorflow_serving/util/net_http/server/internal/evhttp_server.cc create mode 100644 tensorflow_serving/util/net_http/server/internal/evhttp_server.h create mode 100644 tensorflow_serving/util/net_http/server/internal/evhttp_server_test.cc create mode 100644 tensorflow_serving/util/net_http/server/internal/server_support.h create mode 100644 tensorflow_serving/util/net_http/server/public/BUILD create mode 100644 tensorflow_serving/util/net_http/server/public/httpserver.h create mode 100644 tensorflow_serving/util/net_http/server/public/httpserverinterface.h create mode 100644 tensorflow_serving/util/net_http/server/public/response_code_enum.h create mode 100644 tensorflow_serving/util/net_http/server/public/serverrequestinterface.h create mode 100644 tensorflow_serving/util/net_http/server/testing/BUILD create mode 100644 tensorflow_serving/util/net_http/server/testing/evhttp_echo_server.cc create mode 100644 tensorflow_serving/util/net_http/socket/testing/BUILD create mode 100644 tensorflow_serving/util/net_http/socket/testing/ev_fetch_client.cc create mode 100644 tensorflow_serving/util/net_http/socket/testing/ev_print_req_server.cc create mode 100644 third_party/libevent.BUILD create mode 100644 third_party/rapidjson.BUILD diff --git a/tensorflow_serving/g3doc/api_rest.md b/tensorflow_serving/g3doc/api_rest.md new file mode 100644 index 00000000000..760f90c6af7 --- /dev/null +++ b/tensorflow_serving/g3doc/api_rest.md @@ -0,0 +1,290 @@ +# RESTful API + +In addition to [gRPC +APIs](https://github.com/tensorflow/serving/blob/master/tensorflow_serving/apis/prediction_service.proto) +TensorFlow ModelServer also supports RESTful APIs for classification, regression +and prediction on TensorFlow models. This page describes these API endpoints and +format of request/response involved in using them. + +TensorFlow ModelServer running on `host:port` accepts following REST API +requests: + +``` +POST http://host:port/: + +URI: /v1/models/${MODEL_NAME}[/versions/${MODEL_VERSION}] +VERB: classify|regress|predict +``` + +`/versions/${MODEL_VERSION}` is optional. If omitted the latest version is used. + +This API closely follows the gRPC version of +[`PredictionService`](https://github.com/tensorflow/serving/blob/5369880e9143aa00d586ee536c12b04e945a977c/tensorflow_serving/apis/prediction_service.proto#L15) +API. + +Examples of request URLs: + +``` +http://host:port/v1/models/iris:classify +http://host:port/v1/models/mnist/versions/314:predict +``` + +The request and response is a JSON object. The composition of this object +depends on the request type or verb. See the API specific sections below for +details. + +In case of error, all APIs will return a JSON object in the response body with +`error` as key and the error message as the value: + +```json +{ + "error": +} +``` + +## Classify and Regress API + +### Request format + +The request body for the `classify` and `regress` APIs must be a JSON object +formatted as follows: + +```json +{ + // Optional: serving signature to use. + // If unspecifed default serving signature is used. + "signature_name": , + + // Optional: Common context shared by all examples. + // Features that appear here MUST NOT appear in examples (below). + "context": { + "": | + "": | + }, + + // List of Example objects + "examples": [ + { + // Example 1 + "": |, + "": |, + ... + }, + { + // Example 2 + "": |, + "": |, + ... + } + ... + ] +} +``` + +`` is a JSON number (whole or decimal) or string, and `` is a list +of such values. See [Encoding binary values](#encoding-binary-values) section +below for details on how to represent a binary (stream of bytes) value. This +format is similar to gRPC's `ClassificationRequest` and `RegressionRequest` +protos. Both versions accept list of +[`Example`](https://github.com/tensorflow/tensorflow/blob/92e6c3e4f5c1cabfda1e61547a6a1b268ef95fa5/tensorflow/core/example/example.proto#L13) +objects. + +### Response format + +A `classify` request returns a JSON object in the response body, formatted as +follows: + +```json +{ + "result": [ + // List of class label/score pairs for first Example (in request) + [ [, ], [, ], ... ], + + // List of class label/score pairs for next Example (in request) + [ [, ], [, ], ... ], + ... + ] +} +``` + +`