diff --git a/.bazelrc b/.bazelrc index 4b673d3fcfb420..4825d6fba95714 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,3 +1,80 @@ +# TensorFlow Bazel configuration file. +# This file tries to group and simplify build options for TensorFlow +# +# ----CONFIG OPTIONS---- +# Android options: +# android: +# android_arm: +# android_x86: +# android_x86_64: +# +# iOS options: +# ios: +# ios_armv7: +# ios_arm64: +# ios_x86_64: +# ios_fat: +# +# Compiler options: +# cuda_clang: Use clang when building CUDA code. +# c++17: Build with C++17 options +# C++1z: Build with C++17 options +# avx_linux: Build with avx instruction set on linux. +# avx2_linux: Build with avx2 instruction set on linux. +# arch_native_linux: Build with instruction sets available to the host machine on linux +# avx_win: Build with avx instruction set on windows +# avx2_win: Build with avx2 instruction set on windows +# +# Other build options: +# short_logs: Only log errors during build, skip warnings. +# monolithic: Build all TF C++ code into a single shared object. +# dynamic_kernels: Try to link all kernels dynamically (experimental). +# +# +# TF version options; +# v1: Build TF V1 (without contrib) +# v2: Build TF v2 +# +# Feature and Third party library support options: +# xla: Build TF with XLA +# using_cuda: CUDA is available to build system. +# cuda: Build with full cuda support. +# rocm: Build with AMD GPU support (rocm). +# sycl: Build with SYCL support. +# sycl_nodouble: +# sycl_asan: +# sycl_trisycl: +# mkl: Enable full mkl support. +# mkl_open_source_only: Enable MKL support only using open source MKL libraries. +# tensorrt: Enable Tensorrt support. +# ngraph: Enable ngraph support. +# numa: Enable numa using hwloc. +# noaws: Disable AWS S3 storage support +# nogcp: Disable GCS support. +# nohdfs: Disable hadoop hdfs support. +# nonccl: Disable nccl support. +# +# +# Remote build execution options (only configured to work with TF team projects for now.) +# rbe: General RBE options shared by all flavors. +# rbe_linux: General RBE options used on all linux builds. +# rbe_win: General RBE options used on all windows builds. +# +# rbe_cpu_linux: RBE options to build with only CPU support. +# rbe_linux_cuda_nvcc: RBE options to build with GPU support using nvcc. +# rbe_gpu_linux: An alias for rbe_linux_cuda_nvcc +# +# rbe_linux_py2: Linux Python 2 RBE config. +# rbe_linux_py3: Linux Python 3 RBE config +# +# rbe_win_py37: Windows Python 3.7 RBE config +# +# tensorflow_testing_rbe_linux: RBE options to use RBE with tensorflow-testing project on linux +# tensorflow_testing_rbe_win: RBE options to use RBE with tensorflow-testing project on windows +# + + + # Android configs. Bazel needs to have --cpu and --fat_apk_cpu both set to the # target CPU to build transient dependencies correctly. See # https://docs.bazel.build/versions/master/user-manual.html#flag--fat_apk_cpu @@ -48,15 +125,6 @@ build:mkl_open_source_only --define=build_with_mkl_dnn_v1_only=true build:mkl_open_source_only --define=build_with_mkl=true --define=enable_mkl=true build:mkl_open_source_only --define=tensorflow_mkldnn_contraction_kernel=0 -build:download_clang --crosstool_top=@local_config_download_clang//:toolchain -build:download_clang --define=using_clang=true -build:download_clang --action_env TF_DOWNLOAD_CLANG=1 -# Instruct clang to use LLD for linking. -# This only works with GPU builds currently, since Bazel sets -B/usr/bin in -# auto-generated CPU crosstool, forcing /usr/bin/ld.lld to be preferred over -# the downloaded one. -build:download_clang_use_lld --linkopt='-fuse-ld=lld' - # This config refers to building with CUDA available. It does not necessarily # mean that we build CUDA op kernels. build:using_cuda --define=using_cuda=true @@ -109,7 +177,6 @@ build --define=use_fast_cpp_protos=true build --define=allow_oversize_protos=true build --spawn_strategy=standalone -build --strategy=Genrule=standalone build -c opt # Make Bazel print out all options from rc files. @@ -132,36 +199,146 @@ build --define=PREFIX=/usr build --define=LIBDIR=$(PREFIX)/lib build --define=INCLUDEDIR=$(PREFIX)/include +# Suppress C++ compiler warnings, otherwise build logs become 10s of MBs. +build --copt=-w + # Suppress all warning messages. build:short_logs --output_filter=DONT_MATCH_ANYTHING +# Instruction set optimizations +# TODO(gunan): Create a feature in toolchains for avx/avx2 to +# avoid having to define linux/win separately. +build:avx_linux --copt=-mavx +build:avx2_linux --copt=-mavx2 +build:native_arch_linux --copt=-march=native +build:avx_win --copt=/arch=AVX +build:avx2_win --copt=/arch=AVX2 + +# Options to build TensorFlow 1.x or 2.x. +build:v1 --define=tf_api_version=1 +build:v2 --define=tf_api_version=2 +build:v1 --action_env=TF2_BEHAVIOR=0 +build:v2 --action_env=TF2_BEHAVIOR=1 +build --config=v2 +test --config=v2 + +# Enable XLA +build:xla --action_env=TF_ENABLE_XLA=1 +build:xla --define=with_xla_support=true + +# BEGIN TF REMOTE BUILD EXECUTION OPTIONS # Options when using remote execution +# WARNING: THESE OPTIONS WONT WORK IF YOU DO NOT HAVE PROPER AUTHENTICATION AND PERMISSIONS build:rbe --action_env=BAZEL_DO_NOT_DETECT_CPP_TOOLCHAIN=1 build:rbe --auth_enabled=true build:rbe --auth_scope=https://www.googleapis.com/auth/cloud-source-tools +build:rbe --bes_backend=buildeventservice.googleapis.com +build:rbe --bes_best_effort=false +build:rbe --bes_results_url="https://source.cloud.google.com/results/invocations" +build:rbe --bes_timeout=600s build:rbe --define=EXECUTOR=remote build:rbe --flaky_test_attempts=3 build:rbe --jobs=200 -build:rbe --remote_accept_cached=true -build:rbe --remote_cache=remotebuildexecution.googleapis.com -build:rbe --remote_executor=remotebuildexecution.googleapis.com -build:rbe --remote_local_fallback=false -build:rbe --remote_timeout=600 +build:rbe --remote_executor=grpcs://remotebuildexecution.googleapis.com +build:rbe --remote_timeout=3600 build:rbe --spawn_strategy=remote -build:rbe --strategy=Genrule=remote -build:rbe --strategy=Closure=remote -build:rbe --strategy=Javac=remote -build:rbe --strategy=TestRunner=remote -build:rbe --tls_enabled test:rbe --test_env=USER=anon -# Options to build TensorFlow 1.x or 2.x. -build:v1 --define=tf_api_version=1 -build:v2 --define=tf_api_version=2 -test:v1 --test_env=TF2_BEHAVIOR=0 -test:v2 --test_env=TF2_BEHAVIOR=1 -build --config=v2 -test --config=v2 +build:rbe --distinct_host_configuration=false + +build:rbe_linux --config=rbe +build:rbe_linux --action_env=PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/go/bin" +build:rbe_linux --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 +build:rbe_linux --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 +build:rbe_linux --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 +build:rbe_linux --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 + +# Non-rbe settings we should include because we do not run configure +build:rbe_linux --config=xla +build:rbe_linux --config=avx_linux +build:rbe_linux --config=short_logs +# TODO(gunan): Check why we need this specified in rbe, but not in other builds. +build:rbe_linux --linkopt=-lrt + +build:rbe_cpu_linux --config=rbe_linux +build:rbe_cpu_linux --crosstool_top="//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010:toolchain" +build:rbe_cpu_linux --extra_toolchains="//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010:cc-toolchain-k8" +build:rbe_cpu_linux --extra_execution_platforms"=@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010" +build:rbe_cpu_linux --host_platform="@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010" +build:rbe_cpu_linux --platforms="@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010" + +build:rbe_linux_cuda_nvcc --config=rbe_linux +build:rbe_linux_cuda_nvcc --crosstool_top="//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain" +build:rbe_linux_cuda_nvcc --extra_toolchains="//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.1:toolchain-linux-x86_64" +build:rbe_linux_cuda_nvcc --extra_execution_platforms="@org_tensorflow//third_party/toolchains:rbe_cuda10.1-cudnn7-ubuntu16.04-manylinux2010,@org_tensorflow//third_party/toolchains:rbe_cuda10.1-cudnn7-ubuntu16.04-manylinux2010-gpu" +build:rbe_linux_cuda_nvcc --host_platform="@org_tensorflow//third_party/toolchains:rbe_cuda10.1-cudnn7-ubuntu16.04-manylinux2010" +build:rbe_linux_cuda_nvcc --platforms="@org_tensorflow//third_party/toolchains:rbe_cuda10.1-cudnn7-ubuntu16.04-manylinux2010" +build:rbe_linux_cuda_nvcc --repo_env=TF_CUDA_CONFIG_REPO="@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/cuda10.1-cudnn7" +build:rbe_linux_cuda_nvccrepoo_env=TF_TENSORRT_CONFIG_REPO="@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/tensorrt6.0" +build:rbe_linux_cuda_nvcc --repo_env=TF_NEED_TENSORRT=1 +build:rbe_linux_cuda_nvcc --repo_env=TF_CUDA_VERSION=10 +build:rbe_linux_cuda_nvcc --repo_env=TF_CUDNN_VERSION=7 +build:rbe_linux_cuda_nvcc --repo_env=REMOTE_GPU_TESTING=1 +build:rbe_linux_cuda_nvcc --repo_env=TF_NEED_CUDA=1 +build:rbe_linux_cuda_nvcc --define=using_cuda_nvcc=true +test:rbe_linux_cuda_nvcc --test_env=LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64" + +common:rbe_gpu_linux --config=rbe_linux_cuda_nvcc + +build:rbe_linux_py2 --config=rbe_linux +build:rbe_linux_py2 --repo_env=PYTHON_BIN_PATH="/usr/bin/python2" +build:rbe_linux_py2 --python_path="/usr/bin/python2" +build:rbe_linux_py2 --repo_env=TF_PYTHON_CONFIG_REPO="@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/py" + +build:rbe_linux_py3 --config=rbe_linux +build:rbe_linux_py3 --repo_env=PYTHON_BIN_PATH="/usr/bin/python3" +build:rbe_linux_py3 --python_path="/usr/bin/python3" +build:rbe_linux_py3 --repo_env=TF_PYTHON_CONFIG_REPO="@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/py3" + +build:rbe_win --config=rbe +build:rbe_win --crosstool_top="@org_tensorflow//third_party/toolchains/preconfig/win_1803/bazel_026:toolchain" +build:rbe_win --extra_execution_platforms="@org_tensorflow//third_party/toolchains/preconfig/win_1803:rbe_windows_1803" +build:rbe_win --extra_toolchains="@org_tensorflow//third_party/toolchains/preconfig/win_1803/bazel_026:cc-toolchain-x64_windows" +build:rbe_win --host_javabase="@org_tensorflow//third_party/toolchains/preconfig/win_1803:windows_jdk8" +build:rbe_win --host_platform="@org_tensorflow//third_party/toolchains/preconfig/win_1803:rbe_windows_1803" +build:rbe_win --javabase="@org_tensorflow//third_party/toolchains/preconfig/win_1803:windows_jdk8" +build:rbe_win --platforms="@org_tensorflow//third_party/toolchains/preconfig/win_1803:rbe_windows_1803" +build:rbe_win --shell_executable=C:\\tools\\msys64\\usr\\bin\\bash.exe + +# Misc build options we need for windows +build:rbe_win --copt=-DWIN32_LEAN_AND_MEAN +build:rbe_win --host_copt=-DWIN32_LEAN_AND_MEAN +build:rbe_win --copt=-DNOGDI +build:rbe_win --host_copt=-DNOGDI +build:rbe_win --linkopt=/DEBUG +build:rbe_win --host_linkopt=/DEBUG +build:rbe_win --linkopt=/OPT:REF +build:rbe_win --host_linkopt=/OPT:REF +build:rbe_win --linkopt=/OPT:ICF +build:rbe_win --host_linkopt=/OPT:ICF +build:rbe_win --config=monolithic +build:rbe_win --experimental_strict_action_env=true +build:rbe_win --incompatible_windows_native_test_wrapper +# TODO(gunan): Remove once we use MSVC 2019 with latest patches. +build:rbe_win --define=override_eigen_strong_inline=true + +build:rbe_win_py37 --config=rbe +build:rbe_win_py37 --repo_env=PYTHON_BIN_PATH=C:\\Python37\\python.exe +build:rbe_win_py37 --repo_env=PYTHON_LIB_PATH=C:\\Python37\\lib\\site-packages +build:rbe_win_py37 --repo_env=TF_PYTHON_CONFIG_REPO=@org_tensorflow//third_party/toolchains/preconfig/win_1803/py37 +build:rbe_win_py37 --python_path=C:\\Python37\\python.exe + +# These you may need to change for your own GCP project. +build:tensorflow_testing_rbe --project_id=tensorflow-testing +common:tensorflow_testing_rbe_linux --remote_instance_name=tensorflow-testing/instances/default_instance +build:tensorflow_testing_rbe_linux --config=tensorflow_testing_rbe +build:tensorflow_testing_rbe_linux --config=rbe +build:tensorflow_testing_rbe_linux --config=rbe_linux + +common:tensorflow_testing_rbe_win --remote_instance_name=projects/tensorflow-testing/instances/windows +build:tensorflow_testing_rbe_win --config=tensorflow_testing_rbe +build:tensorflow_testing_rbe_win --config=rbe_win +# END TF REMOTE BUILD EXECUTION OPTIONS # Default options should come above this line diff --git a/README.md b/README.md index 5a66b9bb03a029..caf7fd4c6af8dd 100644 --- a/README.md +++ b/README.md @@ -116,6 +116,8 @@ The TensorFlow project strives to abide by generally accepted best practices in Build Type | Status | Artifacts --------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------- +**Linux AMD ROCm GPU** Nightly | [![Build Status](http://ml-ci.amd.com:21096/job/tensorflow-rocm-nightly/badge/icon)](http://ml-ci.amd.com:21096/job/tensorflow-rocm-nightly) | [Nightly](http://ml-ci.amd.com:21096/job/tensorflow-rocm-nightly/lastSuccessfulBuild/) +**Linux AMD ROCm GPU** Stable Release | [![Build Status](http://ml-ci.amd.com:21096/job/tensorflow-rocm-release/badge/icon)](http://ml-ci.amd.com:21096/job/tensorflow-rocm-release/) | [Release](http://ml-ci.amd.com:21096/job/tensorflow-rocm-release/lastSuccessfulBuild/) **Linux s390x** Nightly | [![Build Status](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/badge/icon)](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/) | [Nightly](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_CI/) **Linux s390x CPU** Stable Release | [![Build Status](http://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_Release_Build/badge/icon)](https://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_Release_Build/) | [Release](https://ibmz-ci.osuosl.org/job/TensorFlow_IBMZ_Release_Build/) **Linux ppc64le CPU** Nightly | [![Build Status](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Build/badge/icon)](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Build/) | [Nightly](https://powerci.osuosl.org/job/TensorFlow_PPC64LE_CPU_Nightly_Artifact/) diff --git a/RELEASE.md b/RELEASE.md index 46c7af892a9b74..0d1af5708ae8b5 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,3 +1,11 @@ +# Release 2.0.1 + +## Bug Fixes and Other Changes +* Fixes a security vulnerability where converting a Python string to a `tf.float16` value produces a segmentation fault ([CVE-2020-5215](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2020-5215)) +* Updates `curl` to `7.66.0` to handle [CVE-2019-5482](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-5482) and [CVE-2019-5481](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-5481) +* Updates `sqlite3` to `3.30.01` to handle [CVE-2019-19646](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-19646), [CVE-2019-19645](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-19645) and [CVE-2019-16168](https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2019-16168) + + # Release 2.0.0 ## Major Features and Improvements diff --git a/tensorflow/c/eager/c_api_experimental_test.cc b/tensorflow/c/eager/c_api_experimental_test.cc index 34f325cc2b50f3..e5dc4f44ac5d1f 100644 --- a/tensorflow/c/eager/c_api_experimental_test.cc +++ b/tensorflow/c/eager/c_api_experimental_test.cc @@ -83,7 +83,10 @@ void ExecuteWithProfiling(bool async) { if (!gpu_device_name.empty()) { EXPECT_TRUE(HasSubstr(profile_proto_str, "/device:GPU:0")); // device name with "stream:all" is collected by Device Tracer. +#ifndef TENSORFLOW_USE_ROCM + // ROCm platform does not yet support stream level tracing EXPECT_TRUE(HasSubstr(profile_proto_str, "stream:all")); +#endif } // "/host:CPU" is collected by TraceMe EXPECT_TRUE(HasSubstr(profile_proto_str, "/host:CPU")); diff --git a/tensorflow/c/kernels/BUILD b/tensorflow/c/kernels/BUILD index 7cac7d78235feb..770352c62c1585 100644 --- a/tensorflow/c/kernels/BUILD +++ b/tensorflow/c/kernels/BUILD @@ -1,8 +1,8 @@ load( "//tensorflow:tensorflow.bzl", "tf_cc_test", - "tf_kernel_library", "tf_gen_op_libs", + "tf_kernel_library", ) package( diff --git a/tensorflow/cc/profiler/BUILD b/tensorflow/cc/profiler/BUILD index d18a0bcab0c841..5b4a105eb28fe4 100644 --- a/tensorflow/cc/profiler/BUILD +++ b/tensorflow/cc/profiler/BUILD @@ -9,6 +9,7 @@ tf_cuda_cc_test( name = "profiler_test", srcs = ["profiler_test.cc"], tags = [ + "no_rocm", # stream level tracing not supported on ROCm "nogpu", # b/77649654 ], deps = [ diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD index 1dc2ae0637c709..c14fd04a915686 100644 --- a/tensorflow/compiler/jit/BUILD +++ b/tensorflow/compiler/jit/BUILD @@ -1,4 +1,4 @@ -load("//tensorflow:tensorflow.bzl", "tf_cc_test", "cc_header_only_library") +load("//tensorflow:tensorflow.bzl", "cc_header_only_library", "tf_cc_test") load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda") load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library", "tf_jit_compilation_passes_extra_deps") load("//tensorflow/core/platform:default/build_config.bzl", "tf_additional_all_protos", "tf_proto_library") diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD index 0a4448bdf4047c..a2c63ca3f2807a 100644 --- a/tensorflow/compiler/xla/BUILD +++ b/tensorflow/compiler/xla/BUILD @@ -1,4 +1,4 @@ -load("//tensorflow:tensorflow.bzl", "tf_cc_test", "cc_header_only_library") +load("//tensorflow:tensorflow.bzl", "cc_header_only_library", "tf_cc_test") load("//tensorflow/compiler/xla:xla.bzl", "xla_proto_library") load( "//tensorflow/core/platform:default/build_config.bzl", diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD index 97b3db05263c5b..5194c7be6518a9 100644 --- a/tensorflow/core/BUILD +++ b/tensorflow/core/BUILD @@ -4605,7 +4605,7 @@ tf_cc_test( size = "small", srcs = ["common_runtime/constant_folding_test.cc"], linkstatic = tf_kernel_tests_linkstatic(), - tags = tf_cuda_tests_tags(), + tags = tf_cuda_tests_tags() + ["no_rocm"], deps = [ ":core", ":core_cpu", @@ -4671,6 +4671,7 @@ tf_cuda_cc_test( size = "small", srcs = ["common_runtime/process_function_library_runtime_test.cc"], linkstatic = tf_kernel_tests_linkstatic(), + tags = ["no_rocm"], deps = [ ":core_cpu", ":core_cpu_internal", diff --git a/tensorflow/core/common_runtime/direct_session_test.cc b/tensorflow/core/common_runtime/direct_session_test.cc index 8da13aaca22f62..ef5301fdac054e 100644 --- a/tensorflow/core/common_runtime/direct_session_test.cc +++ b/tensorflow/core/common_runtime/direct_session_test.cc @@ -51,9 +51,11 @@ limitations under the License. #include "tensorflow/core/public/session_options.h" #include "tensorflow/core/util/device_name_utils.h" -#ifdef GOOGLE_CUDA +#if GOOGLE_CUDA #include "third_party/gpus/cuda/include/cuda.h" #include "third_party/gpus/cuda/include/cuda_runtime_api.h" +#elif TENSORFLOW_USE_ROCM +#include "rocm/include/hip/hip_runtime.h" #endif // GOOGLE_CUDA namespace tensorflow { @@ -2089,6 +2091,12 @@ bool IsCUDATensor(const Tensor& t) { if (err == cudaErrorInvalidValue) return false; CHECK_EQ(cudaSuccess, err) << cudaGetErrorString(err); return (attributes.memoryType == cudaMemoryTypeDevice); +#elif TENSORFLOW_USE_ROCM + hipPointerAttribute_t attributes; + hipError_t err = hipPointerGetAttributes(&attributes, t.tensor_data().data()); + if (err == hipErrorInvalidValue) return false; + CHECK_EQ(hipSuccess, err) << hipGetErrorString(err); + return (attributes.memoryType == hipMemoryTypeDevice); #else return false; #endif diff --git a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc index f848bdf74719e2..623cd479364a40 100644 --- a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc +++ b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc @@ -33,9 +33,11 @@ limitations under the License. #include "tensorflow/core/public/session_options.h" #include "tensorflow/core/public/version.h" -#ifdef GOOGLE_CUDA +#if GOOGLE_CUDA #include "third_party/gpus/cuda/include/cuda.h" #include "third_party/gpus/cuda/include/cuda_runtime_api.h" +#elif TENSORFLOW_USE_ROCM +#include "rocm/include/hip/hip_runtime.h" #endif // GOOGLE_CUDA namespace tensorflow { @@ -122,7 +124,7 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test { } Tensor GPUToCPU(const Tensor& device_tensor) { -#ifdef GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM CHECK(gpu_device_); CHECK(gpu_device_->tensorflow_gpu_device_info() != nullptr); DeviceContext* device_context = @@ -146,7 +148,7 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test { } Tensor CPUToGPU(const Tensor& cpu_tensor) { -#ifdef GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM CHECK(gpu_device_); CHECK(gpu_device_->tensorflow_gpu_device_info() != nullptr); DeviceContext* device_context = @@ -461,6 +463,12 @@ bool IsCUDATensor(const Tensor& t) { if (err == cudaErrorInvalidValue) return false; CHECK_EQ(cudaSuccess, err) << cudaGetErrorString(err); return (attributes.memoryType == cudaMemoryTypeDevice); +#elif TENSORFLOW_USE_ROCM + hipPointerAttribute_t attributes; + hipError_t err = hipPointerGetAttributes(&attributes, t.tensor_data().data()); + if (err == hipErrorInvalidValue) return false; + CHECK_EQ(hipSuccess, err) << hipGetErrorString(err); + return (attributes.memoryType == hipMemoryTypeDevice); #else CHECK(false) << "IsCUDATensor should not be called when CUDA is not available"; diff --git a/tensorflow/core/grappler/clusters/utils_test.cc b/tensorflow/core/grappler/clusters/utils_test.cc index 3cf72fd8170ca2..6b7013d3038882 100644 --- a/tensorflow/core/grappler/clusters/utils_test.cc +++ b/tensorflow/core/grappler/clusters/utils_test.cc @@ -40,6 +40,18 @@ TEST(UtilsTest, GetLocalGPUInfo) { properties = GetLocalGPUInfo(PlatformGpuId(0)); EXPECT_EQ("GPU", properties.type()); EXPECT_EQ("NVIDIA", properties.vendor()); +#elif TENSORFLOW_USE_ROCM + LOG(INFO) << "ROCm is enabled."; + DeviceProperties properties; + + // Invalid platform GPU ID. + properties = GetLocalGPUInfo(PlatformGpuId(100)); + EXPECT_EQ("UNKNOWN", properties.type()); + + // Succeed when a valid platform GPU id was inserted. + properties = GetLocalGPUInfo(PlatformGpuId(0)); + EXPECT_EQ("GPU", properties.type()); + EXPECT_EQ("Advanced Micro Devices, Inc", properties.vendor()); #else LOG(INFO) << "CUDA is not enabled."; DeviceProperties properties; @@ -73,6 +85,8 @@ TEST(UtilsTest, GetDeviceInfo) { EXPECT_EQ("GPU", properties.type()); #if GOOGLE_CUDA EXPECT_EQ("NVIDIA", properties.vendor()); +#elif TENSORFLOW_USE_ROCM + EXPECT_EQ("Advanced Micro Devices, Inc", properties.vendor()); #endif // TF to platform GPU id mapping entry doesn't exist. @@ -81,7 +95,7 @@ TEST(UtilsTest, GetDeviceInfo) { properties = GetDeviceInfo(device); EXPECT_EQ("UNKNOWN", properties.type()); -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM // Invalid platform GPU id. TF_ASSERT_OK( GpuIdManager::InsertTfPlatformGpuIdPair(TfGpuId(0), PlatformGpuId(100))); @@ -94,7 +108,11 @@ TEST(UtilsTest, GetDeviceInfo) { device.id = 1; properties = GetDeviceInfo(device); EXPECT_EQ("GPU", properties.type()); +#if GOOGLE_CUDA EXPECT_EQ("NVIDIA", properties.vendor()); +#elif TENSORFLOW_USE_ROCM + EXPECT_EQ("Advanced Micro Devices, Inc", properties.vendor()); +#endif #endif } diff --git a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc index 7a9110e72abcb8..a346856745d75c 100644 --- a/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc +++ b/tensorflow/core/grappler/optimizers/pin_to_host_optimizer_test.cc @@ -203,7 +203,7 @@ TEST_F(PinToHostOptimizerTest, Identity) { // If CUDA, then there is a GPU kernel registration that is pinned to Host // memory. Consequently, `b` will be mapped to Host correct if there is // a GPU kernel registered. -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM EXPECT_EQ(node.device(), "/device:CPU:0"); #else EXPECT_TRUE(node.device().empty()); diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc index ab338a2550c195..fcf86754b5c7e4 100644 --- a/tensorflow/core/kernels/conv_ops_test.cc +++ b/tensorflow/core/kernels/conv_ops_test.cc @@ -1001,6 +1001,10 @@ class FusedConv2DWithBatchNormOpTest : public FusedConv2DOpTest {}; TYPED_TEST_SUITE_P(FusedConv2DWithBiasOpTest); TYPED_TEST_SUITE_P(FusedConv2DWithBatchNormOpTest); +// ROCm does not yet support the _FusedConv2D op, +// Therefore disable tests that check _FusedConv2D, when building with ROCm + +#ifndef TENSORFLOW_USE_ROCM // -------------------------------------------------------------------------- // // Conv2D + BiasAdd + {Activation} // // -------------------------------------------------------------------------- // @@ -1165,4 +1169,5 @@ using FusedBatchNormDataTypes = ::testing::Types; INSTANTIATE_TYPED_TEST_SUITE_P(Test, FusedConv2DWithBatchNormOpTest, FusedBatchNormDataTypes); +#endif // TENSORFLOW_USE_ROCM } // namespace tensorflow diff --git a/tensorflow/core/kernels/in_topk_op.cc b/tensorflow/core/kernels/in_topk_op.cc index a8ee00e080e5ee..22d833395f07e1 100644 --- a/tensorflow/core/kernels/in_topk_op.cc +++ b/tensorflow/core/kernels/in_topk_op.cc @@ -116,7 +116,7 @@ REGISTER_KERNEL_BUILDER(Name("InTopKV2") .TypeConstraint("T"), InTopK); -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM // Forward declarations of the functor specializations for GPU. namespace functor { @@ -142,6 +142,6 @@ REGISTER_KERNEL_BUILDER( Name("InTopKV2").Device(DEVICE_GPU).TypeConstraint("T"), InTopK); -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM } // namespace tensorflow diff --git a/tensorflow/core/kernels/in_topk_op.h b/tensorflow/core/kernels/in_topk_op.h index 52716f2d272709..f48932cdbbfeee 100644 --- a/tensorflow/core/kernels/in_topk_op.h +++ b/tensorflow/core/kernels/in_topk_op.h @@ -16,9 +16,9 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_IN_TOPK_OP_H_ #define TENSORFLOW_CORE_KERNELS_IN_TOPK_OP_H_ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" #include "tensorflow/core/framework/bounds_check.h" diff --git a/tensorflow/core/kernels/in_topk_op_gpu.cu.cc b/tensorflow/core/kernels/in_topk_op_gpu.cu.cc index 1894dedddf108a..4c59e1f7bed119 100644 --- a/tensorflow/core/kernels/in_topk_op_gpu.cu.cc +++ b/tensorflow/core/kernels/in_topk_op_gpu.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if (defined(GOOGLE_CUDA) && GOOGLE_CUDA) +#if (defined(GOOGLE_CUDA) && GOOGLE_CUDA) || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -41,7 +41,7 @@ __global__ void ComputePredictionMaskKernel( const TargetT* targets, // dims: [ num_targets ] int64* mask, // dims: [ num_targets x num_classes ] int num_targets, int num_classes) { - CUDA_1D_KERNEL_LOOP(i, num_targets * num_classes) { + GPU_1D_KERNEL_LOOP(i, num_targets * num_classes) { const int batch_index = i / num_classes; TargetT target_idx = ldg(targets + batch_index); @@ -118,7 +118,7 @@ struct InTopKFunctor { const auto& d = context->eigen_device(); // Compute a mask for all predictions. - CudaLaunchConfig config = GetGpuLaunchConfig(num_targets * num_classes, d); + GpuLaunchConfig config = GetGpuLaunchConfig(num_targets * num_classes, d); OP_REQUIRES_OK( context, GpuLaunchKernel(ComputePredictionMaskKernel, config.block_count, config.thread_per_block, 0, @@ -173,4 +173,4 @@ DEFINE_GPU_KERNELS(float, int64); } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/kernels/reduction_ops_all.cc b/tensorflow/core/kernels/reduction_ops_all.cc index 4a34c4ef513bdd..70ea87a2dfcd43 100644 --- a/tensorflow/core/kernels/reduction_ops_all.cc +++ b/tensorflow/core/kernels/reduction_ops_all.cc @@ -30,7 +30,7 @@ REGISTER_KERNEL_BUILDER( .HostMemory("reduction_indices"), ReductionOp); -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER_KERNEL_BUILDER( Name("All") .TypeConstraint("Tidx") diff --git a/tensorflow/core/kernels/reduction_ops_any.cc b/tensorflow/core/kernels/reduction_ops_any.cc index 6c0519de95e777..cd0ce289e51c66 100644 --- a/tensorflow/core/kernels/reduction_ops_any.cc +++ b/tensorflow/core/kernels/reduction_ops_any.cc @@ -30,7 +30,7 @@ REGISTER_KERNEL_BUILDER( .HostMemory("reduction_indices"), ReductionOp); -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM REGISTER_KERNEL_BUILDER( Name("Any") .TypeConstraint("Tidx") diff --git a/tensorflow/core/kernels/reduction_ops_common_gpu.h b/tensorflow/core/kernels/reduction_ops_common_gpu.h index 9af43f885f91a2..2415f1dbc6d410 100644 --- a/tensorflow/core/kernels/reduction_ops_common_gpu.h +++ b/tensorflow/core/kernels/reduction_ops_common_gpu.h @@ -15,8 +15,8 @@ limitations under the License. #ifndef TENSORFLOW_CORE_KERNELS_REDUCTION_OPS_COMMON_GPU_H_ #define TENSORFLOW_CORE_KERNELS_REDUCTION_OPS_COMMON_GPU_H_ -#if !GOOGLE_CUDA -#error This file must only be included when building with Cuda support +#if !GOOGLE_CUDA && !TENSORFLOW_USE_ROCM +#error This file must only be included when building with GPU support #endif #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor" diff --git a/tensorflow/core/kernels/reduction_ops_euclidean.cc b/tensorflow/core/kernels/reduction_ops_euclidean.cc index 9f4bf50e7ca0ec..cf719e76cd84ce 100644 --- a/tensorflow/core/kernels/reduction_ops_euclidean.cc +++ b/tensorflow/core/kernels/reduction_ops_euclidean.cc @@ -33,7 +33,7 @@ namespace tensorflow { TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER(Name("EuclideanNorm") \ @@ -51,8 +51,10 @@ TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); +#if GOOGLE_CUDA TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); +#endif #undef REGISTER_GPU_KERNELS #endif diff --git a/tensorflow/core/kernels/reduction_ops_gpu_bool.cu.cc b/tensorflow/core/kernels/reduction_ops_gpu_bool.cu.cc index 79ec1d59dfa9ce..89bcf1d7ced8fa 100644 --- a/tensorflow/core/kernels/reduction_ops_gpu_bool.cu.cc +++ b/tensorflow/core/kernels/reduction_ops_gpu_bool.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -59,4 +59,4 @@ DEFINE_FOR_TYPE_AND_R(bool, Eigen::internal::OrReducer); } // end namespace functor } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/kernels/reduction_ops_gpu_double.cu.cc b/tensorflow/core/kernels/reduction_ops_gpu_double.cu.cc index c492308a916259..c952c4c9fa4207 100644 --- a/tensorflow/core/kernels/reduction_ops_gpu_double.cu.cc +++ b/tensorflow/core/kernels/reduction_ops_gpu_double.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -67,4 +67,4 @@ DEFINE_FOR_ALL_REDUCERS(double); } // end namespace functor } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/kernels/reduction_ops_gpu_float.cu.cc b/tensorflow/core/kernels/reduction_ops_gpu_float.cu.cc index b006311c125c1e..92f4b9d707c61e 100644 --- a/tensorflow/core/kernels/reduction_ops_gpu_float.cu.cc +++ b/tensorflow/core/kernels/reduction_ops_gpu_float.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -67,4 +67,4 @@ DEFINE_FOR_ALL_REDUCERS(float); } // end namespace functor } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/kernels/reduction_ops_gpu_int.cu.cc b/tensorflow/core/kernels/reduction_ops_gpu_int.cu.cc index 91a33b92cb6663..c35d8c2ec8607a 100644 --- a/tensorflow/core/kernels/reduction_ops_gpu_int.cu.cc +++ b/tensorflow/core/kernels/reduction_ops_gpu_int.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -68,4 +68,4 @@ DEFINE_FOR_ALL_REDUCERS(int64); } // end namespace functor } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/kernels/reduction_ops_half_mean_sum.cu.cc b/tensorflow/core/kernels/reduction_ops_half_mean_sum.cu.cc index f33d504e25a202..bbb34c9d3ba450 100644 --- a/tensorflow/core/kernels/reduction_ops_half_mean_sum.cu.cc +++ b/tensorflow/core/kernels/reduction_ops_half_mean_sum.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -64,4 +64,4 @@ DEFINE_FOR_ALL_REDUCERS(Eigen::half); } // end namespace functor } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/kernels/reduction_ops_half_prod_max_min.cu.cc b/tensorflow/core/kernels/reduction_ops_half_prod_max_min.cu.cc index 84fd389bb38402..d2a180ba3519bc 100644 --- a/tensorflow/core/kernels/reduction_ops_half_prod_max_min.cu.cc +++ b/tensorflow/core/kernels/reduction_ops_half_prod_max_min.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -64,4 +64,4 @@ DEFINE_FOR_ALL_REDUCERS(Eigen::half); } // end namespace functor } // end namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/kernels/reduction_ops_max.cc b/tensorflow/core/kernels/reduction_ops_max.cc index 8bfa44b2d06e03..fe9775f7f1d4e5 100644 --- a/tensorflow/core/kernels/reduction_ops_max.cc +++ b/tensorflow/core/kernels/reduction_ops_max.cc @@ -33,7 +33,7 @@ namespace tensorflow { TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER( \ diff --git a/tensorflow/core/kernels/reduction_ops_mean.cc b/tensorflow/core/kernels/reduction_ops_mean.cc index 67c974edda284d..d314f1953dc6b1 100644 --- a/tensorflow/core/kernels/reduction_ops_mean.cc +++ b/tensorflow/core/kernels/reduction_ops_mean.cc @@ -33,7 +33,7 @@ namespace tensorflow { TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER( \ @@ -51,8 +51,10 @@ TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); .HostMemory("reduction_indices"), \ ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); +#if GOOGLE_CUDA TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); +#endif #undef REGISTER_GPU_KERNELS #endif diff --git a/tensorflow/core/kernels/reduction_ops_min.cc b/tensorflow/core/kernels/reduction_ops_min.cc index 5c537c5b9c75af..9f1feae969e1a0 100644 --- a/tensorflow/core/kernels/reduction_ops_min.cc +++ b/tensorflow/core/kernels/reduction_ops_min.cc @@ -33,7 +33,7 @@ namespace tensorflow { TF_CALL_REAL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER( \ diff --git a/tensorflow/core/kernels/reduction_ops_prod.cc b/tensorflow/core/kernels/reduction_ops_prod.cc index e9b23df74604da..0642bad92185e6 100644 --- a/tensorflow/core/kernels/reduction_ops_prod.cc +++ b/tensorflow/core/kernels/reduction_ops_prod.cc @@ -33,7 +33,7 @@ namespace tensorflow { TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER(Name("Prod") \ @@ -52,8 +52,10 @@ TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); Eigen::internal::ProdReducer>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_int32(REGISTER_GPU_KERNELS); +#if GOOGLE_CUDA TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); +#endif #undef REGISTER_GPU_KERNELS #endif diff --git a/tensorflow/core/kernels/reduction_ops_sum.cc b/tensorflow/core/kernels/reduction_ops_sum.cc index cf0d0f5c714643..d79684df290cfc 100644 --- a/tensorflow/core/kernels/reduction_ops_sum.cc +++ b/tensorflow/core/kernels/reduction_ops_sum.cc @@ -33,7 +33,7 @@ namespace tensorflow { TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); #undef REGISTER_CPU_KERNELS -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_GPU_KERNELS(type) \ REGISTER_KERNEL_BUILDER( \ @@ -52,8 +52,10 @@ TF_CALL_NUMBER_TYPES(REGISTER_CPU_KERNELS); ReductionOp>); TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU_KERNELS); TF_CALL_int64(REGISTER_GPU_KERNELS); +#if GOOGLE_CUDA TF_CALL_complex64(REGISTER_GPU_KERNELS); TF_CALL_complex128(REGISTER_GPU_KERNELS); +#endif #undef REGISTER_GPU_KERNELS // A special GPU kernel for int32. diff --git a/tensorflow/core/kernels/roll_op.cc b/tensorflow/core/kernels/roll_op.cc index 787d4aa247cead..06e5ae1ff87128 100644 --- a/tensorflow/core/kernels/roll_op.cc +++ b/tensorflow/core/kernels/roll_op.cc @@ -360,7 +360,7 @@ struct Roll { TF_CALL_ALL_TYPES(REGISTER_CPU); #undef REGISTER_CPU -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define REGISTER_KERNEL(type) \ REGISTER_KERNEL_BUILDER(Name("Roll") \ .Device(DEVICE_GPU) \ @@ -402,5 +402,5 @@ TF_CALL_complex64(REGISTER_KERNEL); TF_CALL_complex128(REGISTER_KERNEL); #undef REGISTER_KERNEL -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM } // namespace tensorflow diff --git a/tensorflow/core/kernels/roll_op_gpu.cu.cc b/tensorflow/core/kernels/roll_op_gpu.cu.cc index c5ef02d84a6d9a..e148faf058c225 100644 --- a/tensorflow/core/kernels/roll_op_gpu.cu.cc +++ b/tensorflow/core/kernels/roll_op_gpu.cu.cc @@ -13,7 +13,7 @@ See the License for the specific language governing permissions and limitations under the License. ==============================================================================*/ -#if GOOGLE_CUDA +#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM #define EIGEN_USE_GPU @@ -33,7 +33,7 @@ template __global__ void RollKernel(const int32 nthreads, const int32 num_dims, const T* input, T* output, const int32* dim_size, const int32* threshold, const int64* dim_range) { - CUDA_1D_KERNEL_LOOP(out_idx, nthreads) { + GPU_1D_KERNEL_LOOP(out_idx, nthreads) { int64 offset = 0; for (int i = 0; i < num_dims; i++) { const int64 stride = dim_range[i] / dim_size[i]; @@ -71,7 +71,7 @@ struct Roll { d.memcpyHostToDevice(thres_buf, threshold.data(), thres_bytes); d.memcpyHostToDevice(range_buf, dim_range.data(), range_bytes); - CudaLaunchConfig cfg = GetGpuLaunchConfig(num_elements, d); + GpuLaunchConfig cfg = GetGpuLaunchConfig(num_elements, d); TF_CHECK_OK(GpuLaunchKernel(RollKernel, cfg.block_count, cfg.thread_per_block, 0, d.stream(), @@ -98,4 +98,4 @@ TF_CALL_complex128(DEFINE_GPU_SPECS); } // namespace functor } // namespace tensorflow -#endif // GOOGLE_CUDA +#endif // GOOGLE_CUDA || TENSORFLOW_USE_ROCM diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD index 7439a7f39881bd..3ee02e0f0b1fc5 100644 --- a/tensorflow/core/profiler/internal/BUILD +++ b/tensorflow/core/profiler/internal/BUILD @@ -1,4 +1,4 @@ -load("//tensorflow:tensorflow.bzl", "tf_cc_test", "if_not_windows", "tf_cuda_library") +load("//tensorflow:tensorflow.bzl", "if_not_windows", "tf_cc_test", "tf_cuda_library") load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test") package( diff --git a/tensorflow/core/profiler/internal/gpu/BUILD b/tensorflow/core/profiler/internal/gpu/BUILD index dce7d39c0c2c15..bb4b97b9402787 100644 --- a/tensorflow/core/profiler/internal/gpu/BUILD +++ b/tensorflow/core/profiler/internal/gpu/BUILD @@ -10,9 +10,9 @@ alias( load( "//tensorflow:tensorflow.bzl", + "if_cuda_is_configured_compat", "tf_copts", "tf_cuda_library", - "if_cuda_is_configured_compat", ) tf_cuda_library( diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h index 8d4627f4e2f324..b05208061c8829 100644 --- a/tensorflow/core/public/version.h +++ b/tensorflow/core/public/version.h @@ -22,7 +22,7 @@ limitations under the License. // tensorflow/tools/pip_package/setup.py #define TF_MAJOR_VERSION 2 #define TF_MINOR_VERSION 0 -#define TF_PATCH_VERSION 0 +#define TF_PATCH_VERSION 1 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1", // "-beta", "-rc", "-rc.1") diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD index b4551daf2ce55c..3b9af1331728cc 100644 --- a/tensorflow/python/BUILD +++ b/tensorflow/python/BUILD @@ -1378,6 +1378,16 @@ py_library( ], ) +tf_py_test( + name = "framework_constant_op_test", + size = "small", + srcs = ["framework/constant_op_test.py"], + additional_deps = [ + ":constant_op", + ], + main = "framework/constant_op_test.py", +) + tf_py_test( name = "framework_registry_test", size = "small", @@ -2501,6 +2511,7 @@ cuda_py_test( ], tags = [ "no_cuda_on_cpu_tap", + "no_rocm", "no_windows", ], ) @@ -3431,6 +3442,7 @@ cuda_py_test( "//tensorflow/python/kernel_tests/random:util", "//tensorflow/python/distribute:mirrored_strategy", ], + tags = ["no_rocm"], xla_enable_strict_auto_jit = False, ) diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD index 234d13f2c93083..9f0e3add606447 100644 --- a/tensorflow/python/eager/BUILD +++ b/tensorflow/python/eager/BUILD @@ -259,6 +259,7 @@ cuda_py_test( "//tensorflow/python:training", ], tags = [ + "no_mac", # fails on this branch "no_rocm", "no_windows", #TODO(b/139745667) ], diff --git a/tensorflow/python/framework/constant_op_test.py b/tensorflow/python/framework/constant_op_test.py new file mode 100644 index 00000000000000..da0fb64fde6f2b --- /dev/null +++ b/tensorflow/python/framework/constant_op_test.py @@ -0,0 +1,61 @@ +# Copyright 2020 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +"""Tests for tensorflow.python.framework.constant_op.""" + +from __future__ import absolute_import +from __future__ import division +from __future__ import print_function + +from absl.testing import parameterized + +from tensorflow.python.framework import constant_op +from tensorflow.python.framework import dtypes +from tensorflow.python.framework import ops +from tensorflow.python.platform import test + + +class ConstantOpTest(test.TestCase, parameterized.TestCase): + + @parameterized.parameters( + dtypes.bfloat16, + dtypes.complex128, + dtypes.complex64, + dtypes.double, + dtypes.float16, + dtypes.float32, + dtypes.float64, + dtypes.half, + dtypes.int16, + dtypes.int32, + dtypes.int64, + dtypes.int8, + dtypes.qint16, + dtypes.qint32, + dtypes.qint8, + dtypes.quint16, + dtypes.quint8, + dtypes.uint16, + dtypes.uint32, + dtypes.uint64, + dtypes.uint8, + ) + def test_convert_string_to_number(self, dtype): + with self.assertRaises(TypeError): + constant_op.constant("hello", dtype) + + +if __name__ == "__main__": + ops.enable_eager_execution() + test.main() diff --git a/tensorflow/python/keras/optimizer_v2/BUILD b/tensorflow/python/keras/optimizer_v2/BUILD index 715c7938600d6e..23eef48862016a 100644 --- a/tensorflow/python/keras/optimizer_v2/BUILD +++ b/tensorflow/python/keras/optimizer_v2/BUILD @@ -222,6 +222,7 @@ cuda_py_test( ], shard_count = 8, tags = [ + "no_rocm", "no_windows", ], xla_enable_strict_auto_jit = True, diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD index 3baa43e23c8bf2..2d00b05982e2bd 100644 --- a/tensorflow/python/kernel_tests/BUILD +++ b/tensorflow/python/kernel_tests/BUILD @@ -3275,6 +3275,7 @@ cuda_py_test( "//tensorflow/python:framework_for_generated_wrappers", ], tags = [ + "no_mac", # timeouts on GPU "no_pip", "notap", # http://b/31080670 ], diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py index cfb60884a7884d..bc96774bfa6163 100644 --- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py +++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py @@ -61,7 +61,8 @@ def _DtypesToTest(self, use_gpu): # as we will be using its gradients as reference for fp16 gradients. return optional_float64 + [dtypes.float32, dtypes.float16] else: - return [dtypes.float64, dtypes.float32, dtypes.float16] + return [dtypes.float32, dtypes.float16] + ([dtypes.float64] \ + if not test.is_built_with_rocm else []) def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, stride, padding, data_format, dtype, use_gpu): diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py index 1d935ee81235e6..9032d5a1b635c0 100644 --- a/tensorflow/python/kernel_tests/init_ops_test.py +++ b/tensorflow/python/kernel_tests/init_ops_test.py @@ -753,6 +753,13 @@ def testShapesValues(self): else: shape = [4, 16, 16, 16, 64] convolution = convolutional.conv3d + + if test.is_built_with_rocm(): + # This subtest triggers a known bug in ROCm runtime code + # The bug has been fixed and will be available in ROCm 2.7 + # Re-enable this test once ROCm 2.7 is released + continue + inputs = random_ops.random_normal(shape, dtype=dtype) inputs_2norm = linalg_ops.norm(inputs) outputs = convolution( diff --git a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py index 520e4d3eb8de1c..ed47e8980d993e 100644 --- a/tensorflow/python/kernel_tests/matrix_exponential_op_test.py +++ b/tensorflow/python/kernel_tests/matrix_exponential_op_test.py @@ -63,7 +63,7 @@ def _verifyExponential(self, x, np_type): else: np_ans = np_expm(inp) out = self.evaluate(tf_ans) - self.assertAllClose(np_ans, out, rtol=1e-4, atol=1e-3) + self.assertAllClose(np_ans, out, rtol=1e-3, atol=1e-3) def _verifyExponentialReal(self, x): for np_type in [np.float32, np.float64]: diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py index f4b7a5cb625fb7..afabad961503d6 100644 --- a/tensorflow/python/kernel_tests/pooling_ops_test.py +++ b/tensorflow/python/kernel_tests/pooling_ops_test.py @@ -766,7 +766,8 @@ def testDepthwiseMaxPoolInvalidConfigs(self): # The following are tests that verify that the CPU and GPU implementations # produce the same results. def _CompareMaxPoolingFwd(self, input_shape, ksize, strides, padding): - for dtype in np.float64, np.float32, np.float16: + for dtype in [np.float32, np.float16] + [np.float64] \ + if not test.is_built_with_rocm() else []: tensor_input = np.random.rand(*input_shape).astype(dtype) with self.cached_session(use_gpu=True): t = constant_op.constant(tensor_input, shape=input_shape) @@ -780,7 +781,8 @@ def _CompareMaxPoolingFwd(self, input_shape, ksize, strides, padding): def _CompareMaxPoolingBk(self, input_shape, output_shape, ksize, strides, padding): - for dtype in np.float64, np.float32, np.float16: + for dtype in [np.float32, np.float16] + [np.float64] \ + if not test.is_built_with_rocm() else []: # Generate numbers in a narrow range, so that there are many duplicates # in the input. tensor_input = np.random.random_integers(0, 3, input_shape).astype(dtype) @@ -810,7 +812,8 @@ def _CompareMaxPoolingBk(self, input_shape, output_shape, ksize, strides, def _CompareMaxPoolingGradBk(self, input_shape, output_shape, ksize, strides, padding): - for dtype in np.float64, np.float32, np.float16: + for dtype in [np.float32, np.float16] + [np.float64] \ + if not test.is_built_with_rocm() else []: # Generate numbers in a narrow range, so that there are many duplicates # in the input. tensor_input = np.random.random_integers(0, 3, input_shape).astype(dtype) diff --git a/tensorflow/python/lib/core/py_seq_tensor.cc b/tensorflow/python/lib/core/py_seq_tensor.cc index 8f66a8a7364654..776c5f141fd839 100644 --- a/tensorflow/python/lib/core/py_seq_tensor.cc +++ b/tensorflow/python/lib/core/py_seq_tensor.cc @@ -21,6 +21,7 @@ limitations under the License. #include "tensorflow/core/lib/core/errors.h" #include "tensorflow/core/lib/core/stringpiece.h" #include "tensorflow/core/lib/strings/str_util.h" +#include "tensorflow/core/platform/macros.h" #include "tensorflow/core/platform/types.h" #include "tensorflow/python/lib/core/numpy.h" #include "tensorflow/python/lib/core/py_util.h" @@ -375,37 +376,43 @@ DEFINE_HELPER(ConvertInt32, int32, DT_INT32, ConvertOneInt32); // Floating-point support +// Returns `true` if `out` overflows when converted from `as_double`. +template +static inline bool CheckForOverflow(double as_double, T* out) { + return (sizeof(T) < sizeof(double) && std::isinf(*out) && + std::isfinite(as_double)); +} + +// There is no `std::isinf` that takes `Eigen::half` as argument but Eigen +// provides `Eigen::half_impl::isinf` instead. +template <> +inline bool CheckForOverflow(double as_double, Eigen::half* out) { + return (sizeof(Eigen::half) < sizeof(double) && + Eigen::half_impl::isinf(*out) && std::isfinite(as_double)); +} + template const char* ConvertOneFloat(PyObject* v, T* out) { if (PyErr_Occurred()) { return nullptr; } if (TF_PREDICT_TRUE(PyFloat_Check(v))) { - double as_double = PyFloat_AsDouble(v); - // Handle infinity. - if (as_double == std::numeric_limits::infinity()) { - *out = std::numeric_limits::infinity(); - return nullptr; - } else if (as_double == -1 * std::numeric_limits::infinity()) { - *out = -1 * std::numeric_limits::infinity(); - return nullptr; - } + const double as_double = PyFloat_AS_DOUBLE(v); + *out = static_cast(as_double); // Check for overflow. - if (as_double > std::numeric_limits::max() || - as_double < std::numeric_limits::lowest()) { + if (TF_PREDICT_FALSE(CheckForOverflow(as_double, out))) { return ErrorOutOfRangeDouble; } - *out = static_cast(as_double); return nullptr; } #if PY_MAJOR_VERSION < 3 if (PyInt_Check(v)) { - *out = PyInt_AS_LONG(v); + *out = static_cast(PyInt_AS_LONG(v)); return nullptr; } #endif if (PyLong_Check(v)) { - *out = PyLong_AsDouble(v); + *out = static_cast(PyLong_AsDouble(v)); if (PyErr_Occurred()) return ErrorOutOfRangeDouble; return nullptr; } @@ -432,17 +439,7 @@ const char* ConvertOneFloat(PyObject* v, T* out) { DEFINE_HELPER(ConvertDouble, double, DT_DOUBLE, ConvertOneFloat); DEFINE_HELPER(ConvertFloat, float, DT_FLOAT, ConvertOneFloat); - -const char* ConvertOneNumpyHalf(PyObject* v, Eigen::half* out) { - // NOTE(nareshmodi): Is there a way to convert to C double without the - // intermediate Python double? This will help with ConvertOneFloat as well. - Safe_PyObjectPtr as_float = make_safe(PyNumber_Float(v)); - double v_double = PyFloat_AS_DOUBLE(as_float.get()); - *out = Eigen::half(v_double); - - return nullptr; -} -DEFINE_HELPER(ConvertNumpyHalf, Eigen::half, DT_HALF, ConvertOneNumpyHalf); +DEFINE_HELPER(ConvertNumpyHalf, Eigen::half, DT_HALF, ConvertOneFloat); // String support @@ -537,7 +534,8 @@ Status PySeqToTensor(PyObject* obj, DataType dtype, Tensor* ret) { break; case DT_HALF: - RETURN_STRING_AS_STATUS(ConvertNumpyHalf(obj, shape, ret)); + if (ConvertNumpyHalf(obj, shape, ret) == nullptr) return Status::OK(); + break; case DT_INT64: if (ConvertInt64(obj, shape, ret) == nullptr) return Status::OK(); diff --git a/tensorflow/stream_executor/rocm/rocm_dnn.cc b/tensorflow/stream_executor/rocm/rocm_dnn.cc index e1a7e80f43005b..52a98b6bddb480 100644 --- a/tensorflow/stream_executor/rocm/rocm_dnn.cc +++ b/tensorflow/stream_executor/rocm/rocm_dnn.cc @@ -2280,7 +2280,8 @@ MIOpenSupport::createRnnDescriptor( int batch_size, dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode, dnn::RnnMode rnn_mode, dnn::DataType data_type, const dnn::AlgorithmConfig& algorithm_config, - float dropout, uint64 seed, ScratchAllocator* state_allocator) { + float dropout, uint64 seed, ScratchAllocator* state_allocator, + bool use_padded_io) { // ROCM TODO: cell_size is ignored for now // ROCM TODO: batch_size is ignored for now diff --git a/tensorflow/stream_executor/rocm/rocm_dnn.h b/tensorflow/stream_executor/rocm/rocm_dnn.h index 5bc0914f140d68..594cb860e559b1 100644 --- a/tensorflow/stream_executor/rocm/rocm_dnn.h +++ b/tensorflow/stream_executor/rocm/rocm_dnn.h @@ -50,7 +50,8 @@ class MIOpenSupport : public dnn::DnnSupport { int batch_size, dnn::RnnInputMode input_mode, dnn::RnnDirectionMode direction_mode, dnn::RnnMode rnn_mode, dnn::DataType data_type, const dnn::AlgorithmConfig& algorithm_config, - float dropout, uint64 seed, ScratchAllocator* state_allocator) override; + float dropout, uint64 seed, ScratchAllocator* state_allocator, + bool use_padded_io); port::StatusOr> createRnnSequenceTensorDescriptor(int seq_length, int batch_size, diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl index 97b10bf0d883e8..a402fa438799aa 100644 --- a/tensorflow/tensorflow.bzl +++ b/tensorflow/tensorflow.bzl @@ -58,7 +58,7 @@ def register_extension_info(**kwargs): # not contain rc or alpha, only numbers. # Also update tensorflow/core/public/version.h # and tensorflow/tools/pip_package/setup.py -VERSION = "2.0.0" +VERSION = "2.0.1" VERSION_MAJOR = VERSION.split(".")[0] def if_v2(a): diff --git a/tensorflow/tools/ci_build/Dockerfile.rocm b/tensorflow/tools/ci_build/Dockerfile.rocm index d6601a28a78c8f..10a75989b28d7a 100644 --- a/tensorflow/tools/ci_build/Dockerfile.rocm +++ b/tensorflow/tools/ci_build/Dockerfile.rocm @@ -1,15 +1,15 @@ -# This Dockerfile provides a starting point for a ROCm installation of -# MIOpen and tensorflow. +# This Dockerfile provides a starting point for a ROCm installation of +# MIOpen and tensorflow. FROM ubuntu:xenial MAINTAINER Jeff Poznanovic -ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/debian/ +ARG DEB_ROCM_REPO=http://repo.radeon.com/rocm/apt/2.6/ ARG ROCM_PATH=/opt/rocm ENV DEBIAN_FRONTEND noninteractive ENV TF_NEED_ROCM 1 ENV HOME /root/ -RUN apt update && apt install -y wget software-properties-common +RUN apt update && apt install -y wget software-properties-common # Add rocm repository RUN apt-get clean all diff --git a/tensorflow/tools/ci_build/builds/pip.sh b/tensorflow/tools/ci_build/builds/pip.sh index d1fad98ed7e18d..9f8f8da7106657 100755 --- a/tensorflow/tools/ci_build/builds/pip.sh +++ b/tensorflow/tools/ci_build/builds/pip.sh @@ -131,8 +131,8 @@ echo "Using Bazel flags: ${BAZEL_FLAGS}" PIP_BUILD_TARGET="//tensorflow/tools/pip_package:build_pip_package" GPU_FLAG="" +ROCM_FLAG="" if [[ ${CONTAINER_TYPE} == "cpu" ]] || \ - [[ ${CONTAINER_TYPE} == "rocm" ]] || \ [[ ${CONTAINER_TYPE} == "debian.jessie.cpu" ]]; then bazel build ${BAZEL_FLAGS} ${PIP_BUILD_TARGET} || \ die "Build failed." @@ -140,6 +140,10 @@ elif [[ ${CONTAINER_TYPE} == "gpu" ]]; then bazel build ${BAZEL_FLAGS} ${PIP_BUILD_TARGET} || \ die "Build failed." GPU_FLAG="--gpu" +elif [[ ${CONTAINER_TYPE} == "rocm" ]]; then + bazel build ${BAZEL_FLAGS} ${PIP_BUILD_TARGET} || \ + die "Build failed." + ROCM_FLAG="--rocm" else die "Unrecognized container type: \"${CONTAINER_TYPE}\"" fi @@ -193,7 +197,7 @@ fi PIP_WHL_DIR="${PIP_TEST_ROOT}/whl" PIP_WHL_DIR=$(realpath ${PIP_WHL_DIR}) # Get absolute path rm -rf ${PIP_WHL_DIR} && mkdir -p ${PIP_WHL_DIR} -bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} ${GPU_FLAG} ${NIGHTLY_FLAG} || \ +bazel-bin/tensorflow/tools/pip_package/build_pip_package ${PIP_WHL_DIR} ${GPU_FLAG} ${ROCM_FLAG} ${NIGHTLY_FLAG} || \ die "build_pip_package FAILED" WHL_PATH=$(ls ${PIP_WHL_DIR}/${PROJECT_NAME}*.whl) @@ -406,7 +410,7 @@ do_virtualenv_pip_test() { return ${SKIP_RETURN_CODE} else # Call run_pip_tests.sh to perform test-on-install - "${SCRIPT_DIR}/run_pip_tests.sh" --virtualenv ${GPU_FLAG} ${MAC_FLAG} + "${SCRIPT_DIR}/run_pip_tests.sh" --virtualenv ${GPU_FLAG} ${ROCM_FLAG} ${MAC_FLAG} if [[ $? != 0 ]]; then echo "PIP tests-on-install FAILED" return 1 @@ -426,7 +430,7 @@ do_virtualenv_oss_serial_pip_test() { else # Call run_pip_tests.sh to perform test-on-install "${SCRIPT_DIR}/run_pip_tests.sh" \ - --virtualenv ${GPU_FLAG} ${MAC_FLAG} --oss_serial + --virtualenv ${GPU_FLAG} ${ROCM_FLAG} ${MAC_FLAG} --oss_serial if [[ $? != 0 ]]; then echo "PIP tests-on-install (oss_serial) FAILED" return 1 @@ -439,7 +443,7 @@ do_virtualenv_oss_serial_pip_test() { ################################################################################ do_test_user_ops() { if [[ "${DO_TEST_USER_OPS}" == "1" ]]; then - "${SCRIPT_DIR}/test_user_ops.sh" --virtualenv ${GPU_FLAG} + "${SCRIPT_DIR}/test_user_ops.sh" --virtualenv ${GPU_FLAG} ${ROCM_FLAG} if [[ $? != 0 ]]; then echo "PIP user-op tests-on-install FAILED" return 1 diff --git a/tensorflow/tools/ci_build/linux/gpu/run_cc_core.sh b/tensorflow/tools/ci_build/linux/gpu/run_cc_core.sh index 9d2c8383fae5c1..c07e1a022f5a09 100755 --- a/tensorflow/tools/ci_build/linux/gpu/run_cc_core.sh +++ b/tensorflow/tools/ci_build/linux/gpu/run_cc_core.sh @@ -28,6 +28,7 @@ echo "" export PYTHON_BIN_PATH=`which python3` export CC_OPT_FLAGS='-mavx' +export TF_NEED_ROCM=0 export TF_NEED_CUDA=1 export TF_CUDA_COMPUTE_CAPABILITIES=3.7 diff --git a/tensorflow/tools/ci_build/linux/gpu/run_py3_core.sh b/tensorflow/tools/ci_build/linux/gpu/run_py3_core.sh index 5b3383e1059a18..7cefca0b84b03c 100755 --- a/tensorflow/tools/ci_build/linux/gpu/run_py3_core.sh +++ b/tensorflow/tools/ci_build/linux/gpu/run_py3_core.sh @@ -28,6 +28,7 @@ echo "" export PYTHON_BIN_PATH=`which python3` export CC_OPT_FLAGS='-mavx' +export TF_NEED_ROCM=0 export TF_NEED_CUDA=1 export TF_CUDA_COMPUTE_CAPABILITIES=3.7 diff --git a/tensorflow/tools/ci_build/linux/rocm/run_csb_tests.sh b/tensorflow/tools/ci_build/linux/rocm/run_csb_tests.sh new file mode 100755 index 00000000000000..446a1e39ebce90 --- /dev/null +++ b/tensorflow/tools/ci_build/linux/rocm/run_csb_tests.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Copyright 2017 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# ============================================================================== + +set -e +set -x + +N_JOBS=$(grep -c ^processor /proc/cpuinfo) +N_GPUS=$(lspci|grep 'VGA'|grep 'AMD/ATI'|wc -l) + +echo "" +echo "Bazel will use ${N_JOBS} concurrent build job(s) and ${N_GPUS} concurrent test job(s)." +echo "" + +# Run configure. +export PYTHON_BIN_PATH=`which python3` +export CC_OPT_FLAGS='-mavx' + +export TF_NEED_ROCM=1 +export TF_GPU_COUNT=${N_GPUS} + +yes "" | $PYTHON_BIN_PATH configure.py + +# Run bazel test command. Double test timeouts to avoid flakes. +bazel test \ + --config=rocm \ + -k \ + --test_tag_filters=gpu,-no_gpu,-no_rocm,-benchmark-test,-no_oss,-oss_serial, \ + --test_timeout 600,900,2400,7200 \ + --test_output=errors \ + --jobs=${N_JOBS} \ + --local_test_jobs=${TF_GPU_COUNT} \ + --test_sharding_strategy=disabled \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- \ + //tensorflow/... \ + -//tensorflow/compiler/... \ + -//tensorflow/contrib/... \ + -//tensorflow/lite/... \ + -//tensorflow/python/compiler/tensorrt/... \ + + + diff --git a/tensorflow/tools/ci_build/presubmit/macos/py2_cc/build.sh b/tensorflow/tools/ci_build/presubmit/macos/py2_cc/build.sh new file mode 100644 index 00000000000000..92acb7ab7fe19b --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/macos/py2_cc/build.sh @@ -0,0 +1,70 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# TODO(mihaimaruseac,hyey,ggadde): Convert to py3 + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function setup_pip () { + install_pip2 + python -m virtualenv tf_build_env --system-site-packages + source tf_build_env/bin/activate + install_macos_pip_deps +} + +function run_build () { + # Run configure. + export TF_NEED_CUDA=0 + export PYTHON_BIN_PATH=$(which python2) + yes "" | $PYTHON_BIN_PATH configure.py + tag_filters="-no_oss,-no_oss_py2,-gpu,-tpu,-benchmark-test,-nomac,-no_mac,-v1only" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + "${BAZEL_WRAPPER_PATH}" \ + test \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" \ + --action_env=PATH \ + --remote_accept_cached=true \ + --spawn_strategy=standalone \ + --remote_local_fallback=false \ + --remote_timeout=600 \ + --strategy=Javac=standalone \ + --strategy=Closure=standalone \ + --genrule_strategy=standalone \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_macos +which bazel +set_bazel_outdir + +setup_pip +run_build diff --git a/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh b/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh new file mode 100644 index 00000000000000..ffc823a6e2eb56 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/macos/py37_cc/build.sh @@ -0,0 +1,68 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function setup_pip () { + python3.7 -m virtualenv tf_build_env --system-site-packages + source tf_build_env/bin/activate + install_macos_pip_deps +} + +function run_build () { + # Run configure. + export TF_NEED_CUDA=0 + export PYTHON_BIN_PATH=$(which python3.7) + yes "" | $PYTHON_BIN_PATH configure.py + tag_filters="-no_oss,-no_oss_py2,-gpu,-tpu,-benchmark-test,-nomac,-no_mac,-v1only" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + "${BAZEL_WRAPPER_PATH}" \ + test \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" \ + --action_env=PATH \ + --remote_accept_cached=true \ + --spawn_strategy=standalone \ + --remote_local_fallback=false \ + --remote_timeout=600 \ + --strategy=Javac=standalone \ + --strategy=Closure=standalone \ + --genrule_strategy=standalone \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_macos +which bazel +set_bazel_outdir + +setup_pip +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/android/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/android/build.sh new file mode 100644 index 00000000000000..618b7ad382ec57 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/android/build.sh @@ -0,0 +1,82 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function run_build () { + export ANDROID_NDK_HOME="/opt/android-ndk-r17c" + export NDK_HOME=$ANDROID_NDK_HOME + export ANDROID_SDK_HOME="/opt/android-sdk/current" + export ANDROID_API_LEVEL="23" + export ANDROID_BUILD_TOOLS_VERSION="28.0.0" + + ANDROID_OUT=android.out + ANDROID_OUT_TARGET=gen_android_out + + # Run the presubmit android build. + tensorflow/tools/ci_build/builds/android.sh 2>&1 | tee tensorflow/tools/ci_build/builds/${ANDROID_OUT} + RC=${PIPESTATUS[0]} + + # Since we are running the build remotely (rbe), we need to build a bazel + # target that would output the log generated above and return the expected + # error code. + cat << EOF > tensorflow/tools/ci_build/builds/BUILD +package(default_visibility = ["//tensorflow:internal"]) + +sh_test( + name = "${ANDROID_OUT_TARGET}", + srcs = ["${ANDROID_OUT_TARGET}.sh"], + data = ["${ANDROID_OUT}"], + tags = ["local"], +) +EOF + + cat << EOF > tensorflow/tools/ci_build/builds/${ANDROID_OUT_TARGET}.sh +#!/bin/bash +cat tensorflow/tools/ci_build/builds/${ANDROID_OUT} +exit ${RC} +EOF + + # Now trigger the rbe build that outputs the log + chmod +x tensorflow/tools/ci_build/builds/${ANDROID_OUT_TARGET}.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + "${BAZEL_WRAPPER_PATH}" \ + --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ + test \ + --incompatible_list_based_execution_strategy_selection \ + --test_output=all \ + tensorflow/tools/ci_build/builds:${ANDROID_OUT_TARGET} + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/cpu_py36_full/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/cpu_py36_full/build.sh new file mode 100644 index 00000000000000..3f843f29db1f0b --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/cpu_py36_full/build.sh @@ -0,0 +1,97 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function run_build () { + # Build a unique cache silo string. + UBUNTU_VERSION=$(lsb_release -a | grep Release | awk '{print $2}') + IMAGE_VERSION=$(cat /VERSION) + CACHE_SILO_VAL="cpu-py3-ubuntu-16-${UBUNTU_VERSION}-${IMAGE_VERSION}" + + # Run configure. + # Do not run configure.py when doing remote build & test: + # Most things we set with configure.py are not used in a remote build setting, + # as the build will be defined by pre-configured build files that are checked + # in. + # TODO(klimek): Allow using the right set of bazel flags without the need to + # run configure.py; currently we need to carefully copy them, which is brittle. + export TF_NEED_GCP=0 + export TF_NEED_HDFS=0 + export TF_NEED_CUDA=0 + export ACTION_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + export PYTHON_BIN_PATH="/usr/bin/python3" + export TF2_BEHAVIOR=1 + tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test""$(maybe_skip_v1)" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + "${BAZEL_WRAPPER_PATH}" \ + test \ + --incompatible_list_based_execution_strategy_selection \ + --config=rbe \ + --python_path="${PYTHON_BIN_PATH}" \ + --action_env=PATH="${ACTION_PATH}" \ + --action_env=PYTHON_BIN_PATH="${PYTHON_BIN_PATH}" \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --action_env=TF_PYTHON_CONFIG_REPO=@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/py3 \ + --action_env=TF_ENABLE_XLA=1 \ + --test_tag_filters="${tag_filters}" \ + --build_tag_filters="${tag_filters}" \ + --test_lang_filters=cc,py \ + --define=with_default_optimizations=true \ + --define=framework_shared_object=true \ + --define=with_xla_support=true \ + -c opt \ + --copt="-w" \ + --copt=-mavx \ + --linkopt=-lrt \ + --distinct_host_configuration=false \ + --remote_default_platform_properties="properties:{name:\"build\" value:\"${CACHE_SILO_VAL}\"}" \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010:toolchain \ + --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 \ + --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 \ + --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --extra_toolchains=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010:cc-toolchain-k8 \ + --extra_execution_platforms=@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010 \ + --host_platform=@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010 \ + --remote_timeout=3600 \ + --platforms=@org_tensorflow//third_party/toolchains:rbe_ubuntu16.04-manylinux2010 \ + -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/gpu_py36_full/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/gpu_py36_full/build.sh new file mode 100644 index 00000000000000..3fa4d4f1d7da45 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/gpu_py36_full/build.sh @@ -0,0 +1,114 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function run_build () { + # Build a unique cache silo string. + UBUNTU_VERSION=$(lsb_release -a | grep Release | awk '{print $2}') + IMAGE_VERSION=$(cat /VERSION) + CACHE_SILO_VAL="gpu-py3-ubuntu-16-${UBUNTU_VERSION}-${IMAGE_VERSION}" + + # Run configure. + # Do not run configure.py when doing remote build & test: + # Most things we set with configure.py are not used in a remote build setting, + # as the build will be defined by pre-configured build files that are checked + # in. + # TODO(klimek): Allow using the right set of bazel flags without the need to + # run configure.py; currently we need to carefully copy them, which is brittle. + export LD_LIBRARY_PATH="/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64" + # TODO(klimek): Remove once we don't try to read it while setting up the remote + # config for cuda (we currently don't use it, as it's only used when compiling + # with clang, but we still require it to be set anyway). + export TF_CUDA_COMPUTE_CAPABILITIES=6.0 + export ACTION_PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + export PYTHON_BIN_PATH="/usr/bin/python3" + export TF2_BEHAVIOR=1 + tag_filters="gpu,-no_gpu,-nogpu,-benchmark-test,-no_oss,-oss_serial""$(maybe_skip_v1)" + + # Get the default test targets for bazel. + source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + # TODO(klimek): Re-enable tensorrt tests (with different runtime image) once + # we can build them. + # TODO(klimek): Stop using action_env for things that are only needed during + # setup - we're artificially poisoning the cache. + "${BAZEL_WRAPPER_PATH}" \ + test \ + --config=rbe \ + --python_path="${PYTHON_BIN_PATH}" \ + --action_env=PATH="${ACTION_PATH}" \ + --action_env=PYTHON_BIN_PATH="${PYTHON_BIN_PATH}" \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --action_env=REMOTE_GPU_TESTING=1 \ + --action_env=TF_CUDA_COMPUTE_CAPABILITIES="${TF_CUDA_COMPUTE_CAPABILITIES}" \ + --action_env=TF_CUDA_CONFIG_REPO=@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/cuda10.0-cudnn7 \ + --action_env=TF_CUDA_VERSION=10 \ + --action_env=TF_CUDNN_VERSION=7 \ + --action_env=TF_NEED_TENSORRT=0 \ + --action_env=TF_NEED_CUDA=1 \ + --action_env=TF_PYTHON_CONFIG_REPO=@org_tensorflow//third_party/toolchains/preconfig/ubuntu16.04/py3 \ + --test_env=LD_LIBRARY_PATH \ + --test_tag_filters="${tag_filters}" \ + --build_tag_filters="${tag_filters}" \ + --test_lang_filters=cc,py \ + --define=with_default_optimizations=true \ + --define=framework_shared_object=true \ + --define=with_xla_support=true \ + --define=using_cuda_nvcc=true \ + --define=use_fast_cpp_protos=true \ + --define=allow_oversize_protos=true \ + --define=grpc_no_ares=true \ + -c opt \ + --copt="-w" \ + --copt=-mavx \ + --linkopt=-lrt \ + --distinct_host_configuration=false \ + --remote_default_platform_properties="properties:{name:\"build\" value:\"${CACHE_SILO_VAL}\"}" \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --host_javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.1:jdk8 \ + --javabase=@bazel_toolchains//configs/ubuntu16_04_clang/1.0:jdk8 \ + --host_java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --java_toolchain=@bazel_tools//tools/jdk:toolchain_hostjdk8 \ + --extra_toolchains=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain-linux-x86_64 \ + --extra_execution_platforms=@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010,@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010-gpu \ + --host_platform=@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010 \ + --local_test_jobs=4 \ + --remote_timeout=3600 \ + --platforms=@org_tensorflow//third_party/toolchains:rbe_cuda10.0-cudnn7-ubuntu16.04-manylinux2010 \ + -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +run_build diff --git a/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh b/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh new file mode 100644 index 00000000000000..2fee39be450ff3 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/ubuntu_16/sanity/build.sh @@ -0,0 +1,87 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== + +set -e + +# Error if we somehow forget to set the path to bazel_wrapper.py +set -u +BAZEL_WRAPPER_PATH=$1 +set +u + +# From this point on, logs can be publicly available +set -x + +function install_pylint () { + # TODO(gunan): figure out why we get stuck with later versions of pylint. + # TODO(mihaimaruseac): this is used in the release build in the same way, + # maybe extract out to a common? + sudo python2 -m pip install pylint==1.6.4 + sudo python3 -m pip install pylint==1.6.4 +} + +function run_sanity_checks () { + SANITY_OUT=ci_sanity.out + SANITY_OUT_TARGET=gen_ci_sanity_out + + # Run tensorflow sanity checks. + tensorflow/tools/ci_build/ci_sanity.sh 2>&1 | tee tensorflow/tools/ci_build/${SANITY_OUT} + RC=${PIPESTATUS[0]} + + # Since we are running the sanity build remotely (rbe), we need to build a bazel + # target that would output the log generated above and return the expected + # error code. + cat << EOF > tensorflow/tools/ci_build/BUILD +package(default_visibility = ["//tensorflow:internal"]) + +sh_test( + name = "${SANITY_OUT_TARGET}", + srcs = ["${SANITY_OUT_TARGET}.sh"], + data = ["${SANITY_OUT}"], + tags = ["local"], +) +EOF + + cat << EOF > tensorflow/tools/ci_build/${SANITY_OUT_TARGET}.sh +#!/bin/bash +cat tensorflow/tools/ci_build/${SANITY_OUT} +exit ${RC} +EOF + + # Now trigger the rbe build that outputs the log + chmod +x tensorflow/tools/ci_build/${SANITY_OUT_TARGET}.sh + + # Run bazel test command. Double test timeouts to avoid flakes. + # //tensorflow/core:platform_setround_test is not supported. See b/64264700 + "${BAZEL_WRAPPER_PATH}" \ + --host_jvm_args=-Dbazel.DigestFunction=SHA256 \ + test \ + --incompatible_list_based_execution_strategy_selection \ + --test_output=all \ + tensorflow/tools/ci_build:${SANITY_OUT_TARGET} + + # Copy log to output to be available to GitHub + ls -la "$(bazel info output_base)/java.log" + cp "$(bazel info output_base)/java.log" "${KOKORO_ARTIFACTS_DIR}/" +} + + +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +install_pylint + +run_sanity_checks diff --git a/tensorflow/tools/ci_build/presubmit/windows/cpu_py36_full/build.bat b/tensorflow/tools/ci_build/presubmit/windows/cpu_py36_full/build.bat new file mode 100644 index 00000000000000..fcc079f7b0e010 --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/windows/cpu_py36_full/build.bat @@ -0,0 +1,44 @@ +echo on +setlocal enableextensions enabledelayedexpansion + +@REM This is the path to bazel_wrapper.py, should be set as an argument +set BAZEL_WRAPPER_PATH=%~f1 + +@REM Load common definitions, install bazel +CALL tensorflow\tools\ci_build\release\common_win.bat + +@REM Set up common variables used through the script +set WIN_OUT=win.out +set WIN_OUT_TARGET=gen_win_out +set BUILD_PATH=tensorflow/tools/ci_build/builds +set GEN_SCRIPT=%BUILD_PATH%/%WIN_OUT_TARGET%.sh +set GEN_BUILD=%BUILD_PATH%/BUILD + +@REM Run the presubmit win build. +CALL tensorflow\tools\ci_build\windows\cpu\pip\run.bat --enable_remote_cache %* > %BUILD_PATH%/%WIN_OUT% 2>&1 +set RC=%errorlevel% + +@REM Since we are running the sanity build remotely (rbe), we need to build a bazel +@REM target that would output the log generated above and return the expected +@REM error code. +echo package(default_visibility = ["//visibility:public"]) > %GEN_BUILD% +echo. >> %GEN_BUILD% +echo sh_test( >> %GEN_BUILD% +echo name = "%WIN_OUT_TARGET%", >> %GEN_BUILD% +echo srcs = ["%WIN_OUT_TARGET%.sh"], >> %GEN_BUILD% +echo data = ["%WIN_OUT%"], >> %GEN_BUILD% +echo tags = ["local"], >> %GEN_BUILD% +echo ) >> %GEN_BUILD% + +echo #!/bin/bash > %GEN_SCRIPT% +echo function rlocation() { >> %GEN_SCRIPT% +echo fgrep -m1 "$1 " "$RUNFILES_MANIFEST_FILE" ^| cut -d' ' -f2- >> %GEN_SCRIPT% +echo } >> %GEN_SCRIPT% +echo cat $(rlocation %BUILD_PATH%/%WIN_OUT%) >> %GEN_SCRIPT% +echo exit %RC% >> %GEN_SCRIPT% + +@REM Now trigger the rbe build that outputs the log +chmod +x %GEN_SCRIPT% + +@REM Run bazel test command. +%PY_EXE% %BAZEL_WRAPPER_PATH% --output_user_root=%TMPDIR% --host_jvm_args=-Dbazel.DigestFunction=SHA256 test %BUILD_PATH%:%WIN_OUT_TARGET% --test_output=all diff --git a/tensorflow/tools/ci_build/presubmit/windows/gpu_py36_full/build.bat b/tensorflow/tools/ci_build/presubmit/windows/gpu_py36_full/build.bat new file mode 100644 index 00000000000000..80edefc2bf050e --- /dev/null +++ b/tensorflow/tools/ci_build/presubmit/windows/gpu_py36_full/build.bat @@ -0,0 +1,45 @@ +echo on +setlocal enableextensions enabledelayedexpansion + +@REM This is the path to bazel_wrapper.py, should be set as an argument +set BAZEL_WRAPPER_PATH=%~f1 + +@REM Load common definitions, install bazel +CALL tensorflow\tools\ci_build\release\common_win.bat + +@REM Set up common variables used through the script +set WIN_OUT=win.out +set WIN_OUT_TARGET=gen_win_out +set BUILD_PATH=tensorflow/tools/ci_build/builds +set GEN_SCRIPT=%BUILD_PATH%/%WIN_OUT_TARGET%.sh +set GEN_BUILD=%BUILD_PATH%/BUILD + +@REM Run the presubmit win build. +CALL tensorflow\tools\ci_build\windows\gpu\pip\run.bat --enable_remote_cache %* > %BUILD_PATH%/%WIN_OUT% 2>&1 +set RC=%errorlevel% + +@REM Since we are running the sanity build remotely (rbe), we need to build a bazel +@REM target that would output the log generated above and return the expected +@REM error code. +echo package(default_visibility = ["//visibility:public"]) > %GEN_BUILD% +echo. >> %GEN_BUILD% +echo sh_test( >> %GEN_BUILD% +echo name = "%WIN_OUT_TARGET%", >> %GEN_BUILD% +echo srcs = ["%WIN_OUT_TARGET%.sh"], >> %GEN_BUILD% +echo data = ["%WIN_OUT%"], >> %GEN_BUILD% +echo tags = ["local"], >> %GEN_BUILD% +echo ) >> %GEN_BUILD% + +echo #!/bin/bash > %GEN_SCRIPT% +echo function rlocation() { >> %GEN_SCRIPT% +echo fgrep -m1 "$1 " "$RUNFILES_MANIFEST_FILE" ^| cut -d' ' -f2- >> %GEN_SCRIPT% +echo } >> %GEN_SCRIPT% +echo cat $(rlocation %BUILD_PATH%/%WIN_OUT%) >> %GEN_SCRIPT% +echo exit %RC% >> %GEN_SCRIPT% + +@REM Now trigger the rbe build that outputs the log +chmod +x %GEN_SCRIPT% + +@REM Run bazel test command. +%PY_EXE% %BAZEL_WRAPPER_PATH% --output_user_root=%TMPDIR% --host_jvm_args=-Dbazel.DigestFunction=SHA256 test %BUILD_PATH%:%WIN_OUT_TARGET% --test_output=all + diff --git a/tensorflow/tools/ci_build/release/common.sh b/tensorflow/tools/ci_build/release/common.sh new file mode 100644 index 00000000000000..6f0ab58bd09a0e --- /dev/null +++ b/tensorflow/tools/ci_build/release/common.sh @@ -0,0 +1,278 @@ +#!/usr/bin/env bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +# External `common.sh` + +# Keep in sync with tensorflow_estimator and configure.py. +# LINT.IfChange +LATEST_BAZEL_VERSION=0.26.1 +# LINT.ThenChange( +# //tensorflow/opensource_only/configure.py, +# //tensorflow_estimator/google/kokoro/common.sh, +# //tensorflow/tools/ci_build/install/install_bazel.sh, +# //tensorflow/tools/ci_build/install/install_bazel_from_source.sh) + +# LINT.IfChange +function die() { + echo "$@" 1>&2 ; exit 1; +} + +# A small utility to run the command and only print logs if the command fails. +# On success, all logs are hidden. +function readable_run { + # Disable debug mode to avoid printing of variables here. + set +x + result=$("$@" 2>&1) || die "$result" + echo "$@" + echo "Command completed successfully at $(date)" + set -x +} +# LINT.ThenChange( +# ) # common_.sh + +# LINT.IfChange +# Redirect bazel output dir b/73748835 +function set_bazel_outdir { + mkdir -p /tmpfs/bazel_output + export TEST_TMPDIR=/tmpfs/bazel_output +} + +# Run flaky functions with retries. +# run_with_retry cmd +function run_with_retry { + eval "$1" + # If the command fails retry again in 60 seconds. + if [[ $? -ne 0 ]]; then + sleep 60 + eval "$1" + fi +} + +# Install the given bazel version on linux +function update_bazel_linux { + if [[ -z "$1" ]]; then + BAZEL_VERSION=${LATEST_BAZEL_VERSION} + else + BAZEL_VERSION=$1 + fi + rm -rf ~/bazel + mkdir ~/bazel + + pushd ~/bazel + readable_run wget https://github.com/bazelbuild/bazel/releases/download/"${BAZEL_VERSION}"/bazel-"${BAZEL_VERSION}"-installer-linux-x86_64.sh + chmod +x bazel-*.sh + ./bazel-"${BAZEL_VERSION}"-installer-linux-x86_64.sh --user + rm bazel-"${BAZEL_VERSION}"-installer-linux-x86_64.sh + popd + + PATH="/home/kbuilder/bin:$PATH" + set_bazel_outdir + which bazel + bazel version +} +# LINT.ThenChange( +# //tensorflow_estimator/google/kokoro/common.sh) + +# LINT.IfChange +# Install the given bazel version on macos +function update_bazel_macos { + if [[ -z "$1" ]]; then + BAZEL_VERSION=${LATEST_BAZEL_VERSION} + else + BAZEL_VERSION=$1 + fi + BAZEL_COMMAND="curl -L https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-installer-darwin-x86_64.sh -O && \ + chmod +x bazel-*.sh && ./bazel-${BAZEL_VERSION}-installer-darwin-x86_64.sh --user && \ + rm -f bazel-${BAZEL_VERSION}-installer-darwin-x86_64.sh" + # If the bazel update fails retry again in 60 seconds. + run_with_retry "${BAZEL_COMMAND}" + # Add new bazel installation to path + PATH="/Users/kbuilder/bin:$PATH" + set_bazel_outdir + which bazel + bazel version +} + +function install_pip2 { + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + sudo python2 get-pip.py +} + +function install_pip3.5 { + curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py + sudo python3.5 get-pip.py +} + +function install_pip_deps { + SUDO_CMD="" + PIP_CMD="pip" + + while true; do + if [[ -z "${1}" ]]; then + break + fi + if [[ "$1" == "sudo" ]]; then + SUDO_CMD="sudo " + elif [[ "$1" == "pip"* ]]; then + PIP_CMD="$1" + fi + shift + done + + # TODO(aselle): Change all these to be --user instead of sudo. + # TODO(hyey): Add back IfChange lint check (b/143530103). + # =================================================================== + # Please change dependencies in `install_ubuntu_16_pip_deps` as well. + # =================================================================== + ${SUDO_CMD} ${PIP_CMD} install keras_applications==1.0.8 --no-deps + ${SUDO_CMD} ${PIP_CMD} install keras_preprocessing==1.1.0 --no-deps + ${SUDO_CMD} ${PIP_CMD} install gast==0.2.2 + ${SUDO_CMD} ${PIP_CMD} install h5py==2.8.0 + ${SUDO_CMD} ${PIP_CMD} install six==1.12.0 + ${SUDO_CMD} ${PIP_CMD} install grpcio + ${SUDO_CMD} ${PIP_CMD} install portpicker + ${SUDO_CMD} ${PIP_CMD} install scipy + ${SUDO_CMD} ${PIP_CMD} install scikit-learn==0.20.3 + ${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly==2.1.0a20190927 + ${PIP_CMD} install --user --upgrade attrs + ${PIP_CMD} install --user --upgrade tf-estimator-nightly==1.14.0.dev2019072801 + ${PIP_CMD} install --user --upgrade "future>=0.17.1" + # =================================================================== +} + +function install_ubuntu_16_pip_deps { + PIP_CMD="pip" + + while true; do + if [[ -z "${1}" ]]; then + break + fi + if [[ "$1" == "pip"* ]]; then + PIP_CMD="$1" + fi + shift + done + + # TODO(hyey): Add back IfChange lint check (b/143530103). + # =================================================================== + # Please change dependencies in `install_pip_deps` as well. + # =================================================================== + "${PIP_CMD}" install --user --upgrade attrs + "${PIP_CMD}" install keras_applications==1.0.8 --no-deps --user + "${PIP_CMD}" install keras_preprocessing==1.1.0 --no-deps --user + "${PIP_CMD}" install numpy==1.14.5 --user + "${PIP_CMD}" install --user --upgrade "future>=0.17.1" + "${PIP_CMD}" install gast==0.2.2 --user + "${PIP_CMD}" install h5py==2.8.0 --user + "${PIP_CMD}" install six==1.12.0 --user + "${PIP_CMD}" install grpcio --user + "${PIP_CMD}" install portpicker --user + "${PIP_CMD}" install scipy --user + "${PIP_CMD}" install scikit-learn --user + "${PIP_CMD}" install --user --upgrade tf-estimator-nightly==1.14.0.dev2019072801 + "${PIP_CMD}" install --user --upgrade tb-nightly==2.1.0a20190927 + # =================================================================== +} + +function install_macos_pip_deps { + SUDO_CMD="" + PIP_CMD="pip" + + while true; do + if [[ -z "${1}" ]]; then + break + fi + if [[ "$1" == "sudo" ]]; then + SUDO_CMD="sudo " + elif [[ "$1" == "pip3.7" ]]; then + PIP_CMD="python3.7 -m pip" + SUDO_CMD="sudo -H " + elif [[ "$1" == "pip"* ]]; then + PIP_CMD="$1" + fi + shift + done + + # High Sierra pip for Python2.7 installs don't work as expected. + if [[ "${PIP_CMD}" == "pip" ]]; then + PIP_CMD="python -m pip" + SUDO_CMD="sudo -H " + fi + + # TODO(aselle): Change all these to be --user instead of sudo. + ${SUDO_CMD} ${PIP_CMD} install --upgrade setuptools==39.1.0 + ${SUDO_CMD} ${PIP_CMD} install keras_applications==1.0.8 --no-deps + ${SUDO_CMD} ${PIP_CMD} install keras_preprocessing==1.1.0 --no-deps + ${SUDO_CMD} ${PIP_CMD} install --upgrade mock portpicker scipy grpcio + ${SUDO_CMD} ${PIP_CMD} install six==1.12.0 + ${SUDO_CMD} ${PIP_CMD} install scikit-learn==0.20.3 + ${SUDO_CMD} ${PIP_CMD} install numpy==1.14.5 + ${SUDO_CMD} ${PIP_CMD} install gast==0.2.2 + ${SUDO_CMD} ${PIP_CMD} install h5py==2.8.0 + ${SUDO_CMD} ${PIP_CMD} install --upgrade grpcio + ${SUDO_CMD} ${PIP_CMD} install --upgrade tb-nightly==2.1.0a20190927 + ${PIP_CMD} install --user --upgrade attrs + ${PIP_CMD} install --user --upgrade tf-estimator-nightly==1.14.0.dev2019072801 + ${PIP_CMD} install --user --upgrade "future>=0.17.1" +} + +function maybe_skip_v1 { + # If we are building with v2 by default, skip tests with v1only tag. + if grep -q "build --config=v2" ".bazelrc"; then + echo ",-v1only" + else + echo "" + fi +} + +# Copy and rename a wheel to a new project name. +# Usage: copy_to_new_project_name , for example +# copy_to_new_project_name test_dir/tf_nightly-1.15.0.dev20190813-cp35-cp35m-manylinux2010_x86_64.whl tf_nightly_cpu +# will create a wheel with the same tags, but new project name under the same +# directory at +# test_dir/tf_nightly_cpu-1.15.0.dev20190813-cp35-cp35m-manylinux2010_x86_64.whl +function copy_to_new_project_name { + WHL_PATH="$1" + NEW_PROJECT_NAME="$2" + + ORIGINAL_WHL_NAME=$(basename "${WHL_PATH}") + ORIGINAL_WHL_DIR=$(realpath "$(dirname "${WHL_PATH}")") + ORIGINAL_PROJECT_NAME="$(echo "${ORIGINAL_WHL_NAME}" | cut -d '-' -f 1)" + FULL_TAG="$(echo "${ORIGINAL_WHL_NAME}" | cut -d '-' -f 2-)" + NEW_WHL_NAME="${NEW_PROJECT_NAME}-${FULL_TAG}" + VERSION="$(echo "${FULL_TAG}" | cut -d '-' -f 1)" + + TMP_DIR="$(mktemp -d)" + cp "${WHL_PATH}" "${TMP_DIR}" + pushd "${TMP_DIR}" + unzip -q "${ORIGINAL_WHL_NAME}" + + ORIGINAL_WHL_DIR_PREFIX="${ORIGINAL_PROJECT_NAME}-${VERSION}" + NEW_WHL_DIR_PREFIX="${NEW_PROJECT_NAME}-${VERSION}" + mv "${ORIGINAL_WHL_DIR_PREFIX}.dist-info" "${NEW_WHL_DIR_PREFIX}.dist-info" + mv "${ORIGINAL_WHL_DIR_PREFIX}.data" "${NEW_WHL_DIR_PREFIX}.data" + sed -i.bak "s/${ORIGINAL_PROJECT_NAME}/${NEW_PROJECT_NAME}/g" "${NEW_WHL_DIR_PREFIX}.dist-info/RECORD" + + ORIGINAL_PROJECT_NAME_DASH="${ORIGINAL_PROJECT_NAME//_/-}" + NEW_PROJECT_NAME_DASH="${NEW_PROJECT_NAME//_/-}" + sed -i.bak "s/${ORIGINAL_PROJECT_NAME_DASH}/${NEW_PROJECT_NAME_DASH}/g" "${NEW_WHL_DIR_PREFIX}.dist-info/METADATA" + + zip -rq "${NEW_WHL_NAME}" "${NEW_WHL_DIR_PREFIX}.dist-info" "${NEW_WHL_DIR_PREFIX}.data" "tensorflow" "tensorflow_core" + mv "${NEW_WHL_NAME}" "${ORIGINAL_WHL_DIR}" + popd + rm -rf "${TMP_DIR}" +} +# LINT.ThenChange( +# ) # common.sh diff --git a/tensorflow/tools/ci_build/release/common_win.bat b/tensorflow/tools/ci_build/release/common_win.bat new file mode 100644 index 00000000000000..c18d579f031e5f --- /dev/null +++ b/tensorflow/tools/ci_build/release/common_win.bat @@ -0,0 +1,77 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= +echo on + +@REM +@REM Set Environment Variables +@REM +IF NOT DEFINED PYTHON_DIRECTORY ( + SET PYTHON_DIRECTORY=Python36 +) +SET PY_EXE=C:\%PYTHON_DIRECTORY%\python.exe +SET PIP_EXE=C:\%PYTHON_DIRECTORY%\Scripts\pip.exe +SET PATH=%PATH%;C:\%PYTHON_DIRECTORY% + +@REM TODO(amitpatankar): Make an image with these packages and remove this. + +%PIP_EXE% install setuptools --upgrade +%PIP_EXE% install future>=0.17.1 --no-deps +%PIP_EXE% install tf-estimator-nightly==1.14.0.dev2019072801 --no-deps +%PIP_EXE% install tb-nightly==2.1.0a20190927 --no-deps +%PIP_EXE% install numpy --upgrade --no-deps +%PIP_EXE% install opt_einsum --upgrade +%PIP_EXE% install pandas --upgrade --no-deps +%PIP_EXE% install protobuf --upgrade --no-deps +%PIP_EXE% install keras_applications==1.0.8 --upgrade --no-deps +%PIP_EXE% install keras_preprocessing==1.1.0 --upgrade --no-deps +%PIP_EXE% install wrapt --upgrade --no-deps + +IF "%PYTHON_DIRECTORY%"=="Python37" ( + %PIP_EXE% install astor==0.7.1 + %PIP_EXE% install absl-py==0.5.0 + %PIP_EXE% install colorama==0.3.9 + %PIP_EXE% install cycler==0.10.0 + %PIP_EXE% install gast==0.2.0 + %PIP_EXE% install jedi==0.11.1 + %PIP_EXE% install oauth2client==4.1.2 + %PIP_EXE% install portpicker==1.2.0 + %PIP_EXE% install parso==0.1.1 + %PIP_EXE% install protobuf==3.6.1 + %PIP_EXE% install scikit-learn==0.19.2 + %PIP_EXE% install scipy==1.1.0 + %PIP_EXE% install termcolor==1.1.0 +) + +:: Set cuda related environment variables. If we are not using CUDA, these are not used. +IF NOT DEFINED TF_CUDA_VERSION ( + SET TF_CUDA_VERSION=10.0 +) +SET TF_CUDNN_VERSION=7 +SET TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 +SET CUDA_TOOLKIT_PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v%TF_CUDA_VERSION% +SET CUDNN_INSTALL_PATH=C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v%TF_CUDA_VERSION% +SET PATH=%CUDA_TOOLKIT_PATH%\extras\CUPTI\libx64;%PATH% +SET PATH=%CUDA_TOOLKIT_PATH%\bin;%PATH% +SET PATH=%CUDNN_INSTALL_PATH%\bin;%PATH% + +@REM +@REM Setup Bazel +@REM +:: Download Bazel from github and make sure its found in PATH. +SET BAZEL_VERSION=0.26.1 +md C:\tools\bazel\ +wget -q https://github.com/bazelbuild/bazel/releases/download/%BAZEL_VERSION%/bazel-%BAZEL_VERSION%-windows-x86_64.exe -O C:/tools/bazel/bazel.exe +SET PATH=C:\tools\bazel;%PATH% +bazel version diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh new file mode 100644 index 00000000000000..2c5627feff8f24 --- /dev/null +++ b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/nonpip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +# Install latest bazel +update_bazel_macos +which bazel +bazel version +set_bazel_outdir + +# Pick a more recent version of xcode +sudo xcode-select --switch /Applications/Xcode_10.3.app/Contents/Developer +python3.5 -m virtualenv tf_build_env --system-site-packages +source tf_build_env/bin/activate + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.5 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export TF2_BEHAVIOR=1 +export PYTHON_BIN_PATH=$(which python3.5) +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="-no_oss,-oss_serial,-nomac,-no_mac,-no_oss_py35,-v1only,-gpu,-tpu,-benchmark-test" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} \ + -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py35_full/pip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/pip.sh new file mode 100644 index 00000000000000..99b7e6774e4716 --- /dev/null +++ b/tensorflow/tools/ci_build/release/macos/cpu_py35_full/pip.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +# Install latest bazel +update_bazel_macos +which bazel +bazel version +set_bazel_outdir + +# Pick a more recent version of xcode +sudo xcode-select --switch /Applications/Xcode_10.3.app/Contents/Developer + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.5 + +# Export required variables for running pip.sh +export OS_TYPE="MACOS" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.5' + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="//tensorflow/python/..." +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-nomac,-no_mac,-no_oss,-oss_serial,-no_oss_py35,-gpu,-tpu,-benchmark-test' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh new file mode 100644 index 00000000000000..7ef5d64bf3987a --- /dev/null +++ b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/nonpip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +# Install latest bazel +update_bazel_macos +which bazel +bazel version +set_bazel_outdir + +# Pick a more recent version of xcode +sudo xcode-select --switch /Applications/Xcode_10.3.app/Contents/Developer +python3.6 -m virtualenv tf_build_env --system-site-packages +source tf_build_env/bin/activate + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.6 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export TF2_BEHAVIOR=1 +export PYTHON_BIN_PATH=$(which python3.6) +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="-no_oss,-oss_serial,-nomac,-no_mac,-no_oss_py36,-v1only,-gpu,-tpu,-benchmark-test" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} \ + -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py36_full/pip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/pip.sh new file mode 100644 index 00000000000000..60d8e99bd71be4 --- /dev/null +++ b/tensorflow/tools/ci_build/release/macos/cpu_py36_full/pip.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +# Install latest bazel +update_bazel_macos +which bazel +bazel version +set_bazel_outdir + +# Pick a more recent version of xcode +sudo xcode-select --switch /Applications/Xcode_10.3.app/Contents/Developer + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.6 + +# Export required variables for running pip.sh +export OS_TYPE="MACOS" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.6' + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="//tensorflow/python/..." +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-nomac,-no_mac,-no_oss,-oss_serial,-no_oss_py35,-v1only,-gpu,-tpu,-benchmark-test' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh new file mode 100644 index 00000000000000..9b7c6720bfbad4 --- /dev/null +++ b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/nonpip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +# Install latest bazel +update_bazel_macos +which bazel +bazel version +set_bazel_outdir + +# Pick a more recent version of xcode +sudo xcode-select --switch /Applications/Xcode_10.3.app/Contents/Developer +python -m virtualenv tf_build_env --system-site-packages +source tf_build_env/bin/activate + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.7 + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export TF2_BEHAVIOR=1 +export PYTHON_BIN_PATH=$(which python3.7) +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="-no_oss,-oss_serial,-nomac,-no_mac$(maybe_skip_v1)" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} \ + -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/macos/cpu_py37_full/pip.sh b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/pip.sh new file mode 100644 index 00000000000000..560c82b629364b --- /dev/null +++ b/tensorflow/tools/ci_build/release/macos/cpu_py37_full/pip.sh @@ -0,0 +1,53 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +# Install latest bazel +update_bazel_macos +which bazel +bazel version +set_bazel_outdir + +# Pick a more recent version of xcode +sudo xcode-select --switch /Applications/Xcode_10.3.app/Contents/Developer + +# Install macos pip dependencies +install_macos_pip_deps sudo pip3.7 + +# Export required variables for running pip.sh +export OS_TYPE="MACOS" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.7' + +# Run configure. +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="//tensorflow/python/..." +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-nomac,-no_mac,-no_oss,-oss_serial,-no_oss_py37,-v1only,-gpu,-tpu,-benchmark-test' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/nonpip.sh new file mode 100644 index 00000000000000..e14019908bbe34 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/nonpip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.5) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py35,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/pip.sh new file mode 100644 index 00000000000000..dde6bed4cc3f46 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py35_full/pip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.5' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py35,-v1only' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nonpip.sh new file mode 100644 index 00000000000000..f243eb7944d462 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/nonpip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py36,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/pip.sh new file mode 100644 index 00000000000000..a32610b2c068d4 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py36_full/pip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.6' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py36,-v1only' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/nonpip.sh new file mode 100644 index 00000000000000..900ef053b61f47 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/nonpip.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.7) +export TF2_BEHAVIOR=1 +yes "" | "$PYTHON_BIN_PATH" configure.py +tag_filters="-no_oss,-oss_serial,-gpu,-tpu,-benchmark-test,-no_oss_py37,-v1only" + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Run tests +bazel test --test_output=errors --config=opt --test_lang_filters=py \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --build_tag_filters="${tag_filters}" \ + --test_tag_filters="${tag_filters}" -- \ + ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/pip.sh new file mode 100644 index 00000000000000..7b19819a771d9d --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/cpu_py37_full/pip.sh @@ -0,0 +1,52 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="CPU" +export TF_PYTHON_VERSION='python3.7' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_BUILD_FLAGS="--config=opt --config=v2 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain" +export TF_TEST_FLAGS="--define=no_tensorflow_py_deps=true --test_lang_filters=py --test_output=errors --verbose_failures=true --keep_going --test_env=TF2_BEHAVIOR=1" +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export TF_TEST_FILTER_TAGS='-no_oss,-oss_serial,-no_oss_py37,-v1only' +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME="tensorflow" +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/custom_op/nightly.sh b/tensorflow/tools/ci_build/release/ubuntu_16/custom_op/nightly.sh new file mode 100644 index 00000000000000..3f80dd4253612e --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/custom_op/nightly.sh @@ -0,0 +1,33 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e + +# 1. Build the test server +UBUNTU16_CPU_IMAGE="tensorflow/tensorflow:nightly-custom-op-ubuntu16" +UBUNTU16_GPU_IMAGE="tensorflow/tensorflow:nightly-custom-op-gpu-ubuntu16" + +# Build the docker image +cd tensorflow/tools/ci_build +docker build --no-cache -t "${UBUNTU16_CPU_IMAGE}" -f Dockerfile.custom_op_ubuntu_16 . +docker build --no-cache -t "${UBUNTU16_GPU_IMAGE}" -f Dockerfile.custom_op_ubuntu_16_gpu . + +# Log into docker hub, push the image and log out +docker login -u "${TF_DOCKER_USERNAME}" -p "${TF_DOCKER_PASSWORD}" + +docker push "${UBUNTU16_CPU_IMAGE}" +docker push "${UBUNTU16_GPU_IMAGE}" + +docker logout#!/bin/bash diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_pip_on_cpu/build.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_pip_on_cpu/build.sh new file mode 100755 index 00000000000000..ff7cd318a295f8 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_pip_on_cpu/build.sh @@ -0,0 +1,19 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +echo "This build not enabled for 2.0" diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh new file mode 100644 index 00000000000000..b211c092b7bb6e --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/nonpip.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.5) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-nogpu,-no_oss,-oss_serial,-no_oss_py35" + +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --test_tag_filters=${tag_filters} \ + --build_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip.sh new file mode 100644 index 00000000000000..b74793a3356187 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py35_full/pip.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.5 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.5' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-nogpu,-no_oss,-oss_serial,-no_oss_py35' +export TF_BUILD_FLAGS="--config=opt --config=v2 --config=cuda --distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME=${PROJECT_NAME} +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh new file mode 100644 index 00000000000000..7bc2323a552c92 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/nonpip.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.6) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-nogpu,-no_oss,-oss_serial,-no_oss_py36" + +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --test_tag_filters=${tag_filters} \ + --build_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip.sh new file mode 100644 index 00000000000000..b56942ef5c3910 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py36_full/pip.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.6 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.6' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-nogpu,-no_oss,-oss_serial,-no_oss_py36' +export TF_BUILD_FLAGS="--config=opt --config=v2 --config=cuda --distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME=${PROJECT_NAME} +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh new file mode 100644 index 00000000000000..54094a8870141c --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/nonpip.sh @@ -0,0 +1,58 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +update_bazel_linux + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.7) +export TF2_BEHAVIOR=1 +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +tag_filters="gpu,requires-gpu,-no_gpu,-nogpu,-no_oss,-oss_serial,-no_oss_py37" + +bazel test --config=cuda --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --test_lang_filters=py \ + --build_tag_filters=${tag_filters} \ + --test_tag_filters=${tag_filters} \ + --test_timeout="300,450,1200,3600" --local_test_jobs=4 \ + --test_output=errors --verbose_failures=true --keep_going \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ + -- ${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... -//tensorflow/python:contrib_test -//tensorflow/examples/... diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip.sh new file mode 100644 index 00000000000000..c020387b00a458 --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/gpu_py37_full/pip.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +install_ubuntu_16_pip_deps pip3.7 +# Update bazel +update_bazel_linux + +# Export required variables for running pip.sh +export OS_TYPE="UBUNTU" +export CONTAINER_TYPE="GPU" +export TF_PYTHON_VERSION='python3.7' + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=1 +export TF_CUDA_VERSION=10 +export TF_CUDNN_VERSION=7 +export TF_NEED_TENSORRT=1 +export TENSORRT_INSTALL_PATH=/usr/local/tensorrt +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which ${TF_PYTHON_VERSION}) +export PROJECT_NAME="tensorflow_gpu" +export LD_LIBRARY_PATH="/usr/local/cuda:/usr/local/cuda/lib64:/usr/local/cuda/extras/CUPTI/lib64:$TENSORRT_INSTALL_PATH/lib" +export TF_CUDA_COMPUTE_CAPABILITIES=3.5,3.7,5.2,6.0,6.1,7.0 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +# Get the default test targets for bazel. +source tensorflow/tools/ci_build/build_scripts/PRESUBMIT_BUILD_TARGETS.sh + +# Export optional variables for running pip.sh +export TF_TEST_FILTER_TAGS='gpu,requires-gpu,-no_gpu,-nogpu,-no_oss,-oss_serial,-no_oss_py37' +export TF_BUILD_FLAGS="--config=opt --config=v2 --config=cuda --distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain " +export TF_TEST_FLAGS="--test_tag_filters=${TF_TEST_FILTER_TAGS} --build_tag_filters=${TF_TEST_FILTER_TAGS} \ +--distinct_host_configuration=false \ +--action_env=TF_CUDA_VERSION --action_env=TF_CUDNN_VERSION --test_env=TF2_BEHAVIOR=1 \ +--config=cuda --test_output=errors --local_test_jobs=4 --test_lang_filters=py \ +--verbose_failures=true --keep_going --define=no_tensorflow_py_deps=true \ +--run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute " +export TF_TEST_TARGETS="${DEFAULT_BAZEL_TARGETS} -//tensorflow/lite/... " +export TF_PIP_TESTS="test_pip_virtualenv_non_clean test_pip_virtualenv_clean" +export IS_NIGHTLY=0 # Not nightly +export TF_PROJECT_NAME=${PROJECT_NAME} +export TF_PIP_TEST_ROOT="pip_test" + +./tensorflow/tools/ci_build/builds/pip_new.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh b/tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh new file mode 100644 index 00000000000000..d504650da458fe --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/sanity/build.sh @@ -0,0 +1,35 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e + +# Install latest bazel +source tensorflow/tools/ci_build/release/common.sh +update_bazel_linux +which bazel + +# We need py3 lint +sudo pip3 install pep8 + +# TODO(gunan): figure out why we get stuck with later versions of pylint. +# Install pylint. +sudo python2 -m pip install pylint==1.6.4 +sudo python3 -m pip install pylint==1.6.4 + +# TODO(yifeif): print pylint version for debug. remove later. +python3 -m pylint --version + +# Run tensorflow sanity checks. +tensorflow/tools/ci_build/ci_sanity.sh diff --git a/tensorflow/tools/ci_build/release/ubuntu_16/tpu_py37_full/nonpip.sh b/tensorflow/tools/ci_build/release/ubuntu_16/tpu_py37_full/nonpip.sh new file mode 100644 index 00000000000000..97734e93981e8d --- /dev/null +++ b/tensorflow/tools/ci_build/release/ubuntu_16/tpu_py37_full/nonpip.sh @@ -0,0 +1,51 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh +source tensorflow/tools/ci_build/ctpu/ctpu.sh + +install_ubuntu_16_pip_deps pip3.7 +install_ctpu pip3.7 +ctpu_up -s v2-8 -g tensorflow-windows + +# Run configure. +export TF_NEED_GCP=1 +export TF_NEED_HDFS=1 +export TF_NEED_S3=1 +export TF_NEED_CUDA=0 +export TF_NEED_TENSORRT=0 +export CC_OPT_FLAGS='-mavx' +export PYTHON_BIN_PATH=$(which python3.7) +export TF2_BEHAVIOR=1 + +yes "" | "$PYTHON_BIN_PATH" configure.py + +tag_filters="tpu,requires-tpu,-no_tpu,-notpu,-no_oss,-no_oss_py37" + +bazel test --config=opt \ + --crosstool_top=//third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2010-nvcc-cuda10.0:toolchain \ + --linkopt=-lrt \ + --action_env=TF2_BEHAVIOR="${TF2_BEHAVIOR}" \ + --noincompatible_strict_action_env \ + --build_tag_filters=${tag_filters} \ + --test_tag_filters=${tag_filters} \ + --test_output=errors --verbose_failures=true --keep_going \ + --test_arg=--tpu="${TPU_NAME}" \ + --test_arg=--zone="${TPU_ZONE}" \ + --test_arg=--test_dir_base="gs://kokoro-tpu-testing/tempdir/" \ + -- //tensorflow/... -//tensorflow/compiler/... -//tensorflow/lite/... diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release.bat b/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release.bat new file mode 100644 index 00000000000000..f10ba0ecc4fedc --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python35 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release_pip_rename.sh new file mode 100644 index 00000000000000..43982623109318 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/cpu_py35_full/release_pip_rename.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +# Rename to tensorflow_cpu +for f in $(ls py_test_dir/tensorflow-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow_cpu + rm "${f}" +done diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release.bat b/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release.bat new file mode 100644 index 00000000000000..244e9479eb53e0 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release_pip_rename.sh new file mode 100644 index 00000000000000..43982623109318 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/cpu_py36_full/release_pip_rename.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +# Rename to tensorflow_cpu +for f in $(ls py_test_dir/tensorflow-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow_cpu + rm "${f}" +done diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release.bat b/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release.bat new file mode 100644 index 00000000000000..4164c3cc9b5395 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python37 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\cpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" diff --git a/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release_pip_rename.sh new file mode 100644 index 00000000000000..43982623109318 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/cpu_py37_full/release_pip_rename.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +# Rename to tensorflow_cpu +for f in $(ls py_test_dir/tensorflow-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow_cpu + rm "${f}" +done diff --git a/tensorflow/tools/ci_build/release/windows/gpu_pip_on_cpu/build.bat b/tensorflow/tools/ci_build/release/windows/gpu_pip_on_cpu/build.bat new file mode 100644 index 00000000000000..34c5ac7923c242 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/gpu_pip_on_cpu/build.bat @@ -0,0 +1,16 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +echo "This build not enabled on 2.0" diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release.bat b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release.bat new file mode 100644 index 00000000000000..71db61889bcff2 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python35 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_pip_rename.sh new file mode 100644 index 00000000000000..6a868382777791 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/gpu_py35_full/release_pip_rename.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +# Copy and rename to tensorflow +for f in $(ls py_test_dir/tensorflow_gpu-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow +done diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release.bat b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release.bat new file mode 100644 index 00000000000000..ed0c127afe5512 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python36 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_pip_rename.sh new file mode 100644 index 00000000000000..6a868382777791 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/gpu_py36_full/release_pip_rename.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +# Copy and rename to tensorflow +for f in $(ls py_test_dir/tensorflow_gpu-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow +done diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release.bat b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release.bat new file mode 100644 index 00000000000000..00d85ef9119b71 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release.bat @@ -0,0 +1,20 @@ +:: Copyright 2019 The TensorFlow Authors. All Rights Reserved. +:: +:: Licensed under the Apache License, Version 2.0 (the "License"); +:: you may not use this file except in compliance with the License. +:: You may obtain a copy of the License at +:: +:: http://www.apache.org/licenses/LICENSE-2.0 +:: +:: Unless required by applicable law or agreed to in writing, software +:: distributed under the License is distributed on an "AS IS" BASIS, +:: WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +:: See the License for the specific language governing permissions and +:: limitations under the License. +:: ============================================================================= + +SET PYTHON_DIRECTORY=Python37 + +CALL tensorflow\tools\ci_build\release\common_win.bat + +call tensorflow\tools\ci_build\windows\gpu\pip\run.bat --release_build --extra_build_flags "--config=v2" --extra_test_flags "--test_env=TF2_BEHAVIOR=1" diff --git a/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_pip_rename.sh b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_pip_rename.sh new file mode 100644 index 00000000000000..6a868382777791 --- /dev/null +++ b/tensorflow/tools/ci_build/release/windows/gpu_py37_full/release_pip_rename.sh @@ -0,0 +1,24 @@ +#!/bin/bash +# Copyright 2019 The TensorFlow Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================== +set -e +set -x + +source tensorflow/tools/ci_build/release/common.sh + +# Copy and rename to tensorflow +for f in $(ls py_test_dir/tensorflow_gpu-*cp3*-cp3*m-win_amd64.whl); do + copy_to_new_project_name "${f}" tensorflow +done diff --git a/tensorflow/tools/ci_build/xla/linux/rocm/run_py3.sh b/tensorflow/tools/ci_build/xla/linux/rocm/run_py3.sh index a0de128020661c..efbfbe99f583b4 100755 --- a/tensorflow/tools/ci_build/xla/linux/rocm/run_py3.sh +++ b/tensorflow/tools/ci_build/xla/linux/rocm/run_py3.sh @@ -19,23 +19,27 @@ set -e set -x N_JOBS=$(grep -c ^processor /proc/cpuinfo) +N_GPUS=$(lspci|grep 'VGA'|grep 'AMD/ATI'|wc -l) echo "" -echo "Bazel will use ${N_JOBS} concurrent job(s)." +echo "Bazel will use ${N_JOBS} concurrent build job(s) and ${N_GPUS} concurrent test job(s)." echo "" # Run configure. export PYTHON_BIN_PATH=`which python3` export TF_NEED_ROCM=1 +export TF_GPU_COUNT=${N_GPUS} yes "" | $PYTHON_BIN_PATH configure.py echo "build --distinct_host_configuration=false" >> .tf_configure.bazelrc bazel clean # Run bazel test command. Double test timeouts to avoid flakes. -bazel test --config=rocm --test_tag_filters=-no_gpu,-benchmark-test,-no_oss -k \ - --jobs=${N_JOBS} --test_timeout 300,450,1200,3600 \ - --build_tests_only --test_output=errors --local_test_jobs=1 \ +bazel test --config=rocm --test_tag_filters=-no_gpu,-benchmark-test,-no_oss,-no_rocm -k \ + --jobs=${N_JOBS} --test_timeout 600,900,2400,7200 \ + --build_tests_only --test_output=errors --local_test_jobs=${TF_GPU_COUNT} \ + --test_sharding_strategy=disabled \ + --run_under=//tensorflow/tools/ci_build/gpu_build:parallel_gpu_execute \ --config=xla -- \ //tensorflow/compiler/... diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh index 952b5ee04f00c8..f21499090a8f7f 100755 --- a/tensorflow/tools/pip_package/build_pip_package.sh +++ b/tensorflow/tools/pip_package/build_pip_package.sh @@ -238,6 +238,7 @@ function usage() { echo " --project_name set project name to name" echo " --gpu build tensorflow_gpu" echo " --gpudirect build tensorflow_gpudirect" + echo " --rocm build tensorflow_rocm" echo " --nightly_flag build tensorflow nightly" echo "" exit 1 @@ -247,6 +248,7 @@ function main() { PKG_NAME_FLAG="" PROJECT_NAME="" GPU_BUILD=0 + ROCM_BUILD=0 NIGHTLY_BUILD=0 SRCDIR="" DSTDIR="" @@ -261,6 +263,8 @@ function main() { GPU_BUILD=1 elif [[ "$1" == "--gpudirect" ]]; then PKG_NAME_FLAG="--project_name tensorflow_gpudirect" + elif [[ "$1" == "--rocm" ]]; then + ROCM_BUILD=1 elif [[ "$1" == "--project_name" ]]; then shift if [[ -z "$1" ]]; then @@ -306,10 +310,14 @@ function main() { PKG_NAME_FLAG="--project_name ${PROJECT_NAME}" elif [[ ${NIGHTLY_BUILD} == "1" && ${GPU_BUILD} == "1" ]]; then PKG_NAME_FLAG="--project_name tf_nightly_gpu" + elif [[ ${NIGHTLY_BUILD} == "1" && ${ROCM_BUILD} == "1" ]]; then + PKG_NAME_FLAG="--project_name tf_nightly_rocm" elif [[ ${NIGHTLY_BUILD} == "1" ]]; then PKG_NAME_FLAG="--project_name tf_nightly" elif [[ ${GPU_BUILD} == "1" ]]; then PKG_NAME_FLAG="--project_name tensorflow_gpu" + elif [[ ${ROCM_BUILD} == "1" ]]; then + PKG_NAME_FLAG="--project_name tensorflow_rocm" fi build_wheel "$SRCDIR" "$DSTDIR" "$PKG_NAME_FLAG" diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py index 1fd803f06cc201..7a89a4475e50e3 100644 --- a/tensorflow/tools/pip_package/setup.py +++ b/tensorflow/tools/pip_package/setup.py @@ -47,17 +47,21 @@ # result for pip. # Also update tensorflow/tensorflow.bzl and # tensorflow/core/public/version.h -_VERSION = '2.0.0' +_VERSION = '2.0.1' REQUIRED_PACKAGES = [ 'absl-py >= 0.7.0', 'astor >= 0.6.0', 'backports.weakref >= 1.0rc1;python_version<"3.4"', 'enum34 >= 1.1.6;python_version<"3.4"', + # functools comes with python3, need to install the backport for python2 + 'functools32 >= 3.2.3;python_version<"3"', 'gast == 0.2.2', 'google_pasta >= 0.1.6', 'keras_applications >= 1.0.8', 'keras_preprocessing >= 1.0.5', + # mock comes with unittest.mock for python3, need to install for python2 + 'mock >= 2.0.0;python_version<"3"', 'numpy >= 1.16.0, < 2.0', 'opt_einsum >= 2.3.2', 'six >= 1.10.0', @@ -65,6 +69,9 @@ 'tensorboard >= 2.0.0, < 2.1.0', 'tensorflow_estimator >= 2.0.0, < 2.1.0', 'termcolor >= 1.1.0', + # python3 requires wheel 0.26 + 'wheel >= 0.26;python_version>="3"', + 'wheel;python_version<"3"', 'wrapt >= 1.11.1', ] @@ -81,16 +88,6 @@ sys.argv.remove('--project_name') sys.argv.pop(project_name_idx) -# python3 requires wheel 0.26 -if sys.version_info.major == 3: - REQUIRED_PACKAGES.append('wheel >= 0.26') -else: - REQUIRED_PACKAGES.append('wheel') - # mock comes with unittest.mock for python3, need to install for python2 - REQUIRED_PACKAGES.append('mock >= 2.0.0') - # functools comes with python3, need to install the backport for python2 - REQUIRED_PACKAGES.append('functools32 >= 3.2.3') - # tf-nightly should depend on tb-nightly if 'tf_nightly' in project_name: for i, pkg in enumerate(REQUIRED_PACKAGES): diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl index 5d222431810719..6019e81c5cfd5d 100755 --- a/tensorflow/workspace.bzl +++ b/tensorflow/workspace.bzl @@ -273,12 +273,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "org_sqlite", build_file = clean_dep("//third_party:sqlite.BUILD"), - sha256 = "d02fc4e95cfef672b45052e221617a050b7f2e20103661cda88387349a9b1327", - strip_prefix = "sqlite-amalgamation-3280000", + sha256 = "adf051d4c10781ea5cfabbbc4a2577b6ceca68590d23b58b8260a8e24cc5f081", + strip_prefix = "sqlite-amalgamation-3300100", system_build_file = clean_dep("//third_party/systemlibs:sqlite.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2019/sqlite-amalgamation-3280000.zip", - "https://www.sqlite.org/2019/sqlite-amalgamation-3280000.zip", + "https://storage.googleapis.com/mirror.tensorflow.org/www.sqlite.org/2019/sqlite-amalgamation-3300100.zip", + "https://www.sqlite.org/2019/sqlite-amalgamation-3300100.zip", ], ) @@ -291,7 +291,7 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): system_build_file = clean_dep("//third_party/systemlibs:gif.BUILD"), urls = [ "https://storage.googleapis.com/mirror.tensorflow.org/pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.2.1.tar.gz", - "http://pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.2.1.tar.gz", + "https://pilotfiber.dl.sourceforge.net/project/giflib/giflib-5.2.1.tar.gz", ], ) @@ -500,12 +500,12 @@ def tf_repositories(path_prefix = "", tf_repo_name = ""): tf_http_archive( name = "curl", build_file = clean_dep("//third_party:curl.BUILD"), - sha256 = "4376ac72b95572fb6c4fbffefb97c7ea0dd083e1974c0e44cd7e49396f454839", - strip_prefix = "curl-7.65.3", + sha256 = "d0393da38ac74ffac67313072d7fe75b1fa1010eb5987f63f349b024a36b7ffb", + strip_prefix = "curl-7.66.0", system_build_file = clean_dep("//third_party/systemlibs:curl.BUILD"), urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.65.3.tar.gz", - "https://curl.haxx.se/download/curl-7.65.3.tar.gz", + "https://storage.googleapis.com/mirror.tensorflow.org/curl.haxx.se/download/curl-7.66.0.tar.gz", + "https://curl.haxx.se/download/curl-7.66.0.tar.gz", ], ) diff --git a/third_party/curl.BUILD b/third_party/curl.BUILD index b99a35083339a9..c28dd15461624e 100644 --- a/third_party/curl.BUILD +++ b/third_party/curl.BUILD @@ -158,6 +158,7 @@ cc_library( "lib/pop3.h", "lib/progress.c", "lib/progress.h", + "lib/quic.h", "lib/rand.c", "lib/rand.h", "lib/rtsp.c",