diff --git a/.bazelrc b/.bazelrc index 0123bdae..3217fe32 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,6 +4,8 @@ build --define=grpc_no_ares=true # Suppress all warning messages. build:short_logs --output_filter=DONT_MATCH_ANYTHING +build --copt="-fvisibility=hidden" + # Force python3 build --action_env=PYTHON_BIN_PATH=python3 build --repo_env=PYTHON_BIN_PATH=python3 @@ -14,15 +16,13 @@ build:manylinux2014 --crosstool_top=//third_party/toolchains/preconfig/ubuntu16. build -c opt build --cxxopt="-std=c++17" build --host_cxxopt="-std=c++17" -# We can remove this flag (we keep it explicit for now) -build --cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1" -build --host_cxxopt="-D_GLIBCXX_USE_CXX11_ABI=1" build --auto_output_filter=subpackages -build --copt="-Wall" --copt="-Wno-sign-compare" -build --linkopt="-lrt -lm" + +build --action_env MACOSX_DEPLOYMENT_TARGET=10.14 + # We build with AVX and eigen byte alignment to match tensorflow's (and Eigen) # pip package byte alignment. See b/186669968 for more details. -build --copt=-mavx --copt=-DEIGEN_MAX_ALIGN_BYTES=64 +build --copt=-DEIGEN_MAX_ALIGN_BYTES=64 # TF isn't built in dbg mode, so our dbg builds will segfault due to inconsistency # of defines when using tf's headers. In particular in refcount.h. diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 00000000..e230c839 --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +5.3.0 \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..39108f34 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +python_bin_path.sh +bazel-* +.ipynb_checkpoints +.reverb.bazelrc \ No newline at end of file diff --git a/WORKSPACE b/WORKSPACE index 1d86c9b8..8470fbe8 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -1,38 +1,50 @@ workspace(name = "reverb") -# To change to a version of protoc compatible with tensorflow: -# 1. Convert the required header version to a version string, e.g.: -# 3011004 => "3.11.4" -# 2. Calculate the sha256 of the binary: -# PROTOC_VERSION="3.11.4" -# curl -L "https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip" | sha256sum -# 3. Update the two variables below. -# -PROTOC_VERSION = "21.0" -PROTOC_SHA256 = "a2a92003da7b8c0c08aab530a3c1967d377c2777723482adb9d2eb38c87a9d5f" +load("//tf:tf_configure.bzl", "tf_configure") -load( - "//reverb/cc/platform/default:repo.bzl", - "absl_deps", - "cc_tf_configure", - "github_apple_deps", - "github_grpc_deps", - "googletest_deps", - "protoc_deps", - "python_deps", -) +tf_configure(name = "local_config_tf") -googletest_deps() +load("@bazel_tools//tools/build_defs/repo:git.bzl", "git_repository") +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") -absl_deps() +http_archive( + name = "pybind11", + build_file = "@pybind11_bazel//:pybind11.BUILD", + sha256 = "832e2f309c57da9c1e6d4542dedd34b24e4192ecb4d62f6f4866a737454c9970", + strip_prefix = "pybind11-2.10.4", + urls = ["https://github.com/pybind/pybind11/archive/refs/tags/v2.10.4.tar.gz"], +) +http_archive( + name = "pybind11_bazel", + sha256 = "6426567481ee345eb48661e7db86adc053881cb4dd39fbf527c8986316b682b9", + strip_prefix = "pybind11_bazel-fc56ce8a8b51e3dd941139d329b63ccfea1d304b", + urls = ["https://github.com/pybind/pybind11_bazel/archive/fc56ce8a8b51e3dd941139d329b63ccfea1d304b.zip"], +) +load("@pybind11_bazel//:python_configure.bzl", "python_configure") +python_configure(name = "local_config_python") + +git_repository( + name = "com_google_snappy", + commit = "c9f9edf6d75bb065fa47468bf035e051a57bec7c", + remote = "https://github.com/google/snappy", +) -# Note that the Python dependencies are not tracked by bazel here, but -# in setup.py. +http_archive( + name = "com_github_grpc_grpc", + sha256 = "5e53505a6c84030a26c4fddd71b3f46feec8e0a8eccff2a903b189d349ca6ff5", + strip_prefix = "grpc-1.54.0", + urls = ["https://github.com/grpc/grpc/archive/refs/tags/v1.54.0.tar.gz"], +) -github_apple_deps() +ABSL_COMMIT = "fb3621f4f897824c0dbe0615fa94543df6192f30" +ABSL_SHA256 = "0320586856674d16b0b7a4d4afb22151bdc798490bb7f295eddd8f6a62b46fea" -## Begin GRPC related deps -github_grpc_deps() +http_archive( + name = "com_google_absl", + sha256 = ABSL_SHA256, + strip_prefix = "abseil-cpp-{commit}".format(commit = ABSL_COMMIT), + urls = ["https://github.com/abseil/abseil-cpp/archive/{commit}.tar.gz".format(commit = ABSL_COMMIT)], +) load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") @@ -41,30 +53,3 @@ grpc_deps() load("@com_github_grpc_grpc//bazel:grpc_extra_deps.bzl", "grpc_extra_deps") grpc_extra_deps() - - -load("@upb//bazel:workspace_deps.bzl", "upb_deps") - -upb_deps() - -load( - "@build_bazel_rules_apple//apple:repositories.bzl", - "apple_rules_dependencies", -) - -apple_rules_dependencies() - -load( - "@build_bazel_apple_support//lib:repositories.bzl", - "apple_support_dependencies", -) - -apple_support_dependencies() -## End GRPC related deps - - -cc_tf_configure() - -python_deps() - -protoc_deps(version = PROTOC_VERSION, sha256 = PROTOC_SHA256) diff --git a/configure.py b/configure.py index 15ddcc99..5fed51bd 100644 --- a/configure.py +++ b/configure.py @@ -39,6 +39,8 @@ import os import subprocess import sys +import platform +import tensorflow as tf _REVERB_BAZELRC_FILENAME = '.reverb.bazelrc' _REVERB_WORKSPACE_ROOT = '' @@ -172,6 +174,9 @@ def setup_python(environ_cp, force_defaults: bool): write_to_bazelrc('build --repo_env=PYTHON_BIN_PATH=\"%s"' % python_bin_path) environ_cp['PYTHON_BIN_PATH'] = python_bin_path + write_to_bazelrc('build --spawn_strategy=standalone') + write_to_bazelrc('build --strategy=Genrule=standalone') + # If choosen python_lib_path is from a path specified in the PYTHONPATH # variable, need to tell bazel to include PYTHONPATH if environ_cp.get('PYTHONPATH'): @@ -179,6 +184,25 @@ def setup_python(environ_cp, force_defaults: bool): if python_lib_path in python_paths: write_action_env_to_bazelrc('PYTHONPATH', environ_cp.get('PYTHONPATH')) + cflags = tf.sysconfig.get_compile_flags() + lflags = tf.sysconfig.get_link_flags() + + tf_header_dir = next(cflag[2:] for cflag in cflags if cflag.startswith('-I')) + tf_shared_dir = next(lflag[2:] for lflag in lflags if lflag.startswith('-L')) + tf_shared_lib_name = next(lflag[2:] for lflag in lflags if lflag.startswith('-l')) + lib_ext = { + 'Darwin': 'dylib', + 'Linux': 'so', + }[platform.system()] + tf_lib_prefix = 'lib' if platform.system() in ('Darwin', 'Linux') else '' + + write_action_env_to_bazelrc('TF_HEADER_DIR', tf_header_dir) + write_action_env_to_bazelrc('TF_SHARED_LIBRARY_DIR', tf_shared_dir) + write_action_env_to_bazelrc('TF_SHARED_LIBRARY_NAME', tf_lib_prefix + tf_shared_lib_name + os.path.extsep + lib_ext) + + # if platform.system() == 'Darwin': + # write_to_bazelrc('build --copt="-undefined dynamic_lookup"') + # Write tools/python_bin_path.sh with open(os.path.join(_REVERB_WORKSPACE_ROOT, 'python_bin_path.sh'), 'w') as f: diff --git a/docker/release.dockerfile b/docker/release.dockerfile deleted file mode 100644 index 798d48ff..00000000 --- a/docker/release.dockerfile +++ /dev/null @@ -1,113 +0,0 @@ -ARG cpu_base_image="tensorflow/build:latest-python3.9" -ARG base_image=$cpu_base_image -FROM $base_image - -LABEL maintainer="no-reply@google.com" - -# Re-declare args because the args declared before FROM can't be used in any -# instruction after a FROM. -ARG cpu_base_image="tensorflow/build:latest-python3.9" -ARG base_image=$cpu_base_image -ARG tensorflow_pip="tf-nightly" -ARG python_version="python3.9" -ARG APT_COMMAND="apt-get -o Acquire::Retries=3 -y" - -# Stops tzdata from asking about timezones and blocking install on user input. -ARG DEBIAN_FRONTEND=noninteractive -ENV TZ=America/Los_Angeles - -# Pick up some TF dependencies -RUN ${APT_COMMAND} update && ${APT_COMMAND} install -y --no-install-recommends \ - software-properties-common \ - aria2 \ - build-essential \ - curl \ - git \ - less \ - libfreetype6-dev \ - libhdf5-serial-dev \ - libpng-dev \ - libzmq3-dev \ - lsof \ - pkg-config \ - python3.8-dev \ - python3.9-dev \ - python3.10-dev \ - python3.11-dev \ - # python >= 3.8 needs distutils for packaging. - python3.8-distutils \ - python3.9-distutils \ - python3.10-distutils \ - python3.11-distutils \ - rename \ - rsync \ - sox \ - unzip \ - vim \ - && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* - -RUN curl -O https://bootstrap.pypa.io/get-pip.py - -# Installs known working version of bazel. -ARG bazel_version=5.3.0 -ENV BAZEL_VERSION ${bazel_version} -RUN mkdir /bazel && \ - cd /bazel && \ - curl -fSsL -O https://github.com/bazelbuild/bazel/releases/download/$BAZEL_VERSION/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ - chmod +x bazel-*.sh && \ - ./bazel-$BAZEL_VERSION-installer-linux-x86_64.sh && \ - cd / && \ - rm -f /bazel/bazel-$BAZEL_VERSION-installer-linux-x86_64.sh - -ARG pip_dependencies=' \ - absl-py \ - contextlib2 \ - dataclasses \ - dm-tree>=0.1.5 \ - h5py \ - mock \ - numpy \ - oauth2client \ - pandas \ - portpicker' - -RUN for python in ${python_version}; do \ - $python get-pip.py && \ - $python -mpip uninstall -y tensorflow tensorflow-gpu tf-nightly tf-nightly-gpu && \ - $python -mpip --no-cache-dir install ${tensorflow_pip} --upgrade && \ - $python -mpip --no-cache-dir install $pip_dependencies; \ - done -RUN rm get-pip.py - -# Removes existing links so they can be created to point where we expect. -RUN rm /dt9/usr/include/x86_64-linux-gnu/python3.8 -RUN rm /dt9/usr/include/x86_64-linux-gnu/python3.9 -RUN rm /dt9/usr/include/x86_64-linux-gnu/python3.10 -RUN rm /dt9/usr/include/x86_64-linux-gnu/python3.11 - -# Needed until this is included in the base TF image. -RUN ln -s "/usr/include/x86_64-linux-gnu/python3.8" "/dt9/usr/include/x86_64-linux-gnu/python3.8" -RUN ln -s "/usr/include/x86_64-linux-gnu/python3.9" "/dt9/usr/include/x86_64-linux-gnu/python3.9" -RUN ln -s "/usr/include/x86_64-linux-gnu/python3.10" "/dt9/usr/include/x86_64-linux-gnu/python3.10" -RUN ln -s "/usr/include/x86_64-linux-gnu/python3.11" "/dt9/usr/include/x86_64-linux-gnu/python3.11" - -# bazel build -c opt --copt=-mavx --config=manylinux2014 --test_output=errors //... - -# Update binutils to avoid linker(gold) issue. See b/227299577#comment9 -RUN \ - wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils_2.35.1-1ubuntu1_amd64.deb \ - && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils-x86-64-linux-gnu_2.35.1-1ubuntu1_amd64.deb \ - && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/binutils-common_2.35.1-1ubuntu1_amd64.deb \ - && wget http://old-releases.ubuntu.com/ubuntu/pool/main/b/binutils/libbinutils_2.35.1-1ubuntu1_amd64.deb - -RUN \ - dpkg -i binutils_2.35.1-1ubuntu1_amd64.deb \ - binutils-x86-64-linux-gnu_2.35.1-1ubuntu1_amd64.deb \ - binutils-common_2.35.1-1ubuntu1_amd64.deb \ - libbinutils_2.35.1-1ubuntu1_amd64.deb - -WORKDIR "/tmp/reverb" - -CMD ["/bin/bash"] diff --git a/oss_build.sh b/oss_build.sh old mode 100644 new mode 100755 index e683ad28..360243c1 --- a/oss_build.sh +++ b/oss_build.sh @@ -26,7 +26,9 @@ set -e set -o pipefail # Flags -PYTHON_VERSIONS=3.9 # Options 3.9 (default), 3.8, 3.10 or 3.11. +PYTHON_VERSIONS=3.9 # Options 3.7 (default), 3.8, or 3.9. +# CLEAN=true # Set to true to run bazel clean. +# CLEAR_CACHE=true # Set to true to delete Bazel cache folder. b/279235134 CLEAN=false # Set to true to run bazel clean. CLEAR_CACHE=false # Set to true to delete Bazel cache folder. b/279235134 OUTPUT_DIR=/tmp/reverb/dist/ @@ -39,7 +41,7 @@ PIP_PKG_EXTRA_ARGS="" # Extra args passed to `build_pip_package`. if [[ $# -lt 1 ]] ; then echo "Usage:" echo "--release [Indicates this is a release build. Otherwise nightly.]" - echo "--python [3.9(default)|3.8|3.10|3.11]" + echo "--python [3.7(default)|3.8|3.9|3.10]" echo "--clean [true to run bazel clean]" echo "--clear_bazel_cache [true to delete Bazel cache folder]" echo "--tf_dep_override [Required tensorflow version to pass to setup.py." @@ -98,28 +100,34 @@ for python_version in $PYTHON_VERSIONS; do # Cleans the environment. if [ "$CLEAN" = "true" ]; then if [ "$CLEAR_CACHE" = "true" ]; then - rm -rf $HOME/.cache/bazel + bazel clean --expunge fi bazel clean fi - if [ "$python_version" = "3.8" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3.8 && export PYTHON_LIB_PATH=/usr/local/lib/python3.8/dist-packages + if [ "$python_version" = "3.7" ]; then + ABI=cp37 + elif [ "$python_version" = "3.8" ]; then ABI=cp38 elif [ "$python_version" = "3.9" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3.9 && export PYTHON_LIB_PATH=/usr/local/lib/python3.9/dist-packages ABI=cp39 elif [ "$python_version" = "3.10" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3.10 && export PYTHON_LIB_PATH=/usr/local/lib/python3.10/dist-packages ABI=cp310 - elif [ "$python_version" = "3.11" ]; then - export PYTHON_BIN_PATH=/usr/bin/python3.11 && export PYTHON_LIB_PATH=/usr/local/lib/python3.11/dist-packages - ABI=cp311 else - echo "Error unknown --python. Only [3.8|3.9|3.10|3.11]" + echo "Error unknown --python. Only [3.7|3.8|3.9|3.10]" exit 1 fi + export PYTHON_BIN_PATH=`which python${python_version}` + export PYTHON_LIB_PATH=`${PYTHON_BIN_PATH} -c 'import site; print(site.getsitepackages()[0])'` + + if [ "$(uname)" = "Darwin" ]; then + PLATFORM=`${PYTHON_BIN_PATH} -c "from distutils import util; print(util.get_platform())"` + else + bazel_config="--config=manylinux2014" + PLATFORM="manylinux2014_x86_64" + fi + # Configures Bazel environment for selected Python version. $PYTHON_BIN_PATH configure.py @@ -130,24 +138,26 @@ for python_version in $PYTHON_VERSIONS; do # someone's system unexpectedly. We are executing the python tests after # installing the final package making this approach satisfactory. # TODO(b/157223742): Execute Python tests as well. - bazel test -c opt --copt=-mavx --config=manylinux2014 --test_output=errors //reverb/cc/... + # TODO: Please can I resurrect tests! + # bazel test -c opt $bazel_config --test_output=errors //reverb/cc/... EXTRA_OPT="" if [ "$DEBUG_BUILD" = "true" ]; then EXTRA_OPT="--copt=-g2" fi + # Builds Reverb and creates the wheel package. - bazel build --sandbox_debug --verbose_failures -c opt --copt=-mavx $EXTRA_OPT --config=manylinux2014 reverb/pip_package:build_pip_package + bazel build -c opt $EXTRA_OPT $bazel_config reverb/pip_package:build_pip_package ./bazel-bin/reverb/pip_package/build_pip_package --dst $OUTPUT_DIR $PIP_PKG_EXTRA_ARGS # Installs pip package. - $PYTHON_BIN_PATH -mpip install ${OUTPUT_DIR}*${ABI}*.whl + $PYTHON_BIN_PATH -m pip install --force-reinstall ${OUTPUT_DIR}/*${ABI}*.whl if [ "$PYTHON_TESTS" = "true" ]; then echo "Run Python tests..." set +e - bash run_python_tests.sh |& tee ./unittest_log.txt + bash run_python_tests.sh 2>&1 | tee ./unittest_log.txt UNIT_TEST_ERROR_CODE=$? set -e if [[ $UNIT_TEST_ERROR_CODE != 0 ]]; then diff --git a/reverb/BUILD b/reverb/BUILD index 415ca387..babf0e4b 100644 --- a/reverb/BUILD +++ b/reverb/BUILD @@ -1,14 +1,6 @@ # Description: Reverb is an efficient and easy to use prioritized replay system designed for ML research. - -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_py_test", - "reverb_pybind_deps", - "reverb_pybind_extension", - "reverb_pytype_library", - "reverb_pytype_strict_library", -) +load("@rules_python//python:defs.bzl", "py_library", "py_test") +load("@pybind11_bazel//:build_defs.bzl", "pybind_extension") package(default_visibility = [":__subpackages__"]) @@ -16,7 +8,16 @@ licenses(["notice"]) exports_files(["LICENSE"]) -reverb_pytype_strict_library( +config_setting( + name = "macos", + values = { + "apple_platform_type": "macos", + "cpu": "darwin", + }, + visibility = ["//visibility:public"], +) + +py_library( name = "reverb", srcs = ["__init__.py"], srcs_version = "PY3", @@ -38,23 +39,51 @@ reverb_pytype_strict_library( ], ) -reverb_pytype_strict_library( +pybind_extension( + name = "pybind", + srcs = ["pybind.cc"], + copts = ["-pthread"], + visibility = [":__subpackages__"], + deps = [ + "//reverb/cc:chunker", + "//reverb/cc:client", + "//reverb/cc:conversions", + "//reverb/cc:patterns_cc_proto", + "//reverb/cc:sampler", + "//reverb/cc:structured_writer", + "//reverb/cc:table", + "//reverb/cc:trajectory_writer", + "//reverb/cc:writer", + "//reverb/cc/checkpointing:interface", + "//reverb/cc/platform:checkpointing", + "//reverb/cc/platform:checkpointing_utils", + "//reverb/cc/platform:server", + "//reverb/cc/selectors:fifo", + "//reverb/cc/selectors:heap", + "//reverb/cc/selectors:interface", + "//reverb/cc/selectors:lifo", + "//reverb/cc/selectors:prioritized", + "//reverb/cc/selectors:uniform", + "//reverb/cc/support:tf_util", + "//reverb/cc/table_extensions:interface", + "@com_google_absl//absl/random", + ], +) + +py_library( name = "rate_limiters", srcs = ["rate_limiters.py"], + data = [":pybind.so"], srcs_version = "PY3", - deps = [ - ":pybind", - ], ) -reverb_pytype_library( +py_library( name = "client", srcs = ["client.py"], + data = [":pybind.so"], srcs_version = "PY3", - strict_deps = True, deps = [ ":errors", - ":pybind", ":replay_sample", ":reverb_types", ":structured_writer", @@ -62,147 +91,103 @@ reverb_pytype_library( ], ) -reverb_pytype_library( +py_library( name = "errors", srcs = ["errors.py"], srcs_version = "PY3", - strict_deps = True, deps = [], ) -reverb_pytype_library( +py_library( name = "server", srcs = ["server.py"], + data = [":pybind.so"], srcs_version = "PY3", - strict_deps = True, deps = [ ":client", ":item_selectors", - ":pybind", ":rate_limiters", ":reverb_types", "//reverb/platform/default:checkpointers", ], ) -reverb_pytype_library( +py_library( name = "replay_sample", srcs = ["replay_sample.py"], srcs_version = "PY3", - strict_deps = True, ) -reverb_pybind_extension( - name = "pybind", - srcs = ["pybind.cc"], - module_name = "libpybind", - pytype_srcs = ["pybind.pyi"], - srcs_version = "PY3ONLY", - visibility = [":__subpackages__"], - deps = [ - "//reverb/cc:chunker", - "//reverb/cc:client", - "//reverb/cc:conversions", - "//reverb/cc:patterns_cc_proto", - "//reverb/cc:sampler", - "//reverb/cc:structured_writer", - "//reverb/cc:table", - "//reverb/cc:trajectory_writer", - "//reverb/cc:writer", - "//reverb/cc/checkpointing:interface", - "//reverb/cc/platform:checkpointing", - "//reverb/cc/platform:checkpointing_utils", - "//reverb/cc/platform:server", - "//reverb/cc/selectors:fifo", - "//reverb/cc/selectors:heap", - "//reverb/cc/selectors:interface", - "//reverb/cc/selectors:lifo", - "//reverb/cc/selectors:prioritized", - "//reverb/cc/selectors:uniform", - "//reverb/cc/support:tf_util", - "//reverb/cc/table_extensions:interface", - ] + reverb_pybind_deps() + reverb_absl_deps(), -) - -reverb_pytype_library( +py_library( name = "tf_client", srcs = ["tf_client.py"], + data = ["//reverb/cc/ops:gen_reverb_ops"], srcs_version = "PY3", - strict_deps = True, deps = [ ":replay_sample", - "//reverb/cc/ops:gen_reverb_ops", ], ) -reverb_pytype_library( +py_library( name = "trajectory_dataset", srcs = ["trajectory_dataset.py"], + data = ["//reverb/cc/ops:gen_reverb_ops"], srcs_version = "PY3", - strict_deps = True, visibility = [":__subpackages__"], deps = [ ":client", ":replay_sample", - "//reverb/cc/ops:gen_reverb_ops", ], ) -reverb_pytype_library( +py_library( name = "timestep_dataset", srcs = ["timestep_dataset.py"], + data = ["//reverb/cc/ops:gen_reverb_ops"], srcs_version = "PY3", - strict_deps = True, visibility = ["//visibility:private"], deps = [ ":client", ":replay_sample", - "//reverb/cc/ops:gen_reverb_ops", ], ) -reverb_pytype_library( +py_library( name = "reverb_types", srcs = ["reverb_types.py"], + data = [":pybind.so"], srcs_version = "PY3", - strict_deps = True, - deps = [ - ":pybind", - "//reverb/cc:schema_py_pb2", - ], + deps = ["//reverb/cc:schema_py_pb2"], ) -reverb_pytype_strict_library( +py_library( name = "structured_writer", srcs = ["structured_writer.py"], + data = [":pybind.so"], srcs_version = "PY3", deps = [ ":errors", - ":pybind", ":reverb_types", "//reverb/cc:patterns_py_pb2", ], ) -reverb_pytype_strict_library( +py_library( name = "item_selectors", srcs = ["item_selectors.py"], + data = [":pybind.so"], srcs_version = "PY3", - deps = [":pybind"], ) -reverb_pytype_library( +py_library( name = "trajectory_writer", srcs = ["trajectory_writer.py"], + data = [":pybind.so"], srcs_version = "PY3", - strict_deps = True, - deps = [ - ":errors", - ":pybind", - ], + deps = [":errors"], ) -reverb_py_test( +py_test( name = "client_test", srcs = ["client_test.py"], python_version = "PY3", @@ -216,20 +201,20 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "server_test", srcs = ["server_test.py"], + data = [":pybind.so"], python_version = "PY3", deps = [ ":client", ":item_selectors", - ":pybind", ":rate_limiters", ":server", ], ) -reverb_py_test( +py_test( name = "tf_client_test", timeout = "short", srcs = ["tf_client_test.py"], @@ -244,7 +229,7 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "timestep_dataset_test", srcs = ["timestep_dataset_test.py"], python_version = "PY3", @@ -259,7 +244,7 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "timestep_dataset_eager_test", srcs = ["timestep_dataset_eager_test.py"], deps = [ @@ -268,7 +253,7 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "trajectory_dataset_test", srcs = ["trajectory_dataset_test.py"], python_version = "PY3", @@ -283,7 +268,7 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "trajectory_dataset_eager_test", srcs = ["trajectory_dataset_eager_test.py"], deps = [ @@ -292,7 +277,7 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "rate_limiters_test", srcs = ["rate_limiters_test.py"], python_version = "PY3", @@ -301,7 +286,7 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "pybind_test", srcs = ["pybind_test.py"], python_version = "PY3", @@ -310,20 +295,20 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "trajectory_writer_test", srcs = ["trajectory_writer_test.py"], + data = [":pybind.so"], python_version = "PY3", deps = [ ":client", ":errors", - ":pybind", ":server", ":trajectory_writer", ], ) -reverb_py_test( +py_test( name = "structured_writer_test", srcs = ["structured_writer_test.py"], python_version = "PY3", @@ -334,21 +319,20 @@ reverb_py_test( ], ) -reverb_py_test( +py_test( name = "replay_sample_test", srcs = ["replay_sample_test.py"], + data = [":pybind.so"], python_version = "PY3", deps = [ - ":pybind", ":replay_sample", ], ) -reverb_pytype_library( +py_library( name = "pattern_dataset", srcs = ["pattern_dataset.py"], srcs_version = "PY3", - strict_deps = False, tags = ["pytype_unchecked_annotations"], deps = [ ":replay_sample", @@ -358,7 +342,7 @@ reverb_pytype_library( ], ) -reverb_py_test( +py_test( name = "pattern_dataset_test", srcs = ["pattern_dataset_test.py"], deps = [ diff --git a/reverb/cc/BUILD b/reverb/cc/BUILD index 8aeae419..1682567e 100644 --- a/reverb/cc/BUILD +++ b/reverb/cc/BUILD @@ -1,17 +1,8 @@ # Description: Reverb is an efficient and easy to use prioritized replay system designed for ML research. - -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_grpc_library", - "reverb_cc_library", - "reverb_cc_proto_library", - "reverb_cc_test", - "reverb_grpc_deps", - "reverb_py_proto_library", - "reverb_pybind_deps", - "reverb_tf_deps", -) +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library", "py_grpc_library") +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@com_github_grpc_grpc//bazel:cc_grpc_library.bzl", "cc_grpc_library") package(default_visibility = ["//reverb:__subpackages__"]) @@ -19,7 +10,7 @@ licenses(["notice"]) exports_files(["LICENSE"]) -reverb_cc_test( +cc_test( name = "chunk_store_test", srcs = ["chunk_store_test.cc"], deps = [ @@ -28,10 +19,10 @@ reverb_cc_test( "//reverb/cc/platform:status_matchers", "//reverb/cc/platform:thread", "//reverb/cc/testing:proto_test_util", - ] + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "rate_limiter_test", srcs = ["rate_limiter_test.cc"], deps = [ @@ -40,10 +31,10 @@ reverb_cc_test( "//reverb/cc/platform:thread", "//reverb/cc/selectors:uniform", "//reverb/cc/testing:proto_test_util", - ] + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "table_test", srcs = ["table_test.cc"], deps = [ @@ -58,23 +49,23 @@ reverb_cc_test( "//reverb/cc/support:task_executor", "//reverb/cc/table_extensions:interface", "//reverb/cc/testing:proto_test_util", - ] + reverb_tf_deps() + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "tensor_compression_test", srcs = ["tensor_compression_test.cc"], deps = [ ":tensor_compression", "//reverb/cc/testing:tensor_testutil", - ] + reverb_tf_deps(), + ] ) -reverb_cc_test( +cc_test( name = "sampler_test", srcs = ["sampler_test.cc"], deps = [ - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":sampler", ":table", @@ -86,15 +77,15 @@ reverb_cc_test( "//reverb/cc/testing:proto_test_util", "//reverb/cc/testing:tensor_testutil", "//reverb/cc/testing:time_testutil", - ] + reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "writer_test", srcs = ["writer_test.cc"], deps = [ ":client", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":writer", "//reverb/cc/platform:logging", @@ -106,52 +97,61 @@ reverb_cc_test( "//reverb/cc/support:trajectory_util", "//reverb/cc/support:uint128", "//reverb/cc/testing:proto_test_util", - ] + reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "client_test", srcs = ["client_test.cc"], deps = [ ":chunker", ":client", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":trajectory_writer", "//reverb/cc/platform:status_matchers", "//reverb/cc/support:uint128", "//reverb/cc/testing:proto_test_util", - ] + reverb_grpc_deps(), + ], ) -reverb_cc_library( +cc_library( name = "errors", srcs = ["errors.cc"], hdrs = ["errors.h"], visibility = [ "//reverb:__subpackages__", ], - deps = reverb_absl_deps(), + deps = [ + "@com_google_absl//absl/status", + "@local_config_tf//:tf_header_lib" + ] ) -reverb_cc_library( +cc_library( name = "conversions", srcs = ["conversions.cc"], hdrs = ["conversions.h"], - deps = reverb_pybind_deps() + reverb_absl_deps(), + deps = [ + "@local_config_tf//:numpy_headers", + "@local_config_python//:python_headers", + "@local_config_tf//:tf_header_lib", + ] ) -reverb_cc_library( +cc_library( name = "chunk_store", srcs = ["chunk_store.cc"], hdrs = ["chunk_store.h"], deps = [ ":schema_cc_proto", "//reverb/cc/platform:hash_map", - ] + reverb_absl_deps(), + "@local_config_tf//:tf_header_lib", + "@com_google_absl//absl/status", + ] ) -reverb_cc_library( +cc_library( name = "table", srcs = [ "rate_limiter.cc", @@ -162,7 +162,7 @@ reverb_cc_library( "table.h", ], visibility = ["//reverb:__subpackages__"], - deps = [ + deps = depset([ ":chunk_store", ":errors", ":schema_cc_proto", @@ -176,10 +176,10 @@ reverb_cc_library( "//reverb/cc/support:task_executor", "//reverb/cc/support:trajectory_util", "//reverb/cc/table_extensions:interface", - ] + reverb_absl_deps() + reverb_tf_deps(), + ]), ) -reverb_cc_library( +cc_library( name = "tensor_compression", srcs = ["tensor_compression.cc"], hdrs = ["tensor_compression.h"], @@ -187,17 +187,35 @@ reverb_cc_library( deps = [ "//reverb/cc/platform:logging", "//reverb/cc/platform:snappy", - ] + reverb_tf_deps(), -) - -reverb_cc_library( + "@com_google_absl//absl/base", + "@com_google_absl//absl/base:core_headers", + "@com_google_absl//absl/functional:bind_front", + "@com_google_absl//absl/memory", + "@com_google_absl//absl/numeric:int128", + "@com_google_absl//absl/random", + "@com_google_absl//absl/random:distributions", + "@com_google_absl//absl/status", + "@com_google_absl//absl/status:statusor", + "@com_google_absl//absl/strings", + "@com_google_absl//absl/strings:cord", + "@com_google_absl//absl/strings:str_format", + "@com_google_absl//absl/synchronization", + "@com_google_absl//absl/time", + "@com_google_absl//absl/types:optional", + "@com_google_absl//absl/types:span", + "@com_google_absl//absl/flags:flag", + "@local_config_tf//:tf_header_lib", + ] +) + +cc_library( name = "sampler", srcs = ["sampler.cc"], hdrs = ["sampler.h"], - deps = [ + deps = depset([ ":chunk_store", ":errors", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":schema_cc_proto", ":table", @@ -212,16 +230,16 @@ reverb_cc_library( "//reverb/cc/support:signature", "//reverb/cc/support:tf_util", "//reverb/cc/support:trajectory_util", - ] + reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), + ]), ) -reverb_cc_library( +cc_library( name = "writer", srcs = ["writer.cc"], hdrs = ["writer.h"], visibility = ["//reverb:__subpackages__"], - deps = [ - ":reverb_service_cc_grpc_proto", + deps = depset([ + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":schema_cc_proto", ":tensor_compression", @@ -233,17 +251,19 @@ reverb_cc_library( "//reverb/cc/support:signature", "//reverb/cc/support:tf_util", "//reverb/cc/support:trajectory_util", - ] + reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), + "@com_google_absl//absl/random", + "@com_google_absl//absl/functional:bind_front", + ]), ) -reverb_cc_library( +cc_library( name = "trajectory_writer", srcs = ["trajectory_writer.cc"], hdrs = ["trajectory_writer.h"], visibility = ["//reverb:__subpackages__"], - deps = [ + deps = depset([ ":chunker", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":schema_cc_proto", "//reverb/cc/platform:hash_map", @@ -257,17 +277,18 @@ reverb_cc_library( "//reverb/cc/support:signature", "//reverb/cc/support:tf_util", "//reverb/cc/support:trajectory_util", - ] + reverb_tf_deps() + reverb_absl_deps() + reverb_grpc_deps(), + "@com_google_absl//absl/random", + ]), ) -reverb_cc_library( +cc_library( name = "streaming_trajectory_writer", srcs = ["streaming_trajectory_writer.cc"], hdrs = ["streaming_trajectory_writer.h"], visibility = ["//reverb:__subpackages__"], - deps = [ + deps = depset([ ":chunker", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":schema_cc_proto", ":trajectory_writer", @@ -278,15 +299,15 @@ reverb_cc_library( "//reverb/cc/support:grpc_util", "//reverb/cc/support:key_generators", "//reverb/cc/support:signature", - ] + reverb_tf_deps() + reverb_absl_deps() + reverb_grpc_deps(), + ]), ) -reverb_cc_test( +cc_test( name = "trajectory_writer_test", srcs = ["trajectory_writer_test.cc"], deps = [ ":chunker", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":trajectory_writer", "//reverb/cc/platform:logging", @@ -297,15 +318,15 @@ reverb_cc_test( "//reverb/cc/support:signature", "//reverb/cc/testing:proto_test_util", "//reverb/cc/testing:tensor_testutil", - ] + reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "streaming_trajectory_writer_test", srcs = ["streaming_trajectory_writer_test.cc"], deps = [ ":chunker", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":streaming_trajectory_writer", "//reverb/cc/platform:logging", @@ -314,15 +335,15 @@ reverb_cc_test( "//reverb/cc/support:signature", "//reverb/cc/testing:proto_test_util", "//reverb/cc/testing:tensor_testutil", - ] + reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), + ] ) -reverb_cc_library( +cc_library( name = "chunker", srcs = ["chunker.cc"], hdrs = ["chunker.h"], visibility = ["//reverb:__subpackages__"], - deps = [ + deps = depset([ ":schema_cc_proto", ":tensor_compression", "//reverb/cc/platform:hash_map", @@ -332,10 +353,11 @@ reverb_cc_library( "//reverb/cc/support:signature", "//reverb/cc/support:tf_util", "//reverb/cc/support:trajectory_util", - ] + reverb_tf_deps() + reverb_absl_deps(), + "@com_google_absl//absl/random", + ]), ) -reverb_cc_test( +cc_test( name = "chunker_test", srcs = ["chunker_test.cc"], deps = [ @@ -346,10 +368,10 @@ reverb_cc_test( "//reverb/cc/support:signature", "//reverb/cc/testing:proto_test_util", "//reverb/cc/testing:tensor_testutil", - ] + reverb_tf_deps() + reverb_absl_deps(), + ] ) -reverb_cc_library( +cc_library( name = "client", srcs = ["client.cc"], hdrs = ["client.h"], @@ -357,7 +379,7 @@ reverb_cc_library( deps = [ ":chunker", ":patterns_cc_proto", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":sampler", ":schema_cc_proto", @@ -371,41 +393,75 @@ reverb_cc_library( "//reverb/cc/support:grpc_util", "//reverb/cc/support:signature", "//reverb/cc/support:uint128", - ] + reverb_grpc_deps() + reverb_absl_deps(), + "@com_google_absl//absl/random", + ] ) -reverb_cc_proto_library( - name = "schema_cc_proto", +proto_library( + name = "schema_proto", srcs = ["schema.proto"], + deps = [ + "@local_config_tf//:tensor_proto", + "@local_config_tf//:struct_proto", + "@com_google_protobuf//:timestamp_proto", + ], + # visibility = ["//visibility:public"], ) -reverb_py_proto_library( +cc_proto_library( + name = "schema_cc_proto", + deps = [":schema_proto"], +) + +py_proto_library( name = "schema_py_pb2", - srcs = ["schema.proto"], - deps = [":schema_cc_proto"], + deps = [":schema_proto"], ) -reverb_cc_proto_library( - name = "patterns_cc_proto", +proto_library( + name = "patterns_proto", srcs = ["patterns.proto"], - visibility = ["//reverb:__subpackages__"], + deps = [ + "@local_config_tf//:tensor_proto", + "@local_config_tf//:struct_proto", + "@local_config_tf//:resource_handle_proto", + "@local_config_tf//:tensor_shape_proto", + "@local_config_tf//:types_proto", + ] +) + +cc_proto_library( + name = "patterns_cc_proto", + deps = [":patterns_proto"], ) -reverb_py_proto_library( +py_proto_library( name = "patterns_py_pb2", - srcs = ["patterns.proto"], - deps = [":patterns_cc_proto"], + deps = [":patterns_proto"], ) -reverb_cc_proto_library( - name = "reverb_service_cc_proto", +proto_library( + name = "reverb_service_proto", srcs = ["reverb_service.proto"], - deps = [":schema_cc_proto"], + deps = [ + ":schema_proto", + "@local_config_tf//:tensor_proto", + "@local_config_tf//:struct_proto", + "@local_config_tf//:resource_handle_proto", + "@local_config_tf//:tensor_shape_proto", + "@local_config_tf//:types_proto", + ], ) -reverb_cc_grpc_library( - name = "reverb_service_cc_grpc_proto", - srcs = ["reverb_service.proto"], +cc_proto_library( + name = "reverb_service_cc_proto", + deps = [":reverb_service_proto"], +) + +cc_grpc_library( + name = "reverb_service_cc_grpc", + srcs = [":reverb_service_proto"], + grpc_only = True, generate_mocks = True, visibility = [ "//reverb:__subpackages__", @@ -413,7 +469,19 @@ reverb_cc_grpc_library( deps = [":reverb_service_cc_proto"], ) -reverb_cc_library( +py_proto_library( + name = "reverb_service_py_pb2", + deps = [":reverb_service_proto"], +) + +py_grpc_library( + name = "reverb_service_py_pb2_grpc", + srcs = [":reverb_service_proto"], + deps = [":reverb_service_py_pb2"], +) + + +cc_library( name = "reverb_service_impl", srcs = ["reverb_service_impl.cc"], hdrs = [ @@ -421,7 +489,7 @@ reverb_cc_library( ], deps = [ ":reverb_server_reactor", - ":reverb_service_cc_grpc_proto", + ":reverb_service_cc_grpc", ":reverb_service_cc_proto", ":schema_cc_proto", ":table", @@ -433,11 +501,13 @@ reverb_cc_library( "//reverb/cc/support:grpc_util", "//reverb/cc/support:trajectory_util", "//reverb/cc/support:uint128", - ] + reverb_grpc_deps() + reverb_absl_deps(), - alwayslink = 1, + "@com_google_absl//absl/random", + "@com_google_absl//absl/flags:flag", + ], + alwayslink = True, ) -reverb_cc_library( +cc_library( name = "task_worker", hdrs = ["task_worker.h"], deps = [ @@ -448,10 +518,10 @@ reverb_cc_library( "//reverb/cc/platform:thread", "//reverb/cc/support:periodic_closure", "//reverb/cc/support:unbounded_queue", - ] + reverb_absl_deps(), + ] ) -reverb_cc_library( +cc_library( name = "reverb_server_reactor", hdrs = ["reverb_server_reactor.h"], deps = [ @@ -459,17 +529,20 @@ reverb_cc_library( "//reverb/cc/platform:logging", "//reverb/cc/platform:status_macros", "//reverb/cc/support:grpc_util", - ] + reverb_grpc_deps() + reverb_absl_deps(), + ] ) -reverb_cc_library( +cc_library( name = "thread_stats", srcs = ["thread_stats.cc"], hdrs = ["thread_stats.h"], - deps = reverb_absl_deps(), + deps = [ + "@com_google_absl//absl/time", + "@com_google_absl//absl/status", + ], ) -reverb_cc_test( +cc_test( name = "reverb_service_impl_test", srcs = ["reverb_service_impl_test.cc"], deps = [ @@ -485,18 +558,18 @@ reverb_cc_test( "//reverb/cc/selectors:interface", "//reverb/cc/selectors:uniform", "//reverb/cc/testing:proto_test_util", - ] + reverb_grpc_deps() + reverb_absl_deps() + reverb_tf_deps(), + ] ) -reverb_cc_test( +cc_test( name = "thread_stats_test", srcs = ["thread_stats_test.cc"], deps = [ ":thread_stats", - ] + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "structured_writer_test", srcs = ["structured_writer_test.cc"], deps = [ @@ -509,20 +582,21 @@ reverb_cc_test( "//reverb/cc/support:signature", "//reverb/cc/testing:proto_test_util", "//reverb/cc/testing:tensor_testutil", - ] + reverb_absl_deps() + reverb_tf_deps(), + ] ) -reverb_cc_library( +cc_library( name = "structured_writer", srcs = ["structured_writer.cc"], hdrs = ["structured_writer.h"], visibility = ["//reverb:__subpackages__"], - deps = [ + deps = depset([ ":chunker", ":patterns_cc_proto", ":trajectory_writer", "//reverb/cc/platform:hash_map", "//reverb/cc/platform:logging", "//reverb/cc/platform:status_macros", - ] + reverb_absl_deps() + reverb_tf_deps(), + "@com_google_absl//absl/status:statusor", + ]), ) diff --git a/reverb/cc/__init__.py b/reverb/cc/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reverb/cc/checkpointing/BUILD b/reverb/cc/checkpointing/BUILD index 4bc1bc0f..f26d0ffb 100644 --- a/reverb/cc/checkpointing/BUILD +++ b/reverb/cc/checkpointing/BUILD @@ -1,34 +1,32 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_library", - "reverb_cc_proto_library", - "reverb_py_proto_library", -) +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_proto_library") +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_cc_proto_library( - name = "checkpoint_cc_proto", +proto_library( + name = "checkpoint_proto", srcs = ["checkpoint.proto"], deps = [ - "//reverb/cc:schema_cc_proto", + "//reverb/cc:schema_proto", + "@local_config_tf//:struct_proto", ], ) -reverb_cc_library( +cc_proto_library( + name = "checkpoint_cc_proto", + deps = [":checkpoint_proto"], +) + +cc_library( name = "interface", hdrs = ["interface.h"], - deps = ["//reverb/cc:table"] + reverb_absl_deps(), + deps = ["//reverb/cc:table"] ) -reverb_py_proto_library( +py_proto_library( name = "checkpoint_py_pb2", - srcs = ["checkpoint.proto"], - deps = [ - ":checkpoint_cc_proto", - "//reverb/cc:schema_cc_proto", - ], + deps = [":checkpoint_proto"], ) diff --git a/reverb/cc/conversions.cc b/reverb/cc/conversions.cc index 80586745..5cf4b786 100644 --- a/reverb/cc/conversions.cc +++ b/reverb/cc/conversions.cc @@ -14,9 +14,6 @@ #include "reverb/cc/conversions.h" -#include "absl/status/status.h" -#include "absl/strings/str_cat.h" - namespace deepmind { namespace reverb { namespace pybind { @@ -68,23 +65,23 @@ char const *NumpyTypeName(int numpy_type) { } } -absl::Status PyObjectToString(PyObject *obj, const char **ptr, Py_ssize_t *len, - PyObject **ptr_owner) { +tensorflow::Status PyObjectToString(PyObject *obj, const char **ptr, + Py_ssize_t *len, PyObject **ptr_owner) { *ptr_owner = nullptr; if (PyBytes_Check(obj)) { char *buf; if (PyBytes_AsStringAndSize(obj, &buf, len) != 0) { - return absl::InternalError("Unable to get element as bytes."); + return tensorflow::errors::Internal("Unable to get element as bytes."); } *ptr = buf; } else if (PyUnicode_Check(obj)) { *ptr = PyUnicode_AsUTF8AndSize(obj, len); if (*ptr == nullptr) { - return absl::InternalError("Unable to convert element to UTF-8"); + return tensorflow::errors::Internal("Unable to convert element to UTF-8"); } } else { - return absl::InternalError( - absl::StrCat("Unsupported object type ", obj->ob_type->tp_name)); + return tensorflow::errors::Internal("Unsupported object type ", + obj->ob_type->tp_name); } return tensorflow::OkStatus(); @@ -93,14 +90,14 @@ absl::Status PyObjectToString(PyObject *obj, const char **ptr, Py_ssize_t *len, // Iterate over the string array 'array', extract the ptr and len of each string // element and call f(ptr, len). template -absl::Status PyBytesArrayMap(PyArrayObject *array, F f) { +tensorflow::Status PyBytesArrayMap(PyArrayObject *array, F f) { auto iter = make_safe(PyArray_IterNew(reinterpret_cast(array))); while (PyArray_ITER_NOTDONE(iter.get())) { auto item = make_safe(PyArray_GETITEM( array, static_cast(PyArray_ITER_DATA(iter.get())))); if (!item) { - return absl::InternalError( + return tensorflow::errors::Internal( "Unable to get element from the feed - no item."); } Py_ssize_t len; @@ -114,8 +111,8 @@ absl::Status PyBytesArrayMap(PyArrayObject *array, F f) { return tensorflow::OkStatus(); } -absl::Status StringTensorToPyArray(const tensorflow::Tensor &tensor, - PyArrayObject *dst) { +tensorflow::Status StringTensorToPyArray(const tensorflow::Tensor &tensor, + PyArrayObject *dst) { DCHECK_EQ(tensor.dtype(), tensorflow::DT_STRING); auto iter = make_safe(PyArray_IterNew(reinterpret_cast(dst))); @@ -126,15 +123,15 @@ absl::Status StringTensorToPyArray(const tensorflow::Tensor &tensor, auto py_string = make_safe(PyBytes_FromStringAndSize(value.c_str(), value.size())); if (py_string == nullptr) { - return absl::InternalError(absl::StrCat( + return tensorflow::errors::Internal( "failed to create a python byte array when converting element #", i, - " of a TF_STRING tensor to a numpy ndarray")); + " of a TF_STRING tensor to a numpy ndarray"); } if (PyArray_SETITEM(dst, PyArray_ITER_DATA(iter.get()), py_string.get()) != 0) { - return absl::InternalError( - absl::StrCat("Error settings element #", i, " in the numpy ndarray")); + return tensorflow::errors::Internal("Error settings element #", i, + " in the numpy ndarray"); } PyArray_ITER_NEXT(iter.get()); @@ -143,8 +140,8 @@ absl::Status StringTensorToPyArray(const tensorflow::Tensor &tensor, return tensorflow::OkStatus(); } -absl::Status GetPyDescrFromDataType(tensorflow::DataType dtype, - PyArray_Descr **out_descr) { +tensorflow::Status GetPyDescrFromDataType(tensorflow::DataType dtype, + PyArray_Descr **out_descr) { switch (dtype) { #define TF_TO_PY_ARRAY_TYPE_CASE(TF_DTYPE, PY_ARRAY_TYPE) \ case TF_DTYPE: \ @@ -170,20 +167,20 @@ absl::Status GetPyDescrFromDataType(tensorflow::DataType dtype, #undef TF_DTYPE_TO_PY_ARRAY_TYPE_CASE default: - return absl::InternalError(absl::StrCat( - "Unsupported tf type: ", tensorflow::DataType_Name(dtype))); + return tensorflow::errors::Internal( + "Unsupported tf type: ", tensorflow::DataType_Name(dtype)); } return tensorflow::OkStatus(); } -absl::Status GetPyDescrFromTensor(const tensorflow::Tensor &tensor, - PyArray_Descr **out_descr) { +tensorflow::Status GetPyDescrFromTensor(const tensorflow::Tensor &tensor, + PyArray_Descr **out_descr) { return GetPyDescrFromDataType(tensor.dtype(), out_descr); } -absl::Status GetTensorDtypeFromPyArray(PyArrayObject *array, - tensorflow::DataType *out_tf_datatype) { +tensorflow::Status GetTensorDtypeFromPyArray( + PyArrayObject *array, tensorflow::DataType *out_tf_datatype) { int pyarray_type = PyArray_TYPE(array); switch (pyarray_type) { #define NP_TO_TF_DTYPE_CASE(NP_DTYPE, TF_DTYPE) \ @@ -220,24 +217,25 @@ absl::Status GetTensorDtypeFromPyArray(PyArrayObject *array, default: - return absl::InternalError(absl::StrCat("Unsupported numpy type: ", - NumpyTypeName(pyarray_type))); + return tensorflow::errors::Internal("Unsupported numpy type: ", + NumpyTypeName(pyarray_type)); } return tensorflow::OkStatus(); } -inline absl::Status VerifyDtypeIsSupported(const tensorflow::DataType &dtype) { +inline tensorflow::Status VerifyDtypeIsSupported( + const tensorflow::DataType &dtype) { if (!tensorflow::DataTypeCanUseMemcpy(dtype) && dtype != tensorflow::DT_STRING) { - return absl::UnimplementedError(absl::StrCat( + return tensorflow::errors::Unimplemented( "ndarrays that maps to tensors with dtype ", - tensorflow::DataType_Name(dtype), " are not yet supported")); + tensorflow::DataType_Name(dtype), " are not yet supported"); } return tensorflow::OkStatus(); } -absl::Status NdArrayToTensor(PyObject *ndarray, - tensorflow::Tensor *out_tensor) { +tensorflow::Status NdArrayToTensor(PyObject *ndarray, + tensorflow::Tensor *out_tensor) { DCHECK(out_tensor != nullptr); auto array_safe = make_safe(PyArray_FromAny( /*op=*/ndarray, @@ -247,7 +245,7 @@ absl::Status NdArrayToTensor(PyObject *ndarray, /*requirements=*/NPY_ARRAY_CARRAY_RO, /*context=*/nullptr)); if (!array_safe) { - return absl::InvalidArgumentError( + return tensorflow::errors::InvalidArgument( "Provided input could not be interpreted as an ndarray"); } PyArrayObject *py_array = reinterpret_cast(array_safe.get()); @@ -277,15 +275,15 @@ absl::Status NdArrayToTensor(PyObject *ndarray, out_t[i++] = tensorflow::tstring(ptr, len); })); } else { - return absl::UnimplementedError( - absl::StrCat("Unexpected dtype: ", tensorflow::DataTypeString(dtype))); + return tensorflow::errors::Unimplemented("Unexpected dtype: ", + tensorflow::DataTypeString(dtype)); } return tensorflow::OkStatus(); } -absl::Status TensorToNdArray(const tensorflow::Tensor &tensor, - PyObject **out_ndarray) { +tensorflow::Status TensorToNdArray(const tensorflow::Tensor &tensor, + PyObject **out_ndarray) { TF_RETURN_IF_ERROR(VerifyDtypeIsSupported(tensor.dtype())); // Extract the numpy type and dimensions. @@ -301,7 +299,7 @@ absl::Status TensorToNdArray(const tensorflow::Tensor &tensor, auto safe_out_ndarray = make_safe(PyArray_Empty(dims.size(), dims.data(), descr, 0)); if (!safe_out_ndarray) { - return absl::InternalError("Could not allocate ndarray"); + return tensorflow::errors::Internal("Could not allocate ndarray"); } // Populate the ndarray with data from the tensor. @@ -312,9 +310,9 @@ absl::Status TensorToNdArray(const tensorflow::Tensor &tensor, } else if (tensor.dtype() == tensorflow::DT_STRING) { TF_RETURN_IF_ERROR(StringTensorToPyArray(tensor, py_array)); } else { - return absl::UnimplementedError( - absl::StrCat("Unexpected tensor dtype: ", - tensorflow::DataTypeString(tensor.dtype()))); + return tensorflow::errors::Unimplemented( + "Unexpected tensor dtype: ", + tensorflow::DataTypeString(tensor.dtype())); } *out_ndarray = safe_out_ndarray.release(); diff --git a/reverb/cc/conversions.h b/reverb/cc/conversions.h index 8a347735..cfac4758 100644 --- a/reverb/cc/conversions.h +++ b/reverb/cc/conversions.h @@ -16,26 +16,26 @@ #define REVERB_CC_CONVERSIONS_H_ #include "numpy/arrayobject.h" -#include "absl/status/status.h" -#include "tensorflow/core/framework/tensor.h" #include "tensorflow/core/framework/types.h" +#include "tensorflow/core/framework/tensor.h" namespace deepmind { namespace reverb { namespace pybind { // One MUST initialize Numpy, e.g. within the Pybind11 module definition before -// calling C Numpy functions. See -// https://pythonextensionpatterns.readthedocs.io/en/latest/cpp_and_numpy.html +// calling C Numpy functions. +// See https://pythonextensionpatterns.readthedocs.io/en/latest/cpp_and_numpy.html void ImportNumpy(); -absl::Status TensorToNdArray(const tensorflow::Tensor &tensor, - PyObject **out_ndarray); +tensorflow::Status TensorToNdArray(const tensorflow::Tensor &tensor, + PyObject **out_ndarray); -absl::Status NdArrayToTensor(PyObject *ndarray, tensorflow::Tensor *out_tensor); +tensorflow::Status NdArrayToTensor(PyObject *ndarray, + tensorflow::Tensor *out_tensor); -absl::Status GetPyDescrFromDataType(tensorflow::DataType dtype, - PyArray_Descr **out_descr); +tensorflow::Status GetPyDescrFromDataType(tensorflow::DataType dtype, + PyArray_Descr **out_descr); } // namespace pybind } // namespace reverb diff --git a/reverb/cc/ops/BUILD b/reverb/cc/ops/BUILD index 25eb8118..2dc32988 100644 --- a/reverb/cc/ops/BUILD +++ b/reverb/cc/ops/BUILD @@ -1,26 +1,29 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_library", - "reverb_cc_test", - "reverb_gen_op_wrapper_py", - "reverb_kernel_library", - "reverb_tf_deps", - "reverb_tf_ops_visibility", -) - -package(default_visibility = reverb_tf_ops_visibility()) - +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") +package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_kernel_library( - name = "ops", +py_library( + name = "gen_reverb_ops", + srcs = [ + "__init__.py", + "gen_reverb_ops.py", + ], + data = [ + ":libgen_reverb_ops_gen_op.so", + ], + srcs_version = "PY2AND3", + visibility = ["//visibility:public"], +) + +cc_binary( + name = "libgen_reverb_ops_gen_op.so", srcs = [ "client.cc", "pattern_dataset.cc", "timestep_dataset.cc", "trajectory_dataset.cc", ], + visibility = ["//visibility:public"], deps = [ ":queue_writer", "//reverb/cc:chunker", @@ -32,16 +35,19 @@ reverb_kernel_library( "//reverb/cc:structured_writer", "//reverb/cc/platform:logging", "//reverb/cc/support:tf_util", - ] + reverb_absl_deps(), -) - -reverb_gen_op_wrapper_py( - name = "gen_reverb_ops", - out = "gen_reverb_ops.py", - kernel_lib = ":ops", + "@local_config_tf//:tf_header_lib", + "@local_config_tf//:libtensorflow_framework", + ], + linkshared = True, + copts = [ + "-pthread", + "-std=c++17", + "-undefined dynamic_lookup", + "-D_GLIBCXX_USE_CXX11_ABI=1", + ], ) -reverb_cc_library( +cc_library( name = "queue_writer", srcs = ["queue_writer.cc"], hdrs = ["queue_writer.h"], @@ -55,10 +61,10 @@ reverb_cc_library( "//reverb/cc/support:key_generators", "//reverb/cc/support:tf_util", "//reverb/cc/support:trajectory_util", - ] + reverb_tf_deps() + reverb_absl_deps(), + ], ) -reverb_cc_test( +cc_test( name = "queue_writer_test", srcs = ["queue_writer_test.cc"], deps = [ @@ -71,5 +77,5 @@ reverb_cc_test( "//reverb/cc/support:queue", "//reverb/cc/testing:proto_test_util", "//reverb/cc/testing:tensor_testutil", - ] + reverb_tf_deps() + reverb_absl_deps(), + ], ) diff --git a/reverb/cc/ops/__init__.py b/reverb/cc/ops/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/reverb/cc/ops/gen_reverb_ops.py b/reverb/cc/ops/gen_reverb_ops.py new file mode 100644 index 00000000..731f18a0 --- /dev/null +++ b/reverb/cc/ops/gen_reverb_ops.py @@ -0,0 +1,8 @@ +import tensorflow as _tf +_reverb_gen_op = _tf.load_op_library( + _tf.compat.v1.resource_loader.get_path_to_datafile("libgen_reverb_ops_gen_op.so") +) +_locals = locals() +for k in dir(_reverb_gen_op): + _locals[k] = getattr(_reverb_gen_op, k) +del _locals diff --git a/reverb/cc/platform/BUILD b/reverb/cc/platform/BUILD index a3afde10..33e6f1af 100644 --- a/reverb/cc/platform/BUILD +++ b/reverb/cc/platform/BUILD @@ -1,19 +1,25 @@ # Platform-specific code for reverb - -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_library", - "reverb_cc_test", - "reverb_grpc_deps", - "reverb_tf_deps", -) +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_cc_library( +cc_library( + name = "snappy_hdr", + hdrs = ["snappy.h"], +) + +cc_library( + name = "snappy", + hdrs = ["snappy.h"], + visibility = ["//reverb:__subpackages__"], + deps = [ + "//reverb/cc/platform/default:snappy", + ], +) + +cc_library( name = "tfrecord_checkpointer", srcs = ["tfrecord_checkpointer.cc"], hdrs = ["tfrecord_checkpointer.h"], @@ -32,10 +38,11 @@ reverb_cc_library( "//reverb/cc/support:tf_util", "//reverb/cc/support:trajectory_util", "//reverb/cc/table_extensions:interface", - ] + reverb_tf_deps() + reverb_absl_deps(), + "@com_google_absl//absl/status:statusor", + ], ) -reverb_cc_test( +cc_test( name = "tfrecord_checkpointer_test", srcs = ["tfrecord_checkpointer_test.cc"], deps = [ @@ -49,72 +56,72 @@ reverb_cc_test( "//reverb/cc/selectors:uniform", "//reverb/cc/support:tf_util", "//reverb/cc/testing:proto_test_util", - ] + reverb_tf_deps(), + ], ) -reverb_cc_library( +cc_library( name = "checkpointing_hdr", hdrs = ["checkpointing.h"], deps = [ "//reverb/cc/checkpointing:interface", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "checkpointing", hdrs = ["checkpointing.h"], visibility = ["//reverb:__subpackages__"], deps = [ "//reverb/cc/checkpointing:interface", "//reverb/cc/platform/default:checkpointer", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "checkpointing_utils_hdr", hdrs = ["checkpointing_utils.h"], visibility = ["//reverb:__subpackages__"], deps = [ "//reverb/cc/selectors:interface", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "checkpointing_utils", hdrs = ["checkpointing_utils.h"], visibility = ["//reverb:__subpackages__"], deps = [ "//reverb/cc/platform/default:checkpointing_utils", "//reverb/cc/selectors:interface", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "grpc_utils_hdr", hdrs = ["grpc_utils.h"], - deps = reverb_grpc_deps() + reverb_absl_deps(), + # deps = reverb_grpc_deps() ) -reverb_cc_library( +cc_library( name = "grpc_utils", hdrs = ["grpc_utils.h"], visibility = ["//reverb:__subpackages__"], deps = [ "//reverb/cc/platform/default:grpc_utils", - ] + reverb_grpc_deps() + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "server_hdr", hdrs = ["server.h"], deps = [ "//reverb/cc:client", "//reverb/cc:table", "//reverb/cc/checkpointing:interface", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "server", hdrs = ["server.h"], visibility = ["//reverb:__subpackages__"], @@ -123,105 +130,90 @@ reverb_cc_library( "//reverb/cc:table", "//reverb/cc/checkpointing:interface", "//reverb/cc/platform/default:server", - ] + reverb_grpc_deps() + reverb_absl_deps(), + ], ) -reverb_cc_test( +cc_test( name = "server_test", srcs = ["server_test.cc"], deps = [ ":server", "//reverb/cc/platform:net", "//reverb/cc/platform:status_matchers", - ] + reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), -) - -reverb_cc_library( - name = "snappy_hdr", - hdrs = ["snappy.h"], - deps = reverb_absl_deps(), -) - -reverb_cc_library( - name = "snappy", - hdrs = ["snappy.h"], - visibility = ["//reverb:__subpackages__"], - deps = [ - "//reverb/cc/platform/default:snappy", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "status_macros", hdrs = ["status_macros.h"], visibility = ["//reverb:__subpackages__"], deps = ["//reverb/cc/platform/default:status_macros"], ) -reverb_cc_library( +cc_library( name = "status_matchers", hdrs = ["status_matchers.h"], visibility = ["//reverb:__subpackages__"], deps = ["//reverb/cc/platform/default:status_matchers"], ) -reverb_cc_library( +cc_library( name = "thread_hdr", hdrs = ["thread.h"], - deps = reverb_absl_deps(), + # deps = reverb_absl_deps(), ) -reverb_cc_library( +cc_library( name = "thread", hdrs = ["thread.h"], visibility = ["//reverb:__subpackages__"], deps = [ "//reverb/cc/platform/default:thread", - ] + reverb_absl_deps(), + ], ) -reverb_cc_test( +cc_test( name = "thread_test", srcs = ["thread_test.cc"], deps = [ ":thread", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "hash_map", hdrs = ["hash_map.h"], visibility = ["//reverb:__subpackages__"], deps = ["//reverb/cc/platform/default:hash_map"], ) -reverb_cc_library( +cc_library( name = "hash_set", hdrs = ["hash_set.h"], visibility = ["//reverb:__subpackages__"], deps = ["//reverb/cc/platform/default:hash_set"], ) -reverb_cc_library( +cc_library( name = "logging", hdrs = ["logging.h"], visibility = ["//reverb:__subpackages__"], deps = ["//reverb/cc/platform/default:logging"], ) -reverb_cc_library( +cc_library( name = "net_hdr", hdrs = ["net.h"], ) -reverb_cc_library( +cc_library( name = "net", hdrs = ["net.h"], visibility = ["//reverb:__subpackages__"], deps = ["//reverb/cc/platform/default:net"], ) -reverb_cc_test( +cc_test( name = "net_test", srcs = ["net_test.cc"], deps = [ diff --git a/reverb/cc/platform/build_rules.bzl b/reverb/cc/platform/build_rules.bzl deleted file mode 100644 index 4812ed62..00000000 --- a/reverb/cc/platform/build_rules.bzl +++ /dev/null @@ -1,42 +0,0 @@ -"""Main Starlark code for platform-specific build rules.""" - -load( - "//reverb/cc/platform/default:build_rules.bzl", - _reverb_absl_deps = "reverb_absl_deps", - _reverb_cc_grpc_library = "reverb_cc_grpc_library", - _reverb_cc_library = "reverb_cc_library", - _reverb_cc_proto_library = "reverb_cc_proto_library", - _reverb_cc_test = "reverb_cc_test", - _reverb_gen_op_wrapper_py = "reverb_gen_op_wrapper_py", - _reverb_grpc_deps = "reverb_grpc_deps", - _reverb_kernel_library = "reverb_kernel_library", - _reverb_py_proto_library = "reverb_py_proto_library", - _reverb_py_standard_imports = "reverb_py_standard_imports", - _reverb_py_test = "reverb_py_test", - _reverb_pybind_deps = "reverb_pybind_deps", - _reverb_pybind_extension = "reverb_pybind_extension", - _reverb_pytype_library = "reverb_pytype_library", - _reverb_pytype_strict_binary = "reverb_pytype_strict_binary", - _reverb_pytype_strict_library = "reverb_pytype_strict_library", - _reverb_tf_deps = "reverb_tf_deps", - _reverb_tf_ops_visibility = "reverb_tf_ops_visibility", -) - -reverb_absl_deps = _reverb_absl_deps -reverb_cc_library = _reverb_cc_library -reverb_cc_test = _reverb_cc_test -reverb_cc_grpc_library = _reverb_cc_grpc_library -reverb_cc_proto_library = _reverb_cc_proto_library -reverb_gen_op_wrapper_py = _reverb_gen_op_wrapper_py -reverb_grpc_deps = _reverb_grpc_deps -reverb_kernel_library = _reverb_kernel_library -reverb_py_proto_library = _reverb_py_proto_library -reverb_py_standard_imports = _reverb_py_standard_imports -reverb_py_test = _reverb_py_test -reverb_pybind_deps = _reverb_pybind_deps -reverb_pybind_extension = _reverb_pybind_extension -reverb_pytype_library = _reverb_pytype_library -reverb_pytype_strict_library = _reverb_pytype_strict_library -reverb_pytype_strict_binary = _reverb_pytype_strict_binary -reverb_tf_ops_visibility = _reverb_tf_ops_visibility -reverb_tf_deps = _reverb_tf_deps diff --git a/reverb/cc/platform/default/BUILD b/reverb/cc/platform/default/BUILD index ce83614d..5969fcab 100644 --- a/reverb/cc/platform/default/BUILD +++ b/reverb/cc/platform/default/BUILD @@ -1,29 +1,23 @@ # Platform-specific code for reverb - -load( - "//reverb/cc/platform/default:build_rules.bzl", - "reverb_cc_library", - "reverb_grpc_deps", - "reverb_tf_deps", -) - +load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//reverb/cc/platform:__pkg__"]) licenses(["notice"]) exports_files(["LICENSE"]) -reverb_cc_library( +cc_library( name = "snappy", srcs = ["snappy.cc"], deps = [ "//reverb/cc/platform:snappy_hdr", + "@com_google_snappy//:snappy", "@com_google_absl//absl/strings", - ] + reverb_tf_deps(), - alwayslink = 1, + ], + alwayslink = True, ) -reverb_cc_library( +cc_library( name = "checkpointer", srcs = ["default_checkpointer.cc"], deps = [ @@ -32,7 +26,7 @@ reverb_cc_library( "//reverb/cc/platform:tfrecord_checkpointer", "@com_google_absl//absl/strings", ], - alwayslink = 1, + alwayslink = True, ) cc_library( @@ -49,11 +43,12 @@ cc_library( "//reverb/cc/selectors:prioritized", "//reverb/cc/selectors:uniform", "@com_google_absl//absl/strings", + "@com_google_absl//absl/random", ], - alwayslink = 1, + alwayslink = True, ) -reverb_cc_library( +cc_library( name = "server", srcs = ["server.cc"], deps = [ @@ -67,11 +62,11 @@ reverb_cc_library( "//reverb/cc/platform:tfrecord_checkpointer", "//reverb/cc/support:periodic_closure", "@com_google_absl//absl/strings", - ] + reverb_grpc_deps(), - alwayslink = 1, + ], + alwayslink = True, ) -reverb_cc_library( +cc_library( name = "status_macros", hdrs = [ "status_macros.h", @@ -83,7 +78,7 @@ reverb_cc_library( ], ) -reverb_cc_library( +cc_library( name = "status_matchers", hdrs = ["status_matchers.h"], deps = [ @@ -91,7 +86,7 @@ reverb_cc_library( ], ) -reverb_cc_library( +cc_library( name = "thread", srcs = ["thread.cc"], deps = [ @@ -99,23 +94,23 @@ reverb_cc_library( "@com_google_absl//absl/memory", "@com_google_absl//absl/strings", ], - alwayslink = 1, + alwayslink = True, ) -reverb_cc_library( +cc_library( name = "logging", srcs = ["logging.cc"], hdrs = ["logging.h"], - deps = reverb_tf_deps(), + # deps = reverb_tf_deps(), ) -reverb_cc_library( +cc_library( name = "hash", hdrs = ["hash.h"], - deps = reverb_tf_deps(), + # deps = reverb_tf_deps(), ) -reverb_cc_library( +cc_library( name = "hash_map", hdrs = ["hash_map.h"], deps = [ @@ -124,7 +119,7 @@ reverb_cc_library( ], ) -reverb_cc_library( +cc_library( name = "hash_set", hdrs = ["hash_set.h"], deps = [ @@ -133,21 +128,25 @@ reverb_cc_library( ], ) -reverb_cc_library( +cc_library( name = "grpc_utils", srcs = ["grpc_utils.cc"], + defines = ["BAZEL_BUILD"], deps = [ "//reverb/cc/platform:grpc_utils_hdr", - ] + reverb_grpc_deps(), - alwayslink = 1, + "@com_google_absl//absl/strings", + "@local_config_tf//:tf_header_lib", + "@com_github_grpc_grpc//:grpc++", + ], + alwayslink = True, ) -reverb_cc_library( +cc_library( name = "net", srcs = ["net.cc"], deps = [ "//reverb/cc/platform:logging", "//reverb/cc/platform:net_hdr", ], - alwayslink = 1, + alwayslink = True, ) diff --git a/reverb/cc/platform/default/build_rules.bzl b/reverb/cc/platform/default/build_rules.bzl deleted file mode 100644 index 3fe36c8f..00000000 --- a/reverb/cc/platform/default/build_rules.bzl +++ /dev/null @@ -1,586 +0,0 @@ -"""Default versions of reverb build rule helpers.""" - -def tf_copts(): - return ["-Wno-sign-compare"] - -def reverb_cc_library( - name, - srcs = [], - hdrs = [], - deps = [], - testonly = 0, - **kwargs): - if testonly: - new_deps = [ - "@com_google_googletest//:gtest", - "@tensorflow_includes//:includes", - "@tensorflow_solib//:framework_lib", - ] - else: - new_deps = [] - native.cc_library( - name = name, - srcs = srcs, - hdrs = hdrs, - copts = tf_copts(), - testonly = testonly, - deps = depset(deps + new_deps), - **kwargs - ) - -def reverb_kernel_library(name, srcs = [], deps = [], **kwargs): - deps = deps + reverb_tf_deps() - reverb_cc_library( - name = name, - srcs = srcs, - deps = deps, - alwayslink = 1, - **kwargs - ) - -def _removesuffix(x, txt): - """Backport of x._removesuffix(txt) for Python version earlier than 3.9.""" - if x.endswith(txt): - return x[:-len(txt)] - return x - -def _normalize_proto(x): - if x.endswith("_proto"): - x = _removesuffix(x, "_proto") - if x.endswith("_cc"): - x = _removesuffix(x, "_cc") - if x.endswith("_pb2"): - x = _removesuffix(x, "_pb2") - return x - -def _filegroup_name(x): - return _normalize_proto(x) + "_filegroup" - -def reverb_cc_proto_library(name, srcs = [], deps = [], **kwargs): - """Build a proto cc_library. - - This rule does three things: - - 1) Create a filegroup with name `_filegroup` that contains `srcs` - and any sources from deps named "x_proto" or "x_cc_proto". - - 2) Uses protoc to compile srcs to .h/.cc files, allowing any - tensorflow imports. - - 3) Creates a cc_library with name `name` building the resulting .h/.cc - files. - - Args: - name: The name, should end with "_cc_proto". - srcs: The .proto files. - deps: Any reverb_cc_proto_library targets. - **kwargs: Any additional args for the cc_library rule. - """ - gen_srcs = [_removesuffix(x, ".proto") + ".pb.cc" for x in srcs] - gen_hdrs = [_removesuffix(x, ".proto") + ".pb.h" for x in srcs] - src_paths = ["$(location {})".format(x) for x in srcs] - dep_srcs = [] - for x in deps: - if x.endswith("_proto"): - dep_srcs.append(_filegroup_name(x)) - native.filegroup( - name = _filegroup_name(name), - srcs = srcs + dep_srcs, - **kwargs - ) - native.genrule( - name = name + "_gen", - srcs = srcs, - outs = gen_srcs + gen_hdrs, - tools = dep_srcs + [ - "@protobuf_protoc//:protoc_bin", - "@tensorflow_includes//:protos", - ], - cmd = """ - OUTDIR=$$(echo $(RULEDIR) | sed -E -e 's#reverb(/.*|$$)##') - $(location @protobuf_protoc//:protoc_bin) \ - --proto_path=external/tensorflow_includes/tensorflow_includes/ \ - --proto_path=. \ - --cpp_out=$$OUTDIR {}""".format( - " ".join(src_paths), - ), - ) - - native.cc_library( - name = "{}_static".format(name), - srcs = gen_srcs, - hdrs = gen_hdrs, - deps = depset(deps + reverb_tf_deps()), - alwayslink = 1, - **kwargs - ) - native.cc_binary( - name = "lib{}.so".format(name), - deps = ["{}_static".format(name)], - linkshared = 1, - **kwargs - ) - native.cc_library( - name = name, - hdrs = gen_hdrs, - srcs = ["lib{}.so".format(name)], - deps = depset(deps + reverb_tf_deps()), - alwayslink = 1, - **kwargs - ) - -def reverb_py_proto_library(name, srcs = [], deps = [], **kwargs): - """Build a proto py_library. - - This rule does three things: - - 1) Create a filegroup with name `_filegroup` that contains `srcs` - and any sources from deps named "x_proto" or "x_py_proto". - - 2) Uses protoc to compile srcs to _pb2.py files, allowing any - tensorflow imports. - - 3) Creates a py_library with name `name` building the resulting .py - files. - - Args: - name: The name, should end with "_py_pb2". - srcs: The .proto files. - deps: Any reverb_cc_proto_library targets. - **kwargs: Any additional args for the cc_library rule. - """ - gen_srcs = [_removesuffix(x, ".proto") + "_pb2.py" for x in srcs] - src_paths = ["$(location {})".format(x) for x in srcs] - proto_deps = [] - py_deps = [] - for x in deps: - if x.endswith("_proto"): - proto_deps.append(_filegroup_name(x)) - else: - py_deps.append(x) - native.filegroup( - name = _filegroup_name(name), - srcs = srcs + proto_deps, - **kwargs - ) - native.genrule( - name = name + "_gen", - srcs = srcs, - outs = gen_srcs, - tools = proto_deps + [ - "@protobuf_protoc//:protoc_bin", - "@tensorflow_includes//:protos", - ], - cmd = """ - OUTDIR=$$(echo $(RULEDIR) | sed -E -e 's#reverb(/.*|$$)##') - $(location @protobuf_protoc//:protoc_bin) \ - --proto_path=external/tensorflow_includes/tensorflow_includes/ \ - --proto_path=. \ - --python_out=$$OUTDIR {}""".format( - " ".join(src_paths), - ), - ) - native.py_library( - name = name, - srcs = gen_srcs, - deps = py_deps, - data = proto_deps, - **kwargs - ) - -def reverb_cc_grpc_library( - name, - srcs = [], - deps = [], - generate_mocks = False, - **kwargs): - """Build a grpc cc_library. - - This rule does two things: - - 1) Uses protoc + grpc plugin to compile srcs to .h/.cc files, allowing any - tensorflow imports. Also creates mock headers if requested. - - 2) Creates a cc_library with name `name` building the resulting .h/.cc - files. - - Args: - name: The name, should end with "_cc_grpc_proto". - srcs: The .proto files. - deps: reverb_cc_proto_library targets. Must include src + "_cc_proto", - the cc_proto library, for each src in srcs. - generate_mocks: If true, creates mock headers for each source. - **kwargs: Any additional args for the cc_library rule. - """ - gen_srcs = [_removesuffix(x, ".proto") + ".grpc.pb.cc" for x in srcs] - gen_hdrs = [_removesuffix(x, ".proto") + ".grpc.pb.h" for x in srcs] - proto_src_deps = [] - for x in deps: - if x.endswith("_proto"): - proto_src_deps.append(_filegroup_name(x)) - src_paths = ["$(location {})".format(x) for x in srcs] - - if generate_mocks: - gen_mocks = [_removesuffix(x, ".proto") + "_mock.grpc.pb.h" for x in srcs] - else: - gen_mocks = [] - - native.genrule( - name = name + "_gen", - srcs = srcs, - outs = gen_srcs + gen_hdrs + gen_mocks, - tools = proto_src_deps + [ - "@protobuf_protoc//:protoc_bin", - "@tensorflow_includes//:protos", - "@com_github_grpc_grpc//src/compiler:grpc_cpp_plugin", - ], - cmd = """ - OUTDIR=$$(echo $(RULEDIR) | sed -e 's#reverb/.*##') - $(location @protobuf_protoc//:protoc_bin) \ - --plugin=protoc-gen-grpc=$(location @com_github_grpc_grpc//src/compiler:grpc_cpp_plugin) \ - --proto_path=external/tensorflow_includes/tensorflow_includes/ \ - --proto_path=. \ - --grpc_out={} {}""".format( - "generate_mock_code=true:$$OUTDIR" if generate_mocks else "$$OUTDIR", - " ".join(src_paths), - ), - ) - - native.cc_library( - name = name, - srcs = gen_srcs, - hdrs = gen_hdrs + gen_mocks, - deps = depset(deps + ["@com_github_grpc_grpc//:grpc++_codegen_proto"]), - **kwargs - ) - -def reverb_cc_test(name, srcs, deps = [], **kwargs): - """Reverb-specific version of cc_test. - - Args: - name: Target name. - srcs: Target sources. - deps: Target deps. - **kwargs: Additional args to cc_test. - """ - new_deps = [ - "@com_github_grpc_grpc//:grpc++_test", - "@com_google_googletest//:gtest", - "@tensorflow_includes//:includes", - "@tensorflow_solib//:framework_lib", - "@com_google_googletest//:gtest_main", - ] - size = kwargs.pop("size", "small") - native.cc_test( - name = name, - size = size, - copts = tf_copts(), - srcs = srcs, - deps = depset(deps + new_deps), - **kwargs - ) - -def reverb_gen_op_wrapper_py(name, out, kernel_lib, linkopts = [], **kwargs): - """Generates the py_library `name` with a data dep on the ops in kernel_lib. - - The resulting py_library creates file `$out`, and has a dependency on a - symbolic library called lib{$name}_gen_op.so, which contains the kernels - and ops and can be loaded via `tf.load_op_library`. - - Args: - name: The name of the py_library. - out: The name of the python file. Use "gen_{name}_ops.py". - kernel_lib: A cc_kernel_library target to generate for. - **kwargs: Any args to the `cc_binary` and `py_library` internal rules. - """ - if not out.endswith(".py"): - fail("Argument out must end with '.py', but saw: {}".format(out)) - - module_name = "lib{}_gen_op".format(name) - version_script_file = "%s-version-script.lds" % module_name - native.genrule( - name = module_name + "_version_script", - outs = [version_script_file], - cmd = "echo '{global:\n *tensorflow*;\n *deepmind*;\n local: *;};' >$@", - output_licenses = ["unencumbered"], - visibility = ["//visibility:private"], - ) - native.cc_binary( - name = "{}.so".format(module_name), - deps = [kernel_lib] + reverb_tf_deps() + [version_script_file], - copts = tf_copts() + [ - "-fno-strict-aliasing", # allow a wider range of code [aliasing] to compile. - "-fvisibility=hidden", # avoid symbol clashes between DSOs. - ], - linkshared = 1, - linkopts = linkopts + _rpath_linkopts(module_name) + [ - "-Wl,--version-script", - "$(location %s)" % version_script_file, - ], - **kwargs - ) - native.genrule( - name = "{}_genrule".format(out), - outs = [out], - cmd = """echo 'import tensorflow as _tf -from reverb.platform.default import load_op_library as _load_op_library - -try: - _reverb_gen_op = _tf.load_op_library( - _tf.compat.v1.resource_loader.get_path_to_datafile("lib{}_gen_op.so")) -except _tf.errors.NotFoundError as e: - _load_op_library.reraise_wrapped_error(e) -_locals = locals() -for k in dir(_reverb_gen_op): - _locals[k] = getattr(_reverb_gen_op, k) -del _locals' > $@""".format(name), - ) - deps = kwargs.pop("deps", []) - deps.append("//reverb/platform/default:load_op_library") - native.py_library( - name = name, - srcs = [out], - data = [":lib{}_gen_op.so".format(name)], - deps = deps, - **kwargs - ) - -def reverb_pytype_library(**kwargs): - if "strict_deps" in kwargs: - kwargs.pop("strict_deps") - native.py_library(**kwargs) - -reverb_pytype_strict_library = native.py_library - -reverb_pytype_strict_binary = native.py_binary - -def _make_search_paths(prefix, levels_to_root): - return ",".join( - [ - "-rpath,%s/%s" % (prefix, "/".join([".."] * search_level)) - for search_level in range(levels_to_root + 1) - ], - ) - -def _rpath_linkopts(name): - # Search parent directories up to the TensorFlow root directory for shared - # object dependencies, even if this op shared object is deeply nested - # (e.g. tensorflow/contrib/package:python/ops/_op_lib.so). tensorflow/ is then - # the root and tensorflow/libtensorflow_framework.so should exist when - # deployed. Other shared object dependencies (e.g. shared between contrib/ - # ops) are picked up as long as they are in either the same or a parent - # directory in the tensorflow/ tree. - levels_to_root = native.package_name().count("/") + name.count("/") - return ["-Wl,%s" % (_make_search_paths("$$ORIGIN", levels_to_root),)] - -def reverb_pybind_extension( - name, - srcs, - module_name, - hdrs = [], - features = [], - srcs_version = "PY3", - data = [], - copts = [], - linkopts = [], - deps = [], - defines = [], - visibility = None, - testonly = None, - licenses = None, - compatible_with = None, - restricted_to = None, - deprecation = None, - pytype_srcs = None): - """Builds a generic Python extension module. - - The module can be loaded in python by performing "import ${name}.". - - Args: - name: Name. - srcs: cc files. - module_name: The name of the hidden module. It should be different - from `name`, and *must* match the MODULE declaration in the .cc file. - hdrs: h files. - features: see bazel docs. - srcs_version: srcs_version for py_library. - data: data deps. - copts: compilation opts. - linkopts: linking opts. - deps: cc_library deps. - defines: cc_library defines. - visibility: visibility. - testonly: whether the rule is testonly. - licenses: see bazel docs. - compatible_with: see bazel docs. - restricted_to: see bazel docs. - deprecation: see bazel docs. - pytype_srcs: Unused list of pytype stub files. - """ - if name == module_name: - fail( - "Must have name != module_name ({} vs. {}) because the python ".format(name, module_name) + - "wrapper $name.py needs to add extra logic loading tensorflow.", - ) - py_file = "%s.py" % name - so_file = "%s.so" % module_name - pyd_file = "%s.pyd" % module_name - symbol = "init%s" % module_name - symbol2 = "init_%s" % module_name - symbol3 = "PyInit_%s" % module_name - exported_symbols_file = "%s-exported-symbols.lds" % module_name - version_script_file = "%s-version-script.lds" % module_name - native.genrule( - name = module_name + "_exported_symbols", - outs = [exported_symbols_file], - cmd = "echo '_%s\n_%s\n_%s' >$@" % (symbol, symbol2, symbol3), - output_licenses = ["unencumbered"], - visibility = ["//visibility:private"], - testonly = testonly, - ) - native.genrule( - name = module_name + "_version_script", - outs = [version_script_file], - cmd = "echo '{global:\n %s;\n %s;\n %s;\n local: *;};' >$@" % (symbol, symbol2, symbol3), - output_licenses = ["unencumbered"], - visibility = ["//visibility:private"], - testonly = testonly, - ) - native.cc_binary( - name = so_file, - srcs = srcs + hdrs, - data = data, - copts = copts + [ - "-fno-strict-aliasing", # allow a wider range of code [aliasing] to compile. - "-fexceptions", # pybind relies on exceptions, required to compile. - "-fvisibility=hidden", # avoid pybind symbol clashes between DSOs. - ], - linkopts = linkopts + _rpath_linkopts(module_name) + [ - "-Wl,--version-script", - "$(location %s)" % version_script_file, - ], - deps = depset(deps + [ - exported_symbols_file, - version_script_file, - ]), - defines = defines, - features = features + ["-use_header_modules"], - linkshared = 1, - testonly = testonly, - licenses = licenses, - visibility = visibility, - deprecation = deprecation, - restricted_to = restricted_to, - compatible_with = compatible_with, - ) - native.genrule( - name = module_name + "_pyd_copy", - srcs = [so_file], - outs = [pyd_file], - cmd = "cp $< $@", - output_to_bindir = True, - visibility = visibility, - deprecation = deprecation, - restricted_to = restricted_to, - compatible_with = compatible_with, - ) - native.genrule( - name = name + "_py_file", - outs = [py_file], - cmd = """echo 'import tensorflow as _tf -from reverb.platform.default import load_op_library as _load_op_library -try: - from .%s import * -except ImportError as e: - _load_op_library.reraise_wrapped_error(e) -del _tf' >$@""" % module_name, - output_licenses = ["unencumbered"], - visibility = visibility, - testonly = testonly, - ) - native.py_library( - name = name, - data = [so_file], - deps = ["//reverb/platform/default:load_op_library"], - srcs = [py_file], - srcs_version = srcs_version, - licenses = licenses, - testonly = testonly, - visibility = visibility, - deprecation = deprecation, - restricted_to = restricted_to, - compatible_with = compatible_with, - ) - -def reverb_py_standard_imports(): - return [] - -def reverb_py_test( - name, - srcs = [], - deps = [], - paropts = [], - python_version = "PY3", - **kwargs): - size = kwargs.pop("size", "small") - if "enable_dashboard" in kwargs: - kwargs.pop("enable_dashboard") - native.py_test( - name = name, - size = size, - srcs = srcs, - deps = deps, - python_version = python_version, - **kwargs - ) - return - -def reverb_pybind_deps(): - return [ - "@pybind11", - ] + reverb_tf_deps() - -def reverb_tf_ops_visibility(): - return [ - "//reverb:__subpackages__", - ] - -def reverb_tf_deps(): - # tensorflow_includes doesn't include absl headers, so we bring in - # ours. Note that our absl deps' version must match that of TF - # and we need to include all the absl headers that tensorflow - # includes rely on. - return [ - "@tensorflow_includes//:includes", - "@tensorflow_solib//:framework_lib", - ] + reverb_absl_deps() - -def reverb_grpc_deps(): - return ["@com_github_grpc_grpc//:grpc++"] - -def reverb_absl_deps(): - return [ - # We purposefully don't include absl::flat_hash_{map,set} so that users - # are forced to use platform:hash_{map,set}, which uses a safer hasher. - "@com_google_absl//absl/base", - "@com_google_absl//absl/base:core_headers", - "@com_google_absl//absl/functional:bind_front", - "@com_google_absl//absl/memory", - "@com_google_absl//absl/numeric:int128", - "@com_google_absl//absl/random", - "@com_google_absl//absl/random:distributions", - "@com_google_absl//absl/status", - "@com_google_absl//absl/status:statusor", - "@com_google_absl//absl/strings", - "@com_google_absl//absl/strings:cord", - "@com_google_absl//absl/strings:str_format", - "@com_google_absl//absl/synchronization", - "@com_google_absl//absl/time", - "@com_google_absl//absl/types:optional", - "@com_google_absl//absl/types:span", - "@com_google_absl//absl/flags:flag", - # This is not used within Reverb directly but tensorflow/tsl/platform/env.h introduced this - # dependency on 2023-04-25 in - # https://github.com/tensorflow/tensorflow/commit/134a9f87c22ffb111080f9d6626f202ad0cce2a4. - "@com_google_absl//absl/functional:any_invocable", - ] diff --git a/reverb/cc/platform/default/repo.bzl b/reverb/cc/platform/default/repo.bzl deleted file mode 100644 index e5a53d94..00000000 --- a/reverb/cc/platform/default/repo.bzl +++ /dev/null @@ -1,403 +0,0 @@ -"""Custom external dependencies.""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -# Sanitize a dependency so that it works correctly from code that includes -# codebase as a submodule. -def clean_dep(dep): - return str(Label(dep)) - -def get_python_path(ctx): - path = ctx.os.environ.get("PYTHON_BIN_PATH") - if not path: - fail( - "Could not get environment variable PYTHON_BIN_PATH. " + - "Check your .bazelrc file.", - ) - return path - -def _find_tf_include_path(repo_ctx): - exec_result = repo_ctx.execute( - [ - get_python_path(repo_ctx), - "-c", - "import tensorflow as tf; import sys; " + - "sys.stdout.write(tf.sysconfig.get_include())", - ], - quiet = True, - ) - if exec_result.return_code != 0: - fail("Could not locate tensorflow installation path:\n{}" - .format(exec_result.stderr)) - return exec_result.stdout.splitlines()[-1] - -def _find_tf_lib_path(repo_ctx): - exec_result = repo_ctx.execute( - [ - get_python_path(repo_ctx), - "-c", - "import tensorflow as tf; import sys; " + - "sys.stdout.write(tf.sysconfig.get_lib())", - ], - quiet = True, - ) - if exec_result.return_code != 0: - fail("Could not locate tensorflow installation path:\n{}" - .format(exec_result.stderr)) - return exec_result.stdout.splitlines()[-1] - -def _find_numpy_include_path(repo_ctx): - exec_result = repo_ctx.execute( - [ - get_python_path(repo_ctx), - "-c", - "import numpy; import sys; " + - "sys.stdout.write(numpy.get_include())", - ], - quiet = True, - ) - if exec_result.return_code != 0: - fail("Could not locate numpy includes path:\n{}" - .format(exec_result.stderr)) - return exec_result.stdout.splitlines()[-1] - -def _find_python_include_path(repo_ctx): - exec_result = repo_ctx.execute( - [ - get_python_path(repo_ctx), - "-c", - "from distutils import sysconfig; import sys; " + - "sys.stdout.write(sysconfig.get_python_inc())", - ], - quiet = True, - ) - if exec_result.return_code != 0: - fail("Could not locate python includes path:\n{}" - .format(exec_result.stderr)) - return exec_result.stdout.splitlines()[-1] - -def _find_python_solib_path(repo_ctx): - exec_result = repo_ctx.execute( - [ - get_python_path(repo_ctx), - "-c", - "import sys; vi = sys.version_info; " + - "sys.stdout.write('python{}.{}'.format(vi.major, vi.minor))", - ], - ) - if exec_result.return_code != 0: - fail("Could not locate python shared library path:\n{}" - .format(exec_result.stderr)) - version = exec_result.stdout.splitlines()[-1] - basename = "lib{}.so".format(version) - exec_result = repo_ctx.execute( - ["{}-config".format(version), "--configdir"], - quiet = True, - ) - if exec_result.return_code != 0: - fail("Could not locate python shared library path:\n{}" - .format(exec_result.stderr)) - solib_dir = exec_result.stdout.splitlines()[-1] - full_path = repo_ctx.path("{}/{}".format(solib_dir, basename)) - if not full_path.exists: - fail("Unable to find python shared library file:\n{}/{}" - .format(solib_dir, basename)) - return struct(dir = solib_dir, basename = basename) - -def _eigen_archive_repo_impl(repo_ctx): - tf_include_path = _find_tf_include_path(repo_ctx) - repo_ctx.symlink(tf_include_path, "tf_includes") - repo_ctx.file( - "BUILD", - content = """ -cc_library( - name = "includes", - hdrs = glob(["tf_includes/Eigen/**/*.h", - "tf_includes/Eigen/**", - "tf_includes/unsupported/Eigen/**/*.h", - "tf_includes/unsupported/Eigen/**"]), - # https://groups.google.com/forum/#!topic/bazel-discuss/HyyuuqTxKok - includes = ["tf_includes"], - visibility = ["//visibility:public"], -) -""", - executable = False, - ) - -def _nsync_includes_repo_impl(repo_ctx): - tf_include_path = _find_tf_include_path(repo_ctx) - repo_ctx.symlink(tf_include_path + "/external", "nsync_includes") - repo_ctx.file( - "BUILD", - content = """ -cc_library( - name = "includes", - hdrs = glob(["nsync_includes/nsync/public/*.h"]), - includes = ["nsync_includes"], - visibility = ["//visibility:public"], -) -""", - executable = False, - ) - -def _zlib_includes_repo_impl(repo_ctx): - tf_include_path = _find_tf_include_path(repo_ctx) - repo_ctx.symlink( - tf_include_path + "/external/zlib", - "zlib", - ) - repo_ctx.file( - "BUILD", - content = """ -cc_library( - name = "includes", - hdrs = glob(["zlib/**/*.h"]), - includes = ["zlib"], - visibility = ["//visibility:public"], -) -""", - executable = False, - ) - -def _snappy_includes_repo_impl(repo_ctx): - tf_include_path = _find_tf_include_path(repo_ctx) - repo_ctx.symlink( - tf_include_path + "/external/snappy", - "snappy", - ) - repo_ctx.file( - "BUILD", - content = """ -cc_library( - name = "includes", - hdrs = glob(["snappy/*.h"]), - includes = ["snappy"], - visibility = ["//visibility:public"], -) -""", - executable = False, - ) - -def _protobuf_includes_repo_impl(repo_ctx): - tf_include_path = _find_tf_include_path(repo_ctx) - repo_ctx.symlink(tf_include_path, "tf_includes") - repo_ctx.symlink(Label("//third_party:protobuf.BUILD"), "BUILD") - -def _tensorflow_includes_repo_impl(repo_ctx): - tf_include_path = _find_tf_include_path(repo_ctx) - repo_ctx.symlink(tf_include_path, "tensorflow_includes") - repo_ctx.file( - "BUILD", - content = """ -cc_library( - name = "includes", - hdrs = glob( - [ - "tensorflow_includes/**/*.h", - "tensorflow_includes/third_party/eigen3/**", - ], - exclude = ["tensorflow_includes/absl/**/*.h"], - ), - includes = ["tensorflow_includes"], - deps = [ - "@com_google_absl//absl/container:flat_hash_map", - "@eigen_archive//:includes", - "@protobuf_archive//:includes", - "@zlib_includes//:includes", - "@snappy_includes//:includes", - ], - visibility = ["//visibility:public"], -) -filegroup( - name = "protos", - srcs = glob(["tensorflow_includes/**/*.proto"]), - visibility = ["//visibility:public"], -) -""", - executable = False, - ) - -def _tensorflow_solib_repo_impl(repo_ctx): - tf_lib_path = _find_tf_lib_path(repo_ctx) - repo_ctx.symlink(tf_lib_path, "tensorflow_solib") - repo_ctx.file( - "BUILD", - content = """ -cc_library( - name = "framework_lib", - srcs = ["tensorflow_solib/libtensorflow_framework.so.2"], - deps = ["@python_includes", "@python_includes//:numpy_includes"], - visibility = ["//visibility:public"], -) -""", - ) - -def _python_includes_repo_impl(repo_ctx): - python_include_path = _find_python_include_path(repo_ctx) - python_solib = _find_python_solib_path(repo_ctx) - repo_ctx.symlink(python_include_path, "python_includes") - numpy_include_path = _find_numpy_include_path(repo_ctx) - repo_ctx.symlink(numpy_include_path, "numpy_includes") - repo_ctx.symlink( - "{}/{}".format(python_solib.dir, python_solib.basename), - python_solib.basename, - ) - - repo_ctx.file( - "BUILD", - content = """ -cc_library( - name = "python_includes", - hdrs = glob(["python_includes/**/*.h"]), - includes = ["python_includes"], - visibility = ["//visibility:public"], -) -cc_library( - name = "numpy_includes", - hdrs = glob(["numpy_includes/**/*.h"]), - includes = ["numpy_includes"], - visibility = ["//visibility:public"], -) -""".format(python_solib.basename), - executable = False, - ) - -def cc_tf_configure(): - """Autoconf pre-installed tensorflow repo.""" - make_eigen_repo = repository_rule(implementation = _eigen_archive_repo_impl) - make_eigen_repo(name = "eigen_archive") - make_nsync_repo = repository_rule( - implementation = _nsync_includes_repo_impl, - ) - make_nsync_repo(name = "nsync_includes") - make_zlib_repo = repository_rule( - implementation = _zlib_includes_repo_impl, - ) - make_zlib_repo(name = "zlib_includes") - make_snappy_repo = repository_rule( - implementation = _snappy_includes_repo_impl, - ) - make_snappy_repo(name = "snappy_includes") - make_protobuf_repo = repository_rule( - implementation = _protobuf_includes_repo_impl, - ) - make_protobuf_repo(name = "protobuf_archive") - make_tfinc_repo = repository_rule( - implementation = _tensorflow_includes_repo_impl, - ) - make_tfinc_repo(name = "tensorflow_includes") - make_tflib_repo = repository_rule( - implementation = _tensorflow_solib_repo_impl, - ) - make_tflib_repo(name = "tensorflow_solib") - make_python_inc_repo = repository_rule( - implementation = _python_includes_repo_impl, - ) - make_python_inc_repo(name = "python_includes") - -def python_deps(): - http_archive( - name = "pybind11", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/pybind/pybind11/archive/v2.10.0.tar.gz", - "https://github.com/pybind/pybind11/archive/v2.10.0.tar.gz", - ], - sha256 = "eacf582fa8f696227988d08cfc46121770823839fe9e301a20fbce67e7cd70ec", - strip_prefix = "pybind11-2.10.0", - build_file = clean_dep("//third_party:pybind11.BUILD"), - ) - - http_archive( - name = "absl_py", - sha256 = "a7c51b2a0aa6357a9cbb2d9437e8cd787200531867dc02565218930b6a32166e", - strip_prefix = "abseil-py-pypi-v1.0.0", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-py/archive/refs/tags/v1.0.0.tar.gz", - "https://github.com/abseil/abseil-py/archive/refs/tags/v1.0.0.tar.gz", - ], - ) - -def github_apple_deps(): - http_archive( - name = "build_bazel_rules_apple", - sha256 = "36072d4f3614d309d6a703da0dfe48684ec4c65a89611aeb9590b45af7a3e592", - urls = ["https://github.com/bazelbuild/rules_apple/releases/download/1.0.1/rules_apple.1.0.1.tar.gz"], - ) - http_archive( - name = "build_bazel_apple_support", - sha256 = "ce1042cf936540eaa7b49c4549d7cd9b6b1492acbb6e765840a67a34b8e17a97", - urls = ["https://github.com/bazelbuild/apple_support/releases/download/1.1.0/apple_support.1.1.0.tar.gz"], - ) - -def github_grpc_deps(): - http_archive( - name = "com_github_grpc_grpc", - patch_cmds = [ - """sed -i.bak 's/"python",/"python3",/g' third_party/py/python_configure.bzl""", - """sed -i.bak 's/PYTHONHASHSEED=0/PYTHONHASHSEED=0 python3/g' bazel/cython_library.bzl""", - ], - sha256 = "39bad059a712c6415b168cb3d922cb0e8c16701b475f047426c81b46577d844b", - strip_prefix = "grpc-reverb_fix", - urls = [ - # Patched version of GRPC / boringSSL to make it compile with old TF GCC compiler - # (see b/244280763 for details). - "https://github.com/qstanczyk/grpc/archive/reverb_fix.tar.gz", - ], - ) - -def googletest_deps(): - http_archive( - name = "com_google_googletest", - sha256 = "ff7a82736e158c077e76188232eac77913a15dac0b22508c390ab3f88e6d6d86", - strip_prefix = "googletest-b6cd405286ed8635ece71c72f118e659f4ade3fb", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip", - "https://github.com/google/googletest/archive/b6cd405286ed8635ece71c72f118e659f4ade3fb.zip", - ], - ) - -def absl_deps(): - http_archive( - name = "com_google_absl", - sha256 = "8eeec9382fc0338ef5c60053f3a4b0e0708361375fe51c9e65d0ce46ccfe55a7", # SHARED_ABSL_SHA - strip_prefix = "abseil-cpp-b971ac5250ea8de900eae9f95e06548d14cd95fe", - urls = [ - "https://storage.googleapis.com/mirror.tensorflow.org/github.com/abseil/abseil-cpp/archive/b971ac5250ea8de900eae9f95e06548d14cd95fe.tar.gz", - "https://github.com/abseil/abseil-cpp/archive/b971ac5250ea8de900eae9f95e06548d14cd95fe.tar.gz", - ], - ) - -def _protoc_archive(ctx): - version = ctx.attr.version - sha256 = ctx.attr.sha256 - - urls = [ - "https://github.com/protocolbuffers/protobuf/releases/download/v%s/protoc-%s-linux-x86_64.zip" % (version, version), - ] - ctx.download_and_extract( - url = urls, - sha256 = sha256, - ) - - ctx.file( - "BUILD", - content = """ -filegroup( - name = "protoc_bin", - srcs = ["bin/protoc"], - visibility = ["//visibility:public"], -) -""", - executable = False, - ) - -protoc_archive = repository_rule( - implementation = _protoc_archive, - attrs = { - "version": attr.string(mandatory = True), - "sha256": attr.string(mandatory = True), - }, -) - -def protoc_deps(version, sha256): - protoc_archive(name = "protobuf_protoc", version = version, sha256 = sha256) diff --git a/reverb/cc/selectors/BUILD b/reverb/cc/selectors/BUILD index 68fa5099..5cee76f1 100644 --- a/reverb/cc/selectors/BUILD +++ b/reverb/cc/selectors/BUILD @@ -1,24 +1,17 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_library", - "reverb_cc_test", - "reverb_tf_deps", -) - +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_cc_library( +cc_library( name = "interface", hdrs = ["interface.h"], deps = [ "//reverb/cc/checkpointing:checkpoint_cc_proto", - ] + reverb_tf_deps() + reverb_absl_deps(), + ] ) -reverb_cc_library( +cc_library( name = "uniform", srcs = ["uniform.cc"], hdrs = ["uniform.h"], @@ -28,10 +21,13 @@ reverb_cc_library( "//reverb/cc/checkpointing:checkpoint_cc_proto", "//reverb/cc/platform:hash_map", "//reverb/cc/platform:logging", - ] + reverb_tf_deps() + reverb_absl_deps(), + "@com_google_absl//absl/random", + "@com_google_absl//absl/status", + "@local_config_tf//:tf_header_lib" + ] ) -reverb_cc_library( +cc_library( name = "fifo", srcs = ["fifo.cc"], hdrs = ["fifo.h"], @@ -41,10 +37,12 @@ reverb_cc_library( "//reverb/cc/checkpointing:checkpoint_cc_proto", "//reverb/cc/platform:hash_map", "//reverb/cc/platform:logging", - ] + reverb_tf_deps() + reverb_absl_deps(), + "@local_config_tf//:tf_header_lib", + "@com_google_absl//absl/status", + ] ) -reverb_cc_library( +cc_library( name = "lifo", srcs = ["lifo.cc"], hdrs = ["lifo.h"], @@ -54,10 +52,12 @@ reverb_cc_library( "//reverb/cc/checkpointing:checkpoint_cc_proto", "//reverb/cc/platform:hash_map", "//reverb/cc/platform:logging", - ] + reverb_tf_deps() + reverb_absl_deps(), + "@local_config_tf//:tf_header_lib", + "@com_google_absl//absl/status", + ] ) -reverb_cc_library( +cc_library( name = "prioritized", srcs = ["prioritized.cc"], hdrs = ["prioritized.h"], @@ -68,10 +68,12 @@ reverb_cc_library( "//reverb/cc/platform:hash_map", "//reverb/cc/platform:logging", "//reverb/cc/platform:status_macros", - ] + reverb_tf_deps() + reverb_absl_deps(), + "@local_config_tf//:tf_header_lib", + "@com_google_absl//absl/random", + ] ) -reverb_cc_library( +cc_library( name = "heap", srcs = ["heap.cc"], hdrs = ["heap.h"], @@ -81,10 +83,12 @@ reverb_cc_library( "//reverb/cc/checkpointing:checkpoint_cc_proto", "//reverb/cc/platform:hash_map", "//reverb/cc/support:intrusive_heap", - ] + reverb_tf_deps() + reverb_absl_deps(), + "@local_config_tf//:tf_header_lib", + "@com_google_absl//absl/status", + ] ) -reverb_cc_test( +cc_test( name = "uniform_test", srcs = ["uniform_test.cc"], deps = [ @@ -92,10 +96,10 @@ reverb_cc_test( "//reverb/cc:schema_cc_proto", "//reverb/cc/platform:status_matchers", "//reverb/cc/testing:proto_test_util", - ] + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "fifo_test", srcs = ["fifo_test.cc"], deps = [ @@ -103,10 +107,10 @@ reverb_cc_test( "//reverb/cc:schema_cc_proto", "//reverb/cc/platform:status_matchers", "//reverb/cc/testing:proto_test_util", - ] + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "lifo_test", srcs = ["lifo_test.cc"], deps = [ @@ -114,10 +118,10 @@ reverb_cc_test( "//reverb/cc:schema_cc_proto", "//reverb/cc/platform:status_matchers", "//reverb/cc/testing:proto_test_util", - ] + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "prioritized_test", srcs = ["prioritized_test.cc"], deps = [ @@ -126,10 +130,10 @@ reverb_cc_test( "//reverb/cc/platform:hash_map", "//reverb/cc/platform:status_matchers", "//reverb/cc/testing:proto_test_util", - ] + reverb_absl_deps(), + ] ) -reverb_cc_test( +cc_test( name = "heap_test", srcs = ["heap_test.cc"], deps = [ diff --git a/reverb/cc/support/BUILD b/reverb/cc/support/BUILD index 9bf0b659..166da072 100644 --- a/reverb/cc/support/BUILD +++ b/reverb/cc/support/BUILD @@ -1,45 +1,36 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_library", - "reverb_cc_test", - "reverb_grpc_deps", - "reverb_tf_deps", -) +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_cc_library( +cc_library( name = "uint128", hdrs = ["uint128.h"], deps = [ "//reverb/cc:schema_cc_proto", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "grpc_util", hdrs = ["grpc_util.h"], - deps = reverb_tf_deps() + reverb_grpc_deps() + reverb_absl_deps(), ) -reverb_cc_library( +cc_library( name = "queue", hdrs = ["queue.h"], deps = [ "//reverb/cc/platform:logging", - ] + reverb_absl_deps() + reverb_tf_deps(), + ], ) -reverb_cc_library( +cc_library( name = "cleanup", hdrs = ["cleanup.h"], - deps = reverb_absl_deps(), ) -reverb_cc_test( +cc_test( name = "cleanup_test", srcs = ["cleanup_test.cc"], deps = [ @@ -47,7 +38,7 @@ reverb_cc_test( ], ) -reverb_cc_library( +cc_library( name = "intrusive_heap", srcs = ["intrusive_heap.cc"], hdrs = ["intrusive_heap.h"], @@ -56,25 +47,27 @@ reverb_cc_library( ], ) -reverb_cc_test( +cc_test( name = "intrusive_heap_test", srcs = ["intrusive_heap_test.cc"], deps = [ ":intrusive_heap", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "periodic_closure", srcs = ["periodic_closure.cc"], hdrs = ["periodic_closure.h"], deps = [ "//reverb/cc/platform:logging", "//reverb/cc/platform:thread", - ] + reverb_absl_deps() + reverb_tf_deps(), + "@com_google_absl//absl/status", + "@com_google_absl//absl/strings", + ], ) -reverb_cc_test( +cc_test( name = "queue_test", srcs = ["queue_test.cc"], deps = [ @@ -82,20 +75,20 @@ reverb_cc_test( "//reverb/cc/platform:logging", "//reverb/cc/platform:status_matchers", "//reverb/cc/platform:thread", - ] + reverb_absl_deps(), + ], ) -reverb_cc_test( +cc_test( name = "periodic_closure_test", srcs = ["periodic_closure_test.cc"], deps = [ ":periodic_closure", "//reverb/cc/platform:status_matchers", "//reverb/cc/testing:time_testutil", - ] + reverb_absl_deps() + reverb_tf_deps(), + ], ) -reverb_cc_library( +cc_library( name = "signature", srcs = ["signature.cc"], hdrs = ["signature.h"], @@ -107,38 +100,38 @@ reverb_cc_library( "//reverb/cc/platform:hash_set", "//reverb/cc/platform:logging", "//reverb/cc/platform:status_macros", - ] + reverb_tf_deps() + reverb_absl_deps(), + ], ) -reverb_cc_test( +cc_test( name = "signature_test", srcs = ["signature_test.cc"], deps = [ ":signature", "//reverb/cc/platform:status_matchers", "//reverb/cc/testing:proto_test_util", - ] + reverb_tf_deps() + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "state_statistics", hdrs = [ "state_statistics.h", ], deps = [ "//reverb/cc/platform:hash_map", - ] + reverb_absl_deps() + reverb_tf_deps(), + ], ) -reverb_cc_test( +cc_test( name = "state_statistics_test", srcs = ["state_statistics_test.cc"], deps = [ ":state_statistics", - ] + reverb_tf_deps() + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "task_executor", srcs = ["task_executor.cc"], hdrs = ["task_executor.h"], @@ -147,33 +140,32 @@ reverb_cc_library( "//reverb/cc/platform:thread", "//reverb/cc/support:periodic_closure", "//reverb/cc/support:unbounded_queue", - ] + reverb_absl_deps(), + ], ) -reverb_cc_library( +cc_library( name = "tf_util", hdrs = ["tf_util.h"], deps = [ - ] + reverb_absl_deps() + reverb_tf_deps(), + ], ) -reverb_cc_library( +cc_library( name = "unbounded_queue", hdrs = ["unbounded_queue.h"], - deps = reverb_absl_deps(), ) -reverb_cc_test( +cc_test( name = "unbounded_queue_test", srcs = ["unbounded_queue_test.cc"], deps = [ ":unbounded_queue", "//reverb/cc/platform:logging", "//reverb/cc/platform:thread", - ] + reverb_absl_deps() + reverb_tf_deps(), + ], ) -reverb_cc_library( +cc_library( name = "trajectory_util", srcs = ["trajectory_util.cc"], hdrs = ["trajectory_util.h"], @@ -184,10 +176,10 @@ reverb_cc_library( "//reverb/cc/platform:hash_set", "//reverb/cc/platform:logging", "//reverb/cc/platform:status_macros", - ] + reverb_tf_deps() + reverb_absl_deps(), + ], ) -reverb_cc_test( +cc_test( name = "trajectory_util_test", srcs = ["trajectory_util_test.cc"], deps = [ @@ -198,11 +190,10 @@ reverb_cc_test( "//reverb/cc/platform:status_matchers", "//reverb/cc/testing:proto_test_util", "//reverb/cc/testing:tensor_testutil", - ] + reverb_tf_deps(), + ], ) -reverb_cc_library( +cc_library( name = "key_generators", hdrs = ["key_generators.h"], - deps = reverb_absl_deps(), ) diff --git a/reverb/cc/support/tf_util.h b/reverb/cc/support/tf_util.h index bf815a89..812ba468 100644 --- a/reverb/cc/support/tf_util.h +++ b/reverb/cc/support/tf_util.h @@ -24,21 +24,12 @@ namespace reverb { // Converts a tensorflow::Status object to an absl::Status object. inline absl::Status FromTensorflowStatus(const tensorflow::Status& status) { - if (status.ok()) { - return absl::Status(); - } else { - return absl::Status(static_cast(status.code()), - status.message()); - } + return status; } // Converts an absl::Status object to a tensorflow::Status object. inline tensorflow::Status ToTensorflowStatus(const absl::Status& status) { - if (status.ok()) { - return tensorflow::Status(); - } else { - return tensorflow::Status(status.code(), status.message()); - } + return status; } } // namespace reverb diff --git a/reverb/cc/table.cc b/reverb/cc/table.cc index b5e4e39b..bbb03a75 100644 --- a/reverb/cc/table.cc +++ b/reverb/cc/table.cc @@ -154,10 +154,10 @@ Table::Table(std::string name, std::shared_ptr sampler, num_unique_samples_(0), max_size_(max_size), max_enqueued_inserts_( - std::max(1L, std::min(max_size * kMaxEnqueuedInsertsPerc, + std::max(static_cast(1), std::min(max_size * kMaxEnqueuedInsertsPerc, kMaxEnqueuedInserts))), max_enqueued_extension_ops_( - std::max(1L, std::min(max_size * kMaxPendingExtensionOpsPerc, + std::max(static_cast(1), std::min(max_size * kMaxPendingExtensionOpsPerc, kMaxPendingExtensionOps))), max_times_sampled_(max_times_sampled), name_(std::move(name)), diff --git a/reverb/cc/table_extensions/BUILD b/reverb/cc/table_extensions/BUILD index 7ec61a6b..af089285 100644 --- a/reverb/cc/table_extensions/BUILD +++ b/reverb/cc/table_extensions/BUILD @@ -1,23 +1,17 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_library", - "reverb_tf_deps", -) - +load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_cc_library( +cc_library( name = "interface", hdrs = ["interface.h"], deps = [ "//reverb/cc:schema_cc_proto", - ] + reverb_absl_deps() + reverb_tf_deps(), + ]# + reverb_absl_deps() ) -reverb_cc_library( +cc_library( name = "base", srcs = ["base.cc"], hdrs = ["base.h"], @@ -26,5 +20,5 @@ reverb_cc_library( "//reverb/cc:schema_cc_proto", "//reverb/cc:table", "//reverb/cc/platform:logging", - ] + reverb_absl_deps(), + ] ) diff --git a/reverb/cc/testing/BUILD b/reverb/cc/testing/BUILD index a27ecf06..2e7341f4 100644 --- a/reverb/cc/testing/BUILD +++ b/reverb/cc/testing/BUILD @@ -1,15 +1,9 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_absl_deps", - "reverb_cc_library", - "reverb_tf_deps", -) - +load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_cc_library( +cc_library( name = "proto_test_util", testonly = 1, srcs = ["proto_test_util.cc"], @@ -18,21 +12,20 @@ reverb_cc_library( "//reverb/cc:schema_cc_proto", "//reverb/cc:tensor_compression", "//reverb/cc/platform:logging", - ] + reverb_tf_deps(), + ] ) -reverb_cc_library( +cc_library( name = "tensor_testutil", testonly = 1, hdrs = ["tensor_testutil.h"], deps = [ "//reverb/cc/platform:logging", - ] + reverb_tf_deps(), + ] ) -reverb_cc_library( +cc_library( name = "time_testutil", testonly = 1, hdrs = ["time_testutil.h"], - deps = reverb_absl_deps(), ) diff --git a/reverb/pip_package/BUILD b/reverb/pip_package/BUILD index 718d4245..190b65e3 100644 --- a/reverb/pip_package/BUILD +++ b/reverb/pip_package/BUILD @@ -1,3 +1,4 @@ +load("@rules_python//python:defs.bzl", "py_library") licenses(["notice"]) # Apache 2.0 filegroup( diff --git a/reverb/pip_package/MANIFEST.in b/reverb/pip_package/MANIFEST.in index 3293c9b1..478869af 100644 --- a/reverb/pip_package/MANIFEST.in +++ b/reverb/pip_package/MANIFEST.in @@ -1,2 +1,4 @@ recursive-include * *.py recursive-include * *.so +recursive-include * *.dylib +recursive-include * *.a diff --git a/reverb/pip_package/README.md b/reverb/pip_package/README.md index 74f915ca..3190a157 100644 --- a/reverb/pip_package/README.md +++ b/reverb/pip_package/README.md @@ -30,12 +30,6 @@ There are two steps for building the Reverb package. Execute from the root of the git repository. The end result will end up in `$REVERB_DIR/dist`. -> :warning: Bazel is caching the first version of files or a config of the first - version of Python used. If building reverb concecutively for different - versions of Python `--clear_bazel_cache true` is needed and included in - the commands below. Running the docker once for each python version also - works. This issue may resolve itself in the future. - ```shell ################################## @@ -46,7 +40,7 @@ Execute from the root of the git repository. The end result will end up in # If building against an RC, use == rather than ~= for `tensorflow_pip`. $ docker build --pull --no-cache \ --tag tensorflow:reverb_release \ - --build-arg tensorflow_pip=tensorflow~=2.12.0 \ + --build-arg tensorflow_pip=tensorflow~=2.8.0 \ --build-arg python_version="python3.8 python3.9 python3.10 python3.11" \ - < "$REVERB_DIR/docker/release.dockerfile" @@ -59,16 +53,14 @@ $ docker build --pull --no-cache \ # Packages for Python 3.8, 3.9, 3.10, and 3.11 are created. $ docker run --rm --mount "type=bind,src=$REVERB_DIR,dst=/tmp/reverb" \ tensorflow:reverb_release bash oss_build.sh --clean true \ - --clear_bazel_cache true \ --tf_dep_override "tensorflow~=2.12.0" \ - --release --python "3.8 3.9 3.10 3.11" + --tf_dep_override "tensorflow~=2.11.0" --release --python "3.8 3.9 3.10 3.11" # Builds Reverb against an RC of TensorFlow. `>=` and `~=` are not effective # because pip does not recognize 2.4.0rc0 as greater than 2.3.0. RC builds need # to have a strict dependency on the RC of TensorFlow used. $ docker run --rm --mount "type=bind,src=$REVERB_DIR,dst=/tmp/reverb" \ tensorflow:reverb_release bash oss_build.sh --clean true \ - --clear_bazel_cache true --tf_dep_override "tensorflow==2.12.0rc0" \ - --release --python "3.8 3.9 3.10 3.11" + --tf_dep_override "tensorflow==2.12.0rc0" --release --python "3.8 3.9 3.10 3.11" # Builds a debug version of Reverb. The debug version is not labeled as debug # as that can result in a user installing both the debug and regular packages @@ -77,8 +69,8 @@ $ docker run --rm --mount "type=bind,src=$REVERB_DIR,dst=/tmp/reverb" \ # Debug builds are ~90M compared to normal builds that are closer to 7M. $ docker run --rm --mount "type=bind,src=$REVERB_DIR,dst=/tmp/reverb" \ tensorflow:reverb_release bash oss_build.sh --clean true --debug_build true \ - --clear_bazel_cache true --output_dir /tmp/reverb/dist/debug/ \ - --tf_dep_override "tensorflow~=2.12.0" --release --python "3.8 3.9 3.10 3.11" + --output_dir /tmp/reverb/dist/debug/ --tf_dep_override "tensorflow~=2.10.0" \ + --release --python "3.8 3.9 3.10 3.11" ``` diff --git a/reverb/pip_package/build_pip_package.sh b/reverb/pip_package/build_pip_package.sh index f6a1fd9b..6b6ce85a 100755 --- a/reverb/pip_package/build_pip_package.sh +++ b/reverb/pip_package/build_pip_package.sh @@ -32,7 +32,7 @@ function build_wheel() { pushd ${TMPDIR} > /dev/null echo $(date) : "=== Building wheel" - "${PYTHON_BIN_PATH}" setup.py bdist_wheel ${PKG_NAME_FLAG} ${RELEASE_FLAG} ${TF_VERSION_FLAG} --plat manylinux2014_x86_64 > /dev/null + "${PYTHON_BIN_PATH}" setup.py bdist_wheel sdist --formats=gztar ${PKG_NAME_FLAG} ${RELEASE_FLAG} ${TF_VERSION_FLAG} > /dev/null DEST=${TMPDIR}/dist/ if [[ ! "$TMPDIR" -ef "$DESTDIR" ]]; then mkdir -p ${DESTDIR} @@ -70,7 +70,7 @@ function prepare_src() { # This copies all .so files except for those found in the ops directory, which # must remain where they are for TF to find them. find "${TMPDIR}/reverb/cc" -type d -name ops -prune -o -name '*.so' \ - -exec mv {} "${TMPDIR}/reverb" \; + -exec mv {} "${TMPDIR}/reverb" \; } function usage() { diff --git a/reverb/pip_package/reverb_version.py b/reverb/pip_package/reverb_version.py index 96418321..4f56e6d4 100644 --- a/reverb/pip_package/reverb_version.py +++ b/reverb/pip_package/reverb_version.py @@ -26,7 +26,7 @@ # release branch, the current version is by default assumed to be a # 'development' version, labeled 'dev'. _DEV_SUFFIX = 'dev' -_REL_SUFFIX = 'rc0' +_REL_SUFFIX = '' # Example, '0.5.0rc0' __version__ = '.'.join([ diff --git a/reverb/pip_package/setup.py b/reverb/pip_package/setup.py index 406f2ed9..e0b21647 100644 --- a/reverb/pip_package/setup.py +++ b/reverb/pip_package/setup.py @@ -28,7 +28,7 @@ import reverb_version # Defaults if doing a release build. -TENSORFLOW_VERSION = 'tensorflow~=2.7.0' +TENSORFLOW_VERSION = 'tensorflow~=2.12.0' class BinaryDistribution(Distribution): diff --git a/reverb/platform/BUILD b/reverb/platform/BUILD index 3b668443..b46174f1 100644 --- a/reverb/platform/BUILD +++ b/reverb/platform/BUILD @@ -1,28 +1,19 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_py_test", - "reverb_pytype_strict_library", -) - +load("@rules_python//python:defs.bzl", "py_library", "py_test") package(default_visibility = [":__subpackages__"]) licenses(["notice"]) -reverb_pytype_strict_library( +py_library( name = "checkpointers_lib", srcs = ["checkpointers_lib.py"], srcs_version = "PY3", - deps = [ - "//reverb:pybind", - ], + data = ["//reverb:pybind.so"], ) -reverb_py_test( +py_test( name = "checkpointers_lib_test", srcs = ["checkpointers_lib_test.py"], python_version = "PY3", - deps = [ - ":checkpointers_lib", - "//reverb:pybind", - ], + deps = [":checkpointers_lib"], + data = ["//reverb:pybind.so"], ) diff --git a/reverb/platform/default/BUILD b/reverb/platform/default/BUILD index ee2653a4..71a75756 100644 --- a/reverb/platform/default/BUILD +++ b/reverb/platform/default/BUILD @@ -1,19 +1,16 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_pytype_strict_library", -) +load("@rules_python//python:defs.bzl", "py_library") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_pytype_strict_library( +py_library( name = "load_op_library", srcs = ["load_op_library.py"], srcs_version = "PY3", ) -reverb_pytype_strict_library( +py_library( name = "checkpointers", srcs = ["checkpointers.py"], srcs_version = "PY3", @@ -22,13 +19,13 @@ reverb_pytype_strict_library( ], ) -reverb_pytype_strict_library( +py_library( name = "ensure_tf_install", srcs = ["ensure_tf_install.py"], srcs_version = "PY3", ) -reverb_pytype_strict_library( +py_library( name = "server_main_command_line_args", srcs = ["server_main_command_line_args.py"], srcs_version = "PY3", diff --git a/reverb/pybind.cc b/reverb/pybind.cc index 06ec8e29..993ed2b9 100644 --- a/reverb/pybind.cc +++ b/reverb/pybind.cc @@ -188,7 +188,7 @@ namespace { namespace py = pybind11; -PYBIND11_MODULE(libpybind, m) { +PYBIND11_MODULE(pybind, m) { // Initialization code to use numpy types in the type casters. pybind::ImportNumpy(); diff --git a/reverb/server_executable/BUILD b/reverb/server_executable/BUILD index 504c7d76..efc30c2c 100644 --- a/reverb/server_executable/BUILD +++ b/reverb/server_executable/BUILD @@ -1,32 +1,29 @@ -load( - "//reverb/cc/platform:build_rules.bzl", - "reverb_cc_proto_library", - "reverb_py_proto_library", - "reverb_py_test", - "reverb_pytype_strict_binary", - "reverb_pytype_strict_library", -) +load("@rules_proto//proto:defs.bzl", "proto_library") +load("@rules_cc//cc:defs.bzl", "cc_proto_library") +load("@rules_python//python:defs.bzl", "py_library", "py_binary", "py_test") +load("@com_github_grpc_grpc//bazel:python_rules.bzl", "py_proto_library") package(default_visibility = ["//reverb:__subpackages__"]) licenses(["notice"]) -reverb_cc_proto_library( - name = "reverb_config_cc_proto", +proto_library( + name = "reverb_config_proto", srcs = ["reverb_config.proto"], - deps = ["//reverb/cc/checkpointing:checkpoint_cc_proto"], + deps = ["//reverb/cc/checkpointing:checkpoint_proto"], +) + +cc_proto_library( + name = "reverb_config_cc_proto", + deps = [":reverb_config_proto"], ) -reverb_py_proto_library( +py_proto_library( name = "reverb_config_py_pb2", - srcs = ["reverb_config.proto"], - deps = [ - ":reverb_config_cc_proto", - "//reverb/cc/checkpointing:checkpoint_cc_proto", - ], + deps = [":reverb_config_proto"], ) -reverb_pytype_strict_library( +py_library( name = "server_from_proto", srcs = ["server_from_proto.py"], srcs_version = "PY3", @@ -38,10 +35,9 @@ reverb_pytype_strict_library( ], ) -reverb_py_test( +py_test( name = "server_from_proto_test", srcs = ["server_from_proto_test.py"], - enable_dashboard = False, srcs_version = "PY3", deps = [ ":server_from_proto", @@ -51,17 +47,19 @@ reverb_py_test( ], ) -reverb_pytype_strict_binary( +py_binary( name = "server_main", srcs = ["server_main.py"], srcs_version = "PY3", deps = [ ":server_from_proto", "//reverb", - "//reverb:pybind", "//reverb:reverb_types", "//reverb/cc:schema_py_pb2", "//reverb/cc/checkpointing:checkpoint_py_pb2", "//reverb/platform/default:server_main_command_line_args", ], + data = [ + "//reverb:pybind.so", + ] ) diff --git a/tf/BUILD b/tf/BUILD new file mode 100644 index 00000000..e69de29b diff --git a/tf/BUILD.tpl b/tf/BUILD.tpl new file mode 100644 index 00000000..f87a9d4a --- /dev/null +++ b/tf/BUILD.tpl @@ -0,0 +1,75 @@ +package(default_visibility = ["//visibility:public"]) + +cc_library( + name = "tf_header_lib", + hdrs = [":tf_header_include"], + includes = ["include"], + visibility = ["//visibility:public"], +) + +cc_library( + name = "libtensorflow_framework", + srcs = [":libtensorflow_framework.so"], + #data = ["lib/libtensorflow_framework.so"], + visibility = ["//visibility:public"], +) + +proto_library( + name = "tensor_proto", + srcs = ["include/tensorflow/core/framework/tensor.proto"], + strip_import_prefix = "include/", + deps = [ + ":resource_handle_proto", + ":tensor_shape_proto", + ":types_proto", + ], + visibility = ["//visibility:public"], +) + +proto_library( + name = "struct_proto", + srcs = ["include/tensorflow/core/protobuf/struct.proto"], + strip_import_prefix = "include/", + deps = [ + ":tensor_proto", + ":tensor_shape_proto", + ":types_proto", + ], + visibility = ["//visibility:public"], +) + +proto_library( + name = "resource_handle_proto", + srcs = ["include/tensorflow/core/framework/resource_handle.proto"], + strip_import_prefix = "include/", + deps = [ + ":tensor_shape_proto", + ":types_proto", + ], + visibility = ["//visibility:public"], +) + +proto_library( + name = "tensor_shape_proto", + srcs = ["include/tensorflow/core/framework/tensor_shape.proto"], + strip_import_prefix = "include/", + visibility = ["//visibility:public"], +) + +proto_library( + name = "types_proto", + srcs = ["include/tensorflow/core/framework/types.proto"], + strip_import_prefix = "include/", + visibility = ["//visibility:public"], +) + +cc_library( + name = "numpy_headers", + hdrs = [":numpy_include"], + includes = ["numpy_include"], + strip_include_prefix = "numpy_include/", +) + +%{TF_HEADER_GENRULE} +%{TF_SHARED_LIBRARY_GENRULE} +%{NUMPY_INCLUDE_GENRULE} diff --git a/tf/tf_configure.bzl b/tf/tf_configure.bzl new file mode 100644 index 00000000..edc54acc --- /dev/null +++ b/tf/tf_configure.bzl @@ -0,0 +1,241 @@ +"""Setup TensorFlow as external dependency""" + +_TF_HEADER_DIR = "TF_HEADER_DIR" +_TF_SHARED_LIBRARY_DIR = "TF_SHARED_LIBRARY_DIR" +_TF_SHARED_LIBRARY_NAME = "TF_SHARED_LIBRARY_NAME" +_PYTHON_BIN_PATH = "PYTHON_BIN_PATH" + +def _tpl(repository_ctx, tpl, substitutions = {}, out = None): + if not out: + out = tpl + repository_ctx.template( + out, + Label("//tf:%s.tpl" % tpl), + substitutions, + ) + +def _fail(msg): + """Output failure message when auto configuration fails.""" + red = "\033[0;31m" + no_color = "\033[0m" + fail("%sPython Configuration Error:%s %s\n" % (red, no_color, msg)) + +def _is_windows(repository_ctx): + """Returns true if the host operating system is windows.""" + os_name = repository_ctx.os.name.lower() + if os_name.find("windows") != -1: + return True + return False + +def _execute( + repository_ctx, + cmdline, + error_msg = None, + error_details = None, + empty_stdout_fine = False): + """Executes an arbitrary shell command. + Args: + repository_ctx: the repository_ctx object + cmdline: list of strings, the command to execute + error_msg: string, a summary of the error if the command fails + error_details: string, details about the error or steps to fix it + empty_stdout_fine: bool, if True, an empty stdout result is fine, otherwise + it's an error + Return: + the result of repository_ctx.execute(cmdline) + """ + result = repository_ctx.execute(cmdline) + if result.stderr or not (empty_stdout_fine or result.stdout): + _fail("\n".join([ + error_msg.strip() if error_msg else "Repository command failed", + result.stderr.strip(), + error_details if error_details else "", + ])) + return result + +def _read_dir(repository_ctx, src_dir): + """Returns a string with all files in a directory. + Finds all files inside a directory, traversing subfolders and following + symlinks. The returned string contains the full path of all files + separated by line breaks. + """ + if _is_windows(repository_ctx): + src_dir = src_dir.replace("/", "\\") + find_result = _execute( + repository_ctx, + ["cmd.exe", "/c", "dir", src_dir, "/b", "/s", "/a-d"], + empty_stdout_fine = True, + ) + + # src_files will be used in genrule.outs where the paths must + # use forward slashes. + result = find_result.stdout.replace("\\", "/") + else: + find_result = _execute( + repository_ctx, + ["find", src_dir, "-follow", "-type", "f"], + empty_stdout_fine = True, + ) + result = find_result.stdout + return result + +def _genrule(genrule_name, command, outs): + """Returns a string with a genrule. + + Genrule executes the given command and produces the given outputs. + + Args: + genrule_name: A unique name for genrule target. + command: The command to run. + outs: A list of files generated by this rule. + + Returns: + A genrule target. + """ + return ( + "genrule(\n" + + ' name = "' + + genrule_name + '",\n' + + " outs = [\n" + + outs + + "\n ],\n" + + ' cmd = """\n' + + command + + '\n """,\n' + + ")\n" + ) + +def _norm_path(path): + """Returns a path with '/' and remove the trailing slash.""" + path = path.replace("\\", "/") + if path[-1] == "/": + path = path[:-1] + return path + +def _symlink_genrule_for_dir( + repository_ctx, + src_dir, + dest_dir, + genrule_name, + src_files = [], + dest_files = [], + tf_pip_dir_rename_pair = []): + """Returns a genrule to symlink(or copy if on Windows) a set of files. + + If src_dir is passed, files will be read from the given directory; otherwise + we assume files are in src_files and dest_files. + + Args: + repository_ctx: the repository_ctx object. + src_dir: source directory. + dest_dir: directory to create symlink in. + genrule_name: genrule name. + src_files: list of source files instead of src_dir. + dest_files: list of corresonding destination files. + tf_pip_dir_rename_pair: list of the pair of tf pip parent directory to + replace. For example, in TF pip package, the source code is under + "tensorflow_core", and we might want to replace it with + "tensorflow" to match the header includes. + + Returns: + genrule target that creates the symlinks. + """ + # Check that tf_pip_dir_rename_pair has the right length + tf_pip_dir_rename_pair_len = len(tf_pip_dir_rename_pair) + if tf_pip_dir_rename_pair_len != 0 and tf_pip_dir_rename_pair_len !=2: + _fail("The size of argument tf_pip_dir_rename_pair should be either 0 or 2, but %d is given." % tf_pip_dir_rename_pair_len) + + if src_dir != None: + src_dir = _norm_path(src_dir) + dest_dir = _norm_path(dest_dir) + files = "\n".join(sorted(_read_dir(repository_ctx, src_dir).splitlines())) + + # Create a list with the src_dir stripped to use for outputs. + if tf_pip_dir_rename_pair_len: + dest_files = files.replace(src_dir, "").replace(tf_pip_dir_rename_pair[0], tf_pip_dir_rename_pair[1]).splitlines() + else: + dest_files = files.replace(src_dir, "").splitlines() + src_files = files.splitlines() + command = [] + outs = [] + + for i in range(len(dest_files)): + if dest_files[i] != "": + # If we have only one file to link we do not want to use the dest_dir, as + # $(@D) will include the full path to the file. + dest = "$(@D)/" + dest_dir + dest_files[i] if len(dest_files) != 1 else "$(@D)/" + dest_files[i] + + # Copy the headers to create a sandboxable setup. + cmd = "cp -f" + command.append(cmd + ' "%s" "%s"' % (src_files[i], dest)) + outs.append(' "' + dest_dir + dest_files[i] + '",') + dest_dir = "abc" + genrule = _genrule( + genrule_name, + " && ".join(command), + "\n".join(outs), + ) + return genrule + +def _get_numpy_include(repository_ctx, python_bin): + """Gets the numpy include path.""" + return _execute( + repository_ctx, + [ + python_bin, + "-c", + "from __future__ import print_function;" + + "import numpy;" + + " print(numpy.get_include());", + ], + error_msg = "Problem getting numpy include path.", + error_details = "Is numpy installed?", + ).stdout.splitlines()[0] + +def _tf_pip_impl(repository_ctx): + tf_header_dir = repository_ctx.os.environ[_TF_HEADER_DIR] + tf_header_rule = _symlink_genrule_for_dir( + repository_ctx, + tf_header_dir, + "include", + "tf_header_include", + tf_pip_dir_rename_pair = ["tensorflow_core", "tensorflow"] + ) + + tf_shared_library_dir = repository_ctx.os.environ[_TF_SHARED_LIBRARY_DIR] + tf_shared_library_name = repository_ctx.os.environ[_TF_SHARED_LIBRARY_NAME] + tf_shared_library_path = "%s/%s" % (tf_shared_library_dir, tf_shared_library_name) + + tf_shared_library_rule = _symlink_genrule_for_dir( + repository_ctx, + None, + "", + "libtensorflow_framework.so", + [tf_shared_library_path], + ["_pywrap_tensorflow_internal.lib" if _is_windows(repository_ctx) else "libtensorflow_framework.so"], + ) + + python_bin = repository_ctx.os.environ[_PYTHON_BIN_PATH] + numpy_include = _get_numpy_include(repository_ctx, python_bin) + "/numpy" + numpy_include_rule = _symlink_genrule_for_dir( + repository_ctx, + numpy_include, + "numpy_include/numpy", + "numpy_include", + ) + + _tpl(repository_ctx, "BUILD", { + "%{TF_HEADER_GENRULE}": tf_header_rule, + "%{TF_SHARED_LIBRARY_GENRULE}": tf_shared_library_rule, + "%{NUMPY_INCLUDE_GENRULE}": numpy_include_rule, + }) + +tf_configure = repository_rule( + implementation = _tf_pip_impl, + environ = [ + _PYTHON_BIN_PATH, + _TF_HEADER_DIR, + _TF_SHARED_LIBRARY_DIR, + _TF_SHARED_LIBRARY_NAME, + ], +) diff --git a/third_party/BUILD b/third_party/BUILD deleted file mode 100644 index fe756e1b..00000000 --- a/third_party/BUILD +++ /dev/null @@ -1 +0,0 @@ -licenses(["notice"]) # Apache 2.0 diff --git a/third_party/auditwheel.sh b/third_party/auditwheel.sh deleted file mode 100755 index d6320c70..00000000 --- a/third_party/auditwheel.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# Copyright 2019 The TensorFlow Authors. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# ============================================================================== - -TF_SHARED_LIBRARY_NAME=$(grep -r TF_SHARED_LIBRARY_NAME .bazelrc | awk -F= '{print$2}') - -POLICY_JSON=$(find / -name policy.json) - -sed -i "s/libresolv.so.2\"/libresolv.so.2\", $TF_SHARED_LIBRARY_NAME/g" $POLICY_JSON - -cat $POLICY_JSON - -auditwheel $@ diff --git a/third_party/opensource_only.files b/third_party/opensource_only.files deleted file mode 100644 index 7af2b25b..00000000 --- a/third_party/opensource_only.files +++ /dev/null @@ -1,12 +0,0 @@ -BUILD: -reverb/cc/platform/default/BUILD: -reverb/pip_package/BUILD: -reverb/pip_package/MANIFEST.in: -reverb/pip_package/setup.py: -reverb/platform/default/BUILD: -third_party/BUILD: -third_party/protobuf.BUILD: -third_party/pybind11.BUILD: -third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/BUILD: -third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/cc_toolchain_config.bzl: -third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/dummy_toolchain.bzl: diff --git a/third_party/protobuf.BUILD b/third_party/protobuf.BUILD deleted file mode 100644 index ac55c06b..00000000 --- a/third_party/protobuf.BUILD +++ /dev/null @@ -1,41 +0,0 @@ -_CHECK_VERSION = """ -PROTOC_VERSION=$$($(location @protobuf_protoc//:protoc_bin) --version \ - | cut -d' ' -f2 | sed -e 's/\\./ /g') -PROTOC_VERSION=$$(printf '%d%03d%03d' $${PROTOC_VERSION}) -TF_PROTO_VERSION=$$(grep '#define PROTOBUF_MIN_PROTOC_VERSION' \ - $(location tf_includes/google/protobuf/port_def.inc) | cut -d' ' -f3) -if [ "$${PROTOC_VERSION}" -ne "$${TF_PROTO_VERSION}" ]; then - echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!! 1>&2 - echo Your protoc version does not match the tensorflow proto header \ - required version: "$${PROTOC_VERSION}" vs. "$${TF_PROTO_VERSION}" 1>&2 - echo Please update the PROTOC_VERSION in your WORKSPACE file. 1>&2 - echo https://github.com/deepmind/reverb/tree/master/reverb/pip_package 1>&2 - echo !!!!!!!!!!!!!!!!!!!!!!!!!!!!! 1>&2 - false -else - touch $@ -fi -""" - -genrule( - name = "compare_protobuf_version", - outs = ["versions_compared"], - srcs = [ - "tf_includes/google/protobuf/port_def.inc", - ], - tools = ["@protobuf_protoc//:protoc_bin"], - cmd = _CHECK_VERSION, -) - -cc_library( - name = "includes", - data = [":versions_compared"], - hdrs = glob([ - "tf_includes/google/protobuf/*.h", - "tf_includes/google/protobuf/*.inc", - "tf_includes/google/protobuf/**/*.h", - "tf_includes/google/protobuf/**/*.inc", - ]), - includes = ["tf_includes"], - visibility = ["//visibility:public"], -) diff --git a/third_party/pybind11.BUILD b/third_party/pybind11.BUILD deleted file mode 100644 index 8bee1d63..00000000 --- a/third_party/pybind11.BUILD +++ /dev/null @@ -1,24 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -cc_library( - name = "pybind11", - hdrs = glob( - include = [ - "include/pybind11/*.h", - "include/pybind11/detail/*.h", - ], - exclude = [ - "include/pybind11/common.h", - "include/pybind11/eigen.h", - ], - ), - copts = [ - "-fexceptions", - "-Wno-undefined-inline", - "-Wno-pragma-once-outside-header", - ], - includes = ["include"], - deps = [ - "@python_includes", - ], -) diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/BUILD b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/BUILD deleted file mode 100755 index 9b81d725..00000000 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/BUILD +++ /dev/null @@ -1,121 +0,0 @@ -# Copyright 2016 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# This becomes the BUILD file for @local_config_cc// under non-FreeBSD unixes. - -load(":cc_toolchain_config.bzl", "cc_toolchain_config") - -package(default_visibility = ["//visibility:public"]) - -licenses(["notice"]) # Apache 2.0 - -cc_library( - name = "malloc", -) - -filegroup( - name = "empty", - srcs = [], -) - -filegroup( - name = "cc_wrapper", - srcs = ["cc_wrapper.sh"], -) - -filegroup( - name = "compiler_deps", - srcs = glob(["extra_tools/**"]) + [":empty"], -) - -# This is the entry point for --crosstool_top. Toolchains are found -# by lopping off the name of --crosstool_top and searching for -# the "${CPU}" entry in the toolchains attribute. -cc_toolchain_suite( - name = "toolchain", - toolchains = { - "k8|/dt9/usr/bin/gcc": ":cc-compiler-k8", - "k8": ":cc-compiler-k8", - "armeabi-v7a|compiler": ":cc-compiler-armeabi-v7a", - "armeabi-v7a": ":cc-compiler-armeabi-v7a", - }, -) - -cc_toolchain( - name = "cc-compiler-k8", - all_files = ":compiler_deps", - ar_files = ":empty", - as_files = ":empty", - compiler_files = ":compiler_deps", - dwp_files = ":empty", - linker_files = ":compiler_deps", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, - toolchain_config = ":linux_gnu_x86", - toolchain_identifier = "linux_gnu_x86", -) - -cc_toolchain_config( - name = "linux_gnu_x86", - compiler = "/dt9/usr/bin/gcc", - cpu = "k8", -) - -toolchain( - name = "cc-toolchain-k8", - exec_compatible_with = [ - # TODO(b/154931569): add autodiscovered constraints for host CPU and OS. - ], - target_compatible_with = [ - # TODO(b/154931569): add autodiscovered constraints for host CPU and OS. - ], - toolchain = ":cc-compiler-k8", - toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", -) - -# Android tooling requires a default toolchain for the armeabi-v7a cpu. -cc_toolchain( - name = "cc-compiler-armeabi-v7a", - all_files = ":empty", - ar_files = ":empty", - as_files = ":empty", - compiler_files = ":empty", - dwp_files = ":empty", - linker_files = ":empty", - objcopy_files = ":empty", - strip_files = ":empty", - supports_param_files = 1, - toolchain_config = ":stub_armeabi-v7a", - toolchain_identifier = "stub_armeabi-v7a", -) - -cc_toolchain_config( - name = "stub_armeabi-v7a", - compiler = "compiler", - cpu = "armeabi-v7a", -) - -toolchain( - name = "cc-toolchain-armeabi-v7a", - exec_compatible_with = [ - # TODO(b/154931569): add autodiscovered constraints for host CPU and OS. - ], - target_compatible_with = [ - "@bazel_tools//platforms:arm", - "@bazel_tools//platforms:android", - ], - toolchain = ":cc-compiler-armabi-v7a", - toolchain_type = "@bazel_tools//tools/cpp:toolchain_type", -) diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/WORKSPACE b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/WORKSPACE deleted file mode 100644 index bc05b4c3..00000000 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/WORKSPACE +++ /dev/null @@ -1,2 +0,0 @@ -# DO NOT EDIT: automatically generated WORKSPACE file for cc_autoconf rule -workspace(name = "local_config_cc") diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/cc_toolchain_config.bzl b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/cc_toolchain_config.bzl deleted file mode 100755 index 0168393d..00000000 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/cc_toolchain_config.bzl +++ /dev/null @@ -1,1732 +0,0 @@ -# Copyright 2019 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""A Starlark cc_toolchain configuration rule""" - -load( - "@bazel_tools//tools/cpp:cc_toolchain_config_lib.bzl", - "action_config", - "artifact_name_pattern", - "env_entry", - "env_set", - "feature", - "feature_set", - "flag_group", - "flag_set", - "make_variable", # @unused - "tool", - "tool_path", - "variable_with_value", - "with_feature_set", -) -load("@bazel_tools//tools/build_defs/cc:action_names.bzl", "ACTION_NAMES") - -all_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, - ACTION_NAMES.lto_backend, -] - -all_cpp_compile_actions = [ - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.clif_match, -] - -preprocessor_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.clif_match, -] - -codegen_compile_actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, -] - -all_link_actions = [ - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, -] - -def _windows_msvc_impl(ctx): - toolchain_identifier = "msvc_x64" - host_system_name = "local" - target_system_name = "local" - target_cpu = "x64_windows" - target_libc = "msvcrt" - compiler = "msvc-cl" - abi_version = "local" - abi_libc_version = "local" - cc_target_os = None - builtin_sysroot = None - - cxx_builtin_include_directories = [ - "/dt9/usr/lib/gcc/x86_64-pc-linux-gnu/9/include", - "/dt9/usr/lib/gcc/x86_64-pc-linux-gnu/9/include-fixed", - "/dt9/usr/include", - "/dt9/usr/include/c++/9", - "/dt9/usr/include/c++/9/x86_64-pc-linux-gnu", - "/dt9/usr/include/c++/9/backward", - ] - - cpp_link_nodeps_dynamic_library_action = action_config( - action_name = ACTION_NAMES.cpp_link_nodeps_dynamic_library, - implies = [ - "nologo", - "shared_flag", - "linkstamps", - "output_execpath_flags", - "input_param_flags", - "user_link_flags", - "default_link_flags", - "linker_subsystem_flag", - "linker_param_file", - "msvc_env", - "no_stripping", - "has_configured_linker_path", - "def_file", - ], - tools = [tool(path = "")], - ) - - cpp_link_static_library_action = action_config( - action_name = ACTION_NAMES.cpp_link_static_library, - implies = [ - "nologo", - "archiver_flags", - "input_param_flags", - "linker_param_file", - "msvc_env", - ], - tools = [tool(path = "")], - ) - - assemble_action = action_config( - action_name = ACTION_NAMES.assemble, - implies = [ - "compiler_input_flags", - "compiler_output_flags", - "nologo", - "msvc_env", - "sysroot", - ], - tools = [tool(path = "")], - ) - - preprocess_assemble_action = action_config( - action_name = ACTION_NAMES.preprocess_assemble, - implies = [ - "compiler_input_flags", - "compiler_output_flags", - "nologo", - "msvc_env", - "sysroot", - ], - tools = [tool(path = "")], - ) - - c_compile_action = action_config( - action_name = ACTION_NAMES.c_compile, - implies = [ - "compiler_input_flags", - "compiler_output_flags", - "default_compile_flags", - "nologo", - "msvc_env", - "parse_showincludes", - "user_compile_flags", - "sysroot", - "unfiltered_compile_flags", - ], - tools = [tool(path = "")], - ) - - cpp_compile_action = action_config( - action_name = ACTION_NAMES.cpp_compile, - implies = [ - "compiler_input_flags", - "compiler_output_flags", - "default_compile_flags", - "nologo", - "msvc_env", - "parse_showincludes", - "user_compile_flags", - "sysroot", - "unfiltered_compile_flags", - ], - tools = [tool(path = "")], - ) - - cpp_link_executable_action = action_config( - action_name = ACTION_NAMES.cpp_link_executable, - implies = [ - "nologo", - "linkstamps", - "output_execpath_flags", - "input_param_flags", - "user_link_flags", - "default_link_flags", - "linker_subsystem_flag", - "linker_param_file", - "msvc_env", - "no_stripping", - ], - tools = [tool(path = "")], - ) - - cpp_link_dynamic_library_action = action_config( - action_name = ACTION_NAMES.cpp_link_dynamic_library, - implies = [ - "nologo", - "shared_flag", - "linkstamps", - "output_execpath_flags", - "input_param_flags", - "user_link_flags", - "default_link_flags", - "linker_subsystem_flag", - "linker_param_file", - "msvc_env", - "no_stripping", - "has_configured_linker_path", - "def_file", - ], - tools = [tool(path = "")], - ) - - action_configs = [ - assemble_action, - preprocess_assemble_action, - c_compile_action, - cpp_compile_action, - cpp_link_executable_action, - cpp_link_dynamic_library_action, - cpp_link_nodeps_dynamic_library_action, - cpp_link_static_library_action, - ] - - msvc_link_env_feature = feature( - name = "msvc_link_env", - env_sets = [ - env_set( - actions = all_link_actions + - [ACTION_NAMES.cpp_link_static_library], - env_entries = [env_entry(key = "LIB", value = "")], - ), - ], - ) - - shared_flag_feature = feature( - name = "shared_flag", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], - flag_groups = [flag_group(flags = ["/DLL"])], - ), - ], - ) - - determinism_feature = feature( - name = "determinism", - enabled = True, - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "/wd4117", - "-D__DATE__=\"redacted\"", - "-D__TIMESTAMP__=\"redacted\"", - "-D__TIME__=\"redacted\"", - ], - ), - ], - ), - ], - ) - - sysroot_feature = feature( - name = "sysroot", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = ["--sysroot=%{sysroot}"], - iterate_over = "sysroot", - expand_if_available = "sysroot", - ), - ], - ), - ], - ) - - unfiltered_compile_flags_feature = feature( - name = "unfiltered_compile_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - flags = ["%{unfiltered_compile_flags}"], - iterate_over = "unfiltered_compile_flags", - expand_if_available = "unfiltered_compile_flags", - ), - ], - ), - ], - ) - - copy_dynamic_libraries_to_binary_feature = feature(name = "copy_dynamic_libraries_to_binary") - - input_param_flags_feature = feature( - name = "input_param_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = ["/IMPLIB:%{interface_library_output_path}"], - expand_if_available = "interface_library_output_path", - ), - ], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["%{libopts}"], - iterate_over = "libopts", - expand_if_available = "libopts", - ), - ], - ), - flag_set( - actions = all_link_actions + - [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link", - flag_groups = [ - flag_group( - iterate_over = "libraries_to_link.object_files", - flag_groups = [flag_group(flags = ["%{libraries_to_link.object_files}"])], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file_group", - ), - ), - flag_group( - flag_groups = [flag_group(flags = ["%{libraries_to_link.name}"])], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "object_file", - ), - ), - flag_group( - flag_groups = [flag_group(flags = ["%{libraries_to_link.name}"])], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "interface_library", - ), - ), - flag_group( - flag_groups = [ - flag_group( - flags = ["%{libraries_to_link.name}"], - expand_if_false = "libraries_to_link.is_whole_archive", - ), - flag_group( - flags = ["/WHOLEARCHIVE:%{libraries_to_link.name}"], - expand_if_true = "libraries_to_link.is_whole_archive", - ), - ], - expand_if_equal = variable_with_value( - name = "libraries_to_link.type", - value = "static_library", - ), - ), - ], - expand_if_available = "libraries_to_link", - ), - ], - ), - ], - ) - - fastbuild_feature = feature( - name = "fastbuild", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/Od", "/Z7"])], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["", "/INCREMENTAL:NO"], - ), - ], - ), - ], - implies = ["generate_pdb_file"], - ) - - user_compile_flags_feature = feature( - name = "user_compile_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - flags = ["%{user_compile_flags}"], - iterate_over = "user_compile_flags", - expand_if_available = "user_compile_flags", - ), - ], - ), - ], - ) - - archiver_flags_feature = feature( - name = "archiver_flags", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group( - flags = ["/OUT:%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - ], - ) - - default_link_flags_feature = feature( - name = "default_link_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["/MACHINE:X64"])], - ), - ], - ) - - static_link_msvcrt_feature = feature(name = "static_link_msvcrt") - - dynamic_link_msvcrt_debug_feature = feature( - name = "dynamic_link_msvcrt_debug", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/MDd"])], - ), - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["/DEFAULTLIB:msvcrtd.lib"])], - ), - ], - requires = [feature_set(features = ["dbg"])], - ) - - dbg_feature = feature( - name = "dbg", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/Od", "/Z7"])], - ), - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["", "/INCREMENTAL:NO"], - ), - ], - ), - ], - implies = ["generate_pdb_file"], - ) - - opt_feature = feature( - name = "opt", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/O2"])], - ), - ], - implies = ["frame_pointer"], - ) - - supports_interface_shared_libraries_feature = feature( - name = "supports_interface_shared_libraries", - enabled = True, - ) - - user_link_flags_feature = feature( - name = "user_link_flags", - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["%{user_link_flags}"], - iterate_over = "user_link_flags", - expand_if_available = "user_link_flags", - ), - ], - ), - ], - ) - - default_compile_flags_feature = feature( - name = "default_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = [ - "/DCOMPILER_MSVC", - "/DNOMINMAX", - "/D_WIN32_WINNT=0x0601", - "/D_CRT_SECURE_NO_DEPRECATE", - "/D_CRT_SECURE_NO_WARNINGS", - "/bigobj", - "/Zm500", - "/EHsc", - "/wd4351", - "/wd4291", - "/wd4250", - "/wd4996", - ], - ), - ], - ), - ], - ) - - msvc_compile_env_feature = feature( - name = "msvc_compile_env", - env_sets = [ - env_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ], - env_entries = [env_entry(key = "INCLUDE", value = "")], - ), - ], - ) - - preprocessor_defines_feature = feature( - name = "preprocessor_defines", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group( - flags = ["/D%{preprocessor_defines}"], - iterate_over = "preprocessor_defines", - ), - ], - ), - ], - ) - - generate_pdb_file_feature = feature( - name = "generate_pdb_file", - requires = [ - feature_set(features = ["dbg"]), - feature_set(features = ["fastbuild"]), - ], - ) - - output_execpath_flags_feature = feature( - name = "output_execpath_flags", - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["/OUT:%{output_execpath}"], - expand_if_available = "output_execpath", - ), - ], - ), - ], - ) - - dynamic_link_msvcrt_no_debug_feature = feature( - name = "dynamic_link_msvcrt_no_debug", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/MD"])], - ), - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["/DEFAULTLIB:msvcrt.lib"])], - ), - ], - requires = [ - feature_set(features = ["fastbuild"]), - feature_set(features = ["opt"]), - ], - ) - - disable_assertions_feature = feature( - name = "disable_assertions", - enabled = True, - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/DNDEBUG"])], - with_features = [with_feature_set(features = ["opt"])], - ), - ], - ) - - has_configured_linker_path_feature = feature(name = "has_configured_linker_path") - - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - - no_stripping_feature = feature(name = "no_stripping") - - linker_param_file_feature = feature( - name = "linker_param_file", - flag_sets = [ - flag_set( - actions = all_link_actions + - [ACTION_NAMES.cpp_link_static_library], - flag_groups = [ - flag_group( - flags = ["@%{linker_param_file}"], - expand_if_available = "linker_param_file", - ), - ], - ), - ], - ) - - ignore_noisy_warnings_feature = feature( - name = "ignore_noisy_warnings", - enabled = True, - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.cpp_link_static_library], - flag_groups = [flag_group(flags = ["/ignore:4221"])], - ), - ], - ) - - no_legacy_features_feature = feature(name = "no_legacy_features") - - parse_showincludes_feature = feature( - name = "parse_showincludes", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_header_parsing, - ], - flag_groups = [flag_group(flags = ["/showIncludes"])], - ), - ], - ) - - static_link_msvcrt_no_debug_feature = feature( - name = "static_link_msvcrt_no_debug", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/MT"])], - ), - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["/DEFAULTLIB:libcmt.lib"])], - ), - ], - requires = [ - feature_set(features = ["fastbuild"]), - feature_set(features = ["opt"]), - ], - ) - - treat_warnings_as_errors_feature = feature( - name = "treat_warnings_as_errors", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/WX"])], - ), - ], - ) - - windows_export_all_symbols_feature = feature(name = "windows_export_all_symbols") - - no_windows_export_all_symbols_feature = feature(name = "no_windows_export_all_symbols") - - include_paths_feature = feature( - name = "include_paths", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group( - flags = ["/I%{quote_include_paths}"], - iterate_over = "quote_include_paths", - ), - flag_group( - flags = ["/I%{include_paths}"], - iterate_over = "include_paths", - ), - flag_group( - flags = ["/I%{system_include_paths}"], - iterate_over = "system_include_paths", - ), - ], - ), - ], - ) - - linkstamps_feature = feature( - name = "linkstamps", - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["%{linkstamp_paths}"], - iterate_over = "linkstamp_paths", - expand_if_available = "linkstamp_paths", - ), - ], - ), - ], - ) - - targets_windows_feature = feature( - name = "targets_windows", - enabled = True, - implies = ["copy_dynamic_libraries_to_binary"], - ) - - linker_subsystem_flag_feature = feature( - name = "linker_subsystem_flag", - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["/SUBSYSTEM:CONSOLE"])], - ), - ], - ) - - static_link_msvcrt_debug_feature = feature( - name = "static_link_msvcrt_debug", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/MTd"])], - ), - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["/DEFAULTLIB:libcmtd.lib"])], - ), - ], - requires = [feature_set(features = ["dbg"])], - ) - - frame_pointer_feature = feature( - name = "frame_pointer", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/Oy-"])], - ), - ], - ) - - compiler_output_flags_feature = feature( - name = "compiler_output_flags", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.assemble], - flag_groups = [ - flag_group( - flag_groups = [ - flag_group( - flags = ["/Fo%{output_file}", "/Zi"], - expand_if_available = "output_file", - expand_if_not_available = "output_assembly_file", - ), - ], - expand_if_not_available = "output_preprocess_file", - ), - ], - ), - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - flag_groups = [ - flag_group( - flags = ["/Fo%{output_file}"], - expand_if_not_available = "output_preprocess_file", - ), - ], - expand_if_available = "output_file", - expand_if_not_available = "output_assembly_file", - ), - flag_group( - flag_groups = [ - flag_group( - flags = ["/Fa%{output_file}"], - expand_if_available = "output_assembly_file", - ), - ], - expand_if_available = "output_file", - ), - flag_group( - flag_groups = [ - flag_group( - flags = ["/P", "/Fi%{output_file}"], - expand_if_available = "output_preprocess_file", - ), - ], - expand_if_available = "output_file", - ), - ], - ), - ], - ) - - nologo_feature = feature( - name = "nologo", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_static_library, - ], - flag_groups = [flag_group(flags = ["/nologo"])], - ), - ], - ) - - smaller_binary_feature = feature( - name = "smaller_binary", - enabled = True, - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [flag_group(flags = ["/Gy", "/Gw"])], - with_features = [with_feature_set(features = ["opt"])], - ), - flag_set( - actions = all_link_actions, - flag_groups = [flag_group(flags = ["/OPT:ICF", "/OPT:REF"])], - with_features = [with_feature_set(features = ["opt"])], - ), - ], - ) - - compiler_input_flags_feature = feature( - name = "compiler_input_flags", - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ], - flag_groups = [ - flag_group( - flags = ["/c", "%{source_file}"], - expand_if_available = "source_file", - ), - ], - ), - ], - ) - - def_file_feature = feature( - name = "def_file", - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = [ - flag_group( - flags = ["/DEF:%{def_file_path}", "/ignore:4070"], - expand_if_available = "def_file_path", - ), - ], - ), - ], - ) - - msvc_env_feature = feature( - name = "msvc_env", - env_sets = [ - env_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_static_library, - ], - env_entries = [ - env_entry(key = "PATH", value = ""), - env_entry(key = "TMP", value = ""), - env_entry(key = "TEMP", value = ""), - ], - ), - ], - implies = ["msvc_compile_env", "msvc_link_env"], - ) - - features = [ - no_legacy_features_feature, - nologo_feature, - has_configured_linker_path_feature, - no_stripping_feature, - targets_windows_feature, - copy_dynamic_libraries_to_binary_feature, - default_compile_flags_feature, - msvc_env_feature, - msvc_compile_env_feature, - msvc_link_env_feature, - include_paths_feature, - preprocessor_defines_feature, - parse_showincludes_feature, - generate_pdb_file_feature, - shared_flag_feature, - linkstamps_feature, - output_execpath_flags_feature, - archiver_flags_feature, - input_param_flags_feature, - linker_subsystem_flag_feature, - user_link_flags_feature, - default_link_flags_feature, - linker_param_file_feature, - static_link_msvcrt_feature, - static_link_msvcrt_no_debug_feature, - dynamic_link_msvcrt_no_debug_feature, - static_link_msvcrt_debug_feature, - dynamic_link_msvcrt_debug_feature, - dbg_feature, - fastbuild_feature, - opt_feature, - frame_pointer_feature, - disable_assertions_feature, - determinism_feature, - treat_warnings_as_errors_feature, - smaller_binary_feature, - ignore_noisy_warnings_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - compiler_output_flags_feature, - compiler_input_flags_feature, - def_file_feature, - windows_export_all_symbols_feature, - no_windows_export_all_symbols_feature, - supports_dynamic_linker_feature, - supports_interface_shared_libraries_feature, - ] - - artifact_name_patterns = [ - artifact_name_pattern( - category_name = "object_file", - prefix = "", - extension = ".obj", - ), - artifact_name_pattern( - category_name = "static_library", - prefix = "", - extension = ".lib", - ), - artifact_name_pattern( - category_name = "alwayslink_static_library", - prefix = "", - extension = ".lo.lib", - ), - artifact_name_pattern( - category_name = "executable", - prefix = "", - extension = ".exe", - ), - artifact_name_pattern( - category_name = "dynamic_library", - prefix = "", - extension = ".dll", - ), - artifact_name_pattern( - category_name = "interface_library", - prefix = "", - extension = ".if.lib", - ), - ] - - make_variables = [] - - tool_paths = [ - tool_path(name = "ar", path = ""), - tool_path(name = "ml", path = ""), - tool_path(name = "cpp", path = ""), - tool_path(name = "gcc", path = ""), - tool_path(name = "gcov", path = "wrapper/bin/msvc_nop.bat"), - tool_path(name = "ld", path = ""), - tool_path(name = "nm", path = "wrapper/bin/msvc_nop.bat"), - tool_path( - name = "objcopy", - path = "wrapper/bin/msvc_nop.bat", - ), - tool_path( - name = "objdump", - path = "wrapper/bin/msvc_nop.bat", - ), - tool_path( - name = "strip", - path = "wrapper/bin/msvc_nop.bat", - ), - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = None, - ) - -def _windows_msys_mingw_impl(ctx): - toolchain_identifier = "msys_x64_mingw" - host_system_name = "local" - target_system_name = "local" - target_cpu = "x64_windows" - target_libc = "mingw" - compiler = "mingw-gcc" - abi_version = "local" - abi_libc_version = "local" - cc_target_os = None - builtin_sysroot = None - action_configs = [] - - targets_windows_feature = feature( - name = "targets_windows", - implies = ["copy_dynamic_libraries_to_binary"], - enabled = True, - ) - - copy_dynamic_libraries_to_binary_feature = feature(name = "copy_dynamic_libraries_to_binary") - - gcc_env_feature = feature( - name = "gcc_env", - enabled = True, - env_sets = [ - env_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_static_library, - ], - env_entries = [ - env_entry(key = "PATH", value = "NOT_USED"), - ], - ), - ], - ) - - msys_mingw_flags = [ - ] - msys_mingw_link_flags = [ - ] - - default_compile_flags_feature = feature( - name = "default_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - ), - flag_set( - actions = [ - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = ([flag_group(flags = msys_mingw_flags)] if msys_mingw_flags else []), - ), - ], - ) - - default_link_flags_feature = feature( - name = "default_link_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = ([flag_group(flags = msys_mingw_link_flags)] if msys_mingw_link_flags else []), - ), - ], - ) - - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - - features = [ - targets_windows_feature, - copy_dynamic_libraries_to_binary_feature, - gcc_env_feature, - default_compile_flags_feature, - default_link_flags_feature, - supports_dynamic_linker_feature, - ] - - cxx_builtin_include_directories = [ - ] - - artifact_name_patterns = [ - artifact_name_pattern( - category_name = "executable", - prefix = "", - extension = ".exe", - ), - ] - - make_variables = [] - tool_paths = [ - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = cc_target_os, - ) - -def _armeabi_impl(ctx): - toolchain_identifier = "stub_armeabi-v7a" - host_system_name = "armeabi-v7a" - target_system_name = "armeabi-v7a" - target_cpu = "armeabi-v7a" - target_libc = "armeabi-v7a" - compiler = "compiler" - abi_version = "armeabi-v7a" - abi_libc_version = "armeabi-v7a" - cc_target_os = None - builtin_sysroot = None - action_configs = [] - - supports_pic_feature = feature(name = "supports_pic", enabled = True) - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - features = [supports_dynamic_linker_feature, supports_pic_feature] - - cxx_builtin_include_directories = [] - artifact_name_patterns = [] - make_variables = [] - - tool_paths = [ - tool_path(name = "ar", path = "/bin/false"), - tool_path(name = "compat-ld", path = "/bin/false"), - tool_path(name = "cpp", path = "/bin/false"), - tool_path(name = "dwp", path = "/bin/false"), - tool_path(name = "gcc", path = "/bin/false"), - tool_path(name = "gcov", path = "/bin/false"), - tool_path(name = "ld", path = "/bin/false"), - tool_path(name = "nm", path = "/bin/false"), - tool_path(name = "objcopy", path = "/bin/false"), - tool_path(name = "objdump", path = "/bin/false"), - tool_path(name = "strip", path = "/bin/false"), - ] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = toolchain_identifier, - host_system_name = host_system_name, - target_system_name = target_system_name, - target_cpu = target_cpu, - target_libc = target_libc, - compiler = compiler, - abi_version = abi_version, - abi_libc_version = abi_libc_version, - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = builtin_sysroot, - cc_target_os = cc_target_os, - ) - -def _impl(ctx): - if ctx.attr.cpu == "armeabi-v7a": - return _armeabi_impl(ctx) - elif ctx.attr.cpu == "x64_windows" and ctx.attr.compiler == "msvc-cl": - return _windows_msvc_impl(ctx) - elif ctx.attr.cpu == "x64_windows" and ctx.attr.compiler == "mingw-gcc": - return _windows_msys_mingw_impl(ctx) - - tool_paths = [ - tool_path(name = "ar", path = "/usr/bin/ar"), - tool_path(name = "ld", path = "/usr/bin/ld"), - tool_path(name = "cpp", path = "/usr/bin/cpp"), - tool_path(name = "gcc", path = "/dt9/usr/bin/gcc"), - tool_path(name = "dwp", path = "/usr/bin/dwp"), - tool_path(name = "gcov", path = "/usr/bin/gcov"), - tool_path(name = "nm", path = "/usr/bin/nm"), - tool_path(name = "objcopy", path = "/usr/bin/objcopy"), - tool_path(name = "objdump", path = "/usr/bin/objdump"), - tool_path(name = "strip", path = "/usr/bin/strip"), - ] - - cxx_builtin_include_directories = [ - "/dt9/usr/lib/gcc/x86_64-pc-linux-gnu/9/include", - "/dt9/usr/lib/gcc/x86_64-pc-linux-gnu/9/include-fixed", - "/dt9/usr/include", - "/dt9/usr/include/c++/7", - "/dt9/usr/include/c++/9/x86_64-pc-linux-gnu", - "/dt9/usr/include/c++/9/backward", - ] - - action_configs = [] - - compile_flags = [ - "-U_FORTIFY_SOURCE", - "-fstack-protector", - "-Wall", - "-Wunused-but-set-parameter", - "-Wno-free-nonheap-object", - "-fno-omit-frame-pointer", - ] - - dbg_compile_flags = [ - "-g", - ] - - opt_compile_flags = [ - "-g0", - "-O2", - "-D_FORTIFY_SOURCE=1", - "-DNDEBUG", - "-ffunction-sections", - "-fdata-sections", - ] - - cxx_flags = [ - "-std=c++0x", - ] - - link_flags = [ - "-fuse-ld=gold", - "-Wl,-no-as-needed", - "-Wl,-z,relro,-z,now", - "-B/dt9/usr/bin", - "-pass-exit-codes", - "-lstdc++", - "-lm", - ] - - opt_link_flags = [ - "-Wl,--gc-sections", - ] - - unfiltered_compile_flags = [ - "-fno-canonical-system-headers", - "-Wno-builtin-macro-redefined", - "-D__DATE__=\"redacted\"", - "-D__TIMESTAMP__=\"redacted\"", - "-D__TIME__=\"redacted\"", - ] - - targets_windows_feature = feature( - name = "targets_windows", - implies = ["copy_dynamic_libraries_to_binary"], - enabled = True, - ) - - copy_dynamic_libraries_to_binary_feature = feature(name = "copy_dynamic_libraries_to_binary") - - gcc_env_feature = feature( - name = "gcc_env", - enabled = True, - env_sets = [ - env_set( - actions = [ - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_static_library, - ], - env_entries = [ - env_entry(key = "PATH", value = "NOT_USED"), - ], - ), - ], - ) - - windows_features = [ - targets_windows_feature, - copy_dynamic_libraries_to_binary_feature, - gcc_env_feature, - ] - - coverage_feature = feature( - name = "coverage", - provides = ["profile"], - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ], - flag_groups = [ - flag_group(flags = ["--coverage"]), - ], - ), - flag_set( - actions = [ - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ACTION_NAMES.cpp_link_executable, - ], - flag_groups = [ - flag_group(flags = ["--coverage"]), - ], - ), - ], - ) - - supports_pic_feature = feature( - name = "supports_pic", - enabled = True, - ) - supports_start_end_lib_feature = feature( - name = "supports_start_end_lib", - enabled = True, - ) - - default_compile_flags_feature = feature( - name = "default_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = ([flag_group(flags = compile_flags)] if compile_flags else []), - ), - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = ([flag_group(flags = dbg_compile_flags)] if dbg_compile_flags else []), - with_features = [with_feature_set(features = ["dbg"])], - ), - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = ([flag_group(flags = opt_compile_flags)] if opt_compile_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - flag_set( - actions = [ - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = ([flag_group(flags = cxx_flags)] if cxx_flags else []), - ), - ], - ) - - default_link_flags_feature = feature( - name = "default_link_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = all_link_actions, - flag_groups = ([flag_group(flags = link_flags)] if link_flags else []), - ), - flag_set( - actions = all_link_actions, - flag_groups = ([flag_group(flags = opt_link_flags)] if opt_link_flags else []), - with_features = [with_feature_set(features = ["opt"])], - ), - ], - ) - - dbg_feature = feature(name = "dbg") - - opt_feature = feature(name = "opt") - - sysroot_feature = feature( - name = "sysroot", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ACTION_NAMES.cpp_link_executable, - ACTION_NAMES.cpp_link_dynamic_library, - ACTION_NAMES.cpp_link_nodeps_dynamic_library, - ], - flag_groups = [ - flag_group( - flags = ["--sysroot=%{sysroot}"], - expand_if_available = "sysroot", - ), - ], - ), - ], - ) - - fdo_optimize_feature = feature( - name = "fdo_optimize", - flag_sets = [ - flag_set( - actions = [ACTION_NAMES.c_compile, ACTION_NAMES.cpp_compile], - flag_groups = [ - flag_group( - flags = [ - "-fprofile-use=%{fdo_profile_path}", - "-fprofile-correction", - ], - expand_if_available = "fdo_profile_path", - ), - ], - ), - ], - provides = ["profile"], - ) - - supports_dynamic_linker_feature = feature(name = "supports_dynamic_linker", enabled = True) - - user_compile_flags_feature = feature( - name = "user_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = [ - flag_group( - flags = ["%{user_compile_flags}"], - iterate_over = "user_compile_flags", - expand_if_available = "user_compile_flags", - ), - ], - ), - ], - ) - - unfiltered_compile_flags_feature = feature( - name = "unfiltered_compile_flags", - enabled = True, - flag_sets = [ - flag_set( - actions = [ - ACTION_NAMES.assemble, - ACTION_NAMES.preprocess_assemble, - ACTION_NAMES.linkstamp_compile, - ACTION_NAMES.c_compile, - ACTION_NAMES.cpp_compile, - ACTION_NAMES.cpp_header_parsing, - ACTION_NAMES.cpp_module_compile, - ACTION_NAMES.cpp_module_codegen, - ACTION_NAMES.lto_backend, - ACTION_NAMES.clif_match, - ], - flag_groups = ([flag_group(flags = unfiltered_compile_flags)] if unfiltered_compile_flags else []), - ), - ], - ) - - features = [ - supports_pic_feature, - supports_start_end_lib_feature, - coverage_feature, - default_compile_flags_feature, - default_link_flags_feature, - fdo_optimize_feature, - supports_dynamic_linker_feature, - dbg_feature, - opt_feature, - user_compile_flags_feature, - sysroot_feature, - unfiltered_compile_flags_feature, - ] - - artifact_name_patterns = [ - ] - - make_variables = [] - - return cc_common.create_cc_toolchain_config_info( - ctx = ctx, - features = features, - action_configs = action_configs, - artifact_name_patterns = artifact_name_patterns, - cxx_builtin_include_directories = cxx_builtin_include_directories, - toolchain_identifier = "linux_gnu_x86", - host_system_name = "i686-unknown-linux-gnu", - target_system_name = "x86_64-unknown-linux-gnu", - target_cpu = "k8", - target_libc = "glibc_2.19", - compiler = "/dt9/usr/bin/gcc", - abi_version = "gcc", - abi_libc_version = "glibc_2.19", - tool_paths = tool_paths, - make_variables = make_variables, - builtin_sysroot = "", - cc_target_os = None, - ) - -cc_toolchain_config = rule( - implementation = _impl, - attrs = { - "cpu": attr.string(mandatory = True), - "compiler": attr.string(), - }, - provides = [CcToolchainConfigInfo], -) diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/cc_wrapper.sh b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/cc_wrapper.sh deleted file mode 100755 index 176d488c..00000000 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/cc_wrapper.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash -# -# Copyright 2015 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# Ship the environment to the C++ action -# -set -eu - -# Set-up the environment - - -# Call the C++ compiler -/dt9/usr/bin/gcc "$@" diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/dummy_toolchain.bzl b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/dummy_toolchain.bzl deleted file mode 100755 index 85b3412c..00000000 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/dummy_toolchain.bzl +++ /dev/null @@ -1,23 +0,0 @@ -# pylint: disable=g-bad-file-header -# Copyright 2017 The Bazel Authors. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Starlark rule that stubs a toolchain.""" - -def _dummy_toolchain_impl(ctx): - ctx = ctx # unused argument - toolchain = platform_common.ToolchainInfo() - return [toolchain] - -dummy_toolchain = rule(_dummy_toolchain_impl, attrs = {}) diff --git a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/tools/cpp/empty.cc b/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/tools/cpp/empty.cc deleted file mode 100755 index 40da260e..00000000 --- a/third_party/toolchains/preconfig/ubuntu16.04/gcc7_manylinux2014/tools/cpp/empty.cc +++ /dev/null @@ -1,15 +0,0 @@ -// Copyright 2019 DeepMind Technologies Limited. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -int main() {}