File system implementation for Google Cloud Storage.

This code implements a file system for file paths starting with gs:// using the HTTP API to Google Cloud Storage. No authentication is implemented yet, so only GCS objects with public access can be used.
Change: 122126085

configure

@@ -35,6 +35,37 @@ while true; do
   # Retry
 done
 
+while [ "$TF_NEED_GCP" == "" ]; do
+  read -p "Do you wish to build TensorFlow with "\
+"Google Cloud Platform support? [y/N] " INPUT
+  case $INPUT in
+    [Yy]* ) echo "Google Cloud Platform support will be enabled for "\
+"TensorFlow"; TF_NEED_GCP=1;;
+    [Nn]* ) echo "No Google Cloud Platform support will be enabled for "\
+"TensorFlow"; TF_NEED_GCP=0;;
+    "" ) echo "No Google Cloud Platform support will be enabled for "\
+"TensorFlow"; TF_NEED_GCP=0;;
+    * ) echo "Invalid selection: " $INPUT;;
+  esac
+done
+
+if [ "$TF_NEED_GCP" == "1" ]; then
+
+  ## Verify that libcurl header files are available.
+  # Only check Linux, since on MacOS the header files are installed with XCode.
+  if [[ $(uname -a) =~ Linux ]] && [[ ! -f "/usr/include/curl/curl.h" ]]; then
+    echo "ERROR: It appears that the development version of libcurl is not "\
+"available. Please install the libcurl3-dev package."
+    exit 1
+  fi
+
+  # Update Bazel build configuration.
+  perl -pi -e "s,WITH_GCP_SUPPORT = (False|True),WITH_GCP_SUPPORT = True,s" tensorflow/core/platform/default/build_config.bzl
+else
+  # Update Bazel build configuration.
+  perl -pi -e "s,WITH_GCP_SUPPORT = (False|True),WITH_GCP_SUPPORT = False,s" tensorflow/core/platform/default/build_config.bzl
+fi
+
 ## Find swig path
 if [ -z "$SWIG_PATH" ]; then
   SWIG_PATH=`type -p swig 2> /dev/null`

jsoncpp.BUILD

@@ -0,0 +1,34 @@
+licenses(["notice"])  # MIT
+
+JSON_HEADERS = [
+    "include/json/assertions.h",
+    "include/json/autolink.h",
+    "include/json/config.h",
+    "include/json/features.h",
+    "include/json/forwards.h",
+    "include/json/json.h",
+    "src/lib_json/json_batchallocator.h",
+    "include/json/reader.h",
+    "include/json/value.h",
+    "include/json/writer.h",
+]
+
+JSON_SOURCES = [
+    "src/lib_json/json_reader.cpp",
+    "src/lib_json/json_value.cpp",
+    "src/lib_json/json_writer.cpp",
+    "src/lib_json/json_tool.h",
+]
+
+INLINE_SOURCES = [
+    "src/lib_json/json_valueiterator.inl",
+]
+
+cc_library(
+    name = "jsoncpp",
+    srcs = JSON_SOURCES,
+    hdrs = JSON_HEADERS,
+    includes = ["include"],
+    textual_hdrs = INLINE_SOURCES,
+    visibility = ["//visibility:public"],
+)

tensorflow/BUILD

@@ -91,6 +91,7 @@ filegroup(
         "//tensorflow/core/distributed_runtime/rpc:all_files",
         "//tensorflow/core/kernels:all_files",
         "//tensorflow/core/ops/compat:all_files",
+        "//tensorflow/core/platform/cloud:all_files",
         "//tensorflow/core/platform/default/build_config:all_files",
         "//tensorflow/core/util/ctc:all_files",
         "//tensorflow/examples/android:all_files",

tensorflow/core/BUILD

@@ -67,6 +67,7 @@ load(
     "tf_proto_library",
     "tf_proto_library_cc",
     "tf_additional_lib_srcs",
+    "tf_additional_lib_deps",
     "tf_additional_stream_executor_srcs",
     "tf_additional_test_deps",
     "tf_additional_test_srcs",
@@ -995,9 +996,9 @@ tf_cuda_library(
         ":lib_internal",
         ":proto_text",
         ":protos_all_cc",
-        "//tensorflow/core/kernels:required",
         "//third_party/eigen3",
-    ],
+        "//tensorflow/core/kernels:required",
+    ] + tf_additional_lib_deps(),
     alwayslink = 1,
 )
 

tensorflow/core/platform/cloud/BUILD

@@ -0,0 +1,79 @@
+# Description:
+# Cloud file system implementation.
+
+package(
+    default_visibility = ["//visibility:private"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_test",
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
+cc_library(
+    name = "gcs_file_system",
+    srcs = [
+        "gcs_file_system.cc",
+    ],
+    hdrs = [
+        "gcs_file_system.h",
+    ],
+    linkstatic = 1,  # Needed since alwayslink is broken in bazel b/27630669
+    visibility = ["//visibility:public"],
+    deps = [
+        "@jsoncpp_git//:jsoncpp",
+        ":http_request",
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:lib_internal",
+    ],
+    alwayslink = 1,
+)
+
+cc_library(
+    name = "http_request",
+    srcs = [
+        "http_request.cc",
+    ],
+    hdrs = [
+        "http_request.h",
+    ],
+    deps = [
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+tf_cc_test(
+    name = "gcs_file_system_test",
+    size = "small",
+    deps = [
+        ":gcs_file_system",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+tf_cc_test(
+    name = "http_request_test",
+    size = "small",
+    deps = [
+        ":http_request",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)