Merge remote-tracking branch 'staging/123221709' into 123496073

avro.BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+prefix_dir = "avro-cpp-1.8.0"
+
+cc_library(
+    name = "avrocpp",
+    srcs = glob(
+        [
+            prefix_dir + "/impl/**/*.cc",
+            prefix_dir + "/impl/**/*.hh",
+        ],
+        exclude = [
+            prefix_dir + "/impl/avrogencpp.cc",
+        ],
+    ),
+    hdrs = glob([prefix_dir + "/api/**/*.hh"]),
+    includes = [prefix_dir + "/api"],
+    deps = [
+        "@boost_archive//:boost",
+        "@boost_archive//:filesystem",
+        "@boost_archive//:iostreams",
+        "@boost_archive//:system",
+    ],
+)
+
+cc_binary(
+    name = "avrogencpp",
+    srcs = [prefix_dir + "/impl/avrogencpp.cc"],
+    deps = [
+        ":avrocpp",
+        "@boost_archive//:program_options",
+    ],
+)

boost.BUILD

@@ -0,0 +1,58 @@
+# Description:
+#   The Boost library collection (http://www.boost.org)
+#
+# Most Boost libraries are header-only, in which case you only need to depend
+# on :boost. If you need one of the libraries that has a separately-compiled
+# implementation, depend on the appropriate libs rule.
+
+# This is only needed for Avro.
+package(default_visibility = ["@avro_archive//:__subpackages__"])
+
+licenses(["notice"])  # Boost software license
+
+prefix_dir = "boost_1_61_0"
+
+cc_library(
+    name = "boost",
+    hdrs = glob([
+        prefix_dir + "/boost/**/*.hpp",
+        prefix_dir + "/boost/**/*.h",
+        prefix_dir + "/boost/**/*.ipp",
+    ]),
+    includes = [prefix_dir],
+)
+
+cc_library(
+    name = "filesystem",
+    srcs = glob([prefix_dir + "/libs/filesystem/src/*.cpp"]),
+    deps = [
+        ":boost",
+        ":system",
+    ],
+)
+
+cc_library(
+    name = "iostreams",
+    srcs = glob([prefix_dir + "/libs/iostreams/src/*.cpp"]),
+    deps = [
+        ":boost",
+        "@bzip2_archive//:bz2lib",
+        "@zlib_archive//:zlib",
+    ],
+)
+
+cc_library(
+    name = "program_options",
+    srcs = glob([prefix_dir + "/libs/program_options/src/*.cpp"]),
+    deps = [
+        ":boost",
+    ],
+)
+
+cc_library(
+    name = "system",
+    srcs = glob([prefix_dir + "/libs/system/src/*.cpp"]),
+    deps = [
+        ":boost",
+    ],
+)

bzip2.BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # BSD derivative
+
+prefix_dir = "bzip2-1.0.6"
+
+BZ2LIB_SRCS = [
+    # these are in the same order as their corresponding .o files are in OBJS in
+    # Makefile (rather than lexicographic order) for easy comparison (that they
+    # are identical).
+    "blocksort.c",
+    "huffman.c",
+    "crctable.c",
+    "randtable.c",
+    "compress.c",
+    "decompress.c",
+    "bzlib.c",
+]
+
+cc_library(
+    name = "bz2lib",
+    srcs = [prefix_dir + "/" + source for source in BZ2LIB_SRCS] +
+        [prefix_dir + "/bzlib_private.h"],
+    hdrs = [prefix_dir + "/bzlib.h"],
+    includes = [prefix_dir],
+)
+
+cc_binary(
+    name = "bzip2",
+    srcs = [
+        "bzip2.c",
+    ],
+    deps = [
+        ":bz2lib",
+    ],
+)

tensorflow/contrib/avro/BUILD

@@ -0,0 +1,30 @@
+# Description:
+#   Contains ops for reading and writing Apache Avro files.
+#   (https://avro.apache.org/)
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+package(default_visibility = ["//tensorflow:__subpackages__"])
+
+load("//third_party/avro:build_defs.bzl", "avro_gen_cpp")
+
+avro_gen_cpp(
+    name = "example_h",
+    srcs = ["example.json"],
+    outs = ["example.h"],
+    namespace = "tensorflow",
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)

tensorflow/contrib/avro/README.md

@@ -0,0 +1,4 @@
+# TensorFlow Avro support
+
+This directory contains code for reading and writing
+[Apache Avro](https://avro.apache.org/) data in TensorFlow.

tensorflow/contrib/avro/example.json

@@ -0,0 +1,71 @@
+{
+  "type": "record",
+  "name": "Example",
+  "fields": [
+    {
+      "name": "features",
+      "type": {
+        "type": "record",
+        "name": "Features",
+        "fields": [
+          {
+            "name": "feature",
+            "type": {
+              "type": "map",
+              "values": {
+                "type": "record",
+                "name": "Feature",
+                "fields": [
+                  {
+                    "name": "values",
+                    "type": [
+                      {
+                        "type": "record",
+                        "name": "BytesList",
+                        "fields": [
+                          {
+                            "name": "value",
+                            "type": {
+                              "type": "array",
+                              "items": "bytes"
+                            }
+                          }
+                        ]
+                      },
+                      {
+                        "type": "record",
+                        "name": "FloatList",
+                        "fields": [
+                          {
+                            "name": "value",
+                            "type": {
+                              "type": "array",
+                              "items": "float"
+                            }
+                          }
+                        ]
+                      },
+                      {
+                        "type": "record",
+                        "name": "Int64List",
+                        "fields": [
+                          {
+                            "name": "value",
+                            "type": {
+                              "type": "array",
+                              "items": "long"
+                            }
+                          }
+                        ]
+                      }
+                    ]
+                  }
+                ]
+              }
+            }
+          }
+        ]
+      }
+    }
+  ]
+}

tensorflow/contrib/layers/python/layers/layers.py

@@ -50,6 +50,7 @@
            'fully_connected',
            'max_pool2d',
            'one_hot_encoding',
+           'stack',
            'legacy_convolution2d',
            'legacy_fully_connected',
            'legacy_linear',
@@ -600,6 +601,49 @@ def _apply_activation(y, activation_fn, output_collections):
   return y
 
 
+def stack(inputs, layer, stack_args, **kwargs):
+  """Builds a stack of layers by applying layer repeatedly using stack_args.
+
+  `stack` allows you to repeatedly apply the same operation with different
+  arguments `stack_args[i]`. For each application of the layer, `stack` creates
+  a new scope appended with an increasing number. For example:
+
+  ```python
+    stack(x, fully_connected, [32, 64, 128], scope='fc')
+    # It is equivalent to:
+
+    x = fully_connected(x, 32, scope='fc/fc_1')
+    x = fully_connected(x, 64, scope='fc/fc_2')
+    x = fully_connected(x, 128, scope='fc/fc_3')
+  ```
+
+  Args:
+    inputs: A `Tensor` suitable for layer.
+    layer: A layer(inputs, *args, **kwargs)
+    stack_args: A list/tuple of parameters for each call of layer.
+    **kwargs: Extra kwargs for the layer.
+
+  Returns:
+    a `Tensor` result of applying the stacked layers.
+
+  Raises:
+    ValueError: if the op is unknown or wrong.
+  """
+  scope = kwargs.pop('scope', None)
+  if not isinstance(stack_args, (list, tuple)):
+    raise ValueError('stack_args need to be a list or tuple')
+  with variable_scope.variable_op_scope([inputs], scope, 'Stack'):
+    outputs = inputs
+    scope = scope or layer.__name__
+    for i in range(len(stack_args)):
+      kwargs['scope'] = scope + '_' + str(i+1)
+      layer_args = stack_args[i]
+      if not isinstance(layer_args, (list, tuple)):
+        layer_args = [layer_args]
+      outputs = layer(outputs, *layer_args, **kwargs)
+    return outputs
+
+
 def legacy_fully_connected(x,
                            num_output_units,
                            activation_fn=None,

tensorflow/contrib/layers/python/layers/layers_test.py

@@ -385,7 +385,7 @@ def testCreateFCWithDropout(self):
       num_elem = tf.reduce_mean(tf.to_float(output > 0))
       sess.run(tf.initialize_all_variables())
       num_elem = sess.run(num_elem)
-      self.assertLess(num_elem, 0.3)
+      self.assertLess(num_elem, 0.5)
       self.assertGreater(num_elem, 0.1)
 
 
@@ -878,6 +878,44 @@ def testOneHotEncodingInt32(self):
       self.assertAllClose(output.eval(), one_hot_labels.eval())
 
 
+class StackTests(tf.test.TestCase):
+
+  def testStackFullyConnected(self):
+    height, width = 3, 3
+    with self.test_session():
+      images = tf.random_uniform((5, height * width * 3), seed=1, name='images')
+      output = tf.contrib.layers.stack(images,
+                                       tf.contrib.layers.fully_connected,
+                                       [10, 20, 30])
+      self.assertEquals(output.op.name, 'Stack/fully_connected_3/Relu')
+      self.assertListEqual(output.get_shape().as_list(), [5, 30])
+
+  def testStackConvolution2d(self):
+    height, width = 3, 3
+    with self.test_session():
+      images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
+      output = tf.contrib.layers.stack(images,
+                                       tf.contrib.layers.convolution2d,
+                                       [10, 20, 30],
+                                       kernel_size=[3, 3],
+                                       padding='SAME')
+      self.assertEquals(output.op.name, 'Stack/convolution2d_3/Relu')
+      self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 30])
+
+  def testStackWithScope(self):
+    height, width = 3, 3
+    with self.test_session():
+      images = tf.random_uniform((5, height, width, 3), seed=1, name='images')
+      output = tf.contrib.layers.stack(images,
+                                       tf.contrib.layers.convolution2d,
+                                       [10, 20, 30],
+                                       kernel_size=[3, 3],
+                                       padding='SAME',
+                                       scope='conv1')
+      self.assertEquals(output.op.name, 'conv1/conv1_3/Relu')
+      self.assertListEqual(output.get_shape().as_list(), [5, 3, 3, 30])
+
+
 # TODO(b/28426988): Add separate tests for non-legacy versions.
 class LegacyFullyConnectedTest(tf.test.TestCase):
 

tensorflow/contrib/learn/BUILD

@@ -149,6 +149,18 @@ py_test(
     ],
 )
 
+py_test(
+    name = "test_in_memory_source",
+    size = "small",
+    srcs = ["python/learn/tests/dataframe/test_in_memory_source.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":learn",
+        "//tensorflow:tensorflow_py",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
 py_test(
     name = "test_early_stopping",
     size = "medium",

tensorflow/contrib/learn/python/learn/dataframe/__init__.py

@@ -27,7 +27,8 @@
 from tensorflow.contrib.learn.python.learn.dataframe.transform import Transform
 
 # Transforms
+from tensorflow.contrib.learn.python.learn.dataframe.transforms.in_memory_source import NumpySource
+from tensorflow.contrib.learn.python.learn.dataframe.transforms.in_memory_source import PandasSource
 from tensorflow.contrib.learn.python.learn.dataframe.transforms.reader_source import ReaderSource
 
-
 __all__ = ['Column', 'TransformedColumn', 'DataFrame', 'parameter', 'Transform']

tensorflow/contrib/learn/python/learn/dataframe/dataframe.py

@@ -148,10 +148,7 @@ def to_input_fn(self, feature_keys=None, target_keys=None):
       target_keys = []
 
     if feature_keys is None:
-      if target_keys:
-        feature_keys = self.columns() - set(target_keys)
-      else:
-        feature_keys = self.columns()
+      feature_keys = self.columns() - set(target_keys)
     else:
       in_both = set(feature_keys) & set(target_keys)
       if in_both: