Ignore references in code blocks and code cell output.

PiperOrigin-RevId: 256213223

tools/tensorflow_docs/api_generator/parser.py

@@ -115,6 +115,41 @@ def _get_raw_docstring(py_object):
   else:
     return ''
 
+
+class IgnoreLineInBlock(object):
+  """Ignores the lines in a block.
+
+  Attributes:
+    block_start: Contains the start string of a block to ignore.
+    block_end: Contains the end string of a block to ignore.
+  """
+
+  def __init__(self, block_start, block_end):
+    self._block_start = block_start
+    self._block_end = block_end
+    self._in_block = False
+
+    self._start_end_regex = re.escape(self._block_start) + r'.*' + re.escape(
+        self._block_end)
+
+  def __call__(self, line):
+    # If start and end block are on the same line, return True.
+    if re.match(self._start_end_regex, line):
+      return True
+
+    if not self._in_block:
+      if self._block_start in line:
+        self._in_block = True
+
+    elif self._block_end in line:
+      self._in_block = False
+      # True is being returned here because the last line in the block should
+      # also be ignored.
+      return True
+
+    return self._in_block
+
+
 AUTO_REFERENCE_RE = re.compile(r'`([\w\(\[\)\]\{\}.,=\s]+?)`')
 
 
@@ -178,7 +213,7 @@ def _partial_symbols(self, symbol):
     par_dict = {"keras.layers.Conv2D": "tf.keras.layers.Conv2D",
                 "layers.Conv2D": "tf.keras.layers.Conv2D"}
 
-    There should atleast be one '.' in the partial symbol generated so as to
+    There should at least be one '.' in the partial symbol generated so as to
     avoid guessing for the true symbol.
 
     Args:
@@ -263,8 +298,15 @@ def sloppy_one_ref(match):
         return match.group(0)
 
     fixed_lines = []
+
+    filters = [
+        IgnoreLineInBlock('<pre class="tfo-notebook-code-cell-output">',
+                          '{% endhtmlescape %}</pre>'),
+        IgnoreLineInBlock('```', '```')
+    ]
+
     for line in string.splitlines():
-      if not line.strip().startswith('# '):
+      if not any(filter_block(line) for filter_block in filters):
         line = re.sub(AUTO_REFERENCE_RE, sloppy_one_ref, line)
       fixed_lines.append(line)
 

tools/tensorflow_docs/api_generator/parser_test.py

@@ -22,6 +22,7 @@
 import functools
 import os
 import tempfile
+import textwrap
 import unittest
 
 from absl.testing import absltest
@@ -913,8 +914,6 @@ class TestParseDocstring(absltest.TestCase):
   def test_split_title_blocks(self):
     docstring_parts = parser.TitleBlock.split_string(RELU_DOC)
 
-    print(docstring_parts)
-
     self.assertLen(docstring_parts, 7)
 
     args = docstring_parts[1]
@@ -973,6 +972,79 @@ def test_partial_symbol_references(self, string, link):
     self.assertEqual(expected, ref_string)
 
 
+class TestIgnoreLineInBlock(parameterized.TestCase):
+
+  @parameterized.named_parameters(
+      ('ignore_backticks',
+       ['```'],
+       ['```'],
+       '```\nFiller\n```\n```Same line```\n```python\nDowner\n```'),
+
+      ('ignore_code_cell_output',
+       ['<pre>{% html %}'],
+       ['{% endhtml %}</pre>'],
+       '<pre>{% html %}\nOutput\nmultiline{% endhtml %}</pre>'),
+
+      ('ignore_backticks_and_cell_output',
+       ['<pre>{% html %}', '```'],
+       ['{% endhtml %}</pre>', '```'],
+       ('```\nFiller\n```\n```Same line```\n<pre>{% html %}\nOutput\nmultiline'
+        '{% endhtml %}</pre>\n```python\nDowner\n```'))
+      )
+  def test_ignore_lines(self, block_start, block_end, expected_ignored_lines):
+
+    text = textwrap.dedent('''\
+    ```
+    Filler
+    ```
+
+    ```Same line```
+
+    <pre>{% html %}
+    Output
+    multiline{% endhtml %}</pre>
+
+    ```python
+    Downer
+    ```
+    ''')
+
+    filters = [parser.IgnoreLineInBlock(start, end)
+               for start, end in zip(block_start, block_end)]
+
+    ignored_lines = []
+    for line in text.splitlines():
+      if any(filter_block(line) for filter_block in filters):
+        ignored_lines.append(line)
+
+    self.assertEqual('\n'.join(ignored_lines), expected_ignored_lines)
+
+  def test_clean_text(self):
+    text = textwrap.dedent('''\
+    ```
+    Ignore lines here.
+    ```
+    Useful information.
+    Don't ignore.
+    ```python
+    Ignore here too.
+    ```
+    Stuff.
+    ```Not useful.```
+    ''')
+
+    filters = [parser.IgnoreLineInBlock('```', '```')]
+
+    clean_text = []
+    for line in text.splitlines():
+      if not any(filter_block(line) for filter_block in filters):
+        clean_text.append(line)
+
+    expected_clean_text = 'Useful information.\nDon\'t ignore.\nStuff.'
+
+    self.assertEqual('\n'.join(clean_text), expected_clean_text)
+
+
 class TestGenerateSignature(absltest.TestCase):
 
   def test_known_object(self):