Tell crawlers not to index comment permalink pages

We don't want to spend crawl budget or rank on what are essentially
duplicate pages. In case we have inbound links to these pages, we don't
want the robots.txt to prevent crawlers from accessing them.

r2/r2/lib/pages/pages.py

@@ -1745,7 +1745,13 @@ def __init__(self, link = None, comment = None, disable_comments=False,
             self.num_duplicates = num_duplicates
 
         self.show_promote_button = show_promote_button
-        robots = "noindex,nofollow" if link._deleted or link._spam else None
+        if link._deleted or link._spam:
+            robots = "noindex,nofollow"
+        elif comment:
+            # We don't want crawlers to index the comment permalink pages.
+            robots = "noindex"
+        else:
+            robots = None
 
         if 'extra_js_config' not in kw:
             kw['extra_js_config'] = {}