Add HTML tags removing to default preprocessing pipeline (jbesomi#192)

* Add HTML tags removing to default preprocessing pipeline * Update docstring and add more tests to default cleaning pipeline * Update `get_default_pipeline` docstring
sofean-mso · Oct 21, 2020 · c408c67 · c408c67
1 parent a1c03b3
commit c408c67
Show file tree

Hide file tree

Showing 2 changed files with 30 additions and 10 deletions.
diff --git a/tests/test_preprocessing.py b/tests/test_preprocessing.py
@@ -114,6 +114,23 @@ def test_pipeline_stopwords(self):
  pipeline = [preprocessing.lowercase, preprocessing.remove_stopwords]
  self.assertEqual(preprocessing.clean(s, pipeline=pipeline), s_true)
 
+ def test_pipeline_default(self):
+ s = pd.Series(
+ [
+ "Amazon! < br />< br /> If I was going to order any soft drink online, it would be Diet Coke with Lime",
+ pd.NA,
+ "-1234. Mère, Françoise, noël",
+ ]
+ )
+ s_true = pd.Series(
+ [
+ "amazon going order soft drink online would diet coke lime",
+ "",
+ "mere francoise noel",
+ ]
+ )
+ self.assertEqual(preprocessing.clean(s), s_true)
+
  """
  Test stopwords.
  """
@@ -151,8 +168,8 @@ def test_stopwords_are_set(self):
  """
 
  def test_remove_html_tags(self):
- s = pd.Series("<html>remove <br>html</br> tags<html> &nbsp;")
- s_true = pd.Series("remove html tags ")
+ s = pd.Series("<html>remove <br>html</br> tags<html> &nbsp; < br />< br />")
+ s_true = pd.Series("remove html tags  ")
  self.assertEqual(preprocessing.remove_html_tags(s), s_true)
 
  """

diff --git a/texthero/preprocessing.py b/texthero/preprocessing.py
@@ -389,15 +389,17 @@ def get_default_pipeline() -> List[Callable[[pd.Series], pd.Series]]:
  1. :meth:`texthero.preprocessing.fillna`
  2. :meth:`texthero.preprocessing.lowercase`
  3. :meth:`texthero.preprocessing.remove_digits`
- 4. :meth:`texthero.preprocessing.remove_punctuation`
- 5. :meth:`texthero.preprocessing.remove_diacritics`
- 6. :meth:`texthero.preprocessing.remove_stopwords`
- 7. :meth:`texthero.preprocessing.remove_whitespace`
+ 4. :meth:`texthero.preprocessing.remove_html_tags`
+ 5. :meth:`texthero.preprocessing.remove_punctuation`
+ 6. :meth:`texthero.preprocessing.remove_diacritics`
+ 7. :meth:`texthero.preprocessing.remove_stopwords`
+ 8. :meth:`texthero.preprocessing.remove_whitespace`
  """
  return [
  fillna,
  lowercase,
  remove_digits,
+ remove_html_tags,
  remove_punctuation,
  remove_diacritics,
  remove_stopwords,
@@ -415,10 +417,11 @@ def clean(s: TextSeries, pipeline=None) -> TextSeries:
  1. :meth:`texthero.preprocessing.fillna`
  2. :meth:`texthero.preprocessing.lowercase`
  3. :meth:`texthero.preprocessing.remove_digits`
- 4. :meth:`texthero.preprocessing.remove_punctuation`
- 5. :meth:`texthero.preprocessing.remove_diacritics`
- 6. :meth:`texthero.preprocessing.remove_stopwords`
- 7. :meth:`texthero.preprocessing.remove_whitespace`
+ 4. :meth:`texthero.preprocessing.remove_html_tags`
+ 5. :meth:`texthero.preprocessing.remove_punctuation`
+ 6. :meth:`texthero.preprocessing.remove_diacritics`
+ 7. :meth:`texthero.preprocessing.remove_stopwords`
+ 8. :meth:`texthero.preprocessing.remove_whitespace`
 
  Parameters
  ----------