Skip to content

Commit

Permalink
Dont filter swear words for search indexing
Browse files Browse the repository at this point in the history
  • Loading branch information
brianlovin committed Dec 17, 2017
1 parent 4ae116a commit 1827494
Show file tree
Hide file tree
Showing 3 changed files with 0 additions and 16 deletions.
4 changes: 0 additions & 4 deletions vulcan/models/text-parsing.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
// @flow
const stopword = require('stopword');
const Filter = require('bad-words');
const filter = new Filter({ placeHolder: ' ' });

import createEmojiRegex from 'emoji-regex';

Expand Down Expand Up @@ -52,5 +50,3 @@ export const withoutStopWords = (str: string): string => {
// return the string
return joined;
};

export const withoutSwearWords = (str: string): string => filter.clean(str);
11 changes: 0 additions & 11 deletions vulcan/models/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@ import { toPlainText, toState } from 'shared/draft-utils';
import {
getWordCount,
withoutStopWords,
withoutSwearWords,
onlyContainsEmoji,
} from './text-parsing';

Expand All @@ -31,12 +30,8 @@ export const dbThreadToSearchThread = (thread: DBThread): SearchThread => {

// filter out stop words
body = withoutStopWords(body);
// filter out swear words
body = withoutSwearWords(body);
// filter out stop words
title = withoutStopWords(title);
// filter out swear words
title = withoutSwearWords(title);

// algolia only supports 20kb records
// slice it down until its under 19k, leaving room for the rest of the thread data
Expand Down Expand Up @@ -96,8 +91,6 @@ const filterMessageString = (message: DBMessage): ?string => {

// filter out stop words
messageString = withoutStopWords(messageString);
// filter out swear words
messageString = withoutSwearWords(messageString);

// don't index short messages - will eliminate things like
// +1, nice, lol, cool, etc from being stored
Expand Down Expand Up @@ -145,8 +138,6 @@ export const dbUserToSearchUser = (user: DBUser): SearchUser => {
let description = user.description;
// filter out stop words
description = description && withoutStopWords(description);
// filter out swear words
description = description && withoutSwearWords(description);

return {
name: user.name,
Expand All @@ -164,8 +155,6 @@ export const dbCommunityToSearchCommunity = (
let description = community.description;
// filter out stop words
description = description && withoutStopWords(description);
// filter out swear words
description = description && withoutSwearWords(description);

return {
id: community.id,
Expand Down
1 change: 0 additions & 1 deletion vulcan/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@
},
"dependencies": {
"algoliasearch": "^3.24.7",
"bad-words": "^1.6.1",
"debug": "^2.6.8",
"emoji-regex": "^6.1.1",
"node-env-file": "^0.1.8",
Expand Down

0 comments on commit 1827494

Please sign in to comment.