Skip to content

Commit

Permalink
Compile regexp's at build time. Use raw string literals for patterns …
Browse files Browse the repository at this point in the history 
…to avoid escaping
  • Loading branch information
@mrngm @svishwanath-tw
mrngm authored and svishwanath-tw committed Oct 31, 2019
1 parent dac0d04 commit 601a9bd
Show file tree
Hide file tree
Showing 5 changed files with 94 additions and 94 deletions.
34 changes: 14 additions & 20 deletions detector/filecontent_credit_card_detector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,19 @@ type CreditCardDetector struct {
creditCardRegex []*regexp.Regexp
}

var (
creditCardPatterns = []*regexp.Regexp{
regexp.MustCompile(`(?:3[47][0-9]{13})`),
regexp.MustCompile(`(?:3(?:0[0-5]|[68][0-9])[0-9]{11})`),
regexp.MustCompile(`^65[4-9][0-9]{13}|64[4-9][0-9]{13}|6011[0-9]{12}|(622(?:12[6-9]|1[3-9][0-9]|[2-8][0-9][0-9]|9[01][0-9]|92[0-5])[0-9]{10})$`),
regexp.MustCompile(`^(?:2131|1800|35\d{3})\d{11}$`),
regexp.MustCompile(`^(5018|5020|5038|6304|6759|6761|6763)[0-9]{8,15}$`),
regexp.MustCompile(`(?:(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12})`),
regexp.MustCompile(`((?:4[0-9]{12})(?:[0-9]{3})?)`),

}
)

func (detector CreditCardDetector) checkCreditCardNumber(content string) string {
if !isLuhnNumber(content) {
return ""
Expand All @@ -18,27 +31,8 @@ func (detector CreditCardDetector) checkCreditCardNumber(content string) string
return ""
}

func initPatternForCreditCard() *CreditCardDetector {

patterns := [...]string{
"(?:3[47][0-9]{13})",
"(?:3(?:0[0-5]|[68][0-9])[0-9]{11})",
"^65[4-9][0-9]{13}|64[4-9][0-9]{13}|6011[0-9]{12}|(622(?:12[6-9]|1[3-9][0-9]|[2-8][0-9][0-9]|9[01][0-9]|92[0-5])[0-9]{10})$",
"^(?:2131|1800|35\\d{3})\\d{11}$",
"^(5018|5020|5038|6304|6759|6761|6763)[0-9]{8,15}$",
"(?:(?:5[1-5][0-9]{2}|222[1-9]|22[3-9][0-9]|2[3-6][0-9]{2}|27[01][0-9]|2720)[0-9]{12})",
"((?:4[0-9]{12})(?:[0-9]{3})?)",
}

var creditCardPatterns = make([]*regexp.Regexp, len(patterns))
for i, pattern := range patterns {
creditCardPatterns[i], _ = regexp.Compile(pattern)
}
return &CreditCardDetector{creditCardPatterns}
}

func NewCreditCardDetector() *CreditCardDetector {
return initPatternForCreditCard()
return &CreditCardDetector{creditCardPatterns}
}

func isLuhnNumber(content string) bool {
Expand Down
105 changes: 53 additions & 52 deletions detector/filename_detector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,57 @@ import (
log "github.com/Sirupsen/logrus"
)

var (
filenamePatterns = []*regexp.Regexp{
regexp.MustCompile(`^.+_rsa$`),
regexp.MustCompile(`^.+_dsa.*$`),
regexp.MustCompile(`^.+_ed25519$`),
regexp.MustCompile(`^.+_ecdsa$`),
regexp.MustCompile(`^\.\w+_history$`),
regexp.MustCompile(`^.+\.pem$`),
regexp.MustCompile(`^.+\.ppk$`),
regexp.MustCompile(`^.+\.key(pair)?$`),
regexp.MustCompile(`^.+\.pkcs12$`),
regexp.MustCompile(`^.+\.pfx$`),
regexp.MustCompile(`^.+\.p12$`),
regexp.MustCompile(`^.+\.asc$`),
regexp.MustCompile(`^\.?htpasswd$`),
regexp.MustCompile(`^\.?netrc$`),
regexp.MustCompile(`^.*\.tblk$`),
regexp.MustCompile(`^.*\.ovpn$`),
regexp.MustCompile(`^.*\.kdb$`),
regexp.MustCompile(`^.*\.agilekeychain$`),
regexp.MustCompile(`^.*\.keychain$`),
regexp.MustCompile(`^.*\.key(store|ring)$`),
regexp.MustCompile(`^jenkins\.plugins\.publish_over_ssh\.BapSshPublisherPlugin.xml$`),
regexp.MustCompile(`^credentials\.xml$`),
regexp.MustCompile(`^.*\.pubxml(\.user)?$`),
regexp.MustCompile(`^\.?s3cfg$`),
regexp.MustCompile(`^\.gitrobrc$`),
regexp.MustCompile(`^\.?(bash|zsh)rc$`),
regexp.MustCompile(`^\.?(bash_|zsh_)?profile$`),
regexp.MustCompile(`^\.?(bash_|zsh_)?aliases$`),
regexp.MustCompile(`^secret_token.rb$`),
regexp.MustCompile(`^omniauth.rb$`),
regexp.MustCompile(`^carrierwave.rb$`),
regexp.MustCompile(`^schema.rb$`),
regexp.MustCompile(`^database.yml$`),
regexp.MustCompile(`^settings.py$`),
regexp.MustCompile(`^.*(config)(\.inc)?\.php$`),
regexp.MustCompile(`^LocalSettings.php$`),
regexp.MustCompile(`\.?env`),
regexp.MustCompile(`\bdump|dump\b`),
regexp.MustCompile(`\bsql|sql\b`),
regexp.MustCompile(`\bdump|dump\b`),
regexp.MustCompile(`password`),
regexp.MustCompile(`backup`),
regexp.MustCompile(`private.*key`),
regexp.MustCompile(`(oauth).*(token)`),
regexp.MustCompile(`^.*\.log$`),
regexp.MustCompile(`^\.?kwallet$`),
regexp.MustCompile(`^\.?gnucash$`),
}
)
//FileNameDetector represents tests performed against the fileName of the Additions.
//The Paths of the supplied Additions are tested against the configured patterns and if any of them match, it is logged as a failure during the run
type FileNameDetector struct {
Expand All @@ -17,61 +68,11 @@ type FileNameDetector struct {

//DefaultFileNameDetector returns a FileNameDetector that tests Additions against the pre-configured patterns
func DefaultFileNameDetector() Detector {
return NewFileNameDetector("^.+_rsa$",
"^.+_dsa.*$",
"^.+_ed25519$",
"^.+_ecdsa$",
"^\\.\\w+_history$",
"^.+\\.pem$",
"^.+\\.ppk$",
"^.+\\.key(pair)?$",
"^.+\\.pkcs12$",
"^.+\\.pfx$",
"^.+\\.p12$",
"^.+\\.asc$",
"^\\.?htpasswd$",
"^\\.?netrc$",
"^.*\\.tblk$",
"^.*\\.ovpn$",
"^.*\\.kdb$",
"^.*\\.agilekeychain$",
"^.*\\.keychain$",
"^.*\\.key(store|ring)$",
"^jenkins\\.plugins\\.publish_over_ssh\\.BapSshPublisherPlugin.xml$",
"^credentials\\.xml$",
"^.*\\.pubxml(\\.user)?$",
"^\\.?s3cfg$",
"^\\.gitrobrc$",
"^\\.?(bash|zsh)rc$",
"^\\.?(bash_|zsh_)?profile$",
"^\\.?(bash_|zsh_)?aliases$",
"^secret_token.rb$",
"^omniauth.rb$",
"^carrierwave.rb$",
"^schema.rb$",
"^database.yml$",
"^settings.py$",
"^.*(config)(\\.inc)?\\.php$",
"^LocalSettings.php$",
"\\.?env",
"\\bdump|dump\\b",
"\\bsql|sql\\b",
"\\bdump|dump\\b",
"password",
"backup",
"private.*key",
"(oauth).*(token)",
"^.*\\.log$",
"^\\.?kwallet$",
"^\\.?gnucash$")
return NewFileNameDetector(filenamePatterns)
}

//NewFileNameDetector returns a FileNameDetector that tests Additions against the supplied patterns
func NewFileNameDetector(patternStrings ...string) Detector {
var patterns = make([]*regexp.Regexp, len(patternStrings))
for i, p := range patternStrings {
patterns[i], _ = regexp.Compile(p)
}
func NewFileNameDetector(patterns []*regexp.Regexp) Detector {
return FileNameDetector{patterns}
}

Expand Down
4 changes: 3 additions & 1 deletion detector/filename_detector_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ package detector

import (
"testing"
"regexp"

"talisman/gitrepo"

Expand Down Expand Up @@ -156,7 +157,8 @@ func shouldNotFailWithDefaultDetectorAndIgnores(fileName, ignore string, t *test

func shouldFailWithSpecificPattern(fileName, pattern string, t *testing.T) {
results := NewDetectionResults()
NewFileNameDetector(pattern).Test(additionsNamed(fileName), TalismanRCIgnore{}, results)
pt := regexp.MustCompile(pattern)
NewFileNameDetector([]*regexp.Regexp{pt}).Test(additionsNamed(fileName), TalismanRCIgnore{}, results)
assert.True(t, results.HasFailures(), "Expected file %s to fail the check against the %s pattern", fileName, pattern)
}

Expand Down
17 changes: 10 additions & 7 deletions detector/ignores.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,15 +12,21 @@ import (

const (
//LinePattern represents a line in the ignorefile with an optional comment
LinePattern string = "^([^#]+)?\\s*(#(.*))?$"
LinePattern string = `^([^#]+)?\s*(#(.*))?$`

//IgnoreDetectorCommentPattern represents a special comment that ignores only certain detectors
IgnoreDetectorCommentPattern string = "^ignore:([^\\s]+).*$"
IgnoreDetectorCommentPattern string = `^ignore:([^\s]+).*$`

//DefaultRCFileName represents the name of default file in which all the ignore patterns are configured in new version
DefaultRCFileName string = ".talismanrc"
)

var (
commentPattern = regexp.MustCompile(LinePattern)
ignorePattern = regexp.MustCompile(IgnoreDetectorCommentPattern)
emptyStringPattern = regexp.MustCompile(`^\s*$`)
)

//Ignores represents a set of patterns that have been configured to be ignored by the Detectors.
//Detectors are expected to honor these ignores.
type Ignores struct {
Expand Down Expand Up @@ -74,7 +80,6 @@ func NewTalismanRCIgnore(fileContents []byte) (TalismanRCIgnore) {

func NewIgnore(pattern string, comment string) Ignore {
var ignoredDetectors []string
ignorePattern := regexp.MustCompile(IgnoreDetectorCommentPattern)
match := ignorePattern.FindStringSubmatch(comment)
if match != nil {
ignoredDetectors = strings.Split(match[1], ",")
Expand All @@ -92,14 +97,13 @@ func (i FileIgnoreConfig) isEffective(detectorName string) bool {
contains(i.IgnoreDetectors, detectorName)
}


//NewIgnores builds a new Ignores with the patterns specified in the ignoreSpecs
//Empty lines and comments are ignored.
func NewIgnores(lines ...string) Ignores {
var groups []string
var ignores []Ignore
for _, line := range lines {
var commentPattern = regexp.MustCompile(LinePattern)
groups := commentPattern.FindStringSubmatch(line)
groups = commentPattern.FindStringSubmatch(line)
if len(groups) == 4 {
ignores = append(ignores, NewIgnore(strings.TrimSpace(groups[1]), strings.TrimSpace(groups[3])))
}
Expand Down Expand Up @@ -160,7 +164,6 @@ func (i TalismanRCIgnore) effectiveRules(detectorName string) []string {
}

func isEmptyString(str string) bool {
var emptyStringPattern = regexp.MustCompile("^\\s*$")
return emptyStringPattern.MatchString(str)
}
func contains(s []string, e string) bool {
Expand Down
28 changes: 14 additions & 14 deletions detector/pattern_detector.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,20 +14,20 @@ type PatternDetector struct {

var (
detectorPatterns = []*regexp.Regexp{
regexp.MustCompile("(?i)(['\"_]?password['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pw['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pwd['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pass['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?pword['\"]? *[:=][^,;\n]{8,})"),
regexp.MustCompile("(?i)(['\"_]?adminPassword['\"]? *[:=\n][^,;]{8,})"),
regexp.MustCompile("(?i)(['\"_]?passphrase['\"]? *[:=\n][^,;]{8,})"),
regexp.MustCompile("(<[^(><.)]?password[^(><.)]*?>[^(><.)]+</[^(><.)]?password[^(><.)]*?>)"),
regexp.MustCompile("(<[^(><.)]?passphrase[^(><.)]*?>[^(><.)]+</[^(><.)]?passphrase[^(><.)]*?>)"),
regexp.MustCompile("(?i)(<ConsumerKey>\\S*<\\/ConsumerKey>)"),
regexp.MustCompile("(?i)(<ConsumerSecret>\\S*<\\/ConsumerSecret>)"),
regexp.MustCompile("(?i)(AWS[ \\w]+key[ \\w]+[:=])"),
regexp.MustCompile("(?i)(AWS[ \\w]+secret[ \\w]+[:=])"),
regexp.MustCompile("(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)"),
regexp.MustCompile(`(?i)(['"_]?password['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pw['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pwd['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pass['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?pword['"]? *[:=][^,;\n]{8,})`),
regexp.MustCompile(`(?i)(['"_]?adminPassword['"]? *[:=\n][^,;]{8,})`),
regexp.MustCompile(`(?i)(['"_]?passphrase['"]? *[:=\n][^,;]{8,})`),
regexp.MustCompile(`(<[^(><.)]?password[^(><.)]*?>[^(><.)]+</[^(><.)]?password[^(><.)]*?>)`),
regexp.MustCompile(`(<[^(><.)]?passphrase[^(><.)]*?>[^(><.)]+</[^(><.)]?passphrase[^(><.)]*?>)`),
regexp.MustCompile(`(?i)(<ConsumerKey>\S*</ConsumerKey>)`),
regexp.MustCompile(`(?i)(<ConsumerSecret>\S*</ConsumerSecret>)`),
regexp.MustCompile(`(?i)(AWS[ \w]+key[ \w]+[:=])`),
regexp.MustCompile(`(?i)(AWS[ \w]+secret[ \w]+[:=])`),
regexp.MustCompile(`(?s)(BEGIN RSA PRIVATE KEY.*END RSA PRIVATE KEY)`),
}
)

Expand Down

0 comments on commit 601a9bd

Please sign in to comment.