[enh] make sensitive patterns configurable

Author: asciimoo

config/config.go

@@ -23,11 +23,12 @@ import (
 )
 
 type Config struct {
-	fname   string
-	App     App     `yaml:"app"`
-	Server  Server  `yaml:"server"`
-	Hotkeys Hotkeys `yaml:"hotkeys"`
-	Rules   *Rules  `yaml:"-"`
+	fname                    string
+	App                      App               `yaml:"app"`
+	Server                   Server            `yaml:"server"`
+	Hotkeys                  Hotkeys           `yaml:"hotkeys"`
+	SensitiveContentPatterns map[string]string `yaml:"sensitive_content_patterns"`
+	Rules                    *Rules            `yaml:"-"`
 }
 
 type App struct {
@@ -132,6 +133,14 @@ func CreateDefaultConfig() *Config {
 			"tab":       "autocomplete",
 			"?":         "show_hotkeys",
 		},
+		SensitiveContentPatterns: map[string]string{
+			"aws_access_key":      `AKIA[0-9A-Z]{16}`,
+			"aws_secret_key":      `(?i)aws(.{0,20})?(secret)?(.{0,20})?['"][0-9a-zA-Z\/+]{40}['"]`,
+			"generic_private_key": `-----BEGIN ((RSA|EC|DSA) )?PRIVATE KEY-----`,
+			"github_token":        `(ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36}`,
+			"ssh_private_key":     `-----BEGIN OPENSSH PRIVATE KEY-----`,
+			"pgp_private_key":     `-----BEGIN PGP PRIVATE KEY BLOCK-----`,
+		},
 	}
 }
 

hister.go

@@ -305,7 +305,7 @@ func initDB() {
 }
 
 func initIndex() {
-	err := indexer.Init(cfg.IndexPath())
+	err := indexer.Init(cfg)
 	if err != nil {
 		exit(1, err.Error())
 	}

server/indexer/indexer.go

@@ -76,57 +76,18 @@ type Results struct {
 var i *indexer
 var allFields []string = []string{"url", "title", "text", "favicon", "html", "domain", "added"}
 var ErrSensitiveContent = errors.New("document contains sensitive data")
-var sensitiveContentPatterns = []string{
-	// AWS Access Key
-	`AKIA[0-9A-Z]{16}`,
-	// AWS Secret Key
-	`(?i)aws(.{0,20})?(secret)?(.{0,20})?['"][0-9a-zA-Z\/+]{40}['"]`,
-	// Private Key
-	`-----BEGIN (RSA|EC|DSA)? PRIVATE KEY-----`,
-	// Generic API Key
-	`(?i)(api|token|secret)[\s:=]+['"]?[a-z0-9]{32,}['"]?`,
-	// Slack Token
-	`xox[baprs]-[0-9a-zA-Z]{10,48}`,
-	// GitHub Token
-	`(ghp|gho|ghu|ghs|ghr)_[a-zA-Z0-9]{36}`,
-	// Google API Key
-	`AIza[0-9A-Za-z\-_]{35}`,
-	// Heroku API Key
-	`[hH][eE][rR][oO][kK][uU].{0,30}[0-9A-F]{8}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{4}-[0-9A-F]{12}`,
-	// SSH Private Key
-	`-----BEGIN OPENSSH PRIVATE KEY-----`,
-	// PGP Private Key
-	`-----BEGIN PGP PRIVATE KEY BLOCK-----`,
-	// JWT Token
-	`eyJ[a-zA-Z0-9\/_-]{10,}\.[a-zA-Z0-9\/_-]{10,}\.[a-zA-Z0-9\/_-]{10,}`,
-	// Credit Card Number - disabled, too many false positives - TODO refine regexp
-	//`\b(?:4[0-9]{12}(?:[0-9]{3})?|5[1-5][0-9]{14}|3[47][0-9]{13}|3(?:0[0-5]|[68][0-9])[0-9]{11}|6(?:011|5[0-9]{2})[0-9]{12}|(?:2131|1800|35\d{3})\d{11})\b`,
-	// Basic Auth Credentials - disabled, too many false positives
-	//`(?i)basic [a-z0-9=:_\+\/-]{5,100}`,
-	// Docker Registry Auth
-	`"auth"\s*:\s*"[a-z0-9=:_\+\/-]{5,100}"`,
-	// Azure Storage Key
-	`DefaultEndpointsProtocol=https;AccountName=[a-z0-9]{3,24};AccountKey=[a-z0-9\/+]{88}==`,
-	// Google OAuth Token
-	`ya29\.[a-zA-Z0-9\-_]+`,
-	// Facebook Access Token
-	`EAACEdEose0cBA[0-9A-Za-z]+`,
-	// Twitter API Key
-	`(?i)twitter(.{0,20})?['"][0-9a-z]{35,44}['"]`,
-	// Database Connection String
-	//`(?i)(jdbc:|mongodb:\/\/|postgresql:\/\/|mysql:\/\/).+:[^@]+@[a-z0-9\.-]+`,
-}
 var sensitiveContentRe *regexp.Regexp
 
-func init() {
-	sensitiveContentRe = regexp.MustCompile(fmt.Sprintf("(%s)", strings.Join(sensitiveContentPatterns, "|")))
-}
-
-func Init(idxPath string) error {
-	idx, err := bleve.Open(idxPath)
+func Init(cfg *config.Config) error {
+	sp := make([]string, 0, len(cfg.SensitiveContentPatterns))
+	for _, v := range cfg.SensitiveContentPatterns {
+		sp = append(sp, v)
+	}
+	sensitiveContentRe = regexp.MustCompile(fmt.Sprintf("(%s)", strings.Join(sp, "|")))
+	idx, err := bleve.Open(cfg.IndexPath())
 	if err != nil {
 		mapping := createMapping()
-		idx, err = bleve.New(idxPath, mapping)
+		idx, err = bleve.New(cfg.IndexPath(), mapping)
 		if err != nil {
 			return err
 		}