bot: check for in user agent

Author: crackcomm

bot.go

@@ -75,6 +75,8 @@ func (p *UserAgent) fixOther(sections []section) {
 	}
 }
 
+var botRegex = regexp.MustCompile("(?i)(bot|crawler|sp(i|y)der|search|worm|fetch|nutch)")
+
 // Check if we're dealing with a bot or with some weird browser. If that is the
 // case, the receiver will be modified accordingly.
 func (p *UserAgent) checkBot(sections []section) {
@@ -83,9 +85,8 @@ func (p *UserAgent) checkBot(sections []section) {
 	if len(sections) == 1 && sections[0].name != "Mozilla" {
 		p.mozilla = ""
 
-		// Check whether the name has some suspicious "bot" in his name.
-		reg, _ := regexp.Compile("(?i)bot")
-		if reg.Match([]byte(sections[0].name)) {
+		// Check whether the name has some suspicious "bot" or "crawler" in his name.
+		if botRegex.Match([]byte(sections[0].name)) {
 			p.setSimple(sections[0].name, "", true)
 			return
 		}