feat(lookup): add crawler info on lookup (#2)

* feat(lookup): add crawler info on lookup

* fix(lookup): crawlers should have device info other

Author: GuillaumeBAECHLER

types.go

@@ -18,13 +18,16 @@ type Udger struct {
 	Browsers     map[int]Browser
 	OS           map[int]OS
 	Devices      map[int]Device
+	crawlerTypes map[int]string
+	Crawlers     map[string]Crawler
 }
 
 // Info is the struct returned by the Lookup(ua string) function, contains everything about the UA
 type Info struct {
 	Browser Browser `json:"browser"`
 	OS      OS      `json:"os"`
 	Device  Device  `json:"device"`
+	Crawler Crawler `json:"crawler"`
 }
 
 // Browser contains information about the browser type, engine and off course its name
@@ -58,3 +61,12 @@ type Device struct {
 	Name string `json:"name"`
 	Icon string `json:"icon"`
 }
+
+// Crawler contains all the information about the crawler
+type Crawler struct {
+	Name    string `json:"name"`
+	Family  string `json:"family"`
+	Vendor  string `json:"vendor"`
+	ClassId int
+	Class   string `json:"classification"`
+}

udger.go

@@ -8,6 +8,8 @@ import (
 	"strings"
 )
 
+const CRAWLER_CLASS_ID = 99
+
 // New creates a new instance of Udger from the dbPath database loaded in memory for fast lookup.
 func New(dbPath string) (*Udger, error) {
 	u := &Udger{
@@ -16,6 +18,8 @@ func New(dbPath string) (*Udger, error) {
 		Devices:      make(map[int]Device),
 		browserTypes: make(map[int]string),
 		browserOS:    make(map[int]int),
+		crawlerTypes: make(map[int]string),
+		Crawlers:     make(map[string]Crawler),
 	}
 	var err error
 
@@ -53,6 +57,13 @@ func (udger *Udger) Lookup(ua string) (*Info, error) {
 		info.Browser.typ = -1
 	}
 
+	if crawler, found := udger.Crawlers[ua]; found {
+		info.Crawler = crawler
+		info.Crawler.Class = udger.crawlerTypes[crawler.ClassId]
+		info.Browser.typ = CRAWLER_CLASS_ID
+		info.Browser.Type = udger.browserTypes[CRAWLER_CLASS_ID]
+	}
+
 	if val, ok := udger.browserOS[browserID]; ok {
 		info.OS = udger.OS[val]
 	} else {
@@ -71,7 +82,7 @@ func (udger *Udger) Lookup(ua string) (*Info, error) {
 			Name: "Smartphone",
 			Icon: "phone.png",
 		}
-	} else if info.Browser.typ == 5 || info.Browser.typ == 10 || info.Browser.typ == 20 || info.Browser.typ == 50 {
+	} else if info.Browser.typ == 5 || info.Browser.typ == 10 || info.Browser.typ == 20 || info.Browser.typ == 50 || info.Browser.typ == CRAWLER_CLASS_ID {
 		info.Device = Device{
 			Name: "Other",
 			Icon: "other.png",
@@ -88,13 +99,8 @@ func (udger *Udger) Lookup(ua string) (*Info, error) {
 
 func (udger *Udger) cleanRegex(r string) string {
 	// removes single-line and case-insensitive modifiers
-	if strings.HasSuffix(r, "/si") {
-		r = r[:len(r)-3]
-	}
-	if strings.HasPrefix(r, "/") {
-		r = r[1:]
-	}
-
+	r = strings.TrimSuffix(r, "/si")
+	r = strings.TrimPrefix(r, "/")
 	return r
 }
 
@@ -128,6 +134,9 @@ func (udger *Udger) init() error {
 	if err := udger.initOS(); err != nil {
 		return err
 	}
+	if err := udger.initCrawlers(); err != nil {
+		return err
+	}
 	return nil
 }
 
@@ -253,3 +262,31 @@ func (udger *Udger) initOS() error {
 	rows.Close()
 	return nil
 }
+
+func (udger *Udger) initCrawlers() error {
+	// Uncategorised, Search engine bot, Site monitor, etc.
+	rows, err := udger.db.Query("SELECT id, crawler_classification FROM udger_crawler_class")
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var crawlerClass string
+		var id int
+		rows.Scan(&id, &crawlerClass)
+		udger.crawlerTypes[id] = crawlerClass
+	}
+	rows.Close()
+
+	rows, err = udger.db.Query("SELECT ua_string, name, family, vendor, class_id FROM udger_crawler_list")
+	if err != nil {
+		return err
+	}
+	for rows.Next() {
+		var crawler Crawler
+		var uaString string
+		rows.Scan(&uaString, &crawler.Name, &crawler.Family, &crawler.Vendor, &crawler.ClassId)
+		udger.Crawlers[uaString] = crawler
+	}
+	rows.Close()
+	return nil
+}

udger_test.go

@@ -159,6 +159,35 @@ func TestValidDbName(t *testing.T) {
 					So(info.Browser.Version, ShouldResemble, "")
 				})
 			})
+
+			Convey("test Crawler", func() {
+				info, err := udger.Lookup("Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)")
+				So(err, ShouldBeNil)
+				So(info, ShouldNotBeNil)
+
+				Convey("test lookup info", func() {
+					So(info.OS.Company, ShouldResemble, "")
+					So(info.OS.Family, ShouldResemble, "")
+					So(info.OS.Icon, ShouldResemble, "")
+					So(info.OS.Name, ShouldResemble, "")
+
+					So(info.Device.Name, ShouldResemble, "Other")
+					So(info.Device.Icon, ShouldResemble, "other.png")
+
+					So(info.Browser.Company, ShouldResemble, "")
+					So(info.Browser.Engine, ShouldResemble, "")
+					So(info.Browser.Family, ShouldResemble, "")
+					So(info.Browser.Icon, ShouldResemble, "")
+					So(info.Browser.Name, ShouldResemble, "")
+					So(info.Browser.Type, ShouldResemble, "Crawler")
+					So(info.Browser.Version, ShouldResemble, "")
+
+					So(info.Crawler.Name, ShouldResemble, "Googlebot Desktop")
+					So(info.Crawler.Family, ShouldResemble, "Googlebot")
+					So(info.Crawler.Vendor, ShouldResemble, "Google Inc.")
+					So(info.Crawler.Class, ShouldResemble, "Search engine bot")
+				})
+			})
 		})
 	})
 }