Add Yandex Radar top10k

Closes #15

Author: ilyaglow

README.md

@@ -11,6 +11,7 @@ Fast HTTP service which shows a specified domain rank from following providers:
 - [Open PageRank](https://www.domcop.com/top-10-million-domains)
 - [Tranco List](https://tranco-list.eu/)
 - [Quantcast](https://www.quantcast.com/top-sites/)
+- [YandexRadar](https://radar.yandex.ru/)
 
 Can be used as a base for a domain categorization / network filters /
 suspicious domain detection.
@@ -67,47 +68,54 @@ $: curl 127.0.0.1:8080/rank/github.com
   "ranks": [
     {
       "domain": "github.com",
-      "rank": 40,
-      "date": "2019-06-01T00:00:00Z",
-      "source": "tranco",
+      "rank": 2698,
+      "date": "2019-09-07T00:00:00Z",
+      "source": "umbrella",
       "raw": ""
     },
     {
       "domain": "github.com",
-      "rank": 123,
-      "date": "2019-06-01T00:00:00Z",
-      "source": "quantcast",
-      "raw": ""
+      "rank": 29,
+      "date": "2019-09-07T00:00:00Z",
+      "source": "majestic",
+      "raw": "29,24,github.com,com,176946,489686,github.com,com,29,24,176096,487221"
     },
     {
       "domain": "github.com",
-      "rank": 2555,
-      "date": "2019-06-01T00:00:00Z",
-      "source": "umbrella",
+      "rank": 26,
+      "date": "2019-09-07T00:00:00Z",
+      "source": "pagerank",
       "raw": ""
     },
     {
       "domain": "github.com",
-      "rank": 45,
-      "date": "2019-06-01T00:00:00Z",
+      "rank": 32,
+      "date": "2019-09-07T00:00:00Z",
       "source": "alexa",
       "raw": ""
     },
     {
       "domain": "github.com",
-      "rank": 29,
-      "date": "2019-06-01T00:00:00Z",
-      "source": "majestic",
-      "raw": "29,23,github.com,com,179597,516432,github.com,com,29,23,179410,516889"
+      "rank": 467,
+      "date": "2019-09-07T00:00:00Z",
+      "source": "yandex-radar",
+      "raw": "The world’s leading software development platform · GitHub,github.com,,Сервисы,,,1520000,2340000,,,"
     },
     {
       "domain": "github.com",
-      "rank": 39,
-      "date": "2019-06-01T00:00:00Z",
-      "source": "pagerank",
+      "rank": 43,
+      "date": "2019-09-07T00:00:00Z",
+      "source": "tranco",
+      "raw": ""
+    },
+    {
+      "domain": "github.com",
+      "rank": 168,
+      "date": "2019-09-07T00:00:00Z",
+      "source": "quantcast",
       "raw": ""
     }
   ],
-  "timestamp": "2019-06-01T22:06:12.662492307Z"
+  "timestamp": "2019-09-07T14:32:32.9725943Z"
 }
 ```

ingester.go

@@ -13,6 +13,7 @@ var ingesters = []Ingester{
 	NewPageRank(),
 	NewTranco(),
 	NewQuantcast(),
+	NewYandexRadar(),
 }
 
 // IngesterConf represents a top popular domains provider configuration.

yandex_radar.go

@@ -0,0 +1,80 @@
+package dor
+
+import (
+	"encoding/csv"
+	"io"
+	"log"
+	"net/http"
+	"strings"
+	"time"
+)
+
+const (
+	yandexRadarTop10k = "https://radar.yandex.ru/export?options=%7B%22title%22%3A%22%D0%AF%D0%BD%D0%B4%D0%B5%D0%BA%D1%81.%D0%A0%D0%B0%D0%B4%D0%B0%D1%80%2B-%2B%D0%A2%D0%BE%D0%BF%2B%D0%98%D0%BD%D1%82%D0%B5%D1%80%D0%BD%D0%B5%D1%82-%D0%BF%D1%80%D0%BE%D0%B5%D0%BA%D1%82%D0%BE%D0%B2%22%2C%22exportFormat%22%3A%22csv%22%2C%22limit%22%3A0%2C%22offset%22%3A1000000%7D&mode=top-sites"
+)
+
+// YandexRadarIngester represents Ingester implementation for Yandex Radar.
+type YandexRadarIngester struct {
+	IngesterConf
+}
+
+// NewYandexRadar bootstraps YandexRadarIngester.
+func NewYandexRadar() *YandexRadarIngester {
+	return &YandexRadarIngester{
+		IngesterConf: IngesterConf{
+			Description: "yandex-radar",
+		},
+	}
+
+}
+
+// Do implements Ingester Do func with the data.
+func (in *YandexRadarIngester) Do() (chan *Entry, error) {
+	in.Timestamp = time.Now().UTC()
+	ch := make(chan *Entry)
+
+	go func() {
+		defer close(ch)
+		resp, err := http.Get(yandexRadarTop10k)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		log.Printf("%s: %s downloaded successfully", in.Description, yandexRadarTop10k)
+		defer resp.Body.Close()
+
+		r := csv.NewReader(resp.Body)
+		r.LazyQuotes = true
+		// read the header
+		// "Название ресурса","URL-адрес ресурса","Тематики ресурса","Тип ресурса","Медиахолдинг","Данные Метрики","Посетители (кросс-девайс)","Посетители (браузер)","Среднее время","Доля пользователей приложения","Дневная аудитория"
+		_, err = r.Read()
+		if err != nil {
+			log.Println(err)
+			return
+		}
+		var i uint32
+		now := time.Now()
+		for {
+			i++
+			record, err := r.Read()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				log.Println(err)
+				return
+			}
+
+			ch <- &Entry{
+				Domain:  record[1],
+				Rank:    i,
+				Date:    now,
+				Source:  in.Description,
+				RawData: strings.Join(record, ","),
+			}
+		}
+	}()
+
+	return ch, nil
+}