First commit

Author: Code Express

.gitignore

@@ -11,5 +11,9 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# app specific excludes
+*.log
+binaries/
+
 # Dependency directories (remove the comment below to include it)
 # vendor/

Makefile

@@ -0,0 +1,6 @@
+all:
+	GOOS=windows GOARCH=amd64 go build -o binaries/webpluck.exe main.go logger.go webpluck.go
+	GOOS=linux   GOARCH=amd64 go build -o binaries/webpluck     main.go logger.go webpluck.go
+	GOOS=darwin  GOARCH=amd64 go build -o binaries/webpluck_osx main.go logger.go webpluck.go
+
+

README.md

@@ -1,2 +1,11 @@
-# webpluck
-Extract specific information from a webpage
+# Web Plucker
+
+`webpluck` scrapes a specific values from a web page. It works as a standalone
+binary as well as in a API mode.
+
+`webpluck` takes the following as input:
+ - URL of the webpage
+ - XPATH of the element
+ - optional regex to further narrow the selection
+
+and outputs the selected value.

logger.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"os"
+	"reflect"
+	"time"
+)
+
+var (
+	Logger *log.Logger
+)
+
+// Initializing the logger and customizing prefix
+func initLogger() {
+	f, err := os.OpenFile("run.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
+	if err != nil {
+		panic(err)
+	}
+	Logger = log.New(f, "", 0)
+	Logger.SetPrefix("[" + time.Now().Format("02-Jan-2006 15:04:05 MST") + "]: ")
+}
+
+// Logs to log file
+// takes generic object and then based on the type of object,
+// logs is in appropriate style.
+func logIt(val interface{}, console ...bool) {
+	// if console is passed, print to stdout as well
+	if len(console) != 0 {
+		fmt.Println(val)
+	}
+
+	v := reflect.ValueOf(val)
+	switch v.Kind() {
+	case reflect.Map:
+		str, _ := json.MarshalIndent(val, "", "  ")
+		Logger.Println(string(str))
+	default:
+		Logger.Println(val)
+	}
+}

main.go

@@ -0,0 +1,154 @@
+package main
+
+import (
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io/ioutil"
+	"net"
+	"net/http"
+	"os"
+	"strconv"
+
+	"gopkg.in/yaml.v2"
+)
+
+const (
+	UserAgent = "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.96 Safari/537.36"
+)
+
+var (
+	//argument flags
+	filePtr       *string
+	outputTextPtr *bool
+	serverModePtr *int
+)
+
+func main() {
+	initFlags()
+	initLogger()
+
+	serverMode := isFlagPassed("p")
+
+	if serverMode {
+		serveApi()
+	} else { // things to pluck passed via a yaml file
+		pluckFromFile()
+	}
+}
+
+/* Server Mode
+Listens on a port and answers online queries of type:
+http://localhost:8080?baseUrl="example.com"&xpath="/html/body"&regex=""
+*/
+func serveApi() {
+	logIt("Started HTTP server on localhost: "+strconv.Itoa(*serverModePtr), true)
+
+	http.HandleFunc("/", handleHttp)
+	fmt.Println(http.ListenAndServe(":"+strconv.Itoa(*serverModePtr), nil))
+}
+
+func handleHttp(w http.ResponseWriter, req *http.Request) {
+	results := make(map[string]string)
+	req.ParseForm()
+	baseUrl := req.Form.Get("baseUrl")
+	xpath := req.Form.Get("xpath")
+	regex := req.Form.Get("regex")
+
+	results["baseUrl"] = baseUrl
+	results["xpath"] = xpath
+	results["regex"] = regex
+
+	logIt(getIp(req) + "  " + req.Header.Get("User-Agent") + " Request: ")
+	logIt(results)
+	defer func() { // in case of panic
+		if err := recover(); err != nil {
+			http.Error(w, "my own error message", http.StatusInternalServerError)
+			fmt.Fprintf(w, "Webpluck encountered an error. Make sure that the baseUrl is a valid URL and xpath and regex are valid\n")
+			fmt.Fprintf(w, "Error encountered is:\n%s\n", err)
+			logIt(err)
+		}
+	}()
+	text := ExtractTextFromUrl(baseUrl, xpath, regex)
+	results["pluckedData"] = text
+	jsonString, err := json.MarshalIndent(results, "", "  ")
+	check(err)
+	fmt.Fprintf(w, string(jsonString))
+	logIt("Answer: " + text)
+}
+
+func pluckFromFile() {
+	data, err := ioutil.ReadFile(*filePtr)
+	check(err)
+	var list targetList
+
+	err = yaml.Unmarshal(data, &list)
+	check(err)
+
+	results := make(map[string]string)
+
+	for _, t := range list.TargetList {
+		text := ExtractTextFromUrl(t.BaseUrl, t.Xpath, t.Regex)
+		results[t.Name] = text
+		if *outputTextPtr { // if output to text (t) flag is set
+			fmt.Println(t.Name + ": " + text)
+		}
+	}
+
+	logIt("Webpluck invoked. Reading from file: " + *filePtr)
+	logIt(results)
+
+	if !*outputTextPtr { // default case is to print in JSON
+		jsonString, err := json.MarshalIndent(results, "", "  ")
+		check(err)
+		fmt.Println(string(jsonString))
+	}
+}
+
+func isFlagPassed(name string) bool {
+	found := false
+	flag.Visit(func(f *flag.Flag) {
+		if f.Name == name {
+			found = true
+		}
+	})
+	return found
+}
+
+/* Parse command line flage and initiaize the global flag variables */
+func initFlags() {
+	flag.Usage = func() {
+		fmt.Fprintln(os.Stderr, "---------------------------------------------------------")
+		fmt.Fprintln(os.Stderr, "Usage:   $ ./webpluck [-f filename.yml]")
+		fmt.Fprintln(os.Stderr, "Example: $ ./webpluck -f ./extract_list.yml")
+		fmt.Fprintln(os.Stderr, "---------------------------------------------------------\nFlags:")
+		flag.PrintDefaults()
+	}
+
+	filePtr = flag.String("f", "./targets.yml",
+		"`File name (yml)` with the list of targets to pluck/extract")
+	outputTextPtr = flag.Bool("t", false,
+		"Outputs the results in `text format` instead of JSON (applicable only in non server mode)")
+	serverModePtr = flag.Int("p", 0,
+		"`Port number` to serve webpluck as in HTTP API")
+	flag.Parse()
+}
+
+func check(e error) {
+	if e != nil {
+		panic(e)
+	}
+}
+
+// Get IP address of the incoming HTTP request based on forwarded-for
+// header (present in case of proxy). If not, use the remote address
+func getIp(req *http.Request) string {
+	forwarded := req.Header.Get("X-FORWARDED-FOR")
+	var addr string
+	if forwarded != "" {
+		addr = forwarded
+	}
+	addr = req.RemoteAddr
+	ip, _, _ := net.SplitHostPort(addr)
+	return ip
+}

targets.yml

@@ -0,0 +1,13 @@
+targetList:
+  -   name: example.com
+      baseUrl: http://example.com/
+      xpath:   /html/body/div/p[2]/a/@href
+      regex:   ^(?:https?://)?(?:[^@\n]+@)?([^:/\n]+)
+  -   name: stackoverflow_founders
+      baseUrl: https://stackoverflow.com/company
+      xpath:  //*[@id="content"]/section[3]/ol/li[1]/ol/li[2]
+      regex: ^(.*) launch .*
+  -   name: stackoverflow_example_without_regex
+      baseUrl: https://stackoverflow.com/questions/18361750
+      xpath: //*[@id="question"]/div[2]/div[2]/div[3]/div/div[3]/div/div[1]
+      regex:

webpluck.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"io/ioutil"
+	"net/http"
+	"regexp"
+	"strings"
+
+	"gopkg.in/xmlpath.v2"
+)
+
+type targetList struct {
+	TargetList []dataLocation `yaml:"targetList"`
+}
+
+type dataLocation struct {
+	Name    string `yaml:"name"`
+	BaseUrl string `yaml:"baseUrl"`
+	Xpath   string `yaml:"xpath"`
+	Regex   string `yaml:"regex"`
+}
+
+/* Params:
+   url   - URL of the page to be scraped
+   xpath - the xpath of the target text in the html document
+   regex - the regex to be applied to the extracted text found at xpath to extract final value
+   returns:
+   a string value that is the final plucked value
+*/
+func ExtractTextFromUrl(
+	url string,
+	xpath string,
+	regex string) string {
+
+	text := ""
+
+	//logIt("Fetch from URL: "+url, 1)
+	parsedHtml := fetchUrl(url) // returns a xmlpath.Node object
+	path := xmlpath.MustCompile(xpath)
+	value, ok := path.String(parsedHtml)
+	if ok {
+		if regex != "" {
+			// try applying regex
+			regexMatch := regexp.MustCompile(regex)
+			text = regexMatch.FindStringSubmatch(string(value))[1]
+		} else {
+			text = value // no regex, the xpath element is the value
+		}
+	}
+	return strings.TrimSpace(text)
+}
+
+// does a HTTP GET and returns the HTML body for that URL
+func fetchUrl(url string) *xmlpath.Node {
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		panic(err)
+	}
+	req.Header.Add("User-Agent", UserAgent)
+	resp, err := client.Do(req)
+	if err != nil {
+		panic(err)
+	}
+	defer resp.Body.Close()
+
+	html, _ := ioutil.ReadAll(resp.Body)
+	htmlStr := string(html)
+
+	parsedHtml, err := xmlpath.ParseHTML(strings.NewReader(htmlStr))
+	if err != nil {
+		panic(err)
+	}
+	return parsedHtml
+}