Add process extended config format

Author: triole

src/conf/conf.go

@@ -43,10 +43,13 @@ func (conf *Conf) readConfig() {
 			val.Source, _ = conf.Util.AbsPath(val.Source)
 			val.SourceType = conf.fileOrFolder(val.Source)
 		}
-		if val.SourceType == "url" && val.HTTPMethod == "" {
-			val.HTTPMethod = "get"
+		if val.SourceType == "url" && val.RequestMethod == "" {
+			val.RequestMethod = "get"
 		}
-		val.HTTPMethod = strings.ToUpper(val.HTTPMethod)
+		val.RequestMethod = strings.ToUpper(val.RequestMethod)
+		val.RequestMethod = conf.Util.RxReplaceAll(
+			val.RequestMethod, "^HTTP_", "",
+		)
 		var v datasize.ByteSize
 		if val.MaxReturnSize == "" {
 			val.MaxReturnSize = "10K"

src/conf/struct.go

@@ -24,13 +24,20 @@ type Endpoint struct {
 	Source             string `yaml:"source"`
 	SourceType         string
 	URL                string   `yaml:"url"`
-	HTTPMethod         string   `yaml:"http_method"`
+	RequestMethod      string   `yaml:"request_method"`
 	RxFilter           string   `yaml:"regex_filter"`
 	SortFileName       string   `yaml:"sort_file_name"`
 	IgnoreList         []string `yaml:"regex_ignore_list"`
 	MaxReturnSize      string   `yaml:"max_return_size"`
 	MaxReturnSizeBytes uint64
-	ReturnValues       ReturnValues `yaml:"return_values"`
+	Return             ReturnValues `yaml:"return"`
+}
+
+type Process struct {
+	Strategy      string   `yaml:"strategy"`
+	RequestMethod string   `yaml:"request_method"`
+	JSONPath      string   `yaml:"json_path"`
+	RegexMatch    []string `yaml:"regex_match"`
 }
 
 type ReturnValues struct {
@@ -39,6 +46,7 @@ type ReturnValues struct {
 	Content                  bool       `yaml:"content"`
 	UnmarshalContent         bool       `yaml:"unmarshal_content"`
 	RegexReplace             [][]string `yaml:"regex_replace"`
+	RegexMatch               string     `yaml:"regex_match"`
 	JSONPath                 string     `yaml:"json_path"`
 	Size                     bool       `yaml:"size"`
 	LastMod                  bool       `yaml:"lastmod"`

src/indexer/init_test.go

@@ -23,7 +23,7 @@ func prepareTests(folder, sortBy string, asc bool) (Indexer, JoinerIndex, Params
 }
 
 func newTestEndpoint() conf.Endpoint {
-	return conf.Endpoint{ReturnValues: conf.ReturnValues{
+	return conf.Endpoint{Return: conf.ReturnValues{
 		Created:                  true,
 		LastMod:                  true,
 		Content:                  true,

src/indexer/ji.go

@@ -47,10 +47,10 @@ func (ind Indexer) MakeJoinerIndex(params Params) (ji JoinerIndex) {
 		}
 	case "url":
 		je := JoinerEntry{Path: params.Endpoint.Source}
-		if params.Endpoint.ReturnValues.Content {
+		if params.Endpoint.Return.Content {
 			resp, err := ind.req(
 				params.Endpoint.Source,
-				params.Endpoint.HTTPMethod,
+				params.Endpoint.RequestMethod,
 			)
 			// TODO: maybe later add the possibility to encode base64
 			je.Content = ind.byteToBody(resp)

src/indexer/read.go

@@ -31,11 +31,11 @@ func (ind Indexer) readDataFile(filename string, ps conf.Endpoint, chin chan str
 		Path: pth,
 	}
 	fileSize := ind.Util.GetFileSize(filename)
-	if ps.ReturnValues.Size {
+	if ps.Return.Size {
 		je.Size = fileSize
 	}
 	if ps.MaxReturnSizeBytes > fileSize {
-		if ps.ReturnValues.Content || ps.ReturnValues.SplitMarkdownFrontMatter {
+		if ps.Return.Content || ps.Return.SplitMarkdownFrontMatter {
 			je.Content = ind.readFileContent(filename, ps)
 		}
 	} else {
@@ -48,13 +48,13 @@ func (ind Indexer) readDataFile(filename string, ps conf.Endpoint, chin chan str
 			},
 		)
 	}
-	if ps.ReturnValues.SplitPath {
+	if ps.Return.SplitPath {
 		je.SplitPath = strings.Split(pth, string(filepath.Separator))
 	}
-	if ps.ReturnValues.Created {
+	if ps.Return.Created {
 		je.Created = ind.Util.GetFileCreated(filename)
 	}
-	if ps.ReturnValues.LastMod {
+	if ps.Return.LastMod {
 		je.LastMod = ind.Util.GetFileLastMod(filename)
 	}
 	chout <- je
@@ -68,7 +68,7 @@ func (ind Indexer) readFileContent(filename string, ps conf.Endpoint) (content F
 		case ".json", ".toml", ".yaml", ".yml":
 			content = ind.unmarshal(by, ps)
 		case ".md":
-			content = ind.readMarkdown(by, ps.ReturnValues)
+			content = ind.readMarkdown(by, ps.Return)
 		default:
 			content = ind.byteToBody(by)
 		}
@@ -95,7 +95,7 @@ func (ind Indexer) byteToBody(by []byte) (content FileContent) {
 func (ind Indexer) unmarshal(by []byte, ps conf.Endpoint) (content FileContent) {
 	if ind.Util.IsTextData(by) {
 
-		for _, el := range ps.ReturnValues.RegexReplace {
+		for _, el := range ps.Return.RegexReplace {
 			by = []byte(
 				ind.Util.RxReplaceAll(
 					string(by),
@@ -131,11 +131,11 @@ func (ind Indexer) unmarshal(by []byte, ps conf.Endpoint) (content FileContent)
 			logseal.F{"content": string(by), "err": content.Error},
 		)
 	}
-	if ps.ReturnValues.JSONPath != "" {
+	if ps.Return.JSONPath != "" {
 		ind.Lg.Trace(
 			"parse unmarshalled data using json path",
 			logseal.F{
-				"json_path": ps.ReturnValues.JSONPath,
+				"json_path": ps.Return.JSONPath,
 			},
 		)
 		marsh, err := json.Marshal(content.Body)
@@ -144,11 +144,11 @@ func (ind Indexer) unmarshal(by []byte, ps conf.Endpoint) (content FileContent)
 			logseal.F{"error": err},
 		)
 		if err == nil {
-			result := gjson.GetBytes(marsh, ps.ReturnValues.JSONPath)
+			result := gjson.GetBytes(marsh, ps.Return.JSONPath)
 			if len(result.String()) < 1 {
 				ind.Lg.Warn(
 					"json path result is empty",
-					logseal.F{"json_path": ps.ReturnValues.JSONPath},
+					logseal.F{"json_path": ps.Return.JSONPath},
 				)
 			} else {
 				content = ind.unmarshalJSON([]byte(result.String()))

testdata/http.yaml

@@ -4,19 +4,24 @@ port: 17777
 api:
   repos.json:
     source: http://localhost/repos
-    http_method: get
-    return_values:
+    request_method: http_get
+    return:
       created: true
       content: true
       unmarshal_content: true
       json_path: "#.readme_url"
       regex_replace:
         - ["README\\.md", "public.yaml"]
 
-
   meta.json:
     source: http://localhost:17777/repos.json
-    http_method: get
-    return_values:
+    request_method: http_get
+    process:
+      - strategy: use_as_url_list
+        request_method: http_get
+        json_path: "#.content.body"
+        regex_match:
+          - "http:\\/\\//*?\""
+    return:
       content: true
-      json_path: "#.content.body"
+      unmarshal_content: true