feat: duplicate functions for API request to make it possible using the CNJ code search using the pagination feature. Not the best solution, but it will work

Author: DanielFillol

request/api_async_request_codeSearch.go

@@ -0,0 +1,82 @@
+package request
+
+import (
+	"github.com/DanielFillol/DataJUD_API_CALLER/models"
+	"log"
+	"sync"
+)
+
+// AsyncAPIRequestCode makes API requests asynchronously
+func AsyncAPIRequestCode(users []models.ReadCsvCode, numberOfWorkers int, url string, method string, auth string) ([]models.ResponseBodyNextPage, error) {
+	// Create a channel to signal when the goroutines are done processing inputs
+	done := make(chan struct{})
+	defer close(done)
+	// Create a channel to receive inputs from
+	inputCh := StreamInputsCode(done, users)
+
+	// Create a wait group to wait for the worker goroutines to finish
+	var wg sync.WaitGroup
+	wg.Add(numberOfWorkers)
+
+	// Create a channel to receive results from
+	resultCh := make(chan models.ResponseBodyNextPage)
+
+	// Spawn worker goroutines to process inputs
+	var errorOnApiRequests error
+	for i := 0; i < numberOfWorkers; i++ {
+		go func() {
+			// Each worker goroutine consumes inputs from the shared input channel
+			for input := range inputCh {
+				// Make the API request and send the response to the result channel
+				bodyStr, err := APIRequestCode(url+input.CourtName+API, method, auth, input)
+				resultCh <- bodyStr
+				if err != nil {
+					// If there is an error making the API request, print the error
+					log.Println("error send request: " + err.Error())
+					errorOnApiRequests = err
+					//break
+				}
+				log.Println("success send request: " + "200 " + input.CourtName)
+			}
+			// When the worker goroutine is done processing inputs, signal the wait group
+			wg.Done()
+		}()
+	}
+
+	// Wait for all worker goroutines to finish processing inputs
+	go func() {
+		wg.Wait()
+		close(resultCh)
+	}()
+
+	// Return early on error in any given call on API requests
+	if errorOnApiRequests != nil {
+		return nil, errorOnApiRequests
+	}
+
+	// Collect results from the result channel and return them as a slice
+	var results []models.ResponseBodyNextPage
+	for result := range resultCh {
+		results = append(results, result)
+	}
+
+	return results, nil
+}
+
+// StreamInputsCode sends inputs from a slice to a channel
+func StreamInputsCode(done <-chan struct{}, inputs []models.ReadCsvCode) <-chan models.ReadCsvCode {
+	// Create a channel to send inputs to
+	inputCh := make(chan models.ReadCsvCode)
+	go func() {
+		defer close(inputCh)
+		for _, input := range inputs {
+			select {
+			case inputCh <- input:
+			case <-done:
+				// If the done channel is closed prematurely, finish the loop (closing the input channel)
+				break
+			}
+		}
+	}()
+	return inputCh
+}

request/api_request_codeSearch.go

@@ -0,0 +1,160 @@
+package request
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"github.com/DanielFillol/DataJUD_API_CALLER/models"
+	"io/ioutil"
+	"log"
+	"strconv"
+)
+
+// APIRequestCode makes an API request to the specified URL using the specified HTTP method and authentication header.
+// It returns a models.ResponseBodyNextPage struct containing the API response body and an error (if any).
+func APIRequestCode(url, method string, auth string, request models.ReadCsvCode) (models.ResponseBodyNextPage, error) {
+	// Create responseReturn new BodyRequestLawsuit struct with the document ID and pagination settings for the initial API call.
+	class, err := strconv.Atoi(request.ClassCode)
+	if err != nil {
+		return models.ResponseBodyNextPage{}, errors.New("failed to parse the class code to int: " + request.ClassCode)
+	}
+	court, err := strconv.Atoi(request.CourtCode)
+	if err != nil {
+		return models.ResponseBodyNextPage{}, errors.New("failed to parse the court code to int: " + request.CourtCode)
+	}
+
+	req := models.BodyRequestCode{
+		Size: 100,
+		Query: models.Query{
+			Bool: models.Bool{
+				Must: []models.Must{
+					{Match: models.Match{ClasseCodigo: class}},
+					{Match: models.Match{OrgaoJulgadorCodigo: court}},
+				},
+			},
+		},
+		Sort: []models.Sort{{models.Timestamp{Order: "asc"}}},
+	}
+
+	// Serialize the BodyRequestLawsuit struct to JSON.
+	jsonReq, err := json.Marshal(req)
+	if err != nil {
+		return models.ResponseBodyNextPage{}, errors.New(err.Error())
+	}
+
+	// Create responseReturn new buffer with the JSON-encoded request body.
+	reqBody := bytes.NewBuffer(jsonReq)
+
+	// Make the API call and get the response.
+	res, err := call(url, method, auth, reqBody)
+	if err != nil {
+		return models.ResponseBodyNextPage{}, errors.New(err.Error())
+	}
+
+	// Read the response body.
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		return models.ResponseBodyNextPage{}, errors.New(err.Error())
+	}
+
+	// Unmarshal the response body into responseReturn ResponseBodyLawsuit struct.
+	var response models.ResponseBodyNextPage
+	err = json.Unmarshal(body, &response)
+	if err != nil {
+		return models.ResponseBodyNextPage{}, errors.New(err.Error())
+	}
+
+	responseReturn := models.ResponseBodyNextPage{
+		Took:     response.Took,
+		TimedOut: response.TimedOut,
+		Shards:   response.Shards,
+		Hit:      response.Hit,
+	}
+
+	// If the API response has more pages of data, make additional API calls and append the results to the response.
+	if len(response.Hit.Hits) == 100 {
+		nextPage := strconv.Itoa(int(response.Hit.Hits[99].Sort[0]))
+		// Make the API call and get the response.
+		lawsuits, err := callNextPage(url, method, auth, req, nextPage)
+		if err != nil {
+			return models.ResponseBodyNextPage{}, errors.New(err.Error())
+		}
+
+		responseReturn.Hit.Hits = append(responseReturn.Hit.Hits, lawsuits...)
+		return responseReturn, nil
+	}
+
+	return responseReturn, nil
+}
+
+// callNextPage returns a slice of models.Lawsuit structs containing the data from all pages of the API response.
+func callNextPage(url string, method string, auth string, req models.BodyRequestCode, cursor string) ([]models.Hit2NextPage, error) {
+	var lawsuits []models.Hit2NextPage
+	c, err := strconv.Atoi(cursor)
+	if err != nil {
+		return lawsuits, errors.New("could not covert cursor into int: " + cursor)
+	}
+
+	for {
+		log.Println("success getting next page:" + strconv.Itoa(c))
+		// Create a new BodyRequest struct with the document ID and updated pagination settings for the next API call.
+		request := models.BodyRequestCodeNextPage{
+			Size: 100,
+			Query: models.Query{
+				Bool: models.Bool{
+					Must: []models.Must{
+						{Match: models.Match{ClasseCodigo: req.Query.Bool.Must[0].Match.ClasseCodigo}},
+						{Match: models.Match{OrgaoJulgadorCodigo: req.Query.Bool.Must[1].Match.OrgaoJulgadorCodigo}},
+					},
+				},
+			},
+			Sort:        []models.Sort{{models.Timestamp{Order: "asc"}}},
+			SearchAfter: []int64{int64(c)},
+		}
+
+		// Serialize the BodyRequest struct to JSON.
+		jsonReq, err := json.Marshal(request)
+		if err != nil {
+			log.Println(err)
+			return lawsuits, err
+		}
+
+		// Create a new buffer with the JSON-encoded request body.
+		reqBody := bytes.NewBuffer(jsonReq)
+
+		// Call the API using the provided url, method, authorization, and request body.
+		res, err := call(url, method, auth, reqBody)
+		if err != nil {
+			log.Println(err)
+			return lawsuits, errors.New(err.Error())
+		}
+
+		// Read the response body.
+		body, err := ioutil.ReadAll(res.Body)
+		if err != nil {
+			log.Println(err)
+			return lawsuits, err
+		}
+
+		// Unmarshal the response body into a models.ResponseBody struct.
+		var response models.ResponseBodyNextPage
+		err = json.Unmarshal(body, &response)
+		if err != nil {
+			log.Println(err)
+			return lawsuits, err
+		}
+
+		// Append the current response to the lawsuits slice.
+		lawsuits = append(lawsuits, response.Hit.Hits...)
+
+		// If the API response indicates there are no more pages, break out of the loop.
+		if len(response.Hit.Hits) < 100 {
+			break
+		}
+
+		// Update the cursor for the next API call.
+		c = int(response.Hit.Hits[99].Sort[0])
+	}
+
+	return lawsuits, nil
+}