Add unstructured package components for document processing

- Introduced new types and constants for block types, chunkers, partitioners, and encoders.
- Implemented various chunker types (Character, Title, Page, Similarity) and partitioner strategies (Auto, Fast, HiRes, VLM).
- Added support for encoding types and excludable elements in document processing.
- Enhanced the client with new methods for running workflows and downloading job outputs.
- Updated the Go module to version 1.25.0 and added comprehensive test cases for the new functionality.

Author: aws-gopher

block_types.go

@@ -0,0 +1,10 @@
+package unstructured
+
+// BlockType is a type that represents a block type.
+type BlockType string
+
+// BlockType constants.
+const (
+	BlockTypeImage BlockType = "Image"
+	BlockTypeTable BlockType = "Table"
+)

chunker_character.go

@@ -0,0 +1,54 @@
+package unstructured
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// ChunkerCharacter is a node that chunks text by character.
+type ChunkerCharacter struct {
+	ID                         string           `json:"-"`
+	Name                       string           `json:"-"`
+	APIURL                     string           `json:"unstructured_api_url,omitempty"`
+	APIKey                     string           `json:"unstructured_api_key,omitempty"`
+	IncludeOrigElements        bool             `json:"include_orig_elements,omitempty"`
+	NewAfterNChars             int              `json:"new_after_n_chars,omitempty"`
+	MaxCharacters              int              `json:"max_characters,omitempty"`
+	Overlap                    int              `json:"overlap,omitempty"`
+	OverlapAll                 bool             `json:"overlap_all"`
+	ContextualChunkingStrategy ChunkingStrategy `json:"contextual_chunking_strategy,omitempty"`
+}
+
+// ChunkingStrategy is a strategy for contextual chunking.
+type ChunkingStrategy string
+
+// ChunkingStrategyV1 is a strategy for contextual chunking.
+const ChunkingStrategyV1 = "v1"
+
+var _ WorkflowNode = new(ChunkerCharacter)
+
+// isNode implements the WorkflowNode interface.
+func (c ChunkerCharacter) isNode() {}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (c ChunkerCharacter) MarshalJSON() ([]byte, error) {
+	type alias ChunkerCharacter
+
+	data, err := json.Marshal(alias(c))
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker character: %w", err)
+	}
+
+	headerData, err := json.Marshal(header{
+		ID:       c.ID,
+		Name:     c.Name,
+		Type:     nodeTypeChunk,
+		Subtype:  string(ChunkerSubtypeCharacter),
+		Settings: json.RawMessage(data),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker character header: %w", err)
+	}
+
+	return headerData, nil
+}

chunker_page.go

@@ -0,0 +1,48 @@
+package unstructured
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// ChunkerPage is a node that chunks text by character.
+type ChunkerPage struct {
+	ID                  string           `json:"-"`
+	Name                string           `json:"-"`
+	APIURL              string           `json:"unstructured_api_url,omitempty"`
+	APIKey              string           `json:"unstructured_api_key,omitempty"`
+	IncludeOrigElements bool             `json:"include_orig_elements,omitempty"`
+	NewAfterNChars      int              `json:"new_after_n_chars,omitempty"`
+	MaxCharacters       int              `json:"max_characters,omitempty"`
+	Overlap             int              `json:"overlap,omitempty"`
+	OverlapAll          bool             `json:"overlap_all"`
+	Strategy            ChunkingStrategy `json:"contextual_chunking_strategy,omitempty"`
+}
+
+var _ WorkflowNode = new(ChunkerPage)
+
+// isNode implements the WorkflowNode interface.
+func (c ChunkerPage) isNode() {}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (c ChunkerPage) MarshalJSON() ([]byte, error) {
+	type alias ChunkerPage
+
+	data, err := json.Marshal(alias(c))
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker page: %w", err)
+	}
+
+	headerData, err := json.Marshal(header{
+		ID:       c.ID,
+		Name:     c.Name,
+		Type:     nodeTypeChunk,
+		Subtype:  string(ChunkerSubtypePage),
+		Settings: json.RawMessage(data),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker page header: %w", err)
+	}
+
+	return headerData, nil
+}

chunker_similarity.go

@@ -0,0 +1,48 @@
+package unstructured
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// ChunkerSimilarity is a node that chunks text by character.
+type ChunkerSimilarity struct {
+	ID                  string           `json:"-"`
+	Name                string           `json:"-"`
+	APIURL              string           `json:"unstructured_api_url,omitempty"`
+	APIKey              string           `json:"unstructured_api_key,omitempty"`
+	IncludeOrigElements bool             `json:"include_orig_elements,omitempty"`
+	NewAfterNChars      int              `json:"new_after_n_chars,omitempty"`
+	MaxCharacters       int              `json:"max_characters,omitempty"`
+	Overlap             int              `json:"overlap,omitempty"`
+	OverlapAll          bool             `json:"overlap_all"`
+	Strategy            ChunkingStrategy `json:"contextual_chunking_strategy,omitempty"`
+}
+
+var _ WorkflowNode = new(ChunkerSimilarity)
+
+// isNode implements the WorkflowNode interface.
+func (c ChunkerSimilarity) isNode() {}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (c ChunkerSimilarity) MarshalJSON() ([]byte, error) {
+	type alias ChunkerSimilarity
+
+	data, err := json.Marshal(alias(c))
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker similarity: %w", err)
+	}
+
+	headerData, err := json.Marshal(header{
+		ID:       c.ID,
+		Name:     c.Name,
+		Type:     nodeTypeChunk,
+		Subtype:  string(ChunkerSubtypeSimilarity),
+		Settings: json.RawMessage(data),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker similarity header: %w", err)
+	}
+
+	return headerData, nil
+}

chunker_title.go

@@ -0,0 +1,49 @@
+package unstructured
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// ChunkerTitle is a node that chunks text by character.
+type ChunkerTitle struct {
+	ID                         string           `json:"-"`
+	Name                       string           `json:"-"`
+	APIURL                     string           `json:"unstructured_api_url,omitempty"`
+	APIKey                     string           `json:"unstructured_api_key,omitempty"`
+	CombineTextUnderN          int              `json:"combine_text_under_n_chars,omitempty"`
+	IncludeOrigElements        bool             `json:"include_orig_elements,omitempty"`
+	NewAfterNChars             int              `json:"new_after_n_chars,omitempty"`
+	MaxCharacters              int              `json:"max_characters,omitempty"`
+	Overlap                    int              `json:"overlap,omitempty"`
+	OverlapAll                 bool             `json:"overlap_all"`
+	ContextualChunkingStrategy ChunkingStrategy `json:"contextual_chunking_strategy,omitempty"`
+}
+
+var _ WorkflowNode = new(ChunkerTitle)
+
+// isNode implements the WorkflowNode interface.
+func (c ChunkerTitle) isNode() {}
+
+// MarshalJSON implements the json.Marshaler interface.
+func (c ChunkerTitle) MarshalJSON() ([]byte, error) {
+	type alias ChunkerTitle
+
+	data, err := json.Marshal(alias(c))
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker title: %w", err)
+	}
+
+	headerData, err := json.Marshal(header{
+		ID:       c.ID,
+		Name:     c.Name,
+		Type:     nodeTypeChunk,
+		Subtype:  string(ChunkerSubtypeTitle),
+		Settings: json.RawMessage(data),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("failed to marshal chunker title header: %w", err)
+	}
+
+	return headerData, nil
+}

chunker_type.go

@@ -0,0 +1,56 @@
+package unstructured
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// ChunkerSubtype is a type that represents a chunker subtype.
+type ChunkerSubtype string
+
+// ChunkerSubtype constants.
+const (
+	ChunkerSubtypeCharacter  ChunkerSubtype = "chunk_by_character"
+	ChunkerSubtypeTitle      ChunkerSubtype = "chunk_by_title"
+	ChunkerSubtypePage       ChunkerSubtype = "chunk_by_page"
+	ChunkerSubtypeSimilarity ChunkerSubtype = "chunk_by_similarity"
+)
+
+func unmarshalChunker(header header) (WorkflowNode, error) {
+	var chunker WorkflowNode
+
+	switch ChunkerSubtype(header.Subtype) {
+	case ChunkerSubtypeCharacter:
+		chunker = &ChunkerCharacter{
+			ID:   header.ID,
+			Name: header.Name,
+		}
+
+	case ChunkerSubtypeTitle:
+		chunker = &ChunkerTitle{
+			ID:   header.ID,
+			Name: header.Name,
+		}
+
+	case ChunkerSubtypePage:
+		chunker = &ChunkerPage{
+			ID:   header.ID,
+			Name: header.Name,
+		}
+
+	case ChunkerSubtypeSimilarity:
+		chunker = &ChunkerSimilarity{
+			ID:   header.ID,
+			Name: header.Name,
+		}
+
+	default:
+		return nil, fmt.Errorf("unknown Chunker strategy: %s", header.Subtype)
+	}
+
+	if err := json.Unmarshal(header.Settings, chunker); err != nil {
+		return nil, fmt.Errorf("failed to unmarshal Chunker node: %w", err)
+	}
+
+	return chunker, nil
+}

client.go

@@ -4,6 +4,7 @@ import (
 	"cmp"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"os"
@@ -55,6 +56,15 @@ func WithKey(key string) Option {
 	}
 }
 
+// WithClient returns an Option that sets the HTTP client to use for requests.
+// If no client is provided, the client will default to [http.DefaultClient].
+func WithClient(hc *http.Client) Option {
+	return func(c *Client) error {
+		c.hc = hc
+		return nil
+	}
+}
+
 // New creates a new Client instance with the provided options.
 // If the `UNSTRUCTURED_API_KEY` environment variable is set, it will be used as the API key for authentication.
 // If the `UNSTRUCTURED_API_URL` environment variable is set to a valid URL, it will be used as the base URL for the Unstructured.io API.
@@ -104,17 +114,20 @@ func (c *Client) do(req *http.Request, out any) error {
 	defer func() { _ = resp.Body.Close() }()
 
 	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+		body, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("failed to read response body: %w", err)
+		}
+
 		// Handle 422 validation errors specifically
 		if resp.StatusCode == http.StatusUnprocessableEntity {
 			var validationErr HTTPValidationError
-			if err := json.NewDecoder(resp.Body).Decode(&validationErr); err != nil {
-				return fmt.Errorf("failed to decode validation error response: %w", err)
+			if err := json.Unmarshal(body, &validationErr); err == nil {
+				return &validationErr
 			}
-
-			return &validationErr
 		}
 
-		return fmt.Errorf("unsuccessful response: %s", resp.Status)
+		return fmt.Errorf("[%s]: %s", resp.Status, string(body))
 	}
 
 	if out != nil {

encoding.go

@@ -0,0 +1,45 @@
+package unstructured
+
+import "strings"
+
+// Encoding is a type that represents an encoding.
+type Encoding string
+
+// Encoding constants.
+const (
+	EncodingUTF8          Encoding = "utf_8"
+	EncodingISO88591      Encoding = "iso_8859_1"
+	EncodingISO88596      Encoding = "iso_8859_6"
+	EncodingISO88598      Encoding = "iso_8859_8"
+	EncodingASCII         Encoding = "ascii"
+	EncodingBig5          Encoding = "big5"
+	EncodingUTF16         Encoding = "utf_16"
+	EncodingUTF16Be       Encoding = "utf_16_be"
+	EncodingUTF16Le       Encoding = "utf_16_le"
+	EncodingUTF32         Encoding = "utf_32"
+	EncodingUTF32Be       Encoding = "utf_32_be"
+	EncodingUTF32Le       Encoding = "utf_32_le"
+	EncodingEUCJIS2004    Encoding = "euc_jis_2004"
+	EncodingEUCJISX0213   Encoding = "euc_jisx0213"
+	EncodingEUCJP         Encoding = "euc_jp"
+	EncodingEUCKR         Encoding = "euc_kr"
+	EncodingGb18030       Encoding = "gb18030"
+	EncodingSHIFTJIS      Encoding = "shift_jis"
+	EncodingSHIFTJIS2004  Encoding = "shift_jis_2004"
+	EncodingSHIFTJISX0213 Encoding = "shift_jisx0213"
+)
+
+// String implements the fmt.Stringer interface, canonicalizing the encoding name.
+func (e Encoding) String() string {
+	s := strings.TrimSpace(string(e))
+	s = strings.ToLower(s)
+	s = strings.ReplaceAll(s, "_", "-")
+
+	switch s {
+	case "iso_8859_6_i", "iso_8859_8_i",
+		"iso_8859_6_e", "iso_8859_8_e":
+		s = s[:len(s)-2]
+	}
+
+	return s
+}