Add opt-in identity topic manager and lookup service (#368)

Author: sirdeggen

api/openapi/paths/non_admin/responses.yaml

@@ -88,8 +88,18 @@ components:
       type: object
       properties:
         beef:
-          type: string
-          format: byte
+          type: array
+          description: >-
+            BEEF bytes encoded as the BRC-24 JSON byte array. Earlier Go
+            server releases emitted a base64 string here through Go's []byte
+            JSON encoding; that legacy representation is not BRC-24 JSON
+            interoperable.
+          x-go-type: '[]int32'
+          items:
+            type: integer
+            format: int32
+            minimum: 0
+            maximum: 255
         outputIndex:
           type: integer
           format: uint32

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/spf13/viper v1.21.0
 	github.com/stretchr/testify v1.12.1
 	golang.org/x/sync v0.23.0
+	golang.org/x/text v0.41.0
 	gopkg.in/yaml.v3 v3.0.1
 )
 
@@ -54,7 +55,6 @@ require (
 	golang.org/x/mod v0.38.0 // indirect
 	golang.org/x/net v0.58.0 // indirect
 	golang.org/x/sys v0.47.0 // indirect
-	golang.org/x/text v0.41.0 // indirect
 	golang.org/x/tools v0.48.0 // indirect
 )
 

pkg/core/engine/tests/engine_lookup_test.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"testing"
 
-	"github.com/bsv-blockchain/go-sdk/chainhash"
 	"github.com/bsv-blockchain/go-sdk/overlay/lookup"
 	"github.com/bsv-blockchain/go-sdk/transaction"
 	"github.com/stretchr/testify/require"
@@ -111,13 +110,9 @@ func TestEngine_Lookup_ShouldReturnDirectResult_WhenAnswerTypeIsOutputList(t *te
 func TestEngine_Lookup_ShouldHydrateOutputs_WhenFormulasProvided(t *testing.T) {
 	// given
 	ctx := context.Background()
-	outpoint := &transaction.Outpoint{Txid: fakeTxID(t), Index: 0}
-
-	// Create a proper BEEF object for testing
-	expectedBeef := &transaction.Beef{
-		Version:      transaction.BEEF_V2,
-		Transactions: make(map[chainhash.Hash]*transaction.BeefTx),
-	}
+	beef, subjectTx, _, err := transaction.ParseBeef(createDummyBEEF(t))
+	require.NoError(t, err)
+	outpoint := &transaction.Outpoint{Txid: *subjectTx.TxID(), Index: 0}
 
 	sut := engine.NewEngine(&engine.Config{
 		LookupServices: map[string]engine.LookupService{
@@ -126,7 +121,7 @@ func TestEngine_Lookup_ShouldHydrateOutputs_WhenFormulasProvided(t *testing.T) {
 					return &lookup.LookupAnswer{
 						Type: lookup.AnswerTypeFormula,
 						Formulas: []lookup.LookupFormula{
-							{Outpoint: &transaction.Outpoint{Txid: fakeTxID(t), Index: 0}},
+							{Outpoint: outpoint},
 						},
 					}, nil
 				},
@@ -136,7 +131,7 @@ func TestEngine_Lookup_ShouldHydrateOutputs_WhenFormulasProvided(t *testing.T) {
 			findOutputFunc: func(_ context.Context, outpoint *transaction.Outpoint, _ *string, _ *bool, _ bool) (*engine.Output, error) {
 				return &engine.Output{
 					Outpoint: *outpoint,
-					Beef:     expectedBeef,
+					Beef:     beef,
 				}, nil
 			},
 		},
@@ -150,4 +145,7 @@ func TestEngine_Lookup_ShouldHydrateOutputs_WhenFormulasProvided(t *testing.T) {
 	require.Equal(t, lookup.AnswerTypeOutputList, actualAnswer.Type)
 	require.Len(t, actualAnswer.Outputs, 1)
 	require.Equal(t, outpoint.Index, actualAnswer.Outputs[0].OutputIndex)
+	_, hydratedTx, _, err := transaction.ParseBeef(actualAnswer.Outputs[0].Beef)
+	require.NoError(t, err)
+	require.Equal(t, subjectTx.Bytes(), hydratedTx.Bytes())
 }

pkg/server/internal/ports/lookup_question_handler.go

@@ -71,7 +71,7 @@ func NewLookupQuestionSuccessResponse(dto *app.LookupAnswerDTO) (*openapi.Lookup
 		outputs = make([]openapi.OutputListItem, len(dto.Outputs))
 		for i, output := range dto.Outputs {
 			outputs[i] = openapi.OutputListItem{
-				Beef:        output.BEEF,
+				Beef:        brc24Bytes(output.BEEF),
 				OutputIndex: output.OutputIndex,
 			}
 		}
@@ -83,3 +83,17 @@ func NewLookupQuestionSuccessResponse(dto *app.LookupAnswerDTO) (*openapi.Lookup
 		Type:    dto.Type,
 	}, nil
 }
+
+// brc24Bytes converts Go's base64-marshaled []byte form into BRC-24's
+// portable JSON byte array. This conversion is confined to the JSON lookup
+// response; aggregated binary lookup responses are unchanged.
+func brc24Bytes(bytes []byte) []int32 {
+	if bytes == nil {
+		return nil
+	}
+	result := make([]int32, len(bytes))
+	for i, value := range bytes {
+		result[i] = int32(value)
+	}
+	return result
+}

pkg/server/internal/ports/lookup_question_handler_test.go

@@ -1,6 +1,7 @@
 package ports_test
 
 import (
+	"encoding/json"
 	"testing"
 
 	"github.com/bsv-blockchain/go-sdk/overlay/lookup"
@@ -102,3 +103,39 @@ func TestLookupQuestionHandler_ValidCase(t *testing.T) {
 
 	stub.AssertProvidersState()
 }
+
+func TestLookupQuestionHandlerOutputListUsesBRC24ByteArray(t *testing.T) {
+	stub := testabilities.NewTestOverlayEngineStub(t, testabilities.WithLookupQuestionProvider(testabilities.NewLookupQuestionProviderMock(t, testabilities.LookupQuestionProviderMockExpectations{
+		LookupQuestionCall: true,
+		Answer: &lookup.LookupAnswer{
+			Type: lookup.AnswerTypeOutputList,
+			Outputs: []*lookup.OutputListItem{{
+				Beef:        []byte{1, 2, 255},
+				OutputIndex: 7,
+			}},
+		},
+	})))
+	fixture := server.NewTestFixture(t, server.WithEngine(stub))
+
+	res, err := fixture.Client().
+		R().
+		SetHeader("Content-Type", "application/json").
+		SetBody(openapi.LookupQuestionJSONRequestBody{
+			Query:   map[string]any{},
+			Service: "test-service",
+		}).
+		Post("/api/v1/lookup")
+
+	require.NoError(t, err)
+	require.Equal(t, fiber.StatusOK, res.StatusCode())
+	var body struct {
+		Outputs []struct {
+			Beef json.RawMessage `json:"beef"`
+		} `json:"outputs"`
+	}
+	require.NoError(t, json.Unmarshal(res.Body(), &body))
+	require.Len(t, body.Outputs, 1)
+	require.JSONEq(t, `[1,2,255]`, string(body.Outputs[0].Beef))
+
+	stub.AssertProvidersState()
+}

pkg/server/internal/ports/openapi/openapi_non_admin_response_types.gen.go

@@ -0,0 +1,125 @@
+# Identity companion service
+
+This opt-in package supplies `engine.TopicManager` for `tm_identity` and
+`engine.LookupService` for `ls_identity`. It does not register either service,
+start HTTP, or supply a database adapter. The generic engine must independently
+verify transactions, scripts and chain proofs before topic admission.
+
+`NewTopicManager` evaluates outputs independently. It verifies the exact
+concatenation of PushDrop fields using the subject's `[1, "identity"]`, key
+ID `"1"` signature, then verifies the certificate and decrypts public fields.
+The certificate subject need not equal the derived script locking key.
+Retention is always empty. Admission alone does not establish SPV validity,
+unspentness, certificate revocation status, freshness or trusted-certifier policy.
+
+`NewLookupService(projection)` indexes admitted outputs and removes them on
+spend or legal eviction. History-retention and block-height notifications do
+not change the current-output index. Lookup results are actual outpoint
+formulas for engine BEEF hydration. `ProjectOutput` exposes the same validated
+public record derivation without a storage write for future atomic admission
+and projection-intent integration.
+
+## Projection contract
+
+Implement `Projection` with a durable adapter and explicitly inject the topic
+and lookup service into the engine's configuration. `Upsert` must use the
+outpoint as a unique key, and repeated `Delete` calls must succeed. Public
+decrypted fields remain in `Record.Certificate.Fields`; searchable text excludes
+`profilePhoto` and `icon`. Its concatenation preserves TS keyring property
+enumeration, including numeric field names.
+
+`Find` must honor `Query.Kind`, including an empty-string identity key, all
+selected exact filters, attribute predicates, offset and limit. Attribute
+predicates combine with AND. Empty optional certifier filters are unrestricted;
+an empty certifier-only query is empty. No new ordering guarantee is implied.
+`Field == ""` selects `SearchableAttributes`; otherwise it names a literal
+certificate field. Adapters must use server-owned operators and must not splice
+query values into database operators or regular expressions.
+
+Preserved TS query semantics:
+
+- Precedence: serial number; attributes; identity key with certificate types;
+  identity key; certifiers. Serial number ignores lower-priority filters.
+- Attribute whitespace follows ECMAScript trimming/collapsing. `userName` is an
+  exact, case-sensitive normalized match; other attributes use escaped tokens
+  joined by `.*` with case-insensitive matching.
+- `attributes.any` takes precedence within attributes. After normalization,
+  fewer than two UTF-16 code units yields empty; two uses a fuzzy text regex;
+  more than two requires the native Mongo text-search semantics/index used by
+  the TS service. It must not be approximated by substring or fuzzy matching.
+- Empty/blank attribute searches are empty. Native Mongo text language,
+  stemming, stop words, phrases and index behavior need real adapter tests.
+
+The adapter and host own admission/projection atomicity, ordered replay,
+tombstones or equivalent fencing, outbox retries, read-your-write boundaries,
+rebuilds, migration and readiness. An old admission replay must not resurrect
+an output after spend/eviction. Callback idempotency alone cannot guarantee
+this. The legacy engine callbacks are not an atomic transaction merely because
+this interface exists. Use the separately reviewed engine persistence/outbox
+capability when integrating; this package adds no competing operation ledger.
+
+## Budgets and narrow compatibility limits
+
+`DefaultAdmissionPolicy()` limits one output script to 1 MiB, certificate and
+PushDrop fields to 128, selected outputs to 10,000, total selected transaction
+locking-script bytes to 32 MiB, and engine notification BEEF to 64 MiB.
+`NewTopicManagerWithPolicy` and `NewLookupServiceWithPolicies` accept explicit
+positive policy values. These are application budgets; the host separately
+needs bounded transport and BEEF/transaction/proof parsing. Notification BEEF
+is an already admitted engine payload, not an independent network ingress API.
+
+`DefaultQueryPolicy()` permits at most 10,000 results and offset 100,000.
+The policy may lower those operational ceilings. Explicit positive limits
+within the cap retain limit/offset behavior. Omitted or zero legacy limits
+fetch `cap + 1`: results within the cap succeed; overflow returns `ErrQueryBudget`
+and requires explicit bounded pagination. No truncated success or new wire
+metadata is invented. Large offsets remain workload-dependent scans, not a
+promise of cheap access.
+
+Queries are capped at 64 KiB, 4 KiB per string, 64 attributes and 128 certifiers
+or certificate types. Pagination accepts finite integral JSON numbers and
+rejects negative, fractional, quoted or over-budget values before projection
+work. Attribute selectors must be nonempty and exclude `.`, `$` and NUL to
+prevent unsafe Mongo path construction; certificate data is not normalized or
+renamed. Those selectors and resource ceilings are explicit restrictions on
+the otherwise permissive TS input shape.
+
+## Certificate serialization checkpoint
+
+This component deliberately uses the existing Go SDK v1.4.1 certificate
+verification profile. Its serializer orders UTF-8 field names with Go byte
+ordering. Pinned TS `2bc799a8d8e535242e6de2d305f426ce3975ea7b` reconstructs
+the certificate preimage with `localeCompare`; even mixed ASCII case can
+differ. Collation ties preserve JSON insertion order. The PushDrop envelope
+carries JSON and the subject signature, not a separate signed binary preimage.
+
+The common-compatible real TS fixtures pass. Other legitimate TS certificates
+may not verify under this profile. `ErrCertificateVerification` describes a
+failure to verify under the Go profile, not a diagnosis of forgery. There is
+no alternate-order retry, arbitrary permutation search, ambient locale
+dependency or normalization of signed data. Full TS certificate interoperability
+remains open pending explicit deterministic profile/migration review. The
+[portable fixture provenance](testdata/data/provenance.md) records mixed ASCII,
+accent/combining ties, astral/BMP and searchable-field cases with exact bytes.
+
+## Verification and integration gates
+
+Package tests consume actual TS signed certificate/PushDrop/transaction/BEEF
+bytes. They exercise independently admitted/rejected outputs, exact signature
+and decryption checks, replay/upsert and deletion seams, query behavior and
+budgets, and malformed scripts. In-memory projections exist only in tests;
+they do not claim Mongo text-search or persistent-server acceptance.
+
+Run package tests with `GOTOOLCHAIN=go1.26.8 go test ./pkg/topics/identity`.
+The fixture generator requires the pinned TS sources and built artifacts;
+committed fixture bytes make ordinary Go checks independent of Node or a
+database. Regeneration uses SDK randomness, producing new signed bytes with
+the same tested behavior, and records their SHA-256 digests.
+
+Remaining acceptance includes the durable Mongo adapter/indexes, atomic engine
+admission/projection/outbox integration, actual engine+HTTP lookup consumed by
+the real TS IdentityClient/wallet, restart/spend/eviction, independent chain
+validation and identity-history integration. SDK v1.4.1 also misserializes
+parsed V1 BEEF through `AtomicBytes`; the generic SDK repair is a separate
+primary-profile dependency. Tests here consume exact TS `toAtomicBEEF()` bytes
+and do not conceal that unresolved engine interoperability dependency.

pkg/topics/identity/README.md

@@ -0,0 +1,65 @@
+// Package identity implements the opt-in tm_identity application rules and
+// ls_identity lookup service. Transaction/script/SPV verification belongs to the
+// engine and must precede admission. Certificate admission alone establishes
+// neither transaction validity, unspentness, revocation status nor client trust.
+//
+// A host must inject a durable Projection and register both services explicitly.
+// This package supplies no production storage or default server registration.
+package identity
+
+import (
+	"errors"
+)
+
+const (
+	// Topic is the identity certificate admission topic.
+	Topic = "tm_identity"
+	// Service is the public identity lookup service.
+	Service = "ls_identity"
+)
+
+var (
+	// ErrInvalidOutput means an output does not satisfy identity application rules.
+	ErrInvalidOutput = errors.New("invalid identity output")
+	// ErrCertificateVerification means the certifier signature could not be
+	// verified using the Go SDK serialization profile. This may indicate an
+	// invalid signature or an unsupported TS field-order profile; it does not
+	// diagnose forgery. The package does not try alternative permutations.
+	ErrCertificateVerification = errors.New("identity certificate verification failed under Go SDK serialization profile")
+	// ErrAdmissionBudget means a configured application validation budget was exceeded.
+	ErrAdmissionBudget = errors.New("identity admission budget exceeded")
+	// ErrInvalidTransaction means the selected transaction is absent or malformed.
+	ErrInvalidTransaction = errors.New("invalid identity transaction")
+	// ErrInvalidPolicy means a service resource policy is unusable.
+	ErrInvalidPolicy = errors.New("invalid identity policy")
+)
+
+// AdmissionPolicy bounds work on identity output scripts, separately from the
+// host's transport, transaction graph, proof and engine admission budgets.
+type AdmissionPolicy struct {
+	MaxScriptBytes       int
+	MaxFields            int
+	MaxOutputs           int
+	MaxTotalScriptBytes  int
+	MaxNotificationBytes int
+}
+
+// DefaultAdmissionPolicy returns finite limits for this opt-in application.
+// Larger payload deployments must select and test their own limits explicitly.
+func DefaultAdmissionPolicy() AdmissionPolicy {
+	return AdmissionPolicy{
+		MaxScriptBytes:       1 << 20,
+		MaxFields:            128,
+		MaxOutputs:           10000,
+		MaxTotalScriptBytes:  32 << 20,
+		MaxNotificationBytes: 64 << 20,
+	}
+}
+
+func (p AdmissionPolicy) validate() error {
+	if p.MaxScriptBytes < 1 || p.MaxFields < 1 || p.MaxOutputs < 1 ||
+		p.MaxTotalScriptBytes < p.MaxScriptBytes || p.MaxNotificationBytes < 1 {
+		return ErrInvalidPolicy
+	}
+	return nil
+}