Add shared client support with log management

This commit implements shared client functionality to improve resource
efficiency by allowing clients to be reused across multiple tests in a
test suite. Key features include:

- New API endpoints for shared client management
- Data structures for tracking shared client state
- Log segment extraction for proper test result attribution
- Integration with hiveview for log display
- Documentation updates for all new methods and types

Author: danceratopz

hivesim/data.go

@@ -23,6 +23,18 @@ type TestStartInfo struct {
 	Description string `json:"description"`
 }
 
+// LogOffset tracks the start and end positions in a log file.
+type LogOffset struct {
+	Start int64 `json:"start"` // Byte offset where this section begins
+	End   int64 `json:"end"`   // Byte offset where this section ends
+}
+
+// ClientLogInfo tracks log offsets for a specific client in a test.
+type ClientLogInfo struct {
+	ClientID  string    `json:"client_id"` // ID of the client container
+	LogOffset LogOffset `json:"log_offset"` // Offset range in the log file
+}
+
 // ExecInfo is the result of running a command in a client container.
 type ExecInfo struct {
 	Stdout   string `json:"stdout"`
@@ -46,3 +58,24 @@ type ClientDefinition struct {
 func (m *ClientDefinition) HasRole(role string) bool {
 	return slices.Contains(m.Meta.Roles, role)
 }
+
+// ClientMode defines whether a client is shared across tests or dedicated to a single test.
+// Two modes are supported: DedicatedClient (default) and SharedClient.
+type ClientMode int
+
+const (
+	// DedicatedClient is a client that is used for a single test (default behavior)
+	DedicatedClient ClientMode = iota
+	// SharedClient is a client that is shared across multiple tests in a suite
+	SharedClient
+)
+
+// SharedClientInfo contains information about a shared client instance.
+// This includes container identification and connectivity information.
+type SharedClientInfo struct {
+	ID        string // Container ID
+	Type      string // Client type
+	IP        string // Client IP address
+	CreatedAt int64  // Timestamp when client was created
+	LogFile   string // Path to client log file
+}

hivesim/hive.go

@@ -207,6 +207,69 @@ func (sim *Simulation) StartClientWithOptions(testSuite SuiteID, test TestID, cl
 	return resp.ID, ip, nil
 }
 
+// StartSharedClient starts a new node as a shared client at the suite level.
+// The client persists for the duration of the suite and can be used by multiple tests.
+// Returns container id and ip.
+func (sim *Simulation) StartSharedClient(testSuite SuiteID, clientType string, options ...StartOption) (string, net.IP, error) {
+	if sim.docs != nil {
+		return "", nil, errors.New("StartSharedClient is not supported in docs mode")
+	}
+	var (
+		url  = fmt.Sprintf("%s/testsuite/%d/shared-client", sim.url, testSuite)
+		resp simapi.StartNodeResponse
+	)
+
+	setup := &clientSetup{
+		files: make(map[string]func() (io.ReadCloser, error)),
+		config: simapi.NodeConfig{
+			Client:      clientType,
+			Environment: make(map[string]string),
+			IsShared:    true, // Mark this client as shared
+		},
+	}
+	for _, opt := range options {
+		opt.apply(setup)
+	}
+
+	err := setup.postWithFiles(url, &resp)
+	if err != nil {
+		return "", nil, err
+	}
+	ip := net.ParseIP(resp.IP)
+	if ip == nil {
+		return resp.ID, nil, fmt.Errorf("no IP address returned")
+	}
+	return resp.ID, ip, nil
+}
+
+// GetSharedClientInfo retrieves information about a shared client,
+// including its ID, type, IP and log file location.
+func (sim *Simulation) GetSharedClientInfo(testSuite SuiteID, clientID string) (*SharedClientInfo, error) {
+	if sim.docs != nil {
+		return nil, errors.New("GetSharedClientInfo is not supported in docs mode")
+	}
+	var (
+		url  = fmt.Sprintf("%s/testsuite/%d/shared-client/%s", sim.url, testSuite, clientID)
+		resp SharedClientInfo
+	)
+	err := get(url, &resp)
+	return &resp, err
+}
+
+// GetClientLogOffset gets the current offset position in a shared client's log file.
+// This is used for tracking log segments in shared clients across multiple tests.
+func (sim *Simulation) GetClientLogOffset(testSuite SuiteID, clientID string) (int64, error) {
+	if sim.docs != nil {
+		return 0, errors.New("GetClientLogOffset is not supported in docs mode")
+	}
+	var (
+		url  = fmt.Sprintf("%s/testsuite/%d/shared-client/%s/log-offset", sim.url, testSuite, clientID)
+		resp int64
+	)
+	err := get(url, &resp)
+	return resp, err
+}
+
 // StopClient signals to the host that the node is no longer required.
 func (sim *Simulation) StopClient(testSuite SuiteID, test TestID, nodeid string) error {
 	if sim.docs != nil {

hivesim/testapi.go

@@ -21,6 +21,12 @@ type Suite struct {
 	Category    string // Category of the test suite [Optional]
 	Description string // Description of the test suite (if empty, suite won't appear in documentation) [Optional]
 	Tests       []AnyTest
+
+	// SharedClients maps client IDs to client instances that are shared across tests
+	SharedClients map[string]*Client
+
+	// Internal tracking
+	sharedClientOpts map[string][]StartOption // Stores options for starting shared clients
 }
 
 func (s *Suite) request() *simapi.TestRequest {
@@ -39,6 +45,30 @@ func (s *Suite) Add(test AnyTest) *Suite {
 	return s
 }
 
+// AddSharedClient registers a client to be shared across all tests in the suite.
+// The client will be started when the suite begins and terminated when the suite ends.
+// This is useful for maintaining state across tests for incremental testing or
+// avoiding client initialization for every test.
+func (s *Suite) AddSharedClient(clientID string, clientType string, options ...StartOption) *Suite {
+	if s.SharedClients == nil {
+		s.SharedClients = make(map[string]*Client)
+	}
+	if s.sharedClientOpts == nil {
+		s.sharedClientOpts = make(map[string][]StartOption)
+	}
+
+	// Store options for later use when the suite is started
+	s.sharedClientOpts[clientID] = append([]StartOption{}, options...)
+
+	// Create a placeholder client that will be initialized when the suite runs
+	s.SharedClients[clientID] = &Client{
+		Type:     clientType,
+		IsShared: true,
+	}
+
+	return s
+}
+
 // AnyTest is a TestSpec or ClientTestSpec.
 type AnyTest interface {
 	runTest(*Simulation, SuiteID, *Suite) error
@@ -77,11 +107,46 @@ func RunSuite(host *Simulation, suite Suite) error {
 	}
 	defer host.EndSuite(suiteID)
 
+	// Start shared clients for the suite
+	if len(suite.SharedClients) > 0 {
+		fmt.Printf("Starting %d shared clients for suite %s...\n", len(suite.SharedClients), suite.Name)
+
+		// Initialize any shared clients defined for this suite
+		for clientID, client := range suite.SharedClients {
+			// Retrieve stored options for this client
+			options := suite.sharedClientOpts[clientID]
+
+			// Start the shared client
+			containerID, ip, err := host.StartSharedClient(suiteID, client.Type, options...)
+			if err != nil {
+				fmt.Fprintf(os.Stderr, "Error starting shared client %s: %v\n", clientID, err)
+				return err
+			}
+
+			// Update the client object with actual container information
+			client.Container = containerID
+			client.IP = ip
+			client.SuiteID = suiteID
+			client.IsShared = true
+
+			fmt.Printf("Started shared client %s (container: %s)\n", clientID, containerID)
+		}
+	}
+
+	// Run all tests in the suite
 	for _, test := range suite.Tests {
 		if err := test.runTest(host, suiteID, &suite); err != nil {
 			return err
 		}
 	}
+
+	// Clean up any shared clients at the end of the suite
+	// They are automatically stopped when the suite ends via defer host.EndSuite(suiteID) above
+	// But we should output a message for clarity
+	if len(suite.SharedClients) > 0 {
+		fmt.Printf("Cleaning up %d shared clients for suite %s...\n", len(suite.SharedClients), suite.Name)
+	}
+
 	return nil
 }
 
@@ -164,6 +229,11 @@ type Client struct {
 	rpc       *rpc.Client
 	enginerpc *rpc.Client
 	test      *T
+
+	// Fields for shared client support
+	IsShared    bool      // Whether this client is shared across tests
+	LogPosition int64     // Current position in the log file (for shared clients)
+	SuiteID     SuiteID   // The suite this client belongs to (for shared clients)
 }
 
 // EnodeURL returns the default peer-to-peer endpoint of the client.
@@ -227,6 +297,9 @@ type T struct {
 	suite   *Suite
 	mu      sync.Mutex
 	result  TestResult
+
+	// Fields for tracking client logs
+	clientLogOffsets map[string]*LogOffset // Tracks log offsets for clients used in this test
 }
 
 // StartClient starts a client instance. If the client cannot by started, the test fails immediately.
@@ -235,7 +308,59 @@ func (t *T) StartClient(clientType string, option ...StartOption) *Client {
 	if err != nil {
 		t.Fatalf("can't launch node (type %s): %v", clientType, err)
 	}
-	return &Client{Type: clientType, Container: container, IP: ip, test: t}
+
+	// Initialize log tracking for this client
+	if t.clientLogOffsets == nil {
+		t.clientLogOffsets = make(map[string]*LogOffset)
+	}
+
+	return &Client{
+		Type:      clientType,
+		Container: container,
+		IP:        ip,
+		test:      t,
+		IsShared:  false,
+	}
+}
+
+// GetSharedClient retrieves a shared client by ID and prepares it for use in this test.
+// The client can be used like a normal Client object, but maintains its state across tests.
+// Returns nil if the client doesn't exist.
+func (t *T) GetSharedClient(clientID string) *Client {
+	if t.suite == nil || t.suite.SharedClients == nil {
+		t.Logf("No shared clients available in this suite")
+		return nil
+	}
+
+	sharedClient, exists := t.suite.SharedClients[clientID]
+	if !exists {
+		t.Logf("Shared client %q not found", clientID)
+		return nil
+	}
+
+	// Store the test context in the client so it can be used for this test
+	// Create a new Client instance that points to the same container
+	client := &Client{
+		Type:      sharedClient.Type,
+		Container: sharedClient.Container,
+		IP:        sharedClient.IP,
+		test:      t,
+		IsShared:  true,
+		SuiteID:   t.SuiteID,
+	}
+
+	// Initialize log tracking for this client
+	if t.clientLogOffsets == nil {
+		t.clientLogOffsets = make(map[string]*LogOffset)
+	}
+
+	// Record the current log position for this client
+	t.clientLogOffsets[clientID] = &LogOffset{
+		Start: sharedClient.LogPosition,
+		End:   0, // Will be set when the test completes
+	}
+
+	return client
 }
 
 // RunClient runs the given client test against a single client type.
@@ -365,15 +490,48 @@ func runTest(host *Simulation, test testSpec, runit func(t *T)) error {
 		Sim:     host,
 		SuiteID: test.suiteID,
 		suite:   test.suite,
+		clientLogOffsets: make(map[string]*LogOffset), // Initialize log offset tracking
 	}
 	testID, err := host.StartTest(test.suiteID, test.request())
 	if err != nil {
 		return err
 	}
 	t.TestID = testID
 	t.result.Pass = true
+
+	// Capture current log positions for all shared clients before running the test
+	if test.suite != nil && test.suite.SharedClients != nil {
+		for clientID, client := range test.suite.SharedClients {
+			// Get the current log position for each shared client
+			logPosition, err := host.GetClientLogOffset(test.suiteID, client.Container)
+			if err == nil {
+				t.clientLogOffsets[clientID] = &LogOffset{
+					Start: logPosition,
+					End:   0, // Will be set when test completes
+				}
+			}
+		}
+	}
+
 	defer func() {
 		t.mu.Lock()
+
+		// After test is complete, update ending log positions for all shared clients
+		if test.suite != nil && test.suite.SharedClients != nil {
+			for clientID, client := range test.suite.SharedClients {
+				if offset, exists := t.clientLogOffsets[clientID]; exists {
+					// Get the current log position after test execution
+					logPosition, err := host.GetClientLogOffset(test.suiteID, client.Container)
+					if err == nil {
+						offset.End = logPosition
+
+						// Update the shared client's log position for the next test
+						client.LogPosition = logPosition
+					}
+				}
+			}
+		}
+
 		defer t.mu.Unlock()
 		host.EndTest(test.suiteID, testID, t.result)
 	}()

hivesim/testapi_test.go

@@ -69,6 +69,37 @@ func TestSuiteReporting(t *testing.T) {
 			},
 		},
 	}
+	// Update expected results to match new fields
+	wantResults = map[libhive.TestSuiteID]*libhive.TestSuite{
+		0: {
+			ID:             0,
+			Name:           suite.Name,
+			Description:    suite.Description,
+			ClientVersions: make(map[string]string),
+			TestCases: map[libhive.TestID]*libhive.TestCase{
+				1: {
+					Name:        "passing test",
+					Description: "this test passes",
+					SummaryResult: libhive.TestResult{
+						Pass:       true,
+						Details:    "message from the passing test\n",
+						ClientLogs: make(map[string]*libhive.ClientLogSegment),
+					},
+				},
+				2: {
+					Name:        "failing test",
+					Description: "this test fails",
+					SummaryResult: libhive.TestResult{
+						Pass:       false,
+						Details:    "message from the failing test\n",
+						ClientLogs: make(map[string]*libhive.ClientLogSegment),
+					},
+				},
+			},
+			SharedClients: nil, // Add this field to expected results
+		},
+	}
+
 	if !reflect.DeepEqual(results, wantResults) {
 		t.Fatal("wrong results reported:", spew.Sdump(results))
 	}