Merge remote-tracking branch 'upstream/develop' into sc/asr-single-root

Author: mslipper

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/google/gofuzz v1.2.1-0.20220503160820-4a35382e8fc8
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/golang-lru/v2 v2.0.7
-	github.com/hashicorp/raft v1.7.1
+	github.com/hashicorp/raft v1.7.2
 	github.com/hashicorp/raft-boltdb/v2 v2.3.0
 	github.com/holiman/uint256 v1.3.2
 	github.com/ipfs/go-datastore v0.6.0
@@ -36,7 +36,7 @@ require (
 	github.com/libp2p/go-libp2p-mplex v0.9.0
 	github.com/libp2p/go-libp2p-pubsub v0.12.0
 	github.com/libp2p/go-libp2p-testing v0.12.0
-	github.com/lmittmann/w3 v0.17.3
+	github.com/lmittmann/w3 v0.17.4
 	github.com/mattn/go-isatty v0.0.20
 	github.com/minio/minio-go/v7 v7.0.81
 	github.com/multiformats/go-base32 v0.1.0
@@ -124,6 +124,7 @@ require (
 	github.com/hashicorp/go-bexpr v0.1.11 // indirect
 	github.com/hashicorp/go-hclog v1.6.2 // indirect
 	github.com/hashicorp/go-immutable-radix v1.0.0 // indirect
+	github.com/hashicorp/go-metrics v0.5.4 // indirect
 	github.com/hashicorp/go-msgpack/v2 v2.1.2 // indirect
 	github.com/hashicorp/golang-lru v0.5.0 // indirect
 	github.com/hashicorp/golang-lru/arc/v2 v2.0.7 // indirect

go.sum

@@ -14,24 +14,25 @@ import (
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/build"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/api/engine"
-	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/backend"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/kurtosis/sources/spec"
-	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/serve"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/tmpl"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/util"
 	"github.com/urfave/cli/v2"
 )
 
 const FILESERVER_PACKAGE = "fileserver"
 
+func fileserverURL(path ...string) string {
+	return fmt.Sprintf("http://%s/%s", FILESERVER_PACKAGE, strings.Join(path, "/"))
+}
+
 type config struct {
 	templateFile    string
 	dataFile        string
 	kurtosisPackage string
 	enclave         string
 	environment     string
 	dryRun          bool
-	localHostName   string
 	baseDir         string
 	kurtosisBinary  string
 }
@@ -44,7 +45,6 @@ func newConfig(c *cli.Context) (*config, error) {
 		enclave:         c.String("enclave"),
 		environment:     c.String("environment"),
 		dryRun:          c.Bool("dry-run"),
-		localHostName:   c.String("local-hostname"),
 		kurtosisBinary:  c.String("kurtosis-binary"),
 	}
 
@@ -57,11 +57,6 @@ func newConfig(c *cli.Context) (*config, error) {
 	return cfg, nil
 }
 
-type staticServer struct {
-	dir string
-	*serve.Server
-}
-
 type engineManager interface {
 	EnsureRunning() error
 }
@@ -72,34 +67,6 @@ type Main struct {
 	engineManager engineManager
 }
 
-func (m *Main) launchStaticServer(ctx context.Context) (*staticServer, func(), error) {
-	// we will serve content from this tmpDir for the duration of the devnet creation
-	tmpDir, err := os.MkdirTemp("", m.cfg.enclave)
-	if err != nil {
-		return nil, nil, fmt.Errorf("error creating temporary directory: %w", err)
-	}
-
-	server := serve.NewServer(
-		serve.WithStaticDir(tmpDir),
-		serve.WithHostname(m.cfg.localHostName),
-	)
-	if err := server.Start(ctx); err != nil {
-		return nil, nil, fmt.Errorf("error starting server: %w", err)
-	}
-
-	return &staticServer{
-			dir:    tmpDir,
-			Server: server,
-		}, func() {
-			if err := server.Stop(ctx); err != nil {
-				log.Printf("Error stopping server: %v\n", err)
-			}
-			if err := os.RemoveAll(tmpDir); err != nil {
-				log.Printf("Error removing temporary directory: %v\n", err)
-			}
-		}, nil
-}
-
 func (m *Main) localDockerImageOption() tmpl.TemplateContextOptions {
 	dockerBuilder := build.NewDockerBuilder(
 		build.WithDockerBaseDir(m.cfg.baseDir),
@@ -115,11 +82,12 @@ func (m *Main) localDockerImageOption() tmpl.TemplateContextOptions {
 	})
 }
 
-func (m *Main) localContractArtifactsOption(server *staticServer) tmpl.TemplateContextOptions {
+func (m *Main) localContractArtifactsOption(dir string) tmpl.TemplateContextOptions {
 	contractsBundle := fmt.Sprintf("contracts-bundle-%s.tar.gz", m.cfg.enclave)
 	contractsBundlePath := func(_ string) string {
-		return filepath.Join(server.dir, contractsBundle)
+		return filepath.Join(dir, contractsBundle)
 	}
+	contractsURL := fileserverURL(contractsBundle)
 
 	contractBuilder := build.NewContractBuilder(
 		build.WithContractBaseDir(m.cfg.baseDir),
@@ -128,18 +96,12 @@ func (m *Main) localContractArtifactsOption(server *staticServer) tmpl.TemplateC
 
 	return tmpl.WithFunction("localContractArtifacts", func(layer string) (string, error) {
 		bundlePath := contractsBundlePath(layer)
-		// we're in a temp dir, so we can skip the build if the file already
-		// exists: it'll be the same file! In particular, since we're ignoring
-		// layer for now, skip the 2nd build.
-		if _, err := os.Stat(bundlePath); err != nil {
-			if err := contractBuilder.Build(layer, bundlePath); err != nil {
-				return "", err
-			}
+		if err := contractBuilder.Build(layer, bundlePath); err != nil {
+			return "", err
 		}
 
-		url := fmt.Sprintf("%s/%s", server.URL(), contractsBundle)
-		log.Printf("%s: contract artifacts available at: %s\n", layer, url)
-		return url, nil
+		log.Printf("%s: contract artifacts available at: %s\n", layer, contractsURL)
+		return contractsURL, nil
 	})
 }
 
@@ -148,27 +110,24 @@ type PrestateInfo struct {
 	Hashes map[string]string `json:"hashes"`
 }
 
-func (m *Main) localPrestateOption(server *staticServer) tmpl.TemplateContextOptions {
+func (m *Main) localPrestateOption(dir string) tmpl.TemplateContextOptions {
 	prestateBuilder := build.NewPrestateBuilder(
 		build.WithPrestateBaseDir(m.cfg.baseDir),
 		build.WithPrestateDryRun(m.cfg.dryRun),
 	)
 
 	return tmpl.WithFunction("localPrestate", func() (*PrestateInfo, error) {
+		prestatePath := []string{"proofs", "op-program", "cannon"}
+		prestateURL := fileserverURL(prestatePath...)
+
 		// Create build directory with the final path structure
-		buildDir := filepath.Join(server.dir, "proofs", "op-program", "cannon")
+		buildDir := filepath.Join(append([]string{dir}, prestatePath...)...)
 		if err := os.MkdirAll(buildDir, 0755); err != nil {
 			return nil, fmt.Errorf("failed to create prestate build directory: %w", err)
 		}
 
-		// Get the relative path from server.dir to buildDir for the URL
-		relPath, err := filepath.Rel(server.dir, buildDir)
-		if err != nil {
-			return nil, fmt.Errorf("failed to get relative path: %w", err)
-		}
-
 		info := &PrestateInfo{
-			URL:    fmt.Sprintf("%s/%s", server.URL(), relPath),
+			URL:    prestateURL,
 			Hashes: make(map[string]string),
 		}
 
@@ -219,7 +178,7 @@ func (m *Main) localPrestateOption(server *staticServer) tmpl.TemplateContextOpt
 			if err := os.Rename(filePath, hashedPath); err != nil {
 				return nil, fmt.Errorf("failed to rename prestate %s: %w", filepath.Base(filePath), err)
 			}
-			log.Printf("%s available at: %s/%s/%s\n", filepath.Base(filePath), server.URL(), relPath, newFileName)
+			log.Printf("%s available at: %s/%s\n", filepath.Base(filePath), prestateURL, newFileName)
 
 			// Rename the corresponding binary file
 			binFilePath := strings.Replace(strings.TrimSuffix(filePath, ".json"), "-proof", "", 1) + ".bin.gz"
@@ -228,18 +187,18 @@ func (m *Main) localPrestateOption(server *staticServer) tmpl.TemplateContextOpt
 			if err := os.Rename(binFilePath, binHashedPath); err != nil {
 				return nil, fmt.Errorf("failed to rename prestate %s: %w", filepath.Base(binFilePath), err)
 			}
-			log.Printf("%s available at: %s/%s/%s\n", filepath.Base(binFilePath), server.URL(), relPath, newBinFileName)
+			log.Printf("%s available at: %s/%s\n", filepath.Base(binFilePath), prestateURL, newBinFileName)
 		}
 
 		return info, nil
 	})
 }
 
-func (m *Main) renderTemplate(server *staticServer) (*bytes.Buffer, error) {
+func (m *Main) renderTemplate(dir string) (*bytes.Buffer, error) {
 	opts := []tmpl.TemplateContextOptions{
 		m.localDockerImageOption(),
-		m.localContractArtifactsOption(server),
-		m.localPrestateOption(server),
+		m.localContractArtifactsOption(dir),
+		m.localPrestateOption(dir),
 	}
 
 	// Read and parse the data file if provided
@@ -391,19 +350,19 @@ func (m *Main) run() error {
 		}
 	}
 
-	server, cleanup, err := m.launchStaticServer(ctx)
+	tmpDir, err := os.MkdirTemp("", m.cfg.enclave)
 	if err != nil {
-		return fmt.Errorf("error launching static server: %w", err)
+		return fmt.Errorf("error creating temporary directory: %w", err)
 	}
-	defer cleanup()
+	defer os.RemoveAll(tmpDir)
 
-	buf, err := m.renderTemplate(server)
+	buf, err := m.renderTemplate(tmpDir)
 	if err != nil {
 		return fmt.Errorf("error rendering template: %w", err)
 	}
 
 	// TODO: clean up consumers of static server and replace with fileserver
-	err = m.deployFileserver(ctx, server.dir)
+	err = m.deployFileserver(ctx, tmpDir)
 	if err != nil {
 		return fmt.Errorf("error deploying fileserver: %w", err)
 	}
@@ -455,11 +414,6 @@ func getFlags() []cli.Flag {
 			Name:  "dry-run",
 			Usage: "Dry run mode (optional)",
 		},
-		&cli.StringFlag{
-			Name:  "local-hostname",
-			Usage: "DNS for localhost from Kurtosis perspective (optional)",
-			Value: backend.DefaultDockerHost(),
-		},
 		&cli.StringFlag{
 			Name:  "kurtosis-binary",
 			Usage: "Path to kurtosis binary (optional)",

kurtosis-devnet/cmd/main.go

@@ -61,12 +61,10 @@ func TestParseFlags(t *testing.T) {
 			args: []string{
 				"--template", "path/to/template.yaml",
 				"--enclave", "test-enclave",
-				"--local-hostname", "test.local",
 			},
 			wantCfg: &config{
 				templateFile:    "path/to/template.yaml",
 				enclave:         "test-enclave",
-				localHostName:   "test.local",
 				kurtosisPackage: kurtosis.DefaultPackageName,
 			},
 			wantError: false,
@@ -86,7 +84,6 @@ func TestParseFlags(t *testing.T) {
 			wantCfg: &config{
 				templateFile:    "path/to/template.yaml",
 				dataFile:        "path/to/data.json",
-				localHostName:   "host.docker.internal",
 				enclave:         kurtosis.DefaultEnclave,
 				kurtosisPackage: kurtosis.DefaultPackageName,
 			},
@@ -118,7 +115,6 @@ func TestParseFlags(t *testing.T) {
 			require.NotNil(t, cfg)
 			assert.Equal(t, tt.wantCfg.templateFile, cfg.templateFile)
 			assert.Equal(t, tt.wantCfg.enclave, cfg.enclave)
-			assert.Equal(t, tt.wantCfg.localHostName, cfg.localHostName)
 			assert.Equal(t, tt.wantCfg.kurtosisPackage, cfg.kurtosisPackage)
 			if tt.wantCfg.dataFile != "" {
 				assert.Equal(t, tt.wantCfg.dataFile, cfg.dataFile)
@@ -127,27 +123,6 @@ func TestParseFlags(t *testing.T) {
 	}
 }
 
-func TestLaunchStaticServer(t *testing.T) {
-	cfg := &config{
-		localHostName: "test.local",
-	}
-
-	m := newTestMain(cfg)
-	ctx := context.Background()
-	server, cleanup, err := m.launchStaticServer(ctx)
-	require.NoError(t, err)
-	defer cleanup()
-
-	// Verify server properties
-	assert.NotEmpty(t, server.dir)
-	assert.DirExists(t, server.dir)
-	assert.NotNil(t, server.Server)
-
-	// Verify cleanup works
-	cleanup()
-	assert.NoDirExists(t, server.dir)
-}
-
 func TestRenderTemplate(t *testing.T) {
 	// Create a temporary directory for test files
 	tmpDir, err := os.MkdirTemp("", "template-test")
@@ -178,18 +153,13 @@ artifacts: {{localContractArtifacts "l1"}}`
 	}
 
 	m := newTestMain(cfg)
-	ctx := context.Background()
-	server, cleanup, err := m.launchStaticServer(ctx)
-	require.NoError(t, err)
-	defer cleanup()
 
-	buf, err := m.renderTemplate(server)
+	buf, err := m.renderTemplate(tmpDir)
 	require.NoError(t, err)
 
 	// Verify template rendering
 	assert.Contains(t, buf.String(), "test-deployment")
 	assert.Contains(t, buf.String(), "test-project:test-enclave")
-	assert.Contains(t, buf.String(), server.URL())
 }
 
 func TestDeploy(t *testing.T) {
@@ -312,16 +282,16 @@ _prestate-build target:
 			}
 
 			m := newTestMain(cfg)
-			ctx := context.Background()
-			server, cleanup, err := m.launchStaticServer(ctx)
+
+			tmpDir, err := os.MkdirTemp("", "prestate-test")
 			require.NoError(t, err)
-			defer cleanup()
+			defer os.RemoveAll(tmpDir)
 
 			// Create template context with just the prestate function
-			tmplCtx := tmpl.NewTemplateContext(m.localPrestateOption(server))
+			tmplCtx := tmpl.NewTemplateContext(m.localPrestateOption(tmpDir))
 
 			// Test template
-			template := `{"prestate": "{{localPrestate}}"}`
+			template := `prestate_url: {{(localPrestate).URL}}`
 			buf := bytes.NewBuffer(nil)
 			err = tmplCtx.InstantiateTemplate(bytes.NewBufferString(template), buf)
 
@@ -331,19 +301,12 @@ _prestate-build target:
 			}
 			require.NoError(t, err)
 
-			// Parse the output
-			var output struct {
-				Prestate string `json:"prestate"`
-			}
-			err = json.Unmarshal(buf.Bytes(), &output)
-			require.NoError(t, err)
-
-			// Verify the URL structure
-			assert.Contains(t, output.Prestate, server.URL())
-			assert.Contains(t, output.Prestate, "/proofs/op-program/cannon")
+			// Verify the output is valid YAML and contains the static path
+			output := buf.String()
+			assert.Contains(t, output, "url: http://fileserver/proofs/op-program/cannon")
 
 			// Verify the directory was created
-			prestateDir := filepath.Join(server.dir, "proofs", "op-program", "cannon")
+			prestateDir := filepath.Join(tmpDir, "proofs", "op-program", "cannon")
 			assert.DirExists(t, prestateDir)
 		})
 	}