feat: configure branch dropdown (wip)

Author: reinthal

modules/nessie/client.go

@@ -0,0 +1,69 @@
+package nessie
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"time"
+	"code.gitea.io/gitea/modules/setting"
+	"code.gitea.io/gitea/modules/log"
+
+)
+
+type Reference struct {
+	Name        string    `json:"name"`
+	Hash        string    `json:"hash"`
+	Type        string    `json:"type"`
+}
+
+// ReferencesResponse represents the Nessie API response structure
+type ReferencesResponse struct {
+	Token      *string     `json:"token"`
+	References []Reference `json:"references"`
+	HasMore    bool        `json:"hasMore"`
+}
+
+type Client struct {
+	baseURL    string
+	authToken  string
+	httpClient *http.Client
+}
+
+func NewClient() *Client {
+	return &Client{
+		baseURL:    setting.Nessie.APIURL,
+		authToken:  setting.Nessie.AuthToken,
+		httpClient: &http.Client{Timeout: 10 * time.Second},
+	}
+}
+
+func (c *Client) GetAllReferences(repo string) ([]Reference, error) {
+	// TODO: figure out how to handle multiple repositories
+	url := fmt.Sprintf("%s/api/v1/trees/", c.baseURL)
+	
+	req, err := http.NewRequest("GET", url, nil)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create request: %v", err)
+	}
+
+	if c.authToken != "" {
+		req.Header.Set("Authorization", "Bearer "+c.authToken)
+	}
+
+	resp, err := c.httpClient.Do(req)
+	if err != nil {
+		return nil, fmt.Errorf("failed to make request: %v", err)
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+	}
+
+	var response ReferencesResponse
+	if err := json.NewDecoder(resp.Body).Decode(&response); err != nil {
+		return nil, fmt.Errorf("failed to decode response: %v", err)
+	}
+	log.Info("Nessie references: %v", response.References)
+	return response.References, nil
+} 

modules/setting/nessie.go

@@ -0,0 +1,23 @@
+package setting
+
+import (
+	"code.gitea.io/gitea/modules/log"
+)
+	
+var Nessie = struct {
+	Enabled       bool
+	APIURL        string
+	DefaultBranch string
+	AuthToken     string
+}{
+	Enabled:       true,
+	APIURL:        "http://localhost:19120",
+	DefaultBranch: "main",
+	AuthToken:     "",
+}
+
+func loadNessieFrom(rootCfg ConfigProvider) {
+	if err := rootCfg.Section("nessie").MapTo(&Nessie); err != nil {
+		log.Fatal("Failed to map Nessie settings: %v", err)
+	}
+}

nessie-stack/notebooks/test.ipynb

@@ -9,7 +9,7 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "25/03/22 00:13:18 WARN SparkSession: Using an existing Spark session; only runtime SQL configurations will take effect.\n"
+      "25/03/23 21:11:07 WARN SparkSession: Using an existing Spark session; only runtime SQL configurations will take effect.\n"
      ]
     }
    ],
@@ -28,14 +28,12 @@
      "data": {
       "text/plain": [
        "[('spark.sql.catalog.nessie', 'org.apache.iceberg.spark.SparkCatalog'),\n",
-       " ('spark.app.submitTime', '1742601784570'),\n",
        " ('spark.sql.catalog.nessie.uri', 'http://nessie:19120/api/v1'),\n",
        " ('spark.hadoop.fs.s3a.path.style.access', 'true'),\n",
        " ('spark.sql.catalog.nessie.ref', 'main'),\n",
-       " ('spark.app.startTime', '1742601784818'),\n",
-       " ('spark.driver.port', '39441'),\n",
        " ('spark.sql.warehouse.dir',\n",
        "  'file:/home/iceberg/notebooks/notebooks/spark-warehouse'),\n",
+       " ('spark.app.submitTime', '1742764264501'),\n",
        " ('spark.hadoop.fs.s3a.access.key', 'minioadmin'),\n",
        " ('spark.sql.catalog.nessie.s3.path-style-access', 'true'),\n",
        " ('spark.serializer.objectStreamReset', '100'),\n",
@@ -50,14 +48,15 @@
        "  'file:///root/.ivy2/jars/org.apache.iceberg_iceberg-spark-runtime-3.5_2.12-1.8.1.jar,file:///root/.ivy2/jars/org.projectnessie.nessie-integrations_nessie-spark-extensions-3.5_2.12-0.103.2.jar'),\n",
        " ('spark.driver.extraJavaOptions',\n",
        "  '-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false'),\n",
-       " ('spark.app.id', 'local-1742601785395'),\n",
+       " ('spark.app.startTime', '1742764264699'),\n",
        " ('spark.sql.catalog.nessie.s3.endpoint', 'http://minio:9000'),\n",
+       " ('spark.app.id', 'local-1742764265168'),\n",
        " ('spark.repl.local.jars',\n",
        "  'file:///root/.ivy2/jars/org.apache.iceberg_iceberg-spark-runtime-3.5_2.12-1.8.1.jar,file:///root/.ivy2/jars/org.projectnessie.nessie-integrations_nessie-spark-extensions-3.5_2.12-0.103.2.jar'),\n",
        " ('spark.executor.id', 'driver'),\n",
-       " ('spark.driver.host', '876c347e2cee'),\n",
        " ('spark.app.name', 'PySparkShell'),\n",
        " ('spark.hadoop.fs.s3a.impl', 'org.apache.hadoop.fs.s3a.S3AFileSystem'),\n",
+       " ('spark.driver.host', '137a0ca84ad6'),\n",
        " ('spark.sql.catalogImplementation', 'hive'),\n",
        " ('spark.sql.catalog.nessie.io-impl', 'org.apache.iceberg.aws.s3.S3FileIO'),\n",
        " ('spark.submit.pyFiles',\n",
@@ -66,14 +65,15 @@
        " ('spark.executor.extraJavaOptions',\n",
        "  '-Djava.net.preferIPv6Addresses=false -XX:+IgnoreUnrecognizedVMOptions --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.lang.invoke=ALL-UNNAMED --add-opens=java.base/java.lang.reflect=ALL-UNNAMED --add-opens=java.base/java.io=ALL-UNNAMED --add-opens=java.base/java.net=ALL-UNNAMED --add-opens=java.base/java.nio=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED --add-opens=java.base/sun.nio.ch=ALL-UNNAMED --add-opens=java.base/sun.nio.cs=ALL-UNNAMED --add-opens=java.base/sun.security.action=ALL-UNNAMED --add-opens=java.base/sun.util.calendar=ALL-UNNAMED --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED -Djdk.reflect.useDirectMethodHandle=false'),\n",
        " ('spark.sql.catalog.nessie.warehouse', 's3a://warehouse'),\n",
+       " ('spark.app.initial.jar.urls',\n",
+       "  'spark://137a0ca84ad6:46315/jars/org.projectnessie.nessie-integrations_nessie-spark-extensions-3.5_2.12-0.103.2.jar,spark://137a0ca84ad6:46315/jars/org.apache.iceberg_iceberg-spark-runtime-3.5_2.12-1.8.1.jar'),\n",
        " ('spark.jars',\n",
        "  'file:///root/.ivy2/jars/org.apache.iceberg_iceberg-spark-runtime-3.5_2.12-1.8.1.jar,file:///root/.ivy2/jars/org.projectnessie.nessie-integrations_nessie-spark-extensions-3.5_2.12-0.103.2.jar'),\n",
        " ('spark.hadoop.fs.s3a.endpoint', 'http://minio:9000'),\n",
+       " ('spark.driver.port', '46315'),\n",
+       " ('spark.sql.catalog.nessie.type', 'nessie'),\n",
        " ('spark.files',\n",
        "  'file:///root/.ivy2/jars/org.apache.iceberg_iceberg-spark-runtime-3.5_2.12-1.8.1.jar,file:///root/.ivy2/jars/org.projectnessie.nessie-integrations_nessie-spark-extensions-3.5_2.12-0.103.2.jar'),\n",
-       " ('spark.sql.catalog.nessie.type', 'nessie'),\n",
-       " ('spark.app.initial.jar.urls',\n",
-       "  'spark://876c347e2cee:39441/jars/org.apache.iceberg_iceberg-spark-runtime-3.5_2.12-1.8.1.jar,spark://876c347e2cee:39441/jars/org.projectnessie.nessie-integrations_nessie-spark-extensions-3.5_2.12-0.103.2.jar'),\n",
        " ('spark.ui.showConsoleProgress', 'true')]"
       ]
      },
@@ -95,11 +95,11 @@
      "name": "stderr",
      "output_type": "stream",
      "text": [
-      "25/03/22 00:13:22 WARN HiveConf: HiveConf of name hive.stats.jdbc.timeout does not exist\n",
-      "25/03/22 00:13:22 WARN HiveConf: HiveConf of name hive.stats.retries.wait does not exist\n",
-      "25/03/22 00:13:22 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 2.3.0\n",
-      "25/03/22 00:13:22 WARN ObjectStore: setMetaStoreSchemaVersion called but recording version is disabled: version = 2.3.0, comment = Set by MetaStore [email protected]\n",
-      "25/03/22 00:13:23 WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException\n"
+      "25/03/23 21:11:15 WARN HiveConf: HiveConf of name hive.stats.jdbc.timeout does not exist\n",
+      "25/03/23 21:11:15 WARN HiveConf: HiveConf of name hive.stats.retries.wait does not exist\n",
+      "25/03/23 21:11:16 WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 2.3.0\n",
+      "25/03/23 21:11:16 WARN ObjectStore: setMetaStoreSchemaVersion called but recording version is disabled: version = 2.3.0, comment = Set by MetaStore [email protected]\n",
+      "25/03/23 21:11:16 WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException\n"
      ]
     },
     {
@@ -138,7 +138,7 @@
     }
    ],
    "source": [
-    "spark.sql(\"create namespace default\")"
+    "spark.sql(\"CREATE BRANCH te\")"
    ]
   },
   {

options/locale/locale_en-US.ini

@@ -1025,6 +1025,13 @@ visibility.private_tooltip = Visible only to members of organizations you have j
 
 [repo]
 catalog = Catalog
+catalog.overview = Data catalog Overview
+catalog.datasets = Datasets
+catalog.tags = Tags
+catalog.new_dataset = New Dataset
+catalog.updated = Updated
+catalog.popular_tags = Popular Tags
+
 new_repo_helper = A repository contains all project files, including revision history. Already hosting one elsewhere? <a href="%s">Migrate repository.</a>
 owner = Owner
 owner_helper = Some organizations may not show up in the dropdown due to a maximum repository count limit.

routers/web/repo/view.go

@@ -33,6 +33,7 @@ import (
 	"code.gitea.io/gitea/modules/lfs"
 	"code.gitea.io/gitea/modules/log"
 	"code.gitea.io/gitea/modules/markup"
+	"code.gitea.io/gitea/modules/nessie"
 	"code.gitea.io/gitea/modules/setting"
 	"code.gitea.io/gitea/modules/structs"
 	"code.gitea.io/gitea/modules/templates"
@@ -389,8 +390,24 @@ func Forks(ctx *context.Context) {
 	ctx.HTML(http.StatusOK, tplForks)
 }
 
-// CatalogHome renders the repository catalog home/overview page
 func CatalogHome(ctx *context.Context) {
 	ctx.Data["PageIsCatalog"] = true
+	
+	// Initialize Nessie client
+	client := nessie.NewClient()
+	
+	// Get references from Nessie
+	refs, err := client.GetAllReferences(ctx.Repo.Repository.Name)
+	if err != nil {
+		ctx.ServerError("GetAllReferences", err)
+		return
+	}
+	log.Info("Current branch: %s", ctx.Repo.BranchName)
+	ctx.Data["NessieRefs"] = refs
+	ctx.Data["CurrentBranch"] = ctx.Repo.BranchName // or get from query parameter
+	if ctx.Data["CurrentBranch"] == "" {
+		ctx.Data["CurrentBranch"] = setting.Nessie.DefaultBranch // or your default branch
+	}
+
 	ctx.HTML(http.StatusOK, tplCatalog)
 }

templates/repo/catalog/branch_dropdown.tmpl

@@ -0,0 +1,35 @@
+<div class="ui dropdown custom">
+	<button class="ui basic button nessie-branch-button">
+		{{svg "octicon-git-branch"}} <span class="text">{{.CurrentBranch}}</span>
+		{{svg "octicon-triangle-down" 14 "dropdown icon"}}
+	</button>
+	<div class="menu">
+		<div class="ui icon search input">
+			<i class="icon">{{svg "octicon-search" 16}}</i>
+			<input name="search" placeholder="{{ctx.Locale.Tr "repo.filter_branch_and_tag"}}...">
+		</div>
+		<div class="header">
+			<div class="ui grid">
+				<div class="eight wide column">
+					{{ctx.Locale.Tr "repo.branches"}}
+				</div>
+				<div class="eight wide column">
+					{{ctx.Locale.Tr "repo.tags"}}
+				</div>
+			</div>
+		</div>
+		<div class="scrolling menu">
+			{{range .NessieRefs}}
+				<a class="{{if eq $.CurrentBranch .Name}}active selected{{end}} item" href="{{$.RepoLink}}/catalog/{{.Type}}/{{.Name}}">
+					{{if eq .Type "branch"}}
+						{{svg "octicon-git-branch"}}
+					{{else}}
+						{{svg "octicon-tag"}}
+					{{end}}
+					<span class="text">{{.Name}}</span>
+					<span class="text small">{{.Hash}}</span>
+				</a>
+			{{end}}
+		</div>
+	</div>
+</div> 

templates/repo/catalog/view_content.tmpl

@@ -1,16 +1,11 @@
 {{template "repo/sub_menu" .}}
 <div class="repo-button-row">
 	<div class="repo-button-row-left">
-		{{template "repo/branch_dropdown" dict
-			"Repository" .Repository
-			"ShowTabBranches" true
-			"ShowTabTags" true
-			"CurrentRefType" .RefFullName.RefType
-			"CurrentRefShortName" .RefFullName.ShortName
-			"CurrentTreePath" .TreePath
-			"RefLinkTemplate" "{RepoLink}/catalog/{RefType}/{RefShortName}/{TreePath}"
-			"AllowCreateNewRef" .CanCreateBranch
-			"ShowViewAllRefsEntry" true
+		{{template "repo/catalog/branch_dropdown" dict 
+			"ctx" .ctx
+			"RepoLink" .RepoLink
+			"CurrentBranch" .CurrentBranch
+			"NessieRefs" .NessieRefs
 		}}
 
 		<a href="{{.Repository.Link}}/catalog/find" class="ui compact basic button">{{ctx.Locale.Tr "repo.find_dataset"}}</a>