Switch to docs page to determine latest stable version

The code was previously looking at the /api/updates endpoint on the
register.cockroachdb.com server. However, that endpoint returns
Cloud Only releases, which cannot be downloaded from
binaries.cockroachdb.com.

Now we use the releases list maintained by the docs team, which lets us
filter out undesired releases.

Author: rafiss

testserver/binaries.go

@@ -19,7 +19,6 @@ import (
 	"archive/zip"
 	"bytes"
 	"compress/gzip"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"io"
@@ -35,7 +34,9 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cockroachdb/cockroach-go/v2/testserver/version"
 	"github.com/gofrs/flock"
+	"gopkg.in/yaml.v3"
 )
 
 const (
@@ -45,15 +46,17 @@ const (
 )
 
 const (
-	linuxUrlpat  = "https://binaries.cockroachdb.com/cockroach-v%s.linux-%s.tgz"
-	macUrlpat    = "https://binaries.cockroachdb.com/cockroach-v%s.darwin-%s-%s.tgz"
-	winUrlpat    = "https://binaries.cockroachdb.com/cockroach-v%s.windows-6.2-amd64.zip"
-	sourceUrlPat = "https://binaries.cockroachdb.com/cockroach-v%s.src.tgz)"
+	linuxUrlpat = "https://binaries.cockroachdb.com/cockroach-%s.linux-%s.tgz"
+	macUrlpat   = "https://binaries.cockroachdb.com/cockroach-%s.darwin-%s-%s.tgz"
+	winUrlpat   = "https://binaries.cockroachdb.com/cockroach-%s.windows-6.2-amd64.zip"
 )
 
-// updatesUrl is used to get the info of the latest stable version of CRDB.
-// Note that it may return a withdrawn version, but the risk is low for local tests here.
-const updatesUrl = "https://register.cockroachdb.com/api/updates"
+// releaseDataURL is the location of the YAML file maintained by the
+// docs team where release information is encoded. This data is used
+// to render the public CockroachDB releases page. We leverage the
+// data in structured format to generate release information used
+// for testing purposes.
+const releaseDataURL = "https://raw.githubusercontent.com/cockroachdb/docs/main/src/current/_data/releases.yml"
 
 var muslRE = regexp.MustCompile(`(?i)\bmusl\b`)
 
@@ -249,37 +252,77 @@ func DownloadBinary(tc *TestConfig, desiredVersion string, nonStable bool) (stri
 }
 
 // GetDownloadFilename returns the local filename of the downloaded CRDB binary file.
-func GetDownloadFilename(
-	desiredVersion string,
-) (string, error) {
+func GetDownloadFilename(desiredVersion string) (string, error) {
 	filename := fmt.Sprintf("cockroach-%s", desiredVersion)
 	if runtime.GOOS == "windows" {
 		filename += ".exe"
 	}
 	return filename, nil
 }
 
+// Release contains the information we extract from the YAML file in
+// `releaseDataURL`.
+type Release struct {
+	Name      string `yaml:"release_name"`
+	Withdrawn bool   `yaml:"withdrawn"`
+	CloudOnly bool   `yaml:"cloud_only"`
+}
+
 // getLatestStableVersionInfo returns the latest stable CRDB's download URL,
 // and the formatted corresponding version number. The download URL is based
 // on the runtime OS.
 // Note that it may return a withdrawn version, but the risk is low for local tests here.
 func getLatestStableVersionInfo() (string, string, error) {
-	resp, err := http.Get(updatesUrl)
+	resp, err := http.Get(releaseDataURL)
 	if err != nil {
-		return "", "", err
+		return "", "", fmt.Errorf("could not download release data: %w", err)
 	}
-	var respJson map[string]string
-	if err := json.NewDecoder(resp.Body).Decode(&respJson); err != nil {
-		return "", "", err
+	defer resp.Body.Close()
+
+	var blob bytes.Buffer
+	if _, err := io.Copy(&blob, resp.Body); err != nil {
+		return "", "", fmt.Errorf("error reading response body: %w", err)
 	}
-	latestStableVersion, ok := respJson["version"]
-	if !ok {
-		return "", "", fmt.Errorf("api/updates response is of wrong format")
+
+	var data []Release
+	if err := yaml.Unmarshal(blob.Bytes(), &data); err != nil { //nolint:yaml
+		return "", "", fmt.Errorf("failed to YAML parse release data: %w", err)
+	}
+
+	latestStableVersion := version.MustParse("v0.0.0")
+
+	for _, r := range data {
+		// We ignore versions that cannot be parsed; this should
+		// correspond to really old beta releases.
+		v, err := version.Parse(r.Name)
+		if err != nil {
+			continue
+		}
+
+		// Skip cloud-only releases, since they cannot be downloaded from
+		// binaries.cockroachdb.com.
+		if r.CloudOnly {
+			continue
+		}
+
+		// Ignore any withdrawn releases, since they are known to be broken.
+		if r.Withdrawn {
+			continue
+		}
+
+		// Ignore alphas, betas, and RCs.
+		if v.PreRelease() != "" {
+			continue
+		}
+
+		if v.Compare(latestStableVersion) > 0 {
+			latestStableVersion = v
+		}
 	}
 
-	downloadUrl := getDownloadUrlForVersion(latestStableVersion)
+	downloadUrl := getDownloadUrlForVersion(latestStableVersion.String())
 
-	latestStableVerFormatted := strings.ReplaceAll(latestStableVersion, ".", "-")
+	latestStableVerFormatted := strings.ReplaceAll(latestStableVersion.String(), ".", "-")
 	return downloadUrl, latestStableVerFormatted, nil
 }
 

testserver/testserver.go

@@ -306,9 +306,10 @@ func NonStableDbOpt() TestServerOpt {
 
 // CustomVersionOpt is a TestServer option that can be passed to NewTestServer to
 // download the a specific version of CRDB.
-func CustomVersionOpt(version string) TestServerOpt {
+func CustomVersionOpt(ver string) TestServerOpt {
 	return func(args *testServerArgs) {
-		args.customVersion = version
+		_ = version.MustParse(ver)
+		args.customVersion = ver
 	}
 }
 

testserver/testserver_test.go

@@ -131,7 +131,7 @@ func TestRunServer(t *testing.T) {
 		{
 			name: "InsecureCustomVersion",
 			instantiation: func(t *testing.T) (*sql.DB, func()) {
-				return testserver.NewDBForTest(t, testserver.CustomVersionOpt("21.2.15"))
+				return testserver.NewDBForTest(t, testserver.CustomVersionOpt("v21.2.15"))
 			},
 		},
 		{