wip: commit progress on module validation

Author: buenos-nachos

cmd/readmevalidation/coderResources.go

@@ -4,9 +4,15 @@ import (
 	"bufio"
 	"errors"
 	"fmt"
+	"log"
 	"net/url"
+	"os"
+	"path"
 	"regexp"
+	"slices"
 	"strings"
+
+	"gopkg.in/yaml.v3"
 )
 
 var supportedResourceTypes = []string{"modules", "templates"}
@@ -29,6 +35,13 @@ type coderResourceReadme struct {
 	frontmatter  coderResourceFrontmatter
 }
 
+type coderResourceReadmes map[string]coderResourceReadme
+
+func (crr coderResourceReadmes) Get(filePath string) (coderResourceReadme, bool) {
+	rm, ok := crr[filePath]
+	return rm, ok
+}
+
 func validateCoderResourceDisplayName(displayName *string) error {
 	if displayName != nil && *displayName == "" {
 		return errors.New("if defined, display_name must not be empty string")
@@ -220,3 +233,139 @@ func validateCoderResourceReadme(rm coderResourceReadme) []error {
 
 	return errs
 }
+
+func parseCoderResourceReadme(resourceType string, rm readme) (coderResourceReadme, error) {
+	fm, body, err := separateFrontmatter(rm.rawText)
+	if err != nil {
+		return coderResourceReadme{}, fmt.Errorf("%q: failed to parse frontmatter: %v", rm.filePath, err)
+	}
+
+	yml := coderResourceFrontmatter{}
+	if err := yaml.Unmarshal([]byte(fm), &yml); err != nil {
+		return coderResourceReadme{}, fmt.Errorf("%q: failed to parse: %v", rm.filePath, err)
+	}
+
+	return coderResourceReadme{
+		resourceType: resourceType,
+		filePath:     rm.filePath,
+		body:         body,
+		frontmatter:  yml,
+	}, nil
+}
+
+func parseCoderResourceReadmeFiles(resourceType string, rms []readme) (coderResourceReadmes, error) {
+	resources := coderResourceReadmes(map[string]coderResourceReadme{})
+	var yamlParsingErrs []error
+	for _, rm := range rms {
+		p, err := parseCoderResourceReadme(resourceType, rm)
+		if err != nil {
+			yamlParsingErrs = append(yamlParsingErrs, err)
+			continue
+		}
+
+		resources[p.filePath] = p
+	}
+	if len(yamlParsingErrs) != 0 {
+		return nil, validationPhaseError{
+			phase:  validationPhaseReadmeParsing,
+			errors: yamlParsingErrs,
+		}
+	}
+
+	yamlValidationErrors := []error{}
+	for _, readme := range resources {
+		errors := validateCoderResourceReadme(readme)
+		if len(errors) > 0 {
+			yamlValidationErrors = append(yamlValidationErrors, errors...)
+		}
+	}
+	if len(yamlValidationErrors) != 0 {
+		return nil, validationPhaseError{
+			phase:  validationPhaseReadmeParsing,
+			errors: yamlValidationErrors,
+		}
+	}
+
+	return resources, nil
+}
+
+// Todo: Need to beef up this function by grabbing each image/video URL from
+// the body's AST
+func validateCoderResourceRelativeUrls(resources coderResourceReadmes) error {
+	return nil
+}
+
+func aggregateCoderResourceReadmeFiles(resourceType string) ([]readme, error) {
+	registryFiles, err := os.ReadDir(rootRegistryPath)
+	if err != nil {
+		return nil, err
+	}
+
+	var allReadmeFiles []readme
+	var errs []error
+	for _, rf := range registryFiles {
+		if !rf.IsDir() {
+			continue
+		}
+
+		resourceRootPath := path.Join(rootRegistryPath, rf.Name(), resourceType)
+		resourceDirs, err := os.ReadDir(resourceRootPath)
+		if err != nil {
+			if !errors.Is(err, os.ErrNotExist) {
+				errs = append(errs, err)
+			}
+			continue
+		}
+
+		for _, rd := range resourceDirs {
+			if !rd.IsDir() || rd.Name() == ".coder" {
+				continue
+			}
+
+			resourceReadmePath := path.Join(resourceRootPath, rd.Name(), "README.md")
+			rm, err := os.ReadFile(resourceReadmePath)
+			if err != nil {
+				errs = append(errs, err)
+				continue
+			}
+
+			allReadmeFiles = append(allReadmeFiles, readme{
+				filePath: resourceReadmePath,
+				rawText:  string(rm),
+			})
+		}
+	}
+
+	if len(errs) != 0 {
+		return nil, validationPhaseError{
+			phase:  validationPhaseFileLoad,
+			errors: errs,
+		}
+	}
+	return allReadmeFiles, nil
+}
+
+func validateAllCoderResourceFilesOfType(resourceType string) error {
+	if !slices.Contains(supportedResourceTypes, resourceType) {
+		return fmt.Errorf("resource type %q is not part of supported list [%s]", resourceType, strings.Join(supportedResourceTypes, ", "))
+	}
+
+	allReadmeFiles, err := aggregateCoderResourceReadmeFiles(resourceType)
+	if err != nil {
+		return err
+	}
+
+	log.Printf("Processing %d README files\n", len(allReadmeFiles))
+	resources, err := parseCoderResourceReadmeFiles(resourceType, allReadmeFiles)
+	if err != nil {
+		return err
+	}
+	log.Printf("Processed %d README files as valid Coder resources with type %q", len(resources), resourceType)
+
+	err = validateCoderResourceRelativeUrls(resources)
+	if err != nil {
+		return err
+	}
+	log.Printf("All relative URLs for %s READMEs are valid\n", resourceType)
+	return nil
+}

cmd/readmevalidation/main.go

@@ -23,13 +23,18 @@ func main() {
 		os.Exit(1)
 	}
 
-	errs := []error{}
+	var errs []error
 	err := validateAllContributorFiles()
 	if err != nil {
 		errs = append(errs, err)
 	}
+	err = validateAllCoderResourceFilesOfType("modules")
+	if err != nil {
+		errs = append(errs, err)
+	}
 
 	if len(errs) == 0 {
+		log.Printf("Processed all READMEs in the %q directory\n", rootRegistryPath)
 		os.Exit(0)
 	}
 	for _, err := range errs {