Skip to content

Commit 9f9d9a2

Browse files
AzathothasAzathothas
committed
♻️ Synced dbin 📦 <-- [1.4, 1.5: code cleanup] ⌚
1 parent 28b179f commit 9f9d9a2

File tree

2 files changed

+148
-78
lines changed

2 files changed

+148
-78
lines changed

‎misc/cmd/dbinRepoIndexGenerators/1.4/generator.go‎

Lines changed: 74 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,18 @@ import (
1919
"github.com/tiendc/go-deepcopy"
2020
)
2121

22+
const (
23+
colorRed = "\033[31m"
24+
colorYellow = "\033[33m"
25+
colorReset = "\033[0m"
26+
)
27+
2228
type repository struct {
2329
URLs []string
2430
Name string
2531
Standalone bool
2632
Single bool
33+
Filter func(*[]DbinItem)
2734
}
2835

2936
type PkgForgeItem struct {
@@ -515,15 +522,20 @@ func saveYAML(filename string, metadata DbinMetadata) error {
515522
func main() {
516523
// Load AppStream metadata once at startup
517524
if err := loadAppStreamMetadata(); err != nil {
518-
fmt.Printf("Error loading AppStream metadata: %v\n", err)
525+
fmt.Printf("%serror:%s Error loading AppStream metadata: %v\n", colorRed, colorReset, err)
519526
}
520527

521528
realArchs := map[string]string{
522-
"x86_64-Linux": "amd64_linux",
523-
"aarch64-Linux": "arm64_linux",
524-
"riscv64-Linux": "riscv64_linux",
529+
"x86_64-Linux": "amd64_linux",
530+
"aarch64-Linux": "arm64_linux",
531+
"riscv64-Linux": "riscv64_linux",
532+
"loongarch64-Linux": "loongarch64_linux",
525533
}
526534

535+
// At least the amd64 repo should have succeeded in order for the fetch failure
536+
// of a repo for a specifc arch to be considered a warning instead of an error.
537+
amd64Success := false
538+
527539
repositories := []struct {
528540
Repo repository
529541
Handler RepositoryHandler
@@ -547,6 +559,22 @@ func main() {
547559
"https://meta.pkgforge.dev/pkgcache/%s.json",
548560
},
549561
Single: true,
562+
Filter: func(items *[]DbinItem) {
563+
var filteredItems []DbinItem
564+
for _, item := range *items {
565+
hasPortableNote := false
566+
for _, note := range item.Notes {
567+
if strings.Contains(note, "[PORTABLE]") {
568+
hasPortableNote = true
569+
break
570+
}
571+
}
572+
if hasPortableNote {
573+
filteredItems = append(filteredItems, item)
574+
}
575+
}
576+
*items = filteredItems
577+
},
550578
},
551579
Handler: PkgForgeHandler{},
552580
},
@@ -557,7 +585,18 @@ func main() {
557585
"https://github.com/pkgforge-go/builder/raw/refs/heads/main/data/%s.json",
558586
"https://meta.pkgforge.dev/external/pkgforge-go/%s.json",
559587
},
560-
Standalone: true,
588+
Single: true,
589+
//Filter: func(items *[]DbinItem) {
590+
// var filteredItems []DbinItem
591+
// for _, item := range *items {
592+
// if !strings.Contains(item.Description, "bindings") && !strings.Contains(item.Description, "key") {
593+
// filteredItems = append(filteredItems, item)
594+
// } /* else {
595+
// fmt.Printf("[pkgforge-go]: repo filter: %s#%s contains bad word (%s)", item.Name, item.PkgId, "bindings")
596+
// } */
597+
// }
598+
// *items = filteredItems
599+
//},
561600
},
562601
Handler: PkgForgeHandler{},
563602
},
@@ -605,44 +644,27 @@ func main() {
605644
},
606645
Handler: DbinHandler{},
607646
},
608-
//{
609-
// Repo: repository{
610-
// Name: "dbin",
611-
// URLs: []string{
612-
// "http://192.168.1.59/d/%s",
613-
// },
614-
// Single: true,
615-
// },
616-
// Handler: DbinHandler{},
617-
//},
618647
}
619648

620649
for arch, outputArch := range realArchs {
621650
dbinMetadata := make(DbinMetadata)
651+
archSuccess := false // Track success for the current architecture
622652

623653
for _, repo := range repositories {
624654
items, err := repo.Handler.FetchMetadata(repo.Repo.URLs, arch)
625655
if err != nil {
626-
fmt.Printf("Error downloading %s metadata: %v\n", repo.Repo.Name, err)
627-
continue
656+
// If amd64 succeeded, treat non-amd64 failures as warnings
657+
if arch != "x86_64-Linux" && amd64Success {
658+
fmt.Printf("%swarning:%s Failed to download %s metadata for %s: %v\n", colorYellow, colorReset, repo.Repo.Name, arch, err)
659+
continue
660+
} else {
661+
fmt.Printf("%serror:%s Error downloading %s metadata for %s: %v\n", colorRed, colorReset, repo.Repo.Name, arch, err)
662+
continue
663+
}
628664
}
629665

630-
// Filter items from "pkgcache" repository that do not contain "[PORTABLE]" in their Notes
631-
if repo.Repo.Name == "pkgcache" {
632-
var filteredItems []DbinItem
633-
for _, item := range items {
634-
hasPortableNote := false
635-
for _, note := range item.Notes {
636-
if strings.Contains(note, "[PORTABLE]") {
637-
hasPortableNote = true
638-
break
639-
}
640-
}
641-
if hasPortableNote {
642-
filteredItems = append(filteredItems, item)
643-
}
644-
}
645-
items = filteredItems
666+
if repo.Repo.Filter != nil {
667+
repo.Repo.Filter(&items)
646668
}
647669

648670
if !repo.Repo.Standalone {
@@ -655,20 +677,33 @@ func main() {
655677
singleOutputFile := fmt.Sprintf("%s_%s", repo.Repo.Name, outputArch)
656678

657679
if err := saveMetadata(singleOutputFile, singleMetadata); err != nil {
658-
fmt.Printf("Error saving single metadata to %s: %v\n", singleOutputFile, err)
680+
fmt.Printf("%serror:%s Error saving single metadata to %s: %v\n", colorRed, colorReset, singleOutputFile, err)
659681
continue
660682
}
661683
fmt.Printf("Successfully saved single metadata to %s\n", singleOutputFile)
662684
}
685+
686+
archSuccess = true // Mark this architecture as successful if at least one repo processed
663687
}
664688

665-
outputFile := fmt.Sprintf("%s", outputArch)
666-
if err := saveMetadata(outputFile, dbinMetadata); err != nil {
667-
fmt.Printf("Error saving metadata to %s: %v\n", outputFile, err)
668-
continue
689+
// Update amd64Success if this is the amd64 architecture
690+
if arch == "x86_64-Linux" && archSuccess {
691+
amd64Success = true
669692
}
670693

671-
fmt.Printf("Successfully processed and saved combined metadata to %s\n", outputFile)
694+
// Save combined metadata only if the architecture had at least one successful repo
695+
if archSuccess {
696+
outputFile := fmt.Sprintf("%s", outputArch)
697+
if err := saveMetadata(outputFile, dbinMetadata); err != nil {
698+
fmt.Printf("%serror:%s Error saving metadata to %s: %v\n", colorRed, colorReset, outputFile, err)
699+
continue
700+
}
701+
fmt.Printf("Successfully processed and saved combined metadata to %s\n", outputFile)
702+
} else if arch != "x86_64-Linux" && amd64Success {
703+
fmt.Printf("%swarning:%s No metadata saved for %s: all repositories failed\n", colorYellow, colorReset, outputArch)
704+
} else {
705+
fmt.Printf("%serror:%s No metadata saved for %s: all repositories failed\n", colorRed, colorReset, outputArch)
706+
}
672707
}
673708
}
674709

‎misc/cmd/dbinRepoIndexGenerators/1.5/generator.go‎

Lines changed: 74 additions & 39 deletions
Original file line numberDiff line numberDiff line change
@@ -19,11 +19,18 @@ import (
1919
"github.com/tiendc/go-deepcopy"
2020
)
2121

22+
const (
23+
colorRed = "\033[31m"
24+
colorYellow = "\033[33m"
25+
colorReset = "\033[0m"
26+
)
27+
2228
type repository struct {
2329
URLs []string
2430
Name string
2531
Standalone bool
2632
Single bool
33+
Filter func(*[]DbinItem)
2734
}
2835

2936
type PkgForgeItem struct {
@@ -515,15 +522,20 @@ func saveYAML(filename string, metadata DbinMetadata) error {
515522
func main() {
516523
// Load AppStream metadata once at startup
517524
if err := loadAppStreamMetadata(); err != nil {
518-
fmt.Printf("Error loading AppStream metadata: %v\n", err)
525+
fmt.Printf("%serror:%s Error loading AppStream metadata: %v\n", colorRed, colorReset, err)
519526
}
520527

521528
realArchs := map[string]string{
522-
"x86_64-Linux": "amd64_linux",
523-
"aarch64-Linux": "arm64_linux",
524-
"riscv64-Linux": "riscv64_linux",
529+
"x86_64-Linux": "amd64_linux",
530+
"aarch64-Linux": "arm64_linux",
531+
"riscv64-Linux": "riscv64_linux",
532+
"loongarch64-Linux": "loongarch64_linux",
525533
}
526534

535+
// At least the amd64 repo should have succeeded in order for the fetch failure
536+
// of a repo for a specifc arch to be considered a warning instead of an error.
537+
amd64Success := false
538+
527539
repositories := []struct {
528540
Repo repository
529541
Handler RepositoryHandler
@@ -547,6 +559,22 @@ func main() {
547559
"https://meta.pkgforge.dev/pkgcache/%s.json",
548560
},
549561
Single: true,
562+
Filter: func(items *[]DbinItem) {
563+
var filteredItems []DbinItem
564+
for _, item := range *items {
565+
hasPortableNote := false
566+
for _, note := range item.Notes {
567+
if strings.Contains(note, "[PORTABLE]") {
568+
hasPortableNote = true
569+
break
570+
}
571+
}
572+
if hasPortableNote {
573+
filteredItems = append(filteredItems, item)
574+
}
575+
}
576+
*items = filteredItems
577+
},
550578
},
551579
Handler: PkgForgeHandler{},
552580
},
@@ -557,7 +585,18 @@ func main() {
557585
"https://github.com/pkgforge-go/builder/raw/refs/heads/main/data/%s.json",
558586
"https://meta.pkgforge.dev/external/pkgforge-go/%s.json",
559587
},
560-
Standalone: true,
588+
Single: true,
589+
//Filter: func(items *[]DbinItem) {
590+
// var filteredItems []DbinItem
591+
// for _, item := range *items {
592+
// if !strings.Contains(item.Description, "bindings") && !strings.Contains(item.Description, "key") {
593+
// filteredItems = append(filteredItems, item)
594+
// } /* else {
595+
// fmt.Printf("[pkgforge-go]: repo filter: %s#%s contains bad word (%s)", item.Name, item.PkgId, "bindings")
596+
// } */
597+
// }
598+
// *items = filteredItems
599+
//},
561600
},
562601
Handler: PkgForgeHandler{},
563602
},
@@ -605,44 +644,27 @@ func main() {
605644
},
606645
Handler: DbinHandler{},
607646
},
608-
//{
609-
// Repo: repository{
610-
// Name: "dbin",
611-
// URLs: []string{
612-
// "http://192.168.1.59/d/%s",
613-
// },
614-
// Single: true,
615-
// },
616-
// Handler: DbinHandler{},
617-
//},
618647
}
619648

620649
for arch, outputArch := range realArchs {
621650
dbinMetadata := make(DbinMetadata)
651+
archSuccess := false // Track success for the current architecture
622652

623653
for _, repo := range repositories {
624654
items, err := repo.Handler.FetchMetadata(repo.Repo.URLs, arch)
625655
if err != nil {
626-
fmt.Printf("Error downloading %s metadata: %v\n", repo.Repo.Name, err)
627-
continue
656+
// If amd64 succeeded, treat non-amd64 failures as warnings
657+
if arch != "x86_64-Linux" && amd64Success {
658+
fmt.Printf("%swarning:%s Failed to download %s metadata for %s: %v\n", colorYellow, colorReset, repo.Repo.Name, arch, err)
659+
continue
660+
} else {
661+
fmt.Printf("%serror:%s Error downloading %s metadata for %s: %v\n", colorRed, colorReset, repo.Repo.Name, arch, err)
662+
continue
663+
}
628664
}
629665

630-
// Filter items from "pkgcache" repository that do not contain "[PORTABLE]" in their Notes
631-
if repo.Repo.Name == "pkgcache" {
632-
var filteredItems []DbinItem
633-
for _, item := range items {
634-
hasPortableNote := false
635-
for _, note := range item.Notes {
636-
if strings.Contains(note, "[PORTABLE]") {
637-
hasPortableNote = true
638-
break
639-
}
640-
}
641-
if hasPortableNote {
642-
filteredItems = append(filteredItems, item)
643-
}
644-
}
645-
items = filteredItems
666+
if repo.Repo.Filter != nil {
667+
repo.Repo.Filter(&items)
646668
}
647669

648670
if !repo.Repo.Standalone {
@@ -655,20 +677,33 @@ func main() {
655677
singleOutputFile := fmt.Sprintf("%s_%s", repo.Repo.Name, outputArch)
656678

657679
if err := saveMetadata(singleOutputFile, singleMetadata); err != nil {
658-
fmt.Printf("Error saving single metadata to %s: %v\n", singleOutputFile, err)
680+
fmt.Printf("%serror:%s Error saving single metadata to %s: %v\n", colorRed, colorReset, singleOutputFile, err)
659681
continue
660682
}
661683
fmt.Printf("Successfully saved single metadata to %s\n", singleOutputFile)
662684
}
685+
686+
archSuccess = true // Mark this architecture as successful if at least one repo processed
663687
}
664688

665-
outputFile := fmt.Sprintf("%s", outputArch)
666-
if err := saveMetadata(outputFile, dbinMetadata); err != nil {
667-
fmt.Printf("Error saving metadata to %s: %v\n", outputFile, err)
668-
continue
689+
// Update amd64Success if this is the amd64 architecture
690+
if arch == "x86_64-Linux" && archSuccess {
691+
amd64Success = true
669692
}
670693

671-
fmt.Printf("Successfully processed and saved combined metadata to %s\n", outputFile)
694+
// Save combined metadata only if the architecture had at least one successful repo
695+
if archSuccess {
696+
outputFile := fmt.Sprintf("%s", outputArch)
697+
if err := saveMetadata(outputFile, dbinMetadata); err != nil {
698+
fmt.Printf("%serror:%s Error saving metadata to %s: %v\n", colorRed, colorReset, outputFile, err)
699+
continue
700+
}
701+
fmt.Printf("Successfully processed and saved combined metadata to %s\n", outputFile)
702+
} else if arch != "x86_64-Linux" && amd64Success {
703+
fmt.Printf("%swarning:%s No metadata saved for %s: all repositories failed\n", colorYellow, colorReset, outputArch)
704+
} else {
705+
fmt.Printf("%serror:%s No metadata saved for %s: all repositories failed\n", colorRed, colorReset, outputArch)
706+
}
672707
}
673708
}
674709

0 commit comments

Comments
 (0)