Add custom openstack client implementation

internal/knowledge/datasources/openstack/identity/identity_api.go

@@ -6,14 +6,13 @@ package identity
 import (
 	"context"
 	"log/slog"
+	"net/url"
 	"strings"
 
 	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
 	"github.com/cobaltcore-dev/cortex/internal/knowledge/datasources"
 	"github.com/cobaltcore-dev/cortex/pkg/keystone"
-	"github.com/gophercloud/gophercloud/v2"
-	"github.com/gophercloud/gophercloud/v2/openstack/identity/v3/domains"
-	"github.com/gophercloud/gophercloud/v2/openstack/identity/v3/projects"
+	"github.com/cobaltcore-dev/cortex/pkg/openstack"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -28,7 +27,7 @@ type IdentityAPI interface {
 type identityAPI struct {
 	mon         datasources.Monitor
 	keystoneAPI keystone.KeystoneAPI
-	sc          *gophercloud.ServiceClient
+	client      *openstack.OpenstackClient
 	conf        v1alpha1.IdentityDatasource
 }
 
@@ -37,21 +36,11 @@ func NewIdentityAPI(mon datasources.Monitor, k keystone.KeystoneAPI, conf v1alph
 }
 
 func (api *identityAPI) Init(ctx context.Context) error {
-	if err := api.keystoneAPI.Authenticate(ctx); err != nil {
-		return err
-	}
-	provider := api.keystoneAPI.Client()
-	serviceType := "identity"
-	url, err := api.keystoneAPI.FindEndpoint("public", serviceType)
+	client, err := openstack.IdentityClient(ctx, api.keystoneAPI)
 	if err != nil {
 		return err
 	}
-	slog.Info("using identity endpoint", "url", url)
-	api.sc = &gophercloud.ServiceClient{
-		ProviderClient: provider,
-		Endpoint:       url,
-		Type:           serviceType,
-	}
+	api.client = client
 	return nil
 }
 
@@ -64,19 +53,14 @@ func (api *identityAPI) GetAllDomains(ctx context.Context) ([]Domain, error) {
 		timer := prometheus.NewTimer(hist)
 		defer timer.ObserveDuration()
 	}
-	client := api.sc
-	allPages, err := domains.List(client, nil).AllPages(ctx)
-	if err != nil {
-		return nil, err
-	}
-	var data = &struct {
-		Domains []Domain `json:"domains"`
-	}{}
-	if err := allPages.(domains.DomainPage).ExtractInto(data); err != nil {
+
+	var domains []Domain
+	if err := api.client.List(ctx, "domains", nil, "domains", &domains); err != nil {
 		return nil, err
 	}
-	slog.Info("fetched identity data", "label", "domains", "count", len(data.Domains))
-	return data.Domains, nil
+
+	slog.Info("fetched identity data", "label", "domains", "count", len(domains))
+	return domains, nil
 }
 
 // Get all the projects from the OpenStack identity service.
@@ -88,21 +72,16 @@ func (api *identityAPI) GetAllProjects(ctx context.Context) ([]Project, error) {
 		timer := prometheus.NewTimer(hist)
 		defer timer.ObserveDuration()
 	}
-	client := api.sc
-	allPages, err := projects.List(client, nil).AllPages(ctx)
-	if err != nil {
-		return nil, err
-	}
 
-	var data = &struct {
-		Projects []RawProject `json:"projects"`
-	}{}
-	if err := allPages.(projects.ProjectPage).ExtractInto(data); err != nil {
+	var rawProjects []RawProject
+	if err := api.client.List(ctx, "projects", url.Values{}, "projects", &rawProjects); err != nil {
 		return nil, err
 	}
-	var result []Project
-	for _, p := range data.Projects {
-		result = append(result, Project{
+
+	slog.Info("fetched identity data", "label", "projects", "count", len(rawProjects))
+	var projects []Project
+	for _, p := range rawProjects {
+		projects = append(projects, Project{
 			ID:       p.ID,
 			Name:     p.Name,
 			DomainID: p.DomainID,
@@ -112,6 +91,6 @@ func (api *identityAPI) GetAllProjects(ctx context.Context) ([]Project, error) {
 			Tags:     strings.Join(p.Tags, ","),
 		})
 	}
-	slog.Info("fetched identity data", "label", "projects", "count", len(data.Projects))
-	return result, nil
+	slog.Info("fetched identity data", "label", "projects", "count", len(projects))
+	return projects, nil
 }

internal/knowledge/datasources/openstack/limes/limes_api.go

@@ -5,18 +5,16 @@ package limes
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
 	"log/slog"
-	"net/http"
+	"net/url"
 	"sync"
 	"time"
 
 	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
 	"github.com/cobaltcore-dev/cortex/internal/knowledge/datasources"
 	"github.com/cobaltcore-dev/cortex/internal/knowledge/datasources/openstack/identity"
 	"github.com/cobaltcore-dev/cortex/pkg/keystone"
-	"github.com/gophercloud/gophercloud/v2"
+	"github.com/cobaltcore-dev/cortex/pkg/openstack"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -36,7 +34,7 @@ type limesAPI struct {
 	// Limes configuration.
 	conf v1alpha1.LimesDatasource
 	// Authenticated OpenStack service client to fetch the data.
-	sc *gophercloud.ServiceClient
+	client *openstack.OpenstackClient
 	// Sleep interval to avoid overloading the API.
 	sleepInterval time.Duration
 }
@@ -48,24 +46,11 @@ func NewLimesAPI(mon datasources.Monitor, k keystone.KeystoneAPI, conf v1alpha1.
 
 // Init the limes API.
 func (api *limesAPI) Init(ctx context.Context) error {
-	if err := api.keystoneAPI.Authenticate(ctx); err != nil {
-		return err
-	}
-	// Automatically fetch the limes endpoint from the keystone service catalog.
-	// See: https://github.com/sapcc/limes/blob/5ea068b/docs/users/api-example.md?plain=1#L23
-	provider := api.keystoneAPI.Client()
-	serviceType := "resources"
-	sameAsKeystone := api.keystoneAPI.Availability()
-	url, err := api.keystoneAPI.FindEndpoint(sameAsKeystone, serviceType)
+	client, err := openstack.LimesClient(ctx, api.keystoneAPI)
 	if err != nil {
 		return err
 	}
-	slog.Info("using limes endpoint", "url", url)
-	api.sc = &gophercloud.ServiceClient{
-		ProviderClient: provider,
-		Endpoint:       url,
-		Type:           serviceType,
-	}
+	api.client = client
 	return nil
 }
 
@@ -122,34 +107,20 @@ func (api *limesAPI) GetAllCommitments(ctx context.Context, projects []identity.
 
 // Resolve the commitments for the given project.
 func (api *limesAPI) getCommitments(ctx context.Context, project identity.Project) ([]Commitment, error) {
-	url := api.sc.Endpoint + "v1" +
+	var commitments []Commitment
+
+	path := "v1" +
 		"/domains/" + project.DomainID +
 		"/projects/" + project.ID +
 		"/commitments"
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, http.NoBody)
-	if err != nil {
-		return nil, err
-	}
-	req.Header.Set("X-Auth-Token", api.sc.Token())
-	resp, err := api.sc.HTTPClient.Do(req)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("unexpected status code: %d", resp.StatusCode)
-	}
-	var list struct {
-		Commitments []Commitment `json:"commitments"`
-	}
-	err = json.NewDecoder(resp.Body).Decode(&list)
-	if err != nil {
+
+	if err := api.client.List(ctx, path, url.Values{}, "commitments", &commitments); err != nil {
 		return nil, err
 	}
 	// Add the project information to each commitment.
-	for i := range list.Commitments {
-		list.Commitments[i].ProjectID = project.ID
-		list.Commitments[i].DomainID = project.DomainID
+	for i := range commitments {
+		commitments[i].ProjectID = project.ID
+		commitments[i].DomainID = project.DomainID
 	}
-	return list.Commitments, nil
+	return commitments, nil
 }

internal/knowledge/datasources/openstack/manila/manila_api.go

@@ -5,16 +5,13 @@ package manila
 
 import (
 	"context"
-	"fmt"
 	"log/slog"
+	"net/url"
 
 	"github.com/cobaltcore-dev/cortex/api/v1alpha1"
 	"github.com/cobaltcore-dev/cortex/internal/knowledge/datasources"
 	"github.com/cobaltcore-dev/cortex/pkg/keystone"
-	"github.com/gophercloud/gophercloud/v2"
-	"github.com/gophercloud/gophercloud/v2/openstack"
-	"github.com/gophercloud/gophercloud/v2/openstack/sharedfilesystems/v2/schedulerstats"
-	"github.com/gophercloud/gophercloud/v2/pagination"
+	"github.com/cobaltcore-dev/cortex/pkg/openstack"
 	"github.com/prometheus/client_golang/prometheus"
 )
 
@@ -34,7 +31,7 @@ type manilaAPI struct {
 	// Manila configuration.
 	conf v1alpha1.ManilaDatasource
 	// Authenticated OpenStack service client to fetch the data.
-	sc *gophercloud.ServiceClient
+	client *openstack.OpenstackClient
 }
 
 // Create a new OpenStack Manila api.
@@ -44,52 +41,29 @@ func NewManilaAPI(mon datasources.Monitor, k keystone.KeystoneAPI, conf v1alpha1
 
 // Init the manila API.
 func (api *manilaAPI) Init(ctx context.Context) error {
-	if err := api.keystoneAPI.Authenticate(ctx); err != nil {
-		return err
-	}
-	// Automatically fetch the manila endpoint from the keystone service catalog.
-	provider := api.keystoneAPI.Client()
-	// Workaround to find the v2 service of
-	// See: https://github.com/gophercloud/gophercloud/issues/3347
-	gophercloud.ServiceTypeAliases["shared-file-system"] = []string{"sharev2"}
-	sameAsKeystone := api.keystoneAPI.Availability()
-	sc, err := openstack.NewSharedFileSystemV2(provider, gophercloud.EndpointOpts{
-		Type:         "sharev2",
-		Availability: gophercloud.Availability(sameAsKeystone),
-	})
+	client, err := openstack.ManilaClient(ctx, api.keystoneAPI)
 	if err != nil {
-		return fmt.Errorf("failed to create manila service client: %w", err)
+		return err
 	}
-	sc.Microversion = "2.65"
-	api.sc = sc
+	api.client = client
 	return nil
 }
 
 // Get all Manila storage pools.
 func (api *manilaAPI) GetAllStoragePools(ctx context.Context) ([]StoragePool, error) {
 	label := StoragePool{}.TableName()
 	slog.Info("fetching manila data", "label", label)
-	// Fetch all pages.
-	pages, err := func() (pagination.Page, error) {
-		if api.mon.RequestTimer != nil {
-			hist := api.mon.RequestTimer.WithLabelValues(label)
-			timer := prometheus.NewTimer(hist)
-			defer timer.ObserveDuration()
-		}
-		return schedulerstats.ListDetail(api.sc, schedulerstats.ListDetailOpts{}).AllPages(ctx)
-	}()
-	if err != nil {
-		return nil, err
+
+	if api.mon.RequestTimer != nil {
+		hist := api.mon.RequestTimer.WithLabelValues(label)
+		timer := prometheus.NewTimer(hist)
+		defer timer.ObserveDuration()
 	}
-	// Parse the json data into our custom model.
-	var data = &struct {
-		Pools []StoragePool `json:"pools"`
-	}{}
-	// Log the raw body for debugging purposes.
-	slog.Info("raw response body", "body", pages.(schedulerstats.PoolPage).Body)
-	if err := pages.(schedulerstats.PoolPage).ExtractInto(data); err != nil {
+
+	var pools []StoragePool
+	if err := api.client.List(ctx, "shares/detail", url.Values{"all_tenants": []string{"true"}}, "pools", &pools); err != nil {
 		return nil, err
 	}
-	slog.Info("fetched", "label", label, "count", len(data.Pools))
-	return data.Pools, nil
+	slog.Info("fetched", "label", label, "count", len(pools))
+	return pools, nil
 }