diff --git a/cmd/edit.go b/cmd/edit.go index 38fc7d0168..7586d80d27 100644 --- a/cmd/edit.go +++ b/cmd/edit.go @@ -16,6 +16,7 @@ import ( "github.com/elastic/elastic-package/internal/common" "github.com/elastic/elastic-package/internal/install" "github.com/elastic/elastic-package/internal/kibana" + "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/stack" ) @@ -93,9 +94,26 @@ func editDashboardsCmd(cmd *cobra.Command, args []string) error { } if len(dashboardIDs) == 0 { - dashboardIDs, err = promptDashboardIDs(cmd.Context(), kibanaClient) + // Not mandatory to get the package name here, but it would be helpful for users + // to select by default the package where they are located if any. + defaultPackage := "" + packageRoot, err := packages.MustFindPackageRoot() + if err == nil { + m, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) + if err != nil { + return fmt.Errorf("reading package manifest failed (path: %s): %w", packageRoot, err) + } + defaultPackage = m.Name + } + selectOptions := selectDashboardOptions{ + ctx: cmd.Context(), + kibanaClient: kibanaClient, + kibanaVersion: kibanaVersion, + defaultPackage: defaultPackage, + } + dashboardIDs, err = selectDashboardIDs(selectOptions) if err != nil { - return fmt.Errorf("prompt for dashboard selection failed: %w", err) + return fmt.Errorf("selecting dashboard IDs failed: %w", err) } if len(dashboardIDs) == 0 { diff --git a/cmd/export_dashboards.go b/cmd/export_dashboards.go index b0f446e2ba..026b7677c9 100644 --- a/cmd/export_dashboards.go +++ b/cmd/export_dashboards.go @@ -7,7 +7,9 @@ package cmd import ( "context" "fmt" + "strings" + "github.com/elastic/elastic-package/internal/packages" "github.com/elastic/elastic-package/internal/tui" "github.com/spf13/cobra" @@ -19,10 +21,14 @@ import ( "github.com/elastic/elastic-package/internal/stack" ) -const exportDashboardsLongDescription = `Use this command to export dashboards with referenced objects from the Kibana instance. +const ( + exportDashboardsLongDescription = `Use this command to export dashboards with referenced objects from the Kibana instance. Use this command to download selected dashboards and other associated saved objects from Kibana. This command adjusts the downloaded saved objects according to package naming conventions (prefixes, unique IDs) and writes them locally into folders corresponding to saved object types (dashboard, visualization, map, etc.).` + newDashboardOption = "Working on a new dashboard (show all available dashboards)" +) + func exportDashboardsCmd(cmd *cobra.Command, args []string) error { cmd.Println("Export Kibana dashboards") @@ -67,10 +73,26 @@ func exportDashboardsCmd(cmd *cobra.Command, args []string) error { fmt.Printf("Warning: %s\n", message) } + // Just query for dashboards if none were provided as flags if len(dashboardIDs) == 0 { - dashboardIDs, err = promptDashboardIDs(cmd.Context(), kibanaClient) + packageRoot, err := packages.MustFindPackageRoot() + if err != nil { + return fmt.Errorf("locating package root failed: %w", err) + } + m, err := packages.ReadPackageManifestFromPackageRoot(packageRoot) if err != nil { - return fmt.Errorf("prompt for dashboard selection failed: %w", err) + return fmt.Errorf("reading package manifest failed (path: %s): %w", packageRoot, err) + } + options := selectDashboardOptions{ + ctx: cmd.Context(), + kibanaClient: kibanaClient, + kibanaVersion: kibanaVersion, + defaultPackage: m.Name, + } + + dashboardIDs, err = selectDashboardIDs(options) + if err != nil { + return fmt.Errorf("selecting dashboard IDs failed: %w", err) } if len(dashboardIDs) == 0 { @@ -88,12 +110,171 @@ func exportDashboardsCmd(cmd *cobra.Command, args []string) error { return nil } -func promptDashboardIDs(ctx context.Context, kibanaClient *kibana.Client) ([]string, error) { +type selectDashboardOptions struct { + ctx context.Context + kibanaClient *kibana.Client + kibanaVersion kibana.VersionInfo + defaultPackage string +} + +// selectDashboardIDs prompts the user to select dashboards to export. It handles +// different flows depending on whether the Kibana instance is a Serverless environment or not. +// In non-Serverless environments, it prompts directly for dashboard selection. +// In Serverless environments, it first prompts to select an installed package or choose +// to export new dashboards, and then prompts for dashboard selection accordingly. +func selectDashboardIDs(options selectDashboardOptions) ([]string, error) { + if options.kibanaVersion.BuildFlavor != kibana.ServerlessFlavor { + // This method uses a deprecated API to search for saved objects. + // And this API is not available in Serverless environments. + dashboardIDs, err := promptDashboardIDsNonServerless(options.ctx, options.kibanaClient) + if err != nil { + return nil, fmt.Errorf("prompt for dashboard selection failed: %w", err) + } + return dashboardIDs, nil + } + + installedPackage, err := promptPackagesInstalled(options.ctx, options.kibanaClient, options.defaultPackage) + if err != nil { + return nil, fmt.Errorf("prompt for package selection failed: %w", err) + } + + if installedPackage == "" { + fmt.Println("No installed packages were found in Kibana.") + return nil, nil + } + + if installedPackage == newDashboardOption { + dashboardIDs, err := promptDashboardIDsServerless(options.ctx, options.kibanaClient) + if err != nil { + return nil, fmt.Errorf("prompt for dashboard selection failed: %w", err) + } + return dashboardIDs, nil + } + + // As it can be installed just one version of a package in Elastic, we can split by '-' to get the name. + // This package name will be used to get the data related to a package (kibana.GetPackage). + installedPackageName, _, found := strings.Cut(installedPackage, "-") + if !found { + return nil, fmt.Errorf("invalid package name: %s", installedPackage) + } + + dashboardIDs, err := promptPackageDashboardIDs(options.ctx, options.kibanaClient, installedPackageName) + if err != nil { + return nil, fmt.Errorf("prompt for package dashboard selection failed: %w", err) + } + + return dashboardIDs, nil +} + +func promptPackagesInstalled(ctx context.Context, kibanaClient *kibana.Client, defaultPackageName string) (string, error) { + installedPackages, err := kibanaClient.FindInstalledPackages(ctx) + if err != nil { + return "", fmt.Errorf("finding installed packages failed: %w", err) + } + + // First option is always to list all available dashboards even if they are not related + // to any package. This is helpful in case the user is working on a new dashboard. + options := []string{newDashboardOption} + + options = append(options, installedPackages.Strings()...) + defaultOption := "" + for _, ip := range installedPackages { + if ip.Name == defaultPackageName { + // set default package to the one matching the package in the current directory + defaultOption = ip.String() + break + } + } + + packagesPrompt := tui.NewSelect("Which packages would you like to export dashboards from?", options, defaultOption) + + var selectedOption string + err = tui.AskOne(packagesPrompt, &selectedOption, tui.Required) + if err != nil { + return "", err + } + + return selectedOption, nil +} + +// promptPackageDashboardIDs prompts the user to select dashboards from the given package. +// It requires the package name to fetch the installed package information from Kibana. +func promptPackageDashboardIDs(ctx context.Context, kibanaClient *kibana.Client, packageName string) ([]string, error) { + installedPackage, err := kibanaClient.GetPackage(ctx, packageName) + if err != nil { + return nil, fmt.Errorf("failed to get package status: %w", err) + } + if installedPackage.Status == "not_installed" { + return nil, fmt.Errorf("package %s is not installed", packageName) + } + + // get asset titles from IDs + packageAssets := []packages.Asset{} + for _, asset := range installedPackage.InstallationInfo.InstalledKibanaAssets { + if asset.Type != "dashboard" { + continue + } + + packageAssets = append(packageAssets, packages.Asset{ID: asset.ID, Type: asset.Type}) + } + + assetsResponse, err := kibanaClient.GetDataFromPackageAssetIDs(ctx, packageAssets) + if err != nil { + return nil, fmt.Errorf("failed to get package assets: %w", err) + } + + dashboardIDOptions := []string{} + for _, asset := range assetsResponse { + if asset.Type != "dashboard" { + continue + } + dashboardIDOptions = append(dashboardIDOptions, asset.String()) + } + + if len(dashboardIDOptions) == 0 { + return nil, fmt.Errorf("no dashboards found for package %s", packageName) + } + + dashboardsPrompt := tui.NewMultiSelect("Which dashboards would you like to export?", dashboardIDOptions, []string{}) + dashboardsPrompt.SetPageSize(100) + + var selectedOptions []string + err = tui.AskOne(dashboardsPrompt, &selectedOptions, tui.Required) + if err != nil { + return nil, err + } + + var selectedIDs []string + for _, option := range selectedOptions { + for _, asset := range assetsResponse { + if asset.String() == option { + selectedIDs = append(selectedIDs, asset.ID) + } + } + } + + return selectedIDs, nil +} + +func promptDashboardIDsServerless(ctx context.Context, kibanaClient *kibana.Client) ([]string, error) { + savedDashboards, err := kibanaClient.FindServerlessDashboards(ctx) + if err != nil { + return nil, fmt.Errorf("finding dashboards failed: %w", err) + } + + return promptDashboardIDs(savedDashboards) +} + +func promptDashboardIDsNonServerless(ctx context.Context, kibanaClient *kibana.Client) ([]string, error) { savedDashboards, err := kibanaClient.FindDashboards(ctx) if err != nil { return nil, fmt.Errorf("finding dashboards failed: %w", err) } + return promptDashboardIDs(savedDashboards) +} + +func promptDashboardIDs(savedDashboards kibana.DashboardSavedObjects) ([]string, error) { if len(savedDashboards) == 0 { return []string{}, nil } @@ -102,7 +283,7 @@ func promptDashboardIDs(ctx context.Context, kibanaClient *kibana.Client) ([]str dashboardsPrompt.SetPageSize(100) var selectedOptions []string - err = tui.AskOne(dashboardsPrompt, &selectedOptions, tui.Required) + err := tui.AskOne(dashboardsPrompt, &selectedOptions, tui.Required) if err != nil { return nil, err } diff --git a/internal/kibana/dashboards.go b/internal/kibana/dashboards.go index 347b97f90f..2dce04bb8a 100644 --- a/internal/kibana/dashboards.go +++ b/internal/kibana/dashboards.go @@ -93,3 +93,17 @@ func (c *Client) exportWithDashboardsAPI(ctx context.Context, dashboardIDs []str } return exported.Objects, nil } + +// exportAllDashboards method exports all dashboards using the Kibana APIs without any export details nor including references. +// The number of exported dashboards depends on the "savedObjects.maxImportExportSize" setting, that by default is 10000. +func (c *Client) exportAllDashboards(ctx context.Context) ([]common.MapStr, error) { + logger.Debug("Export dashboards using the Kibana Saved Objects Export API") + + request := ExportSavedObjectsRequest{ + ExcludeExportDetails: true, + IncludeReferencesDeep: false, + Type: "dashboard", + } + + return c.ExportSavedObjects(ctx, request) +} diff --git a/internal/kibana/packages.go b/internal/kibana/packages.go index 3b8950249e..26af579a18 100644 --- a/internal/kibana/packages.go +++ b/internal/kibana/packages.go @@ -11,12 +11,54 @@ import ( "fmt" "net/http" "os" + "sort" + "strings" "github.com/elastic/elastic-package/internal/packages" ) +const findInstalledPackagesPerPage = 100 + var ErrNotSupported error = errors.New("not supported") +type findInstalledPackagesResponse struct { + Items []installedPackage `json:"items"` + Total int `json:"total"` + SearchAfter []string `json:"searchAfter"` +} + +type installedPackages []installedPackage +type installedPackage struct { + Name string `json:"name"` + Version string `json:"version"` +} + +// String method returns string representation for an installed package. +func (ip *installedPackage) String() string { + return fmt.Sprintf("%s-%s", ip.Name, ip.Version) +} + +// Strings method returns string representation for a set of installed packages. +func (ips installedPackages) Strings() []string { + var entries []string + for _, ip := range ips { + entries = append(entries, ip.String()) + } + return entries +} + +type bulkAssetItemResponse struct { + ID string `json:"id"` + Type string `json:"type"` + Attributes struct { + Title string `json:"title"` + } `json:"attributes"` +} + +func (p *bulkAssetItemResponse) String() string { + return fmt.Sprintf("%s (ID: %s, Type: %s)", p.Attributes.Title, p.ID, p.Type) +} + // InstallPackage installs the given package in Fleet. func (c *Client) InstallPackage(ctx context.Context, name, version string) ([]packages.Asset, error) { path := c.epmPackageUrl(name, version) @@ -216,3 +258,79 @@ func processResults(action string, statusCode int, respBody []byte) ([]packages. return resp.Items, nil } + +// FindInstalledPackages retrieves the current installed packages (name and version). Response is sorted by name. +func (c *Client) FindInstalledPackages(ctx context.Context) (installedPackages, error) { + var installed installedPackages + searchAfter := "" + + for { + r, err := c.findInstalledPackagesNextPage(ctx, searchAfter) + if err != nil { + return nil, fmt.Errorf("can't fetch page with results: %w", err) + } + if len(installed) >= r.Total { + break + } + searchAfter = r.SearchAfter[0] + installed = append(installed, r.Items...) + } + sort.Slice(installed, func(i, j int) bool { + return sort.StringsAreSorted([]string{strings.ToLower(installed[i].Name), strings.ToLower(installed[j].Name)}) + }) + + return installed, nil +} + +// findInstalledPackagesNextPage retrieves the next set of packages after the given searchAfter value. +func (c *Client) findInstalledPackagesNextPage(ctx context.Context, searchAfter string) (*findInstalledPackagesResponse, error) { + path := fmt.Sprintf("%s/epm/packages/installed?perPage=%d&searchAfter=[\"%s\"]", FleetAPI, findInstalledPackagesPerPage, searchAfter) + statusCode, respBody, err := c.get(ctx, path) + if err != nil { + return nil, fmt.Errorf("could not find installed packages; API status code = %d; response body = %s: %w", statusCode, string(respBody), err) + } + + if statusCode != http.StatusOK { + return nil, fmt.Errorf("could not find installed packages; API status code = %d; response body = %s", statusCode, string(respBody)) + } + + var resp findInstalledPackagesResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("could not convert installed packages (response) to JSON: %w", err) + } + return &resp, nil +} + +// GetDataFromPackageAssetIDs retrieves data, such as the title, for the given a list of asset IDs +// using the "bulk_assets" Elastic Package Manager API endpoint. Response is sorted by title. +func (c *Client) GetDataFromPackageAssetIDs(ctx context.Context, assets []packages.Asset) ([]bulkAssetItemResponse, error) { + path := fmt.Sprintf("%s/epm/bulk_assets", FleetAPI) + type request struct { + AssetIDs []packages.Asset `json:"assetIds"` + } + reqBody, err := json.Marshal(request{AssetIDs: assets}) + if err != nil { + return nil, fmt.Errorf("could not convert assets (request) to JSON: %w", err) + } + statusCode, respBody, err := c.post(ctx, path, reqBody) + if err != nil { + return nil, fmt.Errorf("could not get assets; API status code = %d; response body = %s: %w", statusCode, string(respBody), err) + } + + if statusCode != http.StatusOK { + return nil, fmt.Errorf("could not get assets; API status code = %d; response body = %s", statusCode, string(respBody)) + } + + type packageAssetsResponse struct { + Items []bulkAssetItemResponse `json:"items"` + } + + var resp packageAssetsResponse + if err := json.Unmarshal(respBody, &resp); err != nil { + return nil, fmt.Errorf("could not convert get assets (response) to JSON: %w", err) + } + sort.Slice(resp.Items, func(i, j int) bool { + return sort.StringsAreSorted([]string{strings.ToLower(resp.Items[i].Attributes.Title), strings.ToLower(resp.Items[j].Attributes.Title)}) + }) + return resp.Items, nil +} diff --git a/internal/kibana/savedobjects.go b/internal/kibana/savedobjects.go index 82bc734e65..d00ef561ed 100644 --- a/internal/kibana/savedobjects.go +++ b/internal/kibana/savedobjects.go @@ -61,10 +61,8 @@ func (dso *DashboardSavedObject) String() string { // FindDashboards method returns dashboards available in the Kibana instance. func (c *Client) FindDashboards(ctx context.Context) (DashboardSavedObjects, error) { logger.Debug("Find dashboards using the Saved Objects API") - var foundObjects DashboardSavedObjects page := 1 - for { r, err := c.findDashboardsNextPage(ctx, page) if err != nil { @@ -112,6 +110,50 @@ func (c *Client) findDashboardsNextPage(ctx context.Context, page int) (*savedOb return &r, nil } +// FindServerlessDashboards method returns dashboards available in the Kibana instance. +func (c *Client) FindServerlessDashboards(ctx context.Context) (DashboardSavedObjects, error) { + logger.Debug("Find dashboards using the Export Objects API") + var foundObjects DashboardSavedObjects + allDashboards, err := c.exportAllDashboards(ctx) + if err != nil { + return nil, fmt.Errorf("can't export all dashboards: %w", err) + } + for _, dashboard := range allDashboards { + dashboardSavedObject, err := dashboardSavedObjectFromMapStr(dashboard) + if err != nil { + return nil, fmt.Errorf("can't parse dashboard saved object: %w", err) + } + foundObjects = append(foundObjects, dashboardSavedObject) + } + + sort.Slice(foundObjects, func(i, j int) bool { + return sort.StringsAreSorted([]string{strings.ToLower(foundObjects[i].Title), strings.ToLower(foundObjects[j].Title)}) + }) + return foundObjects, nil +} + +func dashboardSavedObjectFromMapStr(data common.MapStr) (DashboardSavedObject, error) { + dashboardId, ok := data["id"].(string) + if !ok { + return DashboardSavedObject{}, fmt.Errorf("dashboard object does not contain id field") + } + + attributes, ok := data["attributes"].(map[string]any) + if !ok { + return DashboardSavedObject{}, fmt.Errorf("dashboard object does not contain attributes field") + } + + dashboardTitle, ok := attributes["title"].(string) + if !ok { + return DashboardSavedObject{}, fmt.Errorf("dashboard object does not contain attributes.title field") + } + + return DashboardSavedObject{ + ID: dashboardId, + Title: dashboardTitle, + }, nil +} + // SetManagedSavedObject method sets the managed property in a saved object. // For example managed dashboards cannot be edited, and setting managed to false will // allow to edit them. @@ -141,18 +183,24 @@ func (c *Client) SetManagedSavedObject(ctx context.Context, savedObjectType stri Overwrite: true, Objects: objects, } - _, err = c.ImportSavedObjects(ctx, importRequest) + resp, err := c.ImportSavedObjects(ctx, importRequest) if err != nil { return fmt.Errorf("failed to import %s %s: %w", savedObjectType, id, err) } + // Even if no error is returned, we need to check if the import was successful. + if !resp.Success { + return fmt.Errorf("importing %s %s was not successful", savedObjectType, id) + } + return nil } type ExportSavedObjectsRequest struct { ExcludeExportDetails bool `json:"excludeExportDetails"` IncludeReferencesDeep bool `json:"includeReferencesDeep"` - Objects []ExportSavedObjectsRequestObject `json:"objects"` + Objects []ExportSavedObjectsRequestObject `json:"objects,omitempty"` + Type string `json:"type,omitempty"` } type ExportSavedObjectsRequestObject struct { diff --git a/internal/kibana/status.go b/internal/kibana/status.go index 3a8ad75e21..fec36b5a2e 100644 --- a/internal/kibana/status.go +++ b/internal/kibana/status.go @@ -19,6 +19,8 @@ type VersionInfo struct { BuildFlavor string `json:"build_flavor"` } +const ServerlessFlavor = "serverless" + func (v VersionInfo) Version() string { if v.BuildSnapshot { return fmt.Sprintf("%s%s", v.Number, SNAPSHOT_SUFFIX) diff --git a/internal/packages/assets.go b/internal/packages/assets.go index 01f027e0d5..c1561ad095 100644 --- a/internal/packages/assets.go +++ b/internal/packages/assets.go @@ -50,10 +50,12 @@ var ( // Asset represents a package asset to be loaded into Kibana or Elasticsearch. type Asset struct { - ID string `json:"id"` - Type AssetType `json:"type"` - DataStream string - SourcePath string + ID string `json:"id"` + Type AssetType `json:"type"` + + // These fields are not expected to be part of responses from APIs + DataStream string `json:"-"` + SourcePath string `json:"-"` } // String method returns a string representation of the asset