|
| 1 | +package compute |
| 2 | + |
| 3 | +import ( |
| 4 | + "context" |
| 5 | + "fmt" |
| 6 | + "sort" |
| 7 | + "strings" |
| 8 | + "time" |
| 9 | + |
| 10 | + "github.com/databricks/databricks-sdk-go/databricks/apierr" |
| 11 | + "github.com/databricks/databricks-sdk-go/databricks/log" |
| 12 | + "github.com/databricks/databricks-sdk-go/databricks/retries" |
| 13 | + "github.com/databricks/databricks-sdk-go/databricks/useragent" |
| 14 | +) |
| 15 | + |
| 16 | +type Wait struct { |
| 17 | + ClusterID string |
| 18 | + Libraries []Library |
| 19 | + IsRunning bool |
| 20 | + IsRefresh bool |
| 21 | +} |
| 22 | + |
| 23 | +func (library Library) String() string { |
| 24 | + if library.Whl != "" { |
| 25 | + return fmt.Sprintf("whl:%s", library.Whl) |
| 26 | + } |
| 27 | + if library.Jar != "" { |
| 28 | + return fmt.Sprintf("jar:%s", library.Jar) |
| 29 | + } |
| 30 | + if library.Pypi != nil && library.Pypi.Package != "" { |
| 31 | + return fmt.Sprintf("pypi:%s%s", library.Pypi.Repo, library.Pypi.Package) |
| 32 | + } |
| 33 | + if library.Maven != nil && library.Maven.Coordinates != "" { |
| 34 | + mvn := library.Maven |
| 35 | + return fmt.Sprintf("mvn:%s%s%s", mvn.Repo, mvn.Coordinates, |
| 36 | + strings.Join(mvn.Exclusions, "")) |
| 37 | + } |
| 38 | + if library.Egg != "" { |
| 39 | + return fmt.Sprintf("egg:%s", library.Egg) |
| 40 | + } |
| 41 | + if library.Cran != nil && library.Cran.Package != "" { |
| 42 | + return fmt.Sprintf("cran:%s%s", library.Cran.Repo, library.Cran.Package) |
| 43 | + } |
| 44 | + return "unknown" |
| 45 | +} |
| 46 | + |
| 47 | +func (cll *InstallLibraries) Sort() { |
| 48 | + sort.Slice(cll.Libraries, func(i, j int) bool { |
| 49 | + return cll.Libraries[i].String() < cll.Libraries[j].String() |
| 50 | + }) |
| 51 | +} |
| 52 | + |
| 53 | +// ToLibraryList convert to envity for convenient comparison |
| 54 | +func (cls ClusterLibraryStatuses) ToLibraryList() InstallLibraries { |
| 55 | + cll := InstallLibraries{ClusterId: cls.ClusterId} |
| 56 | + for _, lib := range cls.LibraryStatuses { |
| 57 | + cll.Libraries = append(cll.Libraries, *lib.Library) |
| 58 | + } |
| 59 | + cll.Sort() |
| 60 | + return cll |
| 61 | +} |
| 62 | + |
| 63 | +func (w *Wait) IsNotInScope(lib *Library) bool { |
| 64 | + // if we don't know concrete libraries |
| 65 | + if len(w.Libraries) == 0 { |
| 66 | + return false |
| 67 | + } |
| 68 | + // if we know concrete libraries |
| 69 | + for _, v := range w.Libraries { |
| 70 | + if v.String() == lib.String() { |
| 71 | + return false |
| 72 | + } |
| 73 | + } |
| 74 | + return true |
| 75 | +} |
| 76 | + |
| 77 | +// IsRetryNeeded returns first bool if there needs to be retry. |
| 78 | +// If there needs to be retry, error message will explain why. |
| 79 | +// If retry does not need to happen and error is not nil - it failed. |
| 80 | +func (cls ClusterLibraryStatuses) IsRetryNeeded(w Wait) (bool, error) { |
| 81 | + pending := 0 |
| 82 | + ready := 0 |
| 83 | + errors := []string{} |
| 84 | + for _, lib := range cls.LibraryStatuses { |
| 85 | + if lib.IsLibraryForAllClusters { |
| 86 | + continue |
| 87 | + } |
| 88 | + if w.IsNotInScope(lib.Library) { |
| 89 | + continue |
| 90 | + } |
| 91 | + switch lib.Status { |
| 92 | + // No action has yet been taken to install the library. This state should be very short lived. |
| 93 | + case "PENDING": |
| 94 | + pending++ |
| 95 | + // Metadata necessary to install the library is being retrieved from the provided repository. |
| 96 | + case "RESOLVING": |
| 97 | + pending++ |
| 98 | + // The library is actively being installed, either by adding resources to Spark |
| 99 | + // or executing system commands inside the Spark nodes. |
| 100 | + case "INSTALLING": |
| 101 | + pending++ |
| 102 | + // The library has been successfully installed. |
| 103 | + case "INSTALLED": |
| 104 | + ready++ |
| 105 | + // Installation on a Databricks Runtime 7.0 or above cluster was skipped due to Scala version incompatibility. |
| 106 | + case "SKIPPED": |
| 107 | + ready++ |
| 108 | + // The library has been marked for removal. Libraries can be removed only when clusters are restarted. |
| 109 | + case "UNINSTALL_ON_RESTART": |
| 110 | + ready++ |
| 111 | + //Some step in installation failed. More information can be found in the messages field. |
| 112 | + case "FAILED": |
| 113 | + if w.IsRefresh { |
| 114 | + // we're reading library list on a running cluster and some of the libs failed to install |
| 115 | + continue |
| 116 | + } |
| 117 | + errors = append(errors, fmt.Sprintf("%s failed: %s", lib.Library, strings.Join(lib.Messages, ", "))) |
| 118 | + continue |
| 119 | + } |
| 120 | + } |
| 121 | + if pending > 0 { |
| 122 | + return true, fmt.Errorf("%d libraries are ready, but there are still %d pending", ready, pending) |
| 123 | + } |
| 124 | + if len(errors) > 0 { |
| 125 | + return false, fmt.Errorf("%s", strings.Join(errors, "\n")) |
| 126 | + } |
| 127 | + return false, nil |
| 128 | +} |
| 129 | + |
| 130 | +type Update struct { |
| 131 | + ClusterId string |
| 132 | + // The libraries to install. |
| 133 | + Install []Library |
| 134 | + // The libraries to install. |
| 135 | + Uninstall []Library |
| 136 | +} |
| 137 | + |
| 138 | +type librariesAPIUtilities interface { |
| 139 | + UpdateAndWait(ctx context.Context, update Update, options ...retries.Option[ClusterLibraryStatuses]) error |
| 140 | +} |
| 141 | + |
| 142 | +func (a *LibrariesAPI) UpdateAndWait(ctx context.Context, update Update, |
| 143 | + options ...retries.Option[ClusterLibraryStatuses]) error { |
| 144 | + ctx = useragent.InContext(ctx, "sdk-feature", "update-libraries") |
| 145 | + if len(update.Uninstall) > 0 { |
| 146 | + err := a.Uninstall(ctx, UninstallLibraries{ |
| 147 | + ClusterId: update.ClusterId, |
| 148 | + Libraries: update.Uninstall, |
| 149 | + }) |
| 150 | + if err != nil { |
| 151 | + return fmt.Errorf("uninstall: %w", err) |
| 152 | + } |
| 153 | + } |
| 154 | + if len(update.Install) > 0 { |
| 155 | + err := a.Install(ctx, InstallLibraries{ |
| 156 | + ClusterId: update.ClusterId, |
| 157 | + Libraries: update.Install, |
| 158 | + }) |
| 159 | + if err != nil { |
| 160 | + return fmt.Errorf("install: %w", err) |
| 161 | + } |
| 162 | + } |
| 163 | + // this helps to avoid erroring out when out-of-list library gets added to |
| 164 | + // the cluster manually and thereforce fails the wait on error |
| 165 | + scope := make([]Library, len(update.Install)+len(update.Uninstall)) |
| 166 | + scope = append(scope, update.Install...) |
| 167 | + scope = append(scope, update.Uninstall...) |
| 168 | + _, err := a.Wait(ctx, Wait{ |
| 169 | + ClusterID: update.ClusterId, |
| 170 | + Libraries: scope, |
| 171 | + IsRunning: true, |
| 172 | + IsRefresh: false, |
| 173 | + }, options...) |
| 174 | + return err |
| 175 | +} |
| 176 | + |
| 177 | +// clusterID string, timeout time.Duration, isActive bool, refresh bool |
| 178 | +func (a *LibrariesAPI) Wait(ctx context.Context, wait Wait, |
| 179 | + options ...retries.Option[ClusterLibraryStatuses]) (*ClusterLibraryStatuses, error) { |
| 180 | + ctx = useragent.InContext(ctx, "sdk-feature", "wait-for-libraries") |
| 181 | + i := retries.Info[ClusterLibraryStatuses]{Timeout: 30 * time.Minute} |
| 182 | + for _, o := range options { |
| 183 | + o(&i) |
| 184 | + } |
| 185 | + result, err := retries.Poll(ctx, i.Timeout, func() (*ClusterLibraryStatuses, *retries.Err) { |
| 186 | + status, err := a.ClusterStatusByClusterId(ctx, wait.ClusterID) |
| 187 | + if apierr.IsMissing(err) { |
| 188 | + // eventual consistency error |
| 189 | + return nil, retries.Continue(err) |
| 190 | + } |
| 191 | + for _, o := range options { |
| 192 | + o(&retries.Info[ClusterLibraryStatuses]{ |
| 193 | + Timeout: i.Timeout, |
| 194 | + Info: status, |
| 195 | + }) |
| 196 | + } |
| 197 | + if err != nil { |
| 198 | + return nil, retries.Halt(err) |
| 199 | + } |
| 200 | + if !wait.IsRunning { |
| 201 | + log.InfoContext(ctx, "Cluster %s is currently not running, so just returning list of %d libraries", |
| 202 | + wait.ClusterID, len(status.LibraryStatuses)) |
| 203 | + return status, nil |
| 204 | + } |
| 205 | + retry, err := status.IsRetryNeeded(wait) |
| 206 | + if retry { |
| 207 | + return status, retries.Continue(err) |
| 208 | + } |
| 209 | + if err != nil { |
| 210 | + return status, retries.Halt(err) |
| 211 | + } |
| 212 | + return status, nil |
| 213 | + }) |
| 214 | + if err != nil { |
| 215 | + return nil, err |
| 216 | + } |
| 217 | + if wait.IsRunning { |
| 218 | + installed := []LibraryFullStatus{} |
| 219 | + cleanup := UninstallLibraries{ |
| 220 | + ClusterId: wait.ClusterID, |
| 221 | + Libraries: []Library{}, |
| 222 | + } |
| 223 | + // cleanup libraries that failed to install |
| 224 | + for _, v := range result.LibraryStatuses { |
| 225 | + if v.Status == "FAILED" { |
| 226 | + log.WarningContext(ctx, "Removing failed library %s from %s", |
| 227 | + v.Library, wait.ClusterID) |
| 228 | + cleanup.Libraries = append(cleanup.Libraries, *v.Library) |
| 229 | + continue |
| 230 | + } |
| 231 | + installed = append(installed, v) |
| 232 | + } |
| 233 | + // and result contains only the libraries that were successfully installed |
| 234 | + result.LibraryStatuses = installed |
| 235 | + if len(cleanup.Libraries) > 0 { |
| 236 | + err = a.Uninstall(ctx, cleanup) |
| 237 | + if err != nil { |
| 238 | + return nil, fmt.Errorf("cannot cleanup libraries: %w", err) |
| 239 | + } |
| 240 | + } |
| 241 | + } |
| 242 | + return result, nil |
| 243 | +} |
0 commit comments