Skip to content

Commit ea0161e

Browse files
manish-agrawal-aimanish-agrawal-ai
authored
feat : Modularisation approach MVP v1 (with support of ciCd Module) (#1523)
* API spec * timeout added * spec update * fix * fix * wip * fix * spec updated * status fix * helm operations * fix * sql fixes * fixes * fix * bom url change * canServerUpdate in get server-info api * bug fix * bug fix * informer fix * fix * module name casing change * DB file numbe change * devtronBomUrl from env variable * wire generate * code comments incorporate * fix * dummy * removed cicd hardcoding * log level made configurable * fix : using InstallerCrdObjectExists * buildInformerToListenOnInstallerObject fix * log removed * code comments incorporate * sql file number changed * bom url defualt value change
1 parent 0f6b144 commit ea0161e

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

42 files changed

+2332
-48
lines changed

‎Wire.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,12 @@ import (
3232
"github.com/devtron-labs/devtron/api/deployment"
3333
"github.com/devtron-labs/devtron/api/externalLink"
3434
client "github.com/devtron-labs/devtron/api/helm-app"
35+
"github.com/devtron-labs/devtron/api/module"
3536
"github.com/devtron-labs/devtron/api/restHandler"
3637
pipeline2 "github.com/devtron-labs/devtron/api/restHandler/app"
3738
"github.com/devtron-labs/devtron/api/router"
3839
"github.com/devtron-labs/devtron/api/router/pubsub"
40+
"github.com/devtron-labs/devtron/api/server"
3941
"github.com/devtron-labs/devtron/api/sse"
4042
"github.com/devtron-labs/devtron/api/sso"
4143
"github.com/devtron-labs/devtron/api/team"
@@ -114,6 +116,8 @@ func InitializeApp() (*App, error) {
114116
appStoreDiscover.AppStoreDiscoverWireSet,
115117
appStoreValues.AppStoreValuesWireSet,
116118
appStoreDeployment.AppStoreDeploymentWireSet,
119+
server.ServerWireSet,
120+
module.ModuleWireSet,
117121
// -------wireset end ----------
118122
gitSensor.GetGitSensorConfig,
119123
gitSensor.NewGitSensorSession,

‎api/helm-app/HelmAppService.go‎

Lines changed: 159 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,28 @@ package client
22

33
import (
44
"context"
5+
"errors"
56
"fmt"
67
"github.com/devtron-labs/devtron/api/connector"
78
openapi "github.com/devtron-labs/devtron/api/helm-app/openapiClient"
89
"github.com/devtron-labs/devtron/client/k8s/application"
910
"github.com/devtron-labs/devtron/pkg/cluster"
11+
serverBean "github.com/devtron-labs/devtron/pkg/server/bean"
12+
serverEnvConfig "github.com/devtron-labs/devtron/pkg/server/config"
13+
serverDataStore "github.com/devtron-labs/devtron/pkg/server/store"
14+
util2 "github.com/devtron-labs/devtron/util"
1015
"github.com/devtron-labs/devtron/util/rbac"
16+
jsonpatch "github.com/evanphx/json-patch"
17+
"github.com/ghodss/yaml"
1118
"github.com/gogo/protobuf/proto"
19+
"github.com/tidwall/gjson"
20+
"github.com/tidwall/sjson"
1221
"go.uber.org/zap"
1322
"net/http"
23+
"reflect"
1424
"strconv"
1525
"strings"
26+
"time"
1627
)
1728

1829
const DEFAULT_CLUSTER = "default_cluster"
@@ -34,26 +45,33 @@ type HelmAppService interface {
3445
IsReleaseInstalled(ctx context.Context, app *AppIdentifier) (bool, error)
3546
RollbackRelease(ctx context.Context, app *AppIdentifier, version int32) (bool, error)
3647
GetClusterConf(clusterId int) (*ClusterConfig, error)
48+
GetDevtronHelmAppIdentifier() *AppIdentifier
49+
UpdateApplicationWithChartInfoWithExtraValues(ctx context.Context, appIdentifier *AppIdentifier, chartRepository *ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error)
3750
}
3851

3952
type HelmAppServiceImpl struct {
40-
logger *zap.SugaredLogger
41-
clusterService cluster.ClusterService
42-
helmAppClient HelmAppClient
43-
pump connector.Pump
44-
enforcerUtil rbac.EnforcerUtilHelm
53+
logger *zap.SugaredLogger
54+
clusterService cluster.ClusterService
55+
helmAppClient HelmAppClient
56+
pump connector.Pump
57+
enforcerUtil rbac.EnforcerUtilHelm
58+
serverDataStore *serverDataStore.ServerDataStore
59+
serverEnvConfig *serverEnvConfig.ServerEnvConfig
4560
}
4661

4762
func NewHelmAppServiceImpl(Logger *zap.SugaredLogger,
4863
clusterService cluster.ClusterService,
4964
helmAppClient HelmAppClient,
50-
pump connector.Pump, enforcerUtil rbac.EnforcerUtilHelm) *HelmAppServiceImpl {
65+
pump connector.Pump, enforcerUtil rbac.EnforcerUtilHelm, serverDataStore *serverDataStore.ServerDataStore,
66+
serverEnvConfig *serverEnvConfig.ServerEnvConfig) *HelmAppServiceImpl {
5167
return &HelmAppServiceImpl{
52-
logger: Logger,
53-
clusterService: clusterService,
54-
helmAppClient: helmAppClient,
55-
pump: pump,
56-
enforcerUtil: enforcerUtil,
68+
logger: Logger,
69+
clusterService: clusterService,
70+
helmAppClient: helmAppClient,
71+
pump: pump,
72+
enforcerUtil: enforcerUtil,
73+
serverDataStore: serverDataStore,
74+
serverEnvConfig: serverEnvConfig,
5775
}
5876
}
5977

@@ -179,6 +197,7 @@ func (impl *HelmAppServiceImpl) GetClusterConf(clusterId int) (*ClusterConfig, e
179197
}
180198
return config, nil
181199
}
200+
182201
func (impl *HelmAppServiceImpl) GetApplicationDetail(ctx context.Context, app *AppIdentifier) (*AppDetail, error) {
183202
config, err := impl.GetClusterConf(app.ClusterId)
184203
if err != nil {
@@ -191,6 +210,26 @@ func (impl *HelmAppServiceImpl) GetApplicationDetail(ctx context.Context, app *A
191210
ReleaseName: app.ReleaseName,
192211
}
193212
appdetail, err := impl.helmAppClient.GetAppDetail(ctx, req)
213+
if err != nil {
214+
impl.logger.Errorw("error in fetching app detail", "err", err)
215+
return nil, err
216+
}
217+
218+
// if application is devtron app helm release,
219+
// then for FULL (installer object exists), then status is combination of helm app status and installer object status -
220+
// if installer status is not applied then check for timeout and progressing
221+
devtronHelmAppIdentifier := impl.GetDevtronHelmAppIdentifier()
222+
if app.ClusterId == devtronHelmAppIdentifier.ClusterId && app.Namespace == devtronHelmAppIdentifier.Namespace && app.ReleaseName == devtronHelmAppIdentifier.ReleaseName &&
223+
impl.serverDataStore.InstallerCrdObjectExists {
224+
if impl.serverDataStore.InstallerCrdObjectStatus != serverBean.InstallerCrdObjectStatusApplied {
225+
// if timeout
226+
if time.Now().After(appdetail.GetLastDeployed().AsTime().Add(1 * time.Hour)) {
227+
appdetail.ApplicationStatus = serverBean.AppHealthStatusDegraded
228+
} else {
229+
appdetail.ApplicationStatus = serverBean.AppHealthStatusProgressing
230+
}
231+
}
232+
}
194233
return appdetail, err
195234

196235
}
@@ -427,6 +466,115 @@ func (impl *HelmAppServiceImpl) RollbackRelease(ctx context.Context, app *AppIde
427466
return apiResponse.Result, nil
428467
}
429468

469+
func (impl *HelmAppServiceImpl) GetDevtronHelmAppIdentifier() *AppIdentifier {
470+
return &AppIdentifier{
471+
ClusterId: 1,
472+
Namespace: impl.serverEnvConfig.DevtronHelmReleaseNamespace,
473+
ReleaseName: impl.serverEnvConfig.DevtronHelmReleaseName,
474+
}
475+
}
476+
477+
func (impl *HelmAppServiceImpl) UpdateApplicationWithChartInfoWithExtraValues(ctx context.Context, appIdentifier *AppIdentifier,
478+
chartRepository *ChartRepository, extraValues map[string]interface{}, extraValuesYamlUrl string, useLatestChartVersion bool) (*openapi.UpdateReleaseResponse, error) {
479+
480+
// get release info
481+
releaseInfo, err := impl.GetValuesYaml(context.Background(), appIdentifier)
482+
if err != nil {
483+
impl.logger.Errorw("error in fetching helm release info", "err", err)
484+
return nil, err
485+
}
486+
487+
// initialise object with original values
488+
jsonString := releaseInfo.MergedValues
489+
490+
// handle extra values
491+
// special handling for array
492+
if len(extraValues) > 0 {
493+
for k, v := range extraValues {
494+
var valueI interface{}
495+
if reflect.TypeOf(v).Kind() == reflect.Slice {
496+
currentValue := gjson.Get(jsonString, k).Value()
497+
value := make([]interface{}, 0)
498+
if currentValue != nil {
499+
value = currentValue.([]interface{})
500+
}
501+
for _, singleNewVal := range v.([]interface{}) {
502+
value = append(value, singleNewVal)
503+
}
504+
valueI = value
505+
} else {
506+
valueI = v
507+
}
508+
jsonString, err = sjson.Set(jsonString, k, valueI)
509+
if err != nil {
510+
impl.logger.Errorw("error in handing extra values", "err", err)
511+
return nil, err
512+
}
513+
}
514+
}
515+
516+
// convert to byte array
517+
mergedValuesJsonByteArr := []byte(jsonString)
518+
519+
// handle extra values from url
520+
if len(extraValuesYamlUrl) > 0 {
521+
extraValuesUrlYamlByteArr, err := util2.ReadFromUrlWithRetry(extraValuesYamlUrl)
522+
if err != nil {
523+
impl.logger.Errorw("error in reading content", "extraValuesYamlUrl", extraValuesYamlUrl, "err", err)
524+
return nil, err
525+
} else if extraValuesUrlYamlByteArr == nil {
526+
impl.logger.Errorw("response is empty from url", "extraValuesYamlUrl", extraValuesYamlUrl)
527+
return nil, errors.New("response is empty from values url")
528+
}
529+
530+
extraValuesUrlJsonByteArr, err := yaml.YAMLToJSON(extraValuesUrlYamlByteArr)
531+
if err != nil {
532+
impl.logger.Errorw("error in converting json to yaml", "err", err)
533+
return nil, err
534+
}
535+
536+
mergedValuesJsonByteArr, err = jsonpatch.MergePatch(mergedValuesJsonByteArr, extraValuesUrlJsonByteArr)
537+
if err != nil {
538+
impl.logger.Errorw("error in json patch of extra values from url", "err", err)
539+
return nil, err
540+
}
541+
}
542+
543+
// convert JSON to yaml byte array
544+
mergedValuesYamlByteArr, err := yaml.JSONToYAML(mergedValuesJsonByteArr)
545+
if err != nil {
546+
impl.logger.Errorw("error in converting json to yaml", "err", err)
547+
return nil, err
548+
}
549+
550+
// update in helm
551+
updateReleaseRequest := &InstallReleaseRequest{
552+
ReleaseIdentifier: &ReleaseIdentifier{
553+
ReleaseName: appIdentifier.ReleaseName,
554+
ReleaseNamespace: appIdentifier.Namespace,
555+
},
556+
ChartName: releaseInfo.DeployedAppDetail.ChartName,
557+
ValuesYaml: string(mergedValuesYamlByteArr),
558+
ChartRepository: chartRepository,
559+
}
560+
if !useLatestChartVersion {
561+
updateReleaseRequest.ChartVersion = releaseInfo.DeployedAppDetail.ChartVersion
562+
}
563+
564+
updateResponse, err := impl.UpdateApplicationWithChartInfo(ctx, appIdentifier.ClusterId, updateReleaseRequest)
565+
if err != nil {
566+
impl.logger.Errorw("error in upgrading release", "err", err)
567+
return nil, err
568+
}
569+
// update in helm ends
570+
571+
response := &openapi.UpdateReleaseResponse{
572+
Success: updateResponse.Success,
573+
}
574+
575+
return response, nil
576+
}
577+
430578
type AppIdentifier struct {
431579
ClusterId int `json:"clusterId"`
432580
Namespace string `json:"namespace"`

‎api/module/ModuleRestHandler.go‎

Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
/*
2+
* Copyright (c) 2020 Devtron Labs
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*
16+
*/
17+
18+
package module
19+
20+
import (
21+
"encoding/json"
22+
"errors"
23+
"github.com/devtron-labs/devtron/api/restHandler/common"
24+
"github.com/devtron-labs/devtron/pkg/module"
25+
"github.com/devtron-labs/devtron/pkg/user"
26+
"github.com/devtron-labs/devtron/pkg/user/casbin"
27+
"github.com/gorilla/mux"
28+
"go.uber.org/zap"
29+
"gopkg.in/go-playground/validator.v9"
30+
"net/http"
31+
)
32+
33+
type ModuleRestHandler interface {
34+
GetModuleInfo(w http.ResponseWriter, r *http.Request)
35+
HandleModuleAction(w http.ResponseWriter, r *http.Request)
36+
}
37+
38+
type ModuleRestHandlerImpl struct {
39+
logger *zap.SugaredLogger
40+
moduleService module.ModuleService
41+
userService user.UserService
42+
enforcer casbin.Enforcer
43+
validator *validator.Validate
44+
}
45+
46+
func NewModuleRestHandlerImpl(logger *zap.SugaredLogger,
47+
moduleService module.ModuleService,
48+
userService user.UserService,
49+
enforcer casbin.Enforcer,
50+
validator *validator.Validate,
51+
) *ModuleRestHandlerImpl {
52+
return &ModuleRestHandlerImpl{
53+
logger: logger,
54+
moduleService: moduleService,
55+
userService: userService,
56+
enforcer: enforcer,
57+
validator: validator,
58+
}
59+
}
60+
61+
func (impl ModuleRestHandlerImpl) GetModuleInfo(w http.ResponseWriter, r *http.Request) {
62+
// check if user is logged in or not
63+
userId, err := impl.userService.GetLoggedInUser(r)
64+
if userId == 0 || err != nil {
65+
common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized)
66+
return
67+
}
68+
69+
// check query param
70+
params := mux.Vars(r)
71+
moduleName := params["name"]
72+
if len(moduleName) == 0 {
73+
impl.logger.Error("module name is not supplied")
74+
common.WriteJsonResp(w, errors.New("module name is not supplied"), nil, http.StatusBadRequest)
75+
return
76+
}
77+
78+
// service call
79+
res, err := impl.moduleService.GetModuleInfo(moduleName)
80+
if err != nil {
81+
impl.logger.Errorw("service err, GetModuleInfo", "err", err)
82+
common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
83+
return
84+
}
85+
common.WriteJsonResp(w, err, res, http.StatusOK)
86+
}
87+
88+
func (impl ModuleRestHandlerImpl) HandleModuleAction(w http.ResponseWriter, r *http.Request) {
89+
// check if user is logged in or not
90+
userId, err := impl.userService.GetLoggedInUser(r)
91+
if userId == 0 || err != nil {
92+
common.WriteJsonResp(w, err, "Unauthorized User", http.StatusUnauthorized)
93+
return
94+
}
95+
96+
// check query param
97+
params := mux.Vars(r)
98+
moduleName := params["name"]
99+
if len(moduleName) == 0 {
100+
impl.logger.Error("module name is not supplied")
101+
common.WriteJsonResp(w, errors.New("module name is not supplied"), nil, http.StatusBadRequest)
102+
return
103+
}
104+
105+
// decode request
106+
decoder := json.NewDecoder(r.Body)
107+
var moduleActionRequestDto *module.ModuleActionRequestDto
108+
err = decoder.Decode(&moduleActionRequestDto)
109+
if err != nil {
110+
impl.logger.Errorw("error in decoding request in HandleModuleAction", "err", err)
111+
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
112+
return
113+
}
114+
err = impl.validator.Struct(moduleActionRequestDto)
115+
if err != nil {
116+
impl.logger.Errorw("error in validating request in HandleModuleAction", "err", err)
117+
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
118+
return
119+
}
120+
121+
// handle super-admin RBAC
122+
token := r.Header.Get("token")
123+
if ok := impl.enforcer.Enforce(token, casbin.ResourceGlobal, casbin.ActionUpdate, "*"); !ok {
124+
common.WriteJsonResp(w, errors.New("unauthorized"), nil, http.StatusForbidden)
125+
return
126+
}
127+
128+
// service call
129+
res, err := impl.moduleService.HandleModuleAction(userId, moduleName, moduleActionRequestDto)
130+
if err != nil {
131+
impl.logger.Errorw("service err, HandleModuleAction", "err", err)
132+
common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
133+
return
134+
}
135+
common.WriteJsonResp(w, err, res, http.StatusOK)
136+
}

0 commit comments

Comments
 (0)