Skip to content

Commit 79a0a54

Browse files
kartik-579nishant
kartik-579
and
nishant
authored
feat: Plugin integration - orchestrator (#1435)
* updated spec * updated spec * fixed spec err * added sql script * wip * added plugin config detail api * updated ci pipeline get api * updated script no * fixed sql script * wip * updated sql query * wip * added code for ci pipelien create api - create stages * updated dto for global plugin variable * updated code for ci pipeline update api * fixed old spec error * updated sql model - plugin_tag_relation * updated ci pipeline delete api * updated spec, fixed script mapping bug in update req * wip * added global variable list api * updated command args types * updated script field - mountDirectoryFromHost * wip - fix for create ci pipeline * wip - delete ci stage condition update * wip - ci pipeline create api changes * wip * fixed get api bug * fixed delete pipeline api * wip - delete ci pipeline api changes * wip * wip - fixes * fixed json name * wip * wip * wip * wip * wip * bulk update variables fix * fixed step update bug * review changes - part 1 * review changes -part 2 * updated spec, bean & db model according to UI change * wip * added code for ciStages-plugin ciRunner integration * updated error handling * updated global var api * db model fix * fixed global variable api path * wip * wip * handled err - plugin steps empty case * updated validation * added rbac in global plugin apis * ci runner payload changes * wip * validation removed for pipeline variables * wip * updated ErrNoRows handling for variables db queries * updated ci runner payload, added support for pre stage variables in post-stage * converted dockerBuildScripts(old object) to new stepData obj in WF request * added format in globalVar as per UI requirement * updated wf request as per changes in ci runner: * updated handling for dependent step * updated payload field for variableType * updated pluginVarDto * fixed ref plugin ciRunner payload bug * updated payload fields * updated validation for left fields * updated script no. * updated env variables data in wf request * fixed working dir var * updated sql queries for inserting PRESET plugin data * fixed update condition bug * fixed storeScriptAt data bug * updated ci runner payload * updated steps get query * updated step get query * fixed ci runner payload bug - mount mapping and mountCodeToContainerPath * updated global var description * updated developer guide for variable values & updated code for wf request * updated dev guide * updated plugin var api * updated sql script no. * wip * updated preset plugin script * added executorType for REF_PLUGIN type step * added handling for environment vars in old scripts * updated sql model and code for same name var handling * updated var in sonarqube plugin * updated script * updated script no * updated description for global var to fix UI tooltip len issue * updated description for global var to fix UI tooltip len issue * fix for update api pluginVarIndex value set * updated global var description for UI tooltip fix * updated plugin metadata * fixed ci runner payload - port mapping bug * conditoin update bug fix * updated script no * removed redundant file * added manual plugin creation guide * ci artifact location * blob for post-cd * plugin artifact * update script no. Co-authored-by: nishant <[email protected]>
1 parent 01d0bc1 commit 79a0a54

28 files changed

+4875
-246
lines changed

‎Wire.go‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,9 @@ import (
8787
"github.com/devtron-labs/devtron/pkg/pipeline"
8888
history3 "github.com/devtron-labs/devtron/pkg/pipeline/history"
8989
repository3 "github.com/devtron-labs/devtron/pkg/pipeline/history/repository"
90+
repository5 "github.com/devtron-labs/devtron/pkg/pipeline/repository"
91+
"github.com/devtron-labs/devtron/pkg/plugin"
92+
repository6 "github.com/devtron-labs/devtron/pkg/plugin/repository"
9093
"github.com/devtron-labs/devtron/pkg/projectManagementService/jira"
9194
"github.com/devtron-labs/devtron/pkg/security"
9295
"github.com/devtron-labs/devtron/pkg/sql"
@@ -685,8 +688,28 @@ func InitializeApp() (*App, error) {
685688
wire.Bind(new(history3.ConfigMapHistoryService), new(*history3.ConfigMapHistoryServiceImpl)),
686689
history3.NewPipelineStrategyHistoryServiceImpl,
687690
wire.Bind(new(history3.PipelineStrategyHistoryService), new(*history3.PipelineStrategyHistoryServiceImpl)),
688-
689691
//history ends
692+
693+
//plugin starts
694+
repository6.NewGlobalPluginRepository,
695+
wire.Bind(new(repository6.GlobalPluginRepository), new(*repository6.GlobalPluginRepositoryImpl)),
696+
697+
plugin.NewGlobalPluginService,
698+
wire.Bind(new(plugin.GlobalPluginService), new(*plugin.GlobalPluginServiceImpl)),
699+
700+
restHandler.NewGlobalPluginRestHandler,
701+
wire.Bind(new(restHandler.GlobalPluginRestHandler), new(*restHandler.GlobalPluginRestHandlerImpl)),
702+
703+
router.NewGlobalPluginRouter,
704+
wire.Bind(new(router.GlobalPluginRouter), new(*router.GlobalPluginRouterImpl)),
705+
706+
repository5.NewPipelineStageRepository,
707+
wire.Bind(new(repository5.PipelineStageRepository), new(*repository5.PipelineStageRepositoryImpl)),
708+
709+
pipeline.NewPipelineStageService,
710+
wire.Bind(new(pipeline.PipelineStageService), new(*pipeline.PipelineStageServiceImpl)),
711+
//plugin ends
712+
690713
// AuthWireSet,
691714
)
692715
return &App{}, nil

‎api/restHandler/GlobalPluginRestHandler.go‎

Lines changed: 139 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,139 @@
1+
package restHandler
2+
3+
import (
4+
"fmt"
5+
"github.com/devtron-labs/devtron/api/restHandler/common"
6+
"github.com/devtron-labs/devtron/pkg/pipeline"
7+
"github.com/devtron-labs/devtron/pkg/plugin"
8+
"github.com/devtron-labs/devtron/pkg/user/casbin"
9+
"github.com/devtron-labs/devtron/util/rbac"
10+
"github.com/gorilla/mux"
11+
"go.uber.org/zap"
12+
"net/http"
13+
"strconv"
14+
)
15+
16+
type GlobalPluginRestHandler interface {
17+
GetAllGlobalVariables(w http.ResponseWriter, r *http.Request)
18+
ListAllPlugins(w http.ResponseWriter, r *http.Request)
19+
GetPluginDetailById(w http.ResponseWriter, r *http.Request)
20+
}
21+
22+
func NewGlobalPluginRestHandler(logger *zap.SugaredLogger, globalPluginService plugin.GlobalPluginService,
23+
enforcerUtil rbac.EnforcerUtil, enforcer casbin.Enforcer, pipelineBuilder pipeline.PipelineBuilder) *GlobalPluginRestHandlerImpl {
24+
return &GlobalPluginRestHandlerImpl{
25+
logger: logger,
26+
globalPluginService: globalPluginService,
27+
enforcerUtil: enforcerUtil,
28+
enforcer: enforcer,
29+
pipelineBuilder: pipelineBuilder,
30+
}
31+
}
32+
33+
type GlobalPluginRestHandlerImpl struct {
34+
logger *zap.SugaredLogger
35+
globalPluginService plugin.GlobalPluginService
36+
enforcerUtil rbac.EnforcerUtil
37+
enforcer casbin.Enforcer
38+
pipelineBuilder pipeline.PipelineBuilder
39+
}
40+
41+
func (handler *GlobalPluginRestHandlerImpl) GetAllGlobalVariables(w http.ResponseWriter, r *http.Request) {
42+
token := r.Header.Get("token")
43+
appIdQueryParam := r.URL.Query().Get("appId")
44+
appId, err := strconv.Atoi(appIdQueryParam)
45+
if appIdQueryParam == "" || err != nil {
46+
common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest)
47+
return
48+
}
49+
app, err := handler.pipelineBuilder.GetApp(appId)
50+
if err != nil {
51+
handler.logger.Infow("service error, GetAllGlobalVariables", "err", err, "appId", appId)
52+
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
53+
return
54+
}
55+
//using appId for rbac in plugin(global resource), because this data must be visible to person having create permission
56+
//on atleast one app & we can't check this without iterating through every app
57+
//TODO: update plugin as a resource in casbin and make rbac independent of appId
58+
resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName)
59+
if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok {
60+
common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
61+
return
62+
}
63+
globalVariables, err := handler.globalPluginService.GetAllGlobalVariables()
64+
if err != nil {
65+
handler.logger.Errorw("error in getting global variable list", "err", err)
66+
common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
67+
return
68+
}
69+
common.WriteJsonResp(w, nil, globalVariables, http.StatusOK)
70+
}
71+
72+
func (handler *GlobalPluginRestHandlerImpl) ListAllPlugins(w http.ResponseWriter, r *http.Request) {
73+
token := r.Header.Get("token")
74+
appIdQueryParam := r.URL.Query().Get("appId")
75+
appId, err := strconv.Atoi(appIdQueryParam)
76+
if appIdQueryParam == "" || err != nil {
77+
common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest)
78+
return
79+
}
80+
app, err := handler.pipelineBuilder.GetApp(appId)
81+
if err != nil {
82+
handler.logger.Infow("service error, ListAllPlugins", "err", err, "appId", appId)
83+
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
84+
return
85+
}
86+
//using appId for rbac in plugin(global resource), because this data must be visible to person having create permission
87+
//on atleast one app & we can't check this without iterating through every app
88+
//TODO: update plugin as a resource in casbin and make rbac independent of appId
89+
resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName)
90+
if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok {
91+
common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
92+
return
93+
}
94+
plugins, err := handler.globalPluginService.ListAllPlugins()
95+
if err != nil {
96+
handler.logger.Errorw("error in getting plugin list", "err", err)
97+
common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
98+
return
99+
}
100+
common.WriteJsonResp(w, err, plugins, http.StatusOK)
101+
}
102+
103+
func (handler *GlobalPluginRestHandlerImpl) GetPluginDetailById(w http.ResponseWriter, r *http.Request) {
104+
token := r.Header.Get("token")
105+
appIdQueryParam := r.URL.Query().Get("appId")
106+
appId, err := strconv.Atoi(appIdQueryParam)
107+
if appIdQueryParam == "" || err != nil {
108+
common.WriteJsonResp(w, err, "invalid appId", http.StatusBadRequest)
109+
return
110+
}
111+
app, err := handler.pipelineBuilder.GetApp(appId)
112+
if err != nil {
113+
handler.logger.Infow("service error, GetPluginDetailById", "err", err, "appId", appId)
114+
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
115+
return
116+
}
117+
//using appId for rbac in plugin(global resource), because this data must be visible to person having create permission
118+
//on atleast one app & we can't check this without iterating through every app
119+
//TODO: update plugin as a resource in casbin and make rbac independent of appId
120+
resourceName := handler.enforcerUtil.GetAppRBACName(app.AppName)
121+
if ok := handler.enforcer.Enforce(token, casbin.ResourceApplications, casbin.ActionCreate, resourceName); !ok {
122+
common.WriteJsonResp(w, fmt.Errorf("unauthorized user"), "Unauthorized User", http.StatusForbidden)
123+
return
124+
}
125+
vars := mux.Vars(r)
126+
pluginId, err := strconv.Atoi(vars["pluginId"])
127+
if err != nil {
128+
handler.logger.Errorw("received invalid pluginId, GetPluginDetailById", "err", err, "pluginId", pluginId)
129+
common.WriteJsonResp(w, err, nil, http.StatusBadRequest)
130+
return
131+
}
132+
pluginDetail, err := handler.globalPluginService.GetPluginDetailById(pluginId)
133+
if err != nil {
134+
handler.logger.Errorw("error in getting plugin detail by id", "err", err, "pluginId", pluginId)
135+
common.WriteJsonResp(w, err, nil, http.StatusInternalServerError)
136+
return
137+
}
138+
common.WriteJsonResp(w, err, pluginDetail, http.StatusOK)
139+
}

‎api/router/GlobalPluginRouter.go‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
package router
2+
3+
import (
4+
"github.com/devtron-labs/devtron/api/restHandler"
5+
"github.com/gorilla/mux"
6+
"go.uber.org/zap"
7+
)
8+
9+
type GlobalPluginRouter interface {
10+
initGlobalPluginRouter(globalPluginRouter *mux.Router)
11+
}
12+
13+
func NewGlobalPluginRouter(logger *zap.SugaredLogger, globalPluginRestHandler restHandler.GlobalPluginRestHandler) *GlobalPluginRouterImpl {
14+
return &GlobalPluginRouterImpl{
15+
logger: logger,
16+
globalPluginRestHandler: globalPluginRestHandler,
17+
}
18+
}
19+
20+
type GlobalPluginRouterImpl struct {
21+
logger *zap.SugaredLogger
22+
globalPluginRestHandler restHandler.GlobalPluginRestHandler
23+
}
24+
25+
func (impl *GlobalPluginRouterImpl) initGlobalPluginRouter(globalPluginRouter *mux.Router) {
26+
globalPluginRouter.Path("/global/list/global-variable").
27+
HandlerFunc(impl.globalPluginRestHandler.GetAllGlobalVariables).Methods("GET")
28+
29+
globalPluginRouter.Path("/global/list").
30+
HandlerFunc(impl.globalPluginRestHandler.ListAllPlugins).Methods("GET")
31+
32+
globalPluginRouter.Path("/global/{pluginId}").
33+
HandlerFunc(impl.globalPluginRestHandler.GetPluginDetailById).Methods("GET")
34+
}

‎api/router/router.go‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@ type MuxRouter struct {
101101
deploymentConfigRouter deployment.DeploymentConfigRouter
102102
dashboardTelemetryRouter dashboardEvent.DashboardTelemetryRouter
103103
commonDeploymentRouter appStoreDeployment.CommonDeploymentRouter
104+
globalPluginRouter GlobalPluginRouter
104105
externalLinkRouter externalLink.ExternalLinkRouter
105106
selfRegistrationRolesRouter user.SelfRegistrationRolesRouter
106107
moduleRouter module.ModuleRouter
@@ -127,7 +128,8 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter HelmRouter, PipelineConf
127128
commonRouter CommonRouter, grafanaRouter GrafanaRouter, ssoLoginRouter sso.SsoLoginRouter, telemetryRouter TelemetryRouter, telemetryWatcher telemetry.TelemetryEventClient, bulkUpdateRouter BulkUpdateRouter, webhookListenerRouter WebhookListenerRouter, appLabelsRouter AppLabelRouter,
128129
coreAppRouter CoreAppRouter, helmAppRouter client.HelmAppRouter, k8sApplicationRouter k8s.K8sApplicationRouter,
129130
pProfRouter PProfRouter, deploymentConfigRouter deployment.DeploymentConfigRouter, dashboardTelemetryRouter dashboardEvent.DashboardTelemetryRouter,
130-
commonDeploymentRouter appStoreDeployment.CommonDeploymentRouter, externalLinkRouter externalLink.ExternalLinkRouter, moduleRouter module.ModuleRouter, selfRegistrationRolesRouter user.SelfRegistrationRolesRouter,
131+
commonDeploymentRouter appStoreDeployment.CommonDeploymentRouter, externalLinkRouter externalLink.ExternalLinkRouter,
132+
globalPluginRouter GlobalPluginRouter, selfRegistrationRolesRouter user.SelfRegistrationRolesRouter,moduleRouter module.ModuleRouter,
131133
serverRouter server.ServerRouter) *MuxRouter {
132134
r := &MuxRouter{
133135
Router: mux.NewRouter(),
@@ -185,6 +187,7 @@ func NewMuxRouter(logger *zap.SugaredLogger, HelmRouter HelmRouter, PipelineConf
185187
dashboardTelemetryRouter: dashboardTelemetryRouter,
186188
commonDeploymentRouter: commonDeploymentRouter,
187189
externalLinkRouter: externalLinkRouter,
190+
globalPluginRouter: globalPluginRouter,
188191
selfRegistrationRolesRouter: selfRegistrationRolesRouter,
189192
moduleRouter: moduleRouter,
190193
serverRouter: serverRouter,
@@ -342,6 +345,9 @@ func (r MuxRouter) Init() {
342345
pProfListenerRouter := r.Router.PathPrefix("/orchestrator/debug/pprof").Subrouter()
343346
r.pProfRouter.initPProfRouter(pProfListenerRouter)
344347

348+
globalPluginRouter := r.Router.PathPrefix("/orchestrator/plugin").Subrouter()
349+
r.globalPluginRouter.initGlobalPluginRouter(globalPluginRouter)
350+
345351
// deployment router starts
346352
deploymentConfigSubRouter := r.Router.PathPrefix("/orchestrator/deployment/template").Subrouter()
347353
r.deploymentConfigRouter.Init(deploymentConfigSubRouter)

‎assets/k6-plugin-icon.png‎

1.09 KB
Loading

‎assets/sonarqube-plugin-icon.png‎

965 Bytes
Loading

‎developers-guide/PluginCreationStep.md‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
## Steps to create a new Plugin -
2+
3+
1. **Enter the plugin's metadata in table `plugin_metadata`**
4+
5+
INSERT INTO
6+
"plugin_metadata" ("id", "name", "description","type","icon","deleted", "created_on", "created_by", "updated_on", "updated_by")
7+
VALUES
8+
(nextval('id_seq_plugin_metadata'), 'name_of_plugin','description_of_plugin','SHARED','link_to_icon','f', 'now()', 'user_id', 'now()', 'user_id');
9+
10+
2. **Add tags for plugin if not already present in the table `plugin_tag`(we will use id of this table for connecting with plugin, if a tag is already present note its id).**
11+
12+
To add a new tag -
13+
14+
INSERT INTO "plugin_tag" ("id", "name", "deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES
15+
(nextval('id_seq_plugin_tag'), 'name_of_tag','f', 'now()', 'user_id', 'now()', 'user_id');
16+
17+
3. **Update plugin & tag relation in the table `plugin_tag_relation`**
18+
19+
INSERT INTO "plugin_tag_relation" ("id", "tag_id", "plugin_id", "created_on", "created_by", "updated_on", "updated_by") VALUES
20+
(nextval('id_seq_plugin_tag_relation'), 'id_from_table-plugin_tag','id_from_table-plugin_metadata','now()', 'user_id', 'now()', 'user_id');
21+
22+
Note - there can be multiple tags associated with a plugin.
23+
24+
4. **Create scripts for plugin steps (for every custom type step in your plugin, you need to create script before adding that step)**
25+
26+
To create script-
27+
28+
INSERT INTO "plugin_pipeline_script" ("id", "script", "store_script_at","mount_code_to_container,"mount_code_to_container_path", "mount_directory_from_host","type","deleted","created_on", "created_by", "updated_on", "updated_by") VALUES
29+
(nextval('id_seq_plugin_pipeline_script'), 'script','mount_code_at-field-on-UI',"true/false","path_if_yes_in_previous_field","true/false",'SHELL/CONTAINER_IMAGE','f','now()', 'user_id', 'now()', 'user_id');
30+
31+
After script is created, if there are mappings (filePath mapping in mount_directory_from_host option, command args, port mapping) available in the script add them in table `script_path_arg_port_mapping` -
32+
33+
INSERT INTO "script_path_arg_port_mapping" ("id", "type_of_mapping", "file_path_on_disk","file_path_on_container,"command", "args","port_on_local","port_on_container","script_id","deleted","created_on", "created_by", "updated_on", "updated_by") VALUES
34+
(nextval('id_seq_script_path_arg_port_mapping'), 'FILE_PATH/DOCKER_ARG/PORT','file_path_mapping_entry',"file_path_mapping_entry","command","array_of_args",'port_on_local',''port_on_container','id-from-script-table','f','now()', 'user_id', 'now()', 'user_id');
35+
36+
Note - For multiple mappings (file or port) in a script, separate entry is needed for each mapping
37+
38+
5. **Create plugin steps**
39+
40+
To create a step -
41+
42+
INSERT INTO "plugin_step" ("id", "plugin_id","name","description","index","step_type","script_id","ref_plugin_id","deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES
43+
(nextval('id_seq_plugin_step'), 'id-from-plugin_metadata','name_of_step','description_of_step','index_of_step-start-from-1','INLINE/REF_PLUGIN','id_from_script_table','id-of-ref-plugin-get-from-plugin_metadata','f','now()', 'user_id', 'now()', 'user_id');
44+
45+
6. **Create variables present in steps**
46+
47+
To create variable -
48+
49+
INSERT INTO "plugin_step_variable" ("id", "plugin_step_id", "name", "format", "description", "is_exposed", "allow_empty_value", "variable_type", "value_type", "previous_step_index","default_value", "variable_step_index", "deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES
50+
(nextval('id_seq_plugin_step_variable'), 'id-from-plugin_step','name_of_variable','STRING/BOOL/DATE/NUMBER','description_of_variable','true/false','true/false','INPUT/OUTPUT','NEW/FROM_PREVIOUS_STEP/GLOBAL','index_of_step_of_ref_variable','default_value-nullable','index_of_step_variable_is_present_in','f','now()', 'user_id', 'now()', 'user_id');
51+
52+
7. **Create conditions on variables**
53+
54+
To create condition -
55+
56+
INSERT INTO "plugin_step_condition" ("id", "plugin_step_id","condition_variable_id","condition_type", "conditional_operator","conditional_value","deleted", "created_on", "created_by", "updated_on", "updated_by") VALUES
57+
(nextval('id_seq_plugin_step_condition'), 'id-from-plugin_step',"id-of-variable-on-which-condition-is-applied",'SKIP/TRIGGER/SUCCESS/FAILURE','conditional_operator','conditional_value','f','now()', 'user_id', 'now()', 'user_id');
58+

0 commit comments

Comments
 (0)