feat(plugins): add q dev blueprint

Author: warren830

backend/plugins/q_dev/api/blueprint_v200.go

@@ -0,0 +1,95 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package api
+
+import (
+	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
+	"github.com/apache/incubator-devlake/core/plugin"
+	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
+	"github.com/apache/incubator-devlake/helpers/srvhelper"
+	"github.com/apache/incubator-devlake/plugins/q_dev/models"
+	"github.com/apache/incubator-devlake/plugins/q_dev/tasks"
+)
+
+func MakeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	connectionId uint64,
+	bpScopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	// load connection and scope from the db
+	connection, err := dsHelper.ConnSrv.FindByPk(connectionId)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopeDetails, err := dsHelper.ScopeSrv.MapScopeDetails(connectionId, bpScopes)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	plan, err := makeDataSourcePipelinePlanV200(subtaskMetas, scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+	scopes, err := makeScopesV200(scopeDetails, connection)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	return plan, scopes, nil
+}
+
+func makeDataSourcePipelinePlanV200(
+	subtaskMetas []plugin.SubTaskMeta,
+	scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig],
+	connection *models.QDevConnection,
+) (coreModels.PipelinePlan, errors.Error) {
+	plan := make(coreModels.PipelinePlan, len(scopeDetails))
+	for i, scopeDetail := range scopeDetails {
+		s3Slice := scopeDetail.Scope
+		stage := plan[i]
+		if stage == nil {
+			stage = coreModels.PipelineStage{}
+		}
+
+		// construct task options for q_dev
+		op := &tasks.QDevOptions{
+			ConnectionId: s3Slice.ConnectionId,
+			S3Prefix:     s3Slice.Prefix,
+		}
+
+		// Pass empty entities array to enable all subtasks
+		task, err := helper.MakePipelinePlanTask("q_dev", subtaskMetas, []string{}, op)
+		if err != nil {
+			return nil, err
+		}
+		stage = append(stage, task)
+		plan[i] = stage
+	}
+	return plan, nil
+}
+
+func makeScopesV200(
+	scopeDetails []*srvhelper.ScopeDetail[models.QDevS3Slice, srvhelper.NoScopeConfig],
+	connection *models.QDevConnection,
+) ([]plugin.Scope, errors.Error) {
+	scopes := make([]plugin.Scope, 0)
+	// For Q Developer metrics, we don't need to create domain layer scopes
+	// The data is collected and stored directly in the tool layer
+	return scopes, nil
+}

backend/plugins/q_dev/impl/impl.go

@@ -23,6 +23,7 @@ import (
 	"github.com/apache/incubator-devlake/core/context"
 	"github.com/apache/incubator-devlake/core/dal"
 	"github.com/apache/incubator-devlake/core/errors"
+	coreModels "github.com/apache/incubator-devlake/core/models"
 	"github.com/apache/incubator-devlake/core/plugin"
 	helper "github.com/apache/incubator-devlake/helpers/pluginhelper/api"
 	"github.com/apache/incubator-devlake/plugins/q_dev/api"
@@ -39,6 +40,7 @@ var _ interface {
 	plugin.PluginModel
 	plugin.PluginSource
 	plugin.PluginMigration
+	plugin.DataSourcePluginBlueprintV200
 	plugin.CloseablePluginTask
 } = (*QDev)(nil)
 
@@ -170,3 +172,10 @@ func (p QDev) Close(taskCtx plugin.TaskContext) errors.Error {
 	data.S3Client.Close()
 	return nil
 }
+
+func (p QDev) MakeDataSourcePipelinePlanV200(
+	connectionId uint64,
+	scopes []*coreModels.BlueprintScope,
+) (coreModels.PipelinePlan, []plugin.Scope, errors.Error) {
+	return api.MakeDataSourcePipelinePlanV200(p.SubTaskMetas(), connectionId, scopes)
+}

backend/plugins/q_dev/models/migrationscripts/20251123_add_scope_config_id_to_s3_slice.go

@@ -0,0 +1,48 @@
+/*
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements.  See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License.  You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package migrationscripts
+
+import (
+	"github.com/apache/incubator-devlake/core/context"
+	"github.com/apache/incubator-devlake/core/errors"
+)
+
+type addScopeConfigIdToS3Slice struct{}
+
+func (*addScopeConfigIdToS3Slice) Up(basicRes context.BasicRes) errors.Error {
+	db := basicRes.GetDal()
+
+	// Add scope_config_id column to _tool_q_dev_s3_slices table
+	err := db.Exec(`
+		ALTER TABLE _tool_q_dev_s3_slices
+		ADD COLUMN scope_config_id BIGINT UNSIGNED DEFAULT 0
+	`)
+	if err != nil {
+		return errors.Convert(err)
+	}
+
+	return nil
+}
+
+func (*addScopeConfigIdToS3Slice) Version() uint64 {
+	return 20251123000001
+}
+
+func (*addScopeConfigIdToS3Slice) Name() string {
+	return "Add scope_config_id column to S3 slice table"
+}

backend/plugins/q_dev/models/migrationscripts/register.go

@@ -29,5 +29,6 @@ func All() []plugin.MigrationScript {
 		new(addDisplayNameFields),
 		new(addMissingMetrics),
 		new(addS3SliceTable),
+		new(addScopeConfigIdToS3Slice),
 	}
 }

backend/plugins/q_dev/tasks/identity_client.go

@@ -70,6 +70,11 @@ func NewQDevIdentityClient(connection *models.QDevConnection) (*QDevIdentityClie
 // ResolveUserDisplayName resolves a user ID to a human-readable display name
 // Returns the display name if found, otherwise returns the original userId as fallback
 func (client *QDevIdentityClient) ResolveUserDisplayName(userId string) (string, error) {
+	// Check if client or IdentityStore is nil
+	if client == nil || client.IdentityStore == nil {
+		return userId, nil
+	}
+
 	input := &identitystore.DescribeUserInput{
 		IdentityStoreId: aws.String(client.StoreId),
 		UserId:          aws.String(userId),

backend/plugins/q_dev/tasks/s3_data_extractor.go

@@ -318,4 +318,5 @@ var ExtractQDevS3DataMeta = plugin.SubTaskMeta{
 	EnabledByDefault: true,
 	Description:      "Extract data from S3 CSV files and save to database",
 	DomainTypes:      []string{plugin.DOMAIN_TYPE_CROSS},
+	Dependencies:     []*plugin.SubTaskMeta{&CollectQDevS3FilesMeta},
 }

backend/plugins/q_dev/tasks/s3_file_collector.go

@@ -114,4 +114,5 @@ var CollectQDevS3FilesMeta = plugin.SubTaskMeta{
 	EntryPoint:       CollectQDevS3Files,
 	EnabledByDefault: true,
 	Description:      "Collect S3 file metadata from AWS S3 bucket",
+	DomainTypes:      []string{plugin.DOMAIN_TYPE_CROSS},
 }

config-ui/src/plugins/register/q-dev/connection-fields/connection-test.tsx

@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+import { useState } from 'react';
+import { Button, Alert, Space } from 'antd';
+import { CheckCircleOutlined, ExclamationCircleOutlined, LoadingOutlined } from '@ant-design/icons';
+
+import API from '@/api';
+import { operator } from '@/utils';
+
+interface Props {
+  plugin: string;
+  connectionId?: ID;
+  values: any;
+  initialValues: any;
+  disabled?: boolean;
+}
+
+interface TestResult {
+  success: boolean;
+  message: string;
+  details?: {
+    s3Access?: boolean;
+    identityCenterAccess?: boolean;
+  };
+}
+
+export const QDevConnectionTest = ({ plugin, connectionId, values, initialValues, disabled }: Props) => {
+  const [testing, setTesting] = useState(false);
+  const [testResult, setTestResult] = useState<TestResult | null>(null);
+
+  const handleTest = async () => {
+    setTesting(true);
+    setTestResult(null);
+
+    try {
+      const [success, result] = await operator(
+        () => {
+          if (connectionId) {
+            // Test existing connection with only changed values
+            return API.connection.test(plugin, connectionId, {
+              authType: values.authType !== initialValues.authType ? values.authType : undefined,
+              accessKeyId: values.accessKeyId !== initialValues.accessKeyId ? values.accessKeyId : undefined,
+              secretAccessKey: values.secretAccessKey !== initialValues.secretAccessKey ? values.secretAccessKey : undefined,
+              region: values.region !== initialValues.region ? values.region : undefined,
+              bucket: values.bucket !== initialValues.bucket ? values.bucket : undefined,
+              identityStoreId: values.identityStoreId !== initialValues.identityStoreId ? values.identityStoreId : undefined,
+              identityStoreRegion: values.identityStoreRegion !== initialValues.identityStoreRegion ? values.identityStoreRegion : undefined,
+              rateLimitPerHour: values.rateLimitPerHour !== initialValues.rateLimitPerHour ? values.rateLimitPerHour : undefined,
+              proxy: values.proxy !== initialValues.proxy ? values.proxy : undefined,
+            });
+          } else {
+            // Test new connection with all values
+            return API.connection.testOld(plugin, {
+              name: values.name || '',
+              authType: values.authType || 'access_key',
+              accessKeyId: values.accessKeyId || '',
+              secretAccessKey: values.secretAccessKey || '',
+              region: values.region || '',
+              bucket: values.bucket || '',
+              identityStoreId: values.identityStoreId || '',
+              identityStoreRegion: values.identityStoreRegion || '',
+              rateLimitPerHour: values.rateLimitPerHour || 20000,
+              proxy: values.proxy || '',
+              endpoint: '', // Not used by Q Developer
+              authMethod: '', // Not used by Q Developer
+              username: '', // Not used by Q Developer
+              password: '', // Not used by Q Developer
+              token: '', // Not used by Q Developer
+              appId: '', // Not used by Q Developer
+              secretKey: '', // Not used by Q Developer
+              dbUrl: '', // Not used by Q Developer
+              organization: '', // Not used by Q Developer
+            });
+          }
+        },
+        {
+          setOperating: () => {}, // We handle loading state ourselves
+          hideToast: true, // We show our own success/error messages
+        },
+      );
+
+      if (success && result) {
+        setTestResult({
+          success: true,
+          message: 'Connection test successful! AWS credentials and S3 access verified.',
+          details: {
+            s3Access: true,
+            identityCenterAccess: values.identityStoreId ? true : undefined,
+          },
+        });
+      } else {
+        setTestResult({
+          success: false,
+          message: 'Connection test failed. Please check your configuration.',
+        });
+      }
+    } catch (error: any) {
+      let errorMessage = 'Connection test failed. Please check your configuration.';
+      
+      if (error?.response?.data?.message) {
+        errorMessage = error.response.data.message;
+      } else if (error?.message) {
+        errorMessage = error.message;
+      }
+
+      // Provide more specific error messages based on common issues
+      if (errorMessage.includes('InvalidAccessKeyId') || errorMessage.includes('SignatureDoesNotMatch')) {
+        errorMessage = 'Invalid AWS credentials. Please check your Access Key ID and Secret Access Key.';
+      } else if (errorMessage.includes('NoSuchBucket')) {
+        errorMessage = 'S3 bucket not found. Please check the bucket name and region.';
+      } else if (errorMessage.includes('AccessDenied')) {
+        errorMessage = 'Access denied. Please check your AWS permissions for S3 and IAM Identity Center.';
+      } else if (errorMessage.includes('InvalidBucketName')) {
+        errorMessage = 'Invalid S3 bucket name. Please check the bucket name format.';
+      } else if (errorMessage.includes('NoCredentialsError')) {
+        errorMessage = 'AWS credentials not found. Please provide valid Access Key ID and Secret Access Key, or ensure IAM role is properly configured.';
+      }
+
+      setTestResult({
+        success: false,
+        message: errorMessage,
+      });
+    } finally {
+      setTesting(false);
+    }
+  };
+
+  const getAlertType = () => {
+    if (!testResult) return undefined;
+    return testResult.success ? 'success' : 'error';
+  };
+
+  const getAlertIcon = () => {
+    if (testing) return <LoadingOutlined />;
+    if (!testResult) return undefined;
+    return testResult.success ? <CheckCircleOutlined /> : <ExclamationCircleOutlined />;
+  };
+
+  return (
+    <Space direction="vertical" style={{ width: '100%' }}>
+      <Button
+        type="default"
+        loading={testing}
+        disabled={disabled || testing}
+        onClick={handleTest}
+        style={{ marginTop: 16 }}
+      >
+        {testing ? 'Testing Connection...' : 'Test Connection'}
+      </Button>
+
+      {(testResult || testing) && (
+        <Alert
+          type={getAlertType()}
+          icon={getAlertIcon()}
+          message={testing ? 'Testing connection to AWS S3 and IAM Identity Center...' : testResult?.message}
+          description={
+            testResult?.success && testResult.details ? (
+              <div>
+                <div>✓ S3 Access: Verified</div>
+                {testResult.details.identityCenterAccess && (
+                  <div>✓ IAM Identity Center: Configured</div>
+                )}
+                {!values.identityStoreId && (
+                  <div style={{ marginTop: 8, color: '#faad14' }}>
+                    ⚠️ IAM Identity Center not configured - user display names will show as user IDs
+                  </div>
+                )}
+              </div>
+            ) : undefined
+          }
+          showIcon
+          style={{ marginTop: 8 }}
+        />
+      )}
+    </Space>
+  );
+};