Merge branch 'main' into fix-app-deploy

Author: pmbrull

openmetadata-integration-tests/src/test/java/org/openmetadata/it/tests/TypeResourceIT.java

@@ -7,9 +7,15 @@
 import static org.junit.jupiter.api.Assertions.assertTrue;
 
 import com.fasterxml.jackson.databind.ObjectMapper;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Map;
 import java.util.UUID;
+import java.util.concurrent.CopyOnWriteArrayList;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Disabled;
 import org.junit.jupiter.api.Test;
@@ -542,6 +548,65 @@ void test_addMultipleHyperlinkCustomProperties(TestNamespace ns) throws Exceptio
     assertTrue(hasProp2, "Type should have second hyperlink custom property");
   }
 
+  @Test
+  void test_concurrentCustomPropertyAdditions(TestNamespace ns) throws Exception {
+    OpenMetadataClient client = SdkClients.adminClient();
+    Type pipelineType = getTypeByName(client, "pipeline");
+
+    int threadCount = 5;
+    List<CustomProperty> properties = new ArrayList<>();
+    for (int i = 0; i < threadCount; i++) {
+      CustomProperty prop = new CustomProperty();
+      prop.setName(ns.prefix("concurrentProp" + i));
+      prop.setDescription("Concurrent property " + i);
+      prop.setPropertyType(STRING_TYPE.getEntityReference());
+      properties.add(prop);
+    }
+
+    ExecutorService executor = Executors.newFixedThreadPool(threadCount);
+    CountDownLatch startLatch = new CountDownLatch(1);
+    CountDownLatch doneLatch = new CountDownLatch(threadCount);
+    List<Exception> errors = new CopyOnWriteArrayList<>();
+
+    for (CustomProperty prop : properties) {
+      executor.submit(
+          () -> {
+            try {
+              startLatch.await();
+              addCustomProperty(client, pipelineType.getId(), prop);
+            } catch (InterruptedException e) {
+              Thread.currentThread().interrupt();
+              errors.add(e);
+            } catch (Exception e) {
+              errors.add(e);
+            } finally {
+              doneLatch.countDown();
+            }
+          });
+    }
+
+    startLatch.countDown();
+    assertTrue(doneLatch.await(30, TimeUnit.SECONDS), "All threads should complete within 30s");
+    executor.shutdown();
+
+    assertTrue(errors.isEmpty(), "Concurrent requests should not throw: " + errors);
+
+    Type updatedType = getTypeByName(client, "pipeline", "customProperties");
+    List<String> persistedNames =
+        updatedType.getCustomProperties() != null
+            ? updatedType.getCustomProperties().stream().map(CustomProperty::getName).toList()
+            : List.of();
+
+    for (CustomProperty prop : properties) {
+      assertTrue(
+          persistedNames.contains(prop.getName()),
+          "Property '"
+              + prop.getName()
+              + "' was lost due to a concurrent update. Persisted: "
+              + persistedNames);
+    }
+  }
+
   private static Type createType(OpenMetadataClient client, CreateType createRequest)
       throws Exception {
     return client
@@ -565,6 +630,16 @@ private static Type getTypeByName(OpenMetadataClient client, String name) throws
     return OBJECT_MAPPER.readValue(response, Type.class);
   }
 
+  private static Type getTypeByName(OpenMetadataClient client, String name, String fields)
+      throws Exception {
+    String response =
+        client
+            .getHttpClient()
+            .executeForString(
+                HttpMethod.GET, "/v1/metadata/types/name/" + name + "?fields=" + fields, null);
+    return OBJECT_MAPPER.readValue(response, Type.class);
+  }
+
   private static TypeList listTypes(OpenMetadataClient client) throws Exception {
     String response =
         client.getHttpClient().executeForString(HttpMethod.GET, "/v1/metadata/types", null);

openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TypeRepository.java

@@ -33,6 +33,7 @@
 import java.util.Map;
 import java.util.Set;
 import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
 import java.util.stream.Collectors;
 import lombok.extern.slf4j.Slf4j;
 import org.apache.commons.lang3.tuple.Triple;
@@ -64,6 +65,8 @@
 public class TypeRepository extends EntityRepository<Type> {
   private static final String UPDATE_FIELDS = "customProperties";
   private static final String PATCH_FIELDS = "customProperties";
+  private static final ConcurrentHashMap<UUID, Object> TYPE_PROPERTY_LOCKS =
+      new ConcurrentHashMap<>();
 
   public TypeRepository() {
     super(
@@ -162,31 +165,34 @@ public void postUpdate(Type original, Type updated) {
 
   public PutResponse<Type> addCustomProperty(
       UriInfo uriInfo, String updatedBy, UUID id, CustomProperty property) {
-    Type type = find(id, Include.NON_DELETED);
-    property.setPropertyType(
-        Entity.getEntityReferenceById(
-            Entity.TYPE, property.getPropertyType().getId(), NON_DELETED));
-    validateProperty(property);
-    if (type.getCategory().equals(Category.Field)) {
-      throw new IllegalArgumentException(
-          "Only entity types can be extended and field types can't be extended");
-    }
-    setFieldsInternal(type, putFields);
+    Object lock = TYPE_PROPERTY_LOCKS.computeIfAbsent(id, k -> new Object());
+    synchronized (lock) {
+      Type type = find(id, Include.NON_DELETED);
+      property.setPropertyType(
+          Entity.getEntityReferenceById(
+              Entity.TYPE, property.getPropertyType().getId(), NON_DELETED));
+      validateProperty(property);
+      if (type.getCategory().equals(Category.Field)) {
+        throw new IllegalArgumentException(
+            "Only entity types can be extended and field types can't be extended");
+      }
+      setFieldsInternal(type, putFields);
 
-    find(property.getPropertyType().getId(), NON_DELETED); // Validate customProperty type exists
+      find(property.getPropertyType().getId(), NON_DELETED); // Validate customProperty type exists
 
-    // If property already exists, then update it. Else add the new property.
-    List<CustomProperty> updatedProperties = new ArrayList<>(List.of(property));
-    for (CustomProperty existing : type.getCustomProperties()) {
-      if (!existing.getName().equals(property.getName())) {
-        updatedProperties.add(existing);
+      // If property already exists, then update it. Else add the new property.
+      List<CustomProperty> updatedProperties = new ArrayList<>(List.of(property));
+      for (CustomProperty existing : type.getCustomProperties()) {
+        if (!existing.getName().equals(property.getName())) {
+          updatedProperties.add(existing);
+        }
       }
-    }
 
-    type.setCustomProperties(updatedProperties);
-    type.setUpdatedBy(updatedBy);
-    type.setUpdatedAt(System.currentTimeMillis());
-    return createOrUpdate(uriInfo, type, updatedBy);
+      type.setCustomProperties(updatedProperties);
+      type.setUpdatedBy(updatedBy);
+      type.setUpdatedAt(System.currentTimeMillis());
+      return createOrUpdate(uriInfo, type, updatedBy);
+    }
   }
 
   private List<CustomProperty> getCustomProperties(Type type) {

openmetadata-ui/src/main/resources/ui/package.json

@@ -94,7 +94,7 @@
     "cookie-storage": "^6.1.0",
     "cronstrue": "^2.53.0",
     "crypto-random-string-with-promisify-polyfill": "^5.0.0",
-    "diff": "^5.0.0",
+    "diff": "^5.2.2",
     "dompurify": "^3.2.4",
     "elkjs": "^0.9.3",
     "eventemitter3": "^5.0.1",
@@ -113,7 +113,7 @@
     "notistack": "^3.0.2",
     "oidc-client": "^1.11.5",
     "process": "^0.11.10",
-    "qs": "6.14.1",
+    "qs": "6.14.2",
     "quill-mention": "^6.0.1",
     "quilljs-markdown": "^1.2.0",
     "rapidoc": "^9.3.8",
@@ -264,6 +264,7 @@
     "tar-fs": "2.1.4",
     "js-yaml": "4.1.1",
     "lodash": ">=4.17.23",
-    "lodash-es": ">=4.17.23"
+    "lodash-es": ">=4.17.23",
+    "markdown-it": ">=14.1.1"
   }
 }

openmetadata-ui/src/main/resources/ui/public/locales/en-US/Drive/Sftp.md

@@ -0,0 +1,83 @@
+# SFTP
+
+In this section, we provide guides and references to use the SFTP connector.
+
+## Requirements
+
+To extract metadata from an SFTP server, the user needs to have read access to the directories and files to be catalogued.
+
+You can find further information on the SFTP connector in the <a href="https://docs.open-metadata.org/connectors/drive/sftp" target="_blank">docs</a>.
+
+## Connection Details
+
+$$section
+### Host $(id="host")
+SFTP server hostname or IP address (e.g., `sftp.example.com` or `192.168.1.100`).
+$$
+
+$$section
+### Port $(id="port")
+SFTP server port number. Defaults to `22`.
+$$
+
+$$section
+### Authentication Type $(id="authType")
+Authentication method to connect to the SFTP server. Choose between:
+- **Username/Password**: Authenticate using a username and password.
+- **Private Key**: Authenticate using an SSH private key in PEM format. Supports RSA, Ed25519, ECDSA, and DSS keys.
+$$
+
+$$section
+### Username $(id="username")
+SFTP username used for authentication.
+$$
+
+$$section
+### Password $(id="password")
+Password for username/password authentication.
+$$
+
+$$section
+### Private Key $(id="privateKey")
+SSH private key content in PEM format for key-based authentication. Supports RSA, Ed25519, ECDSA, and DSS keys.
+$$
+
+$$section
+### Private Key Passphrase $(id="privateKeyPassphrase")
+Passphrase to decrypt the private key, if the key is encrypted. Leave blank if the key has no passphrase.
+$$
+
+$$section
+### Root Directories $(id="rootDirectories")
+List of root directories to scan for files and subdirectories. Defaults to `/` (the user's home directory). Multiple directories can be specified to scope the ingestion to specific paths on the server.
+$$
+
+$$section
+### Connection Options $(id="connectionOptions")
+Additional connection options to build the URL that can be sent to the service during the connection.
+$$
+
+$$section
+### Connection Arguments $(id="connectionArguments")
+Additional connection arguments such as security or protocol configs that can be sent to the service during connection.
+$$
+
+$$section
+### Directory Filter Pattern $(id="directoryFilterPattern")
+Regex to only include/exclude directories that match the pattern.
+$$
+
+$$section
+### File Filter Pattern $(id="fileFilterPattern")
+Regex to only include/exclude files that match the pattern.
+$$
+
+$$section
+### Structured Data Files Only $(id="structuredDataFilesOnly")
+When enabled, only catalog structured data files (CSV, TSV) that can have schema extracted. Non-structured files like images, PDFs, and videos will be skipped. Defaults to `false`.
+$$
+
+$$section
+### Extract Sample Data $(id="extractSampleData")
+When enabled, extract sample data from structured files (CSV, TSV). Disabled by default to avoid performance overhead.
+$$

openmetadata-ui/src/main/resources/ui/src/constants/Services.constant.ts

@@ -461,6 +461,8 @@ export const BETA_SERVICES = [
   DatabaseServiceType.MicrosoftFabric,
   PipelineServiceType.MicrosoftFabricPipeline,
   DatabaseServiceType.BurstIQ,
+  DatabaseServiceType.StarRocks,
+  DriveServiceType.SFTP,
 ];
 
 export const TEST_CONNECTION_INITIAL_MESSAGE =