Skip to content

Commit bf48db4

Browse files
mohityadav766mohityadav766
committed
Initial Commit
1 parent 1b295fb commit bf48db4

File tree

28 files changed

+525
-46
lines changed

28 files changed

+525
-46
lines changed

‎bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v016__add_config_source_columns.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
-- Add columns for configuration source tracking
2+
ALTER TABLE openmetadata_settings
3+
ADD COLUMN env_hash VARCHAR(64) NULL,
4+
ADD COLUMN env_sync_timestamp TIMESTAMP(6) NULL,
5+
ADD COLUMN db_modified_timestamp TIMESTAMP(6) NULL;
6+
7+
-- Set initial timestamps for existing records
8+
UPDATE openmetadata_settings
9+
SET db_modified_timestamp = CURRENT_TIMESTAMP(6)
10+
WHERE db_modified_timestamp IS NULL;

‎bootstrap/sql/migrations/flyway/org.postgresql.Driver/v016__add_config_source_columns.sql‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
-- Add columns for configuration source tracking
2+
ALTER TABLE openmetadata_settings
3+
ADD COLUMN env_hash VARCHAR(64) NULL,
4+
ADD COLUMN env_sync_timestamp TIMESTAMP NULL,
5+
ADD COLUMN db_modified_timestamp TIMESTAMP NULL;
6+
7+
-- Set initial timestamps for existing records
8+
UPDATE openmetadata_settings
9+
SET db_modified_timestamp = CURRENT_TIMESTAMP
10+
WHERE db_modified_timestamp IS NULL;

‎conf/openmetadata.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -277,6 +277,7 @@ migrationConfiguration:
277277

278278
# Authorizer Configuration
279279
authorizerConfiguration:
280+
configSource: ${AUTHORIZER_CONFIG_SOURCE:-ENV}
280281
className: ${AUTHORIZER_CLASS_NAME:-org.openmetadata.service.security.DefaultAuthorizer}
281282
containerRequestFilter: ${AUTHORIZER_REQUEST_FILTER:-org.openmetadata.service.security.JwtFilter}
282283
adminPrincipals: ${AUTHORIZER_ADMIN_PRINCIPALS:-[admin]}
@@ -288,6 +289,7 @@ authorizerConfiguration:
288289
useRolesFromProvider: ${AUTHORIZER_USE_ROLES_FROM_PROVIDER:-false}
289290

290291
authenticationConfiguration:
292+
configSource: ${AUTHENTICATION_CONFIG_SOURCE:-ENV}
291293
clientType: ${AUTHENTICATION_CLIENT_TYPE:-public}
292294
provider: ${AUTHENTICATION_PROVIDER:-basic}
293295
# This is used by auth provider provide response as either id_token or code

‎conf/operations.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
email:
2+
configSource: ${EMAIL_CONFIG_SOURCE:-ENV}
23
emailingEntity: ${OM_EMAIL_ENTITY:-"OpenMetadata"}
34
supportUrl: ${OM_SUPPORT_URL:-"https://slack.open-metadata.org"}
45
enableSmtpServer : ${AUTHORIZER_ENABLE_SMTP:-false}
@@ -11,5 +12,6 @@ email:
1112
templates: ${TEMPLATES:-"openmetadata"}
1213

1314
serverUrl:
15+
configSource: ${SERVER_URL_CONFIG_SOURCE:-ENV}
1416
openMetadataUrl: ${OPENMETADATA_SERVER_URL:-"http://localhost:8585"}
1517

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626

2727
import java.sql.ResultSet;
2828
import java.sql.SQLException;
29+
import java.sql.Timestamp;
2930
import java.util.AbstractMap;
3031
import java.util.ArrayList;
3132
import java.util.Arrays;
@@ -7989,6 +7990,47 @@ interface SystemDAO {
79897990
@SqlUpdate(value = "DELETE from openmetadata_settings WHERE configType = :configType")
79907991
void delete(@Bind("configType") String configType);
79917992

7993+
@SqlUpdate(
7994+
"UPDATE openmetadata_settings SET json = :json, env_hash = :envHash, "
7995+
+ "env_sync_timestamp = :envSyncTimestamp, db_modified_timestamp = :dbModifiedTimestamp "
7996+
+ "WHERE configType = :configType")
7997+
void updateConfigWithMetadata(
7998+
@Bind("configType") String configType,
7999+
@Bind("json") String json,
8000+
@Bind("envHash") String envHash,
8001+
@Bind("envSyncTimestamp") Timestamp envSyncTimestamp,
8002+
@Bind("dbModifiedTimestamp") Timestamp dbModifiedTimestamp);
8003+
8004+
@SqlUpdate(
8005+
"UPDATE openmetadata_settings SET env_hash = :envHash, "
8006+
+ "env_sync_timestamp = :envSyncTimestamp, db_modified_timestamp = :dbModifiedTimestamp "
8007+
+ "WHERE configType = :configType")
8008+
void updateConfigMetadata(
8009+
@Bind("configType") String configType,
8010+
@Bind("envHash") String envHash,
8011+
@Bind("envSyncTimestamp") Timestamp envSyncTimestamp,
8012+
@Bind("dbModifiedTimestamp") Timestamp dbModifiedTimestamp);
8013+
8014+
@SqlUpdate(
8015+
"UPDATE openmetadata_settings SET env_hash = :envHash WHERE configType = :configType")
8016+
void updateEnvHash(@Bind("configType") String configType, @Bind("envHash") String envHash);
8017+
8018+
@SqlUpdate(
8019+
"UPDATE openmetadata_settings SET db_modified_timestamp = :dbModifiedTimestamp WHERE configType = :configType")
8020+
void updateDbModifiedTimestamp(
8021+
@Bind("configType") String configType,
8022+
@Bind("dbModifiedTimestamp") Timestamp dbModifiedTimestamp);
8023+
8024+
@SqlQuery("SELECT env_hash FROM openmetadata_settings WHERE configType = :configType")
8025+
String getEnvHash(@Bind("configType") String configType);
8026+
8027+
@SqlQuery("SELECT env_sync_timestamp FROM openmetadata_settings WHERE configType = :configType")
8028+
Timestamp getEnvSyncTimestamp(@Bind("configType") String configType);
8029+
8030+
@SqlQuery(
8031+
"SELECT db_modified_timestamp FROM openmetadata_settings WHERE configType = :configType")
8032+
Timestamp getDbModifiedTimestamp(@Bind("configType") String configType);
8033+
79928034
@SqlQuery("SELECT 42")
79938035
Integer testConnection() throws StatementException;
79948036

‎openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/SystemRepository.java‎

Lines changed: 45 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
import jakarta.json.JsonPatch;
1616
import jakarta.json.JsonValue;
1717
import jakarta.ws.rs.core.Response;
18+
import java.sql.Timestamp;
1819
import java.util.ArrayList;
1920
import java.util.HashSet;
2021
import java.util.List;
@@ -38,6 +39,7 @@
3839
import org.openmetadata.schema.configuration.AssetCertificationSettings;
3940
import org.openmetadata.schema.configuration.ExecutorConfiguration;
4041
import org.openmetadata.schema.configuration.HistoryCleanUpConfiguration;
42+
import org.openmetadata.schema.configuration.OpenMetadataConfig;
4143
import org.openmetadata.schema.configuration.SecurityConfiguration;
4244
import org.openmetadata.schema.configuration.WorkflowSettings;
4345
import org.openmetadata.schema.email.SmtpSettings;
@@ -54,6 +56,7 @@
5456
import org.openmetadata.schema.system.SecurityValidationResponse;
5557
import org.openmetadata.schema.system.StepValidation;
5658
import org.openmetadata.schema.system.ValidationResponse;
59+
import org.openmetadata.schema.type.ConfigSource;
5760
import org.openmetadata.schema.util.EntitiesCount;
5861
import org.openmetadata.schema.util.ServicesCount;
5962
import org.openmetadata.schema.utils.JsonUtils;
@@ -86,6 +89,7 @@
8689
import org.openmetadata.service.security.auth.validator.GoogleAuthValidator;
8790
import org.openmetadata.service.security.auth.validator.OktaAuthValidator;
8891
import org.openmetadata.service.security.auth.validator.SamlValidator;
92+
import org.openmetadata.service.util.ConfigSourceResolver;
8993
import org.openmetadata.service.util.EntityUtil;
9094
import org.openmetadata.service.util.LdapUtil;
9195
import org.openmetadata.service.util.OpenMetadataConnectionBuilder;
@@ -123,6 +127,43 @@ public SystemRepository() {
123127
migrationValidationClient = MigrationValidationClient.getInstance();
124128
}
125129

130+
public boolean isUpdateAllowed(Settings setting) {
131+
Object configValue = setting.getConfigValue();
132+
if (configValue instanceof OpenMetadataConfig openMetadataConfig) {
133+
ConfigSource source = openMetadataConfig.getConfigSource();
134+
return source != ConfigSource.ENV;
135+
}
136+
return true;
137+
}
138+
139+
public String getEnvHash(String configType) {
140+
return dao.getEnvHash(configType);
141+
}
142+
143+
public Timestamp getEnvSyncTimestamp(String configType) {
144+
return dao.getEnvSyncTimestamp(configType);
145+
}
146+
147+
public Timestamp getDbModifiedTimestamp(String configType) {
148+
return dao.getDbModifiedTimestamp(configType);
149+
}
150+
151+
public void updateConfigMetadata(
152+
String configType,
153+
String envHash,
154+
Timestamp envSyncTimestamp,
155+
Timestamp dbModifiedTimestamp) {
156+
dao.updateConfigMetadata(configType, envHash, envSyncTimestamp, dbModifiedTimestamp);
157+
}
158+
159+
public void updateEnvHash(String configType, String envHash) {
160+
dao.updateEnvHash(configType, envHash);
161+
}
162+
163+
public void updateDbModifiedTimestamp(String configType, Timestamp dbModifiedTimestamp) {
164+
dao.updateDbModifiedTimestamp(configType, dbModifiedTimestamp);
165+
}
166+
126167
public EntitiesCount getAllEntitiesCount(ListFilter filter) {
127168
return dao.getAggregatedEntitiesCount(filter.getCondition());
128169
}
@@ -373,9 +414,10 @@ public void updateSetting(Settings setting) {
373414
JsonUtils.validateJsonSchema(authorizerConfig, AuthorizerConfiguration.class);
374415
setting.setConfigValue(authorizerConfig);
375416
}
376-
dao.insertSettings(
377-
setting.getConfigType().toString(), JsonUtils.pojoToJson(setting.getConfigValue()));
378-
// Invalidate Cache
417+
String configType = setting.getConfigType().toString();
418+
dao.insertSettings(configType, JsonUtils.pojoToJson(setting.getConfigValue()));
419+
Timestamp now = ConfigSourceResolver.now();
420+
dao.updateDbModifiedTimestamp(configType, now);
379421
SettingsCache.invalidateSettings(setting.getConfigType().value());
380422
postUpdate(setting.getConfigType());
381423
} catch (Exception ex) {

‎openmetadata-service/src/main/java/org/openmetadata/service/resources/settings/SettingsCache.java‎

Lines changed: 76 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import com.google.common.cache.CacheLoader;
3333
import com.google.common.cache.LoadingCache;
3434
import java.io.IOException;
35+
import java.sql.Timestamp;
3536
import java.util.Collections;
3637
import java.util.HashMap;
3738
import java.util.List;
@@ -62,6 +63,7 @@
6263
import org.openmetadata.schema.security.scim.ScimConfiguration;
6364
import org.openmetadata.schema.settings.Settings;
6465
import org.openmetadata.schema.settings.SettingsType;
66+
import org.openmetadata.schema.type.ConfigSource;
6567
import org.openmetadata.schema.utils.JsonUtils;
6668
import org.openmetadata.search.IndexMapping;
6769
import org.openmetadata.service.Entity;
@@ -70,11 +72,13 @@
7072
import org.openmetadata.service.jdbi3.EntityRepository;
7173
import org.openmetadata.service.search.SearchRepository;
7274
import org.openmetadata.service.search.indexes.SearchIndex;
75+
import org.openmetadata.service.util.ConfigSourceResolver;
7376
import org.openmetadata.service.util.EntityUtil;
7477

7578
@Slf4j
7679
public class SettingsCache {
7780
private static volatile boolean initialized = false;
81+
private static Timestamp applicationStartTime;
7882
protected static final LoadingCache<String, Settings> CACHE =
7983
CacheBuilder.newBuilder()
8084
.maximumSize(1000)
@@ -85,26 +89,21 @@ private SettingsCache() {
8589
// Private constructor for singleton
8690
}
8791

88-
// Expected to be called only once from the DefaultAuthorizer
8992
public static void initialize(OpenMetadataApplicationConfig config) {
9093
if (!initialized) {
9194
initialized = true;
95+
applicationStartTime = ConfigSourceResolver.now();
9296
createDefaultConfiguration(config);
9397
}
9498
}
9599

96100
private static void createDefaultConfiguration(OpenMetadataApplicationConfig applicationConfig) {
97-
// Initialise Email Setting
98-
Settings storedSettings =
99-
Entity.getSystemRepository().getConfigWithKey(EMAIL_CONFIGURATION.toString());
100-
if (storedSettings == null) {
101-
// Only in case a config doesn't exist in DB we insert it
102-
SmtpSettings emailConfig =
103-
applicationConfig.getOperationalApplicationConfigProvider().getEmailSettings();
104-
105-
Settings setting =
106-
new Settings().withConfigType(EMAIL_CONFIGURATION).withConfigValue(emailConfig);
107-
Entity.getSystemRepository().createNewSetting(setting);
101+
SmtpSettings emailConfig =
102+
applicationConfig.getOperationalApplicationConfigProvider().getEmailSettings();
103+
if (emailConfig != null) {
104+
ConfigSource configSource =
105+
emailConfig.getConfigSource() != null ? emailConfig.getConfigSource() : ConfigSource.ENV;
106+
syncConfigWithSource(EMAIL_CONFIGURATION, emailConfig, configSource);
108107
}
109108

110109
// Initialise OM base url setting
@@ -243,41 +242,27 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app
243242
Entity.getSystemRepository().createNewSetting(setting);
244243
}
245244

246-
// Initialize Authentication Configuration
247-
Settings storedAuthConfig =
248-
Entity.getSystemRepository().getConfigWithKey(AUTHENTICATION_CONFIGURATION.toString());
249-
if (storedAuthConfig == null) {
250-
AuthenticationConfiguration authConfig = applicationConfig.getAuthenticationConfiguration();
251-
if (authConfig != null) {
252-
Settings setting =
253-
new Settings().withConfigType(AUTHENTICATION_CONFIGURATION).withConfigValue(authConfig);
254-
255-
Entity.getSystemRepository().createNewSetting(setting);
256-
}
245+
AuthenticationConfiguration authConfig = applicationConfig.getAuthenticationConfiguration();
246+
if (authConfig != null) {
247+
ConfigSource configSource =
248+
authConfig.getConfigSource() != null ? authConfig.getConfigSource() : ConfigSource.ENV;
249+
syncConfigWithSource(AUTHENTICATION_CONFIGURATION, authConfig, configSource);
257250
}
258251

259-
// Initialize Authorizer Configuration
260-
Settings storedAuthzConfig =
261-
Entity.getSystemRepository().getConfigWithKey(AUTHORIZER_CONFIGURATION.toString());
262-
if (storedAuthzConfig == null) {
263-
AuthorizerConfiguration authzConfig = applicationConfig.getAuthorizerConfiguration();
264-
if (authzConfig != null) {
265-
Settings setting =
266-
new Settings().withConfigType(AUTHORIZER_CONFIGURATION).withConfigValue(authzConfig);
267-
268-
Entity.getSystemRepository().createNewSetting(setting);
269-
}
252+
AuthorizerConfiguration authzConfig = applicationConfig.getAuthorizerConfiguration();
253+
if (authzConfig != null) {
254+
ConfigSource configSource =
255+
authzConfig.getConfigSource() != null ? authzConfig.getConfigSource() : ConfigSource.ENV;
256+
syncConfigWithSource(AUTHORIZER_CONFIGURATION, authzConfig, configSource);
270257
}
271258

272-
Settings storedScimConfig =
273-
Entity.getSystemRepository().getConfigWithKey(SCIM_CONFIGURATION.toString());
274-
if (storedScimConfig == null) {
275-
ScimConfiguration scimConfiguration = applicationConfig.getScimConfiguration();
276-
if (scimConfiguration != null) {
277-
Settings setting =
278-
new Settings().withConfigType(SCIM_CONFIGURATION).withConfigValue(scimConfiguration);
279-
Entity.getSystemRepository().createNewSetting(setting);
280-
}
259+
ScimConfiguration scimConfiguration = applicationConfig.getScimConfiguration();
260+
if (scimConfiguration != null) {
261+
ConfigSource configSource =
262+
scimConfiguration.getConfigSource() != null
263+
? scimConfiguration.getConfigSource()
264+
: ConfigSource.ENV;
265+
syncConfigWithSource(SCIM_CONFIGURATION, scimConfiguration, configSource);
281266
}
282267

283268
Settings entityRulesSettings =
@@ -318,6 +303,54 @@ private static void createDefaultConfiguration(OpenMetadataApplicationConfig app
318303
}
319304
}
320305

306+
private static void syncConfigWithSource(
307+
SettingsType settingsType, Object envConfigValue, ConfigSource configSource) {
308+
if (envConfigValue == null) {
309+
return;
310+
}
311+
312+
String currentEnvHash = ConfigSourceResolver.computeHash(envConfigValue);
313+
String storedEnvHash = Entity.getSystemRepository().getEnvHash(settingsType.toString());
314+
Timestamp dbModifiedTimestamp =
315+
Entity.getSystemRepository().getDbModifiedTimestamp(settingsType.toString());
316+
317+
Settings storedSettings =
318+
Entity.getSystemRepository().getConfigWithKey(settingsType.toString());
319+
320+
if (storedSettings == null) {
321+
Settings setting =
322+
new Settings().withConfigType(settingsType).withConfigValue(envConfigValue);
323+
Entity.getSystemRepository().createNewSetting(setting);
324+
Entity.getSystemRepository()
325+
.updateConfigMetadata(
326+
settingsType.toString(), currentEnvHash, applicationStartTime, applicationStartTime);
327+
return;
328+
}
329+
330+
boolean shouldUseEnv =
331+
ConfigSourceResolver.shouldUseEnvValue(
332+
configSource,
333+
currentEnvHash,
334+
storedEnvHash,
335+
null,
336+
dbModifiedTimestamp,
337+
applicationStartTime);
338+
339+
if (shouldUseEnv) {
340+
LOG.info("Config source resolution: using ENV value for {}", settingsType);
341+
Settings setting =
342+
new Settings().withConfigType(settingsType).withConfigValue(envConfigValue);
343+
Entity.getSystemRepository().updateSetting(setting);
344+
Entity.getSystemRepository()
345+
.updateConfigMetadata(
346+
settingsType.toString(), currentEnvHash, applicationStartTime, applicationStartTime);
347+
} else {
348+
if (storedEnvHash == null || !currentEnvHash.equals(storedEnvHash)) {
349+
Entity.getSystemRepository().updateEnvHash(settingsType.toString(), currentEnvHash);
350+
}
351+
}
352+
}
353+
321354
public static <T> T getSetting(SettingsType settingName, Class<T> clazz) {
322355
try {
323356
String json = JsonUtils.pojoToJson(CACHE.get(settingName.toString()).getConfigValue());

‎openmetadata-service/src/main/java/org/openmetadata/service/resources/system/SystemResource.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -360,6 +360,11 @@ public Response createOrUpdateSetting(
360360
}
361361

362362
authorizer.authorizeAdmin(securityContext);
363+
if (!systemRepository.isUpdateAllowed(settingName)) {
364+
throw new IllegalArgumentException(
365+
"This setting is managed by ENV configuration and cannot be updated via API. "
366+
+ "Change configSource to DB or AUTO to enable API updates.");
367+
}
363368
if (SettingsType.SEARCH_SETTINGS
364369
.value()
365370
.equalsIgnoreCase(settingName.getConfigType().toString())) {

0 commit comments

Comments
 (0)