Skip to content

Commit d99996a

Browse files
payertTamas Payer
payert
and
Tamas Payer
authored
KNOX-3222: Integrate OpenSearch REST API in Knox (#462) (#1131)
Co-authored-by: Tamas Payer <[email protected]>
1 parent 1e230d0 commit d99996a

File tree

12 files changed

+384
-7
lines changed

12 files changed

+384
-7
lines changed

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/ClouderaManagerServiceDiscovery.java‎

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.knox.gateway.topology.discovery.ClusterConfigurationMonitor;
4040
import org.apache.knox.gateway.topology.discovery.ServiceDiscovery;
4141
import org.apache.knox.gateway.topology.discovery.ServiceDiscoveryConfig;
42+
import org.apache.knox.gateway.topology.discovery.cm.model.opensearch.OpenSearchApiMasterServiceModelGenerator;
4243
import org.apache.knox.gateway.topology.discovery.cm.monitor.ClouderaManagerClusterConfigurationMonitor;
4344

4445
import java.net.SocketException;
@@ -317,7 +318,7 @@ private Set<ServiceModel> discoverService(DiscoveryApiClient client, String clus
317318
ServiceRoleDetails serviceRoleDetails = new ServiceRoleDetails(service, serviceConfig, role, roleConfigs);
318319
log.discoveringServiceRole(role.getName(), role.getType());
319320

320-
Set<ServiceModel> modelsForRole = generateServiceModels(client, serviceRoleDetails, coreSettingsConfig, modelGenerators);
321+
Set<ServiceModel> modelsForRole = generateServiceModels(client, serviceRoleDetails, coreSettingsConfig, modelGenerators, roleConfigList);
321322

322323
log.discoveredServiceRole(role.getName(), role.getType());
323324

@@ -331,11 +332,17 @@ private Set<ServiceModel> discoverService(DiscoveryApiClient client, String clus
331332
return serviceModels;
332333
}
333334

334-
private Set<ServiceModel> generateServiceModels(DiscoveryApiClient client, ServiceRoleDetails serviceRoleDetails, ApiServiceConfig coreSettingsConfig, List<ServiceModelGenerator> modelGenerators) throws ApiException {
335+
private Set<ServiceModel> generateServiceModels(DiscoveryApiClient client, ServiceRoleDetails serviceRoleDetails,
336+
ApiServiceConfig coreSettingsConfig, List<ServiceModelGenerator> modelGenerators,
337+
ApiRoleConfigList roleConfigList) throws ApiException {
335338
Set<ServiceModel> serviceModels = new HashSet<>();
336339

337340
if (modelGenerators != null) {
338341
for (ServiceModelGenerator serviceModelGenerator : modelGenerators) {
342+
if (OpenSearchApiMasterServiceModelGenerator.shouldSkipGeneratorWhenOpenSearchMaster(serviceModelGenerator, roleConfigList)) {
343+
continue;
344+
}
345+
339346
ServiceModel serviceModel = generateServiceModel(client, serviceRoleDetails, coreSettingsConfig, serviceModelGenerator);
340347
if (serviceModel != null) {
341348
serviceModels.add(serviceModel);
@@ -348,9 +355,9 @@ private Set<ServiceModel> generateServiceModels(DiscoveryApiClient client, Servi
348355

349356
private static ServiceModel generateServiceModel(DiscoveryApiClient client, ServiceRoleDetails sd,
350357
ApiServiceConfig coreSettingsConfig, ServiceModelGenerator serviceModelGenerator) throws ApiException {
358+
serviceModelGenerator.setApiClient(client);
351359
ServiceModelGeneratorHandleResponse response = serviceModelGenerator.handles(sd.getService(), sd.getServiceConfig(), sd.getRole(), sd.getRoleConfig());
352360
if (response.handled()) {
353-
serviceModelGenerator.setApiClient(client);
354361
return serviceModelGenerator.generateService(sd.getService(), sd.getServiceConfig(), sd.getRole(), sd.getRoleConfig(), coreSettingsConfig);
355362
} else if (!response.getConfigurationIssues().isEmpty()) {
356363
log.serviceRoleHasConfigurationIssues(sd.getRole().getName(), String.join(";", response.getConfigurationIssues()));

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/opensearch/AbstractOpenSearchApiServiceModelGenerator.java‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,31 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.knox.gateway.topology.discovery.cm.model.opensearch;
18+
19+
import org.apache.knox.gateway.topology.discovery.cm.ServiceModel;
20+
21+
public abstract class AbstractOpenSearchApiServiceModelGenerator extends AbstractOpenSearchServiceModelGenerator {
22+
@Override
23+
public ServiceModel.Type getModelType() {
24+
return ServiceModel.Type.API;
25+
}
26+
27+
@Override
28+
protected String getPortConfigName() {
29+
return "opensearch_http_port";
30+
}
31+
}

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/opensearch/AbstractOpenSearchServiceModelGenerator.java‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.knox.gateway.topology.discovery.cm.model.opensearch;
18+
19+
import com.cloudera.api.swagger.client.ApiException;
20+
import com.cloudera.api.swagger.model.ApiConfigList;
21+
import com.cloudera.api.swagger.model.ApiRole;
22+
import com.cloudera.api.swagger.model.ApiService;
23+
import com.cloudera.api.swagger.model.ApiServiceConfig;
24+
import org.apache.knox.gateway.topology.discovery.cm.ServiceModel;
25+
import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGenerator;
26+
27+
import java.util.Locale;
28+
29+
public abstract class AbstractOpenSearchServiceModelGenerator extends AbstractServiceModelGenerator {
30+
public static final String SERVICE = "OPENSEARCH";
31+
public static final String SERVICE_TYPE = "OPENSEARCH";
32+
33+
@Override
34+
public String getService() {
35+
return SERVICE;
36+
}
37+
38+
@Override
39+
public String getServiceType() {
40+
return SERVICE_TYPE;
41+
}
42+
43+
protected String getSSLEnabledConfigName() {
44+
return "ssl_enabled";
45+
}
46+
47+
protected abstract String getPortConfigName();
48+
49+
@Override
50+
public ServiceModel generateService(ApiService service, ApiServiceConfig serviceConfig, ApiRole role,
51+
ApiConfigList roleConfig, ApiServiceConfig coreSettingsConfig) throws ApiException {
52+
final String hostname = role.getHostRef().getHostname();
53+
final String port = getRoleConfigValue(roleConfig, getPortConfigName());
54+
final String sslEnabled = getRoleConfigValue(roleConfig, getSSLEnabledConfigName());
55+
56+
final String scheme = Boolean.parseBoolean(sslEnabled) ? "https" : "http";
57+
final String url = String.format(Locale.getDefault(), "%s://%s:%s", scheme, hostname, port);
58+
59+
final ServiceModel model = createServiceModel(url);
60+
model.addRoleProperty(getRoleType(), getPortConfigName(), port);
61+
model.addRoleProperty(getRoleType(), getSSLEnabledConfigName(), sslEnabled);
62+
63+
return model;
64+
}
65+
}

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/opensearch/OpenSearchApiCoordinatorServiceModelGenerator.java‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.knox.gateway.topology.discovery.cm.model.opensearch;
18+
19+
public class OpenSearchApiCoordinatorServiceModelGenerator extends AbstractOpenSearchApiServiceModelGenerator {
20+
static final String OPENSEARCH_COORDINATOR_ROLE_TYPE = "OPENSEARCH_COORDINATOR";
21+
22+
@Override
23+
public String getRoleType() {
24+
return OPENSEARCH_COORDINATOR_ROLE_TYPE;
25+
}
26+
}

‎gateway-discovery-cm/src/main/java/org/apache/knox/gateway/topology/discovery/cm/model/opensearch/OpenSearchApiMasterServiceModelGenerator.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.knox.gateway.topology.discovery.cm.model.opensearch;
18+
19+
import com.cloudera.api.swagger.model.ApiRoleConfigList;
20+
import org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator;
21+
22+
public class OpenSearchApiMasterServiceModelGenerator extends AbstractOpenSearchApiServiceModelGenerator {
23+
static final String OPENSEARCH_MASTER_ROLE_TYPE = "OPENSEARCH_MASTER";
24+
25+
@Override
26+
public String getRoleType() {
27+
return OPENSEARCH_MASTER_ROLE_TYPE;
28+
}
29+
30+
/**
31+
* If here is an OPENSEARCH_COORDINATOR role for this service o not generate service model for
32+
* OPENSEARCH_MASTER role because OPENSEARCH_COORDINATOR is the preferred REST API target.
33+
* @param serviceModelGenerator the actual model generator
34+
* @param roles all the roles of the service
35+
* @return <code>true</code> if <code>serviceModelGenerator</code> is for OpenSearch Master and there are Coordinator roles in the cluster.
36+
*/
37+
public static boolean shouldSkipGeneratorWhenOpenSearchMaster(ServiceModelGenerator serviceModelGenerator, ApiRoleConfigList roles) {
38+
if (!AbstractOpenSearchServiceModelGenerator.SERVICE_TYPE.equals(serviceModelGenerator.getServiceType())
39+
|| !OpenSearchApiMasterServiceModelGenerator.OPENSEARCH_MASTER_ROLE_TYPE.equals(serviceModelGenerator.getRoleType())) {
40+
return false;
41+
}
42+
return roles.getItems().stream().anyMatch(r -> r.getRoleType().equals(OpenSearchApiCoordinatorServiceModelGenerator.OPENSEARCH_COORDINATOR_ROLE_TYPE));
43+
}
44+
}

‎gateway-discovery-cm/src/main/resources/META-INF/services/org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -61,5 +61,5 @@ org.apache.knox.gateway.topology.discovery.cm.model.ozone.OzoneServiceModelGener
6161
org.apache.knox.gateway.topology.discovery.cm.model.ozone.ReconServiceModelGenerator
6262
org.apache.knox.gateway.topology.discovery.cm.model.ozone.SCMServiceModelGenerator
6363
org.apache.knox.gateway.topology.discovery.cm.model.ozone.OzoneHttpfsServiceModelGenerator
64-
65-
64+
org.apache.knox.gateway.topology.discovery.cm.model.opensearch.OpenSearchApiCoordinatorServiceModelGenerator
65+
org.apache.knox.gateway.topology.discovery.cm.model.opensearch.OpenSearchApiMasterServiceModelGenerator

‎gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/AbstractCMDiscoveryTest.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717
package org.apache.knox.gateway.topology.discovery.cm;
1818

19+
import com.cloudera.api.swagger.model.ApiClusterRef;
1920
import com.cloudera.api.swagger.model.ApiConfig;
2021
import com.cloudera.api.swagger.model.ApiConfigList;
2122
import com.cloudera.api.swagger.model.ApiHostRef;
@@ -30,17 +31,26 @@
3031

3132
public class AbstractCMDiscoveryTest {
3233
protected static ApiService createApiServiceMock(final String serviceType) {
33-
return createApiServiceMock(serviceType + "-1", serviceType);
34+
ApiClusterRef clusterRefMock = createApiClusterRefMock("Cluster 1");
35+
return createApiServiceMock(serviceType + "-1", serviceType, clusterRefMock);
3436
}
3537

36-
protected static ApiService createApiServiceMock(final String serviceName, final String serviceType) {
38+
protected static ApiService createApiServiceMock(final String serviceName, final String serviceType, final ApiClusterRef clusterRef) {
3739
ApiService service = EasyMock.createNiceMock(ApiService.class);
3840
EasyMock.expect(service.getName()).andReturn(serviceName).anyTimes();
3941
EasyMock.expect(service.getType()).andReturn(serviceType).anyTimes();
42+
EasyMock.expect(service.getClusterRef()).andReturn(clusterRef).anyTimes();
4043
EasyMock.replay(service);
4144
return service;
4245
}
4346

47+
protected static ApiClusterRef createApiClusterRefMock(final String clusterName) {
48+
ApiClusterRef clusterRef = EasyMock.createNiceMock(ApiClusterRef.class);
49+
EasyMock.expect(clusterRef.getClusterName()).andReturn(clusterName).anyTimes();
50+
EasyMock.replay(clusterRef);
51+
return clusterRef;
52+
}
53+
4454
protected static ApiServiceConfig createApiServiceConfigMock(Map<String, String> configProps) {
4555
ApiServiceConfig serviceConfig = EasyMock.createNiceMock(ApiServiceConfig.class);
4656
EasyMock.expect(serviceConfig.getItems()).andReturn(createMockApiConfigs(configProps)).anyTimes();

‎gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/opensearch/OpenSearchApiCoordinatorServiceModelGeneratorTest.java‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.knox.gateway.topology.discovery.cm.model.opensearch;
18+
19+
import org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator;
20+
import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGeneratorTest;
21+
import org.junit.Test;
22+
23+
import java.util.Collections;
24+
import java.util.HashMap;
25+
import java.util.Map;
26+
27+
public class OpenSearchApiCoordinatorServiceModelGeneratorTest extends AbstractServiceModelGeneratorTest {
28+
29+
@Test
30+
public void testServiceModelMetadata() {
31+
final Map<String, String> serviceConfig = Collections.emptyMap();
32+
final Map<String, String> roleConfig = new HashMap<>();
33+
roleConfig.put("ssl_enabled", "true");
34+
roleConfig.put("opensearch_http_port", "9202");
35+
36+
validateServiceModel(createServiceModel(serviceConfig, roleConfig), serviceConfig, roleConfig);
37+
}
38+
39+
@Override
40+
protected String getServiceType() {
41+
return OpenSearchApiCoordinatorServiceModelGenerator.SERVICE_TYPE;
42+
}
43+
44+
@Override
45+
protected String getRoleType() {
46+
return OpenSearchApiCoordinatorServiceModelGenerator.OPENSEARCH_COORDINATOR_ROLE_TYPE;
47+
}
48+
49+
@Override
50+
protected ServiceModelGenerator newGenerator() {
51+
return new OpenSearchApiCoordinatorServiceModelGenerator();
52+
}
53+
}

‎gateway-discovery-cm/src/test/java/org/apache/knox/gateway/topology/discovery/cm/model/opensearch/OpenSearchApiMasterServiceModelGeneratorTest.java‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with this
4+
* work for additional information regarding copyright ownership. The ASF
5+
* licenses this file to you under the Apache License, Version 2.0 (the
6+
* "License"); you may not use this file except in compliance with the License.
7+
* You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
13+
* WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
14+
* License for the specific language governing permissions and limitations under
15+
* the License.
16+
*/
17+
package org.apache.knox.gateway.topology.discovery.cm.model.opensearch;
18+
19+
import com.cloudera.api.swagger.model.ApiRoleConfig;
20+
import com.cloudera.api.swagger.model.ApiRoleConfigList;
21+
import org.apache.knox.gateway.topology.discovery.cm.ServiceModelGenerator;
22+
import org.apache.knox.gateway.topology.discovery.cm.model.AbstractServiceModelGeneratorTest;
23+
import org.easymock.EasyMock;
24+
import org.junit.Test;
25+
26+
import java.util.Arrays;
27+
import java.util.Collections;
28+
import java.util.HashMap;
29+
import java.util.Map;
30+
31+
import static org.junit.Assert.assertTrue;
32+
33+
public class OpenSearchApiMasterServiceModelGeneratorTest extends AbstractServiceModelGeneratorTest {
34+
35+
@Test
36+
public void testServiceModelMetadata() {
37+
final Map<String, String> serviceConfig = Collections.emptyMap();
38+
final Map<String, String> roleConfig = new HashMap<>();
39+
roleConfig.put("ssl_enabled", "true");
40+
roleConfig.put("opensearch_http_port", "9200");
41+
42+
validateServiceModel(createServiceModel(serviceConfig, roleConfig), serviceConfig, roleConfig);
43+
}
44+
45+
@Override
46+
protected String getServiceType() {
47+
return OpenSearchApiMasterServiceModelGenerator.SERVICE_TYPE;
48+
}
49+
50+
@Override
51+
protected String getRoleType() {
52+
return OpenSearchApiMasterServiceModelGenerator.OPENSEARCH_MASTER_ROLE_TYPE;
53+
}
54+
55+
@Override
56+
protected ServiceModelGenerator newGenerator() {
57+
return new OpenSearchApiMasterServiceModelGenerator();
58+
}
59+
60+
@Test
61+
public void testSkipGeneratorWhenOpenSearchMaster() {
62+
final ApiRoleConfig masterRole = EasyMock.createNiceMock(ApiRoleConfig.class);
63+
EasyMock.expect(masterRole.getRoleType()).andReturn(OpenSearchApiMasterServiceModelGenerator.OPENSEARCH_MASTER_ROLE_TYPE).anyTimes();
64+
EasyMock.replay(masterRole);
65+
66+
final ApiRoleConfig coordinatorRole = EasyMock.createNiceMock(ApiRoleConfig.class);
67+
EasyMock.expect(coordinatorRole.getRoleType()).andReturn(OpenSearchApiCoordinatorServiceModelGenerator.OPENSEARCH_COORDINATOR_ROLE_TYPE).anyTimes();
68+
EasyMock.replay(coordinatorRole);
69+
70+
final ApiRoleConfigList serviceRoleConfigs = EasyMock.createNiceMock(ApiRoleConfigList.class);
71+
EasyMock.expect(serviceRoleConfigs.getItems()).andReturn(Arrays.asList(masterRole, coordinatorRole)).anyTimes();
72+
EasyMock.replay(serviceRoleConfigs);
73+
74+
assertTrue(OpenSearchApiMasterServiceModelGenerator.shouldSkipGeneratorWhenOpenSearchMaster(newGenerator(), serviceRoleConfigs));
75+
}
76+
}

0 commit comments

Comments
 (0)