Skip to content

Commit 37e2542

Browse files
nvazquezPearl1594
nvazquez
authored and
Pearl1594
committed
Enable Autoscaling on Netris for CPU and memory (#51)
* Enable Autoscaling on Netris for CPU and memory * Fix monitor autoscale group and cleanup * Rename autoscaling group method * Integrate Autoscaling by allowing to update LB rules * Refactor according to the SDK changes
1 parent e02ffed commit 37e2542

File tree

12 files changed

+84
-51
lines changed

12 files changed

+84
-51
lines changed

‎api/src/main/java/com/cloud/network/netris/NetrisService.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,6 @@ public interface NetrisService {
5555

5656
boolean releaseNatIp(long zoneId, String publicIp);
5757

58-
boolean createLbRule(NetrisNetworkRule rule);
58+
boolean createOrUpdateLbRule(NetrisNetworkRule rule);
5959
boolean deleteLbRule(NetrisNetworkRule rule);
6060
}

‎engine/schema/src/main/resources/META-INF/db/schema-41910to42000.sql‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -459,3 +459,7 @@ CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.static_routes', 'next_hop', 'varchar
459459

460460
-- Add `for_router` to `user_ip_address` table
461461
CALL `cloud`.`IDEMPOTENT_ADD_COLUMN`('cloud.user_ip_address', 'for_router', 'tinyint(1) DEFAULT 0 COMMENT "True if the ip address is used by Domain Router to expose services"');
462+
463+
-- Add Netris Autoscaling rules
464+
INSERT IGNORE INTO `cloud`.`counter` (uuid, provider, source, name, value, created) VALUES (UUID(), 'Netris', 'cpu', 'VM CPU - average percentage', 'vm.cpu.average.percentage', NOW());
465+
INSERT IGNORE INTO `cloud`.`counter` (uuid, provider, source, name, value, created) VALUES (UUID(), 'Netris', 'memory', 'VM Memory - average percentage', 'vm.memory.average.percentage', NOW());

‎plugins/network-elements/netris/src/main/java/org/apache/cloudstack/resource/NetrisResource.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -359,7 +359,7 @@ private Answer executeRequest(ReleaseNatIpCommand cmd) {
359359
}
360360

361361
private Answer executeRequest(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd) {
362-
boolean result = netrisApiClient.createLbRule(cmd);
362+
boolean result = netrisApiClient.createOrUpdateLbRule(cmd);
363363
if (!result) {
364364
return new NetrisAnswer(cmd, false, String.format("Failed to create Netris LB rule for %s: %s, " +
365365
"for private port: %s and public port: %s", getNetworkType(cmd.isVpc()), cmd.getName(), cmd.getPrivatePort(), cmd.getPublicPort(), cmd.getPublicPort()));

‎plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisApiClient.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,6 @@ public interface NetrisApiClient {
8686
boolean addOrUpdateStaticRoute(AddOrUpdateNetrisStaticRouteCommand cmd);
8787
boolean deleteStaticRoute(DeleteNetrisStaticRouteCommand cmd);
8888
boolean releaseNatIp(ReleaseNatIpCommand cmd);
89-
boolean createLbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd);
89+
boolean createOrUpdateLbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd);
9090
boolean deleteLbRule(DeleteNetrisLoadBalancerRuleCommand cmd);
9191
}

‎plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisApiClientImpl.java‎

Lines changed: 25 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -56,11 +56,13 @@
5656
import io.netris.model.IpTreeSubnet;
5757
import io.netris.model.IpTreeSubnetSites;
5858
import io.netris.model.L4LBSite;
59+
import io.netris.model.L4LbAddItem;
60+
import io.netris.model.L4LbEditItem;
5961
import io.netris.model.L4LbTenant;
6062
import io.netris.model.L4LbVpc;
6163
import io.netris.model.L4LoadBalancerBackendItem;
6264
import io.netris.model.L4LoadBalancerItem;
63-
import io.netris.model.L4lbAddItem;
65+
import io.netris.model.L4lbAddOrUpdateItem;
6466
import io.netris.model.L4lbresBody;
6567
import io.netris.model.NatBodySiteSite;
6668
import io.netris.model.NatBodyVpcVpc;
@@ -97,6 +99,7 @@
9799
import io.netris.model.VnetResListBody;
98100
import io.netris.model.VnetsBody;
99101
import io.netris.model.response.AuthResponse;
102+
import io.netris.model.response.L4LbEditResponse;
100103
import io.netris.model.response.TenantResponse;
101104
import io.netris.model.response.TenantsResponse;
102105
import org.apache.cloudstack.agent.api.CreateNetrisACLCommand;
@@ -116,6 +119,7 @@
116119
import org.apache.cloudstack.agent.api.SetupNetrisPublicRangeCommand;
117120
import org.apache.cloudstack.resource.NetrisResourceObjectUtils;
118121
import org.apache.commons.collections.CollectionUtils;
122+
import org.apache.commons.lang3.ObjectUtils;
119123
import org.apache.commons.lang3.StringUtils;
120124
import org.apache.logging.log4j.LogManager;
121125
import org.apache.logging.log4j.Logger;
@@ -612,7 +616,7 @@ public boolean releaseNatIp(ReleaseNatIpCommand cmd) {
612616
}
613617

614618
@Override
615-
public boolean createLbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd) {
619+
public boolean createOrUpdateLbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd) {
616620
boolean isVpc = cmd.isVpc();
617621
Long networkResourceId = cmd.getId();
618622
String networkResourceName = cmd.getName();
@@ -641,26 +645,31 @@ public boolean createLbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd) {
641645
if (Boolean.FALSE.equals(result)) {
642646
logger.warn("Could not find the Netris LB rule with name {}", lbName);
643647
}
644-
if (!matchingLbId.isEmpty()) {
645-
logger.warn("LB rule by name: {} already exists", lbName);
646-
return true;
647-
}
648-
649-
L4lbAddItem l4lbAddItem = getL4LbRule(cmd, vpcResource, lbName, publicIp, lbBackends);
648+
boolean updateRule = !matchingLbId.isEmpty();
649+
L4lbAddOrUpdateItem l4lbAddItem = getL4LbRule(cmd, vpcResource, lbName, publicIp, lbBackends, updateRule);
650650
L4LoadBalancerApi loadBalancerApi = apiClient.getApiStubForMethod(L4LoadBalancerApi.class);
651-
ResAddEditBody response = loadBalancerApi.apiV2L4lbPost(l4lbAddItem);
652-
if (Objects.isNull(response) || Boolean.FALSE.equals(response.isIsSuccess())) {
653-
throw new CloudRuntimeException("Failed to create Netris LB rule");
651+
boolean success;
652+
L4LbEditResponse editResponse = null;
653+
ResAddEditBody createResponse = null;
654+
if (updateRule) {
655+
editResponse = loadBalancerApi.apiV2L4lbIdPut((L4LbEditItem) l4lbAddItem, matchingLbId.get(0).intValue());
656+
success = editResponse.isIsSuccess();
657+
} else {
658+
createResponse = loadBalancerApi.apiV2L4lbPost((L4LbAddItem) l4lbAddItem);
659+
success = createResponse.isIsSuccess();
660+
}
661+
if (ObjectUtils.allNull(editResponse, createResponse) || Boolean.FALSE.equals(success)) {
662+
throw new CloudRuntimeException(String.format("Failed to %s Netris LB rule", updateRule ? "update" : "create"));
654663
}
655664
} catch (ApiException e) {
656665
logAndThrowException("Failed to create Netris load balancer rule", e);
657666
}
658667
return true;
659668
}
660669

661-
private L4lbAddItem getL4LbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd, VPCListing vpcResource, String lbName,
662-
String publicIp, List<NetrisLbBackend> lbBackends) {
663-
L4lbAddItem l4lbAddItem = new L4lbAddItem();
670+
private L4lbAddOrUpdateItem getL4LbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd, VPCListing vpcResource, String lbName,
671+
String publicIp, List<NetrisLbBackend> lbBackends, boolean updateRule) {
672+
L4lbAddOrUpdateItem l4lbAddItem = updateRule ? new L4LbEditItem() : new L4LbAddItem();
664673
try {
665674
l4lbAddItem.setName(lbName);
666675

@@ -685,7 +694,7 @@ private L4lbAddItem getL4LbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd,
685694
l4lbAddItem.setVpc(vpc);
686695

687696
l4lbAddItem.setAutomatic(false);
688-
l4lbAddItem.setIpFamily(NetUtils.isIpv4(publicIp) ? L4lbAddItem.IpFamilyEnum.IPv4 : L4lbAddItem.IpFamilyEnum.IPv6);
697+
l4lbAddItem.setIpFamily(NetUtils.isIpv4(publicIp) ? L4lbAddOrUpdateItem.IpFamilyEnum.IPv4 : L4lbAddOrUpdateItem.IpFamilyEnum.IPv6);
689698
l4lbAddItem.setIp(publicIp);
690699
l4lbAddItem.setStatus("enable");
691700

@@ -698,7 +707,7 @@ private L4lbAddItem getL4LbRule(CreateOrUpdateNetrisLoadBalancerRuleCommand cmd,
698707
}
699708
l4lbAddItem.setBackend(backends);
700709
l4lbAddItem.setPort(Integer.valueOf(cmd.getPublicPort()));
701-
l4lbAddItem.setHealthCheck(L4lbAddItem.HealthCheckEnum.NONE);
710+
l4lbAddItem.setHealthCheck(L4lbAddOrUpdateItem.HealthCheckEnum.NONE);
702711
} catch (Exception e) {
703712
throw new CloudRuntimeException("Failed to create Netris load balancer rule", e);
704713
}

‎plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisElement.java‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import com.cloud.network.PublicIpAddress;
4545
import com.cloud.network.SDNProviderNetworkRule;
4646
import com.cloud.network.SDNProviderOpObject;
47+
import com.cloud.network.as.AutoScaleCounter;
4748
import com.cloud.network.dao.IPAddressDao;
4849
import com.cloud.network.dao.IPAddressVO;
4950
import com.cloud.network.dao.LoadBalancerVMMapDao;
@@ -91,6 +92,7 @@
9192
import com.cloud.vm.VMInstanceVO;
9293
import com.cloud.vm.VirtualMachineProfile;
9394
import com.cloud.vm.dao.VMInstanceDao;
95+
import com.google.gson.Gson;
9496
import org.apache.cloudstack.StartupNetrisCommand;
9597
import org.apache.cloudstack.api.ApiConstants;
9698
import com.cloud.network.netris.NetrisNetworkRule;
@@ -147,6 +149,16 @@ public class NetrisElement extends AdapterBase implements DhcpServiceProvider, D
147149

148150
private final Map<Network.Service, Map<Network.Capability, String>> capabilities = initCapabilities();
149151

152+
protected static List<AutoScaleCounter> getNetrisAutoScaleCounters() {
153+
AutoScaleCounter counter;
154+
final List<AutoScaleCounter> counterList = new ArrayList<>();
155+
counter = new AutoScaleCounter(AutoScaleCounter.AutoScaleCounterType.Cpu);
156+
counterList.add(counter);
157+
counter = new AutoScaleCounter(AutoScaleCounter.AutoScaleCounterType.Memory);
158+
counterList.add(counter);
159+
return counterList;
160+
}
161+
150162
private static Map<Network.Service, Map<Network.Capability, String>> initCapabilities() {
151163
Map<Network.Service, Map<Network.Capability, String>> capabilities = new HashMap<>();
152164

@@ -166,6 +178,10 @@ private static Map<Network.Service, Map<Network.Capability, String>> initCapabil
166178
lbCapabilities.put(Network.Capability.SupportedProtocols, "tcp, udp");
167179
lbCapabilities.put(Network.Capability.SupportedStickinessMethods, VirtualRouterElement.getHAProxyStickinessCapability());
168180
lbCapabilities.put(Network.Capability.LbSchemes, String.join(",", LoadBalancerContainer.Scheme.Internal.name(), LoadBalancerContainer.Scheme.Public.name()));
181+
final Gson gson = new Gson();
182+
final String autoScaleCounterList = gson.toJson(getNetrisAutoScaleCounters());
183+
lbCapabilities.put(Network.Capability.AutoScaleCounters, autoScaleCounterList);
184+
lbCapabilities.put(Network.Capability.VmAutoScaling, "true");
169185

170186
capabilities.put(Network.Service.Lb, lbCapabilities);
171187
capabilities.put(Network.Service.PortForwarding, null);
@@ -750,7 +766,7 @@ public boolean applyLBRules(Network network, List<LoadBalancingRule> rules) thro
750766
.lbBackends(lbBackends)
751767
.build();
752768
if (Arrays.asList(FirewallRule.State.Add, FirewallRule.State.Active).contains(loadBalancingRule.getState())) {
753-
result &= netrisService.createLbRule(networkRule);
769+
result &= netrisService.createOrUpdateLbRule(networkRule);
754770
} else if (loadBalancingRule.getState() == FirewallRule.State.Revoke) {
755771
result &= netrisService.deleteLbRule(networkRule);
756772
}

‎plugins/network-elements/netris/src/main/java/org/apache/cloudstack/service/NetrisServiceImpl.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,7 @@ public boolean releaseNatIp(long zoneId, String publicIp) {
458458
}
459459

460460
@Override
461-
public boolean createLbRule(NetrisNetworkRule rule) {
461+
public boolean createOrUpdateLbRule(NetrisNetworkRule rule) {
462462
SDNProviderNetworkRule baseRule = rule.getBaseRule();
463463
CreateOrUpdateNetrisLoadBalancerRuleCommand cmd = new CreateOrUpdateNetrisLoadBalancerRuleCommand(baseRule.getZoneId(), baseRule.getAccountId(),
464464
baseRule.getDomainId(), baseRule.getNetworkResourceName(), baseRule.getNetworkResourceId(), baseRule.isVpcResource(),

‎server/src/main/java/com/cloud/network/as/AutoScaleManagerImpl.java‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2250,8 +2250,9 @@ public void checkAutoScaleVmGroup(AutoScaleVmGroupVO asGroup) {
22502250
Network.Provider provider = getLoadBalancerServiceProvider(asGroup.getLoadBalancerId());
22512251
if (Network.Provider.Netscaler.equals(provider)) {
22522252
checkNetScalerAsGroup(asGroup);
2253-
} else if (Network.Provider.VirtualRouter.equals(provider) || Network.Provider.VPCVirtualRouter.equals(provider)) {
2254-
checkVirtualRouterAsGroup(asGroup);
2253+
} else if (Network.Provider.VirtualRouter.equals(provider) || Network.Provider.VPCVirtualRouter.equals(provider) ||
2254+
Network.Provider.Netris.equals(provider)) {
2255+
checkAutoscalingGroup(asGroup);
22552256
}
22562257
}
22572258
}
@@ -2631,7 +2632,7 @@ protected void updateCountersMapWithProcessedData(Map<String, Double> countersMa
26312632
countersNumberMap.put(key, countersNumberMap.get(key) + 1);
26322633
}
26332634

2634-
protected void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) {
2635+
protected void monitorAutoscalingGroup(AutoScaleVmGroupVO asGroup) {
26352636
if (!checkAsGroupMaxAndMinMembers(asGroup)) {
26362637
return;
26372638
}
@@ -2661,7 +2662,7 @@ protected void monitorVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) {
26612662
}
26622663
}
26632664

2664-
protected void checkVirtualRouterAsGroup(AutoScaleVmGroupVO asGroup) {
2665+
protected void checkAutoscalingGroup(AutoScaleVmGroupVO asGroup) {
26652666
AutoScaleVmGroupTO groupTO = lbRulesMgr.toAutoScaleVmGroupTO(asGroup);
26662667

26672668
Map<String, Double> countersMap = new HashMap<>();
@@ -3011,8 +3012,9 @@ protected synchronized void runInContext() {
30113012
Network.Provider provider = getLoadBalancerServiceProvider(asGroup.getLoadBalancerId());
30123013
if (Network.Provider.Netscaler.equals(provider)) {
30133014
logger.debug("Skipping the monitoring on AutoScale VmGroup with Netscaler provider: " + asGroup);
3014-
} else if (Network.Provider.VirtualRouter.equals(provider) || Network.Provider.VPCVirtualRouter.equals(provider)) {
3015-
monitorVirtualRouterAsGroup(asGroup);
3015+
} else if (Network.Provider.VirtualRouter.equals(provider) || Network.Provider.VPCVirtualRouter.equals(provider) ||
3016+
Network.Provider.Netris.equals(provider)) {
3017+
monitorAutoscalingGroup(asGroup);
30163018
}
30173019
}
30183020
} catch (final Exception e) {

‎server/src/main/java/com/cloud/network/lb/LoadBalancingRulesManagerImpl.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1008,6 +1008,7 @@ private boolean isRollBackAllowedForProvider(LoadBalancerVO loadBalancer) {
10081008
return false;
10091009
}
10101010
if (provider.get(0) == Provider.Netscaler || provider.get(0) == Provider.F5BigIp ||
1011+
provider.get(0) == Provider.Netris ||
10111012
provider.get(0) == Provider.VirtualRouter || provider.get(0) == Provider.VPCVirtualRouter) {
10121013
return true;
10131014
}

‎server/src/main/java/com/cloud/server/ConfigurationServerImpl.java‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1199,51 +1199,52 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
11991199

12001200
// Offering #9 - network offering for NSX provider - NATTED mode
12011201
createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING, "Offering for NSX enabled networks - NAT mode",
1202-
NetworkOffering.NetworkMode.NATTED, false, true, Provider.Nsx);
1202+
NetworkOffering.NetworkMode.NATTED, false, true, false, Provider.Nsx);
12031203

12041204
// Offering #10 - network offering for NSX provider - ROUTED mode
12051205
createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_ROUTED_NSX_OFFERING, "Offering for NSX enabled networks - ROUTED mode",
1206-
NetworkOffering.NetworkMode.ROUTED, false, true, Provider.Nsx);
1206+
NetworkOffering.NetworkMode.ROUTED, false, true, false, Provider.Nsx);
12071207

12081208
// Offering #11 - network offering for NSX provider for VPCs - NATTED mode
12091209
createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING_FOR_VPC, "Offering for NSX enabled networks on VPCs - NAT mode",
1210-
NetworkOffering.NetworkMode.NATTED, true, true, Provider.Nsx);
1210+
NetworkOffering.NetworkMode.NATTED, true, true, false, Provider.Nsx);
12111211

12121212
// Offering #12 - network offering for NSX provider for VPCs - ROUTED mode
12131213
createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_ROUTED_NSX_OFFERING_FOR_VPC, "Offering for NSX enabled networks on VPCs - ROUTED mode",
1214-
NetworkOffering.NetworkMode.ROUTED, true, true, Provider.Nsx);
1214+
NetworkOffering.NetworkMode.ROUTED, true, true, false, Provider.Nsx);
12151215

12161216
// Offering #13 - network offering for NSX provider for VPCs with Internal LB - NATTED mode
12171217
createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NSX_OFFERING_FOR_VPC_WITH_ILB, "Offering for NSX enabled networks on VPCs with internal LB - NAT mode",
1218-
NetworkOffering.NetworkMode.NATTED, true, false, Provider.Nsx);
1218+
NetworkOffering.NetworkMode.NATTED, true, false, false, Provider.Nsx);
12191219

12201220
// Offering #14 - network offering for Netris provider for VPCs - ROUTED mode
12211221
createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_ROUTED_NETRIS_OFFERING_FOR_VPC, "Offering for Netris enabled networks on VPCs - ROUTED mode",
1222-
NetworkOffering.NetworkMode.ROUTED, true, true, Provider.Netris);
1222+
NetworkOffering.NetworkMode.ROUTED, true, true, false, Provider.Netris);
12231223

12241224
// Offering #15 - network offering for Netris provider for VPCs - NATTED mode
12251225
createAndPersistDefaultProviderOffering(NetworkOffering.DEFAULT_NAT_NETRIS_OFFERING_FOR_VPC, "Offering for Netris enabled networks on VPCs - NAT mode",
1226-
NetworkOffering.NetworkMode.NATTED, true, true, Provider.Netris);
1226+
NetworkOffering.NetworkMode.NATTED, true, true, true, Provider.Netris);
12271227
}
12281228
});
12291229
}
12301230

12311231
private void createAndPersistDefaultProviderOffering(String name, String displayText, NetworkOffering.NetworkMode networkMode,
1232-
boolean forVpc, boolean publicLB, Provider provider) {
1233-
NetworkOfferingVO defaultNatProviderNetworkOffering =
1232+
boolean forVpc, boolean publicLB, boolean supportVmAutoscaling, Provider provider) {
1233+
NetworkOfferingVO providerNetworkOffering =
12341234
new NetworkOfferingVO(name, displayText, TrafficType.Guest, false, false, null,
12351235
null, true, Availability.Optional, null, GuestType.Isolated, false,
12361236
false, false, false, false, forVpc);
1237-
defaultNatProviderNetworkOffering.setPublicLb(publicLB);
1238-
defaultNatProviderNetworkOffering.setInternalLb(!publicLB);
1239-
defaultNatProviderNetworkOffering.setNetworkMode(networkMode);
1240-
defaultNatProviderNetworkOffering.setState(NetworkOffering.State.Enabled);
1241-
defaultNatProviderNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(defaultNatProviderNetworkOffering);
1237+
providerNetworkOffering.setPublicLb(publicLB);
1238+
providerNetworkOffering.setInternalLb(!publicLB);
1239+
providerNetworkOffering.setNetworkMode(networkMode);
1240+
providerNetworkOffering.setState(NetworkOffering.State.Enabled);
1241+
providerNetworkOffering.setSupportsVmAutoScaling(supportVmAutoscaling);
1242+
providerNetworkOffering = _networkOfferingDao.persistDefaultNetworkOffering(providerNetworkOffering);
12421243

12431244
Map<Service, Provider> serviceProviderMap = getServicesAndProvidersForProviderNetwork(networkMode, forVpc, provider);
12441245
for (Map.Entry<Network.Service, Network.Provider> service : serviceProviderMap.entrySet()) {
12451246
NetworkOfferingServiceMapVO offService =
1246-
new NetworkOfferingServiceMapVO(defaultNatProviderNetworkOffering.getId(), service.getKey(), service.getValue());
1247+
new NetworkOfferingServiceMapVO(providerNetworkOffering.getId(), service.getKey(), service.getValue());
12471248
_ntwkOfferingServiceMapDao.persist(offService);
12481249
logger.trace("Added service for the network offering: " + offService);
12491250
}

0 commit comments

Comments
 (0)