routing: ensure desired routes are in the route table

There is a brief period where the IPv6 CIDR is not yet associated with
the subnets. Thus, CAPA initially creates a route table without a route
to eigw. This change ensures that later reconcilation will add that
missing route.

Note: Route to eigw for destination "::/0" to eigw is required for EC2
instance time sync on start-up.

Author: tthvo

pkg/cloud/services/network/routetables.go

@@ -84,6 +84,11 @@ func (s *Service) reconcileRouteTables() error {
 				}
 			}
 
+			// Make sure desired routes are created in the route table.
+			if err := s.fixMissingRoutes(routes, rt.Routes, rt); err != nil {
+				return err
+			}
+
 			// Make sure tags are up-to-date.
 			if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
 				buildParams := s.getRouteTableTagParams(aws.ToString(rt.RouteTableId), sn.IsPublic, sn.AvailabilityZone)
@@ -145,7 +150,8 @@ func (s *Service) fixMismatchedRouting(specRoute *ec2.CreateRouteInput, currentR
 		if (currentRoute.DestinationIpv6CidrBlock != nil &&
 			aws.ToString(currentRoute.DestinationIpv6CidrBlock) == aws.ToString(specRoute.DestinationIpv6CidrBlock)) &&
 			((currentRoute.GatewayId != nil && aws.ToString(currentRoute.GatewayId) != aws.ToString(specRoute.GatewayId)) ||
-				(currentRoute.NatGatewayId != nil && aws.ToString(currentRoute.NatGatewayId) != aws.ToString(specRoute.NatGatewayId))) {
+				(currentRoute.NatGatewayId != nil && aws.ToString(currentRoute.NatGatewayId) != aws.ToString(specRoute.NatGatewayId)) ||
+				(currentRoute.EgressOnlyInternetGatewayId != nil && aws.ToString(currentRoute.EgressOnlyInternetGatewayId) != aws.ToString(specRoute.EgressOnlyInternetGatewayId))) {
 			input = &ec2.ReplaceRouteInput{
 				RouteTableId:                rt.RouteTableId,
 				DestinationIpv6CidrBlock:    specRoute.DestinationIpv6CidrBlock,
@@ -170,6 +176,36 @@ func (s *Service) fixMismatchedRouting(specRoute *ec2.CreateRouteInput, currentR
 	return nil
 }
 
+func (s *Service) fixMissingRoutes(specRoutes []*ec2.CreateRouteInput, currentRoutes []types.Route, rt types.RouteTable) error {
+	// Routes destination cidr blocks must be unique within a routing table.
+	// Each route can define either an IPv4 or IPv6 CIDR, but not both.
+	currRouteMap := make(map[string]types.Route)
+	for _, route := range currentRoutes {
+		if route.DestinationCidrBlock != nil {
+			currRouteMap[aws.ToString(route.DestinationCidrBlock)] = route
+		}
+		if route.DestinationIpv6CidrBlock != nil {
+			currRouteMap[aws.ToString(route.DestinationIpv6CidrBlock)] = route
+		}
+	}
+
+	routesToAdd := make([]*ec2.CreateRouteInput, 0)
+	for _, route := range specRoutes {
+		var dest string
+		if route.DestinationCidrBlock != nil {
+			dest = aws.ToString(route.DestinationCidrBlock)
+		}
+		if route.DestinationIpv6CidrBlock != nil {
+			dest = aws.ToString(route.DestinationIpv6CidrBlock)
+		}
+		if _, ok := currRouteMap[dest]; !ok {
+			routesToAdd = append(routesToAdd, route)
+		}
+	}
+
+	return s.createRoutesForRouteTable(routesToAdd, rt.RouteTableId)
+}
+
 func (s *Service) describeVpcRouteTablesBySubnet() (map[string]types.RouteTable, error) {
 	rts, err := s.describeVpcRouteTables()
 	if err != nil {
@@ -274,28 +310,34 @@ func (s *Service) createRouteTableWithRoutes(routes []*ec2.CreateRouteInput, isP
 	record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateRouteTable", "Created managed RouteTable %q", aws.ToString(out.RouteTable.RouteTableId))
 	s.scope.Info("Created route table", "route-table-id", *out.RouteTable.RouteTableId)
 
+	if err := s.createRoutesForRouteTable(routes, out.RouteTable.RouteTableId); err != nil {
+		if cleanupErr := s.deleteRouteTable(*out.RouteTable); cleanupErr != nil {
+			record.Warnf(s.scope.InfraCluster(), "FailedDeleteRouteTable", "Failed to delete managed RouteTable %q: %v", aws.ToString(out.RouteTable.RouteTableId), cleanupErr)
+		}
+		return nil, err
+	}
+
+	return &infrav1.RouteTable{
+		ID: *out.RouteTable.RouteTableId,
+	}, nil
+}
+
+func (s *Service) createRoutesForRouteTable(routes []*ec2.CreateRouteInput, routeTableID *string) error {
 	for i := range routes {
 		route := routes[i]
 		if err := wait.WaitForWithRetryable(wait.NewBackoff(), func() (bool, error) {
-			route.RouteTableId = out.RouteTable.RouteTableId
+			route.RouteTableId = routeTableID
 			if _, err := s.EC2Client.CreateRoute(context.TODO(), route); err != nil {
 				return false, err
 			}
 			return true, nil
 		}, awserrors.RouteTableNotFound, awserrors.NATGatewayNotFound, awserrors.GatewayNotFound); err != nil {
-			record.Warnf(s.scope.InfraCluster(), "FailedCreateRoute", "Failed to create route %s for RouteTable %q: %v", route, aws.ToString(out.RouteTable.RouteTableId), err)
-			errDel := s.deleteRouteTable(*out.RouteTable)
-			if errDel != nil {
-				record.Warnf(s.scope.InfraCluster(), "FailedDeleteRouteTable", "Failed to delete managed RouteTable %q: %v", aws.ToString(out.RouteTable.RouteTableId), errDel)
-			}
-			return nil, errors.Wrapf(err, "failed to create route in route table %q: %v", aws.ToString(out.RouteTable.RouteTableId), route)
+			record.Warnf(s.scope.InfraCluster(), "FailedCreateRoute", "Failed to create route %s for RouteTable %q: %v", route, aws.ToString(routeTableID), err)
+			return errors.Wrapf(err, "failed to create route in route table %q: %v", aws.ToString(routeTableID), route)
 		}
-		record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateRoute", "Created route %s for RouteTable %q", route, aws.ToString(out.RouteTable.RouteTableId))
+		record.Eventf(s.scope.InfraCluster(), "SuccessfulCreateRoute", "Created route %s for RouteTable %q", route, aws.ToString(routeTableID))
 	}
-
-	return &infrav1.RouteTable{
-		ID: *out.RouteTable.RouteTableId,
-	}, nil
+	return nil
 }
 
 func (s *Service) associateRouteTable(rt *infrav1.RouteTable, subnetID string) error {