Fix appservers selecting the same port

AdminServer/appscale/admin/instance_manager/instance_manager.py

@@ -4,6 +4,7 @@
 import monotonic
 import json
 import os
+import socket
 import urllib2
 
 from tornado import gen
@@ -442,21 +443,40 @@ def _stop_app_instance(self, instance):
 
     yield self._clean_old_sources()
 
-  def _get_lowest_port(self):
+  def _get_lowest_port(self, excluded_ports):
     """ Determines the lowest usuable port for a new instance.
-
+    Args:
+      excluded_ports: A set of ports that the caller is planning on using.
+        This is used to make sure AppServers do not pick the same port since
+        it's possible for the port to not be in self._running_instances yet.
     Returns:
       An integer specifying a free port.
     """
     existing_ports = {instance.port for instance in self._running_instances}
     port = STARTING_INSTANCE_PORT
     while True:
-      if port in existing_ports:
+      if port in existing_ports or port in excluded_ports or not self._try_port(port):
         port += 1
         continue
 
       return port
 
+  def _try_port(self, port):
+    """ Helper method to check if the port is actually free or not.
+
+    Returns:
+      True if port is free, False otherwise.
+    """
+    sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    result = False
+    try:
+      sock.bind(("0.0.0.0", port))
+      result = True
+    except:
+      logger.info("Port {} is in use".format(port))
+    sock.close()
+    return result
+
   @gen.coroutine
   def _restart_unrouted_instances(self):
     """ Restarts instances that the router considers offline. """
@@ -535,7 +555,9 @@ def _fulfill_assignments(self):
 
       for instance in to_stop:
         yield self._stop_app_instance(instance)
-
+      excluded_ports = set()
+      for assigned_ports in self._assignments:
+        excluded_ports.update(assigned_ports)
       for version_key, assigned_ports in self._assignments.items():
         try:
           version = self._projects_manager.version_from_key(version_key)
@@ -571,7 +593,9 @@ def _fulfill_assignments(self):
                       and instance.port not in assigned_ports]
         to_start = max(new_assignment_count - len(candidates), 0)
         for _ in range(to_start):
-          yield self._start_instance(version, self._get_lowest_port())
+          port = self._get_lowest_port(excluded_ports)
+          excluded_ports.add(port)
+          yield self._start_instance(version, port)
 
   @gen.coroutine
   def _enforce_instance_details(self):