Merge pull request #99 from smarterclayton/more_resilience_in_linking_init

Make gear init --post more resilient

Author: smarterclayton

cmd/gear/commands.go

@@ -82,6 +82,7 @@ var (
 )
 
 func init() {
+	log.SetFlags(0)
 	defaultTransport.Set("http")
 }
 

cmd/gear/support.go

@@ -155,26 +155,35 @@ func InitPostStart(dockerSocket string, id containers.Identifier) error {
 	if file, err := os.Open(id.NetworkLinksPathFor()); err == nil {
 		defer file.Close()
 
-		const ContainerInterval = time.Second / 3
-		const ContainerWait = time.Second * 12
+		const ContainerInterval = time.Second / 10
+		const ContainerWait = time.Second * 15
 		for i := 0; i < int(ContainerWait/ContainerInterval); i++ {
-			if container, err = d.GetContainer(id.ContainerFor(), true); err != nil {
+			if container, err = d.InspectContainer(id.ContainerFor()); err != nil {
+				if err == docker.ErrNoSuchContainer {
+					//log.Printf("Waiting for container to be available.")
+					time.Sleep(ContainerInterval)
+					continue
+				}
 				return err
 			}
-			if container.State.Running {
+			if container.State.Running && container.State.Pid != 0 {
 				break
 			} else {
-				log.Printf("Waiting for container to run.")
+				//log.Printf("Waiting for container to report available.")
 				time.Sleep(ContainerInterval)
 			}
 		}
 
+		if container == nil {
+			return fmt.Errorf("container %s was not visible through Docker before timeout", id.ContainerFor())
+		}
+
 		pid, err := d.ChildProcessForContainer(container)
 		if err != nil {
 			return err
 		}
-		if pid < 2 {
-			return errors.New("support: child PID is not correct")
+		if pid <= 1 {
+			return errors.New("child PID is not correct")
 		}
 		log.Printf("Updating network namespaces for %d", pid)
 		if err := updateNamespaceNetworkLinks(pid, file); err != nil {
@@ -321,6 +330,8 @@ func updateNamespaceNetworkLinks(pid int, ports io.Reader) error {
 				continue
 			}
 
+			log.Printf("Mapping %s(%s):%d -> %s:%d", sourceAddr.String(), srcIP.String(), link.FromPort, destIP.String(), link.ToPort)
+
 			data := containers.OutboundNetworkIptables{sourceAddr.String(), srcIP.IP.String(), link.FromPort, destIP.String(), link.ToPort}
 			if err := containers.OutboundNetworkIptablesTemplate.Execute(stdin, &data); err != nil {
 				log.Printf("gear: Unable to write network link rules: %v", err)

docker/docker.go

@@ -6,7 +6,6 @@ import (
 	"github.com/fsouza/go-dockerclient"
 	"github.com/fsouza/go-dockerclient/engine"
 	"io/ioutil"
-	"log"
 	"os"
 	"path/filepath"
 	"strconv"
@@ -53,7 +52,7 @@ func lookupContainer(containerName string, client *docker.Client, waitForContain
 			return containerLookupResult{container, nil}
 		}
 	}
-	return containerLookupResult{nil, fmt.Errorf("Container not active")}
+	return containerLookupResult{nil, fmt.Errorf("container not active")}
 }
 
 func GetConnection(dockerSocket string) (*DockerClient, error) {
@@ -78,6 +77,16 @@ func GetConnection(dockerSocket string) (*DockerClient, error) {
 	return &DockerClient{client, executionDriver}, nil
 }
 
+var ErrNoSuchContainer = errors.New("can't find container")
+
+func (d *DockerClient) InspectContainer(containerName string) (*docker.Container, error) {
+	c, err := d.client.InspectContainer(containerName)
+	if err != nil && strings.HasPrefix(err.Error(), "No such container") {
+		err = ErrNoSuchContainer
+	}
+	return c, err
+}
+
 func (d *DockerClient) GetContainer(containerName string, waitForContainer bool) (*docker.Container, error) {
 	timeoutChannel := make(chan containerLookupResult, 1)
 	var container *docker.Container
@@ -89,7 +98,7 @@ func (d *DockerClient) GetContainer(containerName string, waitForContainer bool)
 		}
 		container = cInfo.Container
 	case <-time.After(time.Minute):
-		return nil, fmt.Errorf("Timeout waiting for container")
+		return nil, fmt.Errorf("timeout waiting for container")
 	}
 
 	return container, nil
@@ -120,11 +129,11 @@ func (d *DockerClient) GetContainerIPs(ids []string) (map[string]string, error)
 }
 
 func (d *DockerClient) ChildProcessForContainer(container *docker.Container) (int, error) {
-	log.Printf("docker: execution driver %s", d.executionDriver)
+	//log.Printf("docker: execution driver %s", d.executionDriver)
 	if d.executionDriver == "" || strings.HasPrefix(d.executionDriver, "lxc") {
 		//Parent pid (LXC or N-Spawn)
 		ppid := strconv.Itoa(container.State.Pid)
-		log.Printf("docker: parent pid %s", ppid)
+		//log.Printf("docker: parent pid %s", ppid)
 
 		//Lookup any child of parent pid
 		files, _ := filepath.Glob(filepath.Join("/proc", "*", "stat"))
@@ -147,7 +156,7 @@ func (d *DockerClient) ChildProcessForContainer(container *docker.Container) (in
 		if container.State.Pid != 0 {
 			return container.State.Pid, nil
 		}
-		return 0, fmt.Errorf("Container not found")
+		return 0, fmt.Errorf("unable to find child process for container %s - race condition with Docker?", container.ID)
 	}
-	return 0, errors.New(fmt.Sprintf("Unable to find child process for container", container.ID))
+	return 0, fmt.Errorf("unable to find child process for container %s", container.ID)
 }