Avoid nDB stale entries because of intermittent nw issues.

Signed-off-by: Santhosh Manohar <santhosh@docker.com>

endpoint.go

@@ -427,7 +427,7 @@ func (ep *endpoint) Join(sbox Sandbox, options ...EndpointOption) error {
 	return ep.sbJoin(sb, options...)
 }
 
-func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error {
+func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) (err error) {
 	n, err := ep.getNetworkFromStore()
 	if err != nil {
 		return fmt.Errorf("failed to get network from store during join: %v", err)
@@ -462,7 +462,7 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error {
 
 	d, err := n.driver(true)
 	if err != nil {
-		return fmt.Errorf("failed to join endpoint: %v", err)
+		return fmt.Errorf("failed to get driver during join: %v", err)
 	}
 
 	err = d.Join(nid, epid, sb.Key(), ep, sb.Labels())
@@ -471,8 +471,8 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error {
 	}
 	defer func() {
 		if err != nil {
-			if err := d.Leave(nid, epid); err != nil {
-				logrus.Warnf("driver leave failed while rolling back join: %v", err)
+			if e := d.Leave(nid, epid); e != nil {
+				logrus.Warnf("driver leave failed while rolling back join: %v", e)
 			}
 		}
 	}()
@@ -528,8 +528,7 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error {
 	}()
 
 	if sb.needDefaultGW() && sb.getEndpointInGWNetwork() == nil {
-		err = sb.setupDefaultGW()
-		return err
+		return sb.setupDefaultGW()
 	}
 
 	moveExtConn := sb.getGatewayEndpoint() != extEp
@@ -539,11 +538,11 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error {
 			logrus.Debugf("Revoking external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
 			extN, err := extEp.getNetworkFromStore()
 			if err != nil {
-				return fmt.Errorf("failed to get network from store during join: %v", err)
+				return fmt.Errorf("failed to get network from store for revoking external connectivity during join: %v", err)
 			}
 			extD, err := extN.driver(true)
 			if err != nil {
-				return fmt.Errorf("failed to join endpoint: %v", err)
+				return fmt.Errorf("failed to get driver for revoking external connectivity during join: %v", err)
 			}
 			if err = extD.RevokeExternalConnectivity(extEp.network.ID(), extEp.ID()); err != nil {
 				return types.InternalErrorf(
@@ -571,9 +570,9 @@ func (ep *endpoint) sbJoin(sb *sandbox, options ...EndpointOption) error {
 	}
 
 	if !sb.needDefaultGW() {
-		if err := sb.clearDefaultGW(); err != nil {
+		if e := sb.clearDefaultGW(); e != nil {
 			logrus.Warnf("Failure while disconnecting sandbox %s (%s) from gateway network: %v",
-				sb.ID(), sb.ContainerID(), err)
+				sb.ID(), sb.ContainerID(), e)
 		}
 	}
 
@@ -706,7 +705,7 @@ func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption)
 
 	d, err := n.driver(!force)
 	if err != nil {
-		return fmt.Errorf("failed to leave endpoint: %v", err)
+		return fmt.Errorf("failed to get driver during endpoint leave: %v", err)
 	}
 
 	ep.Lock()
@@ -766,11 +765,11 @@ func (ep *endpoint) sbLeave(sb *sandbox, force bool, options ...EndpointOption)
 		logrus.Debugf("Programming external connectivity on endpoint %s (%s)", extEp.Name(), extEp.ID())
 		extN, err := extEp.getNetworkFromStore()
 		if err != nil {
-			return fmt.Errorf("failed to get network from store during leave: %v", err)
+			return fmt.Errorf("failed to get network from store for programming external connectivity during leave: %v", err)
 		}
 		extD, err := extN.driver(true)
 		if err != nil {
-			return fmt.Errorf("failed to leave endpoint: %v", err)
+			return fmt.Errorf("failed to get driver for programming external connectivity during leave: %v", err)
 		}
 		if err := extD.ProgramExternalConnectivity(extEp.network.ID(), extEp.ID(), sb.Labels()); err != nil {
 			logrus.Warnf("driver failed programming external connectivity on endpoint %s: (%s) %v",

networkdb/cluster.go

@@ -17,7 +17,7 @@ import (
 )
 
 const (
-	reapInterval     = 60 * time.Second
+	reapInterval     = 30 * time.Minute
 	reapPeriod       = 5 * time.Second
 	retryInterval    = 1 * time.Second
 	nodeReapInterval = 24 * time.Hour