Merge pull request #959 from sanimej/102

Changelog for v0.6.2-rc1

CHANGELOG.md

@@ -1,5 +1,48 @@
 # Changelog
 
+## 0.6.2-rc1 (2016-02-19)
+- Fixes https://github.com/docker/docker/issues/20350
+
+## 0.6.1-rc3 (2016-02-11)
+- Fixes getNetworksFromStore to not fail on inconsistent network state
+
+## 0.6.1-rc2 (2016-02-09)
+- Fixes https://github.com/docker/docker/issues/20132
+- Fixes https://github.com/docker/docker/issues/20140
+- Fixes https://github.com/docker/docker/issues/20019
+
+## 0.6.1-rc1 (2016-02-05)
+- Fixes https://github.com/docker/docker/issues/20026
+
+## 0.6.0-rc7 (2016-02-01)
+- Allow inter-network connections via exposed ports
+
+## 0.6.0-rc6 (2016-01-30)
+- Properly fixes https://github.com/docker/docker/issues/18814
+
+## 0.6.0-rc5 (2016-01-26)
+- Cleanup stale overlay sandboxes
+
+## 0.6.0-rc4 (2016-01-25)
+- Add Endpoints() API to Sandbox interface
+- Fixed a race-condition in default gateway network creation
+
+## 0.6.0-rc3 (2016-01-25)
+- Fixes docker/docker#19576
+- Fixed embedded DNS to listen in TCP as well
+- Fixed a race-condition in IPAM to choose non-overlapping subnet for concurrent requests
+
+## 0.6.0-rc2 (2016-01-21)
+- Fixes docker/docker#19376
+- Fixes docker/docker#15819
+- Fixes libnetwork/#885, Not filter v6 DNS servers from resolv.conf
+- Fixes docker/docker #19448, also handles the . in service and network names correctly.
+
+## 0.6.0-rc1 (2016-01-14)
+- Fixes docker/docker#19404
+- Fixes the ungraceful daemon restart issue in systemd with remote network plugin
+  (https://github.com/docker/libnetwork/issues/813)
+
 ## 0.5.6 (2016-01-14)
 - Setup embedded DNS server correctly on container restart. Fixes docker/docker#19354
 
@@ -50,6 +93,6 @@
 - Fixed a bunch of issues with osl namespace mgmt
 
 ## 0.3.0 (2015-05-27)
- 
+
 - Introduce CNM (Container Networking Model)
 - Replace docker networking with CNM & Bridge driver

Makefile

@@ -63,7 +63,7 @@ run-tests:
 	    if ls $$dir/*.go &> /dev/null; then \
 		pushd . &> /dev/null ; \
 		cd $$dir ; \
-		$(shell which godep) go test ${INSIDECONTAINER} -test.parallel 3 -test.v -covermode=count -coverprofile=./profile.tmp ; \
+		$(shell which godep) go test ${INSIDECONTAINER} -test.parallel 5 -test.v -covermode=count -coverprofile=./profile.tmp ; \
 		ret=$$? ;\
 		if [ $$ret -ne 0 ]; then exit $$ret; fi ;\
 		popd &> /dev/null; \

controller.go

@@ -387,7 +387,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
 
 	// Make sure we have a driver available for this network type
 	// before we allocate anything.
-	if _, err := network.driver(); err != nil {
+	if _, err := network.driver(true); err != nil {
 		return nil, err
 	}
 
@@ -432,7 +432,7 @@ func (c *controller) NewNetwork(networkType, name string, options ...NetworkOpti
 }
 
 func (c *controller) addNetwork(n *network) error {
-	d, err := n.driver()
+	d, err := n.driver(true)
 	if err != nil {
 		return err
 	}

default_gateway.go

@@ -12,6 +12,8 @@ const (
 	gwEPlen       = 12
 )
 
+var procGwNetwork = make(chan (bool), 1)
+
 /*
    libnetwork creates a bridge network "docker_gw_bridge" for provding
    default gateway for the containers if none of the container's endpoints
@@ -35,13 +37,11 @@ func (sb *sandbox) setupDefaultGW(srcEp *endpoint) error {
 		return nil
 	}
 
+	// Look for default gw network. In case of error (includes not found),
+	// retry and create it if needed in a serialized execution.
 	n, err := c.NetworkByName(libnGWNetwork)
 	if err != nil {
-		if _, ok := err.(types.NotFoundError); !ok {
-			return err
-		}
-		n, err = c.createGWNetwork()
-		if err != nil {
+		if n, err = c.defaultGwNetwork(); err != nil {
 			return err
 		}
 	}
@@ -84,7 +84,7 @@ func (sb *sandbox) clearDefaultGW() error {
 		return nil
 	}
 
-	if err := ep.sbLeave(sb); err != nil {
+	if err := ep.sbLeave(sb, false); err != nil {
 		return fmt.Errorf("container %s: endpoint leaving GW Network failed: %v", sb.containerID, err)
 	}
 	if err := ep.Delete(false); err != nil {
@@ -150,3 +150,18 @@ func (sb *sandbox) getEPwithoutGateway() *endpoint {
 	}
 	return nil
 }
+
+// Looks for the default gw network and creates it if not there.
+// Parallel executions are serialized.
+func (c *controller) defaultGwNetwork() (Network, error) {
+	procGwNetwork <- true
+	defer func() { <-procGwNetwork }()
+
+	n, err := c.NetworkByName(libnGWNetwork)
+	if err != nil {
+		if _, ok := err.(types.NotFoundError); ok {
+			n, err = c.createGWNetwork()
+		}
+	}
+	return n, err
+}

drivers/bridge/setup_ip_tables.go

@@ -115,7 +115,7 @@ func (n *bridgeNetwork) setupIPTables(config *networkConfiguration, i *bridgeInt
 			return iptables.ProgramChain(filterChain, config.BridgeName, hairpinMode, false)
 		})
 
-		n.portMapper.SetIptablesChain(filterChain, n.getNetworkBridgeName())
+		n.portMapper.SetIptablesChain(natChain, n.getNetworkBridgeName())
 	}
 
 	if err := ensureJumpRule("FORWARD", IsolationChain); err != nil {
@@ -138,6 +138,7 @@ func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairp
 		address   = addr.String()
 		natRule   = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-s", address, "!", "-o", bridgeIface, "-j", "MASQUERADE"}}
 		hpNatRule = iptRule{table: iptables.Nat, chain: "POSTROUTING", preArgs: []string{"-t", "nat"}, args: []string{"-m", "addrtype", "--src-type", "LOCAL", "-o", bridgeIface, "-j", "MASQUERADE"}}
+		skipDNAT  = iptRule{table: iptables.Nat, chain: DockerChain, preArgs: []string{"-t", "nat"}, args: []string{"-i", bridgeIface, "-j", "RETURN"}}
 		outRule   = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-i", bridgeIface, "!", "-o", bridgeIface, "-j", "ACCEPT"}}
 		inRule    = iptRule{table: iptables.Filter, chain: "FORWARD", args: []string{"-o", bridgeIface, "-m", "conntrack", "--ctstate", "RELATED,ESTABLISHED", "-j", "ACCEPT"}}
 	)
@@ -149,6 +150,12 @@ func setupIPTablesInternal(bridgeIface string, addr net.Addr, icc, ipmasq, hairp
 		}
 	}
 
+	if ipmasq && !hairpin {
+		if err := programChainRule(skipDNAT, "SKIP DNAT", enable); err != nil {
+			return err
+		}
+	}
+
 	// In hairpin mode, masquerade traffic from localhost
 	if hairpin {
 		if err := programChainRule(hpNatRule, "MASQ LOCAL HOST", enable); err != nil {

drivers/overlay/joinleave.go

@@ -49,33 +49,33 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 
 	sbox := n.sandbox()
 
-	name1, name2, err := createVethPair()
+	overlayIfName, containerIfName, err := createVethPair()
 	if err != nil {
 		return err
 	}
 
-	ep.ifName = name2
+	ep.ifName = overlayIfName
 
 	// Set the container interface and its peer MTU to 1450 to allow
 	// for 50 bytes vxlan encap (inner eth header(14) + outer IP(20) +
 	// outer UDP(8) + vxlan header(8))
-	veth, err := netlink.LinkByName(name1)
+	veth, err := netlink.LinkByName(overlayIfName)
 	if err != nil {
-		return fmt.Errorf("cound not find link by name %s: %v", name1, err)
+		return fmt.Errorf("cound not find link by name %s: %v", overlayIfName, err)
 	}
 	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
 	if err != nil {
 		return err
 	}
 
-	if err := sbox.AddInterface(name1, "veth",
+	if err := sbox.AddInterface(overlayIfName, "veth",
 		sbox.InterfaceOptions().Master(s.brName)); err != nil {
 		return fmt.Errorf("could not add veth pair inside the network sandbox: %v", err)
 	}
 
-	veth, err = netlink.LinkByName(name2)
+	veth, err = netlink.LinkByName(containerIfName)
 	if err != nil {
-		return fmt.Errorf("could not find link by name %s: %v", name2, err)
+		return fmt.Errorf("could not find link by name %s: %v", containerIfName, err)
 	}
 	err = netlink.LinkSetMTU(veth, vxlanVethMTU)
 	if err != nil {
@@ -96,7 +96,7 @@ func (d *driver) Join(nid, eid string, sboxKey string, jinfo driverapi.JoinInfo,
 	}
 
 	if iNames := jinfo.InterfaceName(); iNames != nil {
-		err = iNames.SetNames(name2, "eth")
+		err = iNames.SetNames(containerIfName, "eth")
 		if err != nil {
 			return err
 		}
@@ -136,14 +136,5 @@ func (d *driver) Leave(nid, eid string) error {
 
 	n.leaveSandbox()
 
-	link, err := netlink.LinkByName(ep.ifName)
-	if err != nil {
-		log.Warnf("Failed to retrieve interface link for interface removal on endpoint leave: %v", err)
-		return nil
-	}
-	if err := netlink.LinkDel(link); err != nil {
-		log.Warnf("Failed to delete interface link on endpoint leave: %v", err)
-	}
-
 	return nil
 }

drivers/overlay/ov_endpoint.go

@@ -4,8 +4,10 @@ import (
 	"fmt"
 	"net"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/driverapi"
 	"github.com/docker/libnetwork/netutils"
+	"github.com/vishvananda/netlink"
 )
 
 type endpointTable map[string]*endpoint
@@ -97,6 +99,20 @@ func (d *driver) DeleteEndpoint(nid, eid string) error {
 	}
 
 	n.deleteEndpoint(eid)
+
+	if ep.ifName == "" {
+		return nil
+	}
+
+	link, err := netlink.LinkByName(ep.ifName)
+	if err != nil {
+		log.Debugf("Failed to retrieve interface (%s)'s link on endpoint (%s) delete: %v", ep.ifName, ep.id, err)
+		return nil
+	}
+	if err := netlink.LinkDel(link); err != nil {
+		log.Debugf("Failed to delete interface (%s)'s link on endpoint (%s) delete: %v", ep.ifName, ep.id, err)
+	}
+
 	return nil
 }
 

drivers/overlay/ov_network.go

@@ -5,6 +5,8 @@ import (
 	"fmt"
 	"net"
 	"os"
+	"path/filepath"
+	"strings"
 	"sync"
 	"syscall"
 
@@ -158,7 +160,9 @@ func (n *network) destroySandbox() {
 	sbox := n.sandbox()
 	if sbox != nil {
 		for _, iface := range sbox.Info().Interfaces() {
-			iface.Remove()
+			if err := iface.Remove(); err != nil {
+				logrus.Debugf("Remove interface %s failed: %v", iface.SrcName(), err)
+			}
 		}
 
 		for _, s := range n.subnets {
@@ -298,6 +302,26 @@ func (n *network) initSubnetSandbox(s *subnet) error {
 	return nil
 }
 
+func (n *network) cleanupStaleSandboxes() {
+	filepath.Walk(filepath.Dir(osl.GenerateKey("walk")),
+		func(path string, info os.FileInfo, err error) error {
+			_, fname := filepath.Split(path)
+
+			pList := strings.Split(fname, "-")
+			if len(pList) <= 1 {
+				return nil
+			}
+
+			pattern := pList[1]
+			if strings.Contains(n.id, pattern) {
+				syscall.Unmount(path, syscall.MNT_DETACH)
+				os.Remove(path)
+			}
+
+			return nil
+		})
+}
+
 func (n *network) initSandbox() error {
 	n.Lock()
 	n.initEpoch++
@@ -311,6 +335,10 @@ func (n *network) initSandbox() error {
 		}
 	}
 
+	// If there are any stale sandboxes related to this network
+	// from previous daemon life clean it up here
+	n.cleanupStaleSandboxes()
+
 	sbox, err := osl.NewSandbox(
 		osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
 	if err != nil {

endpoint.go

@@ -406,7 +406,7 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
 
 	ep.processOptions(options...)
 
-	driver, err := network.driver()
+	driver, err := network.driver(true)
 	if err != nil {
 		return fmt.Errorf("failed to join endpoint: %v", err)
 	}
@@ -465,7 +465,7 @@ func (ep *endpoint) sbJoin(sbox Sandbox, options ...EndpointOption) error {
 	if sb.needDefaultGW() {
 		return sb.setupDefaultGW(ep)
 	}
-	return sb.clearDefaultGW()
+	return nil
 }
 
 func (ep *endpoint) rename(name string) error {
@@ -533,10 +533,10 @@ func (ep *endpoint) Leave(sbox Sandbox, options ...EndpointOption) error {
 	sb.joinLeaveStart()
 	defer sb.joinLeaveEnd()
 
-	return ep.sbLeave(sbox, options...)
+	return ep.sbLeave(sbox, false, options...)
 }
 
-func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
+func (ep *endpoint) sbLeave(sbox Sandbox, force bool, options ...EndpointOption) error {
 	sb, ok := sbox.(*sandbox)
 	if !ok {
 		return types.BadRequestErrorf("not a valid Sandbox interface")
@@ -565,7 +565,7 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
 
 	ep.processOptions(options...)
 
-	d, err := n.driver()
+	d, err := n.driver(!force)
 	if err != nil {
 		return fmt.Errorf("failed to leave endpoint: %v", err)
 	}
@@ -575,9 +575,11 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
 	ep.network = n
 	ep.Unlock()
 
-	if err := d.Leave(n.id, ep.id); err != nil {
-		if _, ok := err.(types.MaskableError); !ok {
-			log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
+	if d != nil {
+		if err := d.Leave(n.id, ep.id); err != nil {
+			if _, ok := err.(types.MaskableError); !ok {
+				log.Warnf("driver error disconnecting container %s : %v", ep.name, err)
+			}
 		}
 	}
 
@@ -595,15 +597,7 @@ func (ep *endpoint) sbLeave(sbox Sandbox, options ...EndpointOption) error {
 	}
 
 	sb.deleteHostsEntries(n.getSvcRecords(ep))
-
-	if !sb.inDelete && sb.needDefaultGW() {
-		ep := sb.getEPwithoutGateway()
-		if ep == nil {
-			return fmt.Errorf("endpoint without GW expected, but not found")
-		}
-		return sb.setupDefaultGW(ep)
-	}
-	return sb.clearDefaultGW()
+	return nil
 }
 
 func (n *network) validateForceDelete(locator string) error {
@@ -649,7 +643,7 @@ func (ep *endpoint) Delete(force bool) error {
 	}
 
 	if sb != nil {
-		if e := ep.sbLeave(sb); e != nil {
+		if e := ep.sbLeave(sb, force); e != nil {
 			log.Warnf("failed to leave sandbox for endpoint %s : %v", name, e)
 		}
 	}
@@ -681,7 +675,7 @@ func (ep *endpoint) Delete(force bool) error {
 	// unwatch for service records
 	n.getController().unWatchSvcRecord(ep)
 
-	if err = ep.deleteEndpoint(); err != nil && !force {
+	if err = ep.deleteEndpoint(force); err != nil && !force {
 		return err
 	}
 
@@ -690,18 +684,22 @@ func (ep *endpoint) Delete(force bool) error {
 	return nil
 }
 
-func (ep *endpoint) deleteEndpoint() error {
+func (ep *endpoint) deleteEndpoint(force bool) error {
 	ep.Lock()
 	n := ep.network
 	name := ep.name
 	epid := ep.id
 	ep.Unlock()
 
-	driver, err := n.driver()
+	driver, err := n.driver(!force)
 	if err != nil {
 		return fmt.Errorf("failed to delete endpoint: %v", err)
 	}
 
+	if driver == nil {
+		return nil
+	}
+
 	if err := driver.DeleteEndpoint(n.id, epid); err != nil {
 		if _, ok := err.(types.ForbiddenError); ok {
 			return err
@@ -913,7 +911,7 @@ func (ep *endpoint) assignAddressVersion(ipVer int, ipam ipamapi.Ipam) error {
 		}
 	}
 	if progAdd != nil {
-		return types.BadRequestErrorf("Invalid preferred address %s: It does not belong to any of this network's subnets")
+		return types.BadRequestErrorf("Invalid preferred address %s: It does not belong to any of this network's subnets", prefAdd)
 	}
 	return fmt.Errorf("no available IPv%d addresses on this network's address pools: %s (%s)", ipVer, n.Name(), n.ID())
 }
@@ -956,7 +954,7 @@ func (c *controller) cleanupLocalEndpoints() {
 		}
 
 		for _, ep := range epl {
-			if err := ep.Delete(false); err != nil {
+			if err := ep.Delete(true); err != nil {
 				log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)
 			}
 		}

endpoint_info.go

@@ -188,7 +188,7 @@ func (ep *endpoint) DriverInfo() (map[string]interface{}, error) {
 		return nil, fmt.Errorf("could not find network in store for driver info: %v", err)
 	}
 
-	driver, err := n.driver()
+	driver, err := n.driver(true)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get driver info: %v", err)
 	}

ipam/allocator.go

@@ -145,12 +145,12 @@ func (a *Allocator) GetDefaultAddressSpaces() (string, string, error) {
 // RequestPool returns an address pool along with its unique id.
 func (a *Allocator) RequestPool(addressSpace, pool, subPool string, options map[string]string, v6 bool) (string, *net.IPNet, map[string]string, error) {
 	log.Debugf("RequestPool(%s, %s, %s, %v, %t)", addressSpace, pool, subPool, options, v6)
-	k, nw, ipr, err := a.parsePoolRequest(addressSpace, pool, subPool, v6)
+retry:
+	k, nw, ipr, pdf, err := a.parsePoolRequest(addressSpace, pool, subPool, v6)
 	if err != nil {
 		return "", nil, nil, types.InternalErrorf("failed to parse pool request for address space %q pool %q subpool %q: %v", addressSpace, pool, subPool, err)
 	}
 
-retry:
 	if err := a.refresh(addressSpace); err != nil {
 		return "", nil, nil, err
 	}
@@ -160,8 +160,12 @@ retry:
 		return "", nil, nil, err
 	}
 
-	insert, err := aSpace.updatePoolDBOnAdd(*k, nw, ipr)
+	insert, err := aSpace.updatePoolDBOnAdd(*k, nw, ipr, pdf)
 	if err != nil {
+		if _, ok := err.(types.MaskableError); ok {
+			log.Debugf("Retrying predefined pool search: %v", err)
+			goto retry
+		}
 		return "", nil, nil, err
 	}
 
@@ -221,38 +225,39 @@ func (a *Allocator) getAddrSpace(as string) (*addrSpace, error) {
 	return aSpace, nil
 }
 
-func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *AddressRange, error) {
+func (a *Allocator) parsePoolRequest(addressSpace, pool, subPool string, v6 bool) (*SubnetKey, *net.IPNet, *AddressRange, bool, error) {
 	var (
 		nw  *net.IPNet
 		ipr *AddressRange
 		err error
+		pdf = false
 	)
 
 	if addressSpace == "" {
-		return nil, nil, nil, ipamapi.ErrInvalidAddressSpace
+		return nil, nil, nil, false, ipamapi.ErrInvalidAddressSpace
 	}
 
 	if pool == "" && subPool != "" {
-		return nil, nil, nil, ipamapi.ErrInvalidSubPool
+		return nil, nil, nil, false, ipamapi.ErrInvalidSubPool
 	}
 
 	if pool != "" {
 		if _, nw, err = net.ParseCIDR(pool); err != nil {
-			return nil, nil, nil, ipamapi.ErrInvalidPool
+			return nil, nil, nil, false, ipamapi.ErrInvalidPool
 		}
 		if subPool != "" {
 			if ipr, err = getAddressRange(subPool, nw); err != nil {
-				return nil, nil, nil, err
+				return nil, nil, nil, false, err
 			}
 		}
 	} else {
 		if nw, err = a.getPredefinedPool(addressSpace, v6); err != nil {
-			return nil, nil, nil, err
+			return nil, nil, nil, false, err
 		}
-
+		pdf = true
 	}
 
-	return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, ipr, nil
+	return &SubnetKey{AddressSpace: addressSpace, Subnet: nw.String(), ChildSubnet: subPool}, nw, ipr, pdf, nil
 }
 
 func (a *Allocator) insertBitMask(key SubnetKey, pool *net.IPNet) error {

ipam/allocator_test.go

@@ -2,10 +2,12 @@ package ipam
 
 import (
 	"encoding/json"
+	"flag"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
 	"net"
+	"strconv"
 	"testing"
 	"time"
 
@@ -1168,3 +1170,99 @@ func checkDBEquality(a1, a2 *Allocator, t *testing.T) {
 		}
 	}
 }
+
+const (
+	numInstances = 5
+	first        = 0
+	last         = numInstances - 1
+)
+
+var (
+	allocator *Allocator
+	start     = make(chan struct{})
+	done      = make(chan chan struct{}, numInstances-1)
+	pools     = make([]*net.IPNet, numInstances)
+)
+
+func runParallelTests(t *testing.T, instance int) {
+	var err error
+
+	t.Parallel()
+
+	pTest := flag.Lookup("test.parallel")
+	if pTest == nil {
+		t.Skip("Skipped because test.parallel flag not set;")
+	}
+	numParallel, err := strconv.Atoi(pTest.Value.String())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if numParallel < numInstances {
+		t.Skip("Skipped because t.parallel was less than ", numInstances)
+	}
+
+	// The first instance creates the allocator, gives the start
+	// and finally checks the pools each instance was assigned
+	if instance == first {
+		allocator, err = getAllocator()
+		if err != nil {
+			t.Fatal(err)
+		}
+		close(start)
+	}
+
+	if instance != first {
+		select {
+		case <-start:
+		}
+
+		instDone := make(chan struct{})
+		done <- instDone
+		defer close(instDone)
+
+		if instance == last {
+			defer close(done)
+		}
+	}
+
+	_, pools[instance], _, err = allocator.RequestPool(localAddressSpace, "", "", nil, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if instance == first {
+		for instDone := range done {
+			select {
+			case <-instDone:
+			}
+		}
+		// Now check each instance got a different pool
+		for i := 0; i < numInstances; i++ {
+			for j := i + 1; j < numInstances; j++ {
+				if types.CompareIPNet(pools[i], pools[j]) {
+					t.Fatalf("Instance %d and %d were given the same predefined pool: %v", i, j, pools)
+				}
+			}
+		}
+	}
+}
+
+func TestParallelPredefinedRequest1(t *testing.T) {
+	runParallelTests(t, 0)
+}
+
+func TestParallelPredefinedRequest2(t *testing.T) {
+	runParallelTests(t, 1)
+}
+
+func TestParallelPredefinedRequest3(t *testing.T) {
+	runParallelTests(t, 2)
+}
+
+func TestParallelPredefinedRequest4(t *testing.T) {
+	runParallelTests(t, 3)
+}
+
+func TestParallelPredefinedRequest5(t *testing.T) {
+	runParallelTests(t, 4)
+}

ipam/structures.go

@@ -257,12 +257,15 @@ func (aSpace *addrSpace) New() datastore.KVObject {
 	}
 }
 
-func (aSpace *addrSpace) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange) (func() error, error) {
+func (aSpace *addrSpace) updatePoolDBOnAdd(k SubnetKey, nw *net.IPNet, ipr *AddressRange, pdf bool) (func() error, error) {
 	aSpace.Lock()
 	defer aSpace.Unlock()
 
 	// Check if already allocated
 	if p, ok := aSpace.subnets[k]; ok {
+		if pdf {
+			return nil, types.InternalMaskableErrorf("predefined pool %s is already reserved", nw)
+		}
 		aSpace.incRefCount(p, 1)
 		return func() error { return nil }, nil
 	}

iptables/iptables.go

@@ -325,9 +325,11 @@ func Raw(args ...string) ([]byte, error) {
 		if err == nil || !strings.Contains(err.Error(), "was not provided by any .service files") {
 			return output, err
 		}
-
 	}
+	return raw(args...)
+}
 
+func raw(args ...string) ([]byte, error) {
 	if err := initCheck(); err != nil {
 		return nil, err
 	}
@@ -362,6 +364,15 @@ func RawCombinedOutput(args ...string) error {
 	return nil
 }
 
+// RawCombinedOutputNative behave as RawCombinedOutput with the difference it
+// will always invoke `iptables` binary
+func RawCombinedOutputNative(args ...string) error {
+	if output, err := raw(args...); err != nil || len(output) != 0 {
+		return fmt.Errorf("%s (%v)", string(output), err)
+	}
+	return nil
+}
+
 // ExistChain checks if a chain exists
 func ExistChain(chain string, table Table) bool {
 	if _, err := Raw("-t", string(table), "-L", chain); err == nil {

libnetwork_test.go

@@ -1221,6 +1221,10 @@ func (f *fakeSandbox) ResolveIP(ip string) string {
 	return ""
 }
 
+func (f *fakeSandbox) Endpoints() []libnetwork.Endpoint {
+	return nil
+}
+
 func TestExternalKey(t *testing.T) {
 	externalKeyTest(t, false)
 }
@@ -1706,7 +1710,7 @@ func TestEnableIPv6(t *testing.T) {
 	}
 
 	tmpResolvConf := []byte("search pommesfrites.fr\nnameserver 12.34.56.78\nnameserver 2001:4860:4860::8888\n")
-	expectedResolvConf := []byte("search pommesfrites.fr\nnameserver 127.0.0.11\noptions ndots:0\n")
+	expectedResolvConf := []byte("search pommesfrites.fr\nnameserver 127.0.0.11\nnameserver 2001:4860:4860::8888\noptions ndots:0\n")
 	//take a copy of resolv.conf for restoring after test completes
 	resolvConfSystem, err := ioutil.ReadFile("/etc/resolv.conf")
 	if err != nil {

netutils/utils.go

@@ -14,6 +14,13 @@ import (
 	"github.com/docker/libnetwork/types"
 )
 
+// constants for the IP address type
+const (
+	IP = iota // IPv4 and IPv6
+	IPv4
+	IPv6
+)
+
 var (
 	// ErrNetworkOverlapsWithNameservers preformatted error
 	ErrNetworkOverlapsWithNameservers = errors.New("requested network overlaps with nameserver")

network.go

@@ -149,6 +149,7 @@ type network struct {
 	name         string
 	networkType  string
 	id           string
+	scope        string
 	ipamType     string
 	ipamOptions  map[string]string
 	addrSpace    string
@@ -246,6 +247,7 @@ func (n *network) New() datastore.KVObject {
 	return &network{
 		ctrlr:   n.ctrlr,
 		drvOnce: &sync.Once{},
+		scope:   n.scope,
 	}
 }
 
@@ -295,6 +297,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
 	dstN.name = n.name
 	dstN.id = n.id
 	dstN.networkType = n.networkType
+	dstN.scope = n.scope
 	dstN.ipamType = n.ipamType
 	dstN.enableIPv6 = n.enableIPv6
 	dstN.persist = n.persist
@@ -337,7 +340,7 @@ func (n *network) CopyTo(o datastore.KVObject) error {
 }
 
 func (n *network) DataScope() string {
-	return n.driverScope()
+	return n.Scope()
 }
 
 func (n *network) getEpCnt() *endpointCnt {
@@ -353,6 +356,7 @@ func (n *network) MarshalJSON() ([]byte, error) {
 	netMap["name"] = n.name
 	netMap["id"] = n.id
 	netMap["networkType"] = n.networkType
+	netMap["scope"] = n.scope
 	netMap["ipamType"] = n.ipamType
 	netMap["addrSpace"] = n.addrSpace
 	netMap["enableIPv6"] = n.enableIPv6
@@ -456,6 +460,9 @@ func (n *network) UnmarshalJSON(b []byte) (err error) {
 	if v, ok := netMap["internal"]; ok {
 		n.internal = v.(bool)
 	}
+	if s, ok := netMap["scope"]; ok {
+		n.scope = s.(string)
+	}
 	return nil
 }
 
@@ -566,7 +573,7 @@ func (n *network) driverScope() string {
 	return dd.capability.DataScope
 }
 
-func (n *network) driver() (driverapi.Driver, error) {
+func (n *network) driver(load bool) (driverapi.Driver, error) {
 	c := n.getController()
 
 	c.Lock()
@@ -574,14 +581,20 @@ func (n *network) driver() (driverapi.Driver, error) {
 	dd, ok := c.drivers[n.networkType]
 	c.Unlock()
 
-	if !ok {
+	if !ok && load {
 		var err error
 		dd, err = c.loadDriver(n.networkType)
 		if err != nil {
 			return nil, err
 		}
+	} else if !ok {
+		// dont fail if driver loading is not required
+		return nil, nil
 	}
 
+	n.Lock()
+	n.scope = dd.capability.DataScope
+	n.Unlock()
 	return dd.driver, nil
 }
 
@@ -631,7 +644,7 @@ func (n *network) Delete() error {
 }
 
 func (n *network) deleteNetwork() error {
-	d, err := n.driver()
+	d, err := n.driver(true)
 	if err != nil {
 		return fmt.Errorf("failed deleting network: %v", err)
 	}
@@ -651,7 +664,7 @@ func (n *network) deleteNetwork() error {
 }
 
 func (n *network) addEndpoint(ep *endpoint) error {
-	d, err := n.driver()
+	d, err := n.driver(true)
 	if err != nil {
 		return fmt.Errorf("failed to add endpoint: %v", err)
 	}
@@ -725,7 +738,7 @@ func (n *network) CreateEndpoint(name string, options ...EndpointOption) (Endpoi
 	}
 	defer func() {
 		if err != nil {
-			if e := ep.deleteEndpoint(); e != nil {
+			if e := ep.deleteEndpoint(false); e != nil {
 				log.Warnf("cleaning up endpoint failed %s : %v", name, e)
 			}
 		}
@@ -1169,7 +1182,9 @@ func (n *network) DriverOptions() map[string]string {
 }
 
 func (n *network) Scope() string {
-	return n.driverScope()
+	n.Lock()
+	defer n.Unlock()
+	return n.scope
 }
 
 func (n *network) IpamConfig() (string, map[string]string, []*IpamConf, []*IpamConf) {

osl/interface_linux.go

@@ -8,6 +8,7 @@ import (
 	"sync"
 	"syscall"
 
+	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/types"
 	"github.com/vishvananda/netlink"
 )
@@ -127,7 +128,7 @@ func (i *nwIface) Remove() error {
 
 		err = netlink.LinkSetName(iface, i.SrcName())
 		if err != nil {
-			fmt.Println("LinkSetName failed: ", err)
+			log.Debugf("LinkSetName failed for interface %s: %v", i.SrcName(), err)
 			return err
 		}
 
@@ -139,7 +140,7 @@ func (i *nwIface) Remove() error {
 		} else if !isDefault {
 			// Move the network interface to caller namespace.
 			if err := netlink.LinkSetNsFd(iface, callerFD); err != nil {
-				fmt.Println("LinkSetNsPid failed: ", err)
+				log.Debugf("LinkSetNsPid failed for interface %s: %v", i.SrcName(), err)
 				return err
 			}
 		}

resolvconf/resolvconf.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/resolvconf/dns"
 )
 
@@ -29,6 +30,8 @@ var (
 	localhostNSRegexp = regexp.MustCompile(`(?m)^nameserver\s+` + dns.IPLocalhost + `\s*\n*`)
 	nsIPv6Regexp      = regexp.MustCompile(`(?m)^nameserver\s+` + ipv6Address + `\s*\n*`)
 	nsRegexp          = regexp.MustCompile(`^\s*nameserver\s*((` + ipv4Address + `)|(` + ipv6Address + `))\s*$`)
+	nsIPv6Regexpmatch = regexp.MustCompile(`^\s*nameserver\s*((` + ipv6Address + `))\s*$`)
+	nsIPv4Regexpmatch = regexp.MustCompile(`^\s*nameserver\s*((` + ipv4Address + `))\s*$`)
 	searchRegexp      = regexp.MustCompile(`^\s*search\s*(([^\s]+\s*)*)$`)
 	optionsRegexp     = regexp.MustCompile(`^\s*options\s*(([^\s]+\s*)*)$`)
 )
@@ -119,7 +122,7 @@ func FilterResolvDNS(resolvConf []byte, ipv6Enabled bool) (*File, error) {
 	}
 	// if the resulting resolvConf has no more nameservers defined, add appropriate
 	// default DNS servers for IPv4 and (optionally) IPv6
-	if len(GetNameservers(cleanedResolvConf)) == 0 {
+	if len(GetNameservers(cleanedResolvConf, netutils.IP)) == 0 {
 		logrus.Infof("No non-localhost DNS nameservers are left in resolv.conf. Using default external servers : %v", defaultIPv4Dns)
 		dns := defaultIPv4Dns
 		if ipv6Enabled {
@@ -151,10 +154,17 @@ func getLines(input []byte, commentMarker []byte) [][]byte {
 }
 
 // GetNameservers returns nameservers (if any) listed in /etc/resolv.conf
-func GetNameservers(resolvConf []byte) []string {
+func GetNameservers(resolvConf []byte, kind int) []string {
 	nameservers := []string{}
 	for _, line := range getLines(resolvConf, []byte("#")) {
-		var ns = nsRegexp.FindSubmatch(line)
+		var ns [][]byte
+		if kind == netutils.IP {
+			ns = nsRegexp.FindSubmatch(line)
+		} else if kind == netutils.IPv4 {
+			ns = nsIPv4Regexpmatch.FindSubmatch(line)
+		} else if kind == netutils.IPv6 {
+			ns = nsIPv6Regexpmatch.FindSubmatch(line)
+		}
 		if len(ns) > 0 {
 			nameservers = append(nameservers, string(ns[1]))
 		}
@@ -167,7 +177,7 @@ func GetNameservers(resolvConf []byte) []string {
 // This function's output is intended for net.ParseCIDR
 func GetNameserversAsCIDR(resolvConf []byte) []string {
 	nameservers := []string{}
-	for _, nameserver := range GetNameservers(resolvConf) {
+	for _, nameserver := range GetNameservers(resolvConf, netutils.IP) {
 		nameservers = append(nameservers, nameserver+"/32")
 	}
 	return nameservers

resolvconf/resolvconf_test.go

@@ -7,6 +7,7 @@ import (
 	"testing"
 
 	"github.com/docker/docker/pkg/ioutils"
+	"github.com/docker/libnetwork/netutils"
 	_ "github.com/docker/libnetwork/testutils"
 )
 
@@ -48,7 +49,7 @@ nameserver 1.2.3.4
 		`search example.com
 nameserver 1.2.3.4 # not 4.3.2.1`: {"1.2.3.4"},
 	} {
-		test := GetNameservers([]byte(resolv))
+		test := GetNameservers([]byte(resolv), netutils.IP)
 		if !strSlicesEqual(test, result) {
 			t.Fatalf("Wrong nameserver string {%s} should be %v. Input: %s", test, result, resolv)
 		}

resolver.go

@@ -36,15 +36,18 @@ const (
 	ptrIPv4domain = ".in-addr.arpa."
 	ptrIPv6domain = ".ip6.arpa."
 	respTTL       = 1800
+	maxExtDNS     = 3 //max number of external servers to try
 )
 
 // resolver implements the Resolver interface
 type resolver struct {
-	sb     *sandbox
-	extDNS []string
-	server *dns.Server
-	conn   *net.UDPConn
-	err    error
+	sb        *sandbox
+	extDNS    []string
+	server    *dns.Server
+	conn      *net.UDPConn
+	tcpServer *dns.Server
+	tcpListen *net.TCPListener
+	err       error
 }
 
 // NewResolver creates a new instance of the Resolver
@@ -59,6 +62,7 @@ func (r *resolver) SetupFunc() func() {
 	return (func() {
 		var err error
 
+		// DNS operates primarily on UDP
 		addr := &net.UDPAddr{
 			IP: net.ParseIP(resolverIP),
 		}
@@ -71,13 +75,27 @@ func (r *resolver) SetupFunc() func() {
 		laddr := r.conn.LocalAddr()
 		_, ipPort, _ := net.SplitHostPort(laddr.String())
 
+		// Listen on a TCP as well
+		tcpaddr := &net.TCPAddr{
+			IP: net.ParseIP(resolverIP),
+		}
+
+		r.tcpListen, err = net.ListenTCP("tcp", tcpaddr)
+		if err != nil {
+			r.err = fmt.Errorf("error in opening name TCP server socket %v", err)
+			return
+		}
+		ltcpaddr := r.tcpListen.Addr()
+		_, tcpPort, _ := net.SplitHostPort(ltcpaddr.String())
 		rules := [][]string{
 			{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", laddr.String()},
 			{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+			{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", ltcpaddr.String()},
+			{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
 		}
 
 		for _, rule := range rules {
-			r.err = iptables.RawCombinedOutput(rule...)
+			r.err = iptables.RawCombinedOutputNative(rule...)
 			if r.err != nil {
 				return
 			}
@@ -96,6 +114,12 @@ func (r *resolver) Start() error {
 	go func() {
 		s.ActivateAndServe()
 	}()
+
+	tcpServer := &dns.Server{Handler: r, Listener: r.tcpListen}
+	r.tcpServer = tcpServer
+	go func() {
+		tcpServer.ActivateAndServe()
+	}()
 	return nil
 }
 
@@ -103,7 +127,11 @@ func (r *resolver) Stop() {
 	if r.server != nil {
 		r.server.Shutdown()
 	}
+	if r.tcpServer != nil {
+		r.tcpServer.Shutdown()
+	}
 	r.conn = nil
+	r.tcpServer = nil
 	r.err = fmt.Errorf("setup not done yet")
 }
 
@@ -172,6 +200,9 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
 		err  error
 	)
 
+	if query == nil || len(query.Question) == 0 {
+		return
+	}
 	name := query.Question[0].Name
 	if query.Question[0].Qtype == dns.TypeA {
 		resp, err = r.handleIPv4Query(name, query)
@@ -188,15 +219,25 @@ func (r *resolver) ServeDNS(w dns.ResponseWriter, query *dns.Msg) {
 		if len(r.extDNS) == 0 {
 			return
 		}
-		log.Debugf("Querying ext dns %s for %s[%d]", r.extDNS[0], name, query.Question[0].Qtype)
 
-		c := &dns.Client{Net: "udp"}
-		addr := fmt.Sprintf("%s:%d", r.extDNS[0], 53)
+		num := maxExtDNS
+		if len(r.extDNS) < maxExtDNS {
+			num = len(r.extDNS)
+		}
+		for i := 0; i < num; i++ {
+			log.Debugf("Querying ext dns %s:%s for %s[%d]", w.LocalAddr().Network(), r.extDNS[i], name, query.Question[0].Qtype)
 
-		// TODO: iterate over avilable servers in case of error
-		resp, _, err = c.Exchange(query, addr)
-		if err != nil {
+			c := &dns.Client{Net: w.LocalAddr().Network()}
+			addr := fmt.Sprintf("%s:%d", r.extDNS[i], 53)
+
+			resp, _, err = c.Exchange(query, addr)
+			if err == nil {
+				resp.Compress = true
+				break
+			}
 			log.Errorf("external resolution failed, %s", err)
+		}
+		if resp == nil {
 			return
 		}
 	}

sandbox.go

@@ -14,6 +14,7 @@ import (
 
 	log "github.com/Sirupsen/logrus"
 	"github.com/docker/libnetwork/etchosts"
+	"github.com/docker/libnetwork/netutils"
 	"github.com/docker/libnetwork/osl"
 	"github.com/docker/libnetwork/resolvconf"
 	"github.com/docker/libnetwork/types"
@@ -46,6 +47,8 @@ type Sandbox interface {
 	// ResolveIP returns the service name for the passed in IP. IP is in reverse dotted
 	// notation; the format used for DNS PTR records
 	ResolveIP(name string) string
+	// Endpoints returns all the endpoints connected to the sandbox
+	Endpoints() []Endpoint
 }
 
 // SandboxOption is a option setter function type used to pass varios options to
@@ -160,6 +163,10 @@ func (sb *sandbox) Statistics() (map[string]*types.InterfaceStatistics, error) {
 }
 
 func (sb *sandbox) Delete() error {
+	return sb.delete(false)
+}
+
+func (sb *sandbox) delete(force bool) error {
 	sb.Lock()
 	if sb.inDelete {
 		sb.Unlock()
@@ -181,12 +188,6 @@ func (sb *sandbox) Delete() error {
 	// Detach from all endpoints
 	retain := false
 	for _, ep := range sb.getConnectedEndpoints() {
-		// endpoint in the Gateway network will be cleaned up
-		// when when sandbox no longer needs external connectivity
-		if ep.endpointInGWNetwork() {
-			continue
-		}
-
 		// Retain the sanbdox if we can't obtain the network from store.
 		if _, err := c.getNetworkFromStore(ep.getNetwork().ID()); err != nil {
 			retain = true
@@ -194,11 +195,13 @@ func (sb *sandbox) Delete() error {
 			continue
 		}
 
-		if err := ep.Leave(sb); err != nil {
-			log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
+		if !force {
+			if err := ep.Leave(sb); err != nil {
+				log.Warnf("Failed detaching sandbox %s from endpoint %s: %v\n", sb.ID(), ep.ID(), err)
+			}
 		}
 
-		if err := ep.Delete(false); err != nil {
+		if err := ep.Delete(force); err != nil {
 			log.Warnf("Failed deleting endpoint %s: %v\n", ep.ID(), err)
 		}
 	}
@@ -316,11 +319,15 @@ func (sb *sandbox) startResolver() {
 			}
 		}()
 
-		sb.rebuildDNS()
+		err = sb.rebuildDNS()
+		if err != nil {
+			log.Errorf("Updating resolv.conf failed for container %s, %q", sb.ContainerID(), err)
+			return
+		}
 		sb.resolver.SetExtServers(sb.extDNS)
 
 		sb.osSbox.InvokeFunc(sb.resolver.SetupFunc())
-		if err := sb.resolver.Start(); err != nil {
+		if err = sb.resolver.Start(); err != nil {
 			log.Errorf("Resolver Setup/Start failed for container %s, %q", sb.ContainerID(), err)
 		}
 	})
@@ -342,6 +349,17 @@ func (sb *sandbox) setupResolutionFiles() error {
 	return nil
 }
 
+func (sb *sandbox) Endpoints() []Endpoint {
+	sb.Lock()
+	defer sb.Unlock()
+
+	endpoints := make([]Endpoint, len(sb.endpoints))
+	for i, ep := range sb.endpoints {
+		endpoints[i] = ep
+	}
+	return endpoints
+}
+
 func (sb *sandbox) getConnectedEndpoints() []*endpoint {
 	sb.Lock()
 	defer sb.Unlock()
@@ -421,23 +439,51 @@ func (sb *sandbox) ResolveIP(ip string) string {
 
 func (sb *sandbox) ResolveName(name string) net.IP {
 	var ip net.IP
-	parts := strings.Split(name, ".")
-	log.Debugf("To resolve %v", parts)
 
-	reqName := parts[0]
-	networkName := ""
-	if len(parts) > 1 {
-		networkName = parts[1]
+	// Embedded server owns the docker network domain. Resolution should work
+	// for both container_name and container_name.network_name
+	// We allow '.' in service name and network name. For a name a.b.c.d the
+	// following have to tried;
+	// {a.b.c.d in the networks container is connected to}
+	// {a.b.c in network d},
+	// {a.b in network c.d},
+	// {a in network b.c.d},
+
+	name = strings.TrimSuffix(name, ".")
+	reqName := []string{name}
+	networkName := []string{""}
+
+	if strings.Contains(name, ".") {
+		var i int
+		dup := name
+		for {
+			if i = strings.LastIndex(dup, "."); i == -1 {
+				break
+			}
+			networkName = append(networkName, name[i+1:])
+			reqName = append(reqName, name[:i])
+
+			dup = dup[:i]
+		}
 	}
+
 	epList := sb.getConnectedEndpoints()
-	// First check for local container alias
-	ip = sb.resolveName(reqName, networkName, epList, true)
-	if ip != nil {
-		return ip
-	}
+	for i := 0; i < len(reqName); i++ {
+		log.Debugf("To resolve: %v in %v", reqName[i], networkName[i])
 
-	// Resolve the actual container name
-	return sb.resolveName(reqName, networkName, epList, false)
+		// First check for local container alias
+		ip = sb.resolveName(reqName[i], networkName[i], epList, true)
+		if ip != nil {
+			return ip
+		}
+
+		// Resolve the actual container name
+		ip = sb.resolveName(reqName[i], networkName[i], epList, false)
+		if ip != nil {
+			return ip
+		}
+	}
+	return nil
 }
 
 func (sb *sandbox) resolveName(req string, networkName string, epList []*endpoint, alias bool) net.IP {
@@ -543,7 +589,7 @@ func releaseOSSboxResources(osSbox osl.Sandbox, ep *endpoint) {
 		// Only remove the interfaces owned by this endpoint from the sandbox.
 		if ep.hasInterface(i.SrcName()) {
 			if err := i.Remove(); err != nil {
-				log.Debugf("Remove interface failed: %v", err)
+				log.Debugf("Remove interface %s failed: %v", i.SrcName(), err)
 			}
 		}
 	}
@@ -817,7 +863,7 @@ func (sb *sandbox) setupDNS() error {
 	if len(sb.config.dnsList) > 0 || len(sb.config.dnsSearchList) > 0 || len(sb.config.dnsOptionsList) > 0 {
 		var (
 			err            error
-			dnsList        = resolvconf.GetNameservers(currRC.Content)
+			dnsList        = resolvconf.GetNameservers(currRC.Content, netutils.IP)
 			dnsSearchList  = resolvconf.GetSearchDomains(currRC.Content)
 			dnsOptionsList = resolvconf.GetOptions(currRC.Content)
 		)
@@ -859,6 +905,11 @@ func (sb *sandbox) updateDNS(ipv6Enabled bool) error {
 		hashFile = sb.config.resolvConfHashFile
 	)
 
+	// This is for the host mode networking
+	if sb.config.originResolvConfPath != "" {
+		return nil
+	}
+
 	if len(sb.config.dnsList) > 0 || len(sb.config.dnsSearchList) > 0 || len(sb.config.dnsOptionsList) > 0 {
 		return nil
 	}
@@ -891,36 +942,21 @@ func (sb *sandbox) updateDNS(ipv6Enabled bool) error {
 	if err != nil {
 		return err
 	}
+	err = ioutil.WriteFile(sb.config.resolvConfPath, newRC.Content, 0644)
+	if err != nil {
+		return err
+	}
 
-	// for atomic updates to these files, use temporary files with os.Rename:
+	// write the new hash in a temp file and rename it to make the update atomic
 	dir := path.Dir(sb.config.resolvConfPath)
 	tmpHashFile, err := ioutil.TempFile(dir, "hash")
 	if err != nil {
 		return err
 	}
-	tmpResolvFile, err := ioutil.TempFile(dir, "resolv")
-	if err != nil {
-		return err
-	}
-
-	// Change the perms to filePerm (0644) since ioutil.TempFile creates it by default as 0600
-	if err := os.Chmod(tmpResolvFile.Name(), filePerm); err != nil {
-		return err
-	}
-
-	// write the updates to the temp files
 	if err = ioutil.WriteFile(tmpHashFile.Name(), []byte(newRC.Hash), filePerm); err != nil {
 		return err
 	}
-	if err = ioutil.WriteFile(tmpResolvFile.Name(), newRC.Content, filePerm); err != nil {
-		return err
-	}
-
-	// rename the temp files for atomic replace
-	if err = os.Rename(tmpHashFile.Name(), hashFile); err != nil {
-		return err
-	}
-	return os.Rename(tmpResolvFile.Name(), sb.config.resolvConfPath)
+	return os.Rename(tmpHashFile.Name(), hashFile)
 }
 
 // Embedded DNS server has to be enabled for this sandbox. Rebuild the container's
@@ -935,7 +971,8 @@ func (sb *sandbox) rebuildDNS() error {
 	}
 
 	// localhost entries have already been filtered out from the list
-	sb.extDNS = resolvconf.GetNameservers(currRC.Content)
+	// retain only the v4 servers in sb for forwarding the DNS queries
+	sb.extDNS = resolvconf.GetNameservers(currRC.Content, netutils.IPv4)
 
 	var (
 		dnsList        = []string{sb.resolver.NameServer()}
@@ -943,26 +980,14 @@ func (sb *sandbox) rebuildDNS() error {
 		dnsSearchList  = resolvconf.GetSearchDomains(currRC.Content)
 	)
 
+	// external v6 DNS servers has to be listed in resolv.conf
+	dnsList = append(dnsList, resolvconf.GetNameservers(currRC.Content, netutils.IPv6)...)
+
 	// Resolver returns the options in the format resolv.conf expects
 	dnsOptionsList = append(dnsOptionsList, sb.resolver.ResolverOptions()...)
 
-	dir := path.Dir(sb.config.resolvConfPath)
-	tmpResolvFile, err := ioutil.TempFile(dir, "resolv")
-	if err != nil {
-		return err
-	}
-
-	// Change the perms to filePerm (0644) since ioutil.TempFile creates it by default as 0600
-	if err := os.Chmod(tmpResolvFile.Name(), filePerm); err != nil {
-		return err
-	}
-
-	_, err = resolvconf.Build(tmpResolvFile.Name(), dnsList, dnsSearchList, dnsOptionsList)
-	if err != nil {
-		return err
-	}
-
-	return os.Rename(tmpResolvFile.Name(), sb.config.resolvConfPath)
+	_, err = resolvconf.Build(sb.config.resolvConfPath, dnsList, dnsSearchList, dnsOptionsList)
+	return err
 }
 
 // joinLeaveStart waits to ensure there are no joins or leaves in progress and

sandbox_store.go

@@ -226,7 +226,7 @@ func (c *controller) sandboxCleanup() {
 			heap.Push(&sb.endpoints, ep)
 		}
 
-		if err := sb.Delete(); err != nil {
+		if err := sb.delete(true); err != nil {
 			logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
 		}
 	}

sandbox_test.go

@@ -119,6 +119,10 @@ func TestSandboxAddMultiPrio(t *testing.T) {
 		t.Fatal("Expected ep3 to be at the top of the heap. But did not find ep3 at the top of the heap")
 	}
 
+	if len(sbx.Endpoints()) != 3 {
+		t.Fatal("Expected 3 endpoints to be connected to the sandbox.")
+	}
+
 	if err := ep3.Leave(sbx); err != nil {
 		t.Fatal(err)
 	}

store.go

@@ -75,6 +75,7 @@ func (c *controller) getNetworkFromStore(nid string) (*network, error) {
 		}
 
 		n.epCnt = ec
+		n.scope = store.Scope()
 		return n, nil
 	}
 
@@ -103,10 +104,12 @@ func (c *controller) getNetworksForScope(scope string) ([]*network, error) {
 		ec := &endpointCnt{n: n}
 		err = store.GetObject(datastore.Key(ec.Key()...), ec)
 		if err != nil {
-			return nil, fmt.Errorf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
+			log.Warnf("Could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
+			continue
 		}
 
 		n.epCnt = ec
+		n.scope = scope
 		nl = append(nl, n)
 	}
 
@@ -136,10 +139,12 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
 			ec := &endpointCnt{n: n}
 			err = store.GetObject(datastore.Key(ec.Key()...), ec)
 			if err != nil {
-				return nil, fmt.Errorf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
+				log.Warnf("could not find endpoint count key %s for network %s while listing: %v", datastore.Key(ec.Key()...), n.Name(), err)
+				continue
 			}
 
 			n.epCnt = ec
+			n.scope = store.Scope()
 			nl = append(nl, n)
 		}
 	}
@@ -148,17 +153,21 @@ func (c *controller) getNetworksFromStore() ([]*network, error) {
 }
 
 func (n *network) getEndpointFromStore(eid string) (*endpoint, error) {
-	store := n.ctrlr.getStore(n.Scope())
-	if store == nil {
-		return nil, fmt.Errorf("could not find endpoint %s: datastore not found for scope %s", eid, n.Scope())
+	var errors []string
+	for _, store := range n.ctrlr.getStores() {
+		ep := &endpoint{id: eid, network: n}
+		err := store.GetObject(datastore.Key(ep.Key()...), ep)
+		// Continue searching in the next store if the key is not found in this store
+		if err != nil {
+			if err != datastore.ErrKeyNotFound {
+				errors = append(errors, fmt.Sprintf("{%s:%v}, ", store.Scope(), err))
+				log.Debugf("could not find endpoint %s in %s: %v", eid, store.Scope(), err)
+			}
+			continue
+		}
+		return ep, nil
 	}
-
-	ep := &endpoint{id: eid, network: n}
-	err := store.GetObject(datastore.Key(ep.Key()...), ep)
-	if err != nil {
-		return nil, fmt.Errorf("could not find endpoint %s: %v", eid, err)
-	}
-	return ep, nil
+	return nil, fmt.Errorf("could not find endpoint %s: %v", eid, errors)
 }
 
 func (n *network) getEndpointsFromStore() ([]*endpoint, error) {

test/integration/dnet/helpers.bash

@@ -322,6 +322,42 @@ function test_overlay() {
 	done
     done
 
+    # Setup bridge network and connect containers ot it
+    if [ -z "${2}" -o "${2}" != "skip_add" ]; then
+        if [ -z "${2}" -o "${2}" != "internal" ]; then
+	    dnet_cmd $(inst_id2port 1) network create -d bridge br1
+	    dnet_cmd $(inst_id2port 1) network create -d bridge br2
+	    net_connect ${start} container_${start} br1
+	    net_connect ${start} container_${start} br2
+
+            # Make sure external connectivity works
+	    runc $(dnet_container_name ${start} $dnet_suffix) $(get_sbox_id ${start} container_${start}) \
+	        "ping -c 1 www.google.com"
+	    net_disconnect ${start} container_${start} br1
+	    net_disconnect ${start} container_${start} br2
+
+            # Make sure external connectivity works
+	    runc $(dnet_container_name ${start} $dnet_suffix) $(get_sbox_id ${start} container_${start}) \
+	        "ping -c 1 www.google.com"
+	    dnet_cmd $(inst_id2port 1) network rm br1
+	    dnet_cmd $(inst_id2port 1) network rm br2
+
+            # Disconnect from overlay network
+            net_disconnect ${start} container_${start} multihost
+
+            # Make sure external connectivity works
+            runc $(dnet_container_name ${start} $dnet_suffix) $(get_sbox_id ${start} container_${start}) \
+                "ping -c 1 www.google.com"
+
+            # Connect to overlay network again
+            net_connect ${start} container_${start} multihost
+
+            # Make sure external connectivity still works
+            runc $(dnet_container_name ${start} $dnet_suffix) $(get_sbox_id ${start} container_${start}) \
+                "ping -c 1 www.google.com"
+	fi
+    fi
+
     # Teardown the container connections and the network
     for i in `seq ${start} ${end}`;
     do