Add ConnectivityScope capability for network drivers along with scope network option

- It specifies whether the network driver can
  provide containers connectivity across hosts.
- As of now, the data scope of the driver was
  being overloaded with this notion.
- The driver scope information is still valid
  and it defines whether the data allocation
  of the network resources can be done globally
  or only locally.
- With the scope network option, user can now
  force a network as swarm scoped
  regardless of the driver data scope.
- In case the network is configured as swarm scoped,
  and the network driver is multihost capable,
  a network DB instance will be launched for it.

Signed-off-by: Alessandro Boch <aboch@docker.com>

agent.go

@@ -222,7 +222,7 @@ func (c *controller) agentSetup(clusterProvider cluster.Provider) error {
 			return err
 		}
 		c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool {
-			if capability.DataScope == datastore.GlobalScope {
+			if capability.ConnectivityScope == datastore.GlobalScope {
 				c.agentDriverNotify(driver)
 			}
 			return false
@@ -507,7 +507,7 @@ func (n *network) Services() map[string]ServiceInfo {
 }
 
 func (n *network) isClusterEligible() bool {
-	if n.driverScope() != datastore.GlobalScope {
+	if n.scope != datastore.SwarmScope || !n.driverIsMultihost() {
 		return false
 	}
 	return n.getController().getAgent() != nil

controller.go

@@ -621,7 +621,7 @@ func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capabil
 		}
 	}
 
-	if d == nil || cap.DataScope != datastore.GlobalScope || nodes == nil {
+	if d == nil || cap.ConnectivityScope != datastore.GlobalScope || nodes == nil {
 		return
 	}
 
@@ -747,11 +747,17 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ...
 		return nil, err
 	}
 
+	if network.scope == datastore.LocalScope && cap.DataScope == datastore.GlobalScope {
+		return nil, types.ForbiddenErrorf("cannot downgrade network scope for %s networks", networkType)
+
+	}
 	if network.ingress && cap.DataScope != datastore.GlobalScope {
 		return nil, types.ForbiddenErrorf("Ingress network can only be global scope network")
 	}
 
-	if cap.DataScope == datastore.GlobalScope && !c.isDistributedControl() && !network.dynamic {
+	// At this point the network scope is still unknown if not set by user
+	if (cap.DataScope == datastore.GlobalScope || network.scope == datastore.SwarmScope) &&
+		!c.isDistributedControl() && !network.dynamic {
 		if c.isManager() {
 			// For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager
 			return nil, ManagerRedirectError(name)

datastore/datastore.go

@@ -115,7 +115,10 @@ const (
 	// LocalScope indicates to store the KV object in local datastore such as boltdb
 	LocalScope = "local"
 	// GlobalScope indicates to store the KV object in global datastore such as consul/etcd/zookeeper
-	GlobalScope   = "global"
+	GlobalScope = "global"
+	// SwarmScope is not indicating a datastore location. It is defined here
+	// along with the other two scopes just for consistency.
+	SwarmScope    = "swarm"
 	defaultPrefix = "/var/lib/docker/network/files"
 )
 

docs/remote.md

@@ -56,10 +56,12 @@ Other entries in the list value are allowed; `"NetworkDriver"` indicates that th
 After Handshake, the remote driver will receive another POST message to the URL `/NetworkDriver.GetCapabilities` with no payload. The driver's response should have the form:
 
 	{
-		"Scope": "local"
+		"Scope":             "local"
+		"ConnectivityScope": "global"
 	}
 
-Value of "Scope" should be either "local" or "global" which indicates the capability of remote driver, values beyond these will fail driver's registration and return an error to the caller.
+Value of "Scope" should be either "local" or "global" which indicates whether the resource allocations for this driver's network can be done only locally to the node or globally across the cluster of nodes. Any other value will fail driver's registration and return an error to the caller.
+Similarly, value of "ConnectivityScope" should be either "local" or "global" which indicates whether the driver's network can provide connectivity only locally to this node or globally across the cluster of nodes. If the value is missing, libnetwork will set it to the value of "Scope". should be either "local" or "global" which indicates
 
 ### Create network
 

driverapi/driverapi.go

@@ -161,7 +161,8 @@ type DriverCallback interface {
 
 // Capability represents the high level capabilities of the drivers which libnetwork can make use of
 type Capability struct {
-	DataScope string
+	DataScope         string
+	ConnectivityScope string
 }
 
 // IPAMData represents the per-network ip related

drivers/bridge/bridge.go

@@ -153,7 +153,8 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	}
 
 	c := driverapi.Capability{
-		DataScope: datastore.LocalScope,
+		DataScope:         datastore.LocalScope,
+		ConnectivityScope: datastore.LocalScope,
 	}
 	return dc.RegisterDriver(networkType, d, c)
 }

drivers/host/host.go

@@ -19,7 +19,8 @@ type driver struct {
 // Init registers a new instance of host driver
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	c := driverapi.Capability{
-		DataScope: datastore.LocalScope,
+		DataScope:         datastore.LocalScope,
+		ConnectivityScope: datastore.LocalScope,
 	}
 	return dc.RegisterDriver(networkType, &driver{}, c)
 }

drivers/ipvlan/ipvlan.go

@@ -58,7 +58,8 @@ type network struct {
 // Init initializes and registers the libnetwork ipvlan driver
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	c := driverapi.Capability{
-		DataScope: datastore.LocalScope,
+		DataScope:         datastore.LocalScope,
+		ConnectivityScope: datastore.GlobalScope,
 	}
 	d := &driver{
 		networks: networkTable{},

drivers/macvlan/macvlan.go

@@ -60,7 +60,8 @@ type network struct {
 // Init initializes and registers the libnetwork macvlan driver
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	c := driverapi.Capability{
-		DataScope: datastore.LocalScope,
+		DataScope:         datastore.LocalScope,
+		ConnectivityScope: datastore.GlobalScope,
 	}
 	d := &driver{
 		networks: networkTable{},

drivers/overlay/overlay.go

@@ -56,7 +56,8 @@ type driver struct {
 // Init registers a new instance of overlay driver
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	c := driverapi.Capability{
-		DataScope: datastore.GlobalScope,
+		DataScope:         datastore.GlobalScope,
+		ConnectivityScope: datastore.GlobalScope,
 	}
 	d := &driver{
 		networks: networkTable{},

drivers/overlay/ovmanager/ovmanager.go

@@ -49,7 +49,8 @@ type network struct {
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	var err error
 	c := driverapi.Capability{
-		DataScope: datastore.GlobalScope,
+		DataScope:         datastore.GlobalScope,
+		ConnectivityScope: datastore.GlobalScope,
 	}
 
 	d := &driver{

drivers/remote/api/api.go

@@ -24,7 +24,8 @@ func (r *Response) GetError() string {
 // GetCapabilityResponse is the response of GetCapability request
 type GetCapabilityResponse struct {
 	Response
-	Scope string
+	Scope             string
+	ConnectivityScope string
 }
 
 // AllocateNetworkRequest requests allocation of new network by manager

drivers/remote/driver.go

@@ -74,6 +74,17 @@ func (d *driver) getCapabilities() (*driverapi.Capability, error) {
 		return nil, fmt.Errorf("invalid capability: expecting 'local' or 'global', got %s", capResp.Scope)
 	}
 
+	switch capResp.ConnectivityScope {
+	case "global":
+		c.ConnectivityScope = datastore.GlobalScope
+	case "local":
+		c.ConnectivityScope = datastore.LocalScope
+	case "":
+		c.ConnectivityScope = c.DataScope
+	default:
+		return nil, fmt.Errorf("invalid capability: expecting 'local' or 'global', got %s", capResp.Scope)
+	}
+
 	return c, nil
 }
 

drivers/remote/driver_test.go

@@ -238,8 +238,9 @@ func TestGetExtraCapabilities(t *testing.T) {
 
 	handle(t, mux, "GetCapabilities", func(msg map[string]interface{}) interface{} {
 		return map[string]interface{}{
-			"Scope": "local",
-			"foo":   "bar",
+			"Scope":             "local",
+			"foo":               "bar",
+			"ConnectivityScope": "global",
 		}
 	})
 
@@ -258,6 +259,8 @@ func TestGetExtraCapabilities(t *testing.T) {
 		t.Fatal(err)
 	} else if c.DataScope != datastore.LocalScope {
 		t.Fatalf("get capability '%s', expecting 'local'", c.DataScope)
+	} else if c.ConnectivityScope != datastore.GlobalScope {
+		t.Fatalf("get capability '%s', expecting %q", c.ConnectivityScope, datastore.GlobalScope)
 	}
 }
 

drivers/solaris/bridge/bridge.go

@@ -159,7 +159,8 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	}
 
 	c := driverapi.Capability{
-		DataScope: datastore.LocalScope,
+		DataScope:         datastore.LocalScope,
+		ConnectivityScope: datastore.LocalScope,
 	}
 	return dc.RegisterDriver(networkType, d, c)
 }

drivers/solaris/overlay/overlay.go

@@ -57,7 +57,8 @@ type driver struct {
 // Init registers a new instance of overlay driver
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	c := driverapi.Capability{
-		DataScope: datastore.GlobalScope,
+		DataScope:         datastore.GlobalScope,
+		ConnectivityScope: datastore.GlobalScope,
 	}
 	d := &driver{
 		networks: networkTable{},

drivers/solaris/overlay/ovmanager/ovmanager.go

@@ -49,7 +49,8 @@ type network struct {
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	var err error
 	c := driverapi.Capability{
-		DataScope: datastore.GlobalScope,
+		DataScope:         datastore.GlobalScope,
+		ConnectivityScope: datastore.GlobalScope,
 	}
 
 	d := &driver{

drivers/windows/overlay/overlay_windows.go

@@ -36,7 +36,8 @@ type driver struct {
 // Init registers a new instance of overlay driver
 func Init(dc driverapi.DriverCallback, config map[string]interface{}) error {
 	c := driverapi.Capability{
-		DataScope: datastore.GlobalScope,
+		DataScope:         datastore.GlobalScope,
+		ConnectivityScope: datastore.GlobalScope,
 	}
 
 	d := &driver{

drivers/windows/windows.go

@@ -104,7 +104,8 @@ func GetInit(networkType string) func(dc driverapi.DriverCallback, config map[st
 		}
 
 		return dc.RegisterDriver(networkType, newDriver(networkType), driverapi.Capability{
-			DataScope: datastore.LocalScope,
+			DataScope:         datastore.LocalScope,
+			ConnectivityScope: datastore.LocalScope,
 		})
 	}
 }

network.go

@@ -195,7 +195,7 @@ type network struct {
 	networkType  string
 	id           string
 	created      time.Time
-	scope        string
+	scope        string // network data scope
 	labels       map[string]string
 	ipamType     string
 	ipamOptions  map[string]string
@@ -514,7 +514,12 @@ func (n *network) CopyTo(o datastore.KVObject) error {
 }
 
 func (n *network) DataScope() string {
-	return n.Scope()
+	s := n.Scope()
+	// All swarm scope networks have local datascope
+	if s == datastore.SwarmScope {
+		s = datastore.LocalScope
+	}
+	return s
 }
 
 func (n *network) getEpCnt() *endpointCnt {
@@ -762,6 +767,14 @@ func NetworkOptionAttachable(attachable bool) NetworkOption {
 	}
 }
 
+// NetworkOptionScope returns an option setter to overwrite the network's scope.
+// By default the network's scope is set to the network driver's datascope.
+func NetworkOptionScope(scope string) NetworkOption {
+	return func(n *network) {
+		n.scope = scope
+	}
+}
+
 // NetworkOptionIpam function returns an option setter for the ipam configuration for this network
 func NetworkOptionIpam(ipamDriver string, addrSpace string, ipV4 []*IpamConf, ipV6 []*IpamConf, opts map[string]string) NetworkOption {
 	return func(n *network) {
@@ -877,6 +890,14 @@ func (n *network) driverScope() string {
 	return cap.DataScope
 }
 
+func (n *network) driverIsMultihost() bool {
+	_, cap, err := n.resolveDriver(n.networkType, true)
+	if err != nil {
+		return false
+	}
+	return cap.ConnectivityScope == datastore.GlobalScope
+}
+
 func (n *network) driver(load bool) (driverapi.Driver, error) {
 	d, cap, err := n.resolveDriver(n.networkType, load)
 	if err != nil {
@@ -887,14 +908,14 @@ func (n *network) driver(load bool) (driverapi.Driver, error) {
 	isAgent := c.isAgent()
 	n.Lock()
 	// If load is not required, driver, cap and err may all be nil
-	if cap != nil {
+	if n.scope == "" && cap != nil {
 		n.scope = cap.DataScope
 	}
-	if isAgent || n.dynamic {
-		// If we are running in agent mode then all networks
-		// in libnetwork are local scope regardless of the
-		// backing driver.
-		n.scope = datastore.LocalScope
+	if isAgent && n.dynamic {
+		// If we are running in agent mode and the network
+		// is dynamic, then the networks are swarm scoped
+		// regardless of the backing driver.
+		n.scope = datastore.SwarmScope
 	}
 	n.Unlock()
 	return d, nil

store.go

@@ -350,17 +350,18 @@ func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, ecCh <-chan da
 }
 
 func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) {
-	if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope {
+	n := ep.getNetwork()
+	if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() {
 		return
 	}
 
 	c.Lock()
-	nw, ok := nmap[ep.getNetwork().ID()]
+	nw, ok := nmap[n.ID()]
 	c.Unlock()
 
 	if ok {
 		// Update the svc db for the local endpoint join right away
-		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)
+		n.updateSvcRecord(ep, c.getLocalEps(nw), true)
 
 		c.Lock()
 		nw.localEps[ep.ID()] = ep
@@ -381,15 +382,15 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi
 	// Update the svc db for the local endpoint join right away
 	// Do this before adding this ep to localEps so that we don't
 	// try to update this ep's container's svc records
-	ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true)
+	n.updateSvcRecord(ep, c.getLocalEps(nw), true)
 
 	c.Lock()
 	nw.localEps[ep.ID()] = ep
-	nmap[ep.getNetwork().ID()] = nw
+	nmap[n.ID()] = nw
 	nw.stopCh = make(chan struct{})
 	c.Unlock()
 
-	store := c.getStore(ep.getNetwork().DataScope())
+	store := c.getStore(n.DataScope())
 	if store == nil {
 		return
 	}
@@ -398,7 +399,7 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi
 		return
 	}
 
-	ch, err := store.Watch(ep.getNetwork().getEpCnt(), nw.stopCh)
+	ch, err := store.Watch(n.getEpCnt(), nw.stopCh)
 	if err != nil {
 		logrus.Warnf("Error creating watch for network: %v", err)
 		return
@@ -408,12 +409,13 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi
 }
 
 func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) {
-	if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope {
+	n := ep.getNetwork()
+	if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() {
 		return
 	}
 
 	c.Lock()
-	nw, ok := nmap[ep.getNetwork().ID()]
+	nw, ok := nmap[n.ID()]
 
 	if ok {
 		delete(nw.localEps, ep.ID())
@@ -422,7 +424,7 @@ func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoi
 		// Update the svc db about local endpoint leave right away
 		// Do this after we remove this ep from localEps so that we
 		// don't try to remove this svc record from this ep's container.
-		ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), false)
+		n.updateSvcRecord(ep, c.getLocalEps(nw), false)
 
 		c.Lock()
 		if len(nw.localEps) == 0 {
@@ -430,9 +432,9 @@ func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoi
 
 			// This is the last container going away for the network. Destroy
 			// this network's svc db entry
-			delete(c.svcRecords, ep.getNetwork().ID())
+			delete(c.svcRecords, n.ID())
 
-			delete(nmap, ep.getNetwork().ID())
+			delete(nmap, n.ID())
 		}
 	}
 	c.Unlock()