From 596122e05edd3e9712256ea309746ff903409446 Mon Sep 17 00:00:00 2001 From: Alessandro Boch Date: Fri, 7 Apr 2017 13:31:44 -0700 Subject: [PATCH] Add ConnectivityScope capability for network drivers along with scope network option - It specifies whether the network driver can provide containers connectivity across hosts. - As of now, the data scope of the driver was being overloaded with this notion. - The driver scope information is still valid and it defines whether the data allocation of the network resources can be done globally or only locally. - With the scope network option, user can now force a network as swarm scoped regardless of the driver data scope. - In case the network is configured as swarm scoped, and the network driver is multihost capable, a network DB instance will be launched for it. Signed-off-by: Alessandro Boch --- agent.go | 4 +- controller.go | 10 ++++- datastore/datastore.go | 5 ++- docs/remote.md | 6 ++- driverapi/driverapi.go | 3 +- drivers/bridge/bridge.go | 3 +- drivers/host/host.go | 3 +- drivers/ipvlan/ipvlan.go | 3 +- drivers/macvlan/macvlan.go | 3 +- drivers/overlay/overlay.go | 3 +- drivers/overlay/ovmanager/ovmanager.go | 3 +- drivers/remote/api/api.go | 3 +- drivers/remote/driver.go | 11 ++++++ drivers/remote/driver_test.go | 7 +++- drivers/solaris/bridge/bridge.go | 3 +- drivers/solaris/overlay/overlay.go | 3 +- .../solaris/overlay/ovmanager/ovmanager.go | 3 +- drivers/windows/overlay/overlay_windows.go | 3 +- drivers/windows/windows.go | 3 +- network.go | 37 +++++++++++++++---- store.go | 26 +++++++------ 21 files changed, 103 insertions(+), 42 deletions(-) diff --git a/agent.go b/agent.go index b4b7bdf6..d67d446f 100644 --- a/agent.go +++ b/agent.go @@ -222,7 +222,7 @@ func (c *controller) agentSetup(clusterProvider cluster.Provider) error { return err } c.drvRegistry.WalkDrivers(func(name string, driver driverapi.Driver, capability driverapi.Capability) bool { - if capability.DataScope == datastore.GlobalScope { + if capability.ConnectivityScope == datastore.GlobalScope { c.agentDriverNotify(driver) } return false @@ -507,7 +507,7 @@ func (n *network) Services() map[string]ServiceInfo { } func (n *network) isClusterEligible() bool { - if n.driverScope() != datastore.GlobalScope { + if n.scope != datastore.SwarmScope || !n.driverIsMultihost() { return false } return n.getController().getAgent() != nil diff --git a/controller.go b/controller.go index 84c3ee3d..159e40f6 100644 --- a/controller.go +++ b/controller.go @@ -621,7 +621,7 @@ func (c *controller) pushNodeDiscovery(d driverapi.Driver, cap driverapi.Capabil } } - if d == nil || cap.DataScope != datastore.GlobalScope || nodes == nil { + if d == nil || cap.ConnectivityScope != datastore.GlobalScope || nodes == nil { return } @@ -747,11 +747,17 @@ func (c *controller) NewNetwork(networkType, name string, id string, options ... return nil, err } + if network.scope == datastore.LocalScope && cap.DataScope == datastore.GlobalScope { + return nil, types.ForbiddenErrorf("cannot downgrade network scope for %s networks", networkType) + + } if network.ingress && cap.DataScope != datastore.GlobalScope { return nil, types.ForbiddenErrorf("Ingress network can only be global scope network") } - if cap.DataScope == datastore.GlobalScope && !c.isDistributedControl() && !network.dynamic { + // At this point the network scope is still unknown if not set by user + if (cap.DataScope == datastore.GlobalScope || network.scope == datastore.SwarmScope) && + !c.isDistributedControl() && !network.dynamic { if c.isManager() { // For non-distributed controlled environment, globalscoped non-dynamic networks are redirected to Manager return nil, ManagerRedirectError(name) diff --git a/datastore/datastore.go b/datastore/datastore.go index 19bf0b02..82feef1c 100644 --- a/datastore/datastore.go +++ b/datastore/datastore.go @@ -115,7 +115,10 @@ const ( // LocalScope indicates to store the KV object in local datastore such as boltdb LocalScope = "local" // GlobalScope indicates to store the KV object in global datastore such as consul/etcd/zookeeper - GlobalScope = "global" + GlobalScope = "global" + // SwarmScope is not indicating a datastore location. It is defined here + // along with the other two scopes just for consistency. + SwarmScope = "swarm" defaultPrefix = "/var/lib/docker/network/files" ) diff --git a/docs/remote.md b/docs/remote.md index 48fd5e58..56125080 100644 --- a/docs/remote.md +++ b/docs/remote.md @@ -56,10 +56,12 @@ Other entries in the list value are allowed; `"NetworkDriver"` indicates that th After Handshake, the remote driver will receive another POST message to the URL `/NetworkDriver.GetCapabilities` with no payload. The driver's response should have the form: { - "Scope": "local" + "Scope": "local" + "ConnectivityScope": "global" } -Value of "Scope" should be either "local" or "global" which indicates the capability of remote driver, values beyond these will fail driver's registration and return an error to the caller. +Value of "Scope" should be either "local" or "global" which indicates whether the resource allocations for this driver's network can be done only locally to the node or globally across the cluster of nodes. Any other value will fail driver's registration and return an error to the caller. +Similarly, value of "ConnectivityScope" should be either "local" or "global" which indicates whether the driver's network can provide connectivity only locally to this node or globally across the cluster of nodes. If the value is missing, libnetwork will set it to the value of "Scope". should be either "local" or "global" which indicates ### Create network diff --git a/driverapi/driverapi.go b/driverapi/driverapi.go index 074438ef..48a14ae5 100644 --- a/driverapi/driverapi.go +++ b/driverapi/driverapi.go @@ -161,7 +161,8 @@ type DriverCallback interface { // Capability represents the high level capabilities of the drivers which libnetwork can make use of type Capability struct { - DataScope string + DataScope string + ConnectivityScope string } // IPAMData represents the per-network ip related diff --git a/drivers/bridge/bridge.go b/drivers/bridge/bridge.go index e681b8f7..dd79f04d 100644 --- a/drivers/bridge/bridge.go +++ b/drivers/bridge/bridge.go @@ -153,7 +153,8 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { } c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, } return dc.RegisterDriver(networkType, d, c) } diff --git a/drivers/host/host.go b/drivers/host/host.go index 7b4a986e..a71d4613 100644 --- a/drivers/host/host.go +++ b/drivers/host/host.go @@ -19,7 +19,8 @@ type driver struct { // Init registers a new instance of host driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, } return dc.RegisterDriver(networkType, &driver{}, c) } diff --git a/drivers/ipvlan/ipvlan.go b/drivers/ipvlan/ipvlan.go index 296804dc..c64ad555 100644 --- a/drivers/ipvlan/ipvlan.go +++ b/drivers/ipvlan/ipvlan.go @@ -58,7 +58,8 @@ type network struct { // Init initializes and registers the libnetwork ipvlan driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/drivers/macvlan/macvlan.go b/drivers/macvlan/macvlan.go index 49b9fbae..872e6f3e 100644 --- a/drivers/macvlan/macvlan.go +++ b/drivers/macvlan/macvlan.go @@ -60,7 +60,8 @@ type network struct { // Init initializes and registers the libnetwork macvlan driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/drivers/overlay/overlay.go b/drivers/overlay/overlay.go index 88e1010c..8a932cf3 100644 --- a/drivers/overlay/overlay.go +++ b/drivers/overlay/overlay.go @@ -56,7 +56,8 @@ type driver struct { // Init registers a new instance of overlay driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/drivers/overlay/ovmanager/ovmanager.go b/drivers/overlay/ovmanager/ovmanager.go index dce8f98b..3c3c0bb2 100644 --- a/drivers/overlay/ovmanager/ovmanager.go +++ b/drivers/overlay/ovmanager/ovmanager.go @@ -49,7 +49,8 @@ type network struct { func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { var err error c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ diff --git a/drivers/remote/api/api.go b/drivers/remote/api/api.go index f9a341c5..d24f1901 100644 --- a/drivers/remote/api/api.go +++ b/drivers/remote/api/api.go @@ -24,7 +24,8 @@ func (r *Response) GetError() string { // GetCapabilityResponse is the response of GetCapability request type GetCapabilityResponse struct { Response - Scope string + Scope string + ConnectivityScope string } // AllocateNetworkRequest requests allocation of new network by manager diff --git a/drivers/remote/driver.go b/drivers/remote/driver.go index 49a7fb49..ffe0730c 100644 --- a/drivers/remote/driver.go +++ b/drivers/remote/driver.go @@ -74,6 +74,17 @@ func (d *driver) getCapabilities() (*driverapi.Capability, error) { return nil, fmt.Errorf("invalid capability: expecting 'local' or 'global', got %s", capResp.Scope) } + switch capResp.ConnectivityScope { + case "global": + c.ConnectivityScope = datastore.GlobalScope + case "local": + c.ConnectivityScope = datastore.LocalScope + case "": + c.ConnectivityScope = c.DataScope + default: + return nil, fmt.Errorf("invalid capability: expecting 'local' or 'global', got %s", capResp.Scope) + } + return c, nil } diff --git a/drivers/remote/driver_test.go b/drivers/remote/driver_test.go index e559a612..8a97bacf 100644 --- a/drivers/remote/driver_test.go +++ b/drivers/remote/driver_test.go @@ -238,8 +238,9 @@ func TestGetExtraCapabilities(t *testing.T) { handle(t, mux, "GetCapabilities", func(msg map[string]interface{}) interface{} { return map[string]interface{}{ - "Scope": "local", - "foo": "bar", + "Scope": "local", + "foo": "bar", + "ConnectivityScope": "global", } }) @@ -258,6 +259,8 @@ func TestGetExtraCapabilities(t *testing.T) { t.Fatal(err) } else if c.DataScope != datastore.LocalScope { t.Fatalf("get capability '%s', expecting 'local'", c.DataScope) + } else if c.ConnectivityScope != datastore.GlobalScope { + t.Fatalf("get capability '%s', expecting %q", c.ConnectivityScope, datastore.GlobalScope) } } diff --git a/drivers/solaris/bridge/bridge.go b/drivers/solaris/bridge/bridge.go index 13dd5f14..2547016a 100644 --- a/drivers/solaris/bridge/bridge.go +++ b/drivers/solaris/bridge/bridge.go @@ -159,7 +159,8 @@ func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { } c := driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, } return dc.RegisterDriver(networkType, d, c) } diff --git a/drivers/solaris/overlay/overlay.go b/drivers/solaris/overlay/overlay.go index b001f58a..0a5a1bfd 100644 --- a/drivers/solaris/overlay/overlay.go +++ b/drivers/solaris/overlay/overlay.go @@ -57,7 +57,8 @@ type driver struct { // Init registers a new instance of overlay driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ networks: networkTable{}, diff --git a/drivers/solaris/overlay/ovmanager/ovmanager.go b/drivers/solaris/overlay/ovmanager/ovmanager.go index f053ff97..a3a7cd4c 100644 --- a/drivers/solaris/overlay/ovmanager/ovmanager.go +++ b/drivers/solaris/overlay/ovmanager/ovmanager.go @@ -49,7 +49,8 @@ type network struct { func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { var err error c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ diff --git a/drivers/windows/overlay/overlay_windows.go b/drivers/windows/overlay/overlay_windows.go index 2ae2cec7..111e517d 100644 --- a/drivers/windows/overlay/overlay_windows.go +++ b/drivers/windows/overlay/overlay_windows.go @@ -36,7 +36,8 @@ type driver struct { // Init registers a new instance of overlay driver func Init(dc driverapi.DriverCallback, config map[string]interface{}) error { c := driverapi.Capability{ - DataScope: datastore.GlobalScope, + DataScope: datastore.GlobalScope, + ConnectivityScope: datastore.GlobalScope, } d := &driver{ diff --git a/drivers/windows/windows.go b/drivers/windows/windows.go index b6591a1d..84089a52 100644 --- a/drivers/windows/windows.go +++ b/drivers/windows/windows.go @@ -104,7 +104,8 @@ func GetInit(networkType string) func(dc driverapi.DriverCallback, config map[st } return dc.RegisterDriver(networkType, newDriver(networkType), driverapi.Capability{ - DataScope: datastore.LocalScope, + DataScope: datastore.LocalScope, + ConnectivityScope: datastore.LocalScope, }) } } diff --git a/network.go b/network.go index 7be7b0af..5fcca95f 100644 --- a/network.go +++ b/network.go @@ -195,7 +195,7 @@ type network struct { networkType string id string created time.Time - scope string + scope string // network data scope labels map[string]string ipamType string ipamOptions map[string]string @@ -514,7 +514,12 @@ func (n *network) CopyTo(o datastore.KVObject) error { } func (n *network) DataScope() string { - return n.Scope() + s := n.Scope() + // All swarm scope networks have local datascope + if s == datastore.SwarmScope { + s = datastore.LocalScope + } + return s } func (n *network) getEpCnt() *endpointCnt { @@ -762,6 +767,14 @@ func NetworkOptionAttachable(attachable bool) NetworkOption { } } +// NetworkOptionScope returns an option setter to overwrite the network's scope. +// By default the network's scope is set to the network driver's datascope. +func NetworkOptionScope(scope string) NetworkOption { + return func(n *network) { + n.scope = scope + } +} + // NetworkOptionIpam function returns an option setter for the ipam configuration for this network func NetworkOptionIpam(ipamDriver string, addrSpace string, ipV4 []*IpamConf, ipV6 []*IpamConf, opts map[string]string) NetworkOption { return func(n *network) { @@ -877,6 +890,14 @@ func (n *network) driverScope() string { return cap.DataScope } +func (n *network) driverIsMultihost() bool { + _, cap, err := n.resolveDriver(n.networkType, true) + if err != nil { + return false + } + return cap.ConnectivityScope == datastore.GlobalScope +} + func (n *network) driver(load bool) (driverapi.Driver, error) { d, cap, err := n.resolveDriver(n.networkType, load) if err != nil { @@ -887,14 +908,14 @@ func (n *network) driver(load bool) (driverapi.Driver, error) { isAgent := c.isAgent() n.Lock() // If load is not required, driver, cap and err may all be nil - if cap != nil { + if n.scope == "" && cap != nil { n.scope = cap.DataScope } - if isAgent || n.dynamic { - // If we are running in agent mode then all networks - // in libnetwork are local scope regardless of the - // backing driver. - n.scope = datastore.LocalScope + if isAgent && n.dynamic { + // If we are running in agent mode and the network + // is dynamic, then the networks are swarm scoped + // regardless of the backing driver. + n.scope = datastore.SwarmScope } n.Unlock() return d, nil diff --git a/store.go b/store.go index cb220561..1a8dceca 100644 --- a/store.go +++ b/store.go @@ -350,17 +350,18 @@ func (c *controller) networkWatchLoop(nw *netWatch, ep *endpoint, ecCh <-chan da } func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoint) { - if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope { + n := ep.getNetwork() + if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() { return } c.Lock() - nw, ok := nmap[ep.getNetwork().ID()] + nw, ok := nmap[n.ID()] c.Unlock() if ok { // Update the svc db for the local endpoint join right away - ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true) + n.updateSvcRecord(ep, c.getLocalEps(nw), true) c.Lock() nw.localEps[ep.ID()] = ep @@ -381,15 +382,15 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi // Update the svc db for the local endpoint join right away // Do this before adding this ep to localEps so that we don't // try to update this ep's container's svc records - ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), true) + n.updateSvcRecord(ep, c.getLocalEps(nw), true) c.Lock() nw.localEps[ep.ID()] = ep - nmap[ep.getNetwork().ID()] = nw + nmap[n.ID()] = nw nw.stopCh = make(chan struct{}) c.Unlock() - store := c.getStore(ep.getNetwork().DataScope()) + store := c.getStore(n.DataScope()) if store == nil { return } @@ -398,7 +399,7 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi return } - ch, err := store.Watch(ep.getNetwork().getEpCnt(), nw.stopCh) + ch, err := store.Watch(n.getEpCnt(), nw.stopCh) if err != nil { logrus.Warnf("Error creating watch for network: %v", err) return @@ -408,12 +409,13 @@ func (c *controller) processEndpointCreate(nmap map[string]*netWatch, ep *endpoi } func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoint) { - if !c.isDistributedControl() && ep.getNetwork().driverScope() == datastore.GlobalScope { + n := ep.getNetwork() + if !c.isDistributedControl() && n.Scope() == datastore.SwarmScope && n.driverIsMultihost() { return } c.Lock() - nw, ok := nmap[ep.getNetwork().ID()] + nw, ok := nmap[n.ID()] if ok { delete(nw.localEps, ep.ID()) @@ -422,7 +424,7 @@ func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoi // Update the svc db about local endpoint leave right away // Do this after we remove this ep from localEps so that we // don't try to remove this svc record from this ep's container. - ep.getNetwork().updateSvcRecord(ep, c.getLocalEps(nw), false) + n.updateSvcRecord(ep, c.getLocalEps(nw), false) c.Lock() if len(nw.localEps) == 0 { @@ -430,9 +432,9 @@ func (c *controller) processEndpointDelete(nmap map[string]*netWatch, ep *endpoi // This is the last container going away for the network. Destroy // this network's svc db entry - delete(c.svcRecords, ep.getNetwork().ID()) + delete(c.svcRecords, n.ID()) - delete(nmap, ep.getNetwork().ID()) + delete(nmap, n.ID()) } } c.Unlock()