Add network restore to support docker live restore container

Signed-off-by: Lei Jitang <leijitang@huawei.com>

config/config.go

@@ -15,9 +15,10 @@ import (
 
 // Config encapsulates configurations of various Libnetwork components
 type Config struct {
-	Daemon  DaemonCfg
+	Daemon          DaemonCfg
-	Cluster ClusterCfg
+	Cluster         ClusterCfg
-	Scopes  map[string]*datastore.ScopeCfg
+	Scopes          map[string]*datastore.ScopeCfg
+	ActiveSandboxes map[string]interface{}
 }
 
 // DaemonCfg represents libnetwork core configuration
@@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {
 		c.Scopes[datastore.LocalScope].Client.Config = config
 	}
 }
+
+// OptionActiveSandboxes function returns an option setter for passing the sandboxes
+// which were active during previous daemon life
+func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {
+	return func(c *Config) {
+		c.ActiveSandboxes = sandboxes
+	}
+}

controller.go

@@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
 		}
 	}
 
-	// Reserve pools first before doing cleanup. This is because
+	// Reserve pools first before doing cleanup. Otherwise the
-	// if the pools are not populated properly, the cleanups of
+	// cleanups of endpoint/network and sandbox below will
-	// endpoint/network and sandbox below will not be able to
+	// generate many unnecessary warnings
-	// release ip subnets and addresses properly into the pool
-	// because the pools won't exist.
 	c.reservePools()
 
 	// Cleanup resources
-	c.sandboxCleanup()
+	c.sandboxCleanup(c.cfg.ActiveSandboxes)
 	c.cleanupLocalEndpoints()
 	c.networkCleanup()
 
@@ -832,7 +830,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
 
 	if sb.config.useDefaultSandBox {
 		c.sboxOnce.Do(func() {
-			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
+			c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
 		})
 
 		if err != nil {
@@ -844,7 +842,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
 	}
 
 	if sb.osSbox == nil && !sb.config.useExternalKey {
-		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
+		if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
 			return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
 		}
 	}

drivers/bridge/bridge_store.go

@@ -184,7 +184,7 @@ func (ncfg *networkConfiguration) Exists() bool {
 }
 
 func (ncfg *networkConfiguration) Skip() bool {
-	return ncfg.DefaultBridge
+	return false
 }
 
 func (ncfg *networkConfiguration) New() datastore.KVObject {

drivers/overlay/ov_network.go

@@ -512,7 +512,7 @@ func (n *network) initSandbox() error {
 	n.cleanupStaleSandboxes()
 
 	sbox, err := osl.NewSandbox(
-		osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
+		osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode, false)
 	if err != nil {
 		return fmt.Errorf("could not create network sandbox: %v", err)
 	}

endpoint.go

@@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
 	epMap["name"] = ep.name
 	epMap["id"] = ep.id
 	epMap["ep_iface"] = ep.iface
+	epMap["joinInfo"] = ep.joinInfo
 	epMap["exposed_ports"] = ep.exposedPorts
 	if ep.generic != nil {
 		epMap["generic"] = ep.generic
@@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
 	ib, _ := json.Marshal(epMap["ep_iface"])
 	json.Unmarshal(ib, &ep.iface)
 
+	jb, _ := json.Marshal(epMap["joinInfo"])
+	json.Unmarshal(jb, &ep.joinInfo)
+
 	tb, _ := json.Marshal(epMap["exposed_ports"])
 	var tPorts []types.TransportPort
 	json.Unmarshal(tb, &tPorts)
@@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
 		ep.iface.CopyTo(dstEp.iface)
 	}
 
+	if ep.joinInfo != nil {
+		dstEp.joinInfo = &endpointJoinInfo{}
+		ep.joinInfo.CopyTo(dstEp.joinInfo)
+	}
+
 	dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
 	copy(dstEp.exposedPorts, ep.exposedPorts)
 
@@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() {
 }
 
 func (c *controller) cleanupLocalEndpoints() {
+	// Get used endpoints
+	eps := make(map[string]interface{})
+	for _, sb := range c.sandboxes {
+		for _, ep := range sb.endpoints {
+			eps[ep.id] = true
+		}
+	}
 	nl, err := c.getNetworksForScope(datastore.LocalScope)
 	if err != nil {
 		log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
@@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() {
 		}
 
 		for _, ep := range epl {
+			if _, ok := eps[ep.id]; ok {
+				continue
+			}
 			log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
 			if err := ep.Delete(true); err != nil {
 				log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)

endpoint_info.go

@@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() {
 
 	ep.joinInfo.disableGatewayService = true
 }
+
+func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
+	epMap := make(map[string]interface{})
+	if epj.gw != nil {
+		epMap["gw"] = epj.gw.String()
+	}
+	if epj.gw6 != nil {
+		epMap["gw6"] = epj.gw6.String()
+	}
+	epMap["disableGatewayService"] = epj.disableGatewayService
+	epMap["StaticRoutes"] = epj.StaticRoutes
+	return json.Marshal(epMap)
+}
+
+func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
+	var (
+		err   error
+		epMap map[string]interface{}
+	)
+	if err = json.Unmarshal(b, &epMap); err != nil {
+		return err
+	}
+	if v, ok := epMap["gw"]; ok {
+		epj.gw6 = net.ParseIP(v.(string))
+	}
+	if v, ok := epMap["gw6"]; ok {
+		epj.gw6 = net.ParseIP(v.(string))
+	}
+	epj.disableGatewayService = epMap["disableGatewayService"].(bool)
+
+	var tStaticRoute []types.StaticRoute
+	if v, ok := epMap["StaticRoutes"]; ok {
+		tb, _ := json.Marshal(v)
+		var tStaticRoute []types.StaticRoute
+		json.Unmarshal(tb, &tStaticRoute)
+	}
+	var StaticRoutes []*types.StaticRoute
+	for _, r := range tStaticRoute {
+		StaticRoutes = append(StaticRoutes, &r)
+	}
+	epj.StaticRoutes = StaticRoutes
+
+	return nil
+}
+
+func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
+	dstEpj.disableGatewayService = epj.disableGatewayService
+	dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
+	copy(dstEpj.StaticRoutes, epj.StaticRoutes)
+	dstEpj.gw = types.GetIPCopy(epj.gw)
+	dstEpj.gw = types.GetIPCopy(epj.gw6)
+	return nil
+}

libnetwork_test.go

@@ -1305,7 +1305,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
 	}
 
 	// Create a new OS sandbox using the osl API before using it in SetKey
-	if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil {
+	if extOsBox, err := osl.NewSandbox("ValidKey", true, false); err != nil {
 		t.Fatalf("Failed to create new osl sandbox")
 	} else {
 		defer func() {

osl/namespace_linux.go

@@ -2,10 +2,13 @@ package osl
 
 import (
 	"fmt"
+	"io/ioutil"
 	"net"
 	"os"
 	"os/exec"
 	"runtime"
+	"strconv"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
@@ -133,6 +136,39 @@ func GC() {
 // container id.
 func GenerateKey(containerID string) string {
 	maxLen := 12
+	// Read sandbox key from host for overlay
+	if strings.HasPrefix(containerID, "-") {
+		var (
+			index    int
+			indexStr string
+			tmpkey   string
+		)
+		dir, err := ioutil.ReadDir(prefix)
+		if err != nil {
+			return ""
+		}
+
+		for _, v := range dir {
+			id := v.Name()
+			if strings.HasSuffix(id, containerID[:maxLen-1]) {
+				indexStr = strings.TrimSuffix(id, containerID[:maxLen-1])
+				tmpindex, err := strconv.Atoi(indexStr)
+				if err != nil {
+					return ""
+				}
+				if tmpindex > index {
+					index = tmpindex
+					tmpkey = id
+				}
+
+			}
+		}
+		containerID = tmpkey
+		if containerID == "" {
+			return ""
+		}
+	}
+
 	if len(containerID) < maxLen {
 		maxLen = len(containerID)
 	}
@@ -142,10 +178,12 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
-	err := createNetworkNamespace(key, osCreate)
+	if !isRestore {
-	if err != nil {
+		err := createNetworkNamespace(key, osCreate)
-		return nil, err
+		if err != nil {
+			return nil, err
+		}
 	}
 
 	n := &networkNamespace{path: key, isDefault: !osCreate}
@@ -347,3 +385,108 @@ func (n *networkNamespace) Destroy() error {
 	addToGarbagePaths(n.path)
 	return nil
 }
+
+// Restore restore the network namespace
+func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
+	// restore interfaces
+	for name, opts := range ifsopt {
+		if !strings.Contains(name, "+") {
+			return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name)
+		}
+		seps := strings.Split(name, "+")
+		srcName := seps[0]
+		dstPrefix := seps[1]
+		i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
+		i.processInterfaceOptions(opts...)
+		if i.master != "" {
+			i.dstMaster = n.findDst(i.master, true)
+			if i.dstMaster == "" {
+				return fmt.Errorf("could not find an appropriate master %q for %q",
+					i.master, i.srcName)
+			}
+		}
+		if n.isDefault {
+			i.dstName = i.srcName
+		} else {
+			// due to the docker network connect/disconnect, so the dstName should
+			// restore from the namespace
+			err := nsInvoke(n.path, func(nsFD int) error { return nil }, func(callerFD int) error {
+				ifaces, err := net.Interfaces()
+				if err != nil {
+					return err
+				}
+				for _, iface := range ifaces {
+					addrs, err := iface.Addrs()
+					if err != nil {
+						return err
+					}
+					if strings.HasPrefix(iface.Name, "vxlan") {
+						if i.dstName == "vxlan" {
+							i.dstName = iface.Name
+							break
+						}
+					}
+					// find the interface name by ip
+					if i.address != nil {
+						for _, addr := range addrs {
+							if addr.String() == i.address.String() {
+								i.dstName = iface.Name
+								break
+							}
+							continue
+						}
+						if i.dstName == iface.Name {
+							break
+						}
+					}
+					// This is to find the interface name of the pair in overlay sandbox
+					if strings.HasPrefix(iface.Name, "veth") {
+						if i.master != "" && i.dstName == "veth" {
+							i.dstName = iface.Name
+						}
+					}
+				}
+				return nil
+			})
+			if err != nil {
+				return err
+			}
+			var index int
+			indexStr := strings.TrimPrefix(i.dstName, dstPrefix)
+			if indexStr != "" {
+				index, err = strconv.Atoi(indexStr)
+				if err != nil {
+					return err
+				}
+			}
+			index++
+			n.Lock()
+			if index > n.nextIfIndex {
+				n.nextIfIndex = index
+			}
+			n.iFaces = append(n.iFaces, i)
+			n.Unlock()
+		}
+	}
+
+	// restore routes
+	for _, r := range routes {
+		n.Lock()
+		n.staticRoutes = append(n.staticRoutes, r)
+		n.Unlock()
+	}
+
+	// restore gateway
+	if len(gw) > 0 {
+		n.Lock()
+		n.gw = gw
+		n.Unlock()
+	}
+
+	if len(gw6) > 0 {
+		n.Lock()
+		n.gwv6 = gw6
+		n.Unlock()
+	}
+	return nil
+}

osl/namespace_windows.go

@@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, nil
 }
 

osl/sandbox.go

@@ -58,6 +58,9 @@ type Sandbox interface {
 
 	// Destroy the sandbox
 	Destroy() error
+
+	// restore sandbox
+	Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
 }
 
 // NeighborOptionSetter interface defines the option setter methods for interface options

osl/sandbox_freebsd.go

@@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, nil
 }
 

osl/sandbox_test.go

@@ -25,7 +25,7 @@ func TestSandboxCreate(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -77,7 +77,7 @@ func TestSandboxCreateTwice(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	_, err = NewSandbox(key, true)
+	_, err = NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -85,7 +85,7 @@ func TestSandboxCreateTwice(t *testing.T) {
 
 	// Create another sandbox with the same key to see if we handle it
 	// gracefully.
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -105,7 +105,7 @@ func TestSandboxGC(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}
@@ -127,7 +127,7 @@ func TestAddRemoveInterface(t *testing.T) {
 		t.Fatalf("Failed to obtain a key: %v", err)
 	}
 
-	s, err := NewSandbox(key, true)
+	s, err := NewSandbox(key, true, false)
 	if err != nil {
 		t.Fatalf("Failed to create a new sandbox: %v", err)
 	}

osl/sandbox_unsupported.go

@@ -11,7 +11,7 @@ var (
 
 // NewSandbox provides a new sandbox instance created in an os specific way
 // provided a key which uniquely identifies the sandbox
-func NewSandbox(key string, osCreate bool) (Sandbox, error) {
+func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
 	return nil, ErrNotImplemented
 }
 

resolver_unix.go

@@ -19,6 +19,13 @@ func init() {
 	reexec.Register("setup-resolver", reexecSetupResolver)
 }
 
+const (
+	// outputChain used for docker embed dns
+	outputChain = "DOCKER_OUTPUT"
+	//postroutingchain used for docker embed dns
+	postroutingchain = "DOCKER_POSTROUTING"
+)
+
 func reexecSetupResolver() {
 	runtime.LockOSThread()
 	defer runtime.UnlockOSThread()
@@ -31,10 +38,10 @@ func reexecSetupResolver() {
 	_, ipPort, _ := net.SplitHostPort(os.Args[2])
 	_, tcpPort, _ := net.SplitHostPort(os.Args[3])
 	rules := [][]string{
-		{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
+		{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
-		{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+		{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
-		{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
+		{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
-		{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
+		{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
 	}
 
 	f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0)
@@ -50,6 +57,23 @@ func reexecSetupResolver() {
 		os.Exit(3)
 	}
 
+	// insert outputChain and postroutingchain
+	err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain)
+	if err == nil {
+		iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain)
+	} else {
+		iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain)
+		iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain)
+	}
+
+	err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
+	if err == nil {
+		iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain)
+	} else {
+		iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain)
+		iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
+	}
+
 	for _, rule := range rules {
 		if iptables.RawCombinedOutputNative(rule...) != nil {
 			log.Errorf("setting up rule failed, %v", rule)

sandbox.go

@@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() {
 	osSbox.Destroy()
 }
 
+func (sb *sandbox) restoreOslSandbox() error {
+	var routes []*types.StaticRoute
+
+	// restore osl sandbox
+	Ifaces := make(map[string][]osl.IfaceOption)
+	for _, ep := range sb.endpoints {
+		var ifaceOptions []osl.IfaceOption
+		ep.Lock()
+		joinInfo := ep.joinInfo
+		i := ep.iface
+		ep.Unlock()
+		ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
+		if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
+		}
+		if i.mac != nil {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
+		}
+		if len(i.llAddrs) != 0 {
+			ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
+		}
+		Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions
+		if joinInfo != nil {
+			for _, r := range joinInfo.StaticRoutes {
+				routes = append(routes, r)
+			}
+		}
+		if ep.needResolver() {
+			sb.startResolver()
+		}
+	}
+
+	gwep := sb.getGatewayEndpoint()
+	if gwep == nil {
+		return nil
+	}
+
+	// restore osl sandbox
+	err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
 	sb.Lock()
 	if sb.osSbox == nil {

sandbox_dns_unix.go

@@ -139,6 +139,17 @@ func (sb *sandbox) updateParentHosts() error {
 	return nil
 }
 
+func (sb *sandbox) restorePath() {
+	if sb.config.resolvConfPath == "" {
+		sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf"
+	}
+	sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash"
+	if sb.config.hostsPath == "" {
+		sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts"
+	}
+
+}
+
 func (sb *sandbox) setupDNS() error {
 	var newRC *resolvconf.File
 

sandbox_dns_windows.go

@@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error {
 	return nil
 }
 
+func (sb *sandbox) restorePath() {
+}
+
 func (sb *sandbox) updateHostsFile(ifaceIP string) error {
 	return nil
 }

sandbox_store.go

@@ -20,12 +20,13 @@ type epState struct {
 }
 
 type sbState struct {
-	ID       string
+	ID         string
-	Cid      string
+	Cid        string
-	c        *controller
+	c          *controller
-	dbIndex  uint64
+	dbIndex    uint64
-	dbExists bool
+	dbExists   bool
-	Eps      []epState
+	Eps        []epState
+	EpPriority map[string]int
 }
 
 func (sbs *sbState) Key() []string {
@@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error {
 	dstSbs.Cid = sbs.Cid
 	dstSbs.dbIndex = sbs.dbIndex
 	dstSbs.dbExists = sbs.dbExists
+	dstSbs.EpPriority = sbs.EpPriority
 
 	for _, eps := range sbs.Eps {
 		dstSbs.Eps = append(dstSbs.Eps, eps)
@@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string {
 
 func (sb *sandbox) storeUpdate() error {
 	sbs := &sbState{
-		c:   sb.controller,
+		c:          sb.controller,
-		ID:  sb.id,
+		ID:         sb.id,
-		Cid: sb.containerID,
+		Cid:        sb.containerID,
+		EpPriority: sb.epPriority,
 	}
 
 retry:
@@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error {
 	return sb.controller.deleteFromStore(sbs)
 }
 
-func (c *controller) sandboxCleanup() {
+func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
 	store := c.getStore(datastore.LocalScope)
 	if store == nil {
 		logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes")
@@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() {
 			controller:  sbs.c,
 			containerID: sbs.Cid,
 			endpoints:   epHeap{},
-			epPriority:  map[string]int{},
 			dbIndex:     sbs.dbIndex,
 			isStub:      true,
 			dbExists:    true,
 		}
 
-		sb.osSbox, err = osl.NewSandbox(sb.Key(), true)
+		msg := " for cleanup"
+		create := true
+		isRestore := false
+		if val, ok := activeSandboxes[sb.ID()]; ok {
+			msg = ""
+			sb.isStub = false
+			isRestore = true
+			opts := val.([]SandboxOption)
+			sb.processOptions(opts...)
+			sb.restorePath()
+			create = !sb.config.useDefaultSandBox
+			heap.Init(&sb.endpoints)
+		}
+		sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore)
 		if err != nil {
-			logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err)
+			logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err)
 			continue
 		}
 
@@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() {
 					ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
 				}
 			}
-
 			heap.Push(&sb.endpoints, ep)
 		}
 
-		logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
+		if _, ok := activeSandboxes[sb.ID()]; !ok {
-		if err := sb.delete(true); err != nil {
+			logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
-			logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
+			if err := sb.delete(true); err != nil {
+				logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
+			}
+			continue
+		}
+
+		// reconstruct osl sandbox field
+		if !sb.config.useDefaultSandBox {
+			if err := sb.restoreOslSandbox(); err != nil {
+				logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID())
+				continue
+			}
+		} else {
+			c.sboxOnce.Do(func() {
+				c.defOsSbox = sb.osSbox
+			})
+		}
+
+		for _, ep := range sb.endpoints {
+			// Watch for service records
+			if !c.isAgent() {
+				c.watchSvcRecord(ep)
+			}
 		}
 	}
 }