From be153a13e43f2509c75d86e7f2a7de55ce5ec76c Mon Sep 17 00:00:00 2001 From: Lei Jitang Date: Fri, 10 Jun 2016 17:32:19 -0700 Subject: [PATCH] Add network restore to support docker live restore container Signed-off-by: Lei Jitang --- config/config.go | 15 +++- controller.go | 14 ++- drivers/bridge/bridge_store.go | 2 +- drivers/overlay/ov_network.go | 2 +- endpoint.go | 19 +++++ endpoint_info.go | 53 ++++++++++++ libnetwork_test.go | 2 +- osl/namespace_linux.go | 151 ++++++++++++++++++++++++++++++++- osl/namespace_windows.go | 2 +- osl/sandbox.go | 3 + osl/sandbox_freebsd.go | 2 +- osl/sandbox_test.go | 10 +-- osl/sandbox_unsupported.go | 2 +- resolver_unix.go | 32 ++++++- sandbox.go | 46 ++++++++++ sandbox_dns_unix.go | 11 +++ sandbox_dns_windows.go | 3 + sandbox_store.go | 70 +++++++++++---- 18 files changed, 392 insertions(+), 47 deletions(-) diff --git a/config/config.go b/config/config.go index 2bae6f45..b14691e2 100644 --- a/config/config.go +++ b/config/config.go @@ -15,9 +15,10 @@ import ( // Config encapsulates configurations of various Libnetwork components type Config struct { - Daemon DaemonCfg - Cluster ClusterCfg - Scopes map[string]*datastore.ScopeCfg + Daemon DaemonCfg + Cluster ClusterCfg + Scopes map[string]*datastore.ScopeCfg + ActiveSandboxes map[string]interface{} } // DaemonCfg represents libnetwork core configuration @@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option { c.Scopes[datastore.LocalScope].Client.Config = config } } + +// OptionActiveSandboxes function returns an option setter for passing the sandboxes +// which were active during previous daemon life +func OptionActiveSandboxes(sandboxes map[string]interface{}) Option { + return func(c *Config) { + c.ActiveSandboxes = sandboxes + } +} diff --git a/controller.go b/controller.go index 551e888b..d1ca3f62 100644 --- a/controller.go +++ b/controller.go @@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) { } } - // Reserve pools first before doing cleanup. This is because - // if the pools are not populated properly, the cleanups of - // endpoint/network and sandbox below will not be able to - // release ip subnets and addresses properly into the pool - // because the pools won't exist. + // Reserve pools first before doing cleanup. Otherwise the + // cleanups of endpoint/network and sandbox below will + // generate many unnecessary warnings c.reservePools() // Cleanup resources - c.sandboxCleanup() + c.sandboxCleanup(c.cfg.ActiveSandboxes) c.cleanupLocalEndpoints() c.networkCleanup() @@ -832,7 +830,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s if sb.config.useDefaultSandBox { c.sboxOnce.Do(func() { - c.defOsSbox, err = osl.NewSandbox(sb.Key(), false) + c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false) }) if err != nil { @@ -844,7 +842,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s } if sb.osSbox == nil && !sb.config.useExternalKey { - if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil { + if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil { return nil, fmt.Errorf("failed to create new osl sandbox: %v", err) } } diff --git a/drivers/bridge/bridge_store.go b/drivers/bridge/bridge_store.go index e10a429e..eee8fde4 100644 --- a/drivers/bridge/bridge_store.go +++ b/drivers/bridge/bridge_store.go @@ -184,7 +184,7 @@ func (ncfg *networkConfiguration) Exists() bool { } func (ncfg *networkConfiguration) Skip() bool { - return ncfg.DefaultBridge + return false } func (ncfg *networkConfiguration) New() datastore.KVObject { diff --git a/drivers/overlay/ov_network.go b/drivers/overlay/ov_network.go index 05ed34a8..6bf3b341 100644 --- a/drivers/overlay/ov_network.go +++ b/drivers/overlay/ov_network.go @@ -512,7 +512,7 @@ func (n *network) initSandbox() error { n.cleanupStaleSandboxes() sbox, err := osl.NewSandbox( - osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode) + osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode, false) if err != nil { return fmt.Errorf("could not create network sandbox: %v", err) } diff --git a/endpoint.go b/endpoint.go index f84e8cb7..043c3f16 100644 --- a/endpoint.go +++ b/endpoint.go @@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) { epMap["name"] = ep.name epMap["id"] = ep.id epMap["ep_iface"] = ep.iface + epMap["joinInfo"] = ep.joinInfo epMap["exposed_ports"] = ep.exposedPorts if ep.generic != nil { epMap["generic"] = ep.generic @@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) { ib, _ := json.Marshal(epMap["ep_iface"]) json.Unmarshal(ib, &ep.iface) + jb, _ := json.Marshal(epMap["joinInfo"]) + json.Unmarshal(jb, &ep.joinInfo) + tb, _ := json.Marshal(epMap["exposed_ports"]) var tPorts []types.TransportPort json.Unmarshal(tb, &tPorts) @@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error { ep.iface.CopyTo(dstEp.iface) } + if ep.joinInfo != nil { + dstEp.joinInfo = &endpointJoinInfo{} + ep.joinInfo.CopyTo(dstEp.joinInfo) + } + dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts)) copy(dstEp.exposedPorts, ep.exposedPorts) @@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() { } func (c *controller) cleanupLocalEndpoints() { + // Get used endpoints + eps := make(map[string]interface{}) + for _, sb := range c.sandboxes { + for _, ep := range sb.endpoints { + eps[ep.id] = true + } + } nl, err := c.getNetworksForScope(datastore.LocalScope) if err != nil { log.Warnf("Could not get list of networks during endpoint cleanup: %v", err) @@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() { } for _, ep := range epl { + if _, ok := eps[ep.id]; ok { + continue + } log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id) if err := ep.Delete(true); err != nil { log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err) diff --git a/endpoint_info.go b/endpoint_info.go index cf295a42..60f15518 100644 --- a/endpoint_info.go +++ b/endpoint_info.go @@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() { ep.joinInfo.disableGatewayService = true } + +func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) { + epMap := make(map[string]interface{}) + if epj.gw != nil { + epMap["gw"] = epj.gw.String() + } + if epj.gw6 != nil { + epMap["gw6"] = epj.gw6.String() + } + epMap["disableGatewayService"] = epj.disableGatewayService + epMap["StaticRoutes"] = epj.StaticRoutes + return json.Marshal(epMap) +} + +func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error { + var ( + err error + epMap map[string]interface{} + ) + if err = json.Unmarshal(b, &epMap); err != nil { + return err + } + if v, ok := epMap["gw"]; ok { + epj.gw6 = net.ParseIP(v.(string)) + } + if v, ok := epMap["gw6"]; ok { + epj.gw6 = net.ParseIP(v.(string)) + } + epj.disableGatewayService = epMap["disableGatewayService"].(bool) + + var tStaticRoute []types.StaticRoute + if v, ok := epMap["StaticRoutes"]; ok { + tb, _ := json.Marshal(v) + var tStaticRoute []types.StaticRoute + json.Unmarshal(tb, &tStaticRoute) + } + var StaticRoutes []*types.StaticRoute + for _, r := range tStaticRoute { + StaticRoutes = append(StaticRoutes, &r) + } + epj.StaticRoutes = StaticRoutes + + return nil +} + +func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error { + dstEpj.disableGatewayService = epj.disableGatewayService + dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes)) + copy(dstEpj.StaticRoutes, epj.StaticRoutes) + dstEpj.gw = types.GetIPCopy(epj.gw) + dstEpj.gw = types.GetIPCopy(epj.gw6) + return nil +} diff --git a/libnetwork_test.go b/libnetwork_test.go index b27bde9f..134391fb 100644 --- a/libnetwork_test.go +++ b/libnetwork_test.go @@ -1305,7 +1305,7 @@ func externalKeyTest(t *testing.T, reexec bool) { } // Create a new OS sandbox using the osl API before using it in SetKey - if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil { + if extOsBox, err := osl.NewSandbox("ValidKey", true, false); err != nil { t.Fatalf("Failed to create new osl sandbox") } else { defer func() { diff --git a/osl/namespace_linux.go b/osl/namespace_linux.go index c804caf7..3d38d131 100644 --- a/osl/namespace_linux.go +++ b/osl/namespace_linux.go @@ -2,10 +2,13 @@ package osl import ( "fmt" + "io/ioutil" "net" "os" "os/exec" "runtime" + "strconv" + "strings" "sync" "syscall" "time" @@ -133,6 +136,39 @@ func GC() { // container id. func GenerateKey(containerID string) string { maxLen := 12 + // Read sandbox key from host for overlay + if strings.HasPrefix(containerID, "-") { + var ( + index int + indexStr string + tmpkey string + ) + dir, err := ioutil.ReadDir(prefix) + if err != nil { + return "" + } + + for _, v := range dir { + id := v.Name() + if strings.HasSuffix(id, containerID[:maxLen-1]) { + indexStr = strings.TrimSuffix(id, containerID[:maxLen-1]) + tmpindex, err := strconv.Atoi(indexStr) + if err != nil { + return "" + } + if tmpindex > index { + index = tmpindex + tmpkey = id + } + + } + } + containerID = tmpkey + if containerID == "" { + return "" + } + } + if len(containerID) < maxLen { maxLen = len(containerID) } @@ -142,10 +178,12 @@ func GenerateKey(containerID string) string { // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { - err := createNetworkNamespace(key, osCreate) - if err != nil { - return nil, err +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { + if !isRestore { + err := createNetworkNamespace(key, osCreate) + if err != nil { + return nil, err + } } n := &networkNamespace{path: key, isDefault: !osCreate} @@ -347,3 +385,108 @@ func (n *networkNamespace) Destroy() error { addToGarbagePaths(n.path) return nil } + +// Restore restore the network namespace +func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error { + // restore interfaces + for name, opts := range ifsopt { + if !strings.Contains(name, "+") { + return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name) + } + seps := strings.Split(name, "+") + srcName := seps[0] + dstPrefix := seps[1] + i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n} + i.processInterfaceOptions(opts...) + if i.master != "" { + i.dstMaster = n.findDst(i.master, true) + if i.dstMaster == "" { + return fmt.Errorf("could not find an appropriate master %q for %q", + i.master, i.srcName) + } + } + if n.isDefault { + i.dstName = i.srcName + } else { + // due to the docker network connect/disconnect, so the dstName should + // restore from the namespace + err := nsInvoke(n.path, func(nsFD int) error { return nil }, func(callerFD int) error { + ifaces, err := net.Interfaces() + if err != nil { + return err + } + for _, iface := range ifaces { + addrs, err := iface.Addrs() + if err != nil { + return err + } + if strings.HasPrefix(iface.Name, "vxlan") { + if i.dstName == "vxlan" { + i.dstName = iface.Name + break + } + } + // find the interface name by ip + if i.address != nil { + for _, addr := range addrs { + if addr.String() == i.address.String() { + i.dstName = iface.Name + break + } + continue + } + if i.dstName == iface.Name { + break + } + } + // This is to find the interface name of the pair in overlay sandbox + if strings.HasPrefix(iface.Name, "veth") { + if i.master != "" && i.dstName == "veth" { + i.dstName = iface.Name + } + } + } + return nil + }) + if err != nil { + return err + } + var index int + indexStr := strings.TrimPrefix(i.dstName, dstPrefix) + if indexStr != "" { + index, err = strconv.Atoi(indexStr) + if err != nil { + return err + } + } + index++ + n.Lock() + if index > n.nextIfIndex { + n.nextIfIndex = index + } + n.iFaces = append(n.iFaces, i) + n.Unlock() + } + } + + // restore routes + for _, r := range routes { + n.Lock() + n.staticRoutes = append(n.staticRoutes, r) + n.Unlock() + } + + // restore gateway + if len(gw) > 0 { + n.Lock() + n.gw = gw + n.Unlock() + } + + if len(gw6) > 0 { + n.Lock() + n.gwv6 = gw6 + n.Unlock() + } + return nil +} diff --git a/osl/namespace_windows.go b/osl/namespace_windows.go index 912d4a2e..a735623a 100644 --- a/osl/namespace_windows.go +++ b/osl/namespace_windows.go @@ -15,7 +15,7 @@ func GenerateKey(containerID string) string { // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { return nil, nil } diff --git a/osl/sandbox.go b/osl/sandbox.go index 5264b350..18113e3b 100644 --- a/osl/sandbox.go +++ b/osl/sandbox.go @@ -58,6 +58,9 @@ type Sandbox interface { // Destroy the sandbox Destroy() error + + // restore sandbox + Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error } // NeighborOptionSetter interface defines the option setter methods for interface options diff --git a/osl/sandbox_freebsd.go b/osl/sandbox_freebsd.go index 7c6dcace..0222afe3 100644 --- a/osl/sandbox_freebsd.go +++ b/osl/sandbox_freebsd.go @@ -15,7 +15,7 @@ func GenerateKey(containerID string) string { // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { return nil, nil } diff --git a/osl/sandbox_test.go b/osl/sandbox_test.go index 019ec0e0..d3bd1ace 100644 --- a/osl/sandbox_test.go +++ b/osl/sandbox_test.go @@ -25,7 +25,7 @@ func TestSandboxCreate(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -77,7 +77,7 @@ func TestSandboxCreateTwice(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - _, err = NewSandbox(key, true) + _, err = NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -85,7 +85,7 @@ func TestSandboxCreateTwice(t *testing.T) { // Create another sandbox with the same key to see if we handle it // gracefully. - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -105,7 +105,7 @@ func TestSandboxGC(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } @@ -127,7 +127,7 @@ func TestAddRemoveInterface(t *testing.T) { t.Fatalf("Failed to obtain a key: %v", err) } - s, err := NewSandbox(key, true) + s, err := NewSandbox(key, true, false) if err != nil { t.Fatalf("Failed to create a new sandbox: %v", err) } diff --git a/osl/sandbox_unsupported.go b/osl/sandbox_unsupported.go index 3bc6c385..51a656c8 100644 --- a/osl/sandbox_unsupported.go +++ b/osl/sandbox_unsupported.go @@ -11,7 +11,7 @@ var ( // NewSandbox provides a new sandbox instance created in an os specific way // provided a key which uniquely identifies the sandbox -func NewSandbox(key string, osCreate bool) (Sandbox, error) { +func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) { return nil, ErrNotImplemented } diff --git a/resolver_unix.go b/resolver_unix.go index 2b3734fb..cec2c7d4 100644 --- a/resolver_unix.go +++ b/resolver_unix.go @@ -19,6 +19,13 @@ func init() { reexec.Register("setup-resolver", reexecSetupResolver) } +const ( + // outputChain used for docker embed dns + outputChain = "DOCKER_OUTPUT" + //postroutingchain used for docker embed dns + postroutingchain = "DOCKER_POSTROUTING" +) + func reexecSetupResolver() { runtime.LockOSThread() defer runtime.UnlockOSThread() @@ -31,10 +38,10 @@ func reexecSetupResolver() { _, ipPort, _ := net.SplitHostPort(os.Args[2]) _, tcpPort, _ := net.SplitHostPort(os.Args[3]) rules := [][]string{ - {"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]}, - {"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, - {"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]}, - {"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, + {"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]}, + {"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, + {"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]}, + {"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort}, } f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0) @@ -50,6 +57,23 @@ func reexecSetupResolver() { os.Exit(3) } + // insert outputChain and postroutingchain + err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain) + if err == nil { + iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain) + } else { + iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain) + iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain) + } + + err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain) + if err == nil { + iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain) + } else { + iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain) + iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain) + } + for _, rule := range rules { if iptables.RawCombinedOutputNative(rule...) != nil { log.Errorf("setting up rule failed, %v", rule) diff --git a/sandbox.go b/sandbox.go index 05f44809..dce169bd 100644 --- a/sandbox.go +++ b/sandbox.go @@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() { osSbox.Destroy() } +func (sb *sandbox) restoreOslSandbox() error { + var routes []*types.StaticRoute + + // restore osl sandbox + Ifaces := make(map[string][]osl.IfaceOption) + for _, ep := range sb.endpoints { + var ifaceOptions []osl.IfaceOption + ep.Lock() + joinInfo := ep.joinInfo + i := ep.iface + ep.Unlock() + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes)) + if i.addrv6 != nil && i.addrv6.IP.To16() != nil { + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6)) + } + if i.mac != nil { + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac)) + } + if len(i.llAddrs) != 0 { + ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs)) + } + Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions + if joinInfo != nil { + for _, r := range joinInfo.StaticRoutes { + routes = append(routes, r) + } + } + if ep.needResolver() { + sb.startResolver() + } + } + + gwep := sb.getGatewayEndpoint() + if gwep == nil { + return nil + } + + // restore osl sandbox + err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6) + if err != nil { + return err + } + + return nil +} + func (sb *sandbox) populateNetworkResources(ep *endpoint) error { sb.Lock() if sb.osSbox == nil { diff --git a/sandbox_dns_unix.go b/sandbox_dns_unix.go index 5a3edba4..2b22e0a2 100644 --- a/sandbox_dns_unix.go +++ b/sandbox_dns_unix.go @@ -139,6 +139,17 @@ func (sb *sandbox) updateParentHosts() error { return nil } +func (sb *sandbox) restorePath() { + if sb.config.resolvConfPath == "" { + sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf" + } + sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash" + if sb.config.hostsPath == "" { + sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts" + } + +} + func (sb *sandbox) setupDNS() error { var newRC *resolvconf.File diff --git a/sandbox_dns_windows.go b/sandbox_dns_windows.go index ef90ddae..f2f58d5b 100644 --- a/sandbox_dns_windows.go +++ b/sandbox_dns_windows.go @@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error { return nil } +func (sb *sandbox) restorePath() { +} + func (sb *sandbox) updateHostsFile(ifaceIP string) error { return nil } diff --git a/sandbox_store.go b/sandbox_store.go index ae5ddc15..5aa48394 100644 --- a/sandbox_store.go +++ b/sandbox_store.go @@ -20,12 +20,13 @@ type epState struct { } type sbState struct { - ID string - Cid string - c *controller - dbIndex uint64 - dbExists bool - Eps []epState + ID string + Cid string + c *controller + dbIndex uint64 + dbExists bool + Eps []epState + EpPriority map[string]int } func (sbs *sbState) Key() []string { @@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error { dstSbs.Cid = sbs.Cid dstSbs.dbIndex = sbs.dbIndex dstSbs.dbExists = sbs.dbExists + dstSbs.EpPriority = sbs.EpPriority for _, eps := range sbs.Eps { dstSbs.Eps = append(dstSbs.Eps, eps) @@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string { func (sb *sandbox) storeUpdate() error { sbs := &sbState{ - c: sb.controller, - ID: sb.id, - Cid: sb.containerID, + c: sb.controller, + ID: sb.id, + Cid: sb.containerID, + EpPriority: sb.epPriority, } retry: @@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error { return sb.controller.deleteFromStore(sbs) } -func (c *controller) sandboxCleanup() { +func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) { store := c.getStore(datastore.LocalScope) if store == nil { logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes") @@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() { controller: sbs.c, containerID: sbs.Cid, endpoints: epHeap{}, - epPriority: map[string]int{}, dbIndex: sbs.dbIndex, isStub: true, dbExists: true, } - sb.osSbox, err = osl.NewSandbox(sb.Key(), true) + msg := " for cleanup" + create := true + isRestore := false + if val, ok := activeSandboxes[sb.ID()]; ok { + msg = "" + sb.isStub = false + isRestore = true + opts := val.([]SandboxOption) + sb.processOptions(opts...) + sb.restorePath() + create = !sb.config.useDefaultSandBox + heap.Init(&sb.endpoints) + } + sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore) if err != nil { - logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err) + logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err) continue } @@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() { ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID} } } - heap.Push(&sb.endpoints, ep) } - logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) - if err := sb.delete(true); err != nil { - logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) + if _, ok := activeSandboxes[sb.ID()]; !ok { + logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID) + if err := sb.delete(true); err != nil { + logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err) + } + continue + } + + // reconstruct osl sandbox field + if !sb.config.useDefaultSandBox { + if err := sb.restoreOslSandbox(); err != nil { + logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID()) + continue + } + } else { + c.sboxOnce.Do(func() { + c.defOsSbox = sb.osSbox + }) + } + + for _, ep := range sb.endpoints { + // Watch for service records + if !c.isAgent() { + c.watchSvcRecord(ep) + } } } }