rtyler
/
libnetwork
1
0
Fork 0
Code Issues 1 Pull Requests Releases Wiki Activity

Add network restore to support docker live restore container 

Signed-off-by: Lei Jitang <leijitang@huawei.com>
This commit is contained in:
Lei Jitang 2016-06-10 17:32:19 -07:00 committed by Alessandro Boch
parent 15e7567a2c
commit be153a13e4
18 changed files with 392 additions and 47 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
config/config.go
View File

@ -15,9 +15,10 @@ import (
// Config encapsulates configurations of various Libnetwork components
type Config struct {
Daemon DaemonCfg
Cluster ClusterCfg
Scopes map[string]*datastore.ScopeCfg
Daemon DaemonCfg
Cluster ClusterCfg
Scopes map[string]*datastore.ScopeCfg
ActiveSandboxes map[string]interface{}
}
// DaemonCfg represents libnetwork core configuration
@ -245,3 +246,11 @@ func OptionLocalKVProviderConfig(config *store.Config) Option {
c.Scopes[datastore.LocalScope].Client.Config = config
}
}
// OptionActiveSandboxes function returns an option setter for passing the sandboxes
// which were active during previous daemon life
func OptionActiveSandboxes(sandboxes map[string]interface{}) Option {
return func(c *Config) {
c.ActiveSandboxes = sandboxes
}
}

14
controller.go
View File

@ -203,15 +203,13 @@ func New(cfgOptions ...config.Option) (NetworkController, error) {
}
}
// Reserve pools first before doing cleanup. This is because
// if the pools are not populated properly, the cleanups of
// endpoint/network and sandbox below will not be able to
// release ip subnets and addresses properly into the pool
// because the pools won't exist.
// Reserve pools first before doing cleanup. Otherwise the
// cleanups of endpoint/network and sandbox below will
// generate many unnecessary warnings
c.reservePools()
// Cleanup resources
c.sandboxCleanup()
c.sandboxCleanup(c.cfg.ActiveSandboxes)
c.cleanupLocalEndpoints()
c.networkCleanup()
@ -832,7 +830,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
if sb.config.useDefaultSandBox {
c.sboxOnce.Do(func() {
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false)
c.defOsSbox, err = osl.NewSandbox(sb.Key(), false, false)
})
if err != nil {
@ -844,7 +842,7 @@ func (c *controller) NewSandbox(containerID string, options ...SandboxOption) (s
}
if sb.osSbox == nil && !sb.config.useExternalKey {
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox); err != nil {
if sb.osSbox, err = osl.NewSandbox(sb.Key(), !sb.config.useDefaultSandBox, false); err != nil {
return nil, fmt.Errorf("failed to create new osl sandbox: %v", err)
}
}

2
drivers/bridge/bridge_store.go
View File

@ -184,7 +184,7 @@ func (ncfg *networkConfiguration) Exists() bool {
}
func (ncfg *networkConfiguration) Skip() bool {
return ncfg.DefaultBridge
return false
}
func (ncfg *networkConfiguration) New() datastore.KVObject {

2
drivers/overlay/ov_network.go
View File

@ -512,7 +512,7 @@ func (n *network) initSandbox() error {
n.cleanupStaleSandboxes()
sbox, err := osl.NewSandbox(
osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode)
osl.GenerateKey(fmt.Sprintf("%d-", n.initEpoch)+n.id), !hostMode, false)
if err != nil {
return fmt.Errorf("could not create network sandbox: %v", err)
}

19
endpoint.go
View File

@ -84,6 +84,7 @@ func (ep *endpoint) MarshalJSON() ([]byte, error) {
epMap["name"] = ep.name
epMap["id"] = ep.id
epMap["ep_iface"] = ep.iface
epMap["joinInfo"] = ep.joinInfo
epMap["exposed_ports"] = ep.exposedPorts
if ep.generic != nil {
epMap["generic"] = ep.generic
@ -115,6 +116,9 @@ func (ep *endpoint) UnmarshalJSON(b []byte) (err error) {
ib, _ := json.Marshal(epMap["ep_iface"])
json.Unmarshal(ib, &ep.iface)
jb, _ := json.Marshal(epMap["joinInfo"])
json.Unmarshal(jb, &ep.joinInfo)
tb, _ := json.Marshal(epMap["exposed_ports"])
var tPorts []types.TransportPort
json.Unmarshal(tb, &tPorts)
@ -235,6 +239,11 @@ func (ep *endpoint) CopyTo(o datastore.KVObject) error {
ep.iface.CopyTo(dstEp.iface)
}
if ep.joinInfo != nil {
dstEp.joinInfo = &endpointJoinInfo{}
ep.joinInfo.CopyTo(dstEp.joinInfo)
}
dstEp.exposedPorts = make([]types.TransportPort, len(ep.exposedPorts))
copy(dstEp.exposedPorts, ep.exposedPorts)
@ -1073,6 +1082,13 @@ func (ep *endpoint) releaseAddress() {
}
func (c *controller) cleanupLocalEndpoints() {
// Get used endpoints
eps := make(map[string]interface{})
for _, sb := range c.sandboxes {
for _, ep := range sb.endpoints {
eps[ep.id] = true
}
}
nl, err := c.getNetworksForScope(datastore.LocalScope)
if err != nil {
log.Warnf("Could not get list of networks during endpoint cleanup: %v", err)
@ -1087,6 +1103,9 @@ func (c *controller) cleanupLocalEndpoints() {
}
for _, ep := range epl {
if _, ok := eps[ep.id]; ok {
continue
}
log.Infof("Removing stale endpoint %s (%s)", ep.name, ep.id)
if err := ep.Delete(true); err != nil {
log.Warnf("Could not delete local endpoint %s during endpoint cleanup: %v", ep.name, err)

53
endpoint_info.go
View File

@ -414,3 +414,56 @@ func (ep *endpoint) DisableGatewayService() {
ep.joinInfo.disableGatewayService = true
}
func (epj *endpointJoinInfo) MarshalJSON() ([]byte, error) {
epMap := make(map[string]interface{})
if epj.gw != nil {
epMap["gw"] = epj.gw.String()
}
if epj.gw6 != nil {
epMap["gw6"] = epj.gw6.String()
}
epMap["disableGatewayService"] = epj.disableGatewayService
epMap["StaticRoutes"] = epj.StaticRoutes
return json.Marshal(epMap)
}
func (epj *endpointJoinInfo) UnmarshalJSON(b []byte) error {
var (
err error
epMap map[string]interface{}
)
if err = json.Unmarshal(b, &epMap); err != nil {
return err
}
if v, ok := epMap["gw"]; ok {
epj.gw6 = net.ParseIP(v.(string))
}
if v, ok := epMap["gw6"]; ok {
epj.gw6 = net.ParseIP(v.(string))
}
epj.disableGatewayService = epMap["disableGatewayService"].(bool)
var tStaticRoute []types.StaticRoute
if v, ok := epMap["StaticRoutes"]; ok {
tb, _ := json.Marshal(v)
var tStaticRoute []types.StaticRoute
json.Unmarshal(tb, &tStaticRoute)
}
var StaticRoutes []*types.StaticRoute
for _, r := range tStaticRoute {
StaticRoutes = append(StaticRoutes, &r)
}
epj.StaticRoutes = StaticRoutes
return nil
}
func (epj *endpointJoinInfo) CopyTo(dstEpj *endpointJoinInfo) error {
dstEpj.disableGatewayService = epj.disableGatewayService
dstEpj.StaticRoutes = make([]*types.StaticRoute, len(epj.StaticRoutes))
copy(dstEpj.StaticRoutes, epj.StaticRoutes)
dstEpj.gw = types.GetIPCopy(epj.gw)
dstEpj.gw = types.GetIPCopy(epj.gw6)
return nil
}

2
libnetwork_test.go
View File

@ -1305,7 +1305,7 @@ func externalKeyTest(t *testing.T, reexec bool) {
}
// Create a new OS sandbox using the osl API before using it in SetKey
if extOsBox, err := osl.NewSandbox("ValidKey", true); err != nil {
if extOsBox, err := osl.NewSandbox("ValidKey", true, false); err != nil {
t.Fatalf("Failed to create new osl sandbox")
} else {
defer func() {

151
osl/namespace_linux.go
View File

@ -2,10 +2,13 @@ package osl
import (
"fmt"
"io/ioutil"
"net"
"os"
"os/exec"
"runtime"
"strconv"
"strings"
"sync"
"syscall"
"time"
@ -133,6 +136,39 @@ func GC() {
// container id.
func GenerateKey(containerID string) string {
maxLen := 12
// Read sandbox key from host for overlay
if strings.HasPrefix(containerID, "-") {
var (
index int
indexStr string
tmpkey string
)
dir, err := ioutil.ReadDir(prefix)
if err != nil {
return ""
}
for _, v := range dir {
id := v.Name()
if strings.HasSuffix(id, containerID[:maxLen-1]) {
indexStr = strings.TrimSuffix(id, containerID[:maxLen-1])
tmpindex, err := strconv.Atoi(indexStr)
if err != nil {
return ""
}
if tmpindex > index {
index = tmpindex
tmpkey = id
}
}
}
containerID = tmpkey
if containerID == "" {
return ""
}
}
if len(containerID) < maxLen {
maxLen = len(containerID)
}
@ -142,10 +178,12 @@ func GenerateKey(containerID string) string {
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
err := createNetworkNamespace(key, osCreate)
if err != nil {
return nil, err
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
if !isRestore {
err := createNetworkNamespace(key, osCreate)
if err != nil {
return nil, err
}
}
n := &networkNamespace{path: key, isDefault: !osCreate}
@ -347,3 +385,108 @@ func (n *networkNamespace) Destroy() error {
addToGarbagePaths(n.path)
return nil
}
// Restore restore the network namespace
func (n *networkNamespace) Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error {
// restore interfaces
for name, opts := range ifsopt {
if !strings.Contains(name, "+") {
return fmt.Errorf("wrong iface name in restore osl sandbox interface: %s", name)
}
seps := strings.Split(name, "+")
srcName := seps[0]
dstPrefix := seps[1]
i := &nwIface{srcName: srcName, dstName: dstPrefix, ns: n}
i.processInterfaceOptions(opts...)
if i.master != "" {
i.dstMaster = n.findDst(i.master, true)
if i.dstMaster == "" {
return fmt.Errorf("could not find an appropriate master %q for %q",
i.master, i.srcName)
}
}
if n.isDefault {
i.dstName = i.srcName
} else {
// due to the docker network connect/disconnect, so the dstName should
// restore from the namespace
err := nsInvoke(n.path, func(nsFD int) error { return nil }, func(callerFD int) error {
ifaces, err := net.Interfaces()
if err != nil {
return err
}
for _, iface := range ifaces {
addrs, err := iface.Addrs()
if err != nil {
return err
}
if strings.HasPrefix(iface.Name, "vxlan") {
if i.dstName == "vxlan" {
i.dstName = iface.Name
break
}
}
// find the interface name by ip
if i.address != nil {
for _, addr := range addrs {
if addr.String() == i.address.String() {
i.dstName = iface.Name
break
}
continue
}
if i.dstName == iface.Name {
break
}
}
// This is to find the interface name of the pair in overlay sandbox
if strings.HasPrefix(iface.Name, "veth") {
if i.master != "" && i.dstName == "veth" {
i.dstName = iface.Name
}
}
}
return nil
})
if err != nil {
return err
}
var index int
indexStr := strings.TrimPrefix(i.dstName, dstPrefix)
if indexStr != "" {
index, err = strconv.Atoi(indexStr)
if err != nil {
return err
}
}
index++
n.Lock()
if index > n.nextIfIndex {
n.nextIfIndex = index
}
n.iFaces = append(n.iFaces, i)
n.Unlock()
}
}
// restore routes
for _, r := range routes {
n.Lock()
n.staticRoutes = append(n.staticRoutes, r)
n.Unlock()
}
// restore gateway
if len(gw) > 0 {
n.Lock()
n.gw = gw
n.Unlock()
}
if len(gw6) > 0 {
n.Lock()
n.gwv6 = gw6
n.Unlock()
}
return nil
}

2
osl/namespace_windows.go
View File

@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
return nil, nil
}

3
osl/sandbox.go
View File

@ -58,6 +58,9 @@ type Sandbox interface {
// Destroy the sandbox
Destroy() error
// restore sandbox
Restore(ifsopt map[string][]IfaceOption, routes []*types.StaticRoute, gw net.IP, gw6 net.IP) error
}
// NeighborOptionSetter interface defines the option setter methods for interface options

2
osl/sandbox_freebsd.go
View File

@ -15,7 +15,7 @@ func GenerateKey(containerID string) string {
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
return nil, nil
}

10
osl/sandbox_test.go
View File

@ -25,7 +25,7 @@ func TestSandboxCreate(t *testing.T) {
t.Fatalf("Failed to obtain a key: %v", err)
}
s, err := NewSandbox(key, true)
s, err := NewSandbox(key, true, false)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
@ -77,7 +77,7 @@ func TestSandboxCreateTwice(t *testing.T) {
t.Fatalf("Failed to obtain a key: %v", err)
}
_, err = NewSandbox(key, true)
_, err = NewSandbox(key, true, false)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
@ -85,7 +85,7 @@ func TestSandboxCreateTwice(t *testing.T) {
// Create another sandbox with the same key to see if we handle it
// gracefully.
s, err := NewSandbox(key, true)
s, err := NewSandbox(key, true, false)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
@ -105,7 +105,7 @@ func TestSandboxGC(t *testing.T) {
t.Fatalf("Failed to obtain a key: %v", err)
}
s, err := NewSandbox(key, true)
s, err := NewSandbox(key, true, false)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}
@ -127,7 +127,7 @@ func TestAddRemoveInterface(t *testing.T) {
t.Fatalf("Failed to obtain a key: %v", err)
}
s, err := NewSandbox(key, true)
s, err := NewSandbox(key, true, false)
if err != nil {
t.Fatalf("Failed to create a new sandbox: %v", err)
}

2
osl/sandbox_unsupported.go
View File

@ -11,7 +11,7 @@ var (
// NewSandbox provides a new sandbox instance created in an os specific way
// provided a key which uniquely identifies the sandbox
func NewSandbox(key string, osCreate bool) (Sandbox, error) {
func NewSandbox(key string, osCreate, isRestore bool) (Sandbox, error) {
return nil, ErrNotImplemented
}

32
resolver_unix.go
View File

@ -19,6 +19,13 @@ func init() {
reexec.Register("setup-resolver", reexecSetupResolver)
}
const (
// outputChain used for docker embed dns
outputChain = "DOCKER_OUTPUT"
//postroutingchain used for docker embed dns
postroutingchain = "DOCKER_POSTROUTING"
)
func reexecSetupResolver() {
runtime.LockOSThread()
defer runtime.UnlockOSThread()
@ -31,10 +38,10 @@ func reexecSetupResolver() {
_, ipPort, _ := net.SplitHostPort(os.Args[2])
_, tcpPort, _ := net.SplitHostPort(os.Args[3])
rules := [][]string{
{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
{"-t", "nat", "-A", "OUTPUT", "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
{"-t", "nat", "-A", "POSTROUTING", "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "udp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[2]},
{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "udp", "--sport", ipPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
{"-t", "nat", "-I", outputChain, "-d", resolverIP, "-p", "tcp", "--dport", dnsPort, "-j", "DNAT", "--to-destination", os.Args[3]},
{"-t", "nat", "-I", postroutingchain, "-s", resolverIP, "-p", "tcp", "--sport", tcpPort, "-j", "SNAT", "--to-source", ":" + dnsPort},
}
f, err := os.OpenFile(os.Args[1], os.O_RDONLY, 0)
@ -50,6 +57,23 @@ func reexecSetupResolver() {
os.Exit(3)
}
// insert outputChain and postroutingchain
err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "OUTPUT", "-d", resolverIP, "-j", outputChain)
if err == nil {
iptables.RawCombinedOutputNative("-t", "nat", "-F", outputChain)
} else {
iptables.RawCombinedOutputNative("-t", "nat", "-N", outputChain)
iptables.RawCombinedOutputNative("-t", "nat", "-I", "OUTPUT", "-d", resolverIP, "-j", outputChain)
}
err = iptables.RawCombinedOutputNative("-t", "nat", "-C", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
if err == nil {
iptables.RawCombinedOutputNative("-t", "nat", "-F", postroutingchain)
} else {
iptables.RawCombinedOutputNative("-t", "nat", "-N", postroutingchain)
iptables.RawCombinedOutputNative("-t", "nat", "-I", "POSTROUTING", "-d", resolverIP, "-j", postroutingchain)
}
for _, rule := range rules {
if iptables.RawCombinedOutputNative(rule...) != nil {
log.Errorf("setting up rule failed, %v", rule)

46
sandbox.go
View File

@ -700,6 +700,52 @@ func (sb *sandbox) releaseOSSbox() {
osSbox.Destroy()
}
func (sb *sandbox) restoreOslSandbox() error {
var routes []*types.StaticRoute
// restore osl sandbox
Ifaces := make(map[string][]osl.IfaceOption)
for _, ep := range sb.endpoints {
var ifaceOptions []osl.IfaceOption
ep.Lock()
joinInfo := ep.joinInfo
i := ep.iface
ep.Unlock()
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().Address(i.addr), sb.osSbox.InterfaceOptions().Routes(i.routes))
if i.addrv6 != nil && i.addrv6.IP.To16() != nil {
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().AddressIPv6(i.addrv6))
}
if i.mac != nil {
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().MacAddress(i.mac))
}
if len(i.llAddrs) != 0 {
ifaceOptions = append(ifaceOptions, sb.osSbox.InterfaceOptions().LinkLocalAddresses(i.llAddrs))
}
Ifaces[fmt.Sprintf("%s+%s", i.srcName, i.dstPrefix)] = ifaceOptions
if joinInfo != nil {
for _, r := range joinInfo.StaticRoutes {
routes = append(routes, r)
}
}
if ep.needResolver() {
sb.startResolver()
}
}
gwep := sb.getGatewayEndpoint()
if gwep == nil {
return nil
}
// restore osl sandbox
err := sb.osSbox.Restore(Ifaces, routes, gwep.joinInfo.gw, gwep.joinInfo.gw6)
if err != nil {
return err
}
return nil
}
func (sb *sandbox) populateNetworkResources(ep *endpoint) error {
sb.Lock()
if sb.osSbox == nil {

11
sandbox_dns_unix.go
View File

@ -139,6 +139,17 @@ func (sb *sandbox) updateParentHosts() error {
return nil
}
func (sb *sandbox) restorePath() {
if sb.config.resolvConfPath == "" {
sb.config.resolvConfPath = defaultPrefix + "/" + sb.id + "/resolv.conf"
}
sb.config.resolvConfHashFile = sb.config.resolvConfPath + ".hash"
if sb.config.hostsPath == "" {
sb.config.hostsPath = defaultPrefix + "/" + sb.id + "/hosts"
}
}
func (sb *sandbox) setupDNS() error {
var newRC *resolvconf.File

3
sandbox_dns_windows.go
View File

@ -15,6 +15,9 @@ func (sb *sandbox) setupResolutionFiles() error {
return nil
}
func (sb *sandbox) restorePath() {
}
func (sb *sandbox) updateHostsFile(ifaceIP string) error {
return nil
}

70
sandbox_store.go
View File

@ -20,12 +20,13 @@ type epState struct {
}
type sbState struct {
ID string
Cid string
c *controller
dbIndex uint64
dbExists bool
Eps []epState
ID string
Cid string
c *controller
dbIndex uint64
dbExists bool
Eps []epState
EpPriority map[string]int
}
func (sbs *sbState) Key() []string {
@ -106,6 +107,7 @@ func (sbs *sbState) CopyTo(o datastore.KVObject) error {
dstSbs.Cid = sbs.Cid
dstSbs.dbIndex = sbs.dbIndex
dstSbs.dbExists = sbs.dbExists
dstSbs.EpPriority = sbs.EpPriority
for _, eps := range sbs.Eps {
dstSbs.Eps = append(dstSbs.Eps, eps)
@ -120,9 +122,10 @@ func (sbs *sbState) DataScope() string {
func (sb *sandbox) storeUpdate() error {
sbs := &sbState{
c: sb.controller,
ID: sb.id,
Cid: sb.containerID,
c: sb.controller,
ID: sb.id,
Cid: sb.containerID,
EpPriority: sb.epPriority,
}
retry:
@ -166,7 +169,7 @@ func (sb *sandbox) storeDelete() error {
return sb.controller.deleteFromStore(sbs)
}
func (c *controller) sandboxCleanup() {
func (c *controller) sandboxCleanup(activeSandboxes map[string]interface{}) {
store := c.getStore(datastore.LocalScope)
if store == nil {
logrus.Errorf("Could not find local scope store while trying to cleanup sandboxes")
@ -192,15 +195,27 @@ func (c *controller) sandboxCleanup() {
controller: sbs.c,
containerID: sbs.Cid,
endpoints: epHeap{},
epPriority: map[string]int{},
dbIndex: sbs.dbIndex,
isStub: true,
dbExists: true,
}
sb.osSbox, err = osl.NewSandbox(sb.Key(), true)
msg := " for cleanup"
create := true
isRestore := false
if val, ok := activeSandboxes[sb.ID()]; ok {
msg = ""
sb.isStub = false
isRestore = true
opts := val.([]SandboxOption)
sb.processOptions(opts...)
sb.restorePath()
create = !sb.config.useDefaultSandBox
heap.Init(&sb.endpoints)
}
sb.osSbox, err = osl.NewSandbox(sb.Key(), create, isRestore)
if err != nil {
logrus.Errorf("failed to create new osl sandbox while trying to build sandbox for cleanup: %v", err)
logrus.Errorf("failed to create osl sandbox while trying to restore sandbox %s%s: %v", sb.ID()[0:7], msg, err)
continue
}
@ -222,13 +237,34 @@ func (c *controller) sandboxCleanup() {
ep = &endpoint{id: eps.Eid, network: n, sandboxID: sbs.ID}
}
}
heap.Push(&sb.endpoints, ep)
}
logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
if err := sb.delete(true); err != nil {
logrus.Errorf("failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
if _, ok := activeSandboxes[sb.ID()]; !ok {
logrus.Infof("Removing stale sandbox %s (%s)", sb.id, sb.containerID)
if err := sb.delete(true); err != nil {
logrus.Errorf("Failed to delete sandbox %s while trying to cleanup: %v", sb.id, err)
}
continue
}
// reconstruct osl sandbox field
if !sb.config.useDefaultSandBox {
if err := sb.restoreOslSandbox(); err != nil {
logrus.Errorf("failed to populate fields for osl sandbox %s", sb.ID())
continue
}
} else {
c.sboxOnce.Do(func() {
c.defOsSbox = sb.osSbox
})
}
for _, ep := range sb.endpoints {
// Watch for service records
if !c.isAgent() {
c.watchSvcRecord(ep)
}
}
}
}