Skip to content

Commit dba434c

Browse files
khenidakkhenidak
committed
kubenet for ipv6 dualstack
1 parent 404dbbc commit dba434c

25 files changed

+701
-309
lines changed

‎pkg/kubelet/config/common_test.go

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -63,6 +63,14 @@ func TestDecodeSinglePod(t *testing.T) {
6363
SchedulerName: core.DefaultSchedulerName,
6464
EnableServiceLinks: &enableServiceLinks,
6565
},
66+
Status: v1.PodStatus{
67+
PodIP: "1.2.3.4",
68+
PodIPs: []v1.PodIP{
69+
{
70+
IP: "1.2.3.4",
71+
},
72+
},
73+
},
6674
}
6775
json, err := runtime.Encode(testapi.Default.Codec(), pod)
6876
if err != nil {
@@ -128,6 +136,14 @@ func TestDecodePodList(t *testing.T) {
128136
SchedulerName: core.DefaultSchedulerName,
129137
EnableServiceLinks: &enableServiceLinks,
130138
},
139+
Status: v1.PodStatus{
140+
PodIP: "1.2.3.4",
141+
PodIPs: []v1.PodIP{
142+
{
143+
IP: "1.2.3.4",
144+
},
145+
},
146+
},
131147
}
132148
podList := &v1.PodList{
133149
Items: []v1.Pod{*pod},

‎pkg/kubelet/container/runtime.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -273,8 +273,8 @@ type PodStatus struct {
273273
Name string
274274
// Namespace of the pod.
275275
Namespace string
276-
// IP of the pod.
277-
IP string
276+
// All IPs assigned to this pod
277+
IPs []string
278278
// Status of containers in the pod.
279279
ContainerStatuses []*ContainerStatus
280280
// Status of the pod sandbox.

‎pkg/kubelet/dockershim/docker_sandbox.go

Lines changed: 46 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -324,65 +324,75 @@ func (ds *dockerService) RemovePodSandbox(ctx context.Context, r *runtimeapi.Rem
324324
return nil, utilerrors.NewAggregate(errs)
325325
}
326326

327-
// getIPFromPlugin interrogates the network plugin for an IP.
328-
func (ds *dockerService) getIPFromPlugin(sandbox *dockertypes.ContainerJSON) (string, error) {
327+
// getIPsFromPlugin interrogates the network plugin for sandbox IPs.
328+
func (ds *dockerService) getIPsFromPlugin(sandbox *dockertypes.ContainerJSON) ([]string, error) {
329329
metadata, err := parseSandboxName(sandbox.Name)
330330
if err != nil {
331-
return "", err
331+
return nil, err
332332
}
333333
msg := fmt.Sprintf("Couldn't find network status for %s/%s through plugin", metadata.Namespace, metadata.Name)
334334
cID := kubecontainer.BuildContainerID(runtimeName, sandbox.ID)
335335
networkStatus, err := ds.network.GetPodNetworkStatus(metadata.Namespace, metadata.Name, cID)
336336
if err != nil {
337-
return "", err
337+
return nil, err
338338
}
339339
if networkStatus == nil {
340-
return "", fmt.Errorf("%v: invalid network status for", msg)
340+
return nil, fmt.Errorf("%v: invalid network status for", msg)
341+
}
342+
343+
ips := make([]string, 0)
344+
for _, ip := range networkStatus.IPs {
345+
ips = append(ips, ip.String())
346+
}
347+
// if we don't have any ip in our list then cni is using classic primary IP only
348+
if len(ips) == 0 {
349+
ips = append(ips, networkStatus.IP.String())
341350
}
342-
return networkStatus.IP.String(), nil
351+
return ips, nil
343352
}
344353

345-
// getIP returns the ip given the output of `docker inspect` on a pod sandbox,
354+
// getIPs returns the ip given the output of `docker inspect` on a pod sandbox,
346355
// first interrogating any registered plugins, then simply trusting the ip
347356
// in the sandbox itself. We look for an ipv4 address before ipv6.
348-
func (ds *dockerService) getIP(podSandboxID string, sandbox *dockertypes.ContainerJSON) string {
357+
func (ds *dockerService) getIPs(podSandboxID string, sandbox *dockertypes.ContainerJSON) []string {
349358
if sandbox.NetworkSettings == nil {
350-
return ""
359+
return nil
351360
}
352361
if networkNamespaceMode(sandbox) == runtimeapi.NamespaceMode_NODE {
353362
// For sandboxes using host network, the shim is not responsible for
354363
// reporting the IP.
355-
return ""
364+
return nil
356365
}
357366

358367
// Don't bother getting IP if the pod is known and networking isn't ready
359368
ready, ok := ds.getNetworkReady(podSandboxID)
360369
if ok && !ready {
361-
return ""
370+
return nil
362371
}
363372

364-
ip, err := ds.getIPFromPlugin(sandbox)
373+
ips, err := ds.getIPsFromPlugin(sandbox)
365374
if err == nil {
366-
return ip
375+
return ips
367376
}
368377

378+
ips = make([]string, 0)
369379
// TODO: trusting the docker ip is not a great idea. However docker uses
370380
// eth0 by default and so does CNI, so if we find a docker IP here, we
371381
// conclude that the plugin must have failed setup, or forgotten its ip.
372382
// This is not a sensible assumption for plugins across the board, but if
373383
// a plugin doesn't want this behavior, it can throw an error.
374384
if sandbox.NetworkSettings.IPAddress != "" {
375-
return sandbox.NetworkSettings.IPAddress
385+
ips = append(ips, sandbox.NetworkSettings.IPAddress)
376386
}
377387
if sandbox.NetworkSettings.GlobalIPv6Address != "" {
378-
return sandbox.NetworkSettings.GlobalIPv6Address
388+
ips = append(ips, sandbox.NetworkSettings.GlobalIPv6Address)
379389
}
380390

381391
// If all else fails, warn but don't return an error, as pod status
382392
// should generally not return anything except fatal errors
383393
// FIXME: handle network errors by restarting the pod somehow?
384394
klog.Warningf("failed to read pod IP from plugin/docker: %v", err)
385-
return ""
395+
return ips
386396
}
387397

388398
// Returns the inspect container response, the sandbox metadata, and network namespace mode
@@ -422,11 +432,19 @@ func (ds *dockerService) PodSandboxStatus(ctx context.Context, req *runtimeapi.P
422432
state = runtimeapi.PodSandboxState_SANDBOX_READY
423433
}
424434

425-
var IP string
435+
var ips []string
426436
// TODO: Remove this when sandbox is available on windows
427437
// This is a workaround for windows, where sandbox is not in use, and pod IP is determined through containers belonging to the Pod.
428-
if IP = ds.determinePodIPBySandboxID(podSandboxID); IP == "" {
429-
IP = ds.getIP(podSandboxID, r)
438+
if ips = ds.determinePodIPBySandboxID(podSandboxID); len(ips) == 0 {
439+
ips = ds.getIPs(podSandboxID, r)
440+
}
441+
442+
// ip is primary ips
443+
// ips is all other ips
444+
ip := ""
445+
if len(ips) != 0 {
446+
ip = ips[0]
447+
ips = ips[1:]
430448
}
431449

432450
labels, annotations := extractLabels(r.Config.Labels)
@@ -438,7 +456,7 @@ func (ds *dockerService) PodSandboxStatus(ctx context.Context, req *runtimeapi.P
438456
Labels: labels,
439457
Annotations: annotations,
440458
Network: &runtimeapi.PodSandboxNetworkStatus{
441-
Ip: IP,
459+
Ip: ip,
442460
},
443461
Linux: &runtimeapi.LinuxPodSandboxStatus{
444462
Namespaces: &runtimeapi.Namespace{
@@ -450,6 +468,14 @@ func (ds *dockerService) PodSandboxStatus(ctx context.Context, req *runtimeapi.P
450468
},
451469
},
452470
}
471+
// add additional IPs
472+
additionalPodIPs := make([]*runtimeapi.PodIP, 0, len(ips))
473+
for _, ip := range ips {
474+
additionalPodIPs = append(additionalPodIPs, &runtimeapi.PodIP{
475+
Ip: ip,
476+
})
477+
}
478+
status.Network.AdditionalIps = additionalPodIPs
453479
return &runtimeapi.PodSandboxStatusResponse{Status: status}, nil
454480
}
455481

‎pkg/kubelet/dockershim/docker_sandbox_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -106,7 +106,7 @@ func TestSandboxStatus(t *testing.T) {
106106
State: state,
107107
CreatedAt: ct,
108108
Metadata: config.Metadata,
109-
Network: &runtimeapi.PodSandboxNetworkStatus{Ip: podIP},
109+
Network: &runtimeapi.PodSandboxNetworkStatus{Ip: podIP, AdditionalIps: []*runtimeapi.PodIP{}},
110110
Linux: &runtimeapi.LinuxPodSandboxStatus{
111111
Namespaces: &runtimeapi.Namespace{
112112
Options: &runtimeapi.NamespaceOption{
@@ -142,6 +142,7 @@ func TestSandboxStatus(t *testing.T) {
142142
require.NoError(t, err)
143143
// IP not valid after sandbox stop
144144
expected.Network.Ip = ""
145+
expected.Network.AdditionalIps = []*runtimeapi.PodIP{}
145146
statusResp, err = ds.PodSandboxStatus(getTestCTX(), &runtimeapi.PodSandboxStatusRequest{PodSandboxId: id})
146147
require.NoError(t, err)
147148
assert.Equal(t, expected, statusResp.Status)
@@ -161,14 +162,13 @@ func TestSandboxStatusAfterRestart(t *testing.T) {
161162
config := makeSandboxConfig("foo", "bar", "1", 0)
162163
r := rand.New(rand.NewSource(0)).Uint32()
163164
podIP := fmt.Sprintf("10.%d.%d.%d", byte(r>>16), byte(r>>8), byte(r))
164-
165165
state := runtimeapi.PodSandboxState_SANDBOX_READY
166166
ct := int64(0)
167167
expected := &runtimeapi.PodSandboxStatus{
168168
State: state,
169169
CreatedAt: ct,
170170
Metadata: config.Metadata,
171-
Network: &runtimeapi.PodSandboxNetworkStatus{Ip: podIP},
171+
Network: &runtimeapi.PodSandboxNetworkStatus{Ip: podIP, AdditionalIps: []*runtimeapi.PodIP{}},
172172
Linux: &runtimeapi.LinuxPodSandboxStatus{
173173
Namespaces: &runtimeapi.Namespace{
174174
Options: &runtimeapi.NamespaceOption{

‎pkg/kubelet/dockershim/helpers_linux.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -136,8 +136,8 @@ func (ds *dockerService) updateCreateConfig(
136136
return nil
137137
}
138138

139-
func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
140-
return ""
139+
func (ds *dockerService) determinePodIPBySandboxID(uid string) []string {
140+
return nil
141141
}
142142

143143
func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {

‎pkg/kubelet/dockershim/helpers_unsupported.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -45,9 +45,9 @@ func (ds *dockerService) updateCreateConfig(
4545
return nil
4646
}
4747

48-
func (ds *dockerService) determinePodIPBySandboxID(uid string) string {
48+
func (ds *dockerService) determinePodIPBySandboxID(uid string) []string {
4949
klog.Warningf("determinePodIPBySandboxID is unsupported in this build")
50-
return ""
50+
return nil
5151
}
5252

5353
func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {

‎pkg/kubelet/dockershim/helpers_windows.go

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ func applyWindowsContainerSecurityContext(wsc *runtimeapi.WindowsContainerSecuri
9797
}
9898
}
9999

100-
func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string {
100+
func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) []string {
101101
opts := dockertypes.ContainerListOptions{
102102
All: true,
103103
Filters: dockerfilters.NewArgs(),
@@ -108,7 +108,7 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string {
108108
f.AddLabel(sandboxIDLabelKey, sandboxID)
109109
containers, err := ds.client.ListContainers(opts)
110110
if err != nil {
111-
return ""
111+
return nil
112112
}
113113

114114
for _, c := range containers {
@@ -144,26 +144,26 @@ func (ds *dockerService) determinePodIPBySandboxID(sandboxID string) string {
144144
// Hyper-V only supports one container per Pod yet and the container will have a different
145145
// IP address from sandbox. Return the first non-sandbox container IP as POD IP.
146146
// TODO(feiskyer): remove this workaround after Hyper-V supports multiple containers per Pod.
147-
if containerIP := ds.getIP(c.ID, r); containerIP != "" {
148-
return containerIP
147+
if containerIPs := ds.getIPs(c.ID, r); len(containerIPs) != 0 {
148+
return containerIPs
149149
}
150150
} else {
151151
// Do not return any IP, so that we would continue and get the IP of the Sandbox.
152152
// Windows 1709 and 1803 doesn't have the Namespace support, so getIP() is called
153153
// to replicate the DNS registry key to the Workload container (IP/Gateway/MAC is
154154
// set separately than DNS).
155155
// TODO(feiskyer): remove this workaround after Namespace is supported in Windows RS5.
156-
ds.getIP(sandboxID, r)
156+
ds.getIPs(sandboxID, r)
157157
}
158158
} else {
159159
// ds.getIP will call the CNI plugin to fetch the IP
160-
if containerIP := ds.getIP(c.ID, r); containerIP != "" {
161-
return containerIP
160+
if containerIPs := ds.getIPs(c.ID, r); len(containerIPs) != 0 {
161+
return containerIPs
162162
}
163163
}
164164
}
165165

166-
return ""
166+
return nil
167167
}
168168

169169
func getNetworkNamespace(c *dockertypes.ContainerJSON) (string, error) {

‎pkg/kubelet/dockershim/network/BUILD

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ go_library(
99
importpath = "k8s.io/kubernetes/pkg/kubelet/dockershim/network",
1010
visibility = ["//visibility:public"],
1111
deps = [
12+
"//pkg/features:go_default_library",
1213
"//pkg/kubelet/apis/config:go_default_library",
1314
"//pkg/kubelet/container:go_default_library",
1415
"//pkg/kubelet/dockershim/network/hostport:go_default_library",
@@ -18,6 +19,7 @@ go_library(
1819
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
1920
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
2021
"//staging/src/k8s.io/apimachinery/pkg/util/validation:go_default_library",
22+
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
2123
"//vendor/k8s.io/klog:go_default_library",
2224
"//vendor/k8s.io/utils/exec:go_default_library",
2325
],

‎pkg/kubelet/dockershim/network/cni/cni_others.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -69,12 +69,15 @@ func (plugin *cniNetworkPlugin) GetPodNetworkStatus(namespace string, name strin
6969
return nil, fmt.Errorf("Cannot find the network namespace, skipping pod network status for container %q", id)
7070
}
7171

72-
ip, err := network.GetPodIP(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName)
72+
ips, err := network.GetPodIPs(plugin.execer, plugin.nsenterPath, netnsPath, network.DefaultInterfaceName)
7373
if err != nil {
7474
return nil, err
7575
}
7676

77-
return &network.PodNetworkStatus{IP: ip}, nil
77+
return &network.PodNetworkStatus{
78+
IP: ips[0],
79+
IPs: ips,
80+
}, nil
7881
}
7982

8083
// buildDNSCapabilities builds cniDNSConfig from runtimeapi.DNSConfig.

‎pkg/kubelet/dockershim/network/kubenet/BUILD

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ go_library(
3636
"//pkg/kubelet/dockershim/network:go_default_library",
3737
],
3838
"@io_bazel_rules_go//go/platform:linux": [
39+
"//pkg/features:go_default_library",
3940
"//pkg/kubelet/apis/config:go_default_library",
4041
"//pkg/kubelet/container:go_default_library",
4142
"//pkg/kubelet/dockershim/network:go_default_library",
@@ -48,13 +49,15 @@ go_library(
4849
"//staging/src/k8s.io/apimachinery/pkg/util/errors:go_default_library",
4950
"//staging/src/k8s.io/apimachinery/pkg/util/net:go_default_library",
5051
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
52+
"//staging/src/k8s.io/apiserver/pkg/util/feature:go_default_library",
5153
"//vendor/github.com/containernetworking/cni/libcni:go_default_library",
5254
"//vendor/github.com/containernetworking/cni/pkg/types:go_default_library",
5355
"//vendor/github.com/containernetworking/cni/pkg/types/020:go_default_library",
5456
"//vendor/github.com/vishvananda/netlink:go_default_library",
5557
"//vendor/golang.org/x/sys/unix:go_default_library",
5658
"//vendor/k8s.io/klog:go_default_library",
5759
"//vendor/k8s.io/utils/exec:go_default_library",
60+
"//vendor/k8s.io/utils/net:go_default_library",
5861
],
5962
"@io_bazel_rules_go//go/platform:nacl": [
6063
"//pkg/kubelet/apis/config:go_default_library",
@@ -105,6 +108,7 @@ go_test(
105108
"//pkg/util/bandwidth:go_default_library",
106109
"//pkg/util/iptables/testing:go_default_library",
107110
"//pkg/util/sysctl/testing:go_default_library",
111+
"//staging/src/k8s.io/apimachinery/pkg/util/sets:go_default_library",
108112
"//vendor/github.com/stretchr/testify/assert:go_default_library",
109113
"//vendor/github.com/stretchr/testify/mock:go_default_library",
110114
"//vendor/k8s.io/utils/exec:go_default_library",

0 commit comments

Comments
 (0)