Skip to content
This repository was archived by the owner on Sep 30, 2020. It is now read-only.

Commit e11c828

Browse files
kfr2kfr2
committed
Allow dnsmasq to be backed by a local copy of CoreDNS
This commit allows the user to specify that dnsmasq should be backed by a pod-local copy of CoreDNS rather than relying on the global CoreDNS service. If enabled, the dnsmasq-node DaemonSet will be configured to use a local copy of CoreDN for its resolution while setting the global CoreDNS service as a fallback. This is handy in situations where the number of DNS requests within a cluster grows large and causes resolution issues as dnsmasq reaches out to the global CoreDNS service. Additionally, several values passed to dnsmasq are now configurable including its `--cache-size` and `--dns-forward-max`. See [this postmortem](https://github.com/zalando-incubator/kubernetes-on-aws/blob/dev/docs/postmortems/jan-2019-dns-outage.md) for an investigation into this situation which was instrumental in understanding issues we were facing. Many thanks to dominicgunn for providing the manifests which I codified into this commit. --- These features can be enabled and tuned by setting the following values within cluster.yaml: ```yaml kubeDns: dnsmasq: coreDNSLocal: # When enabled, this will run a copy of CoreDNS within each DNS-masq pod and # configure the utility to use it for resolution. enabled: true # Defines the resource requests/limits for the coredns-local container. # cpu and/or memory constraints can be removed by setting the appropriate value(s) # to an empty string. resources: requests: cpu: 50m memory: 100Mi limits: cpu: 50m memory: 100Mi # The size of dnsmasq's cache. cacheSize: 50000 # The maximum number of concurrent DNS queries. dnsForwardMax: 500 # This option gives a default value for time-to-live (in seconds) which dnsmasq # uses to cache negative replies even in the absence of an SOA record. negTTL: 60 ```
1 parent 2260379 commit e11c828

File tree

5 files changed

+501
-38
lines changed

5 files changed

+501
-38
lines changed

‎builtin/files/cluster.yaml.tmpl

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,7 +1167,7 @@ kubernetes:
11671167
# Tells Kubernetes to enable the autoscaler rest client (not using heapster) without the requirement to use metrics-server.
11681168
podAutoscalerUseRestClient:
11691169
enabled: false
1170-
1170+
11711171
# controllerManager:
11721172
# resources:
11731173
# requests:
@@ -1329,6 +1329,35 @@ kubeDns:
13291329
# - --neg-ttl=10
13301330
# - --no-ping
13311331

1332+
# Settings for the dnsmasq-node DaemonSet which must be enabled by setting
1333+
# `kubeDns.nodeLocalResolver` to true.
1334+
dnsmasq:
1335+
coreDNSLocal:
1336+
# When enabled, this will run a copy of CoreDNS within each DNS-masq pod and
1337+
# configure the utility to use it for resolution.
1338+
enabled: false
1339+
1340+
# Defines the resource requests/limits for the coredns-local container.
1341+
# cpu and/or memory constraints can be removed by setting the appropriate value(s)
1342+
# to an empty string.
1343+
resources:
1344+
requests:
1345+
cpu: 50m
1346+
memory: 100Mi
1347+
limits:
1348+
cpu: 50m
1349+
memory: 100Mi
1350+
1351+
# The size of dnsmasq's cache.
1352+
cacheSize: 50000
1353+
1354+
# The maximum number of concurrent DNS queries.
1355+
dnsForwardMax: 500
1356+
1357+
# This option gives a default value for time-to-live (in seconds) which dnsmasq
1358+
# uses to cache negative replies even in the absence of an SOA record.
1359+
# negTTL: 60
1360+
13321361
# When enabled, will modify the TTL of the coredns service.
13331362
# ttl: 30
13341363

@@ -1351,6 +1380,8 @@ kubeDns:
13511380
#
13521381
# This configuration is injected into the CoreDNS config map after the root
13531382
# zone (".") and can be used to add configuration for additional zones.
1383+
# If coreDNSLocal has been enabled, this configuration will additionally be injected
1384+
# into its ConfigMap.
13541385
# additionalZoneCoreDNSConfig: |
13551386
# global:53 {
13561387
# errors
@@ -1378,7 +1409,7 @@ kubeProxy:
13781409
# When enabled, a security group rule is included on the generated kube-aws SG to allow ICMP Ping from all traffic (0.0.0.0/0).
13791410
# This is applied to all nodes (worker & control plane) in the cluster.
13801411
openICMP: true
1381-
1412+
13821413
# Addon features
13831414
addons:
13841415
# When enabled, Kubernetes rescheduler is deployed to the cluster controller(s)

‎builtin/files/userdata/cloud-config-controller

Lines changed: 176 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,11 @@ write_files:
11271127
"${mfdir}/kube-dns-de.yaml"
11281128
{{- end }}
11291129
{{ if .KubeDns.NodeLocalResolver -}}
1130+
{{ if .KubeDns.dnsmasq.CoreDNSLocal.Enabled -}}
1131+
deploy "${mfdir}/dnsmasq-node-coredns-local.yaml"
1132+
{{- else }}
1133+
remove "${mfdir}/dnsmasq-node-coredns-local.yaml"
1134+
{{ end -}}
11301135
deploy "${mfdir}/dnsmasq-node-ds.yaml"
11311136
{{ end -}}
11321137
forceapply "${mfdir}/kube-dns-pdb.yaml"
@@ -5350,6 +5355,9 @@ write_files:
53505355
namespace: kube-system
53515356
data:
53525357
Corefile: |
5358+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5359+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5360+
{{- end }}
53535361
.:53 {
53545362
errors
53555363
health
@@ -5372,9 +5380,6 @@ write_files:
53725380
reload
53735381
loadbalance
53745382
}
5375-
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5376-
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5377-
{{- end }}
53785383
{{- else }}
53795384
- path: /srv/kubernetes/manifests/kube-dns-sa.yaml
53805385
content: |
@@ -5440,6 +5445,85 @@ write_files:
54405445
- --v=2
54415446
- --logtostderr
54425447

5448+
{{ if and .KubeDns.NodeLocalResolver .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5449+
- path: /srv/kubernetes/manifests/dnsmasq-node-coredns-local.yaml
5450+
content: |
5451+
apiVersion: v1
5452+
kind: ServiceAccount
5453+
metadata:
5454+
name: dnsmasq
5455+
namespace: kube-system
5456+
---
5457+
apiVersion: rbac.authorization.k8s.io/v1
5458+
kind: ClusterRole
5459+
metadata:
5460+
name: dnsmasq
5461+
rules:
5462+
- apiGroups: [""]
5463+
resources: ["endpoints", "services", "pods", "namespaces"]
5464+
verbs: ["list", "watch"]
5465+
---
5466+
apiVersion: rbac.authorization.k8s.io/v1
5467+
kind: ClusterRoleBinding
5468+
metadata:
5469+
name: dnsmasq
5470+
roleRef:
5471+
apiGroup: rbac.authorization.k8s.io
5472+
kind: ClusterRole
5473+
name: dnsmasq
5474+
subjects:
5475+
- kind: ServiceAccount
5476+
name: dnsmasq
5477+
namespace: kube-system
5478+
---
5479+
apiVersion: rbac.authorization.k8s.io/v1
5480+
kind: RoleBinding
5481+
metadata:
5482+
name: dnsmasq-privileged-psp
5483+
namespace: kube-system
5484+
roleRef:
5485+
apiGroup: rbac.authorization.k8s.io
5486+
kind: ClusterRole
5487+
name: privileged-psp
5488+
subjects:
5489+
- kind: ServiceAccount
5490+
name: dnsmasq
5491+
namespace: kube-system
5492+
---
5493+
apiVersion: v1
5494+
kind: ConfigMap
5495+
metadata:
5496+
name: coredns-local
5497+
namespace: kube-system
5498+
labels:
5499+
application: coredns
5500+
data:
5501+
Corefile: |
5502+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5503+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 12 }}
5504+
{{- end }}
5505+
5506+
cluster.local:9254 {{ .PodCIDR }}:9254 {{ .ServiceCIDR }}:9254 {
5507+
errors
5508+
kubernetes {
5509+
pods insecure
5510+
}
5511+
cache 30
5512+
log svc.svc.cluster.local.
5513+
prometheus :9153
5514+
}
5515+
5516+
.:9254 {
5517+
errors
5518+
health :9154 # this is global for all servers
5519+
prometheus :9153
5520+
forward . /etc/resolv.conf
5521+
pprof 127.0.0.1:9156
5522+
cache 30
5523+
reload
5524+
}
5525+
{{ end }}
5526+
54435527
{{ if .KubeDns.NodeLocalResolver }}
54445528
- path: /srv/kubernetes/manifests/dnsmasq-node-ds.yaml
54455529
content: |
@@ -5451,9 +5535,12 @@ write_files:
54515535
labels:
54525536
k8s-app: dnsmasq-node
54535537
spec:
5538+
selector:
5539+
matchLabels:
5540+
k8s-app: dnsmasq-node
54545541
updateStrategy:
54555542
rollingUpdate:
5456-
maxUnavailable: 100%
5543+
maxUnavailable: 10%
54575544
type: RollingUpdate
54585545
selector:
54595546
matchLabels:
@@ -5478,15 +5565,24 @@ write_files:
54785565
configMap:
54795566
name: kube-dns
54805567
optional: true
5568+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5569+
- name: coredns-local-config
5570+
configMap:
5571+
name: coredns-local
5572+
items:
5573+
- key: Corefile
5574+
path: Corefile
5575+
{{ end }}
54815576
containers:
54825577
- name: dnsmasq
54835578
image: {{ .KubeDnsMasqImage.RepoWithTag }}
54845579
livenessProbe:
54855580
httpGet:
54865581
path: /healthcheck/dnsmasq
5487-
port: 10054
5582+
port: 9054
54885583
scheme: HTTP
54895584
initialDelaySeconds: 60
5585+
periodSeconds: 10
54905586
timeoutSeconds: 5
54915587
successThreshold: 1
54925588
failureThreshold: 5
@@ -5497,13 +5593,24 @@ write_files:
54975593
- -restartDnsmasq=true
54985594
- --
54995595
- -k
5500-
- --min-port=1024
5501-
- --cache-size=1000
5596+
- --cache-size={{ .KubeDns.DNSMasq.CacheSize }}
5597+
- --dns-forward-max={{ .KubeDns.DNSMasq.DNSForwardMax }}
5598+
- --log-facility=-
5599+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5600+
- --no-resolv
5601+
- --keep-in-foreground
5602+
- --neg-ttl={{ .KubeDns.DNSMasq.NegTTL }}
5603+
# Send requests to the last server (coredns-local) first and only
5604+
# fallback to the previous one (global coredns) if it's unreachable.
5605+
- --strict-order
5606+
- --server={{.DNSServiceIP}}#53
5607+
- --server=127.0.0.1#9254
5608+
{{ else }}
55025609
- --server=//{{.DNSServiceIP}}
55035610
- --server=/cluster.local/{{.DNSServiceIP}}
55045611
- --server=/in-addr.arpa/{{.DNSServiceIP}}
55055612
- --server=/ip6.arpa/{{.DNSServiceIP}}
5506-
- --log-facility=-
5613+
{{ end }}
55075614
{{- if ne (len .KubeDns.NodeLocalResolverOptions) 0 }}
55085615
{{- range .KubeDns.NodeLocalResolverOptions }}
55095616
- {{.}}
@@ -5519,8 +5626,10 @@ write_files:
55195626
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
55205627
resources:
55215628
requests:
5522-
cpu: 150m
5523-
memory: 20Mi
5629+
ephemeral-storage: 256Mi
5630+
limits:
5631+
cpu: 10m
5632+
memory: 45Mi
55245633
volumeMounts:
55255634
- name: kube-dns-config
55265635
mountPath: /etc/k8s/dns/dnsmasq-nanny
@@ -5529,7 +5638,7 @@ write_files:
55295638
livenessProbe:
55305639
httpGet:
55315640
path: /metrics
5532-
port: 10054
5641+
port: 9054
55335642
scheme: HTTP
55345643
initialDelaySeconds: 60
55355644
timeoutSeconds: 5
@@ -5538,17 +5647,70 @@ write_files:
55385647
args:
55395648
- --v=2
55405649
- --logtostderr
5650+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5651+
- --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
5652+
{{ else }}
55415653
- --probe=dnsmasq,127.0.0.1:53,ec2.amazonaws.com,5,A
5654+
{{ end }}
5655+
- --prometheus-port=9054
55425656
ports:
5543-
- containerPort: 10054
5657+
- containerPort: 9054
55445658
name: metrics
55455659
protocol: TCP
55465660
resources:
55475661
requests:
5548-
memory: 20Mi
5662+
ephemeral-storage: 256Mi
5663+
limits:
5664+
cpu: 100m
5665+
memory: 50Mi
5666+
terminationMessagePath: /dev/termination-log
5667+
terminationMessagePolicy: File
5668+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5669+
- name: coredns
5670+
image: {{ .CoreDnsImage.RepoWithTag }}
5671+
args: ["-conf", "/etc/coredns/Corefile"]
5672+
volumeMounts:
5673+
- name: coredns-local-config
5674+
mountPath: /etc/coredns
5675+
ports:
5676+
- containerPort: 9254
5677+
name: dns
5678+
protocol: UDP
5679+
- containerPort: 9254
5680+
name: dns-tcp
5681+
protocol: TCP
5682+
livenessProbe:
5683+
httpGet:
5684+
path: /health
5685+
port: 9154
5686+
scheme: HTTP
5687+
initialDelaySeconds: 60
5688+
timeoutSeconds: 5
5689+
successThreshold: 1
5690+
failureThreshold: 5
5691+
resources:
5692+
requests:
5693+
ephemeral-storage: 256Mi
5694+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5695+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5696+
{{ end }}
5697+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5698+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5699+
{{ end }}
5700+
{{ if or .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5701+
limits:
5702+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5703+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5704+
{{ end }}
5705+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5706+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5707+
{{ end }}
5708+
{{ end }}
5709+
{{ end }}
55495710
hostNetwork: true
55505711
dnsPolicy: Default
5551-
automountServiceAccountToken: false
5712+
automountServiceAccountToken: true
5713+
serviceAccountName: dnsmasq
55525714
{{ end }}
55535715

55545716
{{- if eq .KubeDns.Provider "coredns" }}

‎pkg/api/cluster.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,26 @@ func NewDefaultCluster() *Cluster {
184184
IPVSMode: ipvsMode,
185185
},
186186
KubeDns: KubeDns{
187-
Provider: "coredns",
188-
NodeLocalResolver: false,
187+
Provider: "coredns",
188+
NodeLocalResolver: false,
189+
DNSMasq: DNSMasq{
190+
CoreDNSLocal: CoreDNSLocal{
191+
Enabled: false,
192+
ComputeResources: ComputeResources{
193+
Requests: ResourceQuota{
194+
Cpu: "50m",
195+
Memory: "100Mi",
196+
},
197+
Limits: ResourceQuota{
198+
Cpu: "50m",
199+
Memory: "100Mi",
200+
},
201+
},
202+
},
203+
CacheSize: 50000,
204+
DNSForwardMax: 500,
205+
NegTTL: 60,
206+
},
189207
DeployToControllers: false,
190208
AntiAffinityAvailabilityZone: false,
191209
TTL: 30,

0 commit comments

Comments
 (0)