Skip to content
This repository was archived by the owner on Sep 30, 2020. It is now read-only.

Commit c0087f0

Browse files
kfr2kfr2
committed
Allow dnsmasq to be backed by a local copy of CoreDNS
This commit allows the user to specify that dnsmasq should be backed by a pod-local copy of CoreDNS rather than relying on the global CoreDNS service. If enabled, the dnsmasq-node DaemonSet will be configured to use a local copy of CoreDN for its resolution while setting the global CoreDNS service as a fallback. This is handy in situations where the number of DNS requests within a cluster grows large and causes resolution issues as dnsmasq reaches out to the global CoreDNS service. Additionally, several values passed to dnsmasq are now configurable including its `--cache-size` and `--dns-forward-max`. See [this postmortem](https://github.com/zalando-incubator/kubernetes-on-aws/blob/dev/docs/postmortems/jan-2019-dns-outage.md) for an investigation into this situation which was instrumental in understanding issues we were facing. Many thanks to dominicgunn for providing the manifests which I codified into this commit. --- These features can be enabled and tuned by setting the following values within cluster.yaml: ```yaml kubeDns: dnsmasq: coreDNSLocal: # When enabled, this will run a copy of CoreDNS within each DNS-masq pod and # configure the utility to use it for resolution. enabled: true # Defines the resource requests/limits for the coredns-local container. # cpu and/or memory constraints can be removed by setting the appropriate value(s) # to an empty string. resources: requests: cpu: 50m memory: 100Mi limits: cpu: 50m memory: 100Mi # The size of dnsmasq's cache. cacheSize: 50000 # The maximum number of concurrent DNS queries. dnsForwardMax: 500 # This option gives a default value for time-to-live (in seconds) which dnsmasq # uses to cache negative replies even in the absence of an SOA record. negTTL: 60 ```
1 parent 2260379 commit c0087f0

File tree

5 files changed

+495
-38
lines changed

5 files changed

+495
-38
lines changed

‎builtin/files/cluster.yaml.tmpl

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1167,7 +1167,7 @@ kubernetes:
11671167
# Tells Kubernetes to enable the autoscaler rest client (not using heapster) without the requirement to use metrics-server.
11681168
podAutoscalerUseRestClient:
11691169
enabled: false
1170-
1170+
11711171
# controllerManager:
11721172
# resources:
11731173
# requests:
@@ -1329,6 +1329,35 @@ kubeDns:
13291329
# - --neg-ttl=10
13301330
# - --no-ping
13311331

1332+
# Settings for the dnsmasq-node DaemonSet which must be enabled by setting
1333+
# `kubeDns.nodeLocalResolver` to true.
1334+
dnsmasq:
1335+
coreDNSLocal:
1336+
# When enabled, this will run a copy of CoreDNS within each DNS-masq pod and
1337+
# configure the utility to use it for resolution.
1338+
enabled: false
1339+
1340+
# Defines the resource requests/limits for the coredns-local container.
1341+
# cpu and/or memory constraints can be removed by setting the appropriate value(s)
1342+
# to an empty string.
1343+
resources:
1344+
requests:
1345+
cpu: 50m
1346+
memory: 100Mi
1347+
limits:
1348+
cpu: 50m
1349+
memory: 100Mi
1350+
1351+
# The size of dnsmasq's cache.
1352+
cacheSize: 50000
1353+
1354+
# The maximum number of concurrent DNS queries.
1355+
dnsForwardMax: 500
1356+
1357+
# This option gives a default value for time-to-live (in seconds) which dnsmasq
1358+
# uses to cache negative replies even in the absence of an SOA record.
1359+
# negTTL: 60
1360+
13321361
# When enabled, will modify the TTL of the coredns service.
13331362
# ttl: 30
13341363

@@ -1351,6 +1380,8 @@ kubeDns:
13511380
#
13521381
# This configuration is injected into the CoreDNS config map after the root
13531382
# zone (".") and can be used to add configuration for additional zones.
1383+
# If coreDNSLocal has been enabled, this configuration will additionally be injected
1384+
# into its ConfigMap.
13541385
# additionalZoneCoreDNSConfig: |
13551386
# global:53 {
13561387
# errors
@@ -1378,7 +1409,7 @@ kubeProxy:
13781409
# When enabled, a security group rule is included on the generated kube-aws SG to allow ICMP Ping from all traffic (0.0.0.0/0).
13791410
# This is applied to all nodes (worker & control plane) in the cluster.
13801411
openICMP: true
1381-
1412+
13821413
# Addon features
13831414
addons:
13841415
# When enabled, Kubernetes rescheduler is deployed to the cluster controller(s)

‎builtin/files/userdata/cloud-config-controller

Lines changed: 170 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1127,6 +1127,11 @@ write_files:
11271127
"${mfdir}/kube-dns-de.yaml"
11281128
{{- end }}
11291129
{{ if .KubeDns.NodeLocalResolver -}}
1130+
{{ if .KubeDns.dnsmasq.CoreDNSLocal.Enabled -}}
1131+
deploy "${mfdir}/dnsmasq-node-coredns-local.yaml"
1132+
{{- else }}
1133+
remove "${mfdir}/dnsmasq-node-coredns-local.yaml"
1134+
{{ end -}}
11301135
deploy "${mfdir}/dnsmasq-node-ds.yaml"
11311136
{{ end -}}
11321137
forceapply "${mfdir}/kube-dns-pdb.yaml"
@@ -5350,6 +5355,9 @@ write_files:
53505355
namespace: kube-system
53515356
data:
53525357
Corefile: |
5358+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5359+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5360+
{{- end }}
53535361
.:53 {
53545362
errors
53555363
health
@@ -5372,9 +5380,6 @@ write_files:
53725380
reload
53735381
loadbalance
53745382
}
5375-
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5376-
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 10 }}
5377-
{{- end }}
53785383
{{- else }}
53795384
- path: /srv/kubernetes/manifests/kube-dns-sa.yaml
53805385
content: |
@@ -5440,6 +5445,85 @@ write_files:
54405445
- --v=2
54415446
- --logtostderr
54425447

5448+
{{ if and .KubeDns.NodeLocalResolver .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5449+
- path: /srv/kubernetes/manifests/dnsmasq-node-coredns-local.yaml
5450+
content: |
5451+
apiVersion: v1
5452+
kind: ServiceAccount
5453+
metadata:
5454+
name: dnsmasq
5455+
namespace: kube-system
5456+
---
5457+
apiVersion: rbac.authorization.k8s.io/v1
5458+
kind: ClusterRole
5459+
metadata:
5460+
name: dnsmasq
5461+
rules:
5462+
- apiGroups: [""]
5463+
resources: ["endpoints", "services", "pods", "namespaces"]
5464+
verbs: ["list", "watch"]
5465+
---
5466+
apiVersion: rbac.authorization.k8s.io/v1
5467+
kind: ClusterRoleBinding
5468+
metadata:
5469+
name: dnsmasq
5470+
roleRef:
5471+
apiGroup: rbac.authorization.k8s.io
5472+
kind: ClusterRole
5473+
name: dnsmasq
5474+
subjects:
5475+
- kind: ServiceAccount
5476+
name: dnsmasq
5477+
namespace: kube-system
5478+
---
5479+
apiVersion: rbac.authorization.k8s.io/v1
5480+
kind: RoleBinding
5481+
metadata:
5482+
name: dnsmasq-privileged-psp
5483+
namespace: kube-system
5484+
roleRef:
5485+
apiGroup: rbac.authorization.k8s.io
5486+
kind: ClusterRole
5487+
name: privileged-psp
5488+
subjects:
5489+
- kind: ServiceAccount
5490+
name: dnsmasq
5491+
namespace: kube-system
5492+
---
5493+
apiVersion: v1
5494+
kind: ConfigMap
5495+
metadata:
5496+
name: coredns-local
5497+
namespace: kube-system
5498+
labels:
5499+
application: coredns
5500+
data:
5501+
Corefile: |
5502+
{{- if and (eq .KubeDns.Provider "coredns") .KubeDns.AdditionalZoneCoreDNSConfig }}
5503+
{{ .KubeDns.AdditionalZoneCoreDNSConfig | indent 12 }}
5504+
{{- end }}
5505+
5506+
cluster.local:9254 {{ .PodCIDR }}:9254 {{ .ServiceCIDR }}:9254 {
5507+
errors
5508+
kubernetes {
5509+
pods insecure
5510+
}
5511+
cache 30
5512+
log svc.svc.cluster.local.
5513+
prometheus :9153
5514+
}
5515+
5516+
.:9254 {
5517+
errors
5518+
health :9154 # this is global for all servers
5519+
prometheus :9153
5520+
forward . /etc/resolv.conf
5521+
pprof 127.0.0.1:9156
5522+
cache 30
5523+
reload
5524+
}
5525+
{{ end }}
5526+
54435527
{{ if .KubeDns.NodeLocalResolver }}
54445528
- path: /srv/kubernetes/manifests/dnsmasq-node-ds.yaml
54455529
content: |
@@ -5451,9 +5535,12 @@ write_files:
54515535
labels:
54525536
k8s-app: dnsmasq-node
54535537
spec:
5538+
selector:
5539+
matchLabels:
5540+
k8s-app: dnsmasq-node
54545541
updateStrategy:
54555542
rollingUpdate:
5456-
maxUnavailable: 100%
5543+
maxUnavailable: 10%
54575544
type: RollingUpdate
54585545
selector:
54595546
matchLabels:
@@ -5478,15 +5565,24 @@ write_files:
54785565
configMap:
54795566
name: kube-dns
54805567
optional: true
5568+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5569+
- name: coredns-local-config
5570+
configMap:
5571+
name: coredns-local
5572+
items:
5573+
- key: Corefile
5574+
path: Corefile
5575+
{{ end }}
54815576
containers:
54825577
- name: dnsmasq
54835578
image: {{ .KubeDnsMasqImage.RepoWithTag }}
54845579
livenessProbe:
54855580
httpGet:
54865581
path: /healthcheck/dnsmasq
5487-
port: 10054
5582+
port: 9054
54885583
scheme: HTTP
54895584
initialDelaySeconds: 60
5585+
periodSeconds: 10
54905586
timeoutSeconds: 5
54915587
successThreshold: 1
54925588
failureThreshold: 5
@@ -5497,13 +5593,24 @@ write_files:
54975593
- -restartDnsmasq=true
54985594
- --
54995595
- -k
5500-
- --min-port=1024
5501-
- --cache-size=1000
5596+
- --cache-size={{ .KubeDns.DNSMasq.CacheSize }}
5597+
- --dns-forward-max={{ .KubeDns.DNSMasq.DNSForwardMax }}
5598+
- --log-facility=-
5599+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5600+
- --no-resolv
5601+
- --keep-in-foreground
5602+
- --neg-ttl={{ .KubeDns.DNSMasq.NegTTL }}
5603+
# Send requests to the last server (coredns-local) first and only
5604+
# fallback to the previous one (global coredns) if it's unreachable.
5605+
- --strict-order
5606+
- --server={{.DNSServiceIP}}#53
5607+
- --server=127.0.0.1#9254
5608+
{{ else }}
55025609
- --server=//{{.DNSServiceIP}}
55035610
- --server=/cluster.local/{{.DNSServiceIP}}
55045611
- --server=/in-addr.arpa/{{.DNSServiceIP}}
55055612
- --server=/ip6.arpa/{{.DNSServiceIP}}
5506-
- --log-facility=-
5613+
{{ end }}
55075614
{{- if ne (len .KubeDns.NodeLocalResolverOptions) 0 }}
55085615
{{- range .KubeDns.NodeLocalResolverOptions }}
55095616
- {{.}}
@@ -5519,8 +5626,7 @@ write_files:
55195626
# see: https://github.com/kubernetes/kubernetes/issues/29055 for details
55205627
resources:
55215628
requests:
5522-
cpu: 150m
5523-
memory: 20Mi
5629+
ephemeral-storage: 256Mi
55245630
volumeMounts:
55255631
- name: kube-dns-config
55265632
mountPath: /etc/k8s/dns/dnsmasq-nanny
@@ -5529,7 +5635,7 @@ write_files:
55295635
livenessProbe:
55305636
httpGet:
55315637
path: /metrics
5532-
port: 10054
5638+
port: 9054
55335639
scheme: HTTP
55345640
initialDelaySeconds: 60
55355641
timeoutSeconds: 5
@@ -5538,17 +5644,67 @@ write_files:
55385644
args:
55395645
- --v=2
55405646
- --logtostderr
5647+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5648+
- --probe=dnsmasq,127.0.0.1:9254,ec2.amazonaws.com,5,A
5649+
{{ else }}
55415650
- --probe=dnsmasq,127.0.0.1:53,ec2.amazonaws.com,5,A
5651+
{{ end }}
5652+
- --prometheus-port=9054
55425653
ports:
5543-
- containerPort: 10054
5654+
- containerPort: 9054
55445655
name: metrics
55455656
protocol: TCP
55465657
resources:
55475658
requests:
5548-
memory: 20Mi
5659+
ephemeral-storage: 256Mi
5660+
terminationMessagePath: /dev/termination-log
5661+
terminationMessagePolicy: File
5662+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.Enabled }}
5663+
- name: coredns
5664+
image: {{ .CoreDnsImage.RepoWithTag }}
5665+
args: ["-conf", "/etc/coredns/Corefile"]
5666+
volumeMounts:
5667+
- name: coredns-local-config
5668+
mountPath: /etc/coredns
5669+
ports:
5670+
- containerPort: 9254
5671+
name: dns
5672+
protocol: UDP
5673+
- containerPort: 9254
5674+
name: dns-tcp
5675+
protocol: TCP
5676+
livenessProbe:
5677+
httpGet:
5678+
path: /health
5679+
port: 9154
5680+
scheme: HTTP
5681+
initialDelaySeconds: 60
5682+
timeoutSeconds: 5
5683+
successThreshold: 1
5684+
failureThreshold: 5
5685+
resources:
5686+
requests:
5687+
ephemeral-storage: 256Mi
5688+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5689+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Cpu }}
5690+
{{ end }}
5691+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5692+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Requests.Memory }}
5693+
{{ end }}
5694+
{{ if or .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5695+
limits:
5696+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5697+
cpu: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Cpu }}
5698+
{{ end }}
5699+
{{ if .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5700+
memory: {{ .KubeDns.DNSMasq.CoreDNSLocal.ComputeResources.Limits.Memory }}
5701+
{{ end }}
5702+
{{ end }}
5703+
{{ end }}
55495704
hostNetwork: true
55505705
dnsPolicy: Default
5551-
automountServiceAccountToken: false
5706+
automountServiceAccountToken: true
5707+
serviceAccountName: dnsmasq
55525708
{{ end }}
55535709

55545710
{{- if eq .KubeDns.Provider "coredns" }}

‎pkg/api/cluster.go

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,26 @@ func NewDefaultCluster() *Cluster {
184184
IPVSMode: ipvsMode,
185185
},
186186
KubeDns: KubeDns{
187-
Provider: "coredns",
188-
NodeLocalResolver: false,
187+
Provider: "coredns",
188+
NodeLocalResolver: false,
189+
DNSMasq: DNSMasq{
190+
CoreDNSLocal: CoreDNSLocal{
191+
Enabled: false,
192+
ComputeResources: ComputeResources{
193+
Requests: ResourceQuota{
194+
Cpu: "50m",
195+
Memory: "100Mi",
196+
},
197+
Limits: ResourceQuota{
198+
Cpu: "50m",
199+
Memory: "100Mi",
200+
},
201+
},
202+
},
203+
CacheSize: 50000,
204+
DNSForwardMax: 500,
205+
NegTTL: 60,
206+
},
189207
DeployToControllers: false,
190208
AntiAffinityAvailabilityZone: false,
191209
TTL: 30,

‎pkg/api/types.go

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -194,6 +194,18 @@ type IPVSMode struct {
194194
MinSyncPeriod string `yaml:"minSyncPeriod"`
195195
}
196196

197+
type CoreDNSLocal struct {
198+
Enabled bool `yaml:"enabled"`
199+
ComputeResources ComputeResources `yaml:"resources,omitempty"`
200+
}
201+
202+
type DNSMasq struct {
203+
CoreDNSLocal CoreDNSLocal `yaml:"coreDNSLocal"`
204+
CacheSize int `yaml:"cacheSize"`
205+
DNSForwardMax int `yaml:"dnsForwardMax"`
206+
NegTTL int `yaml:"negTTL"`
207+
}
208+
197209
type KubeDnsAutoscaler struct {
198210
CoresPerReplica int `yaml:"coresPerReplica"`
199211
NodesPerReplica int `yaml:"nodesPerReplica"`
@@ -204,6 +216,7 @@ type KubeDns struct {
204216
Provider string `yaml:"provider"`
205217
NodeLocalResolver bool `yaml:"nodeLocalResolver"`
206218
NodeLocalResolverOptions []string `yaml:"nodeLocalResolverOptions"`
219+
DNSMasq DNSMasq `yaml:"dnsmasq"`
207220
DeployToControllers bool `yaml:"deployToControllers"`
208221
AntiAffinityAvailabilityZone bool `yaml:"antiAffinityAvailabilityZone"`
209222
TTL int `yaml:"ttl"`

0 commit comments

Comments
 (0)