Manage Kubernetes Secrets
secrets - passords, api keys, creds
k8s secrets source to deployment flow:
code --> git --> api server --> kubelet:pod secret --> api server --> kubelet:pod
simple secret scenario
k create secret generic secret1 --from-literal user=admin
k create secret generic secret2 --from-literal pass=12345
k run pod --image=nginx --dry-run=client -oyaml > pod.yaml
edit pod.yaml add secrets mounts and env var
apiVersion: v1
kind: Pod
metadata:
creationTimestamp: null
labels:
run: pod
name: pod
spec:
containers:
- image: nginx
name: pod
resources: {}
env:
- name: PASSWORD
valueFrom:
secretKeyRef:
name: secret2
key: pass
volumeMounts:
- name: secret1
mountPath: "/etc/secret1"
readOnly: true
volumes:
- name: secret1
secret:
secretName: secret1
dnsPolicy: ClusterFirst
restartPolicy: Always
status: {}
create it k create -f ./pod.yaml
check our ENV var shows our password: k exec pod -- env | grep PASS
check our mount shows our user: k exec pod -- cat /etc/secret1/user
Hack secrets in Docker
on worker node, run docker find out container
root@cks-worker:~# docker ps | grep nginx
b74bd0f17bba nginx "/docker-entrypoint.…" 21 seconds ago Up 20 seconds k8s_pod_pod_default_ce59b0e2-7c2c-4392-a769-49c50c4d75ce_0
inspect that container id and look through the filesystem to find secrets
root@cks-worker:~# docker inspect b74bd0f17bba | grep -i pass
"PASSWORD=12345",
# look at these mounts
root@cks-worker:~# docker inspect b74bd0f17bba | grep -i volume
"/var/lib/kubelet/pods/ce59b0e2-7c2c-4392-a769-49c50c4d75ce/volumes/kubernetes.io~secret/secret1:/etc/secret1:ro",
"/var/lib/kubelet/pods/ce59b0e2-7c2c-4392-a769-49c50c4d75ce/volumes/kubernetes.io~projected/kube-api-access-5khc6:/var/run/secrets/kubernetes.io/serviceaccount:ro",
"VolumeDriver": "",
"VolumesFrom": null,
"Source": "/var/lib/kubelet/pods/ce59b0e2-7c2c-4392-a769-49c50c4d75ce/volumes/kubernetes.io~secret/secret1",
"Source": "/var/lib/kubelet/pods/ce59b0e2-7c2c-4392-a769-49c50c4d75ce/volumes/kubernetes.io~projected/kube-api-access-5khc6",
"Volumes": null,
copy this container filesystem to local
root@cks-worker:~# docker cp b74bd0f17bba:/etc/secret1 secret1
root@cks-worker:~# cd secret1/
root@cks-worker:~/secret1# ls
user
root@cks-worker:~/secret1# cat user
admin
Hack Secrets in ETCD
commands used
note: had to install etcdctl root@cks-master:~# apt install -y etcd-client
# access secret int etcd
root@cks-master:~# cat /etc/kubernetes/manifests/kube-apiserver.yaml | grep etcd
- --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt
- --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt
- --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key
- --etcd-servers=https://127.0.0.1:2379
# check health etcd
root@cks-master:~# ETCDCTL_API=3 etcdctl endpoint health
{"level":"warn","ts":"2021-09-04T22:27:00.050Z","caller":"clientv3/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"endpoint://client-7340418a-6871-4270-ad79-7d02f2a2bb48/127.0.0.1:2379","attempt":0,"error":"rpc error: code = DeadlineExceeded desc = latest connection error: connection closed"}
127.0.0.1:2379 is unhealthy: failed to commit proposal: context deadline exceeded
Error: unhealthy cluster
# use the creds as per kube-apiserver.yaml
ETCDCTL_API=3 etcdctl --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key --cacert /etc/kubernetes/pki/etcd/ca.crt endpoint health
127.0.0.1:2379 is healthy: successfully committed proposal: took = 1.076931ms
# --endpoints "https://127.0.0.1:2379" not necessary because we’re on same node
# get secrets from `/registry/secrets/default/secret1`
root@cks-master:~# ETCDCTL_API=3 etcdctl --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key --cacert /etc/kubernetes/pki/etcd/ca.crt get /registry/secrets/default/secret1
/registry/secrets/default/secret1
k8s
v1Secret�
�
secret1default"*$952203d4-d5f6-4bbe-8f57-386c638c93bf2��ωz�_
kubectl-createUpdatev��ωFieldsV1:-
+{"f:data":{".":{},"f:user":{}},"f:type":{}}
useradminOpaque"
# see the key-pair = "useradmin"? now get password
root@cks-master:~# ETCDCTL_API=3 etcdctl --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key --cacert /etc/kubernetes/pki/etcd/ca.crt get /registry/secrets/default/secret2
/registry/secrets/default/secret2
k8s
v1Secret�
�
secret2default"*$14c16e50-0eb3-469a-9b96-be79622d08ac2��ωz�_
kubectl-createUpdatev��ωFieldsV1:-
+{"f:data":{".":{},"f:pass":{}},"f:type":{}}
pass12345Opaque"
lesson here - unencrypted plain text secrets stored in ETCD.
Encrypt ETCD at rest
API server's only thing allowed to talk to ETCD, so its responsible for encrypt/decrypt in this flow.
read etcd docs page for encrypt/decrypt secrets.
Resource created kind EncryptionConfiguration
- specify what resources to be encrypted by this config e.g. secrets
- providers list array of encryption providers
- the
identity: {}provider is a "no encryption" provider - e.g. aesgcm, aescbc
- the
important: on STORE - providers used in ORDER - so only the first one used. on READ, it will go through ALL providers available to process what it needs to read.
pro-tip - to re-create ALL secrets in ALL namespaces if you want to enforce a new EncryptionConfiguration resource, so the following
kubectl get secrets --all-namespaces -o json | kubectl replace -f -
hands-on example
I am following along the k8s docs myself first.
# create secret
head -c 32 /dev/urandom | base64
save following to /etc/kubernetes/etcd/ec.yaml
apiVersion: apiserver.config.k8s.io/v1
kind: EncryptionConfiguration
resources:
- resources:
- secrets
providers:
- aescbc:
keys:
- name: key1
secret: fNtJ9NkCpeJabcKMocX07jGu1hcL8bUvGvjH2PSIs24=
- identity: {}
update /etc/kubernetes/manifests/kube-apiserver.yaml to add the provider config
spec:
containers:
- command:
- kube-apiserver
- --encryption-provider-config=/etc/kubernetes/etcd/ec.yaml
- --advertise-address=10.152.0.2
- --allow-privileged=true
- --authorization-mode=Node,RBAC
add mount for this file ec.yaml for the kube-apiserver to use
# mount
containers:
...
volumeMounts:
- mountPath: /etc/kubernetes/etcd
name: etcd
# volume
volumes:
- hostPath:
path: /etc/kubernetes/etcd
type: DirectoryOrCreate
name: etcd
API server will restart itself, check:
root@cks-master:/etc/systemd/system/kubelet.service.d# ps aux | grep apiserver
root 3010 9.9 9.4 1101516 380920 ? Ssl 23:17 0:14 kube-apiserver --encryption-provider-config=/etc/kubernetes/etcd/ec.yaml --advertise-address=10.152.0.2 --allow-privileged=true --authorization-mode=Node,RBAC --client-ca-file=/etc/kubernetes/pki/ca.crt --enable-admission-plugins=NodeRestriction --enable-bootstrap-token-auth=true --etcd-cafile=/etc/kubernetes/pki/etcd/ca.crt --etcd-certfile=/etc/kubernetes/pki/apiserver-etcd-client.crt --etcd-keyfile=/etc/kubernetes/pki/apiserver-etcd-client.key --etcd-servers=https://127.0.0.1:2379 --insecure-port=0 --kubelet-client-certificate=/etc/kubernetes/pki/apiserver-kubelet-client.crt --kubelet-client-key=/etc/kubernetes/pki/apiserver-kubelet-client.key --kubelet-preferred-address-types=InternalIP,ExternalIP,Hostname --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key --requestheader-allowed-names=front-proxy-client --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt --requestheader-extra-headers-prefix=X-Remote-Extra- --requestheader-group-headers=X-Remote-Group --requestheader-username-headers=X-Remote-User --secure-port=6443 --service-account-issuer=https://kubernetes.default.svc.cluster.local --service-account-key-file=/etc/kubernetes/pki/sa.pub --service-account-signing-key-file=/etc/kubernetes/pki/sa.key --service-cluster-ip-range=10.96.0.0/12 --tls-cert-file=/etc/kubernetes/pki/apiserver.crt --tls-private-key-file=/etc/kubernetes/pki/apiserver.key
or
root@cks-master:/etc/systemd/system/kubelet.service.d# docker ps | grep api
8fc3c24b04fd 4d217480042e "kube-apiserver --en…" 3 minutes ago Up 3 minutes k8s_kube-apiserver_kube-apiserver-cks-master_kube-system_0fcea4060dc98eb9e7237787e39dca2b_0
a8a8ccb66f88 k8s.gcr.io/pause:3.4.1 "/pause" 3 minutes ago Up 3 minutes k8s_POD_kube-apiserver-cks-master_kube-system_0fcea4060dc98eb9e7237787e39dca2b_0
if you need to troubleshoot
root@cks-master:/etc/systemd/system/kubelet.service.d# ll /var/log/pods/
total 40
drwxr-xr-x 10 root root 4096 Sep 4 23:17 ./
drwxrwxr-x 11 root syslog 4096 Sep 2 06:25 ../
drwxr-xr-x 3 root root 4096 Aug 8 22:34 kube-system_coredns-558bd4d5db-kf8j9_ecb61ac8-aa71-445f-ace2-af33bd88b44f/
drwxr-xr-x 3 root root 4096 Aug 8 22:34 kube-system_coredns-558bd4d5db-nvqqp_e3d25308-e5b6-40c9-a803-b357ac189c22/
drwxr-xr-x 3 root root 4096 Aug 8 22:33 kube-system_etcd-cks-master_59e2fab1e68569ba366888edc129a284/
drwxr-xr-x 3 root root 4096 Sep 4 23:17 kube-system_kube-apiserver-cks-master_0fcea4060dc98eb9e7237787e39dca2b/
drwxr-xr-x 3 root root 4096 Aug 8 22:33 kube-system_kube-controller-manager-cks-master_b9a186e3d4d2fbecf180a0923a24ebcd/
drwxr-xr-x 3 root root 4096 Aug 8 22:34 kube-system_kube-proxy-rzbsd_25749ba6-4d3c-4f2e-a7c6-48a37ff03aa4/
drwxr-xr-x 3 root root 4096 Aug 8 22:33 kube-system_kube-scheduler-cks-master_4eedadadc796f45ef17aba32e2cc0d32/
drwxr-xr-x 5 root root 4096 Aug 8 22:34 kube-system_weave-net-2d9st_b91f6248-8b11-44ff-afec-79a7d3685ef2/
read secrets
read the default token
root@cks-master:/etc/systemd/system/kubelet.service.d# k get secrets default-token-nx5f5 -oyamlapiVersion: v1
data:
ca.crt: LS0tLS1CRUdJTiBDRVJUSUZJQ0FURS0tLS0tCk1JSUM1ekNDQWMrZ0F3SUJBZ0lCQURBTkJna3Foa2lHOXcwQkFRc0ZBREFWTVJNd0VRWURWUVFERXdwcmRXSmwKY201bGRHVnpNQjRYRFRJeE1EZ3dPREl5TXpJeE9Gb1hEVE14TURnd05qSXlNekl4T0Zvd0ZURVRNQkVHQTFVRQpBeE1LYTNWaVpYSnVaWFJsY3pDQ0FTSXdEUVlKS29aSWh2Y05BUUVCQlFBRGdnRVBBRENDQVFvQ2dnRUJBTnNkCnR1K3dNRnJkeUdsSlpaUC9VRFFsTlVEeGFhQ2NFT2x6YXRHL3lwNTNBWTJNRG5PYmxHaXkrblh4S2JWd1BYNE8KdFdDdHh6bWhaUVJzMWpmODZuMkhOZGREWmhpVXMvd2dScUUxVC9ucUE2QTFsZFBuMm03L1BMQW5oTHFJTmJZeQo5NklXc2V0ZGxQRTkzSHQ0S2czU1RMUWJHR09ZVktCL2lMZnRpcUQzWUtzL29mZTA2ZTRMQ2RXc25nNmdsOSt0ClRTL0gyWittN2NJUDhLbW94cy9GMTltUW1NMTl2VnZTUStwbkNBTjJrSGRwbnF5STVjUTN5OUhUck5uOFRQQ3cKTWd0YW5kTXhpeEpzTlN2UnA0RG13cnhQYWp4dXZVblhjZXNRczRkSVYrS0pNL0F5TlQ1aVRiQzBDRUZYRTZPagpHNVhrL2d0ZHZadFg1YmN1S29FQ0F3RUFBYU5DTUVBd0RnWURWUjBQQVFIL0JBUURBZ0trTUE4R0ExVWRFd0VCCi93UUZNQU1CQWY4d0hRWURWUjBPQkJZRUZES1d3Q1V0Rm5sRGtQTHBkU2c1YnNxTGNjSUpNQTBHQ1NxR1NJYjMKRFFFQkN3VUFBNElCQVFDazBUeHNFV2dOT2NCNWdUcTBld3JwTFVOY3M1dUl5ZnlBSHowM0M5SDBJSU12UXJtcwpRL2wvemRVRTB4dG1PTi9mWWduVG5MUzhiSmVxSHpsM1FGeG9OMDBnTS9JNVlxSmtqZENudEN4eTljOENYNTl6ClJ2UlpuVTN4WXdYbDVMMU4waURaZ1hxK2JmUVJGK2tHZnJkNVpTZUxvczNkbUNVMmhMUm94alBNb2ZlVjZvcVcKaXRZQVBPTytQSS9MQjN3WXcraVEzRWE0cjFXU3RnVnVVU0xHVThUZHlhNWRiSTFsVW9nY29ZS0FIN3JsQUc0LwpSK2hTUGo0T1dwWXg0QVdXNDJaUE03RWJ4dTlXcUZ6ajJNSDJlVG11NmhrYlVqR1J6d3VkMHY2eFp4YnJCNjhPCnN2TXNKVUtWcW1UcWZIeWFGajZKU1Q4UmhtWmFYL3VrZGViNAotLS0tLUVORCBDRVJUSUZJQ0FURS0tLS0tCg==
namespace: ZGVmYXVsdA==
token: ZXlKaGJHY2lPaUpTVXpJMU5pSXNJbXRwWkNJNklqQlRUVnBMTldKM1VGbGtSa1p6VDNSNWMxZDJXalI2YldWbFZVRTRaVGcyTTFOZlFXaFNlRUpOU1hNaWZRLmV5SnBjM01pT2lKcmRXSmxjbTVsZEdWekwzTmxjblpwWTJWaFkyTnZkVzUwSWl3aWEzVmlaWEp1WlhSbGN5NXBieTl6WlhKMmFXTmxZV05qYjNWdWRDOXVZVzFsYzNCaFkyVWlPaUprWldaaGRXeDBJaXdpYTNWaVpYSnVaWFJsY3k1cGJ5OXpaWEoyYVdObFlXTmpiM1Z1ZEM5elpXTnlaWFF1Ym1GdFpTSTZJbVJsWm1GMWJIUXRkRzlyWlc0dGJuZzFaalVpTENKcmRXSmxjbTVsZEdWekxtbHZMM05sY25acFkyVmhZMk52ZFc1MEwzTmxjblpwWTJVdFlXTmpiM1Z1ZEM1dVlXMWxJam9pWkdWbVlYVnNkQ0lzSW10MVltVnlibVYwWlhNdWFXOHZjMlZ5ZG1salpXRmpZMjkxYm5RdmMyVnlkbWxqWlMxaFkyTnZkVzUwTG5WcFpDSTZJalUxT1dSaU5qZzRMVEl3Wm1FdE5EWXpNaTA0WVRVeUxXRTNORGRoWmpRMVl6VXlNaUlzSW5OMVlpSTZJbk41YzNSbGJUcHpaWEoyYVdObFlXTmpiM1Z1ZERwa1pXWmhkV3gwT21SbFptRjFiSFFpZlEubVpjX2RzVXhmZWczOHItdklFbVNqVXc0UXBFR2JtMkpGZmMwRE83NlZwblpvd21mOENNNlZPd2pvZmJGQVV5NW0waEpnWFdfaUNRSVAwR3J3ZTZidHI3ZWZOcnVvUXJKbm5meDVrZFo2VURBWVNhRUJLMllfZFlGOGZid3I0ZzRrRElXSzJrSFBxSHNHTjZWX0FJZ3R4VFg0T1NmWDg4S3Roa19EQzJ4Z2tEYTlKMUI0dXI3dTJnMjZfRVVBODlVbndDdURmTWQyOW03WmhBc281VVJNa1o5TTl2SVZjMmczZFJkamlOSWhLeGZsRHFWR2pacTBjNC11Tzd3NE5xZVo2MjczWm5YLXdnNi10SU1pOFJFNUpkWUZ6R2x0blkxb25idUU3bVNBWnBROUFBYWp4U3AtU2xFV1EyUG42WTVBODRzWWlQWlY5ZmdDNE9tek5Hd1RR
kind: Secret
metadata:
annotations:
kubernetes.io/service-account.name: default
kubernetes.io/service-account.uid: 559db688-20fa-4632-8a52-a747af45c522
creationTimestamp: "2021-08-08T22:33:47Z"
name: default-token-nx5f5
namespace: default
resourceVersion: "396"
uid: 1fe556a9-f84d-4e55-b79d-76a24edd4c77
type: kubernetes.io/service-account-token
read the default token key-pair in etcd
root@cks-master:/etc/systemd/system/kubelet.service.d# ETCDCTL_API=3 etcdctl --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key --cacert /etc/kubernetes/pki/etcd/ca.crt get /registry/secrets/default/default-token-nx5f5
/registry/secrets/default/default-token-nx5f5
k8s
v1Secret�
�
default-token-nx5f5default"*$1fe556a9-f84d-4e55-b79d-76a24edd4c772˷��b-
"kubernetes.io/service-account.namedefaultbI
!kubernetes.io/service-account.uid$559db688-20fa-4632-8a52-a747af45c522z��
kube-controller-managerUpdatev˷��FieldsV1:�
�{"f:data":{".":{},"f:ca.crt":{},"f:namespace":{},"f:token":{}},"f:metadata":{"f:annotations":{".":{},"f:kubernetes.io/service-account.name":{},"f:kubernetes.io/service-account.uid":{}}},"f:type":{}}�
ca.crt-----BEGIN CERTIFICATE-----
MIIC5zCCAc+gAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
cm5ldGVzMB4XDTIxMDgwODIyMzIxOFoXDTMxMDgwNjIyMzIxOFowFTETMBEGA1UE
AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANsd
tu+wMFrdyGlJZZP/UDQlNUDxaaCcEOlzatG/yp53AY2MDnOblGiy+nXxKbVwPX4O
tWCtxzmhZQRs1jf86n2HNddDZhiUs/wgRqE1T/nqA6A1ldPn2m7/PLAnhLqINbYy
96IWsetdlPE93Ht4Kg3STLQbGGOYVKB/iLftiqD3YKs/ofe06e4LCdWsng6gl9+t
TS/H2Z+m7cIP8Kmoxs/F19mQmM19vVvSQ+pnCAN2kHdpnqyI5cQ3y9HTrNn8TPCw
MgtandMxixJsNSvRp4DmwrxPajxuvUnXcesQs4dIV+KJM/AyNT5iTbC0CEFXE6Oj
G5Xk/gtdvZtX5bcuKoECAwEAAaNCMEAwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
/wQFMAMBAf8wHQYDVR0OBBYEFDKWwCUtFnlDkPLpdSg5bsqLccIJMA0GCSqGSIb3
DQEBCwUAA4IBAQCk0TxsEWgNOcB5gTq0ewrpLUNcs5uIyfyAHz03C9H0IIMvQrms
Q/l/zdUE0xtmON/fYgnTnLS8bJeqHzl3QFxoN00gM/I5YqJkjdCntCxy9c8CX59z
RvRZnU3xYwXl5L1N0iDZgXq+bfQRF+kGfrd5ZSeLos3dmCU2hLRoxjPMofeV6oqW
itYAPOO+PI/LB3wYw+iQ3Ea4r1WStgVuUSLGU8Tdya5dbI1lUogcoYKAH7rlAG4/
R+hSPj4OWpYx4AWW42ZPM7Ebxu9WqFzj2MH2eTmu6hkbUjGRzwud0v6xZxbrB68O
svMsJUKVqmTqfHyaFj6JST8RhmZaX/ukdeb4
-----END CERTIFICATE-----
namespacedefault�
token�eyJhbGciOiJSUzI1NiIsImtpZCI6IjBTTVpLNWJ3UFlkRkZzT3R5c1d2WjR6bWVlVUE4ZTg2M1NfQWhSeEJNSXMifQ.eyJpc3MiOiJrdWJlcm5ldGVzL3NlcnZpY2VhY2NvdW50Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9uYW1lc3BhY2UiOiJkZWZhdWx0Iiwia3ViZXJuZXRlcy5pby9zZXJ2aWNlYWNjb3VudC9zZWNyZXQubmFtZSI6ImRlZmF1bHQtdG9rZW4tbng1ZjUiLCJrdWJlcm5ldGVzLmlvL3NlcnZpY2VhY2NvdW50L3NlcnZpY2UtYWNjb3VudC5uYW1lIjoiZGVmYXVsdCIsImt1YmVybmV0ZXMuaW8vc2VydmljZWFjY291bnQvc2VydmljZS1hY2NvdW50LnVpZCI6IjU1OWRiNjg4LTIwZmEtNDYzMi04YTUyLWE3NDdhZjQ1YzUyMiIsInN1YiI6InN5c3RlbTpzZXJ2aWNlYWNjb3VudDpkZWZhdWx0OmRlZmF1bHQifQ.mZc_dsUxfeg38r-vIEmSjUw4QpEGbm2JFfc0DO76VpnZowmf8CM6VOwjofbFAUy5m0hJgXW_iCQIP0Grwe6btr7efNruoQrJnnfx5kdZ6UDAYSaEBK2Y_dYF8fbwr4g4kDIWK2kHPqHsGN6V_AIgtxTX4OSfX88Kthk_DC2xgkDa9J1B4ur7u2g26_EUA89UnwCuDfMd29m7ZhAso5URMkZ9M9vIVc2g3dRdjiNIhKxflDqVGjZq0c4-uO7w4NqeZ6273ZnX-wg6-tIMi8RE5JdYFzGltnY1onbuE7mSAZpQ9AAajxSp-SlEWQ2Pn6Y5A84sYiPZV9fgC4OmzNGwTQ#kubernetes.io/service-account-token"
its all unencrypted.
now create a new secret very-secure : k create secret generic very-secure --from-literal cc=1234
now try and read very-secure from etcd
root@cks-master:/etc/systemd/system/kubelet.service.d# ETCDCTL_API=3 etcdctl --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key --cacert /etc/kubernetes/pki/etcd/ca.crt get /registry/secrets/default/very-secure
/registry/secrets/default/very-secure
��S�WK�)aescbc:v1:key1:!���D�#eH� Z�'L"�g)GyQ(��pi�-��,�?̣�9�*�N����6��
R��<� ����ܓ��T`4����L䋻v�~���`1�a���z�m>�$�vӺ��)ǧ�!q�k9��qn�[��
��$��8d��4�^ص+��f\�<"q\af�C�nU��n��8�{�Ā�p��j(���-��V�P9Љ�
BUT- we can still see the secret through API server, which is a good thing- its the gateway, and outside of this = encrypted.
root@cks-master:/etc/systemd/system/kubelet.service.d# k get secrets very-secure -oyaml
apiVersion: v1
data:
cc: MTIzNA==
kind: Secret
metadata:
creationTimestamp: "2021-09-04T23:45:14Z"
name: very-secure
namespace: default
resourceVersion: "433676"
uid: 0ce36039-b49f-4170-bb33-87cc95956bc3
type: Opaque
root@cks-master:/etc/systemd/system/kubelet.service.d# echo MTIzNA== | base64 -d
1234
re-test by commenting out the identity provider from our ec.yaml so there is not more plain-text option, and re-creating ALL the secrets.
# restart api server
root@cks-master:/etc/kubernetes/manifests# mv kube-apiserver.yaml ..
root@cks-master:/etc/kubernetes/manifests# mv ../kube-apiserver.yaml .
try look at secrets
root@cks-master:~# k get secretsError from server (InternalError): Internal error occurred: unable to transform key "/registry/secrets/default/default-token-nx5f5": no matching prefix found
uncomment the identity provider an restart apiserver again!
BUT- the task is to encrypt ALL.
solution:
- add
identityback to get etcd working again with unencrypted secrets - RE-CREATE all secrets, which will now create them encrypted.
- comment out the
identityprovider again
# replace all secrets
root@cks-master:/etc/kubernetes/manifests# k get secrets -A -oyaml | kubectl replace -f -
secret/default-token-vz224 replaced
secret/default-token-vfkf6 replaced
secret/accessor-token-jzdtq replaced
secret/default-token-nx5f5 replaced
secret/secret1 replaced
secret/secret2 replaced
secret/secure-ingress replaced
secret/very-secure replaced
secret/default-token-s4pm8 replaced
secret/ingress-nginx-admission replaced
secret/ingress-nginx-admission-token-87grf replaced
secret/ingress-nginx-token-mqtsr replaced
secret/default-token-wjj9m replaced
secret/default-token-4xcrt replaced
secret/attachdetach-controller-token-x4dwp replaced
secret/bootstrap-signer-token-6svt8 replaced
secret/bootstrap-token-9208wf replaced
secret/certificate-controller-token-fx595 replaced
secret/clusterrole-aggregation-controller-token-w69k6 replaced
secret/coredns-token-cxnf8 replaced
secret/cronjob-controller-token-p74tz replaced
secret/daemon-set-controller-token-plxht replaced
secret/default-token-qr64v replaced
secret/deployment-controller-token-7htn8 replaced
secret/disruption-controller-token-mgbrv replaced
secret/endpoint-controller-token-2grs7 replaced
secret/endpointslice-controller-token-l8z9r replaced
secret/endpointslicemirroring-controller-token-rdcdz replaced
secret/ephemeral-volume-controller-token-ng626 replaced
secret/expand-controller-token-cr6jc replaced
secret/generic-garbage-collector-token-tkswv replaced
secret/horizontal-pod-autoscaler-token-tl7qk replaced
secret/job-controller-token-vjtrs replaced
secret/kube-proxy-token-99z7x replaced
secret/namespace-controller-token-c8fjm replaced
secret/node-controller-token-j6ms7 replaced
secret/persistent-volume-binder-token-dd7zn replaced
secret/pod-garbage-collector-token-j7j2h replaced
secret/pv-protection-controller-token-7qgz2 replaced
secret/pvc-protection-controller-token-2tv7t replaced
secret/replicaset-controller-token-nnbvv replaced
secret/replication-controller-token-9qfnb replaced
secret/resourcequota-controller-token-jhg7j replaced
secret/root-ca-cert-publisher-token-szqv9 replaced
secret/service-account-controller-token-d2wz7 replaced
secret/service-controller-token-vc8v7 replaced
secret/statefulset-controller-token-8dpqb replaced
secret/token-cleaner-token-x2n5g replaced
secret/ttl-after-finished-controller-token-skkr4 replaced
secret/ttl-controller-token-lbvkr replaced
secret/weave-net-token-q8tfq replaced
secret/default-token-fkxcr replaced
secret/kubernetes-dashboard-certs replaced
secret/kubernetes-dashboard-csrf replaced
secret/kubernetes-dashboard-key-holder replaced
secret/kubernetes-dashboard-token-d7f5b replaced
secret/default-token-qwhh6 replaced
# test you can read via API but NOT via etcdctl
k -n kube-system get secrets service-account-controller-token-d2wz7 -oyaml # works
ETCDCTL_API=3 etcdctl --cert /etc/kubernetes/pki/apiserver-etcd-client.crt --key /etc/kubernetes/pki/apiserver-etcd-client.key --cacert /etc/kubernetes/pki/etcd/ca.crt get /registry/secrets/kube-system/service-account-controller-token-d2wz7 # encrypted
# re-comment out indentity and restart apiserver -- from now on all secrets are encrypted in etcd!
recap
configmaps vs secrets - keep them separate, secrets always need more security, configmaps not, so dont complicate configmaps with unnecessary encryption
for PROD - the key in the encryptionConfiguration better managed with a provider like kms with hashicorp vault (wont be on exam)