Container Runtime Sandboxes
1. 介绍
technical overview : container and system calls\
2. Container calls Linux Kernel
root@master:~# k run pod --image=nginx
pod/pod created
root@master:~# k get pod
NAME READY STATUS RESTARTS AGE
pod 1/1 Running 0 9s
root@master:~# k exec pod -ti -- bash
root@pod:/#
root@pod:/# uname -r
4.4.0-198-generic
root@pod:/# exit
exit
root@master:~# uname -r
4.4.0-142-generic
root@master:~# strace uname -r
execve("/bin/uname", ["uname", "-r"], [/* 26 vars */]) = 0
brk(NULL) = 0x163a000
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=47425, ...}) = 0
mmap(NULL, 47425, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbc1c8b3000
close(3) = 0
access("/etc/ld.so.nohwcap", F_OK) = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\35\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2030928, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbc1c8b1000
mmap(NULL, 4131552, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbc1c2a5000
mprotect(0x7fbc1c48c000, 2097152, PROT_NONE) = 0
mmap(0x7fbc1c68c000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7fbc1c68c000
mmap(0x7fbc1c692000, 15072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbc1c692000
close(3) = 0
arch_prctl(ARCH_SET_FS, 0x7fbc1c8b2540) = 0
mprotect(0x7fbc1c68c000, 16384, PROT_READ) = 0
mprotect(0x606000, 4096, PROT_READ) = 0
mprotect(0x7fbc1c8bf000, 4096, PROT_READ) = 0
munmap(0x7fbc1c8b3000, 47425) = 0
brk(NULL) = 0x163a000
brk(0x165b000) = 0x165b000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2999664, ...}) = 0
mmap(NULL, 2999664, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbc1bfc8000
close(3) = 0
uname({sysname="Linux", nodename="master", ...}) = 0
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(136, 0), ...}) = 0
write(1, "4.4.0-142-generic\n", 184.4.0-142-generic
) = 18
close(1) = 0
close(2) = 0
exit_group(0) = ?
+++ exited with 0 +++3. Open Container Initiative OCI
\
4. Crictl
参考链接： https://kubernetes.io/zh/docs/tasks/debug-application-cluster/crictl/ Kubernetes crictl
root@master:~# docker ps
CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES
daa4fcda3062 calico/node "start_runit" 56 minutes ago Up 56 minutes k8s_calico-node_calico-node-ngbm8_kube-system_837fbf7e-0060-4f5c-bd62-fdecf5f7e334_0
47f0bd50cfb9 10cc881966cf "/usr/local/bin/kube…" 2 days ago Up 2 days k8s_kube-proxy_kube-proxy-lfkn9_kube-system_08f4f57e-d10b-4efe-99d7-33509c6492b0_0
bc34cf5b8061 k8s.gcr.io/pause:3.2 "/pause" 2 days ago Up 2 days k8s_POD_calico-node-ngbm8_kube-system_837fbf7e-0060-4f5c-bd62-fdecf5f7e334_0
e687794da368 k8s.gcr.io/pause:3.2 "/pause" 2 days ago Up 2 days k8s_POD_kube-proxy-lfkn9_kube-system_08f4f57e-d10b-4efe-99d7-33509c6492b0_0
8c953da2a0f3 3138b6e3d471 "kube-scheduler --au…" 2 days ago Up 2 days k8s_kube-scheduler_kube-scheduler-master_kube-system_81d2d21449d64d5e6d5e9069a7ca99ed_0
2597fa68a300 b9fa1895dcaa "kube-controller-man…" 2 days ago Up 2 days k8s_kube-controller-manager_kube-controller-manager-master_kube-system_360cd07520ba8dce55b5d403c66acf83_0
27b65081b111 ca9843d3b545 "kube-apiserver --ad…" 2 days ago Up 2 days k8s_kube-apiserver_kube-apiserver-master_kube-system_ee31a01764366141f7c85e23f94828f8_0
ad3166e92a06 0369cf4303ff "etcd --advertise-cl…" 2 days ago Up 2 days k8s_etcd_etcd-master_kube-system_77699ae6105937dbb48c0a720843ce8e_0
794740e27507 k8s.gcr.io/pause:3.2 "/pause" 2 days ago Up 2 days k8s_POD_kube-scheduler-master_kube-system_81d2d21449d64d5e6d5e9069a7ca99ed_0
c666f003a0ad k8s.gcr.io/pause:3.2 "/pause" 2 days ago Up 2 days k8s_POD_kube-controller-manager-master_kube-system_360cd07520ba8dce55b5d403c66acf83_0
1fb5d789dd68 k8s.gcr.io/pause:3.2 "/pause" 2 days ago Up 2 days k8s_POD_kube-apiserver-master_kube-system_ee31a01764366141f7c85e23f94828f8_0
22a1ebc8d2d5 k8s.gcr.io/pause:3.2 "/pause" 2 days ago Up 2 days k8s_POD_etcd-master_kube-system_77699ae6105937dbb48c0a720843ce8e_0
root@master:~# crictl ps
CONTAINER ID IMAGE CREATED STATE NAME ATTEMPT POD ID
daa4fcda30622 calico/node@sha256:04b8a7be6a277000ea4ae12f32692b2f5532cd095fe5d6b6e3ff876d750c336a About an hour ago Running calico-node 0 bc34cf5b8061f
47f0bd50cfb95 10cc881966cfd 2 days ago Running kube-proxy 0 e687794da3683
8c953da2a0f34 3138b6e3d4712 2 days ago Running kube-scheduler 0 794740e275074
2597fa68a300d b9fa1895dcaa6 2 days ago Running kube-controller-manager 0 c666f003a0ad6
27b65081b1114 ca9843d3b5454 2 days ago Running kube-apiserver 0 1fb5d789dd685
ad3166e92a063 0369cf4303ffd 2 days ago Running etcd 0 22a1ebc8d2d50
root@master:~#
root@master:~# crictl pods
POD ID CREATED STATE NAME NAMESPACE ATTEMPT
bc34cf5b8061f 2 days ago Ready calico-node-ngbm8 kube-system 0
e687794da3683 2 days ago Ready kube-proxy-lfkn9 kube-system 0
794740e275074 2 days ago Ready kube-scheduler-master kube-system 0
c666f003a0ad6 2 days ago Ready kube-controller-manager-master kube-system 0
1fb5d789dd685 2 days ago Ready kube-apiserver-master kube-system 0
22a1ebc8d2d50 2 days ago Ready etcd-master kube-system 0
root@master:~# 5. Sandbox Runtime Kata containers
参考链接： https://katacontainers.io/ https://github.com/kata-containers/kata-containers https://www.hi-linux.com/posts/23259.html Kubernetes kata-container 介绍 \
6. Sandbox Runtime gVisor
参考链接： https://github.com/google/gvisor https://gvisor.dev/ https://gvisor.dev/docs/\
7. Create and use RuntimeClasses
参考链接： https://kubernetes.io/docs/concepts/containers/runtime-class/\
root@master:~/cks/runtimeclass# vim rc.yaml
apiVersion: node.k8s.io/v1beta1
kind: RuntimeClass
metadata:
name: gvisor
handler: runsc
root@master:~/cks/runtimeclass# k -f rc.yaml create
runtimeclass.node.k8s.io/gvisor created
root@master:~/cks/runtimeclass# k get runtimeclass
NAME HANDLER AGE
gvisor runsc 8s
root@master:~/cks/runtimeclass# k run gvisor --image=nginx -oyaml --dry-run=client > pod.yaml
root@master:~/cks/runtimeclass# vim pod.yaml
root@master:~/cks/runtimeclass# k get pods
NAME READY STATUS RESTARTS AGE
gvisor 1/1 ContainerCreating 0 19h
root@master:~# k describe pods gvisor
..........
Events:
Type Reason Age From Message
---- ------ ---- ---- -------
Warning FailedCreatePodSandBox 47h (x1778 over 2d19h) kubelet Failed to create pod sandbox: rpc error: code = Unknown desc = RuntimeHandler "runsc" not supported需要接下来的步骤
8. Install and use gVisor
root@master:~# bash <(curl -s https://raw.githubusercontent.com/killer-sh/cks-course-environment/master/course-content/microservice-vulnerabilities/container-runtimes/gvisor/install_gvisor.sh)或者执行
root@master:~/cks/runtimeclass# vim install_gvisor.sh
#!/usr/bin/env bash
# IF THIS FAILS then you can try to change the URL= further down from latest to a specific release
# https://gvisor.dev/docs/user_guide/install
# gvisor
sudo apt-get update && \
sudo apt-get install -y \
apt-transport-https \
ca-certificates \
curl \
gnupg-agent \
software-properties-common
# install from web
{
set -e
URL=https://storage.googleapis.com/gvisor/releases/release/latest
# URL=https://storage.googleapis.com/gvisor/releases/release/20201130.0 # try this version instead if latest doesn't work for you
wget ${URL}/runsc ${URL}/runsc.sha512 \
${URL}/gvisor-containerd-shim ${URL}/gvisor-containerd-shim.sha512 \
${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512
sha512sum -c runsc.sha512 \
-c gvisor-containerd-shim.sha512 \
-c containerd-shim-runsc-v1.sha512
rm -f *.sha512
chmod a+rx runsc gvisor-containerd-shim containerd-shim-runsc-v1
sudo mv runsc gvisor-containerd-shim containerd-shim-runsc-v1 /usr/local/bin
}
# containerd enable runsc
mkdir -p /etc/containerd
cat > /etc/containerd/config.toml <<EOF
disabled_plugins = ["restart"]
[plugins.linux]
shim_debug = true
[plugins.cri.containerd.runtimes.runsc]
runtime_type = "io.containerd.runsc.v1"
EOF
# crictl should use containerd as default
{
cat <<EOF | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
EOF
}
systemctl restart containerd
# kubelet should use containerd
{
cat <<EOF | sudo tee /etc/default/kubelet
KUBELET_EXTRA_ARGS="--container-runtime remote --container-runtime-endpoint unix:///run/containerd/containerd.sock"
EOF
}
systemctl daemon-reload
systemctl restart kubeletroot@master:~/cks/runtimeclass# bash install_gvisor.sh
root@master:~/cks/runtimeclass# systemctl status kubelet
● kubelet.service - kubelet: The Kubernetes Node Agent
Loaded: loaded (/lib/systemd/system/kubelet.service; enabled; vendor preset: enabled)
Drop-In: /etc/systemd/system/kubelet.service.d
└─10-kubeadm.conf
Active: active (running) since Thu 2021-05-13 20:30:54 PDT; 4min 42s ago
Docs: https://kubernetes.io/docs/home/
Main PID: 24673 (kubelet)
Tasks: 14
Memory: 93.2M
CPU: 7.489s
CGroup: /system.slice/kubelet.service
└─24673 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet最后更新于
