Container Runtime Sandboxes

1. 介绍

technical overview : container and system calls\

在这里插入图片描述

2. Container calls Linux Kernel

在这里插入图片描述

root@master:~# k run pod --image=nginx
pod/pod created
root@master:~# k get pod
NAME   READY   STATUS    RESTARTS   AGE
pod    1/1     Running   0          9s
root@master:~# k exec pod -ti -- bash
root@pod:/# 
root@pod:/# uname -r
4.4.0-198-generic
root@pod:/# exit
exit
root@master:~# uname -r
4.4.0-142-generic


root@master:~# strace uname -r
execve("/bin/uname", ["uname", "-r"], [/* 26 vars */]) = 0
brk(NULL)                               = 0x163a000
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
access("/etc/ld.so.preload", R_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/etc/ld.so.cache", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=47425, ...}) = 0
mmap(NULL, 47425, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbc1c8b3000
close(3)                                = 0
access("/etc/ld.so.nohwcap", F_OK)      = -1 ENOENT (No such file or directory)
openat(AT_FDCWD, "/lib/x86_64-linux-gnu/libc.so.6", O_RDONLY|O_CLOEXEC) = 3
read(3, "\177ELF\2\1\1\3\0\0\0\0\0\0\0\0\3\0>\0\1\0\0\0\20\35\2\0\0\0\0\0"..., 832) = 832
fstat(3, {st_mode=S_IFREG|0755, st_size=2030928, ...}) = 0
mmap(NULL, 8192, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_ANONYMOUS, -1, 0) = 0x7fbc1c8b1000
mmap(NULL, 4131552, PROT_READ|PROT_EXEC, MAP_PRIVATE|MAP_DENYWRITE, 3, 0) = 0x7fbc1c2a5000
mprotect(0x7fbc1c48c000, 2097152, PROT_NONE) = 0
mmap(0x7fbc1c68c000, 24576, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_DENYWRITE, 3, 0x1e7000) = 0x7fbc1c68c000
mmap(0x7fbc1c692000, 15072, PROT_READ|PROT_WRITE, MAP_PRIVATE|MAP_FIXED|MAP_ANONYMOUS, -1, 0) = 0x7fbc1c692000
close(3)                                = 0
arch_prctl(ARCH_SET_FS, 0x7fbc1c8b2540) = 0
mprotect(0x7fbc1c68c000, 16384, PROT_READ) = 0
mprotect(0x606000, 4096, PROT_READ)     = 0
mprotect(0x7fbc1c8bf000, 4096, PROT_READ) = 0
munmap(0x7fbc1c8b3000, 47425)           = 0
brk(NULL)                               = 0x163a000
brk(0x165b000)                          = 0x165b000
openat(AT_FDCWD, "/usr/lib/locale/locale-archive", O_RDONLY|O_CLOEXEC) = 3
fstat(3, {st_mode=S_IFREG|0644, st_size=2999664, ...}) = 0
mmap(NULL, 2999664, PROT_READ, MAP_PRIVATE, 3, 0) = 0x7fbc1bfc8000
close(3)                                = 0
uname({sysname="Linux", nodename="master", ...}) = 0
fstat(1, {st_mode=S_IFCHR|0600, st_rdev=makedev(136, 0), ...}) = 0
write(1, "4.4.0-142-generic\n", 184.4.0-142-generic
)     = 18
close(1)                                = 0
close(2)                                = 0
exit_group(0)                           = ?
+++ exited with 0 +++

3. Open Container Initiative OCI

\

4. Crictl

参考链接： https://kubernetes.io/zh/docs/tasks/debug-application-cluster/crictl/ Kubernetes crictl

root@master:~# docker ps
CONTAINER ID        IMAGE                  COMMAND                  CREATED             STATUS              PORTS               NAMES
daa4fcda3062        calico/node            "start_runit"            56 minutes ago      Up 56 minutes                           k8s_calico-node_calico-node-ngbm8_kube-system_837fbf7e-0060-4f5c-bd62-fdecf5f7e334_0
47f0bd50cfb9        10cc881966cf           "/usr/local/bin/kube…"   2 days ago          Up 2 days                               k8s_kube-proxy_kube-proxy-lfkn9_kube-system_08f4f57e-d10b-4efe-99d7-33509c6492b0_0
bc34cf5b8061        k8s.gcr.io/pause:3.2   "/pause"                 2 days ago          Up 2 days                               k8s_POD_calico-node-ngbm8_kube-system_837fbf7e-0060-4f5c-bd62-fdecf5f7e334_0
e687794da368        k8s.gcr.io/pause:3.2   "/pause"                 2 days ago          Up 2 days                               k8s_POD_kube-proxy-lfkn9_kube-system_08f4f57e-d10b-4efe-99d7-33509c6492b0_0
8c953da2a0f3        3138b6e3d471           "kube-scheduler --au…"   2 days ago          Up 2 days                               k8s_kube-scheduler_kube-scheduler-master_kube-system_81d2d21449d64d5e6d5e9069a7ca99ed_0
2597fa68a300        b9fa1895dcaa           "kube-controller-man…"   2 days ago          Up 2 days                               k8s_kube-controller-manager_kube-controller-manager-master_kube-system_360cd07520ba8dce55b5d403c66acf83_0
27b65081b111        ca9843d3b545           "kube-apiserver --ad…"   2 days ago          Up 2 days                               k8s_kube-apiserver_kube-apiserver-master_kube-system_ee31a01764366141f7c85e23f94828f8_0
ad3166e92a06        0369cf4303ff           "etcd --advertise-cl…"   2 days ago          Up 2 days                               k8s_etcd_etcd-master_kube-system_77699ae6105937dbb48c0a720843ce8e_0
794740e27507        k8s.gcr.io/pause:3.2   "/pause"                 2 days ago          Up 2 days                               k8s_POD_kube-scheduler-master_kube-system_81d2d21449d64d5e6d5e9069a7ca99ed_0
c666f003a0ad        k8s.gcr.io/pause:3.2   "/pause"                 2 days ago          Up 2 days                               k8s_POD_kube-controller-manager-master_kube-system_360cd07520ba8dce55b5d403c66acf83_0
1fb5d789dd68        k8s.gcr.io/pause:3.2   "/pause"                 2 days ago          Up 2 days                               k8s_POD_kube-apiserver-master_kube-system_ee31a01764366141f7c85e23f94828f8_0
22a1ebc8d2d5        k8s.gcr.io/pause:3.2   "/pause"                 2 days ago          Up 2 days                               k8s_POD_etcd-master_kube-system_77699ae6105937dbb48c0a720843ce8e_0



root@master:~# crictl ps
CONTAINER ID        IMAGE                                                                                 CREATED             STATE               NAME                      ATTEMPT             POD ID
daa4fcda30622       calico/node@sha256:04b8a7be6a277000ea4ae12f32692b2f5532cd095fe5d6b6e3ff876d750c336a   About an hour ago   Running             calico-node               0                   bc34cf5b8061f
47f0bd50cfb95       10cc881966cfd                                                                         2 days ago          Running             kube-proxy                0                   e687794da3683
8c953da2a0f34       3138b6e3d4712                                                                         2 days ago          Running             kube-scheduler            0                   794740e275074
2597fa68a300d       b9fa1895dcaa6                                                                         2 days ago          Running             kube-controller-manager   0                   c666f003a0ad6
27b65081b1114       ca9843d3b5454                                                                         2 days ago          Running             kube-apiserver            0                   1fb5d789dd685
ad3166e92a063       0369cf4303ffd                                                                         2 days ago          Running             etcd                      0                   22a1ebc8d2d50
root@master:~# 


root@master:~# crictl pods
POD ID              CREATED             STATE               NAME                             NAMESPACE           ATTEMPT
bc34cf5b8061f       2 days ago          Ready               calico-node-ngbm8                kube-system         0
e687794da3683       2 days ago          Ready               kube-proxy-lfkn9                 kube-system         0
794740e275074       2 days ago          Ready               kube-scheduler-master            kube-system         0
c666f003a0ad6       2 days ago          Ready               kube-controller-manager-master   kube-system         0
1fb5d789dd685       2 days ago          Ready               kube-apiserver-master            kube-system         0
22a1ebc8d2d50       2 days ago          Ready               etcd-master                      kube-system         0
root@master:~# 

5. Sandbox Runtime Kata containers

参考链接： https://katacontainers.io/ https://github.com/kata-containers/kata-containers https://www.hi-linux.com/posts/23259.html Kubernetes kata-container 介绍 \

6. Sandbox Runtime gVisor

参考链接： https://github.com/google/gvisor https://gvisor.dev/ https://gvisor.dev/docs/\

7. Create and use RuntimeClasses

参考链接： https://kubernetes.io/docs/concepts/containers/runtime-class/\

root@master:~/cks/runtimeclass# vim rc.yaml
apiVersion: node.k8s.io/v1beta1
kind: RuntimeClass
metadata:
  name: gvisor
handler: runsc



root@master:~/cks/runtimeclass# k -f rc.yaml  create
runtimeclass.node.k8s.io/gvisor created

root@master:~/cks/runtimeclass# k get runtimeclass
NAME     HANDLER   AGE
gvisor   runsc     8s


root@master:~/cks/runtimeclass#  k run gvisor --image=nginx -oyaml --dry-run=client > pod.yaml
root@master:~/cks/runtimeclass# vim pod.yaml


root@master:~/cks/runtimeclass# k get pods
NAME     READY   STATUS              RESTARTS   AGE
gvisor   1/1     ContainerCreating   0          19h


root@master:~# k describe pods gvisor
..........
Events:
  Type     Reason                  Age                     From     Message
  ----     ------                  ----                    ----     -------
  Warning  FailedCreatePodSandBox  47h (x1778 over 2d19h)  kubelet  Failed to create pod sandbox: rpc error: code = Unknown desc = RuntimeHandler "runsc" not supported

需要接下来的步骤

8. Install and use gVisor

在这里插入图片描述

root@master:~# bash <(curl -s https://raw.githubusercontent.com/killer-sh/cks-course-environment/master/course-content/microservice-vulnerabilities/container-runtimes/gvisor/install_gvisor.sh)

或者执行

root@master:~/cks/runtimeclass# vim  install_gvisor.sh 
#!/usr/bin/env bash
# IF THIS FAILS then you can try to change the URL= further down from latest to a specific release
# https://gvisor.dev/docs/user_guide/install

# gvisor
sudo apt-get update && \
sudo apt-get install -y \
    apt-transport-https \
    ca-certificates \
    curl \
    gnupg-agent \
    software-properties-common


# install from web
{
  set -e
  URL=https://storage.googleapis.com/gvisor/releases/release/latest
#  URL=https://storage.googleapis.com/gvisor/releases/release/20201130.0 # try this version instead if latest doesn't work for you
  wget ${URL}/runsc ${URL}/runsc.sha512 \
    ${URL}/gvisor-containerd-shim ${URL}/gvisor-containerd-shim.sha512 \
    ${URL}/containerd-shim-runsc-v1 ${URL}/containerd-shim-runsc-v1.sha512
  sha512sum -c runsc.sha512 \
    -c gvisor-containerd-shim.sha512 \
    -c containerd-shim-runsc-v1.sha512
  rm -f *.sha512
  chmod a+rx runsc gvisor-containerd-shim containerd-shim-runsc-v1
  sudo mv runsc gvisor-containerd-shim containerd-shim-runsc-v1 /usr/local/bin
}

# containerd enable runsc
mkdir -p /etc/containerd
cat > /etc/containerd/config.toml <<EOF
disabled_plugins = ["restart"]
[plugins.linux]
  shim_debug = true
[plugins.cri.containerd.runtimes.runsc]
  runtime_type = "io.containerd.runsc.v1"
EOF

# crictl should use containerd as default
{
cat <<EOF | sudo tee /etc/crictl.yaml
runtime-endpoint: unix:///run/containerd/containerd.sock
EOF
}

systemctl restart containerd

# kubelet should use containerd
{
cat <<EOF | sudo tee /etc/default/kubelet
KUBELET_EXTRA_ARGS="--container-runtime remote --container-runtime-endpoint unix:///run/containerd/containerd.sock"
EOF
}
systemctl daemon-reload
systemctl restart kubelet

root@master:~/cks/runtimeclass# bash install_gvisor.sh
root@master:~/cks/runtimeclass# systemctl status kubelet
● kubelet.service - kubelet: The Kubernetes Node Agent
   Loaded: loaded (/lib/systemd/system/kubelet.service; enabled; vendor preset: enabled)
  Drop-In: /etc/systemd/system/kubelet.service.d
           └─10-kubeadm.conf
   Active: active (running) since Thu 2021-05-13 20:30:54 PDT; 4min 42s ago
     Docs: https://kubernetes.io/docs/home/
 Main PID: 24673 (kubelet)
    Tasks: 14
   Memory: 93.2M
      CPU: 7.489s
   CGroup: /system.slice/kubelet.service
           └─24673 /usr/bin/kubelet --bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet

在这里插入图片描述