root@ubuntu:/# ls /run/libcontainer/ -al
total 0
drwx------ 3 root root 60 Dec 4 06:37 .
drwxr-xr-x 11 root root 220 Dec 4 06:37 ..
drwx------ 3 root root 60 Dec 4 06:37 9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be
root@ubuntu:/# ls /run/libcontainer/*/
9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be
root@ubuntu:/# ls /run/libcontainer/*/*
state.json
root@ubuntu:/# ls /run/libcontainer/*/state.json
ls: cannot access '/run/libcontainer/*/state.json': No such file or directory
root@ubuntu:/# ls /run/libcontainer/*/
9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be
root@ubuntu:/# ls /run/libcontainer/*/*/state.json
/run/libcontainer/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be/state.json
root@ubuntu:/# cat /run/libcontainer/*/*/state.json
{"id":"9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be","init_process_pid":79,
"init_process_start":183,"created":"2020-12-04T06:37:22.27285124Z","config":{"no_pivot_root":false,
"parent_death_signal":0,"rootfs":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be/rootfs",
"readonlyfs":false,"rootPropagation":0,"mounts":[{"source":"proc","destination":"/proc","device":"proc","flags":14,"propagation_flags":null,"data":"","relabel":"",
"extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"tmpfs","destination":"/dev","device":"tmpfs","flags":16777218,"propagation_flags":null,"data":"mode=755,size=65536k",
"relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"devpts","destination":"/dev/pts","device":"devpts","flags":10,"propagation_flags":null,
"data":"newinstance,ptmxmode=0666,mode=0620,gid=5","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"sysfs","destination":"/sys",
"device":"sysfs","flags":15,"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"cgroup","destination":"/sys/fs/cgroup","device":"cgroup","flags":15,"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"mqueue","destination":"/dev/mqueue","device":"mqueue","flags":14,"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be-d7b1133d2132078e-resolv.conf","destination":"/etc/resolv.conf","device":"bind","flags":20480,"propagation_flags":[278528],
"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},
{"source":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be-164d7e3912cd20b1-hostname","destination":"/etc/hostname","device":"bind","flags":20480,"propagation_flags":[278528],
"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"/run/kata-containers/shared/containers/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be-c27b5092d21c0407-hosts","destination":"/etc/hosts",
"device":"bind","flags":20480,"propagation_flags":[278528],"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null},{"source":"/run/kata-containers/sandbox/shm","destination":"/dev/shm","device":"bind","flags":20480,
"propagation_flags":null,"data":"","relabel":"","extensions":0,"premount_cmds":null,"postmount_cmds":null}],"devices":[{"type":99,"path":"/dev/null","major":1,"minor":3,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/random","major":1,"minor":8,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/full","major":1,"minor":7,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/tty","major":5,"minor":0,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/zero","major":1,"minor":5,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/urandom","major":1,"minor":9,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/fuse","major":10,"minor":229,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/binder","major":10,"minor":60,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},{"type":99,"path":"/dev/ashmem","major":10,"minor":61,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false},
{"type":99,"path":"/dev/input/mice","major":13,"minor":63,"permissions":"","file_mode":438,"uid":0,"gid":0,"allow":false}],"mount_label":"","hostname":"9ddf50af4f9d","namespaces":[{"type":"NEWNS","path":""},{"type":"NEWUTS","path":"/var/run/sandbox-ns/uts"},
{"type":"NEWIPC","path":"/var/run/sandbox-ns/ipc"},{"type":"NEWPID","path":""}],"capabilities":{"Bounding":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP",
"CAP_NET_BIND_SERVICE","CAP_SYS_CHROOT","CAP_KILL","CAP_AUDIT_WRITE"],"Effective":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP","CAP_NET_BIND_SERVICE",
"CAP_SYS_CHROOT","CAP_KILL","CAP_AUDIT_WRITE"],"Inheritable":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP","CAP_NET_BIND_SERVICE","CAP_SYS_CHROOT",
"CAP_KILL","CAP_AUDIT_WRITE"],"Permitted":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_FSETID","CAP_FOWNER","CAP_MKNOD","CAP_NET_RAW","CAP_SETGID","CAP_SETUID","CAP_SETFCAP","CAP_SETPCAP","CAP_NET_BIND_SERVICE","CAP_SYS_CHROOT","CAP_KILL","CAP_AUDIT_WRITE"],
"Ambient":[]},"networks":null,"routes":null,"cgroups":{"path":"/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be","scope_prefix":"","Paths":null,"allowed_devices":[{"type":99,"path":"","major":-1,"minor":-1,
"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":98,"path":"","major":-1,"minor":-1,"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/null","major":1,"minor":3,"permissions":
"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/random","major":1,"minor":8,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/full","major":1,"minor":7,"permissions":
"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/tty","major":5,"minor":0,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/zero","major":1,"minor":5,"permissions":"rwm",
"file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/urandom","major":1,"minor":9,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/console","major":5,"minor":1,"permissions":"rwm",
"file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":136,"minor":-1,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":5,"minor":2,"permissions":"rwm","file_mode":0,"uid":0,
"gid":0,"allow":true},{"type":99,"path":"","major":10,"minor":200,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true}],"devices":[{"type":98,"path":"","major":254,"minor":1,"permissions":"rw","file_mode":0,"uid":0,"gid":0,
"allow":false},{"type":99,"path":"","major":-1,"minor":-1,"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":98,"path":"","major":-1,"minor":-1,"permissions":"m","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/null","major":1,"minor":3,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/random","major":1,"minor":8,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/full","major":1,"minor":7,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/tty","major":5,"minor":0,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/zero","major":1,"minor":5,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"/dev/urandom","major":1,"minor":9,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"/dev/console","major":5,"minor":1,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":136,"minor":-1,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},
{"type":99,"path":"","major":5,"minor":2,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true},{"type":99,"path":"","major":10,"minor":200,"permissions":"rwm","file_mode":0,"uid":0,"gid":0,"allow":true}],"memory":0,"memory_reservation":0,
"memory_swap":0,"kernel_memory":0,"kernel_memory_tcp":0,"cpu_shares":0,"cpu_quota":0,"cpu_period":0,"cpu_rt_quota":0,"cpu_rt_period":0,"cpuset_cpus":"","cpuset_mems":"","pids_limit":0,"blkio_weight":0,"blkio_leaf_weight":0,"blkio_weight_device":null,
"blkio_throttle_read_bps_device":null,"blkio_throttle_write_bps_device":null,"blkio_throttle_read_iops_device":null,"blkio_throttle_write_iops_device":null,"freezer":"","hugetlb_limit":null,"oom_kill_disable":false,"memory_swappiness":0,
"net_prio_ifpriomap":null,"net_cls_classid_u":0,"cpu_weight":0,"cpu_max":""},"oom_score_adj":0,"uid_mappings":null,"gid_mappings":null,"mask_paths":["/proc/asound","/proc/acpi","/proc/kcore","/proc/keys","/proc/latency_stats",
"/proc/timer_list","/proc/timer_stats","/proc/sched_debug","/proc/scsi","/sys/firmware"],"readonly_paths":["/proc/bus","/proc/fs","/proc/irq","/proc/sys","/proc/sysrq-trigger"],"sysctl":{},"seccomp":null,
"Hooks":{"poststart":null,"poststop":null,"prestart":null},"version":"1.0.1-dev","labels":["bundle=/"],"no_new_keyring":true},
"rootless":false,"cgroup_paths":{"blkio":"/sys/fs/cgroup/blkio/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"cpu":"/sys/fs/cgroup/cpu,cpuacct/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"cpuacct":"/sys/fs/cgroup/cpu,cpuacct/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be","
cpuset":"/sys/fs/cgroup/cpuset/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"devices":"/sys/fs/cgroup/devices/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"freezer":"/sys/fs/cgroup/freezer/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"memory":"/sys/fs/cgroup/memory/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"name=systemd":"/sys/fs/cgroup/systemd/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"net_cls":"/sys/fs/cgroup/net_cls,net_prio/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"net_prio":"/sys/fs/cgroup/net_cls,net_prio/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"perf_event":"/sys/fs/cgroup/perf_event/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be",
"pids":"/sys/fs/cgroup/pids/docker/9ddf50af4f9d18ab65ce13c7334ff3f0c9c9736aa81a05089042484714e746be"},
"namespace_paths":{"NEWCGROUP":"/proc/79/ns/cgroup","NEWIPC":"/proc/79/ns/ipc","NEWNET":"/proc/79/ns/net","NEWNS":"/proc/79/ns/mnt","NEWPID":"/proc/79/ns/pid","NEWUSER":"/proc/79/ns/user","NEWUTS":"/proc/79/ns/uts"},
"external_descriptors":["/dev/null","/dev/null","/dev/null"],"intel_rdt_path":""}root@ubuntu:/#
root@ubuntu:/# ps -e -o pid,cmd
PID CMD
1 /sbin/init
2 [kthreadd]
3 [rcu_gp]
4 [rcu_par_gp]
6 [kworker/0:0H-kb]
7 [kworker/u2:0-ev]
8 [mm_percpu_wq]
9 [ksoftirqd/0]
10 [rcu_sched]
11 [migration/0]
12 [cpuhp/0]
13 [kdevtmpfs]
14 [netns]
15 [oom_reaper]
16 [writeback]
17 [kcompactd0]
18 [kblockd]
19 [blkcg_punt_bio]
21 [kswapd0]
22 [xfsalloc]
23 [xfs_mru_cache]
24 [kthrotld]
26 [khvcd]
27 [hwrng]
29 [scsi_eh_0]
30 [scsi_tmf_0]
31 [uas]
32 [usbip_event]
33 [ipv6_addrconf]
34 [jbd2/vda1-8]
35 [ext4-rsv-conver]
51 /lib/systemd/systemd-journald
55 /usr/bin/Xvfb :8 -ac -screen 0 720x1280x24
56 /usr/bin/kata-agent
57 /opt/openvmi/bin/openvmi session-manager --run-multiple
59 [kworker/u2:2-ev]
65 /bin/bash
79 /bin/bash
105 [kworker/0:1H-kb]
2814 [kworker/0:1-cgr]
2837 [kworker/0:2-eve]
2850 ps -e -o pid,cmd
root@ubuntu:/# ls -al /proc/79/ns/ total 0 dr-x--x--x 2 root root 0 Dec 4 07:55 . dr-xr-xr-x 9 root root 0 Dec 4 06:37 .. lrwxrwxrwx 1 root root 0 Dec 4 07:55 cgroup -> 'cgroup:[4026531835]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 ipc -> 'ipc:[4026532171]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 mnt -> 'mnt:[4026532173]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 net -> 'net:[4026531897]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 pid -> 'pid:[4026532174]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 pid_for_children -> 'pid:[4026532174]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 user -> 'user:[4026531837]' lrwxrwxrwx 1 root root 0 Dec 4 07:55 uts -> 'uts:[4026532172]' root@ubuntu:/#
agent创建进程
libcontainerPath = "/run/libcontainer"
func (a *agentGRPC) finishCreateContainer(ctr *container, req *pb.CreateContainerRequest, config *configs.Config) (resp *gpb.Empty, err error) {
containerPath := filepath.Join(libcontainerPath, a.sandbox.id)
factory, err := libcontainer.New(containerPath, libcontainer.Cgroupfs)
if err != nil {
return emptyResp, err
}
ctr.container, err = factory.Create(req.ContainerId, config)
if err != nil {
return emptyResp, err
}
ctr.config = *config
ctr.initProcess, err = buildProcess(req.OCI.Process, req.ExecId, true)
if err != nil {
return emptyResp, err
}
func (a *agentGRPC) ExecProcess(ctx context.Context, req *pb.ExecProcessRequest) (*gpb.Empty, error) {
ctr, err := a.getContainer(req.ContainerId)
if err != nil {
return emptyResp, err
}
status, err := ctr.container.Status()
if err != nil {
return nil, err
}
if status == libcontainer.Stopped {
return nil, grpcStatus.Errorf(codes.FailedPrecondition, "Cannot exec in stopped container %s", req.ContainerId)
}
proc, err := buildProcess(req.Process, req.ExecId, false)
if err != nil {
return emptyResp, err
}
if err := a.execProcess(ctr, proc, false); err != nil {
return emptyResp, err
}
return emptyResp, a.postExecProcess(ctr, proc)
}
// Shared function between CreateContainer and ExecProcess, because those expect
// a process to be run.
func (a *agentGRPC) execProcess(ctr *container, proc *process, createContainer bool) (err error) {
if ctr == nil {
return grpcStatus.Error(codes.InvalidArgument, "Container cannot be nil")
}
if proc == nil {
return grpcStatus.Error(codes.InvalidArgument, "Process cannot be nil")
}
// This lock is very important to avoid any race with reaper.reap().
// Indeed, if we don't lock this here, we could potentially get the
// SIGCHLD signal before the channel has been created, meaning we will
// miss the opportunity to get the exit code, leading WaitProcess() to
// wait forever on the new channel.
// This lock has to be taken before we run the new process.
a.sandbox.subreaper.lock()
defer a.sandbox.subreaper.unlock()
if createContainer {
err = ctr.container.Start(&proc.process)
} else {
err = ctr.container.Run(&(proc.process))
}
if err != nil {
return grpcStatus.Errorf(codes.Internal, "Could not run process: %v", err)
}
// Get process PID
pid, err := proc.process.Pid()
if err != nil {
return err
}
proc.exitCodeCh = make(chan int, 1)
// Create process channel to allow WaitProcess to wait on it.
// This channel is buffered so that reaper.reap() will not
// block until WaitProcess listen onto this channel.
a.sandbox.subreaper.setExitCodeCh(pid, proc.exitCodeCh)
return nil
}
创建子进程
func (c *linuxContainer) start(process *Process) error {
parent, err := c.newParentProcess(process) /* 1. 创建parentProcess */
err := parent.start(); /* 2. 启动这个parentProcess */
......
runc进程
/* utils_linux.go */
func startContainer(context *cli.Context, spec *specs.Spec, action CtAct, criuOpts *libcontainer.CriuOpts) (int, error) {
id := context.Args().First()
container, err := createContainer(context, id, spec)
r := &runner{
container: container,
action: action,
init: true,
......
}
return r.run(spec.Process)
}
func (r *runner) run(config *specs.Process) (int, error) {
......
process, err := newProcess(*config, r.init) /* 第1部分 */
......
switch r.action {
case CT_ACT_CREATE:
err = r.container.Start(process) /* runc start */ /* 第2部分 */
case CT_ACT_RESTORE:
err = r.container.Restore(process, r.criuOpts) /* runc restore */
case CT_ACT_RUN:
err = r.container.Run(process) /* runc run */
default:
panic("Unknown action")
}
......
return status, err
}
创建子进程
func (c *linuxContainer) start(process *Process) error {
parent, err := c.newParentProcess(process) /* 1. 创建parentProcess */
err := parent.start(); /* 2. 启动这个parentProcess */
......