!67 kata-containers: support shimv2 with isulad

From: @holyfei
Reviewed-by: @flyflyflypeng,@duguhaotian
Signed-off-by: @duguhaotian

kata-containers.spec

@@ -2,7 +2,7 @@
 %global debug_package %{nil}
 
 %define VERSION v1.11.1
-%define RELEASE 10
+%define RELEASE 11
 
 Name:           kata-containers
 Version:        %{VERSION}
@@ -70,7 +70,8 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot
 
 cd %{_builddir}/kata_integration
 mkdir -p -m 750  %{buildroot}/usr/bin
-install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
+strip ./build/kata-runtime ./build/containerd-shim-kata-v2 ./build/kata-proxy ./build/kata-shim ./build/kata-netmon
+install -p -m 750 ./build/kata-runtime ./build/containerd-shim-kata-v2 ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
 install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
 mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
 install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
@@ -79,6 +80,7 @@ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/u
 
 %files
 /usr/bin/kata-runtime
+/usr/bin/containerd-shim-kata-v2
 /usr/bin/kata-proxy
 /usr/bin/kata-shim
 /usr/bin/kata-netmon
@@ -90,6 +92,12 @@ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/u
 
 
 %changelog
+* Sat Nov 27 2021 yangfeiyu <yangfeiyu2@huawei.com> - 1.11.1-11
+- Type:enhancement
+- ID:NA
+- SUG:NA
+- DESC:support shimv2 with isulad
+
 * Tue Feb 23 2021 xinghe <xinghe1@huawei.com> - 1.11.1-10
 - Type:CVE
 - ID:NA

runtime/kata-runtime.spec

@@ -1,7 +1,7 @@
 %define debug_package %{nil}
 
 %define VERSION 1.11.1
-%define RELEASE 11
+%define RELEASE 12
 
 Name:           kata-runtime
 Version:        %{VERSION}
@@ -98,6 +98,12 @@ install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/de
 /usr/share/defaults/kata-containers/configuration.toml
 
 %changelog
+* Sat Nov 27 2021 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-12
+- Type:enhancement
+- ID:NA
+- SUG:restart
+- DESC:support shimv2 with isulad
+
 * Tue Nov 17 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-11
 - Type:bugfix
 - ID:NA

runtime/patches/0071-fix-containerd-shim-kata-v2-compile-bug.patch

@@ -0,0 +1,32 @@
+From c9737d7702e4042bcc433da720611ebac06f6f6d Mon Sep 17 00:00:00 2001
+From: gaohuatao <gaohuatao@huawei.com>
+Date: Wed, 28 Apr 2021 15:04:02 +0800
+Subject: [PATCH 1/2] fix containerd shim kata v2 compile bug
+
+reason: fix containerd shim kata v2 compile bug
+
+Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+---
+ containerd-shim-v2/metrics.go | 6 +++---
+ 1 file changed, 3 insertions(+), 3 deletions(-)
+
+diff --git a/containerd-shim-v2/metrics.go b/containerd-shim-v2/metrics.go
+index c936966..d59530f 100644
+--- a/containerd-shim-v2/metrics.go
++++ b/containerd-shim-v2/metrics.go
+@@ -172,9 +172,9 @@ func copyBlkio(s []vc.BlkioStatEntry) []*cgroups.BlkIOEntry {
+ 	return ret
+ }
+ 
+-func setNetworkStats(vcNetwork []*vc.NetworkStats) []*cgroups.NetworkStat {
+-	networkStats := make([]*cgroups.NetworkStat, len(vcNetwork))
+-	for i, v := range vcNetwork {
++func setNetworkStats(vcNetwork *vc.NetworkStats) []*cgroups.NetworkStat {
++	networkStats := make([]*cgroups.NetworkStat, len(vcNetwork.Interfaces))
++	for i, v := range vcNetwork.Interfaces {
+ 		networkStats[i] = &cgroups.NetworkStat{
+ 			Name:      v.Name,
+ 			RxBytes:   v.RxBytes,
+-- 
+2.20.1
+

runtime/patches/0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch

@@ -0,0 +1,197 @@
+From 488ebd1f148bc9ddda9376a065d38cbc9bb9a0fe Mon Sep 17 00:00:00 2001
+From: gaohuatao <gaohuatao@huawei.com>
+Date: Wed, 28 Apr 2021 15:31:30 +0800
+Subject: [PATCH] kata shimv2 adapt iSulad and open build flag
+
+Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+---
+ Makefile                                      |  2 +-
+ containerd-shim-v2/container.go               |  9 +++++
+ containerd-shim-v2/service.go                 | 38 +++++++++++++++++++
+ containerd-shim-v2/start.go                   | 10 +++++
+ .../containerd/runtime/v2/shim/shim.go        |  9 ++++-
+ 5 files changed, 65 insertions(+), 3 deletions(-)
+
+diff --git a/Makefile b/Makefile
+index f7a9311..9957db9 100644
+--- a/Makefile
++++ b/Makefile
+@@ -505,7 +505,7 @@ define SHOW_ARCH
+   $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
+ endef
+ 
+-all: runtime netmon
++all: runtime containerd-shim-v2 netmon
+ 
+ # Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
+ # git revision string.  They will only be rebuilt if the revision string
+diff --git a/containerd-shim-v2/container.go b/containerd-shim-v2/container.go
+index 6b5e994..fa7f15b 100644
+--- a/containerd-shim-v2/container.go
++++ b/containerd-shim-v2/container.go
+@@ -6,10 +6,13 @@
+ package containerdshim
+ 
+ import (
++	"os"
++	"path"
+ 	"time"
+ 
+ 	"github.com/containerd/containerd/api/types/task"
+ 	"github.com/containerd/containerd/errdefs"
++	cdshim "github.com/containerd/containerd/runtime/v2/shim"
+ 	taskAPI "github.com/containerd/containerd/runtime/v2/task"
+ 	"github.com/opencontainers/runtime-spec/specs-go"
+ 
+@@ -34,6 +37,8 @@ type container struct {
+ 	status   task.Status
+ 	terminal bool
+ 	mounted  bool
++	exitFifo string
++	exitFd   *os.File
+ }
+ 
+ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) {
+@@ -46,6 +51,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
+ 		spec = &specs.Spec{}
+ 	}
+ 
++	dir := os.Getenv(cdshim.ExitFifoDir)
++	exitFifo := path.Join(dir, r.ID, exitFifoName)
++
+ 	c := &container{
+ 		s:        s,
+ 		spec:     spec,
+@@ -61,6 +69,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
+ 		exitIOch: make(chan struct{}),
+ 		exitCh:   make(chan uint32, 1),
+ 		mounted:  mounted,
++		exitFifo: exitFifo,
+ 	}
+ 	return c, nil
+ }
+diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go
+index 8e9b949..c85eb24 100644
+--- a/containerd-shim-v2/service.go
++++ b/containerd-shim-v2/service.go
+@@ -10,6 +10,7 @@ import (
+ 	"io/ioutil"
+ 	"os"
+ 	sysexec "os/exec"
++	"strconv"
+ 	"sync"
+ 	"syscall"
+ 	"time"
+@@ -48,6 +49,8 @@ const (
+ 	// A time span used to wait for publish a containerd event,
+ 	// once it costs a longer time than timeOut, it will be canceld.
+ 	timeOut = 5 * time.Second
++
++	exitFifoName = "exit_fifo"
+ )
+ 
+ var (
+@@ -883,7 +886,42 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.
+ func (s *service) processExits() {
+ 	for e := range s.ec {
+ 		s.checkProcesses(e)
++
++		if os.Getenv(cdshim.ExitFifoDir) != "" {
++			s.closeExitFifo(e)
++		}
++	}
++}
++
++func (s *service) closeExitFifo(e exit) {
++	if e.execid != "" {
++		// not a container, no need to close exit fifo
++		return
+ 	}
++
++	var ret uint32
++
++	s.mu.Lock()
++	c, err := s.getContainer(e.id)
++	s.mu.Unlock()
++
++	if err != nil {
++		logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id)
++		return
++	}
++
++	ret = <-c.exitCh
++	// refill the exitCh with the container process's exit code in case
++	// there were other waits on this process.
++	c.exitCh <- ret
++	exitStr := strconv.FormatUint(uint64(ret), 10)
++
++	_, err = c.exitFd.Write([]byte(exitStr))
++	if err != nil {
++		logrus.WithError(err).Error("write exit fifo failed")
++	}
++
++	c.exitFd.Close()
+ }
+ 
+ func (s *service) checkProcesses(e exit) {
+diff --git a/containerd-shim-v2/start.go b/containerd-shim-v2/start.go
+index 173ca7c..bb3ce1d 100644
+--- a/containerd-shim-v2/start.go
++++ b/containerd-shim-v2/start.go
+@@ -8,8 +8,11 @@ package containerdshim
+ import (
+ 	"context"
+ 	"fmt"
++	"golang.org/x/sys/unix"
++	"os"
+ 
+ 	"github.com/containerd/containerd/api/types/task"
++	cdshim "github.com/containerd/containerd/runtime/v2/shim"
+ 	"github.com/kata-containers/runtime/pkg/katautils"
+ )
+ 
+@@ -53,6 +56,13 @@ func startContainer(ctx context.Context, s *service, c *container) error {
+ 
+ 	c.status = task.StatusRunning
+ 
++	if os.Getenv(cdshim.ExitFifoDir) != "" {
++		c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
++		if err != nil {
++			return err
++		}
++	}
++
+ 	stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id)
+ 	if err != nil {
+ 		return err
+diff --git a/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
+index d60d496..8bccfef 100644
+--- a/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
++++ b/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
+@@ -84,6 +84,8 @@ var (
+ 	action               string
+ )
+ 
++var ExitFifoDir = "EXIT_FIFO_DIR"
++
+ func parseFlags() {
+ 	flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs")
+ 	flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim")
+@@ -198,9 +200,12 @@ func run(id string, initFunc Init, config Config) error {
+ 		}
+ 		return nil
+ 	default:
+-		if err := setLogger(ctx, idFlag); err != nil {
+-			return err
++		if os.Getenv("EXIT_FIFO_DIR") == "" {
++			if err := setLogger(ctx, idFlag); err != nil {
++				return err
++			}
+ 		}
++
+ 		client := NewShimClient(ctx, service, signals)
+ 		return client.Serve()
+ 	}
+-- 
+2.20.1
+

runtime/patches/0073-write-exit-code-in-integer-byte-order.patch

@@ -0,0 +1,78 @@
+From 3282a8bb49fff14b1eb9742544803255bfa82a82 Mon Sep 17 00:00:00 2001
+From: gaohuatao <gaohuatao@huawei.com>
+Date: Wed, 12 May 2021 17:17:41 +0800
+Subject: [PATCH] write exit code in integer byte order
+
+reason: write exit code in integer byte order just to adapt iSulad
+
+Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+---
+ containerd-shim-v2/service.go | 23 ++++++++++++++++++++---
+ 1 file changed, 20 insertions(+), 3 deletions(-)
+
+diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go
+index c85eb24..9c40f43 100644
+--- a/containerd-shim-v2/service.go
++++ b/containerd-shim-v2/service.go
+@@ -6,14 +6,16 @@
+ package containerdshim
+ 
+ import (
++	"bytes"
+ 	"context"
++	"encoding/binary"
+ 	"io/ioutil"
+ 	"os"
+ 	sysexec "os/exec"
+-	"strconv"
+ 	"sync"
+ 	"syscall"
+ 	"time"
++	"unsafe"
+ 
+ 	eventstypes "github.com/containerd/containerd/api/events"
+ 	"github.com/containerd/containerd/api/types/task"
+@@ -893,6 +895,12 @@ func (s *service) processExits() {
+ 	}
+ }
+ 
++func isBigEndian() (ret bool) {
++	i := int(0x1)
++	bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i))
++	return bs[0] == 0
++}
++
+ func (s *service) closeExitFifo(e exit) {
+ 	if e.execid != "" {
+ 		// not a container, no need to close exit fifo
+@@ -900,6 +908,7 @@ func (s *service) closeExitFifo(e exit) {
+ 	}
+ 
+ 	var ret uint32
++	var nativeEndian binary.ByteOrder
+ 
+ 	s.mu.Lock()
+ 	c, err := s.getContainer(e.id)
+@@ -914,9 +923,17 @@ func (s *service) closeExitFifo(e exit) {
+ 	// refill the exitCh with the container process's exit code in case
+ 	// there were other waits on this process.
+ 	c.exitCh <- ret
+-	exitStr := strconv.FormatUint(uint64(ret), 10)
++	
++	if isBigEndian() {
++		nativeEndian = binary.BigEndian
++	} else {
++		nativeEndian = binary.LittleEndian
++	}
++
++	bytesBuffer := bytes.NewBuffer([]byte{})
++	binary.Write(bytesBuffer, nativeEndian, &ret)
+ 
+-	_, err = c.exitFd.Write([]byte(exitStr))
++	_, err = c.exitFd.Write(bytesBuffer.Bytes())
+ 	if err != nil {
+ 		logrus.WithError(err).Error("write exit fifo failed")
+ 	}
+-- 
+2.20.1
+

runtime/patches/0074-add-timeout-for-grpcWaitProcessRequest.patch

@@ -0,0 +1,37 @@
+From 1972d0ac0236564f0d135cf94baad202126d13c8 Mon Sep 17 00:00:00 2001
+From: gaohuatao <gaohuatao@huawei.com>
+Date: Thu, 3 Jun 2021 16:28:33 +0800
+Subject: [PATCH] add timeout for grpcWaitProcessRequest
+
+reason: add timeout for grpcWaitProcessRequest, it is only used
+in stopping container process
+
+Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+---
+ virtcontainers/kata_agent.go | 3 ++-
+ 1 file changed, 2 insertions(+), 1 deletion(-)
+
+diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
+index 2717e30..058d79e 100644
+--- a/virtcontainers/kata_agent.go
++++ b/virtcontainers/kata_agent.go
+@@ -59,6 +59,7 @@ const (
+ )
+ 
+ var (
++	waitProcessTimeout          = 10 * time.Second
+ 	checkRequestTimeout         = 10 * time.Second
+ 	defaultRequestTimeout       = 10 * time.Second
+ 	createContainerTimeout      = 120 * time.Second
+@@ -2154,7 +2155,7 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
+ 	ctx = context.Background()
+ 	switch reqName {
+ 	case grpcWaitProcessRequest:
+-		// Wait has no timeout
++		ctx, cancel = context.WithTimeout(ctx, waitProcessTimeout)
+ 	case grpcCheckRequest:
+ 		ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
+ 	case grpcCreateContainerRequest:
+-- 
+2.20.1
+

runtime/patches/0075-move-waitProcess-timeout-to-stop-process.patch

@@ -0,0 +1,83 @@
+From 9568f7f2cad915778396dc8a2743c9c3ed7d44b7 Mon Sep 17 00:00:00 2001
+From: gaohuatao <gaohuatao@huawei.com>
+Date: Wed, 16 Jun 2021 14:54:24 +0800
+Subject: [PATCH] move waitProcess timeout to stop process
+
+reason: move timeout to stopping container process, it is only used in stopping container process to support shimv2
+
+Signed-off-by: gaohuatao <gaohuatao@huawei.com>
+---
+ virtcontainers/container.go  | 28 +++++++++++++++++++++++-----
+ virtcontainers/kata_agent.go |  3 +--
+ 2 files changed, 24 insertions(+), 7 deletions(-)
+
+diff --git a/virtcontainers/container.go b/virtcontainers/container.go
+index 34f6ec6..e976453 100644
+--- a/virtcontainers/container.go
++++ b/virtcontainers/container.go
+@@ -51,6 +51,8 @@ var cdromMajors = map[int64]string{
+ 	32: "CM206_CDROM_MAJOR",
+ }
+ 
++var waitProcessTimeout = 10 * time.Second
++
+ var safeCopyFiles = map[string]struct{}{
+ 	"resolv.conf": {},
+ 	"hostname":    {},
+@@ -1174,11 +1176,27 @@ func (c *Container) stop(force bool) error {
+ 		c.Logger().Errorf("send signal to container failed: %v", err)
+ 	}
+ 
+-	// Since the agent has supported the MultiWaitProcess, it's better to
+-	// wait the process here to make sure the process has exited before to
+-	// issue stopContainer, otherwise the RemoveContainerRequest in it will
+-	// get failed if the process hasn't exited.
+-	c.sandbox.agent.waitProcess(c, c.id)
++	done := make(chan error)
++	go func() {
++		// Since the agent has supported the MultiWaitProcess, it's better to
++		// wait the process here to make sure the process has exited before to
++		// issue stopContainer, otherwise the RemoveContainerRequest in it will
++		// get failed if the process hasn't exited.
++		_, err := c.sandbox.agent.waitProcess(c, c.id)
++		if err != nil {
++			done <- err
++		}
++		close(done)
++	}()
++
++	select {
++	case err := <-done:
++		if err != nil {
++			c.Logger().Errorf("wait process failed:%v", err)
++		}
++	case <-time.After(time.Duration(waitProcessTimeout) * time.Second):
++		// Nothing to do
++	}
+ 
+ 	defer func() {
+ 		// Save device and drive data.
+diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
+index 058d79e..2717e30 100644
+--- a/virtcontainers/kata_agent.go
++++ b/virtcontainers/kata_agent.go
+@@ -59,7 +59,6 @@ const (
+ )
+ 
+ var (
+-	waitProcessTimeout          = 10 * time.Second
+ 	checkRequestTimeout         = 10 * time.Second
+ 	defaultRequestTimeout       = 10 * time.Second
+ 	createContainerTimeout      = 120 * time.Second
+@@ -2155,7 +2154,7 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
+ 	ctx = context.Background()
+ 	switch reqName {
+ 	case grpcWaitProcessRequest:
+-		ctx, cancel = context.WithTimeout(ctx, waitProcessTimeout)
++		// Wait has no timeout
+ 	case grpcCheckRequest:
+ 		ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
+ 	case grpcCreateContainerRequest:
+-- 
+2.20.1
+

runtime/series.conf

@@ -68,3 +68,8 @@
 0068-kata-runtime-modify-make-flags.patch
 0069-kata-runtime-add-linkmode-to-resolve-build-error.patch
 0070-kata-runtime-remove-ctty-to-resolve-build-failed.patch
+0071-fix-containerd-shim-kata-v2-compile-bug.patch
+0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch
+0073-write-exit-code-in-integer-byte-order.patch
+0074-add-timeout-for-grpcWaitProcessRequest.patch
+0075-move-waitProcess-timeout-to-stop-process.patch