diff --git a/kata-containers.spec b/kata-containers.spec index 2e27435..00eabd1 100644 --- a/kata-containers.spec +++ b/kata-containers.spec @@ -2,7 +2,7 @@ %global debug_package %{nil} %define VERSION v1.11.1 -%define RELEASE 10 +%define RELEASE 11 Name: kata-containers Version: %{VERSION} @@ -70,7 +70,8 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot cd %{_builddir}/kata_integration mkdir -p -m 750 %{buildroot}/usr/bin -install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/ +strip ./build/kata-runtime ./build/containerd-shim-kata-v2 ./build/kata-proxy ./build/kata-shim ./build/kata-netmon +install -p -m 750 ./build/kata-runtime ./build/containerd-shim-kata-v2 ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/ install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/ mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml @@ -79,6 +80,7 @@ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/u %files /usr/bin/kata-runtime +/usr/bin/containerd-shim-kata-v2 /usr/bin/kata-proxy /usr/bin/kata-shim /usr/bin/kata-netmon @@ -90,6 +92,12 @@ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/u %changelog +* Sat Nov 27 2021 yangfeiyu - 1.11.1-11 +- Type:enhancement +- ID:NA +- SUG:NA +- DESC:support shimv2 with isulad + * Tue Feb 23 2021 xinghe - 1.11.1-10 - Type:CVE - ID:NA diff --git a/runtime/kata-runtime.spec b/runtime/kata-runtime.spec index acef8ce..669d742 100644 --- a/runtime/kata-runtime.spec +++ b/runtime/kata-runtime.spec @@ -1,7 +1,7 @@ %define debug_package %{nil} %define VERSION 1.11.1 -%define RELEASE 11 +%define RELEASE 12 Name: kata-runtime Version: %{VERSION} @@ -98,6 +98,12 @@ install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/de /usr/share/defaults/kata-containers/configuration.toml %changelog +* Sat Nov 27 2021 yangfeiyu - 1.11.1-12 +- Type:enhancement +- ID:NA +- SUG:restart +- DESC:support shimv2 with isulad + * Tue Nov 17 2020 yangfeiyu - 1.11.1-11 - Type:bugfix - ID:NA diff --git a/runtime/patches/0071-fix-containerd-shim-kata-v2-compile-bug.patch b/runtime/patches/0071-fix-containerd-shim-kata-v2-compile-bug.patch new file mode 100644 index 0000000..c5c2210 --- /dev/null +++ b/runtime/patches/0071-fix-containerd-shim-kata-v2-compile-bug.patch @@ -0,0 +1,32 @@ +From c9737d7702e4042bcc433da720611ebac06f6f6d Mon Sep 17 00:00:00 2001 +From: gaohuatao +Date: Wed, 28 Apr 2021 15:04:02 +0800 +Subject: [PATCH 1/2] fix containerd shim kata v2 compile bug + +reason: fix containerd shim kata v2 compile bug + +Signed-off-by: gaohuatao +--- + containerd-shim-v2/metrics.go | 6 +++--- + 1 file changed, 3 insertions(+), 3 deletions(-) + +diff --git a/containerd-shim-v2/metrics.go b/containerd-shim-v2/metrics.go +index c936966..d59530f 100644 +--- a/containerd-shim-v2/metrics.go ++++ b/containerd-shim-v2/metrics.go +@@ -172,9 +172,9 @@ func copyBlkio(s []vc.BlkioStatEntry) []*cgroups.BlkIOEntry { + return ret + } + +-func setNetworkStats(vcNetwork []*vc.NetworkStats) []*cgroups.NetworkStat { +- networkStats := make([]*cgroups.NetworkStat, len(vcNetwork)) +- for i, v := range vcNetwork { ++func setNetworkStats(vcNetwork *vc.NetworkStats) []*cgroups.NetworkStat { ++ networkStats := make([]*cgroups.NetworkStat, len(vcNetwork.Interfaces)) ++ for i, v := range vcNetwork.Interfaces { + networkStats[i] = &cgroups.NetworkStat{ + Name: v.Name, + RxBytes: v.RxBytes, +-- +2.20.1 + diff --git a/runtime/patches/0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch b/runtime/patches/0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch new file mode 100644 index 0000000..6c85157 --- /dev/null +++ b/runtime/patches/0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch @@ -0,0 +1,197 @@ +From 488ebd1f148bc9ddda9376a065d38cbc9bb9a0fe Mon Sep 17 00:00:00 2001 +From: gaohuatao +Date: Wed, 28 Apr 2021 15:31:30 +0800 +Subject: [PATCH] kata shimv2 adapt iSulad and open build flag + +Signed-off-by: gaohuatao +--- + Makefile | 2 +- + containerd-shim-v2/container.go | 9 +++++ + containerd-shim-v2/service.go | 38 +++++++++++++++++++ + containerd-shim-v2/start.go | 10 +++++ + .../containerd/runtime/v2/shim/shim.go | 9 ++++- + 5 files changed, 65 insertions(+), 3 deletions(-) + +diff --git a/Makefile b/Makefile +index f7a9311..9957db9 100644 +--- a/Makefile ++++ b/Makefile +@@ -505,7 +505,7 @@ define SHOW_ARCH + $(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)","")) + endef + +-all: runtime netmon ++all: runtime containerd-shim-v2 netmon + + # Targets that depend on .git-commit can use $(shell cat .git-commit) to get a + # git revision string. They will only be rebuilt if the revision string +diff --git a/containerd-shim-v2/container.go b/containerd-shim-v2/container.go +index 6b5e994..fa7f15b 100644 +--- a/containerd-shim-v2/container.go ++++ b/containerd-shim-v2/container.go +@@ -6,10 +6,13 @@ + package containerdshim + + import ( ++ "os" ++ "path" + "time" + + "github.com/containerd/containerd/api/types/task" + "github.com/containerd/containerd/errdefs" ++ cdshim "github.com/containerd/containerd/runtime/v2/shim" + taskAPI "github.com/containerd/containerd/runtime/v2/task" + "github.com/opencontainers/runtime-spec/specs-go" + +@@ -34,6 +37,8 @@ type container struct { + status task.Status + terminal bool + mounted bool ++ exitFifo string ++ exitFd *os.File + } + + func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) { +@@ -46,6 +51,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con + spec = &specs.Spec{} + } + ++ dir := os.Getenv(cdshim.ExitFifoDir) ++ exitFifo := path.Join(dir, r.ID, exitFifoName) ++ + c := &container{ + s: s, + spec: spec, +@@ -61,6 +69,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con + exitIOch: make(chan struct{}), + exitCh: make(chan uint32, 1), + mounted: mounted, ++ exitFifo: exitFifo, + } + return c, nil + } +diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go +index 8e9b949..c85eb24 100644 +--- a/containerd-shim-v2/service.go ++++ b/containerd-shim-v2/service.go +@@ -10,6 +10,7 @@ import ( + "io/ioutil" + "os" + sysexec "os/exec" ++ "strconv" + "sync" + "syscall" + "time" +@@ -48,6 +49,8 @@ const ( + // A time span used to wait for publish a containerd event, + // once it costs a longer time than timeOut, it will be canceld. + timeOut = 5 * time.Second ++ ++ exitFifoName = "exit_fifo" + ) + + var ( +@@ -883,7 +886,42 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI. + func (s *service) processExits() { + for e := range s.ec { + s.checkProcesses(e) ++ ++ if os.Getenv(cdshim.ExitFifoDir) != "" { ++ s.closeExitFifo(e) ++ } ++ } ++} ++ ++func (s *service) closeExitFifo(e exit) { ++ if e.execid != "" { ++ // not a container, no need to close exit fifo ++ return + } ++ ++ var ret uint32 ++ ++ s.mu.Lock() ++ c, err := s.getContainer(e.id) ++ s.mu.Unlock() ++ ++ if err != nil { ++ logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id) ++ return ++ } ++ ++ ret = <-c.exitCh ++ // refill the exitCh with the container process's exit code in case ++ // there were other waits on this process. ++ c.exitCh <- ret ++ exitStr := strconv.FormatUint(uint64(ret), 10) ++ ++ _, err = c.exitFd.Write([]byte(exitStr)) ++ if err != nil { ++ logrus.WithError(err).Error("write exit fifo failed") ++ } ++ ++ c.exitFd.Close() + } + + func (s *service) checkProcesses(e exit) { +diff --git a/containerd-shim-v2/start.go b/containerd-shim-v2/start.go +index 173ca7c..bb3ce1d 100644 +--- a/containerd-shim-v2/start.go ++++ b/containerd-shim-v2/start.go +@@ -8,8 +8,11 @@ package containerdshim + import ( + "context" + "fmt" ++ "golang.org/x/sys/unix" ++ "os" + + "github.com/containerd/containerd/api/types/task" ++ cdshim "github.com/containerd/containerd/runtime/v2/shim" + "github.com/kata-containers/runtime/pkg/katautils" + ) + +@@ -53,6 +56,13 @@ func startContainer(ctx context.Context, s *service, c *container) error { + + c.status = task.StatusRunning + ++ if os.Getenv(cdshim.ExitFifoDir) != "" { ++ c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0) ++ if err != nil { ++ return err ++ } ++ } ++ + stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id) + if err != nil { + return err +diff --git a/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go +index d60d496..8bccfef 100644 +--- a/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go ++++ b/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go +@@ -84,6 +84,8 @@ var ( + action string + ) + ++var ExitFifoDir = "EXIT_FIFO_DIR" ++ + func parseFlags() { + flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs") + flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim") +@@ -198,9 +200,12 @@ func run(id string, initFunc Init, config Config) error { + } + return nil + default: +- if err := setLogger(ctx, idFlag); err != nil { +- return err ++ if os.Getenv("EXIT_FIFO_DIR") == "" { ++ if err := setLogger(ctx, idFlag); err != nil { ++ return err ++ } + } ++ + client := NewShimClient(ctx, service, signals) + return client.Serve() + } +-- +2.20.1 + diff --git a/runtime/patches/0073-write-exit-code-in-integer-byte-order.patch b/runtime/patches/0073-write-exit-code-in-integer-byte-order.patch new file mode 100644 index 0000000..2667cbf --- /dev/null +++ b/runtime/patches/0073-write-exit-code-in-integer-byte-order.patch @@ -0,0 +1,78 @@ +From 3282a8bb49fff14b1eb9742544803255bfa82a82 Mon Sep 17 00:00:00 2001 +From: gaohuatao +Date: Wed, 12 May 2021 17:17:41 +0800 +Subject: [PATCH] write exit code in integer byte order + +reason: write exit code in integer byte order just to adapt iSulad + +Signed-off-by: gaohuatao +--- + containerd-shim-v2/service.go | 23 ++++++++++++++++++++--- + 1 file changed, 20 insertions(+), 3 deletions(-) + +diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go +index c85eb24..9c40f43 100644 +--- a/containerd-shim-v2/service.go ++++ b/containerd-shim-v2/service.go +@@ -6,14 +6,16 @@ + package containerdshim + + import ( ++ "bytes" + "context" ++ "encoding/binary" + "io/ioutil" + "os" + sysexec "os/exec" +- "strconv" + "sync" + "syscall" + "time" ++ "unsafe" + + eventstypes "github.com/containerd/containerd/api/events" + "github.com/containerd/containerd/api/types/task" +@@ -893,6 +895,12 @@ func (s *service) processExits() { + } + } + ++func isBigEndian() (ret bool) { ++ i := int(0x1) ++ bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i)) ++ return bs[0] == 0 ++} ++ + func (s *service) closeExitFifo(e exit) { + if e.execid != "" { + // not a container, no need to close exit fifo +@@ -900,6 +908,7 @@ func (s *service) closeExitFifo(e exit) { + } + + var ret uint32 ++ var nativeEndian binary.ByteOrder + + s.mu.Lock() + c, err := s.getContainer(e.id) +@@ -914,9 +923,17 @@ func (s *service) closeExitFifo(e exit) { + // refill the exitCh with the container process's exit code in case + // there were other waits on this process. + c.exitCh <- ret +- exitStr := strconv.FormatUint(uint64(ret), 10) ++ ++ if isBigEndian() { ++ nativeEndian = binary.BigEndian ++ } else { ++ nativeEndian = binary.LittleEndian ++ } ++ ++ bytesBuffer := bytes.NewBuffer([]byte{}) ++ binary.Write(bytesBuffer, nativeEndian, &ret) + +- _, err = c.exitFd.Write([]byte(exitStr)) ++ _, err = c.exitFd.Write(bytesBuffer.Bytes()) + if err != nil { + logrus.WithError(err).Error("write exit fifo failed") + } +-- +2.20.1 + diff --git a/runtime/patches/0074-add-timeout-for-grpcWaitProcessRequest.patch b/runtime/patches/0074-add-timeout-for-grpcWaitProcessRequest.patch new file mode 100644 index 0000000..9900748 --- /dev/null +++ b/runtime/patches/0074-add-timeout-for-grpcWaitProcessRequest.patch @@ -0,0 +1,37 @@ +From 1972d0ac0236564f0d135cf94baad202126d13c8 Mon Sep 17 00:00:00 2001 +From: gaohuatao +Date: Thu, 3 Jun 2021 16:28:33 +0800 +Subject: [PATCH] add timeout for grpcWaitProcessRequest + +reason: add timeout for grpcWaitProcessRequest, it is only used +in stopping container process + +Signed-off-by: gaohuatao +--- + virtcontainers/kata_agent.go | 3 ++- + 1 file changed, 2 insertions(+), 1 deletion(-) + +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 2717e30..058d79e 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -59,6 +59,7 @@ const ( + ) + + var ( ++ waitProcessTimeout = 10 * time.Second + checkRequestTimeout = 10 * time.Second + defaultRequestTimeout = 10 * time.Second + createContainerTimeout = 120 * time.Second +@@ -2154,7 +2155,7 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c + ctx = context.Background() + switch reqName { + case grpcWaitProcessRequest: +- // Wait has no timeout ++ ctx, cancel = context.WithTimeout(ctx, waitProcessTimeout) + case grpcCheckRequest: + ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout) + case grpcCreateContainerRequest: +-- +2.20.1 + diff --git a/runtime/patches/0075-move-waitProcess-timeout-to-stop-process.patch b/runtime/patches/0075-move-waitProcess-timeout-to-stop-process.patch new file mode 100644 index 0000000..604108f --- /dev/null +++ b/runtime/patches/0075-move-waitProcess-timeout-to-stop-process.patch @@ -0,0 +1,83 @@ +From 9568f7f2cad915778396dc8a2743c9c3ed7d44b7 Mon Sep 17 00:00:00 2001 +From: gaohuatao +Date: Wed, 16 Jun 2021 14:54:24 +0800 +Subject: [PATCH] move waitProcess timeout to stop process + +reason: move timeout to stopping container process, it is only used in stopping container process to support shimv2 + +Signed-off-by: gaohuatao +--- + virtcontainers/container.go | 28 +++++++++++++++++++++++----- + virtcontainers/kata_agent.go | 3 +-- + 2 files changed, 24 insertions(+), 7 deletions(-) + +diff --git a/virtcontainers/container.go b/virtcontainers/container.go +index 34f6ec6..e976453 100644 +--- a/virtcontainers/container.go ++++ b/virtcontainers/container.go +@@ -51,6 +51,8 @@ var cdromMajors = map[int64]string{ + 32: "CM206_CDROM_MAJOR", + } + ++var waitProcessTimeout = 10 * time.Second ++ + var safeCopyFiles = map[string]struct{}{ + "resolv.conf": {}, + "hostname": {}, +@@ -1174,11 +1176,27 @@ func (c *Container) stop(force bool) error { + c.Logger().Errorf("send signal to container failed: %v", err) + } + +- // Since the agent has supported the MultiWaitProcess, it's better to +- // wait the process here to make sure the process has exited before to +- // issue stopContainer, otherwise the RemoveContainerRequest in it will +- // get failed if the process hasn't exited. +- c.sandbox.agent.waitProcess(c, c.id) ++ done := make(chan error) ++ go func() { ++ // Since the agent has supported the MultiWaitProcess, it's better to ++ // wait the process here to make sure the process has exited before to ++ // issue stopContainer, otherwise the RemoveContainerRequest in it will ++ // get failed if the process hasn't exited. ++ _, err := c.sandbox.agent.waitProcess(c, c.id) ++ if err != nil { ++ done <- err ++ } ++ close(done) ++ }() ++ ++ select { ++ case err := <-done: ++ if err != nil { ++ c.Logger().Errorf("wait process failed:%v", err) ++ } ++ case <-time.After(time.Duration(waitProcessTimeout) * time.Second): ++ // Nothing to do ++ } + + defer func() { + // Save device and drive data. +diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go +index 058d79e..2717e30 100644 +--- a/virtcontainers/kata_agent.go ++++ b/virtcontainers/kata_agent.go +@@ -59,7 +59,6 @@ const ( + ) + + var ( +- waitProcessTimeout = 10 * time.Second + checkRequestTimeout = 10 * time.Second + defaultRequestTimeout = 10 * time.Second + createContainerTimeout = 120 * time.Second +@@ -2155,7 +2154,7 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c + ctx = context.Background() + switch reqName { + case grpcWaitProcessRequest: +- ctx, cancel = context.WithTimeout(ctx, waitProcessTimeout) ++ // Wait has no timeout + case grpcCheckRequest: + ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout) + case grpcCreateContainerRequest: +-- +2.20.1 + diff --git a/runtime/series.conf b/runtime/series.conf index 3d6f95c..3a82ab1 100644 --- a/runtime/series.conf +++ b/runtime/series.conf @@ -68,3 +68,8 @@ 0068-kata-runtime-modify-make-flags.patch 0069-kata-runtime-add-linkmode-to-resolve-build-error.patch 0070-kata-runtime-remove-ctty-to-resolve-build-failed.patch +0071-fix-containerd-shim-kata-v2-compile-bug.patch +0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch +0073-write-exit-code-in-integer-byte-order.patch +0074-add-timeout-for-grpcWaitProcessRequest.patch +0075-move-waitProcess-timeout-to-stop-process.patch