packages/kata-containers
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

kata-containers: support shimv2 with isulad

Browse Source 
Signed-off-by: holyfei <yangfeiyu20092010@163.com>
This commit is contained in:
holyfei 2021-11-27 11:31:40 +08:00
parent 753d3977ea
commit 1666424fd5
8 changed files with 449 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
kata-containers.spec
View File

@ -2,7 +2,7 @@
%global debug_package %{nil}
%define VERSION v1.11.1
%define RELEASE 10
%define RELEASE 11
Name: kata-containers
Version: %{VERSION}
@ -70,7 +70,8 @@ install -p -m 755 -D %{_builddir}/kernel/linux/arch/arm64/boot/Image %{buildroot
cd %{_builddir}/kata_integration
mkdir -p -m 750 %{buildroot}/usr/bin
install -p -m 750 ./build/kata-runtime ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
strip ./build/kata-runtime ./build/containerd-shim-kata-v2 ./build/kata-proxy ./build/kata-shim ./build/kata-netmon
install -p -m 750 ./build/kata-runtime ./build/containerd-shim-kata-v2 ./build/kata-proxy ./build/kata-shim ./build/kata-netmon %{buildroot}/usr/bin/
install -p -m 640 ./build/kata-containers-initrd.img %{buildroot}/var/lib/kata/
mkdir -p -m 750 %{buildroot}/usr/share/defaults/kata-containers/
install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/usr/share/defaults/kata-containers/configuration.toml
@ -79,6 +80,7 @@ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/u
%files
/usr/bin/kata-runtime
/usr/bin/containerd-shim-kata-v2
/usr/bin/kata-proxy
/usr/bin/kata-shim
/usr/bin/kata-netmon
@ -90,6 +92,12 @@ install -p -m 640 -D ./runtime/cli/config/configuration-qemu.toml %{buildroot}/u
%changelog
* Sat Nov 27 2021 yangfeiyu <yangfeiyu2@huawei.com> - 1.11.1-11
- Type:enhancement
- ID:NA
- SUG:NA
- DESC:support shimv2 with isulad
* Tue Feb 23 2021 xinghe <xinghe1@huawei.com> - 1.11.1-10
- Type:CVE
- ID:NA

8
runtime/kata-runtime.spec
View File

@ -1,7 +1,7 @@
%define debug_package %{nil}
%define VERSION 1.11.1
%define RELEASE 11
%define RELEASE 12
Name: kata-runtime
Version: %{VERSION}
@ -98,6 +98,12 @@ install -p -m 640 ./cli/config/configuration-qemu.toml %{buildroot}/usr/share/de
/usr/share/defaults/kata-containers/configuration.toml
%changelog
* Sat Nov 27 2021 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-12
- Type:enhancement
- ID:NA
- SUG:restart
- DESC:support shimv2 with isulad
* Tue Nov 17 2020 yangfeiyu<yangfeiyu20102011@163.com> - 1.11.1-11
- Type:bugfix
- ID:NA

32
runtime/patches/0071-fix-containerd-shim-kata-v2-compile-bug.patch Normal file
View File

@ -0,0 +1,32 @@
From c9737d7702e4042bcc433da720611ebac06f6f6d Mon Sep 17 00:00:00 2001
From: gaohuatao <gaohuatao@huawei.com>
Date: Wed, 28 Apr 2021 15:04:02 +0800
Subject: [PATCH 1/2] fix containerd shim kata v2 compile bug
reason: fix containerd shim kata v2 compile bug
Signed-off-by: gaohuatao <gaohuatao@huawei.com>
---
containerd-shim-v2/metrics.go | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
diff --git a/containerd-shim-v2/metrics.go b/containerd-shim-v2/metrics.go
index c936966..d59530f 100644
--- a/containerd-shim-v2/metrics.go
+++ b/containerd-shim-v2/metrics.go
@@ -172,9 +172,9 @@ func copyBlkio(s []vc.BlkioStatEntry) []*cgroups.BlkIOEntry {
return ret
}
-func setNetworkStats(vcNetwork []*vc.NetworkStats) []*cgroups.NetworkStat {
- networkStats := make([]*cgroups.NetworkStat, len(vcNetwork))
- for i, v := range vcNetwork {
+func setNetworkStats(vcNetwork *vc.NetworkStats) []*cgroups.NetworkStat {
+ networkStats := make([]*cgroups.NetworkStat, len(vcNetwork.Interfaces))
+ for i, v := range vcNetwork.Interfaces {
networkStats[i] = &cgroups.NetworkStat{
Name: v.Name,
RxBytes: v.RxBytes,
--
2.20.1

197
runtime/patches/0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch Normal file
View File

@ -0,0 +1,197 @@
From 488ebd1f148bc9ddda9376a065d38cbc9bb9a0fe Mon Sep 17 00:00:00 2001
From: gaohuatao <gaohuatao@huawei.com>
Date: Wed, 28 Apr 2021 15:31:30 +0800
Subject: [PATCH] kata shimv2 adapt iSulad and open build flag
Signed-off-by: gaohuatao <gaohuatao@huawei.com>
---
Makefile | 2 +-
containerd-shim-v2/container.go | 9 +++++
containerd-shim-v2/service.go | 38 +++++++++++++++++++
containerd-shim-v2/start.go | 10 +++++
.../containerd/runtime/v2/shim/shim.go | 9 ++++-
5 files changed, 65 insertions(+), 3 deletions(-)
diff --git a/Makefile b/Makefile
index f7a9311..9957db9 100644
--- a/Makefile
+++ b/Makefile
@@ -505,7 +505,7 @@ define SHOW_ARCH
$(shell printf "\\t%s%s\\\n" "$(1)" $(if $(filter $(ARCH),$(1))," (default)",""))
endef
-all: runtime netmon
+all: runtime containerd-shim-v2 netmon
# Targets that depend on .git-commit can use $(shell cat .git-commit) to get a
# git revision string. They will only be rebuilt if the revision string
diff --git a/containerd-shim-v2/container.go b/containerd-shim-v2/container.go
index 6b5e994..fa7f15b 100644
--- a/containerd-shim-v2/container.go
+++ b/containerd-shim-v2/container.go
@@ -6,10 +6,13 @@
package containerdshim
import (
+ "os"
+ "path"
"time"
"github.com/containerd/containerd/api/types/task"
"github.com/containerd/containerd/errdefs"
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
taskAPI "github.com/containerd/containerd/runtime/v2/task"
"github.com/opencontainers/runtime-spec/specs-go"
@@ -34,6 +37,8 @@ type container struct {
status task.Status
terminal bool
mounted bool
+ exitFifo string
+ exitFd *os.File
}
func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.ContainerType, spec *specs.Spec, mounted bool) (*container, error) {
@@ -46,6 +51,9 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
spec = &specs.Spec{}
}
+ dir := os.Getenv(cdshim.ExitFifoDir)
+ exitFifo := path.Join(dir, r.ID, exitFifoName)
+
c := &container{
s: s,
spec: spec,
@@ -61,6 +69,7 @@ func newContainer(s *service, r *taskAPI.CreateTaskRequest, containerType vc.Con
exitIOch: make(chan struct{}),
exitCh: make(chan uint32, 1),
mounted: mounted,
+ exitFifo: exitFifo,
}
return c, nil
}
diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go
index 8e9b949..c85eb24 100644
--- a/containerd-shim-v2/service.go
+++ b/containerd-shim-v2/service.go
@@ -10,6 +10,7 @@ import (
"io/ioutil"
"os"
sysexec "os/exec"
+ "strconv"
"sync"
"syscall"
"time"
@@ -48,6 +49,8 @@ const (
// A time span used to wait for publish a containerd event,
// once it costs a longer time than timeOut, it will be canceld.
timeOut = 5 * time.Second
+
+ exitFifoName = "exit_fifo"
)
var (
@@ -883,7 +886,42 @@ func (s *service) Wait(ctx context.Context, r *taskAPI.WaitRequest) (_ *taskAPI.
func (s *service) processExits() {
for e := range s.ec {
s.checkProcesses(e)
+
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
+ s.closeExitFifo(e)
+ }
+ }
+}
+
+func (s *service) closeExitFifo(e exit) {
+ if e.execid != "" {
+ // not a container, no need to close exit fifo
+ return
}
+
+ var ret uint32
+
+ s.mu.Lock()
+ c, err := s.getContainer(e.id)
+ s.mu.Unlock()
+
+ if err != nil {
+ logrus.WithError(err).Errorf("Process container:%v exit fifo failed", e.id)
+ return
+ }
+
+ ret = <-c.exitCh
+ // refill the exitCh with the container process's exit code in case
+ // there were other waits on this process.
+ c.exitCh <- ret
+ exitStr := strconv.FormatUint(uint64(ret), 10)
+
+ _, err = c.exitFd.Write([]byte(exitStr))
+ if err != nil {
+ logrus.WithError(err).Error("write exit fifo failed")
+ }
+
+ c.exitFd.Close()
}
func (s *service) checkProcesses(e exit) {
diff --git a/containerd-shim-v2/start.go b/containerd-shim-v2/start.go
index 173ca7c..bb3ce1d 100644
--- a/containerd-shim-v2/start.go
+++ b/containerd-shim-v2/start.go
@@ -8,8 +8,11 @@ package containerdshim
import (
"context"
"fmt"
+ "golang.org/x/sys/unix"
+ "os"
"github.com/containerd/containerd/api/types/task"
+ cdshim "github.com/containerd/containerd/runtime/v2/shim"
"github.com/kata-containers/runtime/pkg/katautils"
)
@@ -53,6 +56,13 @@ func startContainer(ctx context.Context, s *service, c *container) error {
c.status = task.StatusRunning
+ if os.Getenv(cdshim.ExitFifoDir) != "" {
+ c.exitFd, err = os.OpenFile(c.exitFifo, unix.O_WRONLY|unix.O_NONBLOCK|unix.O_CLOEXEC, 0)
+ if err != nil {
+ return err
+ }
+ }
+
stdin, stdout, stderr, err := s.sandbox.IOStream(c.id, c.id)
if err != nil {
return err
diff --git a/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go b/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
index d60d496..8bccfef 100644
--- a/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
+++ b/vendor/github.com/containerd/containerd/runtime/v2/shim/shim.go
@@ -84,6 +84,8 @@ var (
action string
)
+var ExitFifoDir = "EXIT_FIFO_DIR"
+
func parseFlags() {
flag.BoolVar(&debugFlag, "debug", false, "enable debug output in logs")
flag.StringVar(&namespaceFlag, "namespace", "", "namespace that owns the shim")
@@ -198,9 +200,12 @@ func run(id string, initFunc Init, config Config) error {
}
return nil
default:
- if err := setLogger(ctx, idFlag); err != nil {
- return err
+ if os.Getenv("EXIT_FIFO_DIR") == "" {
+ if err := setLogger(ctx, idFlag); err != nil {
+ return err
+ }
}
+
client := NewShimClient(ctx, service, signals)
return client.Serve()
}
--
2.20.1

78
runtime/patches/0073-write-exit-code-in-integer-byte-order.patch Normal file
View File

@ -0,0 +1,78 @@
From 3282a8bb49fff14b1eb9742544803255bfa82a82 Mon Sep 17 00:00:00 2001
From: gaohuatao <gaohuatao@huawei.com>
Date: Wed, 12 May 2021 17:17:41 +0800
Subject: [PATCH] write exit code in integer byte order
reason: write exit code in integer byte order just to adapt iSulad
Signed-off-by: gaohuatao <gaohuatao@huawei.com>
---
containerd-shim-v2/service.go | 23 ++++++++++++++++++++---
1 file changed, 20 insertions(+), 3 deletions(-)
diff --git a/containerd-shim-v2/service.go b/containerd-shim-v2/service.go
index c85eb24..9c40f43 100644
--- a/containerd-shim-v2/service.go
+++ b/containerd-shim-v2/service.go
@@ -6,14 +6,16 @@
package containerdshim
import (
+ "bytes"
"context"
+ "encoding/binary"
"io/ioutil"
"os"
sysexec "os/exec"
- "strconv"
"sync"
"syscall"
"time"
+ "unsafe"
eventstypes "github.com/containerd/containerd/api/events"
"github.com/containerd/containerd/api/types/task"
@@ -893,6 +895,12 @@ func (s *service) processExits() {
}
}
+func isBigEndian() (ret bool) {
+ i := int(0x1)
+ bs := (*[int(unsafe.Sizeof(i))]byte)(unsafe.Pointer(&i))
+ return bs[0] == 0
+}
+
func (s *service) closeExitFifo(e exit) {
if e.execid != "" {
// not a container, no need to close exit fifo
@@ -900,6 +908,7 @@ func (s *service) closeExitFifo(e exit) {
}
var ret uint32
+ var nativeEndian binary.ByteOrder
s.mu.Lock()
c, err := s.getContainer(e.id)
@@ -914,9 +923,17 @@ func (s *service) closeExitFifo(e exit) {
// refill the exitCh with the container process's exit code in case
// there were other waits on this process.
c.exitCh <- ret
- exitStr := strconv.FormatUint(uint64(ret), 10)
+
+ if isBigEndian() {
+ nativeEndian = binary.BigEndian
+ } else {
+ nativeEndian = binary.LittleEndian
+ }
+
+ bytesBuffer := bytes.NewBuffer([]byte{})
+ binary.Write(bytesBuffer, nativeEndian, &ret)
- _, err = c.exitFd.Write([]byte(exitStr))
+ _, err = c.exitFd.Write(bytesBuffer.Bytes())
if err != nil {
logrus.WithError(err).Error("write exit fifo failed")
}
--
2.20.1

37
runtime/patches/0074-add-timeout-for-grpcWaitProcessRequest.patch Normal file
View File

@ -0,0 +1,37 @@
From 1972d0ac0236564f0d135cf94baad202126d13c8 Mon Sep 17 00:00:00 2001
From: gaohuatao <gaohuatao@huawei.com>
Date: Thu, 3 Jun 2021 16:28:33 +0800
Subject: [PATCH] add timeout for grpcWaitProcessRequest
reason: add timeout for grpcWaitProcessRequest, it is only used
in stopping container process
Signed-off-by: gaohuatao <gaohuatao@huawei.com>
---
virtcontainers/kata_agent.go | 3 ++-
1 file changed, 2 insertions(+), 1 deletion(-)
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index 2717e30..058d79e 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -59,6 +59,7 @@ const (
)
var (
+ waitProcessTimeout = 10 * time.Second
checkRequestTimeout = 10 * time.Second
defaultRequestTimeout = 10 * time.Second
createContainerTimeout = 120 * time.Second
@@ -2154,7 +2155,7 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
ctx = context.Background()
switch reqName {
case grpcWaitProcessRequest:
- // Wait has no timeout
+ ctx, cancel = context.WithTimeout(ctx, waitProcessTimeout)
case grpcCheckRequest:
ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
case grpcCreateContainerRequest:
--
2.20.1

83
runtime/patches/0075-move-waitProcess-timeout-to-stop-process.patch Normal file
View File

@ -0,0 +1,83 @@
From 9568f7f2cad915778396dc8a2743c9c3ed7d44b7 Mon Sep 17 00:00:00 2001
From: gaohuatao <gaohuatao@huawei.com>
Date: Wed, 16 Jun 2021 14:54:24 +0800
Subject: [PATCH] move waitProcess timeout to stop process
reason: move timeout to stopping container process, it is only used in stopping container process to support shimv2
Signed-off-by: gaohuatao <gaohuatao@huawei.com>
---
virtcontainers/container.go | 28 +++++++++++++++++++++++-----
virtcontainers/kata_agent.go | 3 +--
2 files changed, 24 insertions(+), 7 deletions(-)
diff --git a/virtcontainers/container.go b/virtcontainers/container.go
index 34f6ec6..e976453 100644
--- a/virtcontainers/container.go
+++ b/virtcontainers/container.go
@@ -51,6 +51,8 @@ var cdromMajors = map[int64]string{
32: "CM206_CDROM_MAJOR",
}
+var waitProcessTimeout = 10 * time.Second
+
var safeCopyFiles = map[string]struct{}{
"resolv.conf": {},
"hostname": {},
@@ -1174,11 +1176,27 @@ func (c *Container) stop(force bool) error {
c.Logger().Errorf("send signal to container failed: %v", err)
}
- // Since the agent has supported the MultiWaitProcess, it's better to
- // wait the process here to make sure the process has exited before to
- // issue stopContainer, otherwise the RemoveContainerRequest in it will
- // get failed if the process hasn't exited.
- c.sandbox.agent.waitProcess(c, c.id)
+ done := make(chan error)
+ go func() {
+ // Since the agent has supported the MultiWaitProcess, it's better to
+ // wait the process here to make sure the process has exited before to
+ // issue stopContainer, otherwise the RemoveContainerRequest in it will
+ // get failed if the process hasn't exited.
+ _, err := c.sandbox.agent.waitProcess(c, c.id)
+ if err != nil {
+ done <- err
+ }
+ close(done)
+ }()
+
+ select {
+ case err := <-done:
+ if err != nil {
+ c.Logger().Errorf("wait process failed:%v", err)
+ }
+ case <-time.After(time.Duration(waitProcessTimeout) * time.Second):
+ // Nothing to do
+ }
defer func() {
// Save device and drive data.
diff --git a/virtcontainers/kata_agent.go b/virtcontainers/kata_agent.go
index 058d79e..2717e30 100644
--- a/virtcontainers/kata_agent.go
+++ b/virtcontainers/kata_agent.go
@@ -59,7 +59,6 @@ const (
)
var (
- waitProcessTimeout = 10 * time.Second
checkRequestTimeout = 10 * time.Second
defaultRequestTimeout = 10 * time.Second
createContainerTimeout = 120 * time.Second
@@ -2155,7 +2154,7 @@ func (k *kataAgent) getReqContext(reqName string) (ctx context.Context, cancel c
ctx = context.Background()
switch reqName {
case grpcWaitProcessRequest:
- ctx, cancel = context.WithTimeout(ctx, waitProcessTimeout)
+ // Wait has no timeout
case grpcCheckRequest:
ctx, cancel = context.WithTimeout(ctx, checkRequestTimeout)
case grpcCreateContainerRequest:
--
2.20.1

5
runtime/series.conf
View File

@ -68,3 +68,8 @@
0068-kata-runtime-modify-make-flags.patch
0069-kata-runtime-add-linkmode-to-resolve-build-error.patch
0070-kata-runtime-remove-ctty-to-resolve-build-failed.patch
0071-fix-containerd-shim-kata-v2-compile-bug.patch
0072-kata-shimv2-adapt-iSulad-and-open-build-flag.patch
0073-write-exit-code-in-integer-byte-order.patch
0074-add-timeout-for-grpcWaitProcessRequest.patch
0075-move-waitProcess-timeout-to-stop-process.patch