From 4714889cd6ab043a36e595d9bc2c4b077bd17c8f Mon Sep 17 00:00:00 2001 From: zhongjiawei Date: Wed, 21 Sep 2022 16:59:42 +0800 Subject: [PATCH] runc: add logs Signed-off-by: zhongjiawei --- libcontainer/container_linux.go | 21 +++++++++ libcontainer/factory_linux.go | 13 ++++-- libcontainer/init_linux.go | 4 +- libcontainer/logs/logs.go | 68 +++++++++++++++++++++++++++++ libcontainer/nsenter/nsexec.c | 52 ++++++++++++++++++++-- libcontainer/process_linux.go | 11 +++-- libcontainer/setns_init_linux.go | 6 +++ libcontainer/standard_init_linux.go | 18 ++++++-- main.go | 6 ++- main_unix.go | 11 +++++ 10 files changed, 193 insertions(+), 17 deletions(-) create mode 100644 libcontainer/logs/logs.go diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index 7319286..7be84a6 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -23,6 +23,7 @@ import ( "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/configs" "github.com/opencontainers/runc/libcontainer/criurpc" + "github.com/opencontainers/runc/libcontainer/logs" "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/runc/libcontainer/utils" "github.com/syndtr/gocapability/capability" @@ -309,6 +310,17 @@ func (c *linuxContainer) start(process *Process) error { if err != nil { return newSystemErrorWithCause(err, "creating new parent process") } + + if logsDone := logs.ForwardLogs(); logsDone != nil { + defer func() { + select { + case <-logsDone: + case <-time.After(3 * time.Second): + logrus.Warnf("wait child close logfd timeout") + } + }() + } + if err := parent.start(); err != nil { printCgroupInfo(c.config.Cgroups.Path) // terminate the process to ensure that it properly is reaped. @@ -408,6 +420,9 @@ func (c *linuxContainer) newParentProcess(p *Process) (parentProcess, error) { if err != nil { return nil, newSystemErrorWithCause(err, "creating new init pipe") } + if err := logs.InitLogPipe(); err != nil { + return nil, fmt.Errorf("Unable to create the log pipe: %s", err) + } cmd, err := c.commandTemplate(p, childPipe) if err != nil { return nil, newSystemErrorWithCause(err, "creating new command template") @@ -450,6 +465,12 @@ func (c *linuxContainer) commandTemplate(p *Process, childPipe *os.File) (*exec. cmd.Env = append(cmd.Env, fmt.Sprintf("_LIBCONTAINER_INITPIPE=%d", stdioFdCount+len(cmd.ExtraFiles)-1), ) + + cmd.ExtraFiles = append(cmd.ExtraFiles, logs.ChildLogPipe) + cmd.Env = append(cmd.Env, + fmt.Sprintf("_LIBCONTAINER_LOGPIPE=%d", stdioFdCount+len(cmd.ExtraFiles)-1), + ) + // NOTE: when running a container with no PID namespace and the parent process spawning the container is // PID1 the pdeathsig is being delivered to the container's init process by the kernel for some reason // even with the parent still running. diff --git a/libcontainer/factory_linux.go b/libcontainer/factory_linux.go index fe9ce24..e4ef518 100644 --- a/libcontainer/factory_linux.go +++ b/libcontainer/factory_linux.go @@ -4,15 +4,15 @@ package libcontainer import ( "encoding/json" + "errors" "fmt" + "io/ioutil" "os" "path/filepath" "regexp" "runtime/debug" "strconv" "syscall" - "io/ioutil" - "errors" "github.com/docker/docker/pkg/mount" "github.com/opencontainers/runc/libcontainer/cgroups" @@ -281,6 +281,11 @@ func (l *LinuxFactory) StartInitialization() (err error) { defer consoleSocket.Close() } + logPipeFd, err2 := strconv.Atoi(os.Getenv("_LIBCONTAINER_LOGPIPE")) + if err2 != nil { + logPipeFd = 0 + } + // clear the current process's environment to clean any libcontainer // specific env vars. os.Clearenv() @@ -303,7 +308,7 @@ func (l *LinuxFactory) StartInitialization() (err error) { } }() - i, err := newContainerInit(it, pipe, consoleSocket, rootfd) + i, err := newContainerInit(it, pipe, consoleSocket, rootfd, logPipeFd) if err != nil { return err } @@ -346,7 +351,7 @@ func (l *LinuxFactory) updateStateCapabilites(compatState *CompatState, configPa var memSize int64 = int64(memorySwappiness) if memSize < 0 { memSize = 0 - var memUSize uint64 = uint64(memSize-1) + var memUSize uint64 = uint64(memSize - 1) compatState.Config.Cgroups.MemorySwappiness = &memUSize needUpdate = true } diff --git a/libcontainer/init_linux.go b/libcontainer/init_linux.go index ee632b4..e9a83e9 100644 --- a/libcontainer/init_linux.go +++ b/libcontainer/init_linux.go @@ -66,7 +66,7 @@ type initer interface { Init() error } -func newContainerInit(t initType, pipe *os.File, consoleSocket *os.File, stateDirFD int) (initer, error) { +func newContainerInit(t initType, pipe *os.File, consoleSocket *os.File, stateDirFD, logFd int) (initer, error) { var config *initConfig if err := json.NewDecoder(pipe).Decode(&config); err != nil { return nil, err @@ -81,6 +81,7 @@ func newContainerInit(t initType, pipe *os.File, consoleSocket *os.File, stateDi pipe: pipe, consoleSocket: consoleSocket, config: config, + logFd: logFd, }, nil case initStandard: return &linuxStandardInit{ @@ -89,6 +90,7 @@ func newContainerInit(t initType, pipe *os.File, consoleSocket *os.File, stateDi parentPid: syscall.Getppid(), config: config, stateDirFD: stateDirFD, + logFd: logFd, }, nil } return nil, fmt.Errorf("unknown init type %q", t) diff --git a/libcontainer/logs/logs.go b/libcontainer/logs/logs.go new file mode 100644 index 0000000..408a148 --- /dev/null +++ b/libcontainer/logs/logs.go @@ -0,0 +1,68 @@ +package logs + +import ( + "bufio" + "encoding/json" + "os" + + "github.com/Sirupsen/logrus" +) + +var ( + ParentLogPipe *os.File + ChildLogPipe *os.File +) + +func InitLogPipe() error { + var err error + if ParentLogPipe == nil { + ParentLogPipe, ChildLogPipe, err = os.Pipe() + } + return err +} + +func CloseChild() { + if ChildLogPipe != nil { + ChildLogPipe.Close() + ChildLogPipe = nil + } +} + +func ForwardLogs() chan error { + done := make(chan error, 1) + if ParentLogPipe == nil { + close(done) + return done + } + + s := bufio.NewScanner(ParentLogPipe) + go func() { + for s.Scan() { + processEntry(s.Bytes()) + } + if err := ParentLogPipe.Close(); err != nil { + logrus.Errorf("error closing log source: %v", err) + } + // The only error we want to return is when reading from + // logPipe has failed. + done <- s.Err() + close(done) + }() + + return done +} + +func processEntry(text []byte) { + if len(text) == 0 { + return + } + var jl struct { + Level string `json:"level"` + Msg string `json:"msg"` + } + if err := json.Unmarshal(text, &jl); err != nil { + logrus.Errorf("failed to decode %q to json: %v", text, err) + return + } + logrus.Infof("log from child: %s", jl.Msg) +} diff --git a/libcontainer/nsenter/nsexec.c b/libcontainer/nsenter/nsexec.c index 4f73b1a..0075b6e 100644 --- a/libcontainer/nsenter/nsexec.c +++ b/libcontainer/nsenter/nsexec.c @@ -77,6 +77,8 @@ struct nlconfig_t { size_t oom_score_adj_len; }; +int logfd; + /* * List of netlink message types sent to us as part of bootstrapping the init. * These constants are defined in libcontainer/message_linux.go. @@ -111,6 +113,26 @@ int setns(int fd, int nstype) } #endif +void write_log_with_info(const char *level, const char *function, int line, const char *format, ...) +{ + static char message[1024]; + va_list args; + + if (logfd < 0 || level == NULL) + return; + + va_start(args, format); + if (vsnprintf(message, 1024, format, args) < 0) + return; + va_end(args); + + if (dprintf(logfd, "{\"level\":\"%s\", \"msg\": \"%s:%d %s\"}\n", level, function, line, message) < 0) + return; +} + +#define logerr(fmt, ...) \ + write_log_with_info("error", __FUNCTION__, __LINE__, fmt, ##__VA_ARGS__) + /* XXX: This is ugly. */ static int syncfd = -1; @@ -118,13 +140,13 @@ static int syncfd = -1; #define bail(fmt, ...) \ do { \ int ret = __COUNTER__ + 1; \ - fprintf(stderr, "nsenter: " fmt ": %m\n", ##__VA_ARGS__); \ + logerr("nsenter: " fmt ": %m", ##__VA_ARGS__); \ if (syncfd >= 0) { \ enum sync_t s = SYNC_ERR; \ if (write(syncfd, &s, sizeof(s)) != sizeof(s)) \ - fprintf(stderr, "nsenter: failed: write(s)"); \ + logerr("nsenter: failed: write(s)"); \ if (write(syncfd, &ret, sizeof(ret)) != sizeof(ret)) \ - fprintf(stderr, "nsenter: failed: write(ret)"); \ + logerr("nsenter: failed: write(ret)"); \ } \ exit(ret); \ } while(0) @@ -259,6 +281,24 @@ static int initpipe(void) return pipenum; } +static void setup_logpipe(void) +{ + char *logpipe, *endptr; + + logpipe = getenv("_LIBCONTAINER_LOGPIPE"); + if (logpipe == NULL || *logpipe == '\0') { + logfd = -1; + return; + } + + logfd = strtol(logpipe, &endptr, 10); + if (logpipe == endptr || *endptr != '\0') { + fprintf(stderr, "unable to parse _LIBCONTAINER_LOGPIPE, value: %s\n", logpipe); + /* It is too early to use bail */ + exit(1); + } +} + /* Returns the clone(2) flag for a namespace, given the name of a namespace. */ static int nsflag(char *name) { @@ -442,6 +482,12 @@ void nsexec(void) int sync_child_pipe[2], sync_grandchild_pipe[2]; struct nlconfig_t config = {0}; + /* + * Setup a pipe to send logs to the parent. This should happen + * first, because bail will use that pipe. + */ + setup_logpipe(); + /* * If we don't have an init pipe, just return to the go routine. * We'll only get an init pipe for start or exec. diff --git a/libcontainer/process_linux.go b/libcontainer/process_linux.go index 25fe30b..5cdc30c 100644 --- a/libcontainer/process_linux.go +++ b/libcontainer/process_linux.go @@ -16,6 +16,7 @@ import ( "github.com/Sirupsen/logrus" "github.com/opencontainers/runc/libcontainer/cgroups" "github.com/opencontainers/runc/libcontainer/configs" + "github.com/opencontainers/runc/libcontainer/logs" "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/runc/libcontainer/utils" "golang.org/x/sys/unix" @@ -73,6 +74,7 @@ func (p *setnsProcess) start() (err error) { defer p.parentPipe.Close() err = p.cmd.Start() p.childPipe.Close() + logs.CloseChild() if err != nil { return newSystemErrorWithCause(err, "starting setns process") } @@ -135,7 +137,7 @@ func (p *setnsProcess) execSetns() error { } if !status.Success() { p.cmd.Wait() - return newSystemError(&exec.ExitError{ProcessState: status}) + return newSystemErrorWithCause(&exec.ExitError{ProcessState: status}, "getting setns process status") } var pid *pid if err := json.NewDecoder(p.parentPipe).Decode(&pid); err != nil { @@ -222,16 +224,16 @@ func (p *initProcess) execSetns() error { status, err := p.cmd.Process.Wait() if err != nil { p.cmd.Wait() - return err + return newSystemErrorWithCause(err, "waiting on setns process to finish") } if !status.Success() { p.cmd.Wait() - return &exec.ExitError{ProcessState: status} + return newSystemErrorWithCause(&exec.ExitError{ProcessState: status}, "getting setns process status") } var pid *pid if err := json.NewDecoder(p.parentPipe).Decode(&pid); err != nil { p.cmd.Wait() - return err + return newSystemErrorWithCause(err, "reading pid from init pipe") } // Clean up the zombie parent process @@ -256,6 +258,7 @@ func (p *initProcess) start() error { p.process.ops = p p.childPipe.Close() p.rootDir.Close() + logs.CloseChild() if err != nil { p.process.ops = nil return newSystemErrorWithCause(err, "starting init process command") diff --git a/libcontainer/setns_init_linux.go b/libcontainer/setns_init_linux.go index 48cc0ae..e6dfbba 100644 --- a/libcontainer/setns_init_linux.go +++ b/libcontainer/setns_init_linux.go @@ -5,6 +5,7 @@ package libcontainer import ( "fmt" "os" + "syscall" "github.com/opencontainers/runc/libcontainer/apparmor" "github.com/opencontainers/runc/libcontainer/keys" @@ -19,6 +20,7 @@ type linuxSetnsInit struct { pipe *os.File consoleSocket *os.File config *initConfig + logFd int } func (l *linuxSetnsInit) getSessionRingName() string { @@ -59,5 +61,9 @@ func (l *linuxSetnsInit) Init() error { if err := label.SetProcessLabel(l.config.ProcessLabel); err != nil { return err } + if l.logFd != 0 { + syscall.Close(l.logFd) + } + return system.Execv(l.config.Args[0], l.config.Args[0:], os.Environ()) } diff --git a/libcontainer/standard_init_linux.go b/libcontainer/standard_init_linux.go index b985180..b4945c3 100644 --- a/libcontainer/standard_init_linux.go +++ b/libcontainer/standard_init_linux.go @@ -27,6 +27,7 @@ type linuxStandardInit struct { parentPid int stateDirFD int config *initConfig + logFd int } func (l *linuxStandardInit) getSessionRingParams() (string, uint32, uint32) { @@ -181,6 +182,10 @@ func (l *linuxStandardInit) Init() error { // close the pipe to signal that we have completed our init. l.pipe.Close() + if l.logFd != 0 { + syscall.Close(l.logFd) + } + // wait for the fifo to be opened on the other side before // exec'ing the users process. ch := make(chan Error, 1) @@ -222,13 +227,18 @@ func (l *linuxStandardInit) Init() error { } func printCgroupInfo(path string) { + cgroupRoot := "/sys/fs/cgroup" infoFileList := []string{ "/proc/meminfo", "/sys/fs/cgroup/memory/memory.stat", - filepath.Join("/sys/fs/cgroup/files", path, "/files.limit"), - filepath.Join("/sys/fs/cgroup/files", path, "/files.usage"), - filepath.Join("/sys/fs/cgroup/memory", path, "/memory.stat"), - filepath.Join("/sys/fs/cgroup/cpu", path, "/cpu.stat"), + filepath.Join(cgroupRoot, "files", path, "files.limit"), + filepath.Join(cgroupRoot, "files", path, "files.usage"), + filepath.Join(cgroupRoot, "pids", path, "pids.max"), + filepath.Join(cgroupRoot, "pids", path, "pids.current"), + filepath.Join(cgroupRoot, "memory", path, "memory.usage_in_bytes"), + filepath.Join(cgroupRoot, "memory", path, "memory.limit_in_bytes"), + filepath.Join(cgroupRoot, "memory", path, "memory.stat"), + filepath.Join(cgroupRoot, "cpu", path, "cpu.stat"), } for _, file := range infoFileList { printFileContent(file) diff --git a/main.go b/main.go index 0476242..4141ec5 100644 --- a/main.go +++ b/main.go @@ -10,7 +10,7 @@ import ( "time" "github.com/Sirupsen/logrus" - "github.com/Sirupsen/logrus/hooks/syslog" + logrus_syslog "github.com/Sirupsen/logrus/hooks/syslog" "github.com/opencontainers/runtime-spec/specs-go" "github.com/urfave/cli" ) @@ -118,6 +118,10 @@ func main() { updateCommand, } app.Before = func(context *cli.Context) error { + if logrus.StandardLogger().Out != logrus.New().Out { + return nil + } + if path := context.GlobalString("log"); path != "" { f, err := os.OpenFile(path, os.O_CREATE|os.O_WRONLY|os.O_APPEND|os.O_SYNC, 0666) if err != nil { diff --git a/main_unix.go b/main_unix.go index 56904e0..0479949 100644 --- a/main_unix.go +++ b/main_unix.go @@ -6,7 +6,9 @@ import ( "fmt" "os" "runtime" + "strconv" + "github.com/Sirupsen/logrus" "github.com/opencontainers/runc/libcontainer" _ "github.com/opencontainers/runc/libcontainer/nsenter" "github.com/urfave/cli" @@ -16,6 +18,15 @@ func init() { if len(os.Args) > 1 && os.Args[1] == "init" { runtime.GOMAXPROCS(1) runtime.LockOSThread() + + if initType := os.Getenv("_LIBCONTAINER_INITTYPE"); initType == "setns" { + logPipeFd, err := strconv.Atoi(os.Getenv("_LIBCONTAINER_LOGPIPE")) + if err != nil { + return + } + logrus.SetOutput(os.NewFile(uintptr(logPipeFd), "logpipe")) + logrus.SetFormatter(new(logrus.JSONFormatter)) + } } } -- 2.30.0