From b047798b17e7d26523d0c03b78f6912f392fed2e Mon Sep 17 00:00:00 2001 From: Emily Chui Date: Thu, 9 May 2024 17:18:08 +1000 Subject: [PATCH] Gitaly pod init container binary The current containerized Gitaly pod does not have sufficient permissions to create cgroups in the typical cgroups path for Kubernetes pods. This makes it unable to manage cgroup (v2) controllers. Hence, an init container is required to run first before the Gitaly pod to change permissions on the host's cgroup path, so by the time Gitaly pod runs, it will have the right permissions to write to cgroups. --- tools/gitaly-init-cgroups/README.md | 2 + tools/gitaly-init-cgroups/main.go | 73 ++++++++++++++++++++++++++ tools/gitaly-init-cgroups/main_test.go | 68 ++++++++++++++++++++++++ 3 files changed, 143 insertions(+) create mode 100644 tools/gitaly-init-cgroups/README.md create mode 100644 tools/gitaly-init-cgroups/main.go create mode 100644 tools/gitaly-init-cgroups/main_test.go diff --git a/tools/gitaly-init-cgroups/README.md b/tools/gitaly-init-cgroups/README.md new file mode 100644 index 0000000000..db405762d6 --- /dev/null +++ b/tools/gitaly-init-cgroups/README.md @@ -0,0 +1,2 @@ +# gitaly-init-cgroups +gitaly-init-cgroups is a helper executable that will be used to configure cgroups for Gitaly to run in a Kubernetes environment. It will be containerized and run as an init container for the Gitaly pod. The main purpose of this helper executable is to modify permissions for cgroup paths so that the Gitaly Pod has access to write the cgroup controllers it manages. Finally it will output the pod cgroup path to a file which will be mounted to the Gitaly pod as a volume. diff --git a/tools/gitaly-init-cgroups/main.go b/tools/gitaly-init-cgroups/main.go new file mode 100644 index 0000000000..914719a650 --- /dev/null +++ b/tools/gitaly-init-cgroups/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func main() { + podID := getEnvOrExit("GITALY_POD_UID") + cgroupPath := getEnvOrExit("CGROUP_PATH") + outputPath := getEnvOrExit("OUTPUT_PATH") + + k8sPodID := strings.Replace(podID, "-", "_", -1) // eg. 2953d396_fd5e_4fc6_910f_ab687c9d08a8 + podCgroupPath := findPodCgroup(cgroupPath, k8sPodID) + if podCgroupPath == "" { + fmt.Printf("Error: failed to find cgroup for Gitaly pod %v\n", podID) + os.Exit(1) + } + /* Minimal permissions that a pod will need in order to manage its own subtree. + This is one way a cgroup can be delegated to a less privileged user, by granting write access + of the directory and its "cgroup.procs", "cgroup.threads", "cgroup.subtree_control" files. + Reference: https://docs.kernel.org/admin-guide/cgroup-v2.html#model-of-delegation + */ + modifyPaths := []string{"", "cgroup.procs", "cgroup.threads", "cgroup.subtree_control"} + if err := changePermissions(podCgroupPath, modifyPaths); err != nil { + fmt.Printf("Error: changing permissions for Gitaly pod %v cgroups: %v\n", podID, err) + os.Exit(1) + } + + if err := writePodCgroupPath(outputPath, podCgroupPath); err != nil { + fmt.Printf("Error: failed to write pod cgroup path: %v\n", err) + os.Exit(1) + } +} + +func getEnvOrExit(key string) string { + value := os.Getenv(key) + if value == "" { + fmt.Printf("Error: %s environment variable is not set\n", key) + os.Exit(1) + } + return value +} + +// findPodCgroup finds the path to the pod's cgroup given the pod id. +func findPodCgroup(path string, uid string) string { + directory := filepath.Join(path) + files, _ := filepath.Glob(directory + "/*/*-pod" + uid + ".slice") + + if len(files) > 0 { + return files[0] + } + return "" +} + +// changePermissions changes permissions of cgroup paths enabling write access for the Gitaly pod +func changePermissions(podCgroupPath string, paths []string) error { + for _, path := range paths { + chownCmd := fmt.Sprintf("chown 1000:1000 %s/%s", podCgroupPath, path) + if err := exec.Command("/bin/sh", "-c", chownCmd).Run(); err != nil { + return fmt.Errorf("failed to chown %q: %w", path, err) + } + } + return nil +} + +// writePodCgroupPath outputs the pod level cgroup path to a file +func writePodCgroupPath(outputPath string, podCgroupPath string) error { + return os.WriteFile(outputPath, []byte(podCgroupPath), 0o644) +} diff --git a/tools/gitaly-init-cgroups/main_test.go b/tools/gitaly-init-cgroups/main_test.go new file mode 100644 index 0000000000..d2c13caaab --- /dev/null +++ b/tools/gitaly-init-cgroups/main_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFindPodCgroup(t *testing.T) { + tests := []struct { + name string + uid string + expectedCgroupPath string + additionalCgroups []string + }{ + { + name: "no matching cgroup path", + uid: "test-uid", + expectedCgroupPath: "", + }, + { + name: "matching cgroup path exists", + uid: "123456", + expectedCgroupPath: "kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice", + }, + { + name: "multiple cgroup paths", + uid: "123456", + expectedCgroupPath: "kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice", + additionalCgroups: []string{"kubepods-besteffort.slice/kubepods-besteffort-pod789012.slice"}, + }, + { + name: "multiple matching cgroup paths", + uid: "123456", + expectedCgroupPath: "kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice", + additionalCgroups: []string{"kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice/cri-containerd-a69be83419aceb5000a9925552d67dc949e150c251b101357c242292579ed5d7.scope"}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tempDir, err := os.MkdirTemp("", "kubepods.slice") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(tempDir)) + }) + expectedPath := tt.expectedCgroupPath + if tt.expectedCgroupPath != "" { + expectedPath = filepath.Join(tempDir, tt.expectedCgroupPath) + require.NoError(t, os.MkdirAll(expectedPath, 0o755)) + } + + for _, additionalCgroup := range tt.additionalCgroups { + additionalPath := filepath.Join(tempDir, additionalCgroup) + require.NoError(t, os.MkdirAll(additionalPath, 0o755)) + } + + cgroupPath := findPodCgroup(tempDir, tt.uid) + require.Equal(t, expectedPath, cgroupPath) + }) + } +} -- GitLab