diff --git a/tools/gitaly-init-cgroups/README.md b/tools/gitaly-init-cgroups/README.md new file mode 100644 index 0000000000000000000000000000000000000000..db405762d603bc559af3bbbc1737efe4a16168e1 --- /dev/null +++ b/tools/gitaly-init-cgroups/README.md @@ -0,0 +1,2 @@ +# gitaly-init-cgroups +gitaly-init-cgroups is a helper executable that will be used to configure cgroups for Gitaly to run in a Kubernetes environment. It will be containerized and run as an init container for the Gitaly pod. The main purpose of this helper executable is to modify permissions for cgroup paths so that the Gitaly Pod has access to write the cgroup controllers it manages. Finally it will output the pod cgroup path to a file which will be mounted to the Gitaly pod as a volume. diff --git a/tools/gitaly-init-cgroups/main.go b/tools/gitaly-init-cgroups/main.go new file mode 100644 index 0000000000000000000000000000000000000000..914719a65063bc34a318dbf625157afae2353bc3 --- /dev/null +++ b/tools/gitaly-init-cgroups/main.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" +) + +func main() { + podID := getEnvOrExit("GITALY_POD_UID") + cgroupPath := getEnvOrExit("CGROUP_PATH") + outputPath := getEnvOrExit("OUTPUT_PATH") + + k8sPodID := strings.Replace(podID, "-", "_", -1) // eg. 2953d396_fd5e_4fc6_910f_ab687c9d08a8 + podCgroupPath := findPodCgroup(cgroupPath, k8sPodID) + if podCgroupPath == "" { + fmt.Printf("Error: failed to find cgroup for Gitaly pod %v\n", podID) + os.Exit(1) + } + /* Minimal permissions that a pod will need in order to manage its own subtree. + This is one way a cgroup can be delegated to a less privileged user, by granting write access + of the directory and its "cgroup.procs", "cgroup.threads", "cgroup.subtree_control" files. + Reference: https://docs.kernel.org/admin-guide/cgroup-v2.html#model-of-delegation + */ + modifyPaths := []string{"", "cgroup.procs", "cgroup.threads", "cgroup.subtree_control"} + if err := changePermissions(podCgroupPath, modifyPaths); err != nil { + fmt.Printf("Error: changing permissions for Gitaly pod %v cgroups: %v\n", podID, err) + os.Exit(1) + } + + if err := writePodCgroupPath(outputPath, podCgroupPath); err != nil { + fmt.Printf("Error: failed to write pod cgroup path: %v\n", err) + os.Exit(1) + } +} + +func getEnvOrExit(key string) string { + value := os.Getenv(key) + if value == "" { + fmt.Printf("Error: %s environment variable is not set\n", key) + os.Exit(1) + } + return value +} + +// findPodCgroup finds the path to the pod's cgroup given the pod id. +func findPodCgroup(path string, uid string) string { + directory := filepath.Join(path) + files, _ := filepath.Glob(directory + "/*/*-pod" + uid + ".slice") + + if len(files) > 0 { + return files[0] + } + return "" +} + +// changePermissions changes permissions of cgroup paths enabling write access for the Gitaly pod +func changePermissions(podCgroupPath string, paths []string) error { + for _, path := range paths { + chownCmd := fmt.Sprintf("chown 1000:1000 %s/%s", podCgroupPath, path) + if err := exec.Command("/bin/sh", "-c", chownCmd).Run(); err != nil { + return fmt.Errorf("failed to chown %q: %w", path, err) + } + } + return nil +} + +// writePodCgroupPath outputs the pod level cgroup path to a file +func writePodCgroupPath(outputPath string, podCgroupPath string) error { + return os.WriteFile(outputPath, []byte(podCgroupPath), 0o644) +} diff --git a/tools/gitaly-init-cgroups/main_test.go b/tools/gitaly-init-cgroups/main_test.go new file mode 100644 index 0000000000000000000000000000000000000000..d2c13caaabecc95f4cccfcb094775cf538242086 --- /dev/null +++ b/tools/gitaly-init-cgroups/main_test.go @@ -0,0 +1,68 @@ +package main + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestFindPodCgroup(t *testing.T) { + tests := []struct { + name string + uid string + expectedCgroupPath string + additionalCgroups []string + }{ + { + name: "no matching cgroup path", + uid: "test-uid", + expectedCgroupPath: "", + }, + { + name: "matching cgroup path exists", + uid: "123456", + expectedCgroupPath: "kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice", + }, + { + name: "multiple cgroup paths", + uid: "123456", + expectedCgroupPath: "kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice", + additionalCgroups: []string{"kubepods-besteffort.slice/kubepods-besteffort-pod789012.slice"}, + }, + { + name: "multiple matching cgroup paths", + uid: "123456", + expectedCgroupPath: "kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice", + additionalCgroups: []string{"kubepods-besteffort.slice/kubepods-besteffort-pod123456.slice/cri-containerd-a69be83419aceb5000a9925552d67dc949e150c251b101357c242292579ed5d7.scope"}, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tempDir, err := os.MkdirTemp("", "kubepods.slice") + require.NoError(t, err) + + t.Cleanup(func() { + require.NoError(t, os.RemoveAll(tempDir)) + }) + expectedPath := tt.expectedCgroupPath + if tt.expectedCgroupPath != "" { + expectedPath = filepath.Join(tempDir, tt.expectedCgroupPath) + require.NoError(t, os.MkdirAll(expectedPath, 0o755)) + } + + for _, additionalCgroup := range tt.additionalCgroups { + additionalPath := filepath.Join(tempDir, additionalCgroup) + require.NoError(t, os.MkdirAll(additionalPath, 0o755)) + } + + cgroupPath := findPodCgroup(tempDir, tt.uid) + require.Equal(t, expectedPath, cgroupPath) + }) + } +}