From 9d3b8f0bb4c5127f48a8ceac50ac50aeba89bd5d Mon Sep 17 00:00:00 2001 From: Soren Mathiasen Date: Sun, 21 Jan 2018 11:48:50 +0100 Subject: [PATCH] refactor code, create a steps package This creates a steps package for handling all steps related code --- .gitlab-ci.yml | 126 ++++++++++++++++++++++++--------------- builder/builder.go | 129 +++++----------------------------------- builder/steps/common.go | 27 +++++++++ builder/steps/git.go | 55 +++++++++++++++++ builder/steps/ssh.go | 66 ++++++++++++++++++++ web/web.go | 8 ++- 6 files changed, 248 insertions(+), 163 deletions(-) create mode 100644 builder/steps/common.go create mode 100644 builder/steps/git.go create mode 100644 builder/steps/ssh.go diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index dc46512..e0a96e6 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -5,72 +5,100 @@ variables: CONTAINER_RELEASE_IMAGE: registry.gitlab.com/sorenmat/seneferu:latest POSTGRES_HOST: postgres POSTGRES_DB: seneferu + CI_BUILD_REF_NAME: $CI_COMMIT_REF_NAME + CI_BUILD_ID: $CI_JOB_ID + CI_BUILD_REPO: $CI_REPOSITORY_URL + CI_BUILD_REF: $CI_COMMIT_SHA before_script: - mkdir -p $GOPATH/src/$REPO_NAME - - ln -svf $CI_PROJECT_DIR/* $GOPATH/src/$REPO_NAME + - cp -r $CI_PROJECT_DIR/* $GOPATH/src/$REPO_NAME - cd $GOPATH/src/$REPO_NAME stages: - validate - build - coverage + - coverage-upload - release -format: - stage: validate - script: - - go fmt ./... -vet: - stage: validate - script: - - go vet ./... +#format: +# stage: validate +# script: +# - go fmt ./... +#vet: +# stage: validate +# script: +# - go vet ./... + +#test: +# stage: validate +# services: +# - postgres:latest +# script: +# - go test -race ./... + +#compile: +# stage: build +# script: +# - GOOS=linux CGO_ENABLED=0 go build -o $CI_PROJECT_DIR/seneferu +# artifacts: +# untracked: true + +#compile-js: +# stage: build +# image: node:9.2.0 +# script: +# - cd $CI_PROJECT_DIR +# - npm install +# - npm run build +# artifacts: +# paths: +# - js/ + -test: - stage: validate +coverage: + image: golang:1.10beta2-alpine + stage: coverage services: - postgres:latest - script: - - go test -race ./... - -compile: - stage: build - script: - - GOOS=linux CGO_ENABLED=0 go build -o $CI_PROJECT_DIR/seneferu - artifacts: - untracked: true -compile-js: - stage: build - image: node:9.2.0 - script: - - cd $CI_PROJECT_DIR - - npm install - - npm run build artifacts: paths: - - js/ + - coverage.txt -#coverage: -# image: golang:1.10-rc-stretch -# stage: coverage -# script: -# - go test -race -coverprofile=coverage.txt -covermode=atomic ./... -# - bash <(curl -s https://codecov.io/bash) -release-image: - stage: release - image: docker:git - only: - - master - services: - - docker:dind - dependencies: - - compile - - compile-js script: - - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.gitlab.com - - docker version - - cd $CI_PROJECT_DIR - - docker build -t registry.gitlab.com/sorenmat/seneferu:$CI_COMMIT_REF_NAME . - - docker push registry.gitlab.com/sorenmat/seneferu:$CI_COMMIT_REF_NAME + - ls + - go version + - go test -coverprofile=coverage.txt -covermode=atomic ./... +coverage-upload: + image: ubuntu + stage: coverage-upload + dependencies: + - coverage + script: + - ls -la + - apt update + - apt install -y curl + - echo ${CODECOV_TOKEN} + - bash <(curl -s https://codecov.io/bash) -t d943264a-ca45-410e-a190-75d02601ffae +# +# +#release-image: +# stage: release +# image: docker:git +# only: +# - master +# services: +# - docker:dind +# dependencies: +# - compile +# - compile-js +# script: +# - docker login -u gitlab-ci-token -p $CI_BUILD_TOKEN registry.gitlab.com +# - docker version +# - cd $CI_PROJECT_DIR +# - docker build -t registry.gitlab.com/sorenmat/seneferu:$CI_COMMIT_REF_NAME . +# - docker push registry.gitlab.com/sorenmat/seneferu:$CI_COMMIT_REF_NAME +# diff --git a/builder/builder.go b/builder/builder.go index 1a0f483..2d31d1a 100644 --- a/builder/builder.go +++ b/builder/builder.go @@ -16,6 +16,7 @@ import ( "github.com/pborman/uuid" "github.com/pkg/errors" "github.com/sorenmat/pipeline/pipeline/frontend/yaml" + "gitlab.com/sorenmat/seneferu/builder/steps" "gitlab.com/sorenmat/seneferu/github" "gitlab.com/sorenmat/seneferu/model" "gitlab.com/sorenmat/seneferu/storage" @@ -45,11 +46,11 @@ func CreateSSHKeySecret(kubectl *kubernetes.Clientset, sshkey string) error { log.Println("sshkey exsists, will try to update it") _, err := kubectl.CoreV1().Secrets("default").Update(&cm) return err - } else { - log.Println("sshkey mssing, will try to create it") - _, err := kubectl.CoreV1().Secrets("default").Create(&cm) - return err } + log.Println("sshkey mssing, will try to create it") + _, err = kubectl.CoreV1().Secrets("default").Create(&cm) + return err + } func volumemounts() []v1.Volume { @@ -82,6 +83,9 @@ func volumemounts() []v1.Volume { } } +// ExecuteBuild main entry point for executing the build. +// The will parse the config file, create and setup the steps and services +// handle the execution of the containers in Kubernetes func ExecuteBuild(kubectl *kubernetes.Clientset, service storage.Service, build *model.Build, repo *model.Repo, token string) error { pod := &v1.Pod{Spec: v1.PodSpec{ RestartPolicy: "Never", @@ -109,9 +113,9 @@ func ExecuteBuild(kubectl *kubernetes.Clientset, service storage.Service, build } var prepareSteps []v1.Container var buildSteps []v1.Container - prepareSteps = append(prepareSteps, createSSHAgentContainer()) - prepareSteps = append(prepareSteps, createSSHKeyAdd()) - prepareSteps = append(prepareSteps, createGitContainer(build)) + prepareSteps = append(prepareSteps, steps.CreateSSHAgentContainer(shareddir)) + prepareSteps = append(prepareSteps, steps.CreateSSHKeyAdd(shareddir)) + prepareSteps = append(prepareSteps, steps.CreateGitContainer(shareddir, build)) x, err := createBuildSteps(build, cfg) buildSteps = append(buildSteps, x...) if err != nil { @@ -321,7 +325,7 @@ func createBuildSteps(build *model.Build, cfg *yaml.Config) ([]v1.Container, err if count > 0 { cmds = append(cmds, waitForContainerCmd(fmt.Sprintf("build%v", count-1))) // wait for the previous build step } - provider := "github.com" + provider := "github.com" //TODO this should be configurable at some point workspace := shareddir + "/go/src/" + provider + "/" + projectName cmds = append(cmds, fmt.Sprintf("cd %v", workspace)) @@ -375,6 +379,9 @@ func createBuildSteps(build *model.Build, cfg *yaml.Config) ([]v1.Container, err return containers, nil } +// createServiceSteps will create service steps for the build +// a service is an external service to the build. This is mainly used for spinning up +// databases and that sort of thing that the build could be dependend on func createServiceSteps(cfg *yaml.Config) ([]v1.Container, error) { var containers []v1.Container @@ -395,109 +402,6 @@ func createServiceSteps(cfg *yaml.Config) ([]v1.Container, error) { return containers, nil } -func createGitContainer(build *model.Build) v1.Container { - - projectName := build.Org + "/" + build.Name - url := "git@github.com:" + projectName - provider := "github.com" - workspace := shareddir + "/go/src/" + provider + "/" + projectName - - sshTrustCmd := "ssh-keyscan -t rsa github.com > ~/.ssh/known_hosts" - cloneCmd := fmt.Sprintf("git clone %v %v", url, workspace) - curWDCmd := fmt.Sprintf("cd %v", workspace) - // TODO take the last element of the refs/head/init this is most likely not a good idea - checkoutCmd := fmt.Sprintf("git checkout %v", strings.Split(build.Ref, "/")[2]) - fmt.Println(checkoutCmd) - - doneCmd := "touch " + shareddir + "/git.done" - cmds := []string{waitForContainerCmd("ssh-key-add"), "mkdir ~/.ssh", sshTrustCmd, cloneCmd, curWDCmd, checkoutCmd, doneCmd} - return v1.Container{ - Name: "git", - Image: "sorenmat/git:1.0", - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: []v1.VolumeMount{ - { - Name: "shared-data", - MountPath: shareddir, - }, - { - Name: "ssh-agent", - MountPath: "/.ssh-agent", - }, - }, - - Command: []string{"/bin/sh", "-c", "echo $CI_SCRIPT | base64 -d |/bin/sh -e"}, - - Env: []v1.EnvVar{ - {Name: "SSH_AUTH_SOCK", Value: "/.ssh-agent/socket"}, - {Name: "CI_SCRIPT", Value: generateScript(cmds)}, - }, - Lifecycle: &v1.Lifecycle{ - PreStop: &v1.Handler{Exec: &v1.ExecAction{Command: []string{"/usr/bin/touch", shareddir + "/git.done"}}}, - }, - } -} - -func createSSHAgentContainer() v1.Container { - cmds := []string{"ssh-agent -a /.ssh-agent/socket -D"} - return v1.Container{ - Name: "ssh-agent", - Image: "nardeas/ssh-agent", - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: []v1.VolumeMount{ - { - Name: "shared-data", - MountPath: shareddir, - }, - { - Name: "sshvolume", - MountPath: "/root/.ssh", - }, - { - Name: "ssh-agent", - MountPath: "/.ssh-agent", - }, - }, - - Command: []string{"/bin/sh", "-c", "echo $CI_SCRIPT | base64 -d |/bin/sh -e"}, - - Env: []v1.EnvVar{ - {Name: "CI_SCRIPT", Value: generateScript(cmds)}, - }, - } -} - -func createSSHKeyAdd() v1.Container { - doneCmd := "touch " + shareddir + "/ssh-key-add.done" - - cmds := []string{"ssh-add /root/.ssh/id_rsa", doneCmd} - return v1.Container{ - Name: "ssh-key-add", - Image: "nardeas/ssh-agent", - ImagePullPolicy: v1.PullIfNotPresent, - VolumeMounts: []v1.VolumeMount{ - { - Name: "shared-data", - MountPath: shareddir, - }, - { - Name: "sshvolume", - MountPath: "/root/.ssh", - }, - { - Name: "ssh-agent", - MountPath: "/.ssh-agent", - }, - }, - - Command: []string{"/bin/sh", "-c", "echo $CI_SCRIPT | base64 -d |/bin/sh -e"}, - - Env: []v1.EnvVar{ - {Name: "CI_SCRIPT", Value: generateScript(cmds)}, - }, - } -} - func formatDuration(d time.Duration) string { // taken from github.com/hako/durafmt var ( @@ -565,9 +469,8 @@ func waitForContainerTermination(kubectl *kubernetes.Clientset, b v1.Container, if v.Name == b.Name { if v.State.Terminated != nil && v.State.Terminated.Reason != "" { return v.State.Terminated.ExitCode, nil - } else { - time.Sleep(2 * time.Second) } + time.Sleep(2 * time.Second) } } } diff --git a/builder/steps/common.go b/builder/steps/common.go new file mode 100644 index 0000000..6316a28 --- /dev/null +++ b/builder/steps/common.go @@ -0,0 +1,27 @@ +package steps + +import ( + "bytes" + "encoding/base64" + "fmt" +) + +func waitForContainerCmd(name, shareddir string) string { + command := `while ! test -f "` + shareddir + `/` + name + `.done"; do + sleep 1 + done + ` + return command +} + +// generateScript is a helper function that generates a build script and base64 encode it. +func generateScript(commands []string) string { + var buf bytes.Buffer + for _, command := range commands { + buf.WriteString(fmt.Sprintf(` +%s +`, command, + )) + } + return base64.StdEncoding.EncodeToString([]byte(buf.String())) +} diff --git a/builder/steps/git.go b/builder/steps/git.go new file mode 100644 index 0000000..c5fc063 --- /dev/null +++ b/builder/steps/git.go @@ -0,0 +1,55 @@ +package steps + +import ( + "fmt" + + "strings" + + "gitlab.com/sorenmat/seneferu/model" + "k8s.io/api/core/v1" +) + +// CreateGitContainer this create the git container used for cloning the sourcecode. +// this is a mandatory build step in Seneferu +func CreateGitContainer(shareddir string, build *model.Build) v1.Container { + + projectName := build.Org + "/" + build.Name + url := "git@github.com:" + projectName + provider := "github.com" + workspace := shareddir + "/go/src/" + provider + "/" + projectName + + sshTrustCmd := "ssh-keyscan -t rsa github.com > ~/.ssh/known_hosts" + cloneCmd := fmt.Sprintf("git clone %v %v", url, workspace) + curWDCmd := fmt.Sprintf("cd %v", workspace) + // TODO take the last element of the refs/head/init this is most likely not a good idea + checkoutCmd := fmt.Sprintf("git checkout %v", strings.Split(build.Ref, "/")[2]) + fmt.Println(checkoutCmd) + + doneCmd := "touch " + shareddir + "/git.done" + cmds := []string{waitForContainerCmd("ssh-key-add", shareddir), "mkdir ~/.ssh", sshTrustCmd, cloneCmd, curWDCmd, checkoutCmd, doneCmd} + return v1.Container{ + Name: "git", + Image: "sorenmat/git:1.0", + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: []v1.VolumeMount{ + { + Name: "shared-data", + MountPath: shareddir, + }, + { + Name: "ssh-agent", + MountPath: "/.ssh-agent", + }, + }, + + Command: []string{"/bin/sh", "-c", "echo $CI_SCRIPT | base64 -d |/bin/sh -e"}, + + Env: []v1.EnvVar{ + {Name: "SSH_AUTH_SOCK", Value: "/.ssh-agent/socket"}, + {Name: "CI_SCRIPT", Value: generateScript(cmds)}, + }, + Lifecycle: &v1.Lifecycle{ + PreStop: &v1.Handler{Exec: &v1.ExecAction{Command: []string{"/usr/bin/touch", shareddir + "/git.done"}}}, + }, + } +} diff --git a/builder/steps/ssh.go b/builder/steps/ssh.go new file mode 100644 index 0000000..6c344c5 --- /dev/null +++ b/builder/steps/ssh.go @@ -0,0 +1,66 @@ +package steps + +import "k8s.io/api/core/v1" + +// CreateSSHAgentContainer creates a container with a ssh agent running +// this will be used for sharing git credentials in the build steps +func CreateSSHAgentContainer(shareddir string) v1.Container { + cmds := []string{"ssh-agent -a /.ssh-agent/socket -D"} + return v1.Container{ + Name: "ssh-agent", + Image: "nardeas/ssh-agent", + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: []v1.VolumeMount{ + { + Name: "shared-data", + MountPath: shareddir, + }, + { + Name: "sshvolume", + MountPath: "/root/.ssh", + }, + { + Name: "ssh-agent", + MountPath: "/.ssh-agent", + }, + }, + + Command: []string{"/bin/sh", "-c", "echo $CI_SCRIPT | base64 -d |/bin/sh -e"}, + + Env: []v1.EnvVar{ + {Name: "CI_SCRIPT", Value: generateScript(cmds)}, + }, + } +} + +// CreateSSHKeyAdd will add the ssh key to the ssh agent +func CreateSSHKeyAdd(shareddir string) v1.Container { + doneCmd := "touch " + shareddir + "/ssh-key-add.done" + + cmds := []string{"ssh-add /root/.ssh/id_rsa", doneCmd} + return v1.Container{ + Name: "ssh-key-add", + Image: "nardeas/ssh-agent", + ImagePullPolicy: v1.PullIfNotPresent, + VolumeMounts: []v1.VolumeMount{ + { + Name: "shared-data", + MountPath: shareddir, + }, + { + Name: "sshvolume", + MountPath: "/root/.ssh", + }, + { + Name: "ssh-agent", + MountPath: "/.ssh-agent", + }, + }, + + Command: []string{"/bin/sh", "-c", "echo $CI_SCRIPT | base64 -d |/bin/sh -e"}, + + Env: []v1.EnvVar{ + {Name: "CI_SCRIPT", Value: generateScript(cmds)}, + }, + } +} diff --git a/web/web.go b/web/web.go index b062cba..500122b 100644 --- a/web/web.go +++ b/web/web.go @@ -44,9 +44,11 @@ func HandlePullRequest(payload interface{}, header webhooks.Header) { fmt.Printf("%+v", pl) } +// HandlePush handle push notifications from Github, this will then trigger a build in Seneferu +// assuming the conditions for a build is met func HandlePush(service storage.Service, kubectl *kubernetes.Clientset, token string) webhooks.ProcessPayloadFunc { return func(payload interface{}, header webhooks.Header) { - fmt.Println("Handling Push Request") + log.Println("Handling Push Request") pl := payload.(github.PushPayload) @@ -79,6 +81,7 @@ func HandlePush(service storage.Service, kubectl *kubernetes.Clientset, token st } } +// StartWebServer starts the web server for Seneferu handeling incoming request on the API func StartWebServer(db storage.Service, kubectl *kubernetes.Clientset, secret string, helmHost string, token string) { // Github hook @@ -124,12 +127,14 @@ func StartWebServer(db storage.Service, kubectl *kubernetes.Clientset, secret st var sockets = NewSockets() +// Sockets structure holding the websockets type Sockets struct { Add chan *websocket.Conn Remove chan *websocket.Conn sockets []*websocket.Conn } +// GetSockets return all the registered web sockets func (s *Sockets) GetSockets() []*websocket.Conn { return s.sockets } @@ -149,6 +154,7 @@ func (s *Sockets) handle() { } } +// NewSockets create a new Sockets structure containing all the web hooks func NewSockets() *Sockets { s := Sockets{} s.Add = make(chan *websocket.Conn) -- GitLab