From 9a5b9d1b55dfbecd4dd33ba4502688c25d58919c Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Sun, 29 Oct 2023 22:44:39 +0100 Subject: [PATCH 1/7] akeleton config and launch template --- .woodpecker.yml | 37 +++++++++++++++++++++ Dockerfile | 18 +++++++++++ cmd/woodpecker-autoscaler.go | 38 ++++++++++++++++++++++ go.mod | 15 +++++++++ go.sum | 24 ++++++++++++++ internal/config/config.go | 38 ++++++++++++++++++++++ internal/health/healthcheck.go | 31 ++++++++++++++++++ internal/hetzner/hetzneragent.go | 55 ++++++++++++++++++++++++++++++++ internal/logging/logging.go | 28 ++++++++++++++++ internal/woodpecker/metrics.go | 7 ++++ 10 files changed, 291 insertions(+) create mode 100644 .woodpecker.yml create mode 100644 Dockerfile create mode 100644 cmd/woodpecker-autoscaler.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 internal/config/config.go create mode 100644 internal/health/healthcheck.go create mode 100644 internal/hetzner/hetzneragent.go create mode 100644 internal/logging/logging.go create mode 100644 internal/woodpecker/metrics.go diff --git a/.woodpecker.yml b/.woodpecker.yml new file mode 100644 index 0000000..8d37d4c --- /dev/null +++ b/.woodpecker.yml @@ -0,0 +1,37 @@ +steps: + test: + image: golang:1.21 + commands: + - go test ./... + when: + event: + - push + - pull_request + build: + image: woodpeckerci/plugin-docker-buildx + settings: + platforms: linux/arm64/v8 + repo: lerentis/metallb-ip-floater + tags: + - latest + - ${CI_COMMIT_SHA} + dry-run: true + when: + event: + - push + - pull_request + notify: + image: appleboy/drone-telegram + settings: + message: "Commit {{ commit.message }} ({{ commit.link }}) ran with build {{ build.number }} and finished with status {{ build.status }}." + to: + from_secret: telegram_userid + token: + from_secret: telegram_secret + when: + status: + - failure + - success + event: + - push + - pull_request diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..546d439 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,18 @@ +FROM golang:1.21 as build + +WORKDIR /app + +COPY . . + +RUN go mod tidy && CGO_ENABLED=0 GOOS=linux go build -a -tags netgo -ldflags '-w -extldflags "-static"' -o woodpecker-autoscaler ./cmd/ + +FROM scratch + +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ca-certificates.crt +COPY --from=build /etc/passwd /etc/passwd +COPY --from=build /etc/group /etc/group +COPY --from=build --chown=65534:65534 /app/woodpecker-autoscaler /usr/local/bin/woodpecker-autoscaler + +USER nobody + +ENTRYPOINT ["/usr/local/bin/woodpecker-autoscaler"] diff --git a/cmd/woodpecker-autoscaler.go b/cmd/woodpecker-autoscaler.go new file mode 100644 index 0000000..22d5d2d --- /dev/null +++ b/cmd/woodpecker-autoscaler.go @@ -0,0 +1,38 @@ +package main + +import ( + "fmt" + "time" + + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging" + log "github.com/sirupsen/logrus" +) + +func main() { + + cfg, err := config.GenConfig() + logging.ConfigureLogger(cfg) + + if err != nil { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Fatal(fmt.Sprintf("Error generating Config: %s", err.Error())) + } + + go func() { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Info("Starting Health Endpoint") + health.StartHealthEndpoint() + }() + + log.WithFields(log.Fields{ + "Caller": "Main", + }).Info("Entering main event loop") + + for { + time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..4aa429d --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler + +go 1.21.1 + +require ( + github.com/gorilla/mux v1.8.0 + github.com/jinzhu/configor v1.2.1 + github.com/sirupsen/logrus v1.9.3 +) + +require ( + github.com/BurntSushi/toml v0.3.1 // indirect + golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect + gopkg.in/yaml.v2 v2.2.2 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..05c5af2 --- /dev/null +++ b/go.sum @@ -0,0 +1,24 @@ +github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/jinzhu/configor v1.2.1 h1:OKk9dsR8i6HPOCZR8BcMtcEImAFjIhbJFZNyn5GCZko= +github.com/jinzhu/configor v1.2.1/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 0000000..1eaf563 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,38 @@ +package config + +import ( + "errors" + "fmt" + "time" + + "github.com/jinzhu/configor" +) + +type Config = struct { + LogLevel string `default:"Info" env:"WOODPECKER_AUTOSCALER_LOGLEVEL"` + CheckInterval int `default:"15" env:"WOODPECKER_AUTOSCALER_CHECK_INTERVAL"` + LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"` + WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"` + WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"` + Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"` + HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"` + InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"` + Zone string `default:"" env:"WOODPECKER_AUTOSCALER_ZONE"` + DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"` + SSHKey string `default:"" env:"WOODPECKER_AUTOSCALER_SSH_KEY"` +} + +func GenConfig() (cfg *Config, err error) { + + cfg = &Config{} + + err = configor.New(&configor.Config{ + ENVPrefix: "WOODPECKER_AUTOSCALER", + AutoReload: true, + Silent: true, + AutoReloadInterval: time.Minute}).Load(cfg, "config.json") + if err != nil { + return nil, errors.New(fmt.Sprintf("Error generating Config: %s", err.Error())) + } + return cfg, nil +} diff --git a/internal/health/healthcheck.go b/internal/health/healthcheck.go new file mode 100644 index 0000000..f945381 --- /dev/null +++ b/internal/health/healthcheck.go @@ -0,0 +1,31 @@ +package health + +import ( + "fmt" + "net/http" + + "github.com/gorilla/mux" + log "github.com/sirupsen/logrus" +) + +func StartHealthEndpoint() { + r := mux.NewRouter() + r.Use(mux.CORSMethodMiddleware(r)) + r.HandleFunc("/health", send200).Methods(http.MethodGet) + err := http.ListenAndServe("0.0.0.0:8080", r) + if err != nil { + log.WithFields(log.Fields{ + "Caller": "StartHealthEndpoint", + }).Error(fmt.Sprintf("Error creating health endpoint: %s", err.Error())) + } +} + +func send200(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err := w.Write([]byte{}) + if err != nil { + log.WithFields(log.Fields{ + "Caller": "send200", + }).Error(fmt.Sprintf("Error answering health endpoint: %s", err.Error())) + } +} diff --git a/internal/hetzner/hetzneragent.go b/internal/hetzner/hetzneragent.go new file mode 100644 index 0000000..2357b2f --- /dev/null +++ b/internal/hetzner/hetzneragent.go @@ -0,0 +1,55 @@ +package hetzner + +import ( + "os" + "text/template" + + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" +) + +var USER_DATA_TEMPLATE = ` +write_files: +- content: | + # docker-compose.yml + version: '3' + + services: + + woodpecker-agent: + image: {{ .Image }} + command: agent + restart: always + volumes: + - /var/run/docker.sock:/var/run/docker.sock + environment: + {{- range $key, $val := .EnvConfig }} + - {{ $key }}: {{ $val }} + {{- end }} + path: /root/docker-compose.yml +runcmd: +- [ sh, -xc, "cd /root; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes; docker compose up -d" ] +` + +type UserDataConfig struct { + Image string + EnvConfig map[string]string +} + +func generateConfig(cfg *config.Config) { + envConfig := map[string]string{} + envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance + envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret + envConfig["WOODPECKER_FILTER_LABELS"] = cfg.LabelSelector + config := UserDataConfig{ + Image: "woodpeckerci/woodpecker-agent:latest", + EnvConfig: envConfig, + } + tmpl, err := template.New("test").Parse(USER_DATA_TEMPLATE) + if err != nil { + panic(err) + } + err = tmpl.Execute(os.Stdout, config) + if err != nil { + panic(err) + } +} diff --git a/internal/logging/logging.go b/internal/logging/logging.go new file mode 100644 index 0000000..871543a --- /dev/null +++ b/internal/logging/logging.go @@ -0,0 +1,28 @@ +package logging + +import ( + "os" + + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" + log "github.com/sirupsen/logrus" +) + +func ConfigureLogger(cfg *config.Config) { + + switch cfg.LogLevel { + case "Debug": + log.SetLevel(log.DebugLevel) + case "Info": + log.SetLevel(log.InfoLevel) + case "Warn": + log.SetLevel(log.WarnLevel) + case "Error": + log.SetLevel(log.ErrorLevel) + default: + log.SetLevel(log.InfoLevel) + log.Warnf("Home: invalid log level supplied: '%s'", cfg.LogLevel) + } + + log.SetFormatter(&log.JSONFormatter{}) + log.SetOutput(os.Stdout) +} diff --git a/internal/woodpecker/metrics.go b/internal/woodpecker/metrics.go new file mode 100644 index 0000000..ea3f8e4 --- /dev/null +++ b/internal/woodpecker/metrics.go @@ -0,0 +1,7 @@ +package woodpecker + +import "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" + +func GetPendingJobs(cfg *config.Config) { + +} From e61514439634035ffcc63d28c7658870703bcdee Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Mon, 30 Oct 2023 22:46:17 +0100 Subject: [PATCH 2/7] added models and getting information from woodpecker --- cmd/woodpecker-autoscaler.go | 2 ++ internal/config/config.go | 1 + internal/hetzner/hetzneragent.go | 17 ++++++---- internal/models/structs.go | 56 ++++++++++++++++++++++++++++++++ internal/woodpecker/metrics.go | 56 ++++++++++++++++++++++++++++++-- 5 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 internal/models/structs.go diff --git a/cmd/woodpecker-autoscaler.go b/cmd/woodpecker-autoscaler.go index 22d5d2d..d528995 100644 --- a/cmd/woodpecker-autoscaler.go +++ b/cmd/woodpecker-autoscaler.go @@ -7,6 +7,7 @@ import ( "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/woodpecker" log "github.com/sirupsen/logrus" ) @@ -33,6 +34,7 @@ func main() { }).Info("Entering main event loop") for { + woodpecker.CheckPending(cfg) time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute) } } diff --git a/internal/config/config.go b/internal/config/config.go index 1eaf563..7d59d32 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,6 +14,7 @@ type Config = struct { LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"` WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"` WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"` + WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"` Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"` HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"` InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"` diff --git a/internal/hetzner/hetzneragent.go b/internal/hetzner/hetzneragent.go index 2357b2f..142a00f 100644 --- a/internal/hetzner/hetzneragent.go +++ b/internal/hetzner/hetzneragent.go @@ -1,7 +1,9 @@ package hetzner import ( - "os" + "bytes" + "errors" + "fmt" "text/template" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" @@ -35,7 +37,7 @@ type UserDataConfig struct { EnvConfig map[string]string } -func generateConfig(cfg *config.Config) { +func generateConfig(cfg *config.Config) (string, error) { envConfig := map[string]string{} envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret @@ -44,12 +46,15 @@ func generateConfig(cfg *config.Config) { Image: "woodpeckerci/woodpecker-agent:latest", EnvConfig: envConfig, } - tmpl, err := template.New("test").Parse(USER_DATA_TEMPLATE) + tmpl, err := template.New("userdata").Parse(USER_DATA_TEMPLATE) if err != nil { - panic(err) + return "", errors.New(fmt.Sprintf("Errors in userdata template: %s", err.Error())) } - err = tmpl.Execute(os.Stdout, config) + var buf bytes.Buffer + err = tmpl.Execute(&buf, &config) if err != nil { - panic(err) + return "", errors.New(fmt.Sprintf("Could not render userdata template: %s", err.Error())) } + + return buf.String(), nil } diff --git a/internal/models/structs.go b/internal/models/structs.go new file mode 100644 index 0000000..a7508a5 --- /dev/null +++ b/internal/models/structs.go @@ -0,0 +1,56 @@ +package models + +/* + { + "pending": [ + { + "id": "146", + "data": "REDACTED", + "labels": { + "repo": "REDACTED", + "type": "picus" + }, + "dependencies": null, + "run_on": null, + "dep_status": {}, + "agent_id": 0 + } + ], + "waiting_on_deps": null, + "running": null, + "stats": { + "worker_count": 40, + "pending_count": 1, + "waiting_on_deps_count": 0, + "running_count": 0, + "completed_count": 0 + }, + "paused": false + } +*/ + +type PendingInformation struct { + ID int `json:"id"` + Data string `json:"data"` + Labels map[string]string `json:"labels"` + Dependencies string `json:"dependencies"` + RunOn string `json:"run_on"` + DepStatus string `json:"-"` // dont need those + AgentId int `json:"agent_id"` +} + +type Stats struct { + WorkerCount int `json:"worker_count"` + PendingCount int `json:"pending_count"` + WaitingOnDepsCount int `json:"waiting_on_deps_count"` + RunningCount int `json:"running_count"` + CompletedCount int `json:"completed_count"` +} + +type QueueInfo struct { + Pending []PendingInformation `json:"pending"` + WaitingOnDeps string `json:"-"` // dont need those + Running int `json:"running"` + Stats Stats `json:"stats"` + Paused bool `json:"paused"` +} diff --git a/internal/woodpecker/metrics.go b/internal/woodpecker/metrics.go index ea3f8e4..18d107a 100644 --- a/internal/woodpecker/metrics.go +++ b/internal/woodpecker/metrics.go @@ -1,7 +1,59 @@ package woodpecker -import "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" +import ( + "encoding/json" + "errors" + "fmt" + "net/http" -func GetPendingJobs(cfg *config.Config) { + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models" + log "github.com/sirupsen/logrus" +) + +func QueueInfo(cfg *config.Config, target interface{}) error { + apiRoute := fmt.Sprintf("%s/api/queue/info", cfg.WoodpeckerInstance) + req, err := http.NewRequest("GET", apiRoute, nil) + if err != nil { + return errors.New(fmt.Sprintf("Could not create queue request: %s", err.Error())) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return errors.New(fmt.Sprintf("Could not query queue info: %s", err.Error())) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return errors.New(fmt.Sprintf("Error from queue info api: %s", err.Error())) + } + + return json.NewDecoder(resp.Body).Decode(target) +} + +func CheckPending(cfg *config.Config) error { + queueInfo := new(models.QueueInfo) + err := QueueInfo(cfg, queueInfo) + if err != nil { + return errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error())) + } + if queueInfo.Stats.PendingCount > 0 { + for _, pendingJobs := range queueInfo.Pending { + // TODO: separate key and value from LabelSelector and compare them deeply + _, exists := pendingJobs.Labels[cfg.LabelSelector] + if exists { + log.WithFields(log.Fields{ + "Caller": "CheckPending", + }).Info("Found pending job for us. Requesting new Agent") + } else { + log.WithFields(log.Fields{ + "Caller": "CheckPending", + }).Info("No Jobs for us in Queue") + } + } + } + return nil } From b0577f3c5532e45b16b84df4aea5b1fac8a46ed3 Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Mon, 30 Oct 2023 22:49:48 +0100 Subject: [PATCH 3/7] omitempty on possible null flields --- internal/models/structs.go | 6 +++--- internal/woodpecker/metrics.go | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/models/structs.go b/internal/models/structs.go index a7508a5..9512d05 100644 --- a/internal/models/structs.go +++ b/internal/models/structs.go @@ -33,7 +33,7 @@ type PendingInformation struct { ID int `json:"id"` Data string `json:"data"` Labels map[string]string `json:"labels"` - Dependencies string `json:"dependencies"` + Dependencies string `json:"dependencies,omitempty"` RunOn string `json:"run_on"` DepStatus string `json:"-"` // dont need those AgentId int `json:"agent_id"` @@ -48,9 +48,9 @@ type Stats struct { } type QueueInfo struct { - Pending []PendingInformation `json:"pending"` + Pending []PendingInformation `json:"pending,omitempty"` WaitingOnDeps string `json:"-"` // dont need those - Running int `json:"running"` + Running int `json:"running,omitempty"` Stats Stats `json:"stats"` Paused bool `json:"paused"` } diff --git a/internal/woodpecker/metrics.go b/internal/woodpecker/metrics.go index 18d107a..a48517d 100644 --- a/internal/woodpecker/metrics.go +++ b/internal/woodpecker/metrics.go @@ -41,6 +41,7 @@ func CheckPending(cfg *config.Config) error { return errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error())) } if queueInfo.Stats.PendingCount > 0 { + // TODO: queueInfo.Pending may be empty for _, pendingJobs := range queueInfo.Pending { // TODO: separate key and value from LabelSelector and compare them deeply _, exists := pendingJobs.Labels[cfg.LabelSelector] From bb1c7fb1d337c0fdf45da6be6adf98fdca51eb3d Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Mon, 30 Oct 2023 23:12:56 +0100 Subject: [PATCH 4/7] hopefully no more duplicate jobs --- .woodpecker.yml | 54 +++++++++++++++++++++++++++++++++++++------------ 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/.woodpecker.yml b/.woodpecker.yml index 8d37d4c..8910fea 100644 --- a/.woodpecker.yml +++ b/.woodpecker.yml @@ -4,10 +4,11 @@ steps: commands: - go test ./... when: - event: - - push - - pull_request - build: + - event: pull_request + - event: push + branch: main + - event: tag + pr-build: image: woodpeckerci/plugin-docker-buildx settings: platforms: linux/arm64/v8 @@ -17,9 +18,36 @@ steps: - ${CI_COMMIT_SHA} dry-run: true when: - event: - - push - - pull_request + - event: pull_request + pre-release: + image: woodpeckerci/plugin-docker-buildx + settings: + platforms: linux/arm64/v8 + repo: lerentis/metallb-ip-floater + tags: + - latest + - ${CI_COMMIT_SHA} + password: + from_secret: docker_hub_password + username: + from_secret: docker_hub_username + when: + - event: push + branch: main + release: + image: woodpeckerci/plugin-docker-buildx + settings: + platforms: linux/arm64/v8 + repo: lerentis/metallb-ip-floater + tags: + - latest + - ${CI_COMMIT_TAG} + password: + from_secret: docker_hub_password + username: + from_secret: docker_hub_username + when: + - event: tag notify: image: appleboy/drone-telegram settings: @@ -29,9 +57,9 @@ steps: token: from_secret: telegram_secret when: - status: - - failure - - success - event: - - push - - pull_request + - event: pull_request + - event: push + branch: main + - event: tag + - status: success + - status: failure From 7f2c465391f9e53526094389c47c6edbe3ccd00f Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Tue, 31 Oct 2023 23:18:08 +0100 Subject: [PATCH 5/7] implemented spawning of new agents split pipeline --- .woodpecker.yml | 65 ----------------------------- .woodpecker/main.yml | 29 +++++++++++++ .woodpecker/pr.yml | 25 ++++++++++++ .woodpecker/release.yml | 28 +++++++++++++ cmd/woodpecker-autoscaler.go | 49 +++++++++++++++++++++- go.mod | 18 +++++++- go.sum | 56 ++++++++++++++++++++++--- internal/config/config.go | 3 +- internal/hetzner/hetzneragent.go | 70 +++++++++++++++++++++++++++++++- internal/models/structs.go | 36 ++++++++++++++++ internal/utils/utils.go | 17 ++++++++ internal/woodpecker/agent.go | 60 +++++++++++++++++++++++++++ internal/woodpecker/metrics.go | 47 ++++++++++++++------- 13 files changed, 412 insertions(+), 91 deletions(-) delete mode 100644 .woodpecker.yml create mode 100644 .woodpecker/main.yml create mode 100644 .woodpecker/pr.yml create mode 100644 .woodpecker/release.yml create mode 100644 internal/utils/utils.go create mode 100644 internal/woodpecker/agent.go diff --git a/.woodpecker.yml b/.woodpecker.yml deleted file mode 100644 index 8910fea..0000000 --- a/.woodpecker.yml +++ /dev/null @@ -1,65 +0,0 @@ -steps: - test: - image: golang:1.21 - commands: - - go test ./... - when: - - event: pull_request - - event: push - branch: main - - event: tag - pr-build: - image: woodpeckerci/plugin-docker-buildx - settings: - platforms: linux/arm64/v8 - repo: lerentis/metallb-ip-floater - tags: - - latest - - ${CI_COMMIT_SHA} - dry-run: true - when: - - event: pull_request - pre-release: - image: woodpeckerci/plugin-docker-buildx - settings: - platforms: linux/arm64/v8 - repo: lerentis/metallb-ip-floater - tags: - - latest - - ${CI_COMMIT_SHA} - password: - from_secret: docker_hub_password - username: - from_secret: docker_hub_username - when: - - event: push - branch: main - release: - image: woodpeckerci/plugin-docker-buildx - settings: - platforms: linux/arm64/v8 - repo: lerentis/metallb-ip-floater - tags: - - latest - - ${CI_COMMIT_TAG} - password: - from_secret: docker_hub_password - username: - from_secret: docker_hub_username - when: - - event: tag - notify: - image: appleboy/drone-telegram - settings: - message: "Commit {{ commit.message }} ({{ commit.link }}) ran with build {{ build.number }} and finished with status {{ build.status }}." - to: - from_secret: telegram_userid - token: - from_secret: telegram_secret - when: - - event: pull_request - - event: push - branch: main - - event: tag - - status: success - - status: failure diff --git a/.woodpecker/main.yml b/.woodpecker/main.yml new file mode 100644 index 0000000..804c829 --- /dev/null +++ b/.woodpecker/main.yml @@ -0,0 +1,29 @@ +when: + - event: push + branch: main + +steps: + test: + image: golang:1.21 + commands: + - go test ./... + pre-release: + image: woodpeckerci/plugin-docker-buildx + settings: + platforms: linux/arm64/v8 + repo: lerentis/metallb-ip-floater + tags: + - latest + - ${CI_COMMIT_SHA} + password: + from_secret: docker_hub_password + username: + from_secret: docker_hub_username + notify: + image: appleboy/drone-telegram + settings: + message: "Commit {{ commit.message }} ({{ commit.link }}) ran with build {{ build.number }} and finished with status {{ build.status }}." + to: + from_secret: telegram_userid + token: + from_secret: telegram_secret diff --git a/.woodpecker/pr.yml b/.woodpecker/pr.yml new file mode 100644 index 0000000..7fbdc8a --- /dev/null +++ b/.woodpecker/pr.yml @@ -0,0 +1,25 @@ +when: + - event: pull_request + +steps: + test: + image: golang:1.21 + commands: + - go test ./... + pr-build: + image: woodpeckerci/plugin-docker-buildx + settings: + platforms: linux/arm64/v8 + repo: lerentis/metallb-ip-floater + tags: + - latest + - ${CI_COMMIT_SHA} + dry-run: true + notify: + image: appleboy/drone-telegram + settings: + message: "Commit {{ commit.message }} ({{ commit.link }}) ran with build {{ build.number }} and finished with status {{ build.status }}." + to: + from_secret: telegram_userid + token: + from_secret: telegram_secret diff --git a/.woodpecker/release.yml b/.woodpecker/release.yml new file mode 100644 index 0000000..c3e6998 --- /dev/null +++ b/.woodpecker/release.yml @@ -0,0 +1,28 @@ +when: + - event: tag + +steps: + test: + image: golang:1.21 + commands: + - go test ./... + release: + image: woodpeckerci/plugin-docker-buildx + settings: + platforms: linux/arm64/v8 + repo: lerentis/metallb-ip-floater + tags: + - latest + - ${CI_COMMIT_TAG} + password: + from_secret: docker_hub_password + username: + from_secret: docker_hub_username + notify: + image: appleboy/drone-telegram + settings: + message: "Commit {{ commit.message }} ({{ commit.link }}) ran with build {{ build.number }} and finished with status {{ build.status }}." + to: + from_secret: telegram_userid + token: + from_secret: telegram_secret diff --git a/cmd/woodpecker-autoscaler.go b/cmd/woodpecker-autoscaler.go index d528995..fc89d70 100644 --- a/cmd/woodpecker-autoscaler.go +++ b/cmd/woodpecker-autoscaler.go @@ -6,8 +6,10 @@ import ( "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/hetzner" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/woodpecker" + "github.com/hetznercloud/hcloud-go/hcloud" log "github.com/sirupsen/logrus" ) @@ -34,7 +36,52 @@ func main() { }).Info("Entering main event loop") for { - woodpecker.CheckPending(cfg) + pendingTasks, err := woodpecker.CheckPending(cfg) + if err != nil { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error())) + } + if pendingTasks { + server, err := hetzner.CreateNewAgent(cfg) + if err != nil { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Fatal(fmt.Sprintf("Error spawning new agent: %s", err.Error())) + } + for { + if server.Status == hcloud.ServerStatusRunning { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Infof("%s started!", server.Name) + break + } + log.WithFields(log.Fields{ + "Caller": "Main", + }).Infof("Waiting for agent %s to start", server.Name) + } + } else { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Info("Checking if agents can be removed") + runningTasks, err := woodpecker.CheckRunning(cfg) + if err != nil { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error())) + } + if runningTasks { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Info("Still found running tasks. No agent to be removed") + } else { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Info("No tasks running. Will remove agents") + // TODO: iterate over agents and remove ours + // agent name should match hetzner name + } + } time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute) } } diff --git a/go.mod b/go.mod index 4aa429d..01119a7 100644 --- a/go.mod +++ b/go.mod @@ -4,12 +4,26 @@ go 1.21.1 require ( github.com/gorilla/mux v1.8.0 + github.com/hetznercloud/hcloud-go v1.52.0 github.com/jinzhu/configor v1.2.1 github.com/sirupsen/logrus v1.9.3 ) require ( github.com/BurntSushi/toml v0.3.1 // indirect - golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect - gopkg.in/yaml.v2 v2.2.2 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/kr/text v0.2.0 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/prometheus/client_golang v1.16.0 // indirect + github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/common v0.42.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + golang.org/x/net v0.12.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/text v0.11.0 // indirect + google.golang.org/protobuf v1.30.0 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index 05c5af2..f9ca42b 100644 --- a/go.sum +++ b/go.sum @@ -1,24 +1,70 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/hetznercloud/hcloud-go v1.52.0 h1:3r9pEulTOBB9BoArSgpQYUQVTy+Xjkg0k/QAU4c6dQ8= +github.com/hetznercloud/hcloud-go v1.52.0/go.mod h1:VzDWThl47lOnZXY0q5/LPFD+M62pfe/52TV+mOrpp9Q= github.com/jinzhu/configor v1.2.1 h1:OKk9dsR8i6HPOCZR8BcMtcEImAFjIhbJFZNyn5GCZko= github.com/jinzhu/configor v1.2.1/go.mod h1:nX89/MOmDba7ZX7GCyU/VIaQ2Ar2aizBl2d3JLF/rDc= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/config/config.go b/internal/config/config.go index 7d59d32..7661b75 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -18,7 +18,8 @@ type Config = struct { Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"` HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"` InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"` - Zone string `default:"" env:"WOODPECKER_AUTOSCALER_ZONE"` + Region string `default:"" env:"WOODPECKER_AUTOSCALER_REGION"` + Datacenter string `default:"" env:"WOODPECKER_AUTOSCALER_DATACENTER"` DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"` SSHKey string `default:"" env:"WOODPECKER_AUTOSCALER_SSH_KEY"` } diff --git a/internal/hetzner/hetzneragent.go b/internal/hetzner/hetzneragent.go index 142a00f..ded5b3a 100644 --- a/internal/hetzner/hetzneragent.go +++ b/internal/hetzner/hetzneragent.go @@ -2,11 +2,16 @@ package hetzner import ( "bytes" + "context" "errors" "fmt" "text/template" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/utils" + "github.com/hetznercloud/hcloud-go/hcloud" + + log "github.com/sirupsen/logrus" ) var USER_DATA_TEMPLATE = ` @@ -37,11 +42,12 @@ type UserDataConfig struct { EnvConfig map[string]string } -func generateConfig(cfg *config.Config) (string, error) { +func generateConfig(cfg *config.Config, name string) (string, error) { envConfig := map[string]string{} envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret envConfig["WOODPECKER_FILTER_LABELS"] = cfg.LabelSelector + envConfig["WOODPECKER_HOSTNAME"] = name config := UserDataConfig{ Image: "woodpeckerci/woodpecker-agent:latest", EnvConfig: envConfig, @@ -55,6 +61,66 @@ func generateConfig(cfg *config.Config) (string, error) { if err != nil { return "", errors.New(fmt.Sprintf("Could not render userdata template: %s", err.Error())) } - return buf.String(), nil } + +func CreateNewAgent(cfg *config.Config) (*hcloud.Server, error) { + client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken)) + name := fmt.Sprintf("woodpecker-autoscaler-agent-%s", utils.RandStringBytes(5)) + userdata, err := generateConfig(cfg, name) + img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker", "amd64") + loc, _, err := client.Location.GetByName(context.Background(), cfg.Region) + pln, _, err := client.ServerType.GetByName(context.Background(), cfg.InstanceType) + key, _, err := client.SSHKey.GetByName(context.Background(), cfg.SSHKey) + dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.Datacenter) + labels := map[string]string{} + labels["Role"] = "WoodpeckerAgent" + labels["ControledBy"] = "WoodpeckerAutoscaler" + + if err != nil { + return nil, errors.New(fmt.Sprintf("Could not create new Agent: %s", err.Error())) + } + + res, _, err := client.Server.Create(context.Background(), hcloud.ServerCreateOpts{ + Name: name, + ServerType: pln, + Image: img, + SSHKeys: []*hcloud.SSHKey{key}, + Location: loc, + Datacenter: dc, + UserData: userdata, + StartAfterCreate: utils.BoolPointer(true), + Labels: labels, + }) + + log.WithFields(log.Fields{ + "Caller": "CreateNewAgent", + }).Infof("Created new Build Agent %s", res.Server.Name) + + return res.Server, nil +} + +func ListAgents(cfg *config.Config) ([]hcloud.Server, error) { + client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken)) + allServers, err := client.Server.All(context.Background()) + if err != nil { + return nil, errors.New(fmt.Sprintf("Could not query Server list: %s", err.Error())) + } + myServers := []hcloud.Server{} + for _, server := range allServers { + val, exists := server.Labels["ControledBy"] + if exists && val == "WoodpeckerAutoscaler" { + myServers = append(myServers, *server) + } + } + return myServers, nil +} + +func DecomAgent(cfg *config.Config, server *hcloud.Server) error { + client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken)) + _, _, err := client.Server.DeleteWithResult(context.Background(), server) + if err != nil { + return errors.New(fmt.Sprintf("Could not delete Agent: %s", err.Error())) + } + return nil +} diff --git a/internal/models/structs.go b/internal/models/structs.go index 9512d05..ab85bc3 100644 --- a/internal/models/structs.go +++ b/internal/models/structs.go @@ -54,3 +54,39 @@ type QueueInfo struct { Stats Stats `json:"stats"` Paused bool `json:"paused"` } + +/*[ + { + "id": 2, + "created": 1693567407, + "updated": 1694013270, + "name": "", + "owner_id": -1, + "token": "redacted", + "last_contact": 1694013270, + "platform": "linux/arm64", + "backend": "kubernetes", + "capacity": 4, + "version": "next-971534929c", + "no_schedule": false + } +]*/ + +type Agent struct { + ID int64 `json:"id"` + Created int64 `json:"created"` + Updated int64 `json:"updated"` + Name string `json:"name"` + OwnerID int64 `json:"owner_id"` + Token string `json:"token"` + LastContact int64 `json:"last_contact"` + Platform string `json:"platform"` + Backend string `json:"backend"` + Capacity int32 `json:"capacity"` + Version string `json:"version"` + NoSchedule bool `json:"no_schedule"` +} + +type AgentList struct { + Agents []Agent +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go new file mode 100644 index 0000000..b2103f4 --- /dev/null +++ b/internal/utils/utils.go @@ -0,0 +1,17 @@ +package utils + +import "math/rand" + +const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + +func RandStringBytes(n int) string { + b := make([]byte, n) + for i := range b { + b[i] = letterBytes[rand.Intn(len(letterBytes))] + } + return string(b) +} + +func BoolPointer(b bool) *bool { + return &b +} diff --git a/internal/woodpecker/agent.go b/internal/woodpecker/agent.go new file mode 100644 index 0000000..c9d61e2 --- /dev/null +++ b/internal/woodpecker/agent.go @@ -0,0 +1,60 @@ +package woodpecker + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models" +) + +func DecomAgent(cfg *config.Config, agentId int) error { + apiRoute := fmt.Sprintf("%s/api/agents/%d", cfg.WoodpeckerInstance, agentId) + req, err := http.NewRequest("DELETE", apiRoute, nil) + if err != nil { + return errors.New(fmt.Sprintf("Could not create delete request: %s", err.Error())) + } + req.Header.Set("Accept", "text/plain") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return errors.New(fmt.Sprintf("Could not delete agent: %s", err.Error())) + } + defer resp.Body.Close() + return nil +} + +func GetAgentIdByName(cfg *config.Config, name string) (int, error) { + apiRoute := fmt.Sprintf("%s/api/agents?page=1&perPage=100", cfg.WoodpeckerInstance) + req, err := http.NewRequest("GET", apiRoute, nil) + if err != nil { + return 0, errors.New(fmt.Sprintf("Could not create agent query request: %s", err.Error())) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return 0, errors.New(fmt.Sprintf("Could not query agent list: %s", err.Error())) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return 0, errors.New(fmt.Sprintf("Invalid status code from API: %d", resp.StatusCode)) + } + agentList := new(models.AgentList) + err = json.NewDecoder(resp.Body).Decode(agentList) + if err != nil { + return 0, errors.New(fmt.Sprintf("Could not unmarshal api response: %s", err.Error())) + } + + for _, agent := range agentList.Agents { + if agent.Name == name { + return int(agent.ID), nil + } + } + return 0, errors.New(fmt.Sprintf("Agent with name %s is not in server", name)) +} diff --git a/internal/woodpecker/metrics.go b/internal/woodpecker/metrics.go index a48517d..96f2759 100644 --- a/internal/woodpecker/metrics.go +++ b/internal/woodpecker/metrics.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "net/http" + "strings" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models" @@ -34,27 +35,43 @@ func QueueInfo(cfg *config.Config, target interface{}) error { return json.NewDecoder(resp.Body).Decode(target) } -func CheckPending(cfg *config.Config) error { +func CheckPending(cfg *config.Config) (bool, error) { + expectedKV := strings.Split(cfg.LabelSelector, "=") queueInfo := new(models.QueueInfo) err := QueueInfo(cfg, queueInfo) if err != nil { - return errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error())) + return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error())) } if queueInfo.Stats.PendingCount > 0 { - // TODO: queueInfo.Pending may be empty - for _, pendingJobs := range queueInfo.Pending { - // TODO: separate key and value from LabelSelector and compare them deeply - _, exists := pendingJobs.Labels[cfg.LabelSelector] - if exists { - log.WithFields(log.Fields{ - "Caller": "CheckPending", - }).Info("Found pending job for us. Requesting new Agent") - } else { - log.WithFields(log.Fields{ - "Caller": "CheckPending", - }).Info("No Jobs for us in Queue") + if queueInfo.Pending != nil { + for _, pendingJobs := range queueInfo.Pending { + val, exists := pendingJobs.Labels[expectedKV[0]] + if exists && val == expectedKV[1] { + log.WithFields(log.Fields{ + "Caller": "CheckPending", + }).Info("Found pending job for us") + return true, nil + } else { + log.WithFields(log.Fields{ + "Caller": "CheckPending", + }).Info("No Jobs for us in Queue") + return false, nil + } } } } - return nil + return false, nil +} + +func CheckRunning(cfg *config.Config) (bool, error) { + queueInfo := new(models.QueueInfo) + err := QueueInfo(cfg, queueInfo) + if err != nil { + return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error())) + } + // TODO: create and parse running object. there may be jobs that are not for us + if queueInfo.Stats.RunningCount > 0 { + return true, nil + } + return false, nil } From db1b560896f336a021cd2e62ba8247d77a0b3d18 Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Wed, 1 Nov 2023 20:48:45 +0100 Subject: [PATCH 6/7] decom agents as well added helm chart added docs --- .woodpecker/main.yml | 2 +- .woodpecker/pr.yml | 2 +- .woodpecker/release.yml | 2 +- README.md | 93 +++++++++++++++++++ chart/metallb-ip-floater/.helmignore | 23 +++++ chart/metallb-ip-floater/Chart.yaml | 24 +++++ chart/metallb-ip-floater/templates/NOTES.txt | 3 + .../metallb-ip-floater/templates/_helpers.tpl | 62 +++++++++++++ .../templates/deployment.yaml | 68 ++++++++++++++ .../templates/serviceaccount.yaml | 12 +++ chart/metallb-ip-floater/values.yaml | 83 +++++++++++++++++ cmd/woodpecker-autoscaler.go | 22 ++++- internal/config/config.go | 26 +++--- internal/hetzner/hetzneragent.go | 20 ++-- internal/woodpecker/agent.go | 9 ++ internal/woodpecker/metrics.go | 2 +- 16 files changed, 427 insertions(+), 26 deletions(-) create mode 100644 chart/metallb-ip-floater/.helmignore create mode 100644 chart/metallb-ip-floater/Chart.yaml create mode 100644 chart/metallb-ip-floater/templates/NOTES.txt create mode 100644 chart/metallb-ip-floater/templates/_helpers.tpl create mode 100644 chart/metallb-ip-floater/templates/deployment.yaml create mode 100644 chart/metallb-ip-floater/templates/serviceaccount.yaml create mode 100644 chart/metallb-ip-floater/values.yaml diff --git a/.woodpecker/main.yml b/.woodpecker/main.yml index 804c829..6e96d32 100644 --- a/.woodpecker/main.yml +++ b/.woodpecker/main.yml @@ -11,7 +11,7 @@ steps: image: woodpeckerci/plugin-docker-buildx settings: platforms: linux/arm64/v8 - repo: lerentis/metallb-ip-floater + repo: lerentis/woodpecker-autoscaler tags: - latest - ${CI_COMMIT_SHA} diff --git a/.woodpecker/pr.yml b/.woodpecker/pr.yml index 7fbdc8a..628cbf0 100644 --- a/.woodpecker/pr.yml +++ b/.woodpecker/pr.yml @@ -10,7 +10,7 @@ steps: image: woodpeckerci/plugin-docker-buildx settings: platforms: linux/arm64/v8 - repo: lerentis/metallb-ip-floater + repo: lerentis/woodpecker-autoscaler tags: - latest - ${CI_COMMIT_SHA} diff --git a/.woodpecker/release.yml b/.woodpecker/release.yml index c3e6998..1bcf531 100644 --- a/.woodpecker/release.yml +++ b/.woodpecker/release.yml @@ -10,7 +10,7 @@ steps: image: woodpeckerci/plugin-docker-buildx settings: platforms: linux/arm64/v8 - repo: lerentis/metallb-ip-floater + repo: lerentis/woodpecker-autoscaler tags: - latest - ${CI_COMMIT_TAG} diff --git a/README.md b/README.md index c7f4a6d..d99406c 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,95 @@ # woodpecker-autoscaler +[![status-badge](https://woodpecker.uploadfilter24.eu/api/badges/8/status.svg)](https://woodpecker.uploadfilter24.eu/repos/8) + +Dynamically spawns woodpecker ci build agents in hetzner cloud. + +## Installing with Helm + +The deployment will use helm and the chart in `chart/woodpecker-autoscaler`. +You will need a [hcloud api token](https://docs.hetzner.com/cloud/api/getting-started/generating-api-token/), a [woodpecker agent secret](https://woodpecker-ci.org/docs/administration/agent-config#woodpecker_agent_secret), a woodpecker api token and a definition of the build agents to be created. +Expose these information to the floater as described in this example: + +```yaml +env: + - name: WOODPECKER_AUTOSCALER_LOGLEVEL + value: "Info" + - name: WOODPECKER_AUTOSCALER_CHECK_INTERVAL + value: "15" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR + value: "uploadfilter24.eu/instance-role=Woodpecker" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE + value: "define_it" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET + value: "define_it" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_TOKEN + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE + value: "cpx21" + - name: WOODPECKER_AUTOSCALER_HCLOUD_REGION + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY + value: "define_it" +``` + +you can also create a secret manually with these information and reference the existing secret like this in the `values.yaml`: + +```yaml +externalConfigSecret: + enabled: true + name: "my-existing-secret" +``` + +Now you are able to deploy: + +```bash +kubectl create namespace woodpecker-autoscaler +cd chart/woodpecker-autoscaler +helm upgrade --install -f values.yaml -n woodpecker-autoscaler woodpecker-autoscaler ./ +``` + +## Installing Manually + +Download the binary from the release section and place it somewhere on your system; `/usr/bin/woodpecker-autoscaler` for example. +Create a systemd service like the one in this template: + +```systemd +[Unit] +Description=Dynamically spawn woodpecker ci build agents in hetzner cloud + +[Service] +Type=simple +Nice=10 +ExecStart=/usr/bin/woodpecker-autoscaler +EnvironmentFile="/etc/default/woodpecker-autoscaler.env" +``` + +Now place the environment variable configuration in the specified file: + +```bash +WOODPECKER_AUTOSCALER_LOGLEVEL=Info +WOODPECKER_AUTOSCALER_CHECK_INTERVAL=15 +WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR="uploadfilter24.eu/instance-role=Woodpecker" +WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE="define_it" +WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET="define_it" +WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN="define_it" +WOODPECKER_AUTOSCALER_HCLOUD_TOKEN="define_it" +WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE=cpx21 +WOODPECKER_AUTOSCALER_HCLOUD_REGION="define_it" +WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER="define_it" +WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY="define_it" +``` + +Now reload the systemd daemons and start the service: + +```bash +sudo systemctl daemon-reload +sudo systemctl enable woodpecker-autoscaler +sudo systemctl start woodpecker-autoscaler +``` + +> Made with ♡ by the folkes at uploadfilter24.eu :) \ No newline at end of file diff --git a/chart/metallb-ip-floater/.helmignore b/chart/metallb-ip-floater/.helmignore new file mode 100644 index 0000000..0e8a0eb --- /dev/null +++ b/chart/metallb-ip-floater/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/metallb-ip-floater/Chart.yaml b/chart/metallb-ip-floater/Chart.yaml new file mode 100644 index 0000000..c0a188c --- /dev/null +++ b/chart/metallb-ip-floater/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: woodpecker-autoscaler +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.0.1 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "0.0.1" diff --git a/chart/metallb-ip-floater/templates/NOTES.txt b/chart/metallb-ip-floater/templates/NOTES.txt new file mode 100644 index 0000000..dab6847 --- /dev/null +++ b/chart/metallb-ip-floater/templates/NOTES.txt @@ -0,0 +1,3 @@ +Service has been deployed +export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "woodpecker-autoscaler.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") +get logs: kubectl logs -f --namespace {{ .Release.Namespace }} $POD_NAME diff --git a/chart/metallb-ip-floater/templates/_helpers.tpl b/chart/metallb-ip-floater/templates/_helpers.tpl new file mode 100644 index 0000000..ca474ff --- /dev/null +++ b/chart/metallb-ip-floater/templates/_helpers.tpl @@ -0,0 +1,62 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "woodpecker-autoscaler.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "woodpecker-autoscaler.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "woodpecker-autoscaler.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "woodpecker-autoscaler.labels" -}} +helm.sh/chart: {{ include "woodpecker-autoscaler.chart" . }} +{{ include "woodpecker-autoscaler.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "woodpecker-autoscaler.selectorLabels" -}} +app.kubernetes.io/name: {{ include "woodpecker-autoscaler.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "woodpecker-autoscaler.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "woodpecker-autoscaler.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} diff --git a/chart/metallb-ip-floater/templates/deployment.yaml b/chart/metallb-ip-floater/templates/deployment.yaml new file mode 100644 index 0000000..0a793ea --- /dev/null +++ b/chart/metallb-ip-floater/templates/deployment.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "woodpecker-autoscaler.fullname" . }} + labels: + {{- include "woodpecker-autoscaler.labels" . | nindent 4 }} +spec: + replicas: {{ .Values.replicaCount }} + selector: + matchLabels: + {{- include "woodpecker-autoscaler.selectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "woodpecker-autoscaler.selectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "woodpecker-autoscaler.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + env: + {{- with .Values.env }} + {{- . | toYaml | trim | nindent 12 }} + {{- end }} + {{- if .Values.externalConfigSecret.enabled }} + envFrom: + - secretRef: + name: {{ .Values.externalConfigSecret.name }} + {{- end }} + ports: + - name: healthcheck + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /health + port: healthcheck + readinessProbe: + httpGet: + path: /health + port: healthcheck + resources: + {{- toYaml .Values.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/chart/metallb-ip-floater/templates/serviceaccount.yaml b/chart/metallb-ip-floater/templates/serviceaccount.yaml new file mode 100644 index 0000000..51bfe6b --- /dev/null +++ b/chart/metallb-ip-floater/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "woodpecker-autoscaler.serviceAccountName" . }} + labels: + {{- include "woodpecker-autoscaler.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/metallb-ip-floater/values.yaml b/chart/metallb-ip-floater/values.yaml new file mode 100644 index 0000000..dfdccc3 --- /dev/null +++ b/chart/metallb-ip-floater/values.yaml @@ -0,0 +1,83 @@ +# Default values for woodpecker-autoscaler. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +image: + repository: lerentis/woodpecker-autoscaler + pullPolicy: Always + # Overrides the image tag whose default is the chart appVersion. + tag: "latest" + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +env: + - name: WOODPECKER_AUTOSCALER_LOGLEVEL + value: "Info" + - name: WOODPECKER_AUTOSCALER_CHECK_INTERVAL + value: "15" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR + value: "uploadfilter24.eu/instance-role=Woodpecker" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE + value: "define_it" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET + value: "define_it" + - name: WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_TOKEN + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE + value: "cpx21" + - name: WOODPECKER_AUTOSCALER_HCLOUD_REGION + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER + value: "define_it" + - name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY + value: "define_it" + +externalConfigSecret: + enabled: false + name: "" + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/cmd/woodpecker-autoscaler.go b/cmd/woodpecker-autoscaler.go index fc89d70..0f6c3b6 100644 --- a/cmd/woodpecker-autoscaler.go +++ b/cmd/woodpecker-autoscaler.go @@ -42,6 +42,15 @@ func main() { "Caller": "Main", }).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error())) } + ownedNodes, err := hetzner.ListAgents(cfg) + if err != nil { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Fatal(fmt.Sprintf("Error collecting owned hetzner nodes: %s", err.Error())) + } + log.WithFields(log.Fields{ + "Caller": "Main", + }).Infof("Currently owning %d Agents", len(ownedNodes)) if pendingTasks { server, err := hetzner.CreateNewAgent(cfg) if err != nil { @@ -78,8 +87,17 @@ func main() { log.WithFields(log.Fields{ "Caller": "Main", }).Info("No tasks running. Will remove agents") - // TODO: iterate over agents and remove ours - // agent name should match hetzner name + for _, server := range ownedNodes { + hetzner.DecomNode(cfg, &server) + agentId, err := woodpecker.GetAgentIdByName(cfg, server.Name) + if err != nil { + log.WithFields(log.Fields{ + "Caller": "Main", + }).Warnf("Could not find agent %s in woodpecker. Assuming it was never added", server.Name) + } else { + woodpecker.DecomAgent(cfg, agentId) + } + } } } time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute) diff --git a/internal/config/config.go b/internal/config/config.go index 7661b75..2c43aff 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -9,19 +9,19 @@ import ( ) type Config = struct { - LogLevel string `default:"Info" env:"WOODPECKER_AUTOSCALER_LOGLEVEL"` - CheckInterval int `default:"15" env:"WOODPECKER_AUTOSCALER_CHECK_INTERVAL"` - LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"` - WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"` - WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"` - WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"` - Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"` - HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"` - InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"` - Region string `default:"" env:"WOODPECKER_AUTOSCALER_REGION"` - Datacenter string `default:"" env:"WOODPECKER_AUTOSCALER_DATACENTER"` - DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"` - SSHKey string `default:"" env:"WOODPECKER_AUTOSCALER_SSH_KEY"` + LogLevel string `default:"Info" env:"WOODPECKER_AUTOSCALER_LOGLEVEL"` + CheckInterval int `default:"15" env:"WOODPECKER_AUTOSCALER_CHECK_INTERVAL"` + DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"` + WoodpeckerLabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR"` + WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"` + WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"` + WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"` + WoodpeckerProtocol string `default:"http" env:"WOODPECKER_AUTOSCALER_WOODPECKER_PROTOCOL"` + HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"` + HcloudInstanceType string `default:"cpx21" env:"WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE"` + HcloudRegion string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_REGION"` + HcloudDatacenter string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER"` + HcloudSSHKey string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY"` } func GenConfig() (cfg *Config, err error) { diff --git a/internal/hetzner/hetzneragent.go b/internal/hetzner/hetzneragent.go index ded5b3a..34f947c 100644 --- a/internal/hetzner/hetzneragent.go +++ b/internal/hetzner/hetzneragent.go @@ -46,7 +46,7 @@ func generateConfig(cfg *config.Config, name string) (string, error) { envConfig := map[string]string{} envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret - envConfig["WOODPECKER_FILTER_LABELS"] = cfg.LabelSelector + envConfig["WOODPECKER_FILTER_LABELS"] = cfg.WoodpeckerLabelSelector envConfig["WOODPECKER_HOSTNAME"] = name config := UserDataConfig{ Image: "woodpeckerci/woodpecker-agent:latest", @@ -68,11 +68,11 @@ func CreateNewAgent(cfg *config.Config) (*hcloud.Server, error) { client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken)) name := fmt.Sprintf("woodpecker-autoscaler-agent-%s", utils.RandStringBytes(5)) userdata, err := generateConfig(cfg, name) - img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker", "amd64") - loc, _, err := client.Location.GetByName(context.Background(), cfg.Region) - pln, _, err := client.ServerType.GetByName(context.Background(), cfg.InstanceType) - key, _, err := client.SSHKey.GetByName(context.Background(), cfg.SSHKey) - dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.Datacenter) + img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker-ce", "amd64") + loc, _, err := client.Location.GetByName(context.Background(), cfg.HcloudRegion) + pln, _, err := client.ServerType.GetByName(context.Background(), cfg.HcloudInstanceType) + key, _, err := client.SSHKey.GetByName(context.Background(), cfg.HcloudSSHKey) + dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.HcloudDatacenter) labels := map[string]string{} labels["Role"] = "WoodpeckerAgent" labels["ControledBy"] = "WoodpeckerAutoscaler" @@ -111,13 +111,19 @@ func ListAgents(cfg *config.Config) ([]hcloud.Server, error) { val, exists := server.Labels["ControledBy"] if exists && val == "WoodpeckerAutoscaler" { myServers = append(myServers, *server) + log.WithFields(log.Fields{ + "Caller": "ListAgents", + }).Debugf("Owning %s Hetzner node", server.Name) } } return myServers, nil } -func DecomAgent(cfg *config.Config, server *hcloud.Server) error { +func DecomNode(cfg *config.Config, server *hcloud.Server) error { client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken)) + log.WithFields(log.Fields{ + "Caller": "DecomNode", + }).Debugf("Deleting %s node", server.Name) _, _, err := client.Server.DeleteWithResult(context.Background(), server) if err != nil { return errors.New(fmt.Sprintf("Could not delete Agent: %s", err.Error())) diff --git a/internal/woodpecker/agent.go b/internal/woodpecker/agent.go index c9d61e2..e1c6c4f 100644 --- a/internal/woodpecker/agent.go +++ b/internal/woodpecker/agent.go @@ -8,6 +8,8 @@ import ( "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models" + + log "github.com/sirupsen/logrus" ) func DecomAgent(cfg *config.Config, agentId int) error { @@ -19,6 +21,10 @@ func DecomAgent(cfg *config.Config, agentId int) error { req.Header.Set("Accept", "text/plain") req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken)) + log.WithFields(log.Fields{ + "Caller": "DecomAgent", + }).Debugf("Deleting %d agent from woodpecker", agentId) + resp, err := http.DefaultClient.Do(req) if err != nil { return errors.New(fmt.Sprintf("Could not delete agent: %s", err.Error())) @@ -53,6 +59,9 @@ func GetAgentIdByName(cfg *config.Config, name string) (int, error) { for _, agent := range agentList.Agents { if agent.Name == name { + log.WithFields(log.Fields{ + "Caller": "GetAgentIdByName", + }).Debugf("Found ID %d for Agent %s", agent.ID, name) return int(agent.ID), nil } } diff --git a/internal/woodpecker/metrics.go b/internal/woodpecker/metrics.go index 96f2759..49fe6eb 100644 --- a/internal/woodpecker/metrics.go +++ b/internal/woodpecker/metrics.go @@ -36,7 +36,7 @@ func QueueInfo(cfg *config.Config, target interface{}) error { } func CheckPending(cfg *config.Config) (bool, error) { - expectedKV := strings.Split(cfg.LabelSelector, "=") + expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=") queueInfo := new(models.QueueInfo) err := QueueInfo(cfg, queueInfo) if err != nil { From b313c1b5aa228a49d4c35f4db7602dd07767457f Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Wed, 1 Nov 2023 20:54:44 +0100 Subject: [PATCH 7/7] renamed PendingInformation to JobInformation filter running jobs better --- internal/models/structs.go | 12 ++++++------ internal/woodpecker/metrics.go | 17 +++++++++++++++-- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/internal/models/structs.go b/internal/models/structs.go index ab85bc3..1abe885 100644 --- a/internal/models/structs.go +++ b/internal/models/structs.go @@ -29,7 +29,7 @@ package models } */ -type PendingInformation struct { +type JobInformation struct { ID int `json:"id"` Data string `json:"data"` Labels map[string]string `json:"labels"` @@ -48,11 +48,11 @@ type Stats struct { } type QueueInfo struct { - Pending []PendingInformation `json:"pending,omitempty"` - WaitingOnDeps string `json:"-"` // dont need those - Running int `json:"running,omitempty"` - Stats Stats `json:"stats"` - Paused bool `json:"paused"` + Pending []JobInformation `json:"pending,omitempty"` + WaitingOnDeps string `json:"-"` // dont need those + Running []JobInformation `json:"running,omitempty"` + Stats Stats `json:"stats"` + Paused bool `json:"paused"` } /*[ diff --git a/internal/woodpecker/metrics.go b/internal/woodpecker/metrics.go index 49fe6eb..43dc92b 100644 --- a/internal/woodpecker/metrics.go +++ b/internal/woodpecker/metrics.go @@ -64,14 +64,27 @@ func CheckPending(cfg *config.Config) (bool, error) { } func CheckRunning(cfg *config.Config) (bool, error) { + expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=") queueInfo := new(models.QueueInfo) err := QueueInfo(cfg, queueInfo) if err != nil { return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error())) } - // TODO: create and parse running object. there may be jobs that are not for us if queueInfo.Stats.RunningCount > 0 { - return true, nil + for _, runningJobs := range queueInfo.Running { + val, exists := runningJobs.Labels[expectedKV[0]] + if exists && val == expectedKV[1] { + log.WithFields(log.Fields{ + "Caller": "CheckRunning", + }).Info("Found running job for us") + return true, nil + } else { + log.WithFields(log.Fields{ + "Caller": "CheckRunning", + }).Info("No running job for us") + return false, nil + } + } } return false, nil }