From e61514439634035ffcc63d28c7658870703bcdee Mon Sep 17 00:00:00 2001 From: Tobias Trabelsi Date: Mon, 30 Oct 2023 22:46:17 +0100 Subject: [PATCH] added models and getting information from woodpecker --- cmd/woodpecker-autoscaler.go | 2 ++ internal/config/config.go | 1 + internal/hetzner/hetzneragent.go | 17 ++++++---- internal/models/structs.go | 56 ++++++++++++++++++++++++++++++++ internal/woodpecker/metrics.go | 56 ++++++++++++++++++++++++++++++-- 5 files changed, 124 insertions(+), 8 deletions(-) create mode 100644 internal/models/structs.go diff --git a/cmd/woodpecker-autoscaler.go b/cmd/woodpecker-autoscaler.go index 22d5d2d..d528995 100644 --- a/cmd/woodpecker-autoscaler.go +++ b/cmd/woodpecker-autoscaler.go @@ -7,6 +7,7 @@ import ( "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/woodpecker" log "github.com/sirupsen/logrus" ) @@ -33,6 +34,7 @@ func main() { }).Info("Entering main event loop") for { + woodpecker.CheckPending(cfg) time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute) } } diff --git a/internal/config/config.go b/internal/config/config.go index 1eaf563..7d59d32 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -14,6 +14,7 @@ type Config = struct { LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"` WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"` WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"` + WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"` Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"` HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"` InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"` diff --git a/internal/hetzner/hetzneragent.go b/internal/hetzner/hetzneragent.go index 2357b2f..142a00f 100644 --- a/internal/hetzner/hetzneragent.go +++ b/internal/hetzner/hetzneragent.go @@ -1,7 +1,9 @@ package hetzner import ( - "os" + "bytes" + "errors" + "fmt" "text/template" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" @@ -35,7 +37,7 @@ type UserDataConfig struct { EnvConfig map[string]string } -func generateConfig(cfg *config.Config) { +func generateConfig(cfg *config.Config) (string, error) { envConfig := map[string]string{} envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret @@ -44,12 +46,15 @@ func generateConfig(cfg *config.Config) { Image: "woodpeckerci/woodpecker-agent:latest", EnvConfig: envConfig, } - tmpl, err := template.New("test").Parse(USER_DATA_TEMPLATE) + tmpl, err := template.New("userdata").Parse(USER_DATA_TEMPLATE) if err != nil { - panic(err) + return "", errors.New(fmt.Sprintf("Errors in userdata template: %s", err.Error())) } - err = tmpl.Execute(os.Stdout, config) + var buf bytes.Buffer + err = tmpl.Execute(&buf, &config) if err != nil { - panic(err) + return "", errors.New(fmt.Sprintf("Could not render userdata template: %s", err.Error())) } + + return buf.String(), nil } diff --git a/internal/models/structs.go b/internal/models/structs.go new file mode 100644 index 0000000..a7508a5 --- /dev/null +++ b/internal/models/structs.go @@ -0,0 +1,56 @@ +package models + +/* + { + "pending": [ + { + "id": "146", + "data": "REDACTED", + "labels": { + "repo": "REDACTED", + "type": "picus" + }, + "dependencies": null, + "run_on": null, + "dep_status": {}, + "agent_id": 0 + } + ], + "waiting_on_deps": null, + "running": null, + "stats": { + "worker_count": 40, + "pending_count": 1, + "waiting_on_deps_count": 0, + "running_count": 0, + "completed_count": 0 + }, + "paused": false + } +*/ + +type PendingInformation struct { + ID int `json:"id"` + Data string `json:"data"` + Labels map[string]string `json:"labels"` + Dependencies string `json:"dependencies"` + RunOn string `json:"run_on"` + DepStatus string `json:"-"` // dont need those + AgentId int `json:"agent_id"` +} + +type Stats struct { + WorkerCount int `json:"worker_count"` + PendingCount int `json:"pending_count"` + WaitingOnDepsCount int `json:"waiting_on_deps_count"` + RunningCount int `json:"running_count"` + CompletedCount int `json:"completed_count"` +} + +type QueueInfo struct { + Pending []PendingInformation `json:"pending"` + WaitingOnDeps string `json:"-"` // dont need those + Running int `json:"running"` + Stats Stats `json:"stats"` + Paused bool `json:"paused"` +} diff --git a/internal/woodpecker/metrics.go b/internal/woodpecker/metrics.go index ea3f8e4..18d107a 100644 --- a/internal/woodpecker/metrics.go +++ b/internal/woodpecker/metrics.go @@ -1,7 +1,59 @@ package woodpecker -import "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" +import ( + "encoding/json" + "errors" + "fmt" + "net/http" -func GetPendingJobs(cfg *config.Config) { + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" + "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models" + log "github.com/sirupsen/logrus" +) + +func QueueInfo(cfg *config.Config, target interface{}) error { + apiRoute := fmt.Sprintf("%s/api/queue/info", cfg.WoodpeckerInstance) + req, err := http.NewRequest("GET", apiRoute, nil) + if err != nil { + return errors.New(fmt.Sprintf("Could not create queue request: %s", err.Error())) + } + req.Header.Set("Accept", "application/json") + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken)) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return errors.New(fmt.Sprintf("Could not query queue info: %s", err.Error())) + } + defer resp.Body.Close() + + if resp.StatusCode != 200 { + return errors.New(fmt.Sprintf("Error from queue info api: %s", err.Error())) + } + + return json.NewDecoder(resp.Body).Decode(target) +} + +func CheckPending(cfg *config.Config) error { + queueInfo := new(models.QueueInfo) + err := QueueInfo(cfg, queueInfo) + if err != nil { + return errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error())) + } + if queueInfo.Stats.PendingCount > 0 { + for _, pendingJobs := range queueInfo.Pending { + // TODO: separate key and value from LabelSelector and compare them deeply + _, exists := pendingJobs.Labels[cfg.LabelSelector] + if exists { + log.WithFields(log.Fields{ + "Caller": "CheckPending", + }).Info("Found pending job for us. Requesting new Agent") + } else { + log.WithFields(log.Fields{ + "Caller": "CheckPending", + }).Info("No Jobs for us in Queue") + } + } + } + return nil }