covidnetes/woodpecker-autoscaler
3
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 3 Wiki Activity

initial implementation #1

Merged
lerentis merged 7 commits from init into main 2023-11-04 11:17:41 +00:00
Conversation 0 Commits 7 Files Changed 23 +124 -8
5 changed files with 124 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit e615144396 - Show all commits

2
cmd/woodpecker-autoscaler.go
View File

@ -7,6 +7,7 @@ import (
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/woodpecker"
log "github.com/sirupsen/logrus"
)
@ -33,6 +34,7 @@ func main() {
}).Info("Entering main event loop")
for {
woodpecker.CheckPending(cfg)
time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute)
}
}

1
internal/config/config.go
View File

@ -14,6 +14,7 @@ type Config = struct {
LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"`
WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"`
WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"`
Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"`
HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"`
InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"`

17
internal/hetzner/hetzneragent.go
View File

@ -1,7 +1,9 @@
package hetzner
import (
"os"
"bytes"
"errors"
"fmt"
"text/template"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
@ -35,7 +37,7 @@ type UserDataConfig struct {
EnvConfig map[string]string
}
func generateConfig(cfg *config.Config) {
func generateConfig(cfg *config.Config) (string, error) {
envConfig := map[string]string{}
envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance
envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret
@ -44,12 +46,15 @@ func generateConfig(cfg *config.Config) {
Image: "woodpeckerci/woodpecker-agent:latest",
EnvConfig: envConfig,
}
tmpl, err := template.New("test").Parse(USER_DATA_TEMPLATE)
tmpl, err := template.New("userdata").Parse(USER_DATA_TEMPLATE)
if err != nil {
panic(err)
return "", errors.New(fmt.Sprintf("Errors in userdata template: %s", err.Error()))
}
err = tmpl.Execute(os.Stdout, config)
var buf bytes.Buffer
err = tmpl.Execute(&buf, &config)
if err != nil {
panic(err)
return "", errors.New(fmt.Sprintf("Could not render userdata template: %s", err.Error()))
}
return buf.String(), nil
}

56
internal/models/structs.go Normal file
View File

@ -0,0 +1,56 @@
package models
/*
{
"pending": [
{
"id": "146",
"data": "REDACTED",
"labels": {
"repo": "REDACTED",
"type": "picus"
},
"dependencies": null,
"run_on": null,
"dep_status": {},
"agent_id": 0
}
],
"waiting_on_deps": null,
"running": null,
"stats": {
"worker_count": 40,
"pending_count": 1,
"waiting_on_deps_count": 0,
"running_count": 0,
"completed_count": 0
},
"paused": false
}
*/
type PendingInformation struct {
ID int `json:"id"`
Data string `json:"data"`
Labels map[string]string `json:"labels"`
Dependencies string `json:"dependencies"`
RunOn string `json:"run_on"`
DepStatus string `json:"-"` // dont need those
AgentId int `json:"agent_id"`
}
type Stats struct {
WorkerCount int `json:"worker_count"`
PendingCount int `json:"pending_count"`
WaitingOnDepsCount int `json:"waiting_on_deps_count"`
RunningCount int `json:"running_count"`
CompletedCount int `json:"completed_count"`
}
type QueueInfo struct {
Pending []PendingInformation `json:"pending"`
WaitingOnDeps string `json:"-"` // dont need those
Running int `json:"running"`
Stats Stats `json:"stats"`
Paused bool `json:"paused"`
}

56
internal/woodpecker/metrics.go
View File

@ -1,7 +1,59 @@
package woodpecker
import "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
import (
"encoding/json"
"errors"
"fmt"
"net/http"
func GetPendingJobs(cfg *config.Config) {
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
log "github.com/sirupsen/logrus"
)
func QueueInfo(cfg *config.Config, target interface{}) error {
apiRoute := fmt.Sprintf("%s/api/queue/info", cfg.WoodpeckerInstance)
req, err := http.NewRequest("GET", apiRoute, nil)
if err != nil {
return errors.New(fmt.Sprintf("Could not create queue request: %s", err.Error()))
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken))
resp, err := http.DefaultClient.Do(req)
if err != nil {
return errors.New(fmt.Sprintf("Could not query queue info: %s", err.Error()))
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.New(fmt.Sprintf("Error from queue info api: %s", err.Error()))
}
return json.NewDecoder(resp.Body).Decode(target)
}
func CheckPending(cfg *config.Config) error {
queueInfo := new(models.QueueInfo)
err := QueueInfo(cfg, queueInfo)
if err != nil {
return errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
}
if queueInfo.Stats.PendingCount > 0 {
for _, pendingJobs := range queueInfo.Pending {
// TODO: separate key and value from LabelSelector and compare them deeply
_, exists := pendingJobs.Labels[cfg.LabelSelector]
if exists {
log.WithFields(log.Fields{
"Caller": "CheckPending",
}).Info("Found pending job for us. Requesting new Agent")
} else {
log.WithFields(log.Fields{
"Caller": "CheckPending",
}).Info("No Jobs for us in Queue")
}
}
}
return nil
}