added models and getting information from woodpecker
All checks were successful
ci/woodpecker/push/woodpecker Pipeline was successful
ci/woodpecker/pr/woodpecker Pipeline was successful

This commit is contained in:
Tobias Trabelsi 2023-10-30 22:46:17 +01:00
parent 9a5b9d1b55
commit e615144396
Signed by: lerentis
GPG Key ID: FF0C2839718CAF2E
5 changed files with 124 additions and 8 deletions

View File

@ -7,6 +7,7 @@ import (
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/woodpecker"
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
@ -33,6 +34,7 @@ func main() {
}).Info("Entering main event loop") }).Info("Entering main event loop")
for { for {
woodpecker.CheckPending(cfg)
time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute) time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute)
} }
} }

View File

@ -14,6 +14,7 @@ type Config = struct {
LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"` LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"`
WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"` WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"` WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"`
WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"`
Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"` Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"`
HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"` HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"`
InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"` InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"`

View File

@ -1,7 +1,9 @@
package hetzner package hetzner
import ( import (
"os" "bytes"
"errors"
"fmt"
"text/template" "text/template"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
@ -35,7 +37,7 @@ type UserDataConfig struct {
EnvConfig map[string]string EnvConfig map[string]string
} }
func generateConfig(cfg *config.Config) { func generateConfig(cfg *config.Config) (string, error) {
envConfig := map[string]string{} envConfig := map[string]string{}
envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance
envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret
@ -44,12 +46,15 @@ func generateConfig(cfg *config.Config) {
Image: "woodpeckerci/woodpecker-agent:latest", Image: "woodpeckerci/woodpecker-agent:latest",
EnvConfig: envConfig, EnvConfig: envConfig,
} }
tmpl, err := template.New("test").Parse(USER_DATA_TEMPLATE) tmpl, err := template.New("userdata").Parse(USER_DATA_TEMPLATE)
if err != nil { if err != nil {
panic(err) return "", errors.New(fmt.Sprintf("Errors in userdata template: %s", err.Error()))
} }
err = tmpl.Execute(os.Stdout, config) var buf bytes.Buffer
err = tmpl.Execute(&buf, &config)
if err != nil { if err != nil {
panic(err) return "", errors.New(fmt.Sprintf("Could not render userdata template: %s", err.Error()))
} }
return buf.String(), nil
} }

View File

@ -0,0 +1,56 @@
package models
/*
{
"pending": [
{
"id": "146",
"data": "REDACTED",
"labels": {
"repo": "REDACTED",
"type": "picus"
},
"dependencies": null,
"run_on": null,
"dep_status": {},
"agent_id": 0
}
],
"waiting_on_deps": null,
"running": null,
"stats": {
"worker_count": 40,
"pending_count": 1,
"waiting_on_deps_count": 0,
"running_count": 0,
"completed_count": 0
},
"paused": false
}
*/
type PendingInformation struct {
ID int `json:"id"`
Data string `json:"data"`
Labels map[string]string `json:"labels"`
Dependencies string `json:"dependencies"`
RunOn string `json:"run_on"`
DepStatus string `json:"-"` // dont need those
AgentId int `json:"agent_id"`
}
type Stats struct {
WorkerCount int `json:"worker_count"`
PendingCount int `json:"pending_count"`
WaitingOnDepsCount int `json:"waiting_on_deps_count"`
RunningCount int `json:"running_count"`
CompletedCount int `json:"completed_count"`
}
type QueueInfo struct {
Pending []PendingInformation `json:"pending"`
WaitingOnDeps string `json:"-"` // dont need those
Running int `json:"running"`
Stats Stats `json:"stats"`
Paused bool `json:"paused"`
}

View File

@ -1,7 +1,59 @@
package woodpecker package woodpecker
import "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config" import (
"encoding/json"
"errors"
"fmt"
"net/http"
func GetPendingJobs(cfg *config.Config) { "git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
log "github.com/sirupsen/logrus"
)
func QueueInfo(cfg *config.Config, target interface{}) error {
apiRoute := fmt.Sprintf("%s/api/queue/info", cfg.WoodpeckerInstance)
req, err := http.NewRequest("GET", apiRoute, nil)
if err != nil {
return errors.New(fmt.Sprintf("Could not create queue request: %s", err.Error()))
}
req.Header.Set("Accept", "application/json")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken))
resp, err := http.DefaultClient.Do(req)
if err != nil {
return errors.New(fmt.Sprintf("Could not query queue info: %s", err.Error()))
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return errors.New(fmt.Sprintf("Error from queue info api: %s", err.Error()))
}
return json.NewDecoder(resp.Body).Decode(target)
}
func CheckPending(cfg *config.Config) error {
queueInfo := new(models.QueueInfo)
err := QueueInfo(cfg, queueInfo)
if err != nil {
return errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
}
if queueInfo.Stats.PendingCount > 0 {
for _, pendingJobs := range queueInfo.Pending {
// TODO: separate key and value from LabelSelector and compare them deeply
_, exists := pendingJobs.Labels[cfg.LabelSelector]
if exists {
log.WithFields(log.Fields{
"Caller": "CheckPending",
}).Info("Found pending job for us. Requesting new Agent")
} else {
log.WithFields(log.Fields{
"Caller": "CheckPending",
}).Info("No Jobs for us in Queue")
}
}
}
return nil
} }