woodpecker-autoscaler/cmd/woodpecker-autoscaler.go

128 lines
3.8 KiB
Go
Raw Normal View History

2023-10-29 21:44:39 +00:00
package main
import (
"fmt"
"time"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/health"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/hetzner"
2023-10-29 21:44:39 +00:00
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/logging"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/woodpecker"
"github.com/hetznercloud/hcloud-go/hcloud"
2023-10-29 21:44:39 +00:00
log "github.com/sirupsen/logrus"
)
func main() {
cfg, err := config.GenConfig()
logging.ConfigureLogger(cfg)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error generating Config: %s", err.Error()))
}
go func() {
log.WithFields(log.Fields{
"Caller": "Main",
}).Info("Starting Health Endpoint")
health.StartHealthEndpoint()
}()
log.WithFields(log.Fields{
"Caller": "Main",
}).Info("Entering main event loop")
for {
pendingTasks, err := woodpecker.CheckPending(cfg)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
}
ownedNodes, err := hetzner.ListAgents(cfg)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error collecting owned hetzner nodes: %s", err.Error()))
}
log.WithFields(log.Fields{
"Caller": "Main",
}).Infof("Currently owning %d Agents", len(ownedNodes))
2023-11-06 20:28:14 +00:00
if pendingTasks > len(ownedNodes) {
agent, err := woodpecker.CreateWoodpeckerAgent(cfg)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error creating new agent: %s", err.Error()))
}
server, err := hetzner.CreateNewAgent(cfg, agent)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error spawning new agent: %s", err.Error()))
}
for {
2023-11-04 21:37:09 +00:00
server, err = hetzner.RefreshNodeInfo(cfg, server.ID)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Failed to start Agent: %s", err.Error()))
}
if server.Status == hcloud.ServerStatusRunning {
log.WithFields(log.Fields{
"Caller": "Main",
}).Infof("%s started!", server.Name)
break
}
log.WithFields(log.Fields{
"Caller": "Main",
2023-11-04 21:08:30 +00:00
}).Infof("%s is in status %s", server.Name, server.Status)
time.Sleep(30 * time.Second)
}
} else {
log.WithFields(log.Fields{
"Caller": "Main",
}).Info("Checking if agents can be removed")
runningTasks, err := woodpecker.CheckRunning(cfg)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
}
2023-11-08 20:32:22 +00:00
if (runningTasks <= len(ownedNodes) && runningTasks != 0) || pendingTasks > 0 {
log.WithFields(log.Fields{
"Caller": "Main",
}).Info("Still found running tasks. No agent to be removed")
} else {
2023-11-08 19:42:20 +00:00
if len(ownedNodes) == 0 {
log.WithFields(log.Fields{
"Caller": "Main",
}).Infof("Nothing running and not owning any nodes. Recheck in %d", cfg.CheckInterval)
} else {
log.WithFields(log.Fields{
"Caller": "Main",
}).Info("No tasks running. Will remove agents")
for _, server := range ownedNodes {
agentId, err := hetzner.DecomNode(cfg, &server)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Warnf("Error while deleting node %s: %s", server.Name, err.Error())
}
err = woodpecker.DecomAgent(cfg, agentId)
2023-11-08 19:42:20 +00:00
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Warnf("Could not delete node %s in woodpecker: %s", server.Name, err.Error())
2023-11-08 19:42:20 +00:00
}
}
}
}
}
2023-10-29 21:44:39 +00:00
time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute)
}
}