Compare commits
No commits in common. "d4357767f5f0a20290c774f7796ffb7d461eba6f" and "a190c1e0fea653709954e831a07d4890933f7381" have entirely different histories.
d4357767f5
...
a190c1e0fe
19
CHANGELOG
19
CHANGELOG
@ -1,19 +0,0 @@
|
|||||||
CHANGELOG
|
|
||||||
|
|
||||||
v1.1.0
|
|
||||||
|
|
||||||
Updated Dependencies
|
|
||||||
Restructured Main event loop
|
|
||||||
Cost optimized mode to make use of the fully hour that is billed by hetzner
|
|
||||||
|
|
||||||
v1.0.1
|
|
||||||
|
|
||||||
Fix woodpecker agent decom
|
|
||||||
|
|
||||||
v1.0.0
|
|
||||||
|
|
||||||
First stable release
|
|
||||||
|
|
||||||
v0.0.1
|
|
||||||
|
|
||||||
First test release
|
|
@ -32,8 +32,6 @@ env:
|
|||||||
value: "define_it"
|
value: "define_it"
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
|
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
|
||||||
value: "define_it"
|
value: "define_it"
|
||||||
- name: WOODPECKER_AUTOSCALER_COST_OPTIMIZED
|
|
||||||
value: "true"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
you can also create a secret manually with these information and reference the existing secret like this in the `values.yaml`:
|
you can also create a secret manually with these information and reference the existing secret like this in the `values.yaml`:
|
||||||
@ -81,7 +79,6 @@ WOODPECKER_AUTOSCALER_HCLOUD_TOKEN="define_it"
|
|||||||
WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE=cpx21
|
WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE=cpx21
|
||||||
WOODPECKER_AUTOSCALER_HCLOUD_LOCATION="define_it"
|
WOODPECKER_AUTOSCALER_HCLOUD_LOCATION="define_it"
|
||||||
WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY="define_it"
|
WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY="define_it"
|
||||||
WOODPECKER_AUTOSCALER_COST_OPTIMIZED="true"
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Now reload the systemd daemons and start the service:
|
Now reload the systemd daemons and start the service:
|
||||||
|
@ -13,106 +13,6 @@ import (
|
|||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
func SpawnNewAgent(cfg *config.Config) {
|
|
||||||
agent, err := woodpecker.CreateWoodpeckerAgent(cfg)
|
|
||||||
if err != nil {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "SpawnNewAgent",
|
|
||||||
}).Fatal(fmt.Sprintf("Error creating new agent: %s", err.Error()))
|
|
||||||
}
|
|
||||||
server, err := hetzner.CreateNewAgent(cfg, agent)
|
|
||||||
if err != nil {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "SpawnNewAgent",
|
|
||||||
}).Fatal(fmt.Sprintf("Error spawning new agent: %s", err.Error()))
|
|
||||||
}
|
|
||||||
for {
|
|
||||||
server, err = hetzner.RefreshNodeInfo(cfg, server.ID)
|
|
||||||
if err != nil {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "SpawnNewAgent",
|
|
||||||
}).Fatal(fmt.Sprintf("Failed to start Agent: %s", err.Error()))
|
|
||||||
}
|
|
||||||
if server.Status == hcloud.ServerStatusRunning {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "SpawnNewAgent",
|
|
||||||
}).Infof("%s started!", server.Name)
|
|
||||||
break
|
|
||||||
}
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "SpawnNewAgent",
|
|
||||||
}).Infof("%s is in status %s", server.Name, server.Status)
|
|
||||||
time.Sleep(30 * time.Second)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func CheckJobs(cfg *config.Config, ownedNodes []hcloud.Server, pendingTasks int) {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "CheckJobs",
|
|
||||||
}).Info("Checking if agents can be removed")
|
|
||||||
runningTasks, err := woodpecker.CheckRunning(cfg)
|
|
||||||
if err != nil {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "CheckJobs",
|
|
||||||
}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
|
|
||||||
}
|
|
||||||
if (runningTasks <= len(ownedNodes) && runningTasks != 0) || pendingTasks > 0 {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "CheckJobs",
|
|
||||||
}).Info("Still found running tasks. No agent to be removed")
|
|
||||||
} else {
|
|
||||||
if len(ownedNodes) == 0 {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "CheckJobs",
|
|
||||||
}).Info("Nothing running and not owning any nodes")
|
|
||||||
} else {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "CheckJobs",
|
|
||||||
}).Info("No tasks running. Will remove agents")
|
|
||||||
Decom(cfg, ownedNodes)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func Decom(cfg *config.Config, ownedNodes []hcloud.Server) {
|
|
||||||
for _, server := range ownedNodes {
|
|
||||||
if cfg.CostOptimizedMode {
|
|
||||||
runtime, err := hetzner.CheckRuntime(cfg, &server)
|
|
||||||
if err != nil {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "Decom",
|
|
||||||
}).Warnf("Error while checking runtime of node %s: %s", server.Name, err.Error())
|
|
||||||
}
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "Decom",
|
|
||||||
}).Debugf("Node %s is running for %f", server.Name, runtime.Minutes())
|
|
||||||
// Check if next check if sooner than the 60 Minute mark of the next hetzner check
|
|
||||||
// https://docs.hetzner.com/cloud/billing/faq/#how-do-you-bill-your-servers
|
|
||||||
if (runtime + time.Duration(cfg.CheckInterval)*time.Minute) < 60 {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "Decom",
|
|
||||||
}).Infof("Skipping node termination of %s (running for %f Minutes) in Cost Optimized Mode", server.Name, runtime.Minutes())
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
}
|
|
||||||
agentId, err := hetzner.DecomNode(cfg, &server)
|
|
||||||
if err != nil {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "Decom",
|
|
||||||
}).Warnf("Error while deleting node %s: %s", server.Name, err.Error())
|
|
||||||
}
|
|
||||||
err = woodpecker.DecomAgent(cfg, agentId)
|
|
||||||
if err != nil {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "Decom",
|
|
||||||
}).Warnf("Could not delete node %s in woodpecker: %s", server.Name, err.Error())
|
|
||||||
}
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "Decom",
|
|
||||||
}).Infof("Deleted node %s", server.Name)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
|
|
||||||
cfg, err := config.GenConfig()
|
cfg, err := config.GenConfig()
|
||||||
@ -152,9 +52,75 @@ func main() {
|
|||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
}).Infof("Currently owning %d Agents", len(ownedNodes))
|
}).Infof("Currently owning %d Agents", len(ownedNodes))
|
||||||
if pendingTasks > len(ownedNodes) {
|
if pendingTasks > len(ownedNodes) {
|
||||||
SpawnNewAgent(cfg)
|
agent, err := woodpecker.CreateWoodpeckerAgent(cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Fatal(fmt.Sprintf("Error creating new agent: %s", err.Error()))
|
||||||
|
}
|
||||||
|
server, err := hetzner.CreateNewAgent(cfg, agent)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Fatal(fmt.Sprintf("Error spawning new agent: %s", err.Error()))
|
||||||
|
}
|
||||||
|
for {
|
||||||
|
server, err = hetzner.RefreshNodeInfo(cfg, server.ID)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Fatal(fmt.Sprintf("Failed to start Agent: %s", err.Error()))
|
||||||
|
}
|
||||||
|
if server.Status == hcloud.ServerStatusRunning {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Infof("%s started!", server.Name)
|
||||||
|
break
|
||||||
|
}
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Infof("%s is in status %s", server.Name, server.Status)
|
||||||
|
time.Sleep(30 * time.Second)
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
CheckJobs(cfg, ownedNodes, pendingTasks)
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Info("Checking if agents can be removed")
|
||||||
|
runningTasks, err := woodpecker.CheckRunning(cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
|
||||||
|
}
|
||||||
|
if (runningTasks <= len(ownedNodes) && runningTasks != 0) || pendingTasks > 0 {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Info("Still found running tasks. No agent to be removed")
|
||||||
|
} else {
|
||||||
|
if len(ownedNodes) == 0 {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Info("Nothing running and not owning any nodes")
|
||||||
|
} else {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Info("No tasks running. Will remove agents")
|
||||||
|
for _, server := range ownedNodes {
|
||||||
|
agentId, err := hetzner.DecomNode(cfg, &server)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Warnf("Error while deleting node %s: %s", server.Name, err.Error())
|
||||||
|
}
|
||||||
|
err = woodpecker.DecomAgent(cfg, agentId)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Warnf("Could not delete node %s in woodpecker: %s", server.Name, err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
}
|
}
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
|
@ -12,7 +12,6 @@ type Config = struct {
|
|||||||
LogLevel string `default:"Info" env:"WOODPECKER_AUTOSCALER_LOGLEVEL"`
|
LogLevel string `default:"Info" env:"WOODPECKER_AUTOSCALER_LOGLEVEL"`
|
||||||
CheckInterval int `default:"15" env:"WOODPECKER_AUTOSCALER_CHECK_INTERVAL"`
|
CheckInterval int `default:"15" env:"WOODPECKER_AUTOSCALER_CHECK_INTERVAL"`
|
||||||
DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
|
DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
|
||||||
CostOptimizedMode bool `default:"false" env:"WOODPECKER_AUTOSCALER_COST_OPTIMIZED"`
|
|
||||||
WoodpeckerLabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR"`
|
WoodpeckerLabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR"`
|
||||||
WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
|
WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
|
||||||
WoodpeckerGrpc string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_GRPC"`
|
WoodpeckerGrpc string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_GRPC"`
|
||||||
|
@ -8,7 +8,6 @@ import (
|
|||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"text/template"
|
"text/template"
|
||||||
"time"
|
|
||||||
|
|
||||||
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
|
||||||
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
|
||||||
@ -174,12 +173,3 @@ func RefreshNodeInfo(cfg *config.Config, serverID int) (*hcloud.Server, error) {
|
|||||||
}
|
}
|
||||||
return server, nil
|
return server, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckRuntime(cfg *config.Config, server *hcloud.Server) (time.Duration, error) {
|
|
||||||
server, err := RefreshNodeInfo(cfg, server.ID)
|
|
||||||
now := time.Now()
|
|
||||||
if err != nil {
|
|
||||||
return time.Duration(0), errors.New(fmt.Sprintf("Could not check Runtime: %s", err.Error()))
|
|
||||||
}
|
|
||||||
return server.Created.Sub(now), nil
|
|
||||||
}
|
|
||||||
|
Loading…
Reference in New Issue
Block a user