Compare commits
31 Commits
Author | SHA1 | Date | |
---|---|---|---|
174c40c72c
|
|||
48b4b273ee
|
|||
09c85fc3bd
|
|||
c431131d5f | |||
fe52d864a4
|
|||
b536c88db8
|
|||
41c9abbe3a
|
|||
6a63daab73
|
|||
24be598758
|
|||
be83d204dc
|
|||
2e787818d1
|
|||
f9cebaf216
|
|||
9f90962d92
|
|||
98ec02ea0b
|
|||
1a03808847
|
|||
bbaa89d4c2
|
|||
089df0ff7f
|
|||
9349af6ad8
|
|||
2d03c2dcc4
|
|||
bbdcedf6de
|
|||
98d48f006f
|
|||
fe3a28f84b
|
|||
e9cd1521fb
|
|||
6b6aa0ad69 | |||
f7f7e5ffde
|
|||
e304781ba3
|
|||
047e859efa
|
|||
3280b21f9b
|
|||
80686523f5
|
|||
72c63f3508
|
|||
01c8dc0229
|
@ -28,9 +28,7 @@ env:
|
|||||||
value: "define_it"
|
value: "define_it"
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE
|
- name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE
|
||||||
value: "cpx21"
|
value: "cpx21"
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_REGION
|
- name: WOODPECKER_AUTOSCALER_HCLOUD_LOCATION
|
||||||
value: "define_it"
|
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER
|
|
||||||
value: "define_it"
|
value: "define_it"
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
|
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
|
||||||
value: "define_it"
|
value: "define_it"
|
||||||
@ -79,8 +77,7 @@ WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET="define_it"
|
|||||||
WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN="define_it"
|
WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN="define_it"
|
||||||
WOODPECKER_AUTOSCALER_HCLOUD_TOKEN="define_it"
|
WOODPECKER_AUTOSCALER_HCLOUD_TOKEN="define_it"
|
||||||
WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE=cpx21
|
WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE=cpx21
|
||||||
WOODPECKER_AUTOSCALER_HCLOUD_REGION="define_it"
|
WOODPECKER_AUTOSCALER_HCLOUD_LOCATION="define_it"
|
||||||
WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER="define_it"
|
|
||||||
WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY="define_it"
|
WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY="define_it"
|
||||||
```
|
```
|
||||||
|
|
||||||
|
@ -53,9 +53,7 @@ env:
|
|||||||
value: "define_it"
|
value: "define_it"
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE
|
- name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE
|
||||||
value: "cpx21"
|
value: "cpx21"
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_REGION
|
- name: WOODPECKER_AUTOSCALER_HCLOUD_LOCATION
|
||||||
value: "define_it"
|
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER
|
|
||||||
value: "define_it"
|
value: "define_it"
|
||||||
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
|
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
|
||||||
value: "define_it"
|
value: "define_it"
|
@ -51,14 +51,26 @@ func main() {
|
|||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
}).Infof("Currently owning %d Agents", len(ownedNodes))
|
}).Infof("Currently owning %d Agents", len(ownedNodes))
|
||||||
if pendingTasks {
|
if pendingTasks > len(ownedNodes) {
|
||||||
server, err := hetzner.CreateNewAgent(cfg)
|
agent, err := woodpecker.CreateWoodpeckerAgent(cfg)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Fatal(fmt.Sprintf("Error creating new agent: %s", err.Error()))
|
||||||
|
}
|
||||||
|
server, err := hetzner.CreateNewAgent(cfg, agent)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
}).Fatal(fmt.Sprintf("Error spawning new agent: %s", err.Error()))
|
}).Fatal(fmt.Sprintf("Error spawning new agent: %s", err.Error()))
|
||||||
}
|
}
|
||||||
for {
|
for {
|
||||||
|
server, err = hetzner.RefreshNodeInfo(cfg, server.ID)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Fatal(fmt.Sprintf("Failed to start Agent: %s", err.Error()))
|
||||||
|
}
|
||||||
if server.Status == hcloud.ServerStatusRunning {
|
if server.Status == hcloud.ServerStatusRunning {
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
@ -67,7 +79,8 @@ func main() {
|
|||||||
}
|
}
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
}).Infof("Waiting for agent %s to start", server.Name)
|
}).Infof("%s is in status %s", server.Name, server.Status)
|
||||||
|
time.Sleep(30 * time.Second)
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
@ -79,27 +92,39 @@ func main() {
|
|||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
|
}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
|
||||||
}
|
}
|
||||||
if runningTasks {
|
if (runningTasks <= len(ownedNodes) && runningTasks != 0) || pendingTasks > 0 {
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "Main",
|
"Caller": "Main",
|
||||||
}).Info("Still found running tasks. No agent to be removed")
|
}).Info("Still found running tasks. No agent to be removed")
|
||||||
} else {
|
} else {
|
||||||
log.WithFields(log.Fields{
|
if len(ownedNodes) == 0 {
|
||||||
"Caller": "Main",
|
log.WithFields(log.Fields{
|
||||||
}).Info("No tasks running. Will remove agents")
|
"Caller": "Main",
|
||||||
for _, server := range ownedNodes {
|
}).Info("Nothing running and not owning any nodes")
|
||||||
hetzner.DecomNode(cfg, &server)
|
} else {
|
||||||
agentId, err := woodpecker.GetAgentIdByName(cfg, server.Name)
|
log.WithFields(log.Fields{
|
||||||
if err != nil {
|
"Caller": "Main",
|
||||||
log.WithFields(log.Fields{
|
}).Info("No tasks running. Will remove agents")
|
||||||
"Caller": "Main",
|
for _, server := range ownedNodes {
|
||||||
}).Warnf("Could not find agent %s in woodpecker. Assuming it was never added", server.Name)
|
agentId, err := hetzner.DecomNode(cfg, &server)
|
||||||
} else {
|
if err != nil {
|
||||||
woodpecker.DecomAgent(cfg, agentId)
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Warnf("Error while deleting node %s: %s", server.Name, err.Error())
|
||||||
|
}
|
||||||
|
err = woodpecker.DecomAgent(cfg, agentId)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Warnf("Could not delete node %s in woodpecker: %s", server.Name, err.Error())
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "Main",
|
||||||
|
}).Infof("Recheck in %d", cfg.CheckInterval)
|
||||||
time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute)
|
time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -14,14 +14,15 @@ type Config = struct {
|
|||||||
DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
|
DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
|
||||||
WoodpeckerLabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR"`
|
WoodpeckerLabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR"`
|
||||||
WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
|
WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
|
||||||
|
WoodpeckerGrpc string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_GRPC"`
|
||||||
WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"`
|
WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"`
|
||||||
WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"`
|
WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"`
|
||||||
WoodpeckerProtocol string `default:"http" env:"WOODPECKER_AUTOSCALER_WOODPECKER_PROTOCOL"`
|
WoodpeckerAgentVersion string `default:"latest" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_VERSION"`
|
||||||
HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"`
|
HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"`
|
||||||
HcloudInstanceType string `default:"cpx21" env:"WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE"`
|
HcloudInstanceType string `default:"cpx21" env:"WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE"`
|
||||||
HcloudRegion string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_REGION"`
|
HcloudLocation string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_LOCATION"`
|
||||||
HcloudDatacenter string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER"`
|
HcloudSSHKeys string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEYS"`
|
||||||
HcloudSSHKey string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY"`
|
HcloudIPv6Only bool `default:"false" env:"WOODPECKER_AUTOSCALER_HCLOUD_IPV6_ONLY"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func GenConfig() (cfg *Config, err error) {
|
func GenConfig() (cfg *Config, err error) {
|
||||||
|
@ -5,9 +5,12 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"text/template"
|
"text/template"
|
||||||
|
|
||||||
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
|
||||||
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
|
||||||
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/utils"
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/utils"
|
||||||
"github.com/hetznercloud/hcloud-go/hcloud"
|
"github.com/hetznercloud/hcloud-go/hcloud"
|
||||||
|
|
||||||
@ -15,13 +18,12 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
var USER_DATA_TEMPLATE = `
|
var USER_DATA_TEMPLATE = `
|
||||||
|
#cloud-config
|
||||||
write_files:
|
write_files:
|
||||||
- content: |
|
- content: |
|
||||||
# docker-compose.yml
|
# docker-compose.yml
|
||||||
version: '3'
|
version: '3'
|
||||||
|
|
||||||
services:
|
services:
|
||||||
|
|
||||||
woodpecker-agent:
|
woodpecker-agent:
|
||||||
image: {{ .Image }}
|
image: {{ .Image }}
|
||||||
command: agent
|
command: agent
|
||||||
@ -30,7 +32,7 @@ write_files:
|
|||||||
- /var/run/docker.sock:/var/run/docker.sock
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
environment:
|
environment:
|
||||||
{{- range $key, $val := .EnvConfig }}
|
{{- range $key, $val := .EnvConfig }}
|
||||||
- {{ $key }}: {{ $val }}
|
- {{ $key }}={{ $val }}
|
||||||
{{- end }}
|
{{- end }}
|
||||||
path: /root/docker-compose.yml
|
path: /root/docker-compose.yml
|
||||||
runcmd:
|
runcmd:
|
||||||
@ -39,17 +41,20 @@ runcmd:
|
|||||||
|
|
||||||
type UserDataConfig struct {
|
type UserDataConfig struct {
|
||||||
Image string
|
Image string
|
||||||
EnvConfig map[string]string
|
EnvConfig map[string]interface{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func generateConfig(cfg *config.Config, name string) (string, error) {
|
func generateConfig(cfg *config.Config, name string, agentToken string) (string, error) {
|
||||||
envConfig := map[string]string{}
|
envConfig := map[string]interface{}{
|
||||||
envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance
|
"WOODPECKER_SERVER": fmt.Sprintf("%s", cfg.WoodpeckerGrpc),
|
||||||
envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret
|
"WOODPECKER_GRPC_SECURE": true,
|
||||||
envConfig["WOODPECKER_FILTER_LABELS"] = cfg.WoodpeckerLabelSelector
|
"WOODPECKER_AGENT_SECRET": fmt.Sprintf("%s", agentToken),
|
||||||
envConfig["WOODPECKER_HOSTNAME"] = name
|
"WOODPECKER_FILTER_LABELS": fmt.Sprintf("%s", cfg.WoodpeckerLabelSelector),
|
||||||
|
"WOODPECKER_HOSTNAME": fmt.Sprintf("%s", name),
|
||||||
|
"WOODPECKER_MAX_WORKFLOWS": 4,
|
||||||
|
}
|
||||||
config := UserDataConfig{
|
config := UserDataConfig{
|
||||||
Image: "woodpeckerci/woodpecker-agent:latest",
|
Image: fmt.Sprintf("woodpeckerci/woodpecker-agent:%s", cfg.WoodpeckerAgentVersion),
|
||||||
EnvConfig: envConfig,
|
EnvConfig: envConfig,
|
||||||
}
|
}
|
||||||
tmpl, err := template.New("userdata").Parse(USER_DATA_TEMPLATE)
|
tmpl, err := template.New("userdata").Parse(USER_DATA_TEMPLATE)
|
||||||
@ -64,35 +69,52 @@ func generateConfig(cfg *config.Config, name string) (string, error) {
|
|||||||
return buf.String(), nil
|
return buf.String(), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func CreateNewAgent(cfg *config.Config) (*hcloud.Server, error) {
|
func CreateNewAgent(cfg *config.Config, woodpeckerAgent *models.Agent) (*hcloud.Server, error) {
|
||||||
client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
|
client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
|
||||||
name := fmt.Sprintf("woodpecker-autoscaler-agent-%s", utils.RandStringBytes(5))
|
userdata, err := generateConfig(cfg, woodpeckerAgent.Name, woodpeckerAgent.Token)
|
||||||
userdata, err := generateConfig(cfg, name)
|
keys := []*hcloud.SSHKey{}
|
||||||
img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker-ce", "amd64")
|
for _, keyName := range strings.Split(cfg.HcloudSSHKeys, ",") {
|
||||||
loc, _, err := client.Location.GetByName(context.Background(), cfg.HcloudRegion)
|
key, _, err := client.SSHKey.GetByName(context.Background(), keyName)
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "CreateNewAgent",
|
||||||
|
}).Warnf("Failed to look up ssh key %s: %s", keyName, err.Error())
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
keys = append(keys, key)
|
||||||
|
}
|
||||||
|
img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker-ce", "x86")
|
||||||
|
utils.CheckError(err, "GetImageByNameAndArchitecture")
|
||||||
|
loc, _, err := client.Location.GetByName(context.Background(), cfg.HcloudLocation)
|
||||||
|
utils.CheckError(err, "GetRegionByName")
|
||||||
pln, _, err := client.ServerType.GetByName(context.Background(), cfg.HcloudInstanceType)
|
pln, _, err := client.ServerType.GetByName(context.Background(), cfg.HcloudInstanceType)
|
||||||
key, _, err := client.SSHKey.GetByName(context.Background(), cfg.HcloudSSHKey)
|
utils.CheckError(err, "GetServerTypeByName")
|
||||||
dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.HcloudDatacenter)
|
|
||||||
labels := map[string]string{}
|
labels := map[string]string{}
|
||||||
labels["Role"] = "WoodpeckerAgent"
|
labels["Role"] = "WoodpeckerAgent"
|
||||||
labels["ControledBy"] = "WoodpeckerAutoscaler"
|
labels["ControledBy"] = "WoodpeckerAutoscaler"
|
||||||
|
labels["ID"] = fmt.Sprintf("%d", woodpeckerAgent.ID)
|
||||||
|
|
||||||
|
networkConf := hcloud.ServerCreatePublicNet{
|
||||||
|
EnableIPv4: !cfg.HcloudIPv6Only,
|
||||||
|
EnableIPv6: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
res, _, err := client.Server.Create(context.Background(), hcloud.ServerCreateOpts{
|
||||||
|
Name: woodpeckerAgent.Name,
|
||||||
|
ServerType: pln,
|
||||||
|
Image: img,
|
||||||
|
SSHKeys: keys,
|
||||||
|
Location: loc,
|
||||||
|
UserData: userdata,
|
||||||
|
StartAfterCreate: utils.BoolPointer(true),
|
||||||
|
Labels: labels,
|
||||||
|
PublicNet: &networkConf,
|
||||||
|
})
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, errors.New(fmt.Sprintf("Could not create new Agent: %s", err.Error()))
|
return nil, errors.New(fmt.Sprintf("Could not create new Agent: %s", err.Error()))
|
||||||
}
|
}
|
||||||
|
|
||||||
res, _, err := client.Server.Create(context.Background(), hcloud.ServerCreateOpts{
|
|
||||||
Name: name,
|
|
||||||
ServerType: pln,
|
|
||||||
Image: img,
|
|
||||||
SSHKeys: []*hcloud.SSHKey{key},
|
|
||||||
Location: loc,
|
|
||||||
Datacenter: dc,
|
|
||||||
UserData: userdata,
|
|
||||||
StartAfterCreate: utils.BoolPointer(true),
|
|
||||||
Labels: labels,
|
|
||||||
})
|
|
||||||
|
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "CreateNewAgent",
|
"Caller": "CreateNewAgent",
|
||||||
}).Infof("Created new Build Agent %s", res.Server.Name)
|
}).Infof("Created new Build Agent %s", res.Server.Name)
|
||||||
@ -119,14 +141,35 @@ func ListAgents(cfg *config.Config) ([]hcloud.Server, error) {
|
|||||||
return myServers, nil
|
return myServers, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func DecomNode(cfg *config.Config, server *hcloud.Server) error {
|
func DecomNode(cfg *config.Config, server *hcloud.Server) (int64, error) {
|
||||||
client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
|
client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
|
||||||
|
var woodpeckerAgentID int64
|
||||||
|
val, exists := server.Labels["ID"]
|
||||||
|
if exists {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "DecomNode",
|
||||||
|
}).Debugf("Found woodpecker agent id: %s", val)
|
||||||
|
woodpeckerAgentID, _ = strconv.ParseInt(val, 10, 64)
|
||||||
|
} else {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "DecomNode",
|
||||||
|
}).Warnf("Did not find woodpecker agent id for node %s", server.Name)
|
||||||
|
}
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "DecomNode",
|
"Caller": "DecomNode",
|
||||||
}).Debugf("Deleting %s node", server.Name)
|
}).Debugf("Deleting %s node", server.Name)
|
||||||
_, _, err := client.Server.DeleteWithResult(context.Background(), server)
|
_, _, err := client.Server.DeleteWithResult(context.Background(), server)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.New(fmt.Sprintf("Could not delete Agent: %s", err.Error()))
|
return woodpeckerAgentID, errors.New(fmt.Sprintf("Could not delete Agent: %s", err.Error()))
|
||||||
}
|
}
|
||||||
return nil
|
return woodpeckerAgentID, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func RefreshNodeInfo(cfg *config.Config, serverID int) (*hcloud.Server, error) {
|
||||||
|
client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
|
||||||
|
server, _, err := client.Server.GetByID(context.Background(), serverID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New(fmt.Sprintf("Could not refresh server info: %s", err.Error()))
|
||||||
|
}
|
||||||
|
return server, nil
|
||||||
}
|
}
|
||||||
|
56
internal/hetzner/hetzneragent_test.go
Normal file
56
internal/hetzner/hetzneragent_test.go
Normal file
@ -0,0 +1,56 @@
|
|||||||
|
package hetzner
|
||||||
|
|
||||||
|
import (
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateUserData(t *testing.T) {
|
||||||
|
cfg := config.Config{
|
||||||
|
LogLevel: "Info",
|
||||||
|
CheckInterval: 5,
|
||||||
|
DryRun: false,
|
||||||
|
WoodpeckerLabelSelector: "uploadfilter24.eu/instance-role=WoodpeckerTest",
|
||||||
|
WoodpeckerInstance: "http://woodpecker.test.tld",
|
||||||
|
WoodpeckerGrpc: "grpc-test.woodpecker.test.tld:443",
|
||||||
|
WoodpeckerAgentSecret: "Geheim1!",
|
||||||
|
WoodpeckerApiToken: "VeryGeheim1!",
|
||||||
|
WoodpeckerAgentVersion: "latest",
|
||||||
|
HcloudToken: "EvenMoreGeheim1!",
|
||||||
|
HcloudInstanceType: "cpx21",
|
||||||
|
HcloudLocation: "fsn1",
|
||||||
|
HcloudSSHKeys: "test-key",
|
||||||
|
}
|
||||||
|
wanted := `
|
||||||
|
#cloud-config
|
||||||
|
write_files:
|
||||||
|
- content: |
|
||||||
|
# docker-compose.yml
|
||||||
|
version: '3'
|
||||||
|
services:
|
||||||
|
woodpecker-agent:
|
||||||
|
image: woodpeckerci/woodpecker-agent:latest
|
||||||
|
command: agent
|
||||||
|
restart: always
|
||||||
|
volumes:
|
||||||
|
- /var/run/docker.sock:/var/run/docker.sock
|
||||||
|
environment:
|
||||||
|
- WOODPECKER_AGENT_SECRET=Geheim1!
|
||||||
|
- WOODPECKER_FILTER_LABELS=uploadfilter24.eu/instance-role=WoodpeckerTest
|
||||||
|
- WOODPECKER_GRPC_SECURE=true
|
||||||
|
- WOODPECKER_HOSTNAME=test-instance
|
||||||
|
- WOODPECKER_MAX_WORKFLOWS=4
|
||||||
|
- WOODPECKER_SERVER=grpc-test.woodpecker.test.tld:443
|
||||||
|
path: /root/docker-compose.yml
|
||||||
|
runcmd:
|
||||||
|
- [ sh, -xc, "cd /root; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes; docker compose up -d" ]
|
||||||
|
`
|
||||||
|
got, err := generateConfig(&cfg, "test-instance", "Geheim1!")
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Error in generating Config: %v", err)
|
||||||
|
}
|
||||||
|
if wanted != got {
|
||||||
|
t.Errorf("got:\n%v\n, wanted:\n%v", got, wanted)
|
||||||
|
}
|
||||||
|
}
|
@ -30,7 +30,7 @@ package models
|
|||||||
*/
|
*/
|
||||||
|
|
||||||
type JobInformation struct {
|
type JobInformation struct {
|
||||||
ID int `json:"id"`
|
ID string `json:"id"`
|
||||||
Data string `json:"data"`
|
Data string `json:"data"`
|
||||||
Labels map[string]string `json:"labels"`
|
Labels map[string]string `json:"labels"`
|
||||||
Dependencies string `json:"dependencies,omitempty"`
|
Dependencies string `json:"dependencies,omitempty"`
|
||||||
@ -90,3 +90,8 @@ type Agent struct {
|
|||||||
type AgentList struct {
|
type AgentList struct {
|
||||||
Agents []Agent
|
Agents []Agent
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type AgentRequest struct {
|
||||||
|
Name string `json:"name"`
|
||||||
|
NoSchedule bool `json:"no_schedule"`
|
||||||
|
}
|
||||||
|
@ -1,6 +1,10 @@
|
|||||||
package utils
|
package utils
|
||||||
|
|
||||||
import "math/rand"
|
import (
|
||||||
|
"math/rand"
|
||||||
|
|
||||||
|
log "github.com/sirupsen/logrus"
|
||||||
|
)
|
||||||
|
|
||||||
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
|
||||||
|
|
||||||
@ -15,3 +19,11 @@ func RandStringBytes(n int) string {
|
|||||||
func BoolPointer(b bool) *bool {
|
func BoolPointer(b bool) *bool {
|
||||||
return &b
|
return &b
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func CheckError(err error, caller string) {
|
||||||
|
if err != nil {
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": caller,
|
||||||
|
}).Warnf("Error from hetzner API: %s", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
@ -1,6 +1,7 @@
|
|||||||
package woodpecker
|
package woodpecker
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"bytes"
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
@ -8,11 +9,12 @@ import (
|
|||||||
|
|
||||||
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
|
||||||
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
|
||||||
|
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/utils"
|
||||||
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
func DecomAgent(cfg *config.Config, agentId int) error {
|
func DecomAgent(cfg *config.Config, agentId int64) error {
|
||||||
apiRoute := fmt.Sprintf("%s/api/agents/%d", cfg.WoodpeckerInstance, agentId)
|
apiRoute := fmt.Sprintf("%s/api/agents/%d", cfg.WoodpeckerInstance, agentId)
|
||||||
req, err := http.NewRequest("DELETE", apiRoute, nil)
|
req, err := http.NewRequest("DELETE", apiRoute, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -23,7 +25,7 @@ func DecomAgent(cfg *config.Config, agentId int) error {
|
|||||||
|
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "DecomAgent",
|
"Caller": "DecomAgent",
|
||||||
}).Debugf("Deleting %d agent from woodpecker", agentId)
|
}).Debugf("Deleting agent with id %d from woodpecker", agentId)
|
||||||
|
|
||||||
resp, err := http.DefaultClient.Do(req)
|
resp, err := http.DefaultClient.Do(req)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -35,7 +37,7 @@ func DecomAgent(cfg *config.Config, agentId int) error {
|
|||||||
|
|
||||||
func GetAgentIdByName(cfg *config.Config, name string) (int, error) {
|
func GetAgentIdByName(cfg *config.Config, name string) (int, error) {
|
||||||
apiRoute := fmt.Sprintf("%s/api/agents?page=1&perPage=100", cfg.WoodpeckerInstance)
|
apiRoute := fmt.Sprintf("%s/api/agents?page=1&perPage=100", cfg.WoodpeckerInstance)
|
||||||
req, err := http.NewRequest("GET", apiRoute, nil)
|
req, err := http.NewRequest(http.MethodGet, apiRoute, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, errors.New(fmt.Sprintf("Could not create agent query request: %s", err.Error()))
|
return 0, errors.New(fmt.Sprintf("Could not create agent query request: %s", err.Error()))
|
||||||
}
|
}
|
||||||
@ -67,3 +69,67 @@ func GetAgentIdByName(cfg *config.Config, name string) (int, error) {
|
|||||||
}
|
}
|
||||||
return 0, errors.New(fmt.Sprintf("Agent with name %s is not in server", name))
|
return 0, errors.New(fmt.Sprintf("Agent with name %s is not in server", name))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func ListAgents(cfg *config.Config) (*models.AgentList, error) {
|
||||||
|
agentList := new(models.AgentList)
|
||||||
|
apiRoute := fmt.Sprintf("%s/api/agents?page=1&perPage=100", cfg.WoodpeckerInstance)
|
||||||
|
req, err := http.NewRequest(http.MethodGet, apiRoute, nil)
|
||||||
|
if err != nil {
|
||||||
|
return agentList, errors.New(fmt.Sprintf("Could not create agent query request: %s", err.Error()))
|
||||||
|
}
|
||||||
|
req.Header.Set("Accept", "application/json")
|
||||||
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken))
|
||||||
|
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return agentList, errors.New(fmt.Sprintf("Could not query agent list: %s", err.Error()))
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return agentList, errors.New(fmt.Sprintf("Invalid status code from API: %d", resp.StatusCode))
|
||||||
|
}
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(agentList)
|
||||||
|
if err != nil {
|
||||||
|
return agentList, errors.New(fmt.Sprintf("Could not unmarshal api response: %s", err.Error()))
|
||||||
|
}
|
||||||
|
return agentList, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func CreateWoodpeckerAgent(cfg *config.Config) (*models.Agent, error) {
|
||||||
|
name := fmt.Sprintf("woodpecker-autoscaler-agent-%s", utils.RandStringBytes(5))
|
||||||
|
agentRequest := models.AgentRequest{
|
||||||
|
Name: name,
|
||||||
|
NoSchedule: false,
|
||||||
|
}
|
||||||
|
jsonBody, _ := json.Marshal(agentRequest)
|
||||||
|
bodyReader := bytes.NewReader(jsonBody)
|
||||||
|
|
||||||
|
apiRoute := fmt.Sprintf("%s/api/agents", cfg.WoodpeckerInstance)
|
||||||
|
log.WithFields(log.Fields{
|
||||||
|
"Caller": "CreateWoodpeckerAgent",
|
||||||
|
}).Debugf("Sending the following data to %s: %s", apiRoute, jsonBody)
|
||||||
|
req, err := http.NewRequest(http.MethodPost, apiRoute, bodyReader)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New(fmt.Sprintf("Could not create agent request: %s", err.Error()))
|
||||||
|
}
|
||||||
|
req.Header.Set("Content-Type", "application/json")
|
||||||
|
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken))
|
||||||
|
|
||||||
|
resp, err := http.DefaultClient.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New(fmt.Sprintf("Could not create new Agent: %s", err.Error()))
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, errors.New(fmt.Sprintf("Invalid status code from API: %d", resp.StatusCode))
|
||||||
|
}
|
||||||
|
newAgent := new(models.Agent)
|
||||||
|
err = json.NewDecoder(resp.Body).Decode(newAgent)
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.New(fmt.Sprintf("Could not unmarshal api response: %s", err.Error()))
|
||||||
|
}
|
||||||
|
return newAgent, nil
|
||||||
|
|
||||||
|
}
|
||||||
|
@ -15,7 +15,7 @@ import (
|
|||||||
|
|
||||||
func QueueInfo(cfg *config.Config, target interface{}) error {
|
func QueueInfo(cfg *config.Config, target interface{}) error {
|
||||||
apiRoute := fmt.Sprintf("%s/api/queue/info", cfg.WoodpeckerInstance)
|
apiRoute := fmt.Sprintf("%s/api/queue/info", cfg.WoodpeckerInstance)
|
||||||
req, err := http.NewRequest("GET", apiRoute, nil)
|
req, err := http.NewRequest(http.MethodGet, apiRoute, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return errors.New(fmt.Sprintf("Could not create queue request: %s", err.Error()))
|
return errors.New(fmt.Sprintf("Could not create queue request: %s", err.Error()))
|
||||||
}
|
}
|
||||||
@ -35,56 +35,48 @@ func QueueInfo(cfg *config.Config, target interface{}) error {
|
|||||||
return json.NewDecoder(resp.Body).Decode(target)
|
return json.NewDecoder(resp.Body).Decode(target)
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckPending(cfg *config.Config) (bool, error) {
|
func CheckPending(cfg *config.Config) (int, error) {
|
||||||
expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=")
|
expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=")
|
||||||
queueInfo := new(models.QueueInfo)
|
queueInfo := new(models.QueueInfo)
|
||||||
err := QueueInfo(cfg, queueInfo)
|
err := QueueInfo(cfg, queueInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
|
return 0, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
|
||||||
}
|
}
|
||||||
|
count := 0
|
||||||
if queueInfo.Stats.PendingCount > 0 {
|
if queueInfo.Stats.PendingCount > 0 {
|
||||||
if queueInfo.Pending != nil {
|
if queueInfo.Pending != nil {
|
||||||
for _, pendingJobs := range queueInfo.Pending {
|
for _, pendingJobs := range queueInfo.Pending {
|
||||||
val, exists := pendingJobs.Labels[expectedKV[0]]
|
val, exists := pendingJobs.Labels[expectedKV[0]]
|
||||||
if exists && val == expectedKV[1] {
|
if exists && val == expectedKV[1] {
|
||||||
|
count++
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "CheckPending",
|
"Caller": "CheckPending",
|
||||||
}).Info("Found pending job for us")
|
}).Debugf("Currently serving %d Jobs", count)
|
||||||
return true, nil
|
|
||||||
} else {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "CheckPending",
|
|
||||||
}).Info("No Jobs for us in Queue")
|
|
||||||
return false, nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckRunning(cfg *config.Config) (bool, error) {
|
func CheckRunning(cfg *config.Config) (int, error) {
|
||||||
expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=")
|
expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=")
|
||||||
queueInfo := new(models.QueueInfo)
|
queueInfo := new(models.QueueInfo)
|
||||||
err := QueueInfo(cfg, queueInfo)
|
err := QueueInfo(cfg, queueInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
|
return 0, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
|
||||||
}
|
}
|
||||||
|
count := 0
|
||||||
if queueInfo.Stats.RunningCount > 0 {
|
if queueInfo.Stats.RunningCount > 0 {
|
||||||
for _, runningJobs := range queueInfo.Running {
|
for _, runningJobs := range queueInfo.Running {
|
||||||
val, exists := runningJobs.Labels[expectedKV[0]]
|
val, exists := runningJobs.Labels[expectedKV[0]]
|
||||||
if exists && val == expectedKV[1] {
|
if exists && val == expectedKV[1] {
|
||||||
|
count++
|
||||||
log.WithFields(log.Fields{
|
log.WithFields(log.Fields{
|
||||||
"Caller": "CheckRunning",
|
"Caller": "CheckRunning",
|
||||||
}).Info("Found running job for us")
|
}).Debugf("Currently serving %d Jobs", count)
|
||||||
return true, nil
|
|
||||||
} else {
|
|
||||||
log.WithFields(log.Fields{
|
|
||||||
"Caller": "CheckRunning",
|
|
||||||
}).Info("No running job for us")
|
|
||||||
return false, nil
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return false, nil
|
return count, nil
|
||||||
}
|
}
|
||||||
|
Reference in New Issue
Block a user