drop datacenter config as it is ignored by the api anyway

cmd/woodpecker-autoscaler.go

@@ -51,7 +51,7 @@ func main() {
 		log.WithFields(log.Fields{
 			"Caller": "Main",
 		}).Infof("Currently owning %d Agents", len(ownedNodes))
-		if pendingTasks {
+		if pendingTasks > len(ownedNodes) {
 			server, err := hetzner.CreateNewAgent(cfg)
 			if err != nil {
 				log.WithFields(log.Fields{
@@ -59,6 +59,12 @@ func main() {
 				}).Fatal(fmt.Sprintf("Error spawning new agent: %s", err.Error()))
 			}
 			for {
+				server, err = hetzner.RefreshNodeInfo(cfg, server.ID)
+				if err != nil {
+					log.WithFields(log.Fields{
+						"Caller": "Main",
+					}).Fatal(fmt.Sprintf("Failed to start Agent: %s", err.Error()))
+				}
 				if server.Status == hcloud.ServerStatusRunning {
 					log.WithFields(log.Fields{
 						"Caller": "Main",
@@ -67,7 +73,8 @@ func main() {
 				}
 				log.WithFields(log.Fields{
 					"Caller": "Main",
-				}).Infof("Waiting for agent %s to start", server.Name)
+				}).Infof("%s is in status %s", server.Name, server.Status)
+				time.Sleep(30 * time.Second)
 			}
 		} else {
 			log.WithFields(log.Fields{
@@ -79,23 +86,29 @@ func main() {
 					"Caller": "Main",
 				}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
 			}
-			if runningTasks {
+			if (runningTasks <= len(ownedNodes) && runningTasks != 0) || pendingTasks > 0 {
 				log.WithFields(log.Fields{
 					"Caller": "Main",
 				}).Info("Still found running tasks. No agent to be removed")
 			} else {
-				log.WithFields(log.Fields{
-					"Caller": "Main",
-				}).Info("No tasks running. Will remove agents")
-				for _, server := range ownedNodes {
-					hetzner.DecomNode(cfg, &server)
-					agentId, err := woodpecker.GetAgentIdByName(cfg, server.Name)
-					if err != nil {
-						log.WithFields(log.Fields{
-							"Caller": "Main",
-						}).Warnf("Could not find agent %s in woodpecker. Assuming it was never added", server.Name)
-					} else {
-						woodpecker.DecomAgent(cfg, agentId)
+				if len(ownedNodes) == 0 {
+					log.WithFields(log.Fields{
+						"Caller": "Main",
+					}).Infof("Nothing running and not owning any nodes. Recheck in %d", cfg.CheckInterval)
+				} else {
+					log.WithFields(log.Fields{
+						"Caller": "Main",
+					}).Info("No tasks running. Will remove agents")
+					for _, server := range ownedNodes {
+						hetzner.DecomNode(cfg, &server)
+						agentId, err := woodpecker.GetAgentIdByName(cfg, server.Name)
+						if err != nil {
+							log.WithFields(log.Fields{
+								"Caller": "Main",
+							}).Warnf("Could not find agent %s in woodpecker. Assuming it was never added", server.Name)
+						} else {
+							woodpecker.DecomAgent(cfg, agentId)
+						}
 					}
 				}
 			}

internal/config/config.go

@@ -14,14 +14,15 @@ type Config = struct {
 	DryRun                  bool   `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
 	WoodpeckerLabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR"`
 	WoodpeckerInstance      string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
+	WoodpeckerGrpc          string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_GRPC"`
 	WoodpeckerAgentSecret   string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"`
 	WoodpeckerApiToken      string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"`
-	WoodpeckerProtocol      string `default:"http" env:"WOODPECKER_AUTOSCALER_WOODPECKER_PROTOCOL"`
+	WoodpeckerAgentVersion  string `default:"latest" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_VERSION"`
 	HcloudToken             string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"`
 	HcloudInstanceType      string `default:"cpx21" env:"WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE"`
-	HcloudRegion            string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_REGION"`
-	HcloudDatacenter        string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER"`
-	HcloudSSHKey            string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY"`
+	HcloudLocation          string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_LOCATION"`
+	HcloudSSHKeys           string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEYS"`
+	HcloudIPv6Only          bool   `default:"false" env:"WOODPECKER_AUTOSCALER_HCLOUD_IPV6_ONLY"`
 }
 
 func GenConfig() (cfg *Config, err error) {

internal/hetzner/hetzneragent.go

@@ -5,6 +5,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"strings"
 	"text/template"
 
 	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
@@ -15,13 +16,12 @@ import (
 )
 
 var USER_DATA_TEMPLATE = `
+#cloud-config
 write_files:
 - content: |
     # docker-compose.yml
     version: '3'
-
     services:
-
       woodpecker-agent:
         image: {{ .Image }}
         command: agent
@@ -30,7 +30,7 @@ write_files:
           - /var/run/docker.sock:/var/run/docker.sock
         environment:
 		{{- range $key, $val := .EnvConfig }}
-			- {{ $key }}: {{ $val }}
+          - {{ $key }}={{ $val }}
 	  	{{- end }}
   path: /root/docker-compose.yml
 runcmd:
@@ -39,17 +39,19 @@ runcmd:
 
 type UserDataConfig struct {
 	Image     string
-	EnvConfig map[string]string
+	EnvConfig map[string]interface{}
 }
 
 func generateConfig(cfg *config.Config, name string) (string, error) {
-	envConfig := map[string]string{}
-	envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance
-	envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret
-	envConfig["WOODPECKER_FILTER_LABELS"] = cfg.WoodpeckerLabelSelector
-	envConfig["WOODPECKER_HOSTNAME"] = name
+	envConfig := map[string]interface{}{
+		"WOODPECKER_SERVER":        fmt.Sprintf("%s", cfg.WoodpeckerGrpc),
+		"WOODPECKER_GRPC_SECURE":   true,
+		"WOODPECKER_AGENT_SECRET":  fmt.Sprintf("%s", cfg.WoodpeckerAgentSecret),
+		"WOODPECKER_FILTER_LABELS": fmt.Sprintf("%s", cfg.WoodpeckerLabelSelector),
+		"WOODPECKER_HOSTNAME":      fmt.Sprintf("%s", name),
+	}
 	config := UserDataConfig{
-		Image:     "woodpeckerci/woodpecker-agent:latest",
+		Image:     fmt.Sprintf("woodpeckerci/woodpecker-agent:%s", cfg.WoodpeckerAgentVersion),
 		EnvConfig: envConfig,
 	}
 	tmpl, err := template.New("userdata").Parse(USER_DATA_TEMPLATE)
@@ -68,31 +70,48 @@ func CreateNewAgent(cfg *config.Config) (*hcloud.Server, error) {
 	client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
 	name := fmt.Sprintf("woodpecker-autoscaler-agent-%s", utils.RandStringBytes(5))
 	userdata, err := generateConfig(cfg, name)
-	img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker-ce", "amd64")
-	loc, _, err := client.Location.GetByName(context.Background(), cfg.HcloudRegion)
+	keys := []*hcloud.SSHKey{}
+	for _, keyName := range strings.Split(cfg.HcloudSSHKeys, ",") {
+		key, _, err := client.SSHKey.GetByName(context.Background(), keyName)
+		if err != nil {
+			log.WithFields(log.Fields{
+				"Caller": "CreateNewAgent",
+			}).Warnf("Failed to look up ssh key %s: %s", keyName, err.Error())
+			continue
+		}
+		keys = append(keys, key)
+	}
+	img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker-ce", "x86")
+	utils.CheckError(err, "GetImageByNameAndArchitecture")
+	loc, _, err := client.Location.GetByName(context.Background(), cfg.HcloudLocation)
+	utils.CheckError(err, "GetRegionByName")
 	pln, _, err := client.ServerType.GetByName(context.Background(), cfg.HcloudInstanceType)
-	key, _, err := client.SSHKey.GetByName(context.Background(), cfg.HcloudSSHKey)
-	dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.HcloudDatacenter)
+	utils.CheckError(err, "GetServerTypeByName")
 	labels := map[string]string{}
 	labels["Role"] = "WoodpeckerAgent"
 	labels["ControledBy"] = "WoodpeckerAutoscaler"
 
-	if err != nil {
-		return nil, errors.New(fmt.Sprintf("Could not create new Agent: %s", err.Error()))
+	networkConf := hcloud.ServerCreatePublicNet{
+		EnableIPv4: !cfg.HcloudIPv6Only,
+		EnableIPv6: true,
 	}
 
 	res, _, err := client.Server.Create(context.Background(), hcloud.ServerCreateOpts{
 		Name:             name,
 		ServerType:       pln,
 		Image:            img,
-		SSHKeys:          []*hcloud.SSHKey{key},
+		SSHKeys:          keys,
 		Location:         loc,
-		Datacenter:       dc,
 		UserData:         userdata,
 		StartAfterCreate: utils.BoolPointer(true),
 		Labels:           labels,
+		PublicNet:        &networkConf,
 	})
 
+	if err != nil {
+		return nil, errors.New(fmt.Sprintf("Could not create new Agent: %s", err.Error()))
+	}
+
 	log.WithFields(log.Fields{
 		"Caller": "CreateNewAgent",
 	}).Infof("Created new Build Agent %s", res.Server.Name)
@@ -130,3 +149,12 @@ func DecomNode(cfg *config.Config, server *hcloud.Server) error {
 	}
 	return nil
 }
+
+func RefreshNodeInfo(cfg *config.Config, serverID int) (*hcloud.Server, error) {
+	client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
+	server, _, err := client.Server.GetByID(context.Background(), serverID)
+	if err != nil {
+		return nil, errors.New(fmt.Sprintf("Could not refresh server info: %s", err.Error()))
+	}
+	return server, nil
+}

internal/hetzner/hetzneragent_test.go

@@ -0,0 +1,55 @@
+package hetzner
+
+import (
+	"testing"
+
+	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
+)
+
+func TestGenerateUserData(t *testing.T) {
+	cfg := config.Config{
+		LogLevel:                "Info",
+		CheckInterval:           5,
+		DryRun:                  false,
+		WoodpeckerLabelSelector: "uploadfilter24.eu/instance-role=WoodpeckerTest",
+		WoodpeckerInstance:      "http://woodpecker.test.tld",
+		WoodpeckerGrpc:          "grpc-test.woodpecker.test.tld:443",
+		WoodpeckerAgentSecret:   "Geheim1!",
+		WoodpeckerApiToken:      "VeryGeheim1!",
+		WoodpeckerAgentVersion:  "latest",
+		HcloudToken:             "EvenMoreGeheim1!",
+		HcloudInstanceType:      "cpx21",
+		HcloudLocation:          "fsn1",
+		HcloudSSHKeys:           "test-key",
+	}
+	wanted := `
+#cloud-config
+write_files:
+- content: |
+    # docker-compose.yml
+    version: '3'
+    services:
+      woodpecker-agent:
+        image: woodpeckerci/woodpecker-agent:latest
+        command: agent
+        restart: always
+        volumes:
+          - /var/run/docker.sock:/var/run/docker.sock
+        environment:
+          - WOODPECKER_AGENT_SECRET=Geheim1!
+          - WOODPECKER_FILTER_LABELS=uploadfilter24.eu/instance-role=WoodpeckerTest
+          - WOODPECKER_GRPC_SECURE=true
+          - WOODPECKER_HOSTNAME=test-instance
+          - WOODPECKER_SERVER=grpc-test.woodpecker.test.tld:443
+  path: /root/docker-compose.yml
+runcmd:
+- [ sh, -xc, "cd /root; docker run --rm --privileged multiarch/qemu-user-static --reset -p yes; docker compose up -d" ]
+`
+	got, err := generateConfig(&cfg, "test-instance")
+	if err != nil {
+		t.Errorf("Error in generating Config: %v", err)
+	}
+	if wanted != got {
+		t.Errorf("got:\n%v\n, wanted:\n%v", got, wanted)
+	}
+}

internal/models/structs.go

@@ -30,7 +30,7 @@ package models
 */
 
 type JobInformation struct {
-	ID           int               `json:"id"`
+	ID           string            `json:"id"`
 	Data         string            `json:"data"`
 	Labels       map[string]string `json:"labels"`
 	Dependencies string            `json:"dependencies,omitempty"`

internal/utils/utils.go

@@ -1,6 +1,10 @@
 package utils
 
-import "math/rand"
+import (
+	"math/rand"
+
+	log "github.com/sirupsen/logrus"
+)
 
 const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
 
@@ -15,3 +19,11 @@ func RandStringBytes(n int) string {
 func BoolPointer(b bool) *bool {
 	return &b
 }
+
+func CheckError(err error, caller string) {
+	if err != nil {
+		log.WithFields(log.Fields{
+			"Caller": caller,
+		}).Warnf("Error from hetzner API: %s", err.Error())
+	}
+}

internal/woodpecker/metrics.go

@@ -35,56 +35,48 @@ func QueueInfo(cfg *config.Config, target interface{}) error {
 	return json.NewDecoder(resp.Body).Decode(target)
 }
 
-func CheckPending(cfg *config.Config) (bool, error) {
+func CheckPending(cfg *config.Config) (int, error) {
 	expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=")
 	queueInfo := new(models.QueueInfo)
 	err := QueueInfo(cfg, queueInfo)
 	if err != nil {
-		return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
+		return 0, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
 	}
+	count := 0
 	if queueInfo.Stats.PendingCount > 0 {
 		if queueInfo.Pending != nil {
 			for _, pendingJobs := range queueInfo.Pending {
 				val, exists := pendingJobs.Labels[expectedKV[0]]
 				if exists && val == expectedKV[1] {
+					count++
 					log.WithFields(log.Fields{
 						"Caller": "CheckPending",
-					}).Info("Found pending job for us")
-					return true, nil
-				} else {
-					log.WithFields(log.Fields{
-						"Caller": "CheckPending",
-					}).Info("No Jobs for us in Queue")
-					return false, nil
+					}).Debugf("Currently serving %d Jobs", count)
 				}
 			}
 		}
 	}
-	return false, nil
+	return count, nil
 }
 
-func CheckRunning(cfg *config.Config) (bool, error) {
+func CheckRunning(cfg *config.Config) (int, error) {
 	expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=")
 	queueInfo := new(models.QueueInfo)
 	err := QueueInfo(cfg, queueInfo)
 	if err != nil {
-		return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
+		return 0, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
 	}
+	count := 0
 	if queueInfo.Stats.RunningCount > 0 {
 		for _, runningJobs := range queueInfo.Running {
 			val, exists := runningJobs.Labels[expectedKV[0]]
 			if exists && val == expectedKV[1] {
+				count++
 				log.WithFields(log.Fields{
 					"Caller": "CheckRunning",
-				}).Info("Found running job for us")
-				return true, nil
-			} else {
-				log.WithFields(log.Fields{
-					"Caller": "CheckRunning",
-				}).Info("No running job for us")
-				return false, nil
+				}).Debugf("Currently serving %d Jobs", count)
 			}
 		}
 	}
-	return false, nil
+	return count, nil
 }