implemented spawning of new agents

split pipeline

internal/config/config.go

@@ -18,7 +18,8 @@ type Config = struct {
 	Protocol              string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"`
 	HcloudToken           string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"`
 	InstanceType          string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"`
-	Zone                  string `default:"" env:"WOODPECKER_AUTOSCALER_ZONE"`
+	Region                string `default:"" env:"WOODPECKER_AUTOSCALER_REGION"`
+	Datacenter            string `default:"" env:"WOODPECKER_AUTOSCALER_DATACENTER"`
 	DryRun                bool   `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
 	SSHKey                string `default:"" env:"WOODPECKER_AUTOSCALER_SSH_KEY"`
 }

internal/hetzner/hetzneragent.go

@@ -2,11 +2,16 @@ package hetzner
 
 import (
 	"bytes"
+	"context"
 	"errors"
 	"fmt"
 	"text/template"
 
 	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
+	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/utils"
+	"github.com/hetznercloud/hcloud-go/hcloud"
+
+	log "github.com/sirupsen/logrus"
 )
 
 var USER_DATA_TEMPLATE = `
@@ -37,11 +42,12 @@ type UserDataConfig struct {
 	EnvConfig map[string]string
 }
 
-func generateConfig(cfg *config.Config) (string, error) {
+func generateConfig(cfg *config.Config, name string) (string, error) {
 	envConfig := map[string]string{}
 	envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance
 	envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret
 	envConfig["WOODPECKER_FILTER_LABELS"] = cfg.LabelSelector
+	envConfig["WOODPECKER_HOSTNAME"] = name
 	config := UserDataConfig{
 		Image:     "woodpeckerci/woodpecker-agent:latest",
 		EnvConfig: envConfig,
@@ -55,6 +61,66 @@ func generateConfig(cfg *config.Config) (string, error) {
 	if err != nil {
 		return "", errors.New(fmt.Sprintf("Could not render userdata template: %s", err.Error()))
 	}
-
 	return buf.String(), nil
 }
+
+func CreateNewAgent(cfg *config.Config) (*hcloud.Server, error) {
+	client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
+	name := fmt.Sprintf("woodpecker-autoscaler-agent-%s", utils.RandStringBytes(5))
+	userdata, err := generateConfig(cfg, name)
+	img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker", "amd64")
+	loc, _, err := client.Location.GetByName(context.Background(), cfg.Region)
+	pln, _, err := client.ServerType.GetByName(context.Background(), cfg.InstanceType)
+	key, _, err := client.SSHKey.GetByName(context.Background(), cfg.SSHKey)
+	dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.Datacenter)
+	labels := map[string]string{}
+	labels["Role"] = "WoodpeckerAgent"
+	labels["ControledBy"] = "WoodpeckerAutoscaler"
+
+	if err != nil {
+		return nil, errors.New(fmt.Sprintf("Could not create new Agent: %s", err.Error()))
+	}
+
+	res, _, err := client.Server.Create(context.Background(), hcloud.ServerCreateOpts{
+		Name:             name,
+		ServerType:       pln,
+		Image:            img,
+		SSHKeys:          []*hcloud.SSHKey{key},
+		Location:         loc,
+		Datacenter:       dc,
+		UserData:         userdata,
+		StartAfterCreate: utils.BoolPointer(true),
+		Labels:           labels,
+	})
+
+	log.WithFields(log.Fields{
+		"Caller": "CreateNewAgent",
+	}).Infof("Created new Build Agent %s", res.Server.Name)
+
+	return res.Server, nil
+}
+
+func ListAgents(cfg *config.Config) ([]hcloud.Server, error) {
+	client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
+	allServers, err := client.Server.All(context.Background())
+	if err != nil {
+		return nil, errors.New(fmt.Sprintf("Could not query Server list: %s", err.Error()))
+	}
+	myServers := []hcloud.Server{}
+	for _, server := range allServers {
+		val, exists := server.Labels["ControledBy"]
+		if exists && val == "WoodpeckerAutoscaler" {
+			myServers = append(myServers, *server)
+		}
+	}
+	return myServers, nil
+}
+
+func DecomAgent(cfg *config.Config, server *hcloud.Server) error {
+	client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
+	_, _, err := client.Server.DeleteWithResult(context.Background(), server)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Could not delete Agent: %s", err.Error()))
+	}
+	return nil
+}

internal/models/structs.go

@@ -54,3 +54,39 @@ type QueueInfo struct {
 	Stats         Stats                `json:"stats"`
 	Paused        bool                 `json:"paused"`
 }
+
+/*[
+  {
+    "id": 2,
+    "created": 1693567407,
+    "updated": 1694013270,
+    "name": "",
+    "owner_id": -1,
+    "token": "redacted",
+    "last_contact": 1694013270,
+    "platform": "linux/arm64",
+    "backend": "kubernetes",
+    "capacity": 4,
+    "version": "next-971534929c",
+    "no_schedule": false
+  }
+]*/
+
+type Agent struct {
+	ID          int64  `json:"id"`
+	Created     int64  `json:"created"`
+	Updated     int64  `json:"updated"`
+	Name        string `json:"name"`
+	OwnerID     int64  `json:"owner_id"`
+	Token       string `json:"token"`
+	LastContact int64  `json:"last_contact"`
+	Platform    string `json:"platform"`
+	Backend     string `json:"backend"`
+	Capacity    int32  `json:"capacity"`
+	Version     string `json:"version"`
+	NoSchedule  bool   `json:"no_schedule"`
+}
+
+type AgentList struct {
+	Agents []Agent
+}

internal/utils/utils.go

@@ -0,0 +1,17 @@
+package utils
+
+import "math/rand"
+
+const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
+
+func RandStringBytes(n int) string {
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}
+
+func BoolPointer(b bool) *bool {
+	return &b
+}

internal/woodpecker/agent.go

@@ -0,0 +1,60 @@
+package woodpecker
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
+	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
+)
+
+func DecomAgent(cfg *config.Config, agentId int) error {
+	apiRoute := fmt.Sprintf("%s/api/agents/%d", cfg.WoodpeckerInstance, agentId)
+	req, err := http.NewRequest("DELETE", apiRoute, nil)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Could not create delete request: %s", err.Error()))
+	}
+	req.Header.Set("Accept", "text/plain")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken))
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return errors.New(fmt.Sprintf("Could not delete agent: %s", err.Error()))
+	}
+	defer resp.Body.Close()
+	return nil
+}
+
+func GetAgentIdByName(cfg *config.Config, name string) (int, error) {
+	apiRoute := fmt.Sprintf("%s/api/agents?page=1&perPage=100", cfg.WoodpeckerInstance)
+	req, err := http.NewRequest("GET", apiRoute, nil)
+	if err != nil {
+		return 0, errors.New(fmt.Sprintf("Could not create agent query request: %s", err.Error()))
+	}
+	req.Header.Set("Accept", "application/json")
+	req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken))
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return 0, errors.New(fmt.Sprintf("Could not query agent list: %s", err.Error()))
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return 0, errors.New(fmt.Sprintf("Invalid status code from API: %d", resp.StatusCode))
+	}
+	agentList := new(models.AgentList)
+	err = json.NewDecoder(resp.Body).Decode(agentList)
+	if err != nil {
+		return 0, errors.New(fmt.Sprintf("Could not unmarshal api response: %s", err.Error()))
+	}
+
+	for _, agent := range agentList.Agents {
+		if agent.Name == name {
+			return int(agent.ID), nil
+		}
+	}
+	return 0, errors.New(fmt.Sprintf("Agent with name %s is not in server", name))
+}

internal/woodpecker/metrics.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
+	"strings"
 
 	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
 	"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
@@ -34,27 +35,43 @@ func QueueInfo(cfg *config.Config, target interface{}) error {
 	return json.NewDecoder(resp.Body).Decode(target)
 }
 
-func CheckPending(cfg *config.Config) error {
+func CheckPending(cfg *config.Config) (bool, error) {
+	expectedKV := strings.Split(cfg.LabelSelector, "=")
 	queueInfo := new(models.QueueInfo)
 	err := QueueInfo(cfg, queueInfo)
 	if err != nil {
-		return errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
+		return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
 	}
 	if queueInfo.Stats.PendingCount > 0 {
-		// TODO: queueInfo.Pending may be empty
-		for _, pendingJobs := range queueInfo.Pending {
-			// TODO: separate key and value from LabelSelector and compare them deeply
-			_, exists := pendingJobs.Labels[cfg.LabelSelector]
-			if exists {
-				log.WithFields(log.Fields{
-					"Caller": "CheckPending",
-				}).Info("Found pending job for us. Requesting new Agent")
-			} else {
-				log.WithFields(log.Fields{
-					"Caller": "CheckPending",
-				}).Info("No Jobs for us in Queue")
+		if queueInfo.Pending != nil {
+			for _, pendingJobs := range queueInfo.Pending {
+				val, exists := pendingJobs.Labels[expectedKV[0]]
+				if exists && val == expectedKV[1] {
+					log.WithFields(log.Fields{
+						"Caller": "CheckPending",
+					}).Info("Found pending job for us")
+					return true, nil
+				} else {
+					log.WithFields(log.Fields{
+						"Caller": "CheckPending",
+					}).Info("No Jobs for us in Queue")
+					return false, nil
+				}
 			}
 		}
 	}
-	return nil
+	return false, nil
+}
+
+func CheckRunning(cfg *config.Config) (bool, error) {
+	queueInfo := new(models.QueueInfo)
+	err := QueueInfo(cfg, queueInfo)
+	if err != nil {
+		return false, errors.New(fmt.Sprintf("Error from QueueInfo: %s", err.Error()))
+	}
+	// TODO: create and parse running object. there may be jobs that are not for us
+	if queueInfo.Stats.RunningCount > 0 {
+		return true, nil
+	}
+	return false, nil
 }