covidnetes/woodpecker-autoscaler
3
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases 3 Wiki Activity

initial implementation #1

Merged
lerentis merged 7 commits from init into main 2023-11-04 11:17:41 +00:00
Conversation 0 Commits 7 Files Changed 23 +427 -26
16 changed files with 427 additions and 26 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit db1b560896 - Show all commits

2
.woodpecker/main.yml
View File

@ -11,7 +11,7 @@ steps:
image: woodpeckerci/plugin-docker-buildx
settings:
platforms: linux/arm64/v8
repo: lerentis/metallb-ip-floater
repo: lerentis/woodpecker-autoscaler
tags:
- latest
- ${CI_COMMIT_SHA}

2
.woodpecker/pr.yml
View File

@ -10,7 +10,7 @@ steps:
image: woodpeckerci/plugin-docker-buildx
settings:
platforms: linux/arm64/v8
repo: lerentis/metallb-ip-floater
repo: lerentis/woodpecker-autoscaler
tags:
- latest
- ${CI_COMMIT_SHA}

2
.woodpecker/release.yml
View File

@ -10,7 +10,7 @@ steps:
image: woodpeckerci/plugin-docker-buildx
settings:
platforms: linux/arm64/v8
repo: lerentis/metallb-ip-floater
repo: lerentis/woodpecker-autoscaler
tags:
- latest
- ${CI_COMMIT_TAG}

93
README.md
View File

@ -1,2 +1,95 @@
# woodpecker-autoscaler
[![status-badge](https://woodpecker.uploadfilter24.eu/api/badges/8/status.svg)](https://woodpecker.uploadfilter24.eu/repos/8)
Dynamically spawns woodpecker ci build agents in hetzner cloud.
## Installing with Helm
The deployment will use helm and the chart in `chart/woodpecker-autoscaler`.
You will need a [hcloud api token](https://docs.hetzner.com/cloud/api/getting-started/generating-api-token/), a [woodpecker agent secret](https://woodpecker-ci.org/docs/administration/agent-config#woodpecker_agent_secret), a woodpecker api token and a definition of the build agents to be created.
Expose these information to the floater as described in this example:
```yaml
env:
- name: WOODPECKER_AUTOSCALER_LOGLEVEL
value: "Info"
- name: WOODPECKER_AUTOSCALER_CHECK_INTERVAL
value: "15"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR
value: "uploadfilter24.eu/instance-role=Woodpecker"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE
value: "define_it"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET
value: "define_it"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_TOKEN
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE
value: "cpx21"
- name: WOODPECKER_AUTOSCALER_HCLOUD_REGION
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
value: "define_it"
```
you can also create a secret manually with these information and reference the existing secret like this in the `values.yaml`:
```yaml
externalConfigSecret:
enabled: true
name: "my-existing-secret"
```
Now you are able to deploy:
```bash
kubectl create namespace woodpecker-autoscaler
cd chart/woodpecker-autoscaler
helm upgrade --install -f values.yaml -n woodpecker-autoscaler woodpecker-autoscaler ./
```
## Installing Manually
Download the binary from the release section and place it somewhere on your system; `/usr/bin/woodpecker-autoscaler` for example.
Create a systemd service like the one in this template:
```systemd
[Unit]
Description=Dynamically spawn woodpecker ci build agents in hetzner cloud
[Service]
Type=simple
Nice=10
ExecStart=/usr/bin/woodpecker-autoscaler
EnvironmentFile="/etc/default/woodpecker-autoscaler.env"
```
Now place the environment variable configuration in the specified file:
```bash
WOODPECKER_AUTOSCALER_LOGLEVEL=Info
WOODPECKER_AUTOSCALER_CHECK_INTERVAL=15
WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR="uploadfilter24.eu/instance-role=Woodpecker"
WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE="define_it"
WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET="define_it"
WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN="define_it"
WOODPECKER_AUTOSCALER_HCLOUD_TOKEN="define_it"
WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE=cpx21
WOODPECKER_AUTOSCALER_HCLOUD_REGION="define_it"
WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER="define_it"
WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY="define_it"
```
Now reload the systemd daemons and start the service:
```bash
sudo systemctl daemon-reload
sudo systemctl enable woodpecker-autoscaler
sudo systemctl start woodpecker-autoscaler
```
> Made with ♡ by the folkes at uploadfilter24.eu :)

23
chart/metallb-ip-floater/.helmignore Normal file
View File

@ -0,0 +1,23 @@
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/

24
chart/metallb-ip-floater/Chart.yaml Normal file
View File

@ -0,0 +1,24 @@
apiVersion: v2
name: woodpecker-autoscaler
description: A Helm chart for Kubernetes
# A chart can be either an 'application' or a 'library' chart.
#
# Application charts are a collection of templates that can be packaged into versioned archives
# to be deployed.
#
# Library charts provide useful utilities or functions for the chart developer. They're included as
# a dependency of application charts to inject those utilities and functions into the rendering
# pipeline. Library charts do not define any templates and therefore cannot be deployed.
type: application
# This is the chart version. This version number should be incremented each time you make changes
# to the chart and its templates, including the app version.
# Versions are expected to follow Semantic Versioning (https://semver.org/)
version: 0.0.1
# This is the version number of the application being deployed. This version number should be
# incremented each time you make changes to the application. Versions are not expected to
# follow Semantic Versioning. They should reflect the version the application is using.
# It is recommended to use it with quotes.
appVersion: "0.0.1"

3
chart/metallb-ip-floater/templates/NOTES.txt Normal file
View File

@ -0,0 +1,3 @@
Service has been deployed
export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "woodpecker-autoscaler.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}")
get logs: kubectl logs -f --namespace {{ .Release.Namespace }} $POD_NAME

62
chart/metallb-ip-floater/templates/_helpers.tpl Normal file
View File

@ -0,0 +1,62 @@
{{/*
Expand the name of the chart.
*/}}
{{- define "woodpecker-autoscaler.name" -}}
{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Create a default fully qualified app name.
We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
If release name contains chart name it will be used as a full name.
*/}}
{{- define "woodpecker-autoscaler.fullname" -}}
{{- if .Values.fullnameOverride }}
{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- $name := default .Chart.Name .Values.nameOverride }}
{{- if contains $name .Release.Name }}
{{- .Release.Name | trunc 63 | trimSuffix "-" }}
{{- else }}
{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }}
{{- end }}
{{- end }}
{{- end }}
{{/*
Create chart name and version as used by the chart label.
*/}}
{{- define "woodpecker-autoscaler.chart" -}}
{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }}
{{- end }}
{{/*
Common labels
*/}}
{{- define "woodpecker-autoscaler.labels" -}}
helm.sh/chart: {{ include "woodpecker-autoscaler.chart" . }}
{{ include "woodpecker-autoscaler.selectorLabels" . }}
{{- if .Chart.AppVersion }}
app.kubernetes.io/version: {{ .Chart.AppVersion | quote }}
{{- end }}
app.kubernetes.io/managed-by: {{ .Release.Service }}
{{- end }}
{{/*
Selector labels
*/}}
{{- define "woodpecker-autoscaler.selectorLabels" -}}
app.kubernetes.io/name: {{ include "woodpecker-autoscaler.name" . }}
app.kubernetes.io/instance: {{ .Release.Name }}
{{- end }}
{{/*
Create the name of the service account to use
*/}}
{{- define "woodpecker-autoscaler.serviceAccountName" -}}
{{- if .Values.serviceAccount.create }}
{{- default (include "woodpecker-autoscaler.fullname" .) .Values.serviceAccount.name }}
{{- else }}
{{- default "default" .Values.serviceAccount.name }}
{{- end }}
{{- end }}

68
chart/metallb-ip-floater/templates/deployment.yaml Normal file
View File

@ -0,0 +1,68 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: {{ include "woodpecker-autoscaler.fullname" . }}
labels:
{{- include "woodpecker-autoscaler.labels" . | nindent 4 }}
spec:
replicas: {{ .Values.replicaCount }}
selector:
matchLabels:
{{- include "woodpecker-autoscaler.selectorLabels" . | nindent 6 }}
template:
metadata:
{{- with .Values.podAnnotations }}
annotations:
{{- toYaml . | nindent 8 }}
{{- end }}
labels:
{{- include "woodpecker-autoscaler.selectorLabels" . | nindent 8 }}
spec:
{{- with .Values.imagePullSecrets }}
imagePullSecrets:
{{- toYaml . | nindent 8 }}
{{- end }}
serviceAccountName: {{ include "woodpecker-autoscaler.serviceAccountName" . }}
securityContext:
{{- toYaml .Values.podSecurityContext | nindent 8 }}
containers:
- name: {{ .Chart.Name }}
securityContext:
{{- toYaml .Values.securityContext | nindent 12 }}
image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}"
imagePullPolicy: {{ .Values.image.pullPolicy }}
env:
{{- with .Values.env }}
{{- . | toYaml | trim | nindent 12 }}
{{- end }}
{{- if .Values.externalConfigSecret.enabled }}
envFrom:
- secretRef:
name: {{ .Values.externalConfigSecret.name }}
{{- end }}
ports:
- name: healthcheck
containerPort: 8080
protocol: TCP
livenessProbe:
httpGet:
path: /health
port: healthcheck
readinessProbe:
httpGet:
path: /health
port: healthcheck
resources:
{{- toYaml .Values.resources | nindent 12 }}
{{- with .Values.nodeSelector }}
nodeSelector:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.affinity }}
affinity:
{{- toYaml . | nindent 8 }}
{{- end }}
{{- with .Values.tolerations }}
tolerations:
{{- toYaml . | nindent 8 }}
{{- end }}

12
chart/metallb-ip-floater/templates/serviceaccount.yaml Normal file
View File

@ -0,0 +1,12 @@
{{- if .Values.serviceAccount.create -}}
apiVersion: v1
kind: ServiceAccount
metadata:
name: {{ include "woodpecker-autoscaler.serviceAccountName" . }}
labels:
{{- include "woodpecker-autoscaler.labels" . | nindent 4 }}
{{- with .Values.serviceAccount.annotations }}
annotations:
{{- toYaml . | nindent 4 }}
{{- end }}
{{- end }}

83
chart/metallb-ip-floater/values.yaml Normal file
View File

@ -0,0 +1,83 @@
# Default values for woodpecker-autoscaler.
# This is a YAML-formatted file.
# Declare variables to be passed into your templates.
replicaCount: 1
image:
repository: lerentis/woodpecker-autoscaler
pullPolicy: Always
# Overrides the image tag whose default is the chart appVersion.
tag: "latest"
imagePullSecrets: []
nameOverride: ""
fullnameOverride: ""
serviceAccount:
# Specifies whether a service account should be created
create: true
# Annotations to add to the service account
annotations: {}
# The name of the service account to use.
# If not set and create is true, a name is generated using the fullname template
name: ""
podAnnotations: {}
podSecurityContext: {}
# fsGroup: 2000
securityContext: {}
# capabilities:
# drop:
# - ALL
# readOnlyRootFilesystem: true
# runAsNonRoot: true
# runAsUser: 1000
env:
- name: WOODPECKER_AUTOSCALER_LOGLEVEL
value: "Info"
- name: WOODPECKER_AUTOSCALER_CHECK_INTERVAL
value: "15"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR
value: "uploadfilter24.eu/instance-role=Woodpecker"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE
value: "define_it"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET
value: "define_it"
- name: WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_TOKEN
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE
value: "cpx21"
- name: WOODPECKER_AUTOSCALER_HCLOUD_REGION
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER
value: "define_it"
- name: WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY
value: "define_it"
externalConfigSecret:
enabled: false
name: ""
resources: {}
# We usually recommend not to specify default resources and to leave this as a conscious
# choice for the user. This also increases chances charts run on environments with little
# resources, such as Minikube. If you do want to specify resources, uncomment the following
# lines, adjust them as necessary, and remove the curly braces after 'resources:'.
# limits:
# cpu: 100m
# memory: 128Mi
# requests:
# cpu: 100m
# memory: 128Mi
nodeSelector: {}
tolerations: []
affinity: {}

22
cmd/woodpecker-autoscaler.go
View File

@ -42,6 +42,15 @@ func main() {
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error checking woodpecker queue: %s", err.Error()))
}
ownedNodes, err := hetzner.ListAgents(cfg)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Fatal(fmt.Sprintf("Error collecting owned hetzner nodes: %s", err.Error()))
}
log.WithFields(log.Fields{
"Caller": "Main",
}).Infof("Currently owning %d Agents", len(ownedNodes))
if pendingTasks {
server, err := hetzner.CreateNewAgent(cfg)
if err != nil {
@ -78,8 +87,17 @@ func main() {
log.WithFields(log.Fields{
"Caller": "Main",
}).Info("No tasks running. Will remove agents")
// TODO: iterate over agents and remove ours
// agent name should match hetzner name
for _, server := range ownedNodes {
hetzner.DecomNode(cfg, &server)
agentId, err := woodpecker.GetAgentIdByName(cfg, server.Name)
if err != nil {
log.WithFields(log.Fields{
"Caller": "Main",
}).Warnf("Could not find agent %s in woodpecker. Assuming it was never added", server.Name)
} else {
woodpecker.DecomAgent(cfg, agentId)
}
}
}
}
time.Sleep(time.Duration(cfg.CheckInterval) * time.Minute)

14
internal/config/config.go
View File

@ -11,17 +11,17 @@ import (
type Config = struct {
LogLevel string `default:"Info" env:"WOODPECKER_AUTOSCALER_LOGLEVEL"`
CheckInterval int `default:"15" env:"WOODPECKER_AUTOSCALER_CHECK_INTERVAL"`
LabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_LABELSELECTOR"`
DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
WoodpeckerLabelSelector string `default:"uploadfilter24.eu/instance-role=Woodpecker" env:"WOODPECKER_AUTOSCALER_WOODPECKER_LABEL_SELECTOR"`
WoodpeckerInstance string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_INSTANCE"`
WoodpeckerAgentSecret string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_AGENT_SECRET"`
WoodpeckerApiToken string `default:"" env:"WOODPECKER_AUTOSCALER_WOODPECKER_API_TOKEN"`
Protocol string `default:"http" env:"WOODPECKER_AUTOSCALER_PROTOCOL"`
WoodpeckerProtocol string `default:"http" env:"WOODPECKER_AUTOSCALER_WOODPECKER_PROTOCOL"`
HcloudToken string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_TOKEN"`
InstanceType string `default:"" env:"WOODPECKER_AUTOSCALER_INSTANCE_TYPE"`
Region string `default:"" env:"WOODPECKER_AUTOSCALER_REGION"`
Datacenter string `default:"" env:"WOODPECKER_AUTOSCALER_DATACENTER"`
DryRun bool `default:"false" env:"WOODPECKER_AUTOSCALER_DRY_RUN"`
SSHKey string `default:"" env:"WOODPECKER_AUTOSCALER_SSH_KEY"`
HcloudInstanceType string `default:"cpx21" env:"WOODPECKER_AUTOSCALER_HCLOUD_INSTANCE_TYPE"`
HcloudRegion string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_REGION"`
HcloudDatacenter string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_DATACENTER"`
HcloudSSHKey string `default:"" env:"WOODPECKER_AUTOSCALER_HCLOUD_SSH_KEY"`
}
func GenConfig() (cfg *Config, err error) {

20
internal/hetzner/hetzneragent.go
View File

@ -46,7 +46,7 @@ func generateConfig(cfg *config.Config, name string) (string, error) {
envConfig := map[string]string{}
envConfig["WOODPECKER_SERVER"] = cfg.WoodpeckerInstance
envConfig["WOODPECKER_AGENT_SECRET"] = cfg.WoodpeckerAgentSecret
envConfig["WOODPECKER_FILTER_LABELS"] = cfg.LabelSelector
envConfig["WOODPECKER_FILTER_LABELS"] = cfg.WoodpeckerLabelSelector
envConfig["WOODPECKER_HOSTNAME"] = name
config := UserDataConfig{
Image: "woodpeckerci/woodpecker-agent:latest",
@ -68,11 +68,11 @@ func CreateNewAgent(cfg *config.Config) (*hcloud.Server, error) {
client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
name := fmt.Sprintf("woodpecker-autoscaler-agent-%s", utils.RandStringBytes(5))
userdata, err := generateConfig(cfg, name)
img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker", "amd64")
loc, _, err := client.Location.GetByName(context.Background(), cfg.Region)
pln, _, err := client.ServerType.GetByName(context.Background(), cfg.InstanceType)
key, _, err := client.SSHKey.GetByName(context.Background(), cfg.SSHKey)
dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.Datacenter)
img, _, err := client.Image.GetByNameAndArchitecture(context.Background(), "docker-ce", "amd64")
loc, _, err := client.Location.GetByName(context.Background(), cfg.HcloudRegion)
pln, _, err := client.ServerType.GetByName(context.Background(), cfg.HcloudInstanceType)
key, _, err := client.SSHKey.GetByName(context.Background(), cfg.HcloudSSHKey)
dc, _, err := client.Datacenter.GetByName(context.Background(), cfg.HcloudDatacenter)
labels := map[string]string{}
labels["Role"] = "WoodpeckerAgent"
labels["ControledBy"] = "WoodpeckerAutoscaler"
@ -111,13 +111,19 @@ func ListAgents(cfg *config.Config) ([]hcloud.Server, error) {
val, exists := server.Labels["ControledBy"]
if exists && val == "WoodpeckerAutoscaler" {
myServers = append(myServers, *server)
log.WithFields(log.Fields{
"Caller": "ListAgents",
}).Debugf("Owning %s Hetzner node", server.Name)
}
}
return myServers, nil
}
func DecomAgent(cfg *config.Config, server *hcloud.Server) error {
func DecomNode(cfg *config.Config, server *hcloud.Server) error {
client := hcloud.NewClient(hcloud.WithToken(cfg.HcloudToken))
log.WithFields(log.Fields{
"Caller": "DecomNode",
}).Debugf("Deleting %s node", server.Name)
_, _, err := client.Server.DeleteWithResult(context.Background(), server)
if err != nil {
return errors.New(fmt.Sprintf("Could not delete Agent: %s", err.Error()))

9
internal/woodpecker/agent.go
View File

@ -8,6 +8,8 @@ import (
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/config"
"git.uploadfilter24.eu/covidnetes/woodpecker-autoscaler/internal/models"
log "github.com/sirupsen/logrus"
)
func DecomAgent(cfg *config.Config, agentId int) error {
@ -19,6 +21,10 @@ func DecomAgent(cfg *config.Config, agentId int) error {
req.Header.Set("Accept", "text/plain")
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", cfg.WoodpeckerApiToken))
log.WithFields(log.Fields{
"Caller": "DecomAgent",
}).Debugf("Deleting %d agent from woodpecker", agentId)
resp, err := http.DefaultClient.Do(req)
if err != nil {
return errors.New(fmt.Sprintf("Could not delete agent: %s", err.Error()))
@ -53,6 +59,9 @@ func GetAgentIdByName(cfg *config.Config, name string) (int, error) {
for _, agent := range agentList.Agents {
if agent.Name == name {
log.WithFields(log.Fields{
"Caller": "GetAgentIdByName",
}).Debugf("Found ID %d for Agent %s", agent.ID, name)
return int(agent.ID), nil
}
}

2
internal/woodpecker/metrics.go
View File

@ -36,7 +36,7 @@ func QueueInfo(cfg *config.Config, target interface{}) error {
}
func CheckPending(cfg *config.Config) (bool, error) {
expectedKV := strings.Split(cfg.LabelSelector, "=")
expectedKV := strings.Split(cfg.WoodpeckerLabelSelector, "=")
queueInfo := new(models.QueueInfo)
err := QueueInfo(cfg, queueInfo)
if err != nil {