lerentis/bitwarden-crd-operator
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 18 Wiki Activity

getting started with kubebuilder

Browse Source
This commit is contained in:
Tobias Trabelsi
2025-03-19 22:21:54 +01:00
parent 3b98f9aced
commit fbfe7b9986
69 changed files with 4066 additions and 217 deletions
Download Patch File Download Diff File Expand all files Collapse all files

92
test/e2e/e2e_suite_test.go Normal file
View File

@ -0,0 +1,92 @@
/*
MIT License
Copyright (c) 2025 lerentis, https://git.uploadfilter24.eu/lerentis
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package e2e
import (
"fmt"
"os"
"os/exec"
"testing"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/lerentis/bitwarden-crd-operator/test/utils"
)
var (
// Optional Environment Variables:
// - CERT_MANAGER_INSTALL_SKIP=true: Skips CertManager installation during test setup.
// These variables are useful if CertManager is already installed, avoiding
// re-installation and conflicts.
skipCertManagerInstall = os.Getenv("CERT_MANAGER_INSTALL_SKIP") == "true"
// isCertManagerAlreadyInstalled will be set true when CertManager CRDs be found on the cluster
isCertManagerAlreadyInstalled = false
// projectImage is the name of the image which will be build and loaded
// with the code source changes to be tested.
projectImage = "example.com/new:v0.0.1"
)
// TestE2E runs the end-to-end (e2e) test suite for the project. These tests execute in an isolated,
// temporary environment to validate project changes with the the purposed to be used in CI jobs.
// The default setup requires Kind, builds/loads the Manager Docker image locally, and installs
// CertManager.
func TestE2E(t *testing.T) {
RegisterFailHandler(Fail)
_, _ = fmt.Fprintf(GinkgoWriter, "Starting new integration test suite\n")
RunSpecs(t, "e2e suite")
}
var _ = BeforeSuite(func() {
By("building the manager(Operator) image")
cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", projectImage))
_, err := utils.Run(cmd)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager(Operator) image")
// TODO(user): If you want to change the e2e test vendor from Kind, ensure the image is
// built and available before running the tests. Also, remove the following block.
By("loading the manager(Operator) image on Kind")
err = utils.LoadImageToKindClusterWithName(projectImage)
ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager(Operator) image into Kind")
// The tests-e2e are intended to run on a temporary cluster that is created and destroyed for testing.
// To prevent errors when tests run in environments with CertManager already installed,
// we check for its presence before execution.
// Setup CertManager before the suite if not skipped and if not already installed
if !skipCertManagerInstall {
By("checking if cert manager is installed already")
isCertManagerAlreadyInstalled = utils.IsCertManagerCRDsInstalled()
if !isCertManagerAlreadyInstalled {
_, _ = fmt.Fprintf(GinkgoWriter, "Installing CertManager...\n")
Expect(utils.InstallCertManager()).To(Succeed(), "Failed to install CertManager")
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "WARNING: CertManager is already installed. Skipping installation...\n")
}
}
})
var _ = AfterSuite(func() {
// Teardown CertManager after the suite if not skipped and if it was not already installed
if !skipCertManagerInstall && !isCertManagerAlreadyInstalled {
_, _ = fmt.Fprintf(GinkgoWriter, "Uninstalling CertManager...\n")
utils.UninstallCertManager()
}
})

332
test/e2e/e2e_test.go Normal file
View File

@ -0,0 +1,332 @@
/*
MIT License
Copyright (c) 2025 lerentis, https://git.uploadfilter24.eu/lerentis
Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"),
to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense,
and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
*/
package e2e
import (
"encoding/json"
"fmt"
"os"
"os/exec"
"path/filepath"
"time"
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
"github.com/lerentis/bitwarden-crd-operator/test/utils"
)
// namespace where the project is deployed in
const namespace = "new-system"
// serviceAccountName created for the project
const serviceAccountName = "new-controller-manager"
// metricsServiceName is the name of the metrics service of the project
const metricsServiceName = "new-controller-manager-metrics-service"
// metricsRoleBindingName is the name of the RBAC that will be created to allow get the metrics data
const metricsRoleBindingName = "new-metrics-binding"
var _ = Describe("Manager", Ordered, func() {
var controllerPodName string
// Before running the tests, set up the environment by creating the namespace,
// enforce the restricted security policy to the namespace, installing CRDs,
// and deploying the controller.
BeforeAll(func() {
By("creating manager namespace")
cmd := exec.Command("kubectl", "create", "ns", namespace)
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to create namespace")
By("labeling the namespace to enforce the restricted security policy")
cmd = exec.Command("kubectl", "label", "--overwrite", "ns", namespace,
"pod-security.kubernetes.io/enforce=restricted")
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to label namespace with restricted policy")
By("installing CRDs")
cmd = exec.Command("make", "install")
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to install CRDs")
By("deploying the controller-manager")
cmd = exec.Command("make", "deploy", fmt.Sprintf("IMG=%s", projectImage))
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to deploy the controller-manager")
})
// After all tests have been executed, clean up by undeploying the controller, uninstalling CRDs,
// and deleting the namespace.
AfterAll(func() {
By("cleaning up the curl pod for metrics")
cmd := exec.Command("kubectl", "delete", "pod", "curl-metrics", "-n", namespace)
_, _ = utils.Run(cmd)
By("undeploying the controller-manager")
cmd = exec.Command("make", "undeploy")
_, _ = utils.Run(cmd)
By("uninstalling CRDs")
cmd = exec.Command("make", "uninstall")
_, _ = utils.Run(cmd)
By("removing manager namespace")
cmd = exec.Command("kubectl", "delete", "ns", namespace)
_, _ = utils.Run(cmd)
})
// After each test, check for failures and collect logs, events,
// and pod descriptions for debugging.
AfterEach(func() {
specReport := CurrentSpecReport()
if specReport.Failed() {
By("Fetching controller manager pod logs")
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
controllerLogs, err := utils.Run(cmd)
if err == nil {
_, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs)
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Controller logs: %s", err)
}
By("Fetching Kubernetes events")
cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp")
eventsOutput, err := utils.Run(cmd)
if err == nil {
_, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput)
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get Kubernetes events: %s", err)
}
By("Fetching curl-metrics logs")
cmd = exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
metricsOutput, err := utils.Run(cmd)
if err == nil {
_, _ = fmt.Fprintf(GinkgoWriter, "Metrics logs:\n %s", metricsOutput)
} else {
_, _ = fmt.Fprintf(GinkgoWriter, "Failed to get curl-metrics logs: %s", err)
}
By("Fetching controller manager pod description")
cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace)
podDescription, err := utils.Run(cmd)
if err == nil {
fmt.Println("Pod description:\n", podDescription)
} else {
fmt.Println("Failed to describe controller pod")
}
}
})
SetDefaultEventuallyTimeout(2 * time.Minute)
SetDefaultEventuallyPollingInterval(time.Second)
Context("Manager", func() {
It("should run successfully", func() {
By("validating that the controller-manager pod is running as expected")
verifyControllerUp := func(g Gomega) {
// Get the name of the controller-manager pod
cmd := exec.Command("kubectl", "get",
"pods", "-l", "control-plane=controller-manager",
"-o", "go-template={{ range .items }}"+
"{{ if not .metadata.deletionTimestamp }}"+
"{{ .metadata.name }}"+
"{{ \"\\n\" }}{{ end }}{{ end }}",
"-n", namespace,
)
podOutput, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information")
podNames := utils.GetNonEmptyLines(podOutput)
g.Expect(podNames).To(HaveLen(1), "expected 1 controller pod running")
controllerPodName = podNames[0]
g.Expect(controllerPodName).To(ContainSubstring("controller-manager"))
// Validate the pod's status
cmd = exec.Command("kubectl", "get",
"pods", controllerPodName, "-o", "jsonpath={.status.phase}",
"-n", namespace,
)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("Running"), "Incorrect controller-manager pod status")
}
Eventually(verifyControllerUp).Should(Succeed())
})
It("should ensure the metrics endpoint is serving metrics", func() {
By("creating a ClusterRoleBinding for the service account to allow access to metrics")
cmd := exec.Command("kubectl", "create", "clusterrolebinding", metricsRoleBindingName,
"--clusterrole=new-metrics-reader",
fmt.Sprintf("--serviceaccount=%s:%s", namespace, serviceAccountName),
)
_, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to create ClusterRoleBinding")
By("validating that the metrics service is available")
cmd = exec.Command("kubectl", "get", "service", metricsServiceName, "-n", namespace)
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Metrics service should exist")
By("getting the service account token")
token, err := serviceAccountToken()
Expect(err).NotTo(HaveOccurred())
Expect(token).NotTo(BeEmpty())
By("waiting for the metrics endpoint to be ready")
verifyMetricsEndpointReady := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "endpoints", metricsServiceName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("8443"), "Metrics endpoint is not ready")
}
Eventually(verifyMetricsEndpointReady).Should(Succeed())
By("verifying that the controller manager is serving the metrics server")
verifyMetricsServerStarted := func(g Gomega) {
cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(ContainSubstring("controller-runtime.metrics\tServing metrics server"),
"Metrics server not yet started")
}
Eventually(verifyMetricsServerStarted).Should(Succeed())
By("creating the curl-metrics pod to access the metrics endpoint")
cmd = exec.Command("kubectl", "run", "curl-metrics", "--restart=Never",
"--namespace", namespace,
"--image=curlimages/curl:latest",
"--overrides",
fmt.Sprintf(`{
"spec": {
"containers": [{
"name": "curl",
"image": "curlimages/curl:latest",
"command": ["/bin/sh", "-c"],
"args": ["curl -v -k -H 'Authorization: Bearer %s' https://%s.%s.svc.cluster.local:8443/metrics"],
"securityContext": {
"allowPrivilegeEscalation": false,
"capabilities": {
"drop": ["ALL"]
},
"runAsNonRoot": true,
"runAsUser": 1000,
"seccompProfile": {
"type": "RuntimeDefault"
}
}
}],
"serviceAccount": "%s"
}
}`, token, metricsServiceName, namespace, serviceAccountName))
_, err = utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to create curl-metrics pod")
By("waiting for the curl-metrics pod to complete.")
verifyCurlUp := func(g Gomega) {
cmd := exec.Command("kubectl", "get", "pods", "curl-metrics",
"-o", "jsonpath={.status.phase}",
"-n", namespace)
output, err := utils.Run(cmd)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(output).To(Equal("Succeeded"), "curl pod in wrong status")
}
Eventually(verifyCurlUp, 5*time.Minute).Should(Succeed())
By("getting the metrics by checking curl-metrics logs")
metricsOutput := getMetricsOutput()
Expect(metricsOutput).To(ContainSubstring(
"controller_runtime_reconcile_total",
))
})
// +kubebuilder:scaffold:e2e-webhooks-checks
// TODO: Customize the e2e test suite with scenarios specific to your project.
// Consider applying sample/CR(s) and check their status and/or verifying
// the reconciliation by using the metrics, i.e.:
// metricsOutput := getMetricsOutput()
// Expect(metricsOutput).To(ContainSubstring(
// fmt.Sprintf(`controller_runtime_reconcile_total{controller="%s",result="success"} 1`,
// strings.ToLower(<Kind>),
// ))
})
})
// serviceAccountToken returns a token for the specified service account in the given namespace.
// It uses the Kubernetes TokenRequest API to generate a token by directly sending a request
// and parsing the resulting token from the API response.
func serviceAccountToken() (string, error) {
const tokenRequestRawString = `{
"apiVersion": "authentication.k8s.io/v1",
"kind": "TokenRequest"
}`
// Temporary file to store the token request
secretName := fmt.Sprintf("%s-token-request", serviceAccountName)
tokenRequestFile := filepath.Join("/tmp", secretName)
err := os.WriteFile(tokenRequestFile, []byte(tokenRequestRawString), os.FileMode(0o644))
if err != nil {
return "", err
}
var out string
verifyTokenCreation := func(g Gomega) {
// Execute kubectl command to create the token
cmd := exec.Command("kubectl", "create", "--raw", fmt.Sprintf(
"/api/v1/namespaces/%s/serviceaccounts/%s/token",
namespace,
serviceAccountName,
), "-f", tokenRequestFile)
output, err := cmd.CombinedOutput()
g.Expect(err).NotTo(HaveOccurred())
// Parse the JSON output to extract the token
var token tokenRequest
err = json.Unmarshal(output, &token)
g.Expect(err).NotTo(HaveOccurred())
out = token.Status.Token
}
Eventually(verifyTokenCreation).Should(Succeed())
return out, err
}
// getMetricsOutput retrieves and returns the logs from the curl pod used to access the metrics endpoint.
func getMetricsOutput() string {
By("getting the curl-metrics logs")
cmd := exec.Command("kubectl", "logs", "curl-metrics", "-n", namespace)
metricsOutput, err := utils.Run(cmd)
Expect(err).NotTo(HaveOccurred(), "Failed to retrieve logs from curl pod")
Expect(metricsOutput).To(ContainSubstring("< HTTP/1.1 200 OK"))
return metricsOutput
}
// tokenRequest is a simplified representation of the Kubernetes TokenRequest API response,
// containing only the token field that we need to extract.
type tokenRequest struct {
Status struct {
Token string `json:"token"`
} `json:"status"`
}