Getting started with Kubernetes Operators in Go

At the beginning of last year, RedHat released the operator-sdk which helps to create the scaffolding for writing your own operators in Ansible, Helm or natively in Go. There has been quite a few changes along the way around the operator-sdk and it is maturing a lot over the course of the past year.

$ wget https://github.com/operator-framework/operator-sdk/releases/download/v1.2.0/operator-sdk-v1.2.0-x86_64-linux-gnu
$ mv operator-sdk-v1.2.0-x86_64-linux-gnu operator-sdk
$ sudo mv operator-sdk /usr/local/bin/
$ mkdir k8s-helloworld-operator
$ cd k8s-helloworld-operator
$ operator-sdk init --domain=helloworld.io --repo=github.com/berndonline/k8s-helloworld-operator
$ operator-sdk create api --group app --version v1alpha1 --kind Operator
Create Resource [y/n]
y
Create Controller [y/n]
y
Writing scaffold for you to edit...
api/v1alpha1/operator_types.go
controllers/operator_controller.go
...
// OperatorSpec defines the desired state of Operator
type OperatorSpec struct {
// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
// Important: Run "make" to regenerate code after modifying this file

// Foo is an example field of Operator. Edit Operator_types.go to remove/update
Size int32 `json:"size"`
Image string `json:"image"`
Response string `json:"response"`
}
// OperatorStatus defines the observed state of Operator
type OperatorStatus struct {
// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
// Important: Run "make" to regenerate code after modifying this file
Nodes []string `json:"nodes"`
}

// Operator is the Schema for the operators API
// +kubebuilder:subresource:status
type Operator struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`

Spec OperatorSpec `json:"spec,omitempty"`
Status OperatorStatus `json:"status,omitempty"`
}
$ make generate 
/home/ubuntu/.go/bin/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."
$ make manifests
/home/ubuntu/.go/bin/controller-gen "crd:trivialVersions=true" rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
apiVersion: app.helloworld.io/v1alpha1
kind: Operator
metadata:
name: operator-sample
spec:
size: 1
response: "Hello, World!"
image: "ghcr.io/berndonline/k8s/go-helloworld:latest"
// deploymentForOperator returns a operator Deployment object
func (r *OperatorReconciler) deploymentForOperator(m *appv1alpha1.Operator) *appsv1.Deployment {
ls := labelsForOperator(m.Name)
replicas := m.Spec.Size

dep := &appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: m.Name,
Namespace: m.Namespace,
},
Spec: appsv1.DeploymentSpec{
Replicas: &replicas,
Selector: &metav1.LabelSelector{
MatchLabels: ls,
},
Template: corev1.PodTemplateSpec{
ObjectMeta: metav1.ObjectMeta{
Labels: ls,
},
Spec: corev1.PodSpec{
Containers: []corev1.Container{{
Image: m.Spec.Image,
ImagePullPolicy: "Always",
Name: "helloworld",
Ports: []corev1.ContainerPort{{
ContainerPort: 8080,
Name: "operator",
}},
Env: []corev1.EnvVar{{
Name: "RESPONSE",
Value: m.Spec.Response,
}},
EnvFrom: []corev1.EnvFromSource{{
ConfigMapRef: &corev1.ConfigMapEnvSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: m.Name,
},
},
}},
VolumeMounts: []corev1.VolumeMount{{
Name: m.Name,
ReadOnly: true,
MountPath: "/helloworld/",
}},
}},
Volumes: []corev1.Volume{{
Name: m.Name,
VolumeSource: corev1.VolumeSource{
ConfigMap: &corev1.ConfigMapVolumeSource{
LocalObjectReference: corev1.LocalObjectReference{
Name: m.Name,
},
},
},
}},
},
},
},
}

// Set Operator instance as the owner and controller
ctrl.SetControllerReference(m, dep, r.Scheme)
return dep
}
// Check if the deployment already exists, if not create a new one
found := &appsv1.Deployment{}
err = r.Get(ctx, types.NamespacedName{Name: operator.Name, Namespace: operator.Namespace}, found)
if err != nil && errors.IsNotFound(err) {
// Define a new deployment
dep := r.deploymentForOperator(operator)
log.Info("Creating a new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
err = r.Create(ctx, dep)
if err != nil {
log.Error(err, "Failed to create new Deployment", "Deployment.Namespace", dep.Namespace, "Deployment.Name", dep.Name)
return ctrl.Result{}, err
}
// Deployment created successfully - return and requeue
return ctrl.Result{Requeue: true}, nil
} else if err != nil {
log.Error(err, "Failed to get Deployment")
return ctrl.Result{}, err
}
// Check if the deployment Spec.Template, matches the found Spec.Template
deploy := r.deploymentForOperator(operator)
if !equality.Semantic.DeepDerivative(deploy.Spec.Template, found.Spec.Template) {
found = deploy
log.Info("Updating Deployment", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name)
err := r.Update(ctx, found)
if err != nil {
log.Error(err, "Failed to update Deployment", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name)
return ctrl.Result{}, err
}
return ctrl.Result{Requeue: true}, nil
}

// Ensure the deployment size is the same as the spec
size := operator.Spec.Size
if *found.Spec.Replicas != size {
found.Spec.Replicas = &size
err = r.Update(ctx, found)
if err != nil {
log.Error(err, "Failed to update Deployment", "Deployment.Namespace", found.Namespace, "Deployment.Name", found.Name)
return ctrl.Result{}, err
}
// Spec updated - return and requeue
return ctrl.Result{Requeue: true}, nil
}
func (r *OperatorReconciler) SetupWithManager(mgr ctrl.Manager) error {
return ctrl.NewControllerManagedBy(mgr).
For(&appv1alpha1.Operator{}).
Owns(&appsv1.Deployment{}).
Owns(&corev1.ConfigMap{}).
Owns(&corev1.Service{}).
Owns(&networkingv1beta1.Ingress{}).
Complete(r)
}
// +kubebuilder:rbac:groups=app.helloworld.io,resources=operators,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=app.helloworld.io,resources=operators/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=app.helloworld.io,resources=operators/finalizers,verbs=update
// +kubebuilder:rbac:groups=apps,resources=deployments,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=configmaps,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=pods,verbs=get;list;watch
$ make manifests 
/home/ubuntu/.go/bin/controller-gen "crd:trivialVersions=true" rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
$ ./scripts/create-kind-cluster.sh 
Creating cluster "kind" ...
✓ Ensuring node image (kindest/node:v1.19.1) 🖼
✓ Preparing nodes 📦
✓ Writing configuration 📜
✓ Starting control-plane 🕹️
✓ Installing CNI 🔌
✓ Installing StorageClass 💾
Set kubectl context to "kind-kind"
You can now use your cluster with:

kubectl cluster-info --context kind-kind

Have a question, bug, or feature request? Let us know! https://kind.sigs.k8s.io/#community 🙂
$ make install
/home/ubuntu/.go/bin/controller-gen "crd:trivialVersions=true" rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
/usr/bin/kustomize build config/crd | kubectl apply -f -
Warning: apiextensions.k8s.io/v1beta1 CustomResourceDefinition is deprecated in v1.16+, unavailable in v1.22+; use apiextensions.k8s.io/v1 CustomResourceDefinition
customresourcedefinition.apiextensions.k8s.io/operators.app.helloworld.io created
$ make run
/home/ubuntu/.go/bin/controller-gen object:headerFile="hack/boilerplate.go.txt" paths="./..."
go fmt ./...
go vet ./...
/home/ubuntu/.go/bin/controller-gen "crd:trivialVersions=true" rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
go run ./main.go
2020-11-22T18:12:49.023Z INFO controller-runtime.metrics metrics server is starting to listen {"addr": ":8080"}
2020-11-22T18:12:49.024Z INFO setup starting manager
2020-11-22T18:12:49.025Z INFO controller-runtime.manager starting metrics server {"path": "/metrics"}
2020-11-22T18:12:49.025Z INFO controller Starting EventSource {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "source": "kind source: /, Kind="}
2020-11-22T18:12:49.126Z INFO controller Starting EventSource {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "source": "kind source: /, Kind="}
2020-11-22T18:12:49.226Z INFO controller Starting EventSource {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "source": "kind source: /, Kind="}
2020-11-22T18:12:49.327Z INFO controller Starting EventSource {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "source": "kind source: /, Kind="}
2020-11-22T18:12:49.428Z INFO controller Starting EventSource {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "source": "kind source: /, Kind="}
2020-11-22T18:12:49.528Z INFO controller Starting Controller {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator"}
2020-11-22T18:12:49.528Z INFO controller Starting workers {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "worker count": 1}
$ kubectl apply -f config/samples/app_v1alpha1_operator.yaml 
operator.app.helloworld.io/operator-sample created
2020-11-22T18:15:30.412Z	INFO	controllers.Operator	Creating a new Deployment	{"operator": "default/operator-sample", "Deployment.Namespace": "default", "Deployment.Name": "operator-sample"}
2020-11-22T18:15:30.446Z INFO controllers.Operator Creating a new ConfigMap {"operator": "default/operator-sample", "ConfigMap.Namespace": "default", "ConfigMap.Name": "operator-sample"}
2020-11-22T18:15:30.453Z INFO controllers.Operator Creating a new Service {"operator": "default/operator-sample", "Service.Namespace": "default", "Service.Name": "operator-sample"}
2020-11-22T18:15:30.470Z INFO controllers.Operator Creating a new Ingress {"operator": "default/operator-sample", "Ingress.Namespace": "default", "Ingress.Name": "operator-sample"}
2020-11-22T18:15:30.927Z DEBUG controller Successfully Reconciled {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "name": "operator-sample", "namespace": "default"}
2020-11-22T18:15:30.927Z DEBUG controller Successfully Reconciled {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "name": "operator-sample", "namespace": "default"}
2020-11-22T18:15:33.776Z DEBUG controller Successfully Reconciled {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "name": "operator-sample", "namespace": "default"}
2020-11-22T18:15:35.181Z DEBUG controller Successfully Reconciled {"reconcilerGroup": "app.helloworld.io", "reconcilerKind": "Operator", "controller": "operator", "name": "operator-sample", "namespace": "default"}
$ kubectl get operators.app.helloworld.io 
NAME AGE
operator-sample 6m11s
$ kubectl get all
NAME READY STATUS RESTARTS AGE
pod/operator-sample-767897c4b9-8zwsd 1/1 Running 0 2m59s

NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE
service/kubernetes ClusterIP 10.96.0.1 443/TCP 29m
service/operator-sample ClusterIP 10.96.199.188 8080/TCP 2m59s

NAME READY UP-TO-DATE AVAILABLE AGE
deployment.apps/operator-sample 1/1 1 1 2m59s

NAME DESIRED CURRENT READY AGE
replicaset.apps/operator-sample-767897c4b9 1 1 1 2m59s
$ make docker-build IMG=ghcr.io/berndonline/k8s/helloworld-operator:latest
$ make docker-push IMG=ghcr.io/berndonline/k8s/helloworld-operator:latest
$ kustomize build config/default | kubectl apply -f -

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Bernd Malmqvist

Bernd Malmqvist

Highly versatile Senior technical Lead Engineer, I am a consummate and competent qualified IT Professional specialising in distributed systems