From 3103a2ad4c040b4bf1db0324e0d5adb18ca6590d Mon Sep 17 00:00:00 2001 From: Carl Andersson Date: Mon, 13 Oct 2025 16:25:19 +0200 Subject: [PATCH] Initial commit --- README.md | 80 +++ apiResources/v1.33.json | 1150 +++++++++++++++++++++++++++++++++++++++ default.nix | 51 ++ demo/default.nix | 6 + demo/helm.nix | 27 + demo/resources.nix | 15 + helm.nix | 151 +++++ importyaml.nix | 132 +++++ internal.nix | 27 + kluctl.nix | 169 ++++++ kubernetes.nix | 131 +++++ validation.nix | 160 ++++++ 12 files changed, 2099 insertions(+) create mode 100644 README.md create mode 100644 apiResources/v1.33.json create mode 100644 default.nix create mode 100644 demo/default.nix create mode 100644 demo/helm.nix create mode 100644 demo/resources.nix create mode 100644 helm.nix create mode 100644 importyaml.nix create mode 100644 internal.nix create mode 100644 kluctl.nix create mode 100644 kubernetes.nix create mode 100644 validation.nix diff --git a/README.md b/README.md new file mode 100644 index 0000000..7e43076 --- /dev/null +++ b/README.md @@ -0,0 +1,80 @@ +# easykubenix + +`easykubenix` uses the NixOS module system to generate Kubernetes manifests. It +avoids generating Nix types for the entire Kubernetes API, resulting in faster +evaluations and a simpler user experience compared to alternatives. + +Manifest validation is performed by a script that applies the generated +resources against an ephemeral `etcd` and `kube-apiserver` instance. This +approach uses the Kubernetes API server as the single source of truth for +validation. + +## Usage + +Define your resources using the NixOS module system. The top-level attribute +is `kubernetes`, followed by the resource `kind`, then the resource name. + +```nix +# my-manifests.nix +{ + kubernetes.ConfigMap.my-awesome-configmap = { + metadata.namespace = "mynamespace123"; + data."config.json" = builtins.toJSON { key = "value"; }; + }; + + kubernetes.Deployment.my-app = { + metadata.name = "my-app"; + spec.replicas = 3; + # ... + }; +} +``` + +To generate the final YAML manifests, import your modules into the provided +`eval` function. + +```nix +# default.nix +{ pkgs ? import {} }: + +(import { + inherit pkgs; + modules = [ ./my-manifests.nix ]; +}).eval +``` + +## Features + +### Manifest Validation + +To validate your manifests against a real Kubernetes API server without +affecting a live cluster, run the validation script. + +```sh +nix run --file $youreasynix easykubenix.validator +``` + +This command builds your manifests, spins up a temporary API server, and +applies the configuration to it, reporting any errors from `kubectl`. + +### Helm Chart Rendering + +`easykubenix` can render Helm charts and import their resources into the NixOS +module system. This allows you to override values from rendered charts using +standard module system functions like `lib.mkForce`. + +The import is performed via Import From Derivation (IFD), which is necessary +as it requires running `helm template` during Nix evaluation. + +### Kluctl Integration + +`kluctl` is a CLI and GitOps tool that deploys manifests. It's distinctive +feature is that it adds a label (discriminator) to every resource deployed which +means when applying manifests with --prune it scans the watch-cache for resources +with this label and removes them if they're not in the manifest we're applying. + +Essentially kubectl apply --prune -l on steroids. It also integrates SOPS +which can be used to keep secrets out of the Nix store. + +`easykubenix` supports generating a minimal kluctl project and deployment script. + diff --git a/apiResources/v1.33.json b/apiResources/v1.33.json new file mode 100644 index 0000000..8eb8aba --- /dev/null +++ b/apiResources/v1.33.json @@ -0,0 +1,1150 @@ +{ + "kind": "APIResourceList", + "apiVersion": "v1", + "groupVersion": "", + "resources": [ + { + "name": "bindings", + "singularName": "binding", + "namespaced": true, + "version": "v1", + "kind": "Binding", + "verbs": [ + "create" + ] + }, + { + "name": "componentstatuses", + "singularName": "componentstatus", + "namespaced": false, + "version": "v1", + "kind": "ComponentStatus", + "verbs": [ + "get", + "list" + ], + "shortNames": [ + "cs" + ] + }, + { + "name": "configmaps", + "singularName": "configmap", + "namespaced": true, + "version": "v1", + "kind": "ConfigMap", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "cm" + ] + }, + { + "name": "endpoints", + "singularName": "endpoints", + "namespaced": true, + "version": "v1", + "kind": "Endpoints", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ep" + ] + }, + { + "name": "events", + "singularName": "event", + "namespaced": true, + "version": "v1", + "kind": "Event", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ev" + ] + }, + { + "name": "limitranges", + "singularName": "limitrange", + "namespaced": true, + "version": "v1", + "kind": "LimitRange", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "limits" + ] + }, + { + "name": "namespaces", + "singularName": "namespace", + "namespaced": false, + "version": "v1", + "kind": "Namespace", + "verbs": [ + "create", + "delete", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ns" + ] + }, + { + "name": "nodes", + "singularName": "node", + "namespaced": false, + "version": "v1", + "kind": "Node", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "no" + ] + }, + { + "name": "persistentvolumeclaims", + "singularName": "persistentvolumeclaim", + "namespaced": true, + "version": "v1", + "kind": "PersistentVolumeClaim", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "pvc" + ] + }, + { + "name": "persistentvolumes", + "singularName": "persistentvolume", + "namespaced": false, + "version": "v1", + "kind": "PersistentVolume", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "pv" + ] + }, + { + "name": "pods", + "singularName": "pod", + "namespaced": true, + "version": "v1", + "kind": "Pod", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "po" + ], + "categories": [ + "all" + ] + }, + { + "name": "podtemplates", + "singularName": "podtemplate", + "namespaced": true, + "version": "v1", + "kind": "PodTemplate", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "replicationcontrollers", + "singularName": "replicationcontroller", + "namespaced": true, + "version": "v1", + "kind": "ReplicationController", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "rc" + ], + "categories": [ + "all" + ] + }, + { + "name": "resourcequotas", + "singularName": "resourcequota", + "namespaced": true, + "version": "v1", + "kind": "ResourceQuota", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "quota" + ] + }, + { + "name": "secrets", + "singularName": "secret", + "namespaced": true, + "version": "v1", + "kind": "Secret", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "serviceaccounts", + "singularName": "serviceaccount", + "namespaced": true, + "version": "v1", + "kind": "ServiceAccount", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "sa" + ] + }, + { + "name": "services", + "singularName": "service", + "namespaced": true, + "version": "v1", + "kind": "Service", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "svc" + ], + "categories": [ + "all" + ] + }, + { + "name": "mutatingwebhookconfigurations", + "singularName": "mutatingwebhookconfiguration", + "namespaced": false, + "group": "admissionregistration.k8s.io", + "version": "v1", + "kind": "MutatingWebhookConfiguration", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "categories": [ + "api-extensions" + ] + }, + { + "name": "validatingadmissionpolicies", + "singularName": "validatingadmissionpolicy", + "namespaced": false, + "group": "admissionregistration.k8s.io", + "version": "v1", + "kind": "ValidatingAdmissionPolicy", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "categories": [ + "api-extensions" + ] + }, + { + "name": "validatingadmissionpolicybindings", + "singularName": "validatingadmissionpolicybinding", + "namespaced": false, + "group": "admissionregistration.k8s.io", + "version": "v1", + "kind": "ValidatingAdmissionPolicyBinding", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "categories": [ + "api-extensions" + ] + }, + { + "name": "validatingwebhookconfigurations", + "singularName": "validatingwebhookconfiguration", + "namespaced": false, + "group": "admissionregistration.k8s.io", + "version": "v1", + "kind": "ValidatingWebhookConfiguration", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "categories": [ + "api-extensions" + ] + }, + { + "name": "customresourcedefinitions", + "singularName": "customresourcedefinition", + "namespaced": false, + "group": "apiextensions.k8s.io", + "version": "v1", + "kind": "CustomResourceDefinition", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "crd", + "crds" + ], + "categories": [ + "api-extensions" + ] + }, + { + "name": "apiservices", + "singularName": "apiservice", + "namespaced": false, + "group": "apiregistration.k8s.io", + "version": "v1", + "kind": "APIService", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "categories": [ + "api-extensions" + ] + }, + { + "name": "controllerrevisions", + "singularName": "controllerrevision", + "namespaced": true, + "group": "apps", + "version": "v1", + "kind": "ControllerRevision", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "daemonsets", + "singularName": "daemonset", + "namespaced": true, + "group": "apps", + "version": "v1", + "kind": "DaemonSet", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ds" + ], + "categories": [ + "all" + ] + }, + { + "name": "deployments", + "singularName": "deployment", + "namespaced": true, + "group": "apps", + "version": "v1", + "kind": "Deployment", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "deploy" + ], + "categories": [ + "all" + ] + }, + { + "name": "replicasets", + "singularName": "replicaset", + "namespaced": true, + "group": "apps", + "version": "v1", + "kind": "ReplicaSet", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "rs" + ], + "categories": [ + "all" + ] + }, + { + "name": "statefulsets", + "singularName": "statefulset", + "namespaced": true, + "group": "apps", + "version": "v1", + "kind": "StatefulSet", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "sts" + ], + "categories": [ + "all" + ] + }, + { + "name": "selfsubjectreviews", + "singularName": "selfsubjectreview", + "namespaced": false, + "group": "authentication.k8s.io", + "version": "v1", + "kind": "SelfSubjectReview", + "verbs": [ + "create" + ] + }, + { + "name": "tokenreviews", + "singularName": "tokenreview", + "namespaced": false, + "group": "authentication.k8s.io", + "version": "v1", + "kind": "TokenReview", + "verbs": [ + "create" + ] + }, + { + "name": "localsubjectaccessreviews", + "singularName": "localsubjectaccessreview", + "namespaced": true, + "group": "authorization.k8s.io", + "version": "v1", + "kind": "LocalSubjectAccessReview", + "verbs": [ + "create" + ] + }, + { + "name": "selfsubjectaccessreviews", + "singularName": "selfsubjectaccessreview", + "namespaced": false, + "group": "authorization.k8s.io", + "version": "v1", + "kind": "SelfSubjectAccessReview", + "verbs": [ + "create" + ] + }, + { + "name": "selfsubjectrulesreviews", + "singularName": "selfsubjectrulesreview", + "namespaced": false, + "group": "authorization.k8s.io", + "version": "v1", + "kind": "SelfSubjectRulesReview", + "verbs": [ + "create" + ] + }, + { + "name": "subjectaccessreviews", + "singularName": "subjectaccessreview", + "namespaced": false, + "group": "authorization.k8s.io", + "version": "v1", + "kind": "SubjectAccessReview", + "verbs": [ + "create" + ] + }, + { + "name": "horizontalpodautoscalers", + "singularName": "horizontalpodautoscaler", + "namespaced": true, + "group": "autoscaling", + "version": "v2", + "kind": "HorizontalPodAutoscaler", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "hpa" + ], + "categories": [ + "all" + ] + }, + { + "name": "cronjobs", + "singularName": "cronjob", + "namespaced": true, + "group": "batch", + "version": "v1", + "kind": "CronJob", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "cj" + ], + "categories": [ + "all" + ] + }, + { + "name": "jobs", + "singularName": "job", + "namespaced": true, + "group": "batch", + "version": "v1", + "kind": "Job", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "categories": [ + "all" + ] + }, + { + "name": "certificatesigningrequests", + "singularName": "certificatesigningrequest", + "namespaced": false, + "group": "certificates.k8s.io", + "version": "v1", + "kind": "CertificateSigningRequest", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "csr" + ] + }, + { + "name": "leases", + "singularName": "lease", + "namespaced": true, + "group": "coordination.k8s.io", + "version": "v1", + "kind": "Lease", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "endpointslices", + "singularName": "endpointslice", + "namespaced": true, + "group": "discovery.k8s.io", + "version": "v1", + "kind": "EndpointSlice", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "events", + "singularName": "event", + "namespaced": true, + "group": "events.k8s.io", + "version": "v1", + "kind": "Event", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ev" + ] + }, + { + "name": "flowschemas", + "singularName": "flowschema", + "namespaced": false, + "group": "flowcontrol.apiserver.k8s.io", + "version": "v1", + "kind": "FlowSchema", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "prioritylevelconfigurations", + "singularName": "prioritylevelconfiguration", + "namespaced": false, + "group": "flowcontrol.apiserver.k8s.io", + "version": "v1", + "kind": "PriorityLevelConfiguration", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "ingressclasses", + "singularName": "ingressclass", + "namespaced": false, + "group": "networking.k8s.io", + "version": "v1", + "kind": "IngressClass", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "ingresses", + "singularName": "ingress", + "namespaced": true, + "group": "networking.k8s.io", + "version": "v1", + "kind": "Ingress", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ing" + ] + }, + { + "name": "ipaddresses", + "singularName": "ipaddress", + "namespaced": false, + "group": "networking.k8s.io", + "version": "v1", + "kind": "IPAddress", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "ip" + ] + }, + { + "name": "networkpolicies", + "singularName": "networkpolicy", + "namespaced": true, + "group": "networking.k8s.io", + "version": "v1", + "kind": "NetworkPolicy", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "netpol" + ] + }, + { + "name": "servicecidrs", + "singularName": "servicecidr", + "namespaced": false, + "group": "networking.k8s.io", + "version": "v1", + "kind": "ServiceCIDR", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "runtimeclasses", + "singularName": "runtimeclass", + "namespaced": false, + "group": "node.k8s.io", + "version": "v1", + "kind": "RuntimeClass", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "poddisruptionbudgets", + "singularName": "poddisruptionbudget", + "namespaced": true, + "group": "policy", + "version": "v1", + "kind": "PodDisruptionBudget", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "pdb" + ] + }, + { + "name": "clusterrolebindings", + "singularName": "clusterrolebinding", + "namespaced": false, + "group": "rbac.authorization.k8s.io", + "version": "v1", + "kind": "ClusterRoleBinding", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "clusterroles", + "singularName": "clusterrole", + "namespaced": false, + "group": "rbac.authorization.k8s.io", + "version": "v1", + "kind": "ClusterRole", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "rolebindings", + "singularName": "rolebinding", + "namespaced": true, + "group": "rbac.authorization.k8s.io", + "version": "v1", + "kind": "RoleBinding", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "roles", + "singularName": "role", + "namespaced": true, + "group": "rbac.authorization.k8s.io", + "version": "v1", + "kind": "Role", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "priorityclasses", + "singularName": "priorityclass", + "namespaced": false, + "group": "scheduling.k8s.io", + "version": "v1", + "kind": "PriorityClass", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "pc" + ] + }, + { + "name": "csidrivers", + "singularName": "csidriver", + "namespaced": false, + "group": "storage.k8s.io", + "version": "v1", + "kind": "CSIDriver", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "csinodes", + "singularName": "csinode", + "namespaced": false, + "group": "storage.k8s.io", + "version": "v1", + "kind": "CSINode", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "csistoragecapacities", + "singularName": "csistoragecapacity", + "namespaced": true, + "group": "storage.k8s.io", + "version": "v1", + "kind": "CSIStorageCapacity", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + }, + { + "name": "storageclasses", + "singularName": "storageclass", + "namespaced": false, + "group": "storage.k8s.io", + "version": "v1", + "kind": "StorageClass", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ], + "shortNames": [ + "sc" + ] + }, + { + "name": "volumeattachments", + "singularName": "volumeattachment", + "namespaced": false, + "group": "storage.k8s.io", + "version": "v1", + "kind": "VolumeAttachment", + "verbs": [ + "create", + "delete", + "deletecollection", + "get", + "list", + "patch", + "update", + "watch" + ] + } + ] +} \ No newline at end of file diff --git a/default.nix b/default.nix new file mode 100644 index 0000000..55affdb --- /dev/null +++ b/default.nix @@ -0,0 +1,51 @@ +# default.nix +{ + pkgs ? import { }, + modules ? [ ./demo ], + specialArgs ? { }, + debug ? true, +}: +let + inherit (pkgs) lib; + attrIf = condition: content: if condition then content else { }; + + kubenix = builtins.fetchTree { + type = "github"; + owner = "hall"; + repo = "kubenix"; + ref = "e8577e661f3286624f79ce5ca3240a0ffcea9a7d"; + }; + + eval = lib.evalModules { + modules = [ + ./internal.nix + ./kubernetes.nix + ./helm.nix + ./kluctl.nix + ./validation.nix + ./importyaml.nix + ] + ++ modules; + specialArgs = { + inherit pkgs; + inherit (pkgs) lib; + helm = import "${kubenix}/lib/helm" { inherit pkgs; }; + } + // specialArgs; + }; +in +{ + inherit (eval.config.internal) + manifestAttrs + manifestJSON + manifestJSONFile + manifestYAML + manifestYAMLFile + ; + deploymentScript = eval.config.kluctl.script; + validationScript = eval.config.validation.script; +} +# Add debug attributes if debug is set +// (attrIf debug { + inherit pkgs lib eval; +}) diff --git a/demo/default.nix b/demo/default.nix new file mode 100644 index 0000000..f8abc32 --- /dev/null +++ b/demo/default.nix @@ -0,0 +1,6 @@ +{ ... }: +{ + imports = [ + ./helm.nix + ]; +} diff --git a/demo/helm.nix b/demo/helm.nix new file mode 100644 index 0000000..809c180 --- /dev/null +++ b/demo/helm.nix @@ -0,0 +1,27 @@ +# Helm example +{ helm, ... }: +{ + config = { + kubernetes.resources.kube-system.Secret.hcloud.data.token = ""; + helm.releases.hccm = { + namespace = "hccm-system"; + + chart = helm.fetch { + repo = "https://charts.hetzner.cloud"; + chart = "hcloud-cloud-controller-manager"; + version = "1.27.0"; + sha256 = "sha256-mzW5gQaRSPPKixaaJTTO+z7eQZcYhCRGulPnv9k/3Hg="; + }; + + values = { + env.HCLOUD_INSTANCES_ADDRESS_FAMILY.value = "dualstack"; + additionalTolerations = [ + { + key = "node.cilium.io/agent-not-ready"; + operator = "Exists"; + } + ]; + }; + }; + }; +} diff --git a/demo/resources.nix b/demo/resources.nix new file mode 100644 index 0000000..be3f1b6 --- /dev/null +++ b/demo/resources.nix @@ -0,0 +1,15 @@ +{ pkgs, ... }: +{ + config = { + kubernetes.resources.default.ConfigMap."my-app-config" = { + # No need to set namespace or name here + metadata = { + labels.app = "my-app"; + }; + stringData = { + "config.yaml" = "some-value"; + "storePath" = toString pkgs.fishMinimal; + }; + }; + }; +} diff --git a/helm.nix b/helm.nix new file mode 100644 index 0000000..011baab --- /dev/null +++ b/helm.nix @@ -0,0 +1,151 @@ +# helm defines kubenix module with options for using helm charts with kubenix +# Copied from hall/kubenix +{ + config, + lib, + pkgs, + helm, + ... +}: +with lib; +let + cfg = config.helm; +in +{ + options.helm = { + releases = mkOption { + description = "Attribute set of helm releases"; + type = types.attrsOf ( + types.submodule ( + { config, name, ... }: + { + options = { + name = mkOption { + description = "Helm release name"; + type = types.str; + default = name; + }; + + chart = mkOption { + description = "Helm chart to use"; + type = types.package; + }; + + namespace = mkOption { + description = "Namespace to install helm chart to"; + type = types.nullOr types.str; + default = null; + }; + + values = mkOption { + description = "Values to pass to chart"; + type = lib.types.anything; + default = { }; + }; + + kubeVersion = mkOption { + description = "Kubernetes version to build chart for"; + type = types.str; + # default = globalConfig.kubernetes.version; + default = "1.34.1"; + }; + + overrides = mkOption { + description = "Overrides to apply to all chart resources"; + type = types.listOf types.unspecified; + default = [ ]; + }; + + overrideNamespace = mkOption { + description = "Whether to apply namespace override"; + type = types.bool; + default = true; + }; + + includeCRDs = mkOption { + description = '' + Whether to include CRDs. + + Warning: Always including CRDs here is dangerous and can break CRs in your cluster as CRDs may be updated unintentionally. + An interactive `helm install` NEVER updates CRDs, only installs them when they are not existing. + See https://github.com/helm/community/blob/aa8e13054d91ee69857b13149a9652be09133a61/hips/hip-0011.md + + Only set this to true if you know what you are doing and are manually checking the included CRDs for breaking changes whenever updating the Helm chart. + ''; + type = types.bool; + default = false; + }; + + noHooks = mkOption { + description = '' + Wether to include Helm hooks. + + Without this all hooks run immediately on apply since we are bypassing the Helm CLI. + However, some charts only have minor validation hooks (e.g., upgrade version skew validation) and are safe to ignore. + ''; + type = types.bool; + default = false; + }; + + apiVersions = mkOption { + description = '' + Inform Helm about which CRDs are available in the cluster (`--api-versions` option). + This is useful for charts which contain `.Capabilities.APIVersions.Has` checks. + If you use `kubernetes.customTypes` to make kubenix aware of CRDs, it will include those as well by default. + ''; + type = types.listOf types.str; + default = [ ]; + }; + + objects = mkOption { + description = "Generated kubernetes objects"; + type = types.listOf types.attrs; + default = [ ]; + }; + }; + + config.overrides = mkIf (config.overrideNamespace && config.namespace != null) [ + { + metadata.namespace = config.namespace; + } + ]; + + config.objects = importJSON ( + helm.chart2json { + inherit (config) + chart + name + namespace + values + kubeVersion + includeCRDs + noHooks + apiVersions + ; + } + ); + } + ) + ); + default = { }; + }; + }; + + config = { + kubernetes.resources = mkMerge ( + flatten ( + mapAttrsToList ( + _: release: + map (object: { + ${object.metadata.namespace or "none"}.${object.kind}."${object.metadata.name}" = mkMerge ( + [ + object + ] + ++ release.overrides + ); + }) release.objects + ) cfg.releases + ) + ); + }; +} diff --git a/importyaml.nix b/importyaml.nix new file mode 100644 index 0000000..613d946 --- /dev/null +++ b/importyaml.nix @@ -0,0 +1,132 @@ +{ + config, + pkgs, + lib, + ... +}: +with lib; +let + cfg = config.importyaml; + /* + Recursively applies a function `f` to every non-list, non-attrset value + in a nested structure. The function `f` receives two arguments: + `f key value`, where `key` is the attribute name. For list elements, + `key` is null. + */ + deepMapWithKey = + f: + let + # Recursive helper function + recurse = + key: value: + if lib.isAttrs value then + lib.mapAttrs (name: val: recurse name val) value + else if lib.isList value then + # List elements don't have a key, so pass null + lib.map (elem: recurse null elem) value + else + # At a leaf, apply the user's function with the key and value + f key value; + in + # Start the recursion; the top-level value has no parent key + value: recurse null value; + + importyaml = types.submodule ( + { config, ... }: + { + options = { + src = mkOption { + description = "Should be either a derivation or URL for builtins.fetchTree"; + type = types.either types.package types.str; + }; + overrideNamespace = lib.mkOption { + description = "Override any attribute with name namespace DEEPLY"; + type = types.nullOr types.str; + default = null; + }; + overrides = mkOption { + description = "Overrides to apply to all resources"; + type = types.listOf types.anything; + default = [ ]; + }; + manifests = lib.mkOption { + type = types.listOf types.anything; + internal = true; + }; + }; + config = { + manifests = + let + src = + if isDerivation config.src then + config.src + else + builtins.fetchTree { + type = "file"; + url = config.src; + }; + + # Thanks kubenix + jsonFile = + pkgs.runCommandNoCCLocal "yaml2json" { } # bash + '' + # Remove null values + ${lib.getExe pkgs.yq} -Scs 'walk( + if type == "object" then + with_entries(select(.value != null)) + elif type == "array" then + map(select(. != null)) + else + . + end)' ${toString src} >$out + ''; + jsonStr = builtins.readFile jsonFile; + list = builtins.fromJSON jsonStr; + in + if config.overrideNamespace != null then + # Update attribute with name namespace through the entire manifest + lib.pipe list [ + # Special case for updating namespace resources + (lib.map ( + v: + if v.kind or null == "Namespace" then + lib.recursiveUpdate v { + metadata.name = config.overrideNamespace; + } + else + v + )) + # Recursively update anything called "namespace" with the new value + # There's no way to import a YAML with multiple namespace specs correctly. + (deepMapWithKey (n: v: if n == "namespace" then config.overrideNamespace else v)) + ] + else + list; + }; + } + ); +in +{ + options.importyaml = mkOption { + type = types.attrsOf importyaml; + default = { }; + }; + config = { + # Thanks kubenix for this voodoo magic function + kubernetes.resources = mkMerge ( + flatten ( + mapAttrsToList ( + _: yaml: + map (object: { + ${object.metadata.namespace or "none"}.${object.kind}."${object.metadata.name}" = mkMerge ( + [ + object + ] + ++ yaml.overrides + ); + }) yaml.manifests + ) cfg + ) + ); + }; +} diff --git a/internal.nix b/internal.nix new file mode 100644 index 0000000..fd37340 --- /dev/null +++ b/internal.nix @@ -0,0 +1,27 @@ +{ + config, + pkgs, + lib, + ... +}: +{ + options.internal = lib.mkOption { + type = lib.types.anything; + }; + config.internal = rec { + manifestAttrs = { + apiVersion = "v1"; + kind = "List"; + items = config.kubernetes.generated; + }; + manifestJSON = builtins.toJSON manifestAttrs; + manifestJSONFile = pkgs.writeText "manifest.json" manifestJSON; + # Beware that YAML rendering requires IFD + manifestYAML = builtins.readFile manifestYAMLFile; + manifestYAMLFile = + pkgs.runCommandNoCCLocal "manifest.yaml" { } # bash + '' + ${lib.getExe pkgs.yq} --yaml-output '.' ${manifestJSONFile} > $out + ''; + }; +} diff --git a/kluctl.nix b/kluctl.nix new file mode 100644 index 0000000..2a0a73a --- /dev/null +++ b/kluctl.nix @@ -0,0 +1,169 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.kluctl; + + writeMultipleFiles = + { + name, + files, + extraCommands ? "", + }: + let + fileList = lib.mapAttrsToList (path: file: { + inherit path; + content = file.content or file; + mode = if file.executable or false then "755" else file.mode or "644"; + }) files; + + # Create attribute names for passAsFile + passAsFileAttrs = builtins.listToAttrs ( + lib.imap0 (i: file: { + name = "file${toString i}"; + value = file.content; + }) fileList + ); + + passAsFileNames = builtins.attrNames passAsFileAttrs; + + commands = + (lib.imap0 (i: file: '' + mkdir -p $out/$(dirname "${file.path}") + cp "$file${toString i}Path" $out/${file.path} + chmod ${file.mode} $out/${file.path} + '') fileList) + ++ (lib.toList extraCommands); + + in + pkgs.runCommand name ( + passAsFileAttrs + // { + passAsFile = passAsFileNames; + } + ) (builtins.concatStringsSep "\n" commands); +in +{ + options = { + kluctl = { + package = lib.mkPackageOption pkgs "kluctl" { }; + discriminator = lib.mkOption { + type = lib.types.str; + description = "kluctl deployment label discriminator, AKA K8s label conformant project name"; + default = "easykubenix"; + }; + project = lib.mkOption { + type = lib.types.anything; + description = "Anything to be rendered into .kluctl.yaml"; + default = { + targets = [ { name = "local"; } ]; + }; + }; + resourcePriority = lib.mkOption { + type = lib.types.attrsOf lib.types.int; + description = "Priority of which order to apply resource types in"; + default = { + CustomResourceDefinition = 10; + Namespace = 20; + }; + }; + deployment = lib.mkOption { + type = lib.types.anything; + description = "Anything to be rendered into deployment.yaml"; + default = { }; + }; + files = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + description = "Attribute set where name is filename and value is string to be put into the file"; + default = { }; + }; + retryCount = lib.mkOption { + type = lib.types.int; + default = 3; + apply = toString; + }; + projectDir = lib.mkOption { + type = lib.types.package; + internal = true; + }; + script = lib.mkOption { + type = lib.types.package; + internal = true; + }; + }; + }; + config = { + kluctl.deployment = { + deployments = + (lib.pipe cfg.resourcePriority [ + lib.attrsToList + (lib.sort (a: b: a.value < b.value)) + (lib.map (v: { + path = v.name; + barrier = true; + })) + ]) + ++ [ + { + path = "default"; + } + ]; + }; + kluctl.projectDir = writeMultipleFiles { + name = "kluctlProject"; + files = { + ".kluctl.yaml" = { + content = builtins.toJSON config.kluctl.project; + }; + "deployment.yaml" = { + content = builtins.toJSON config.kluctl.deployment; + }; + # Don't apply prioritized resources again. + "default/easykubenix.yaml" = { + content = builtins.toJSON { + apiVersion = "v1"; + kind = "List"; + items = lib.filter ( + v: !lib.elem (v.kind or null) (lib.attrNames cfg.resourcePriority) + ) config.kubernetes.generated; + }; + }; + } + # Prioritized resources + // (lib.mapAttrs' (n: v: { + name = "${n}/easykubenix.yaml"; + value = builtins.toJSON { + apiVersion = "v1"; + kind = "List"; + items = lib.filter (v: v.kind or null == n) config.kubernetes.generated; + }; + }) cfg.resourcePriority) + # Other user-supplied files + // cfg.files; + }; + kluctl.script = + pkgs.writeScriptBin "kubenixDeploy" # fish + '' + #! ${lib.getExe pkgs.fishMinimal} + echo "Calling kluctl with ${cfg.retryCount} retries." + set command ${lib.getExe cfg.package} deploy \ + --no-update-check \ + --target local \ + --discriminator ${cfg.discriminator} \ + --project-dir ${cfg.projectDir} \ + $argv # --dry-run? --yes? --prune! + echo $command + + for i in (seq ${cfg.retryCount}) + $command && begin + echo "Success!" + exit 0 + end + end + exit $status + ''; + }; +} diff --git a/kubernetes.nix b/kubernetes.nix new file mode 100644 index 0000000..717c6ef --- /dev/null +++ b/kubernetes.nix @@ -0,0 +1,131 @@ +{ config, lib, ... }: +let + cfg = config.kubernetes; + + parseApiResources = + file: + let + data = builtins.fromJSON (builtins.readFile file); + resourceToAttr = resource: { + name = resource.kind; + value = + if !(resource ? group) || resource.group == "" then + resource.version + else + "${resource.group}/${resource.version}"; + }; + in + lib.listToAttrs (map resourceToAttr data.resources); + + resourceBody = lib.types.submodule ( + { name, ... }: + { + freeformType = lib.types.anything; + options = { + apiVersion = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Kubernetes API version. If null, it's inferred from `apiMappings`."; + }; + kind = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "Kubernetes resource kind. If null, it's inferred from the attribute path."; + }; + metadata = lib.mkOption { + type = lib.types.submodule { + freeformType = lib.types.anything; + options.name = lib.mkOption { + type = lib.types.str; + default = name; + }; + }; + default = { }; + description = "Kubernetes resource metadata."; + }; + }; + } + ); +in +{ + options.kubernetes = { + # In your options definition + resources = lib.mkOption { + type = lib.types.attrsOf (lib.types.attrsOf (lib.types.attrsOf resourceBody)); + default = { }; + description = "Kubernetes resources, grouped by namespace, then kind."; + }; + + apiMappings = lib.mkOption { + type = lib.types.attrsOf lib.types.str; + default = { }; + example = { + Cluster = "cluster.x-k8s.io/v1beta1"; + HCloudMachineTemplate = "infrastructure.cluster.x-k8s.io/v1beta1"; + HCloudRemediationTemplate = "infrastructure.cluster.x-k8s.io/v1beta1"; + HelmChartProxy = "addons.cluster.x-k8s.io/v1alpha1"; + HelmReleaseProxy = "addons.cluster.x-k8s.io/v1alpha1"; + HetznerCluster = "infrastructure.cluster.x-k8s.io/v1beta1"; + KubeadmConfigTemplate = "bootstrap.cluster.x-k8s.io/v1beta1"; + KubeadmControlPlane = "controlplane.cluster.x-k8s.io/v1beta1"; + MachineDeployment = "cluster.x-k8s.io/v1beta1"; + MachineHealthCheck = "cluster.x-k8s.io/v1beta1"; + }; + description = "Map of Kind to apiVersion. Merged with mappings from `apiMappingFile`."; + }; + + apiMappingFile = lib.mkOption { + type = lib.types.path; + default = ./apiResources/v1.33.json; + description = '' + A JSON file to extend apiMappings. + Generated by calling `kubectl api-resources --output=json > mappings.json` + ''; + }; + + generated = lib.mkOption { + type = lib.types.anything; + internal = true; + description = "The final, generated Kubernetes list object."; + }; + }; + + config.kubernetes = { + # The module system merges this definition with the default and any user-provided values. + # Values from the file will override the defaults. + apiMappings = parseApiResources cfg.apiMappingFile; + + generated = + let + cleanedResources = lib.filterAttrsRecursive (n: v: v != null) cfg.resources; + mappedResources = lib.mapAttrs ( + namespace: resourcesByKind: + lib.mapAttrs ( + kind: resourcesByName: + lib.mapAttrs ( + name: attrs: + # Provide a base set of attributes. `recursiveUpdate` will merge `attrs` on top, + # so any user-defined values for these fields will take precedence. + lib.recursiveUpdate ( + { + apiVersion = cfg.apiMappings.${kind}; + kind = kind; + metadata.name = name; + } + // lib.optionalAttrs (namespace != "none") { + metadata.namespace = namespace; + } + ) attrs + ) resourcesByName + ) resourcesByKind + ) cleanedResources; + + resourceList = lib.flatten ( + lib.mapAttrsToList ( + nsName: kinds: lib.mapAttrsToList (kindName: names: lib.attrValues names) kinds + ) mappedResources + ); + in + resourceList; + }; +} diff --git a/validation.nix b/validation.nix new file mode 100644 index 0000000..e3956c1 --- /dev/null +++ b/validation.nix @@ -0,0 +1,160 @@ +{ + config, + pkgs, + lib, + ... +}: +let + cfg = config.validation; + debugpipe = if (!cfg.debug) then "&>/dev/null" else ""; +in +{ + options.validation = { + debug = lib.mkEnableOption "validation debugging"; + etcdPackage = lib.mkPackageOption pkgs "etcd" { }; + kubernetesPackage = lib.mkPackageOption pkgs "kubernetes" { }; + kubeadmConfig = lib.mkOption { + type = lib.types.anything; + }; + podSubnet = lib.mkOption { + type = lib.types.str; + default = "10.97.0.0/16"; + }; + serviceSubnet = lib.mkOption { + type = lib.types.str; + default = "10.96.0.0/16"; + }; + script = lib.mkOption { + type = lib.types.package; + internal = true; + }; + }; + config.validation = + let + cfgFile = pkgs.writeText "ClusterConfiguration.json" (builtins.toJSON cfg.kubeadmConfig); + in + { + # Set kubeadmConfig sane defaults + kubeadmConfig = lib.mapAttrsRecursive (n: v: lib.mkDefault v) { + apiVersion = "kubeadm.k8s.io/v1beta4"; + kind = "ClusterConfiguration"; + kubernetesVersion = "v${cfg.kubernetesPackage.version}"; + controlPlaneEndpoint = "$BIND_ADDRESS:$KUBERNETES_PORT"; + certificatesDir = "$CERT_DIR"; + networking = { + podSubnet = cfg.podSubnet; + serviceSubnet = cfg.serviceSubnet; + }; + }; + + script = + pkgs.writeScriptBin "kubeval" # fish + '' + #! ${pkgs.fishMinimal}/bin/fish + set --prepend PATH ${cfg.etcdPackage}/bin + set --prepend PATH ${cfg.kubernetesPackage}/bin + set --prepend PATH ${pkgs.retry}/bin + set TMPDIR (mktemp --directory --suffix=eknvalidation) + ${if cfg.debug then "echo TMPDIR: $TMPDIR" else ""} + set --export CERT_DIR $TMPDIR/pki + set KUBEADM_CONFIG $TMPDIR/kubeadm-config.json + set --export KUBECONFIG $TMPDIR/admin.conf + + function cleanup --on-event fish_exit + kill -9 $ETCD_PID 2>/dev/null || true + kill -9 $APISERVER_PID 2>/dev/null || true + ${if cfg.debug then "" else "rm -rf $TMPDIR"} + end + + function get_free_port + ${lib.getExe pkgs.python3Minimal} -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()' + end + + set --export BIND_ADDRESS 127.0.0.1 + set --export KUBERNETES_PORT $(get_free_port) + set ETCD_CLIENT_PORT $(get_free_port) + set ETCD_PEER_PORT $(get_free_port) + + ${lib.getExe pkgs.envsubst} -i ${cfgFile} > $KUBEADM_CONFIG + + echo initializing with kubeadm + kubeadm init phase certs all --config=$KUBEADM_CONFIG ${debugpipe} || exit 1 + kubeadm init phase kubeconfig admin --config=$KUBEADM_CONFIG --kubeconfig-dir=$TMPDIR ${debugpipe} || exit 1 + + echo "kubeadm initialized; starting etcd" + + set command etcd --data-dir=$TMPDIR/etcd-data \ + --name=default \ + --listen-client-urls=https://127.0.0.1:$ETCD_CLIENT_PORT \ + --advertise-client-urls=https://127.0.0.1:$ETCD_CLIENT_PORT \ + --listen-peer-urls=https://127.0.0.1:$ETCD_PEER_PORT \ + --initial-advertise-peer-urls=https://127.0.0.1:$ETCD_PEER_PORT \ + --initial-cluster=default=https://127.0.0.1:$ETCD_PEER_PORT \ + --client-cert-auth=true \ + --trusted-ca-file=$CERT_DIR/etcd/ca.crt \ + --cert-file=$CERT_DIR/etcd/server.crt \ + --key-file=$CERT_DIR/etcd/server.key \ + --peer-client-cert-auth=true \ + --peer-trusted-ca-file=$CERT_DIR/etcd/ca.crt \ + --peer-cert-file=$CERT_DIR/etcd/peer.crt \ + --peer-key-file=$CERT_DIR/etcd/peer.key \ + --log-level=error + ${if cfg.debug then "echo $command" else ""} + $command ${debugpipe} & + set ETCD_PID $last_pid + + retry \ + --times=10 \ + --delay=0 \ + --jitter=1 \ + -- \ + etcdctl \ + --endpoints=https://127.0.0.1:$ETCD_CLIENT_PORT \ + --cacert=$CERT_DIR/etcd/ca.crt \ + --cert=$CERT_DIR/etcd/healthcheck-client.crt \ + --key=$CERT_DIR/etcd/healthcheck-client.key \ + endpoint health ${debugpipe} || exit 1 + + echo "etcd is ready; starting kube-apiserver" + + set command kube-apiserver \ + --watch-cache=false \ + --anonymous-auth=false \ + --etcd-cafile=$CERT_DIR/etcd/ca.crt \ + --etcd-certfile=$CERT_DIR/apiserver-etcd-client.crt \ + --etcd-keyfile=$CERT_DIR/apiserver-etcd-client.key \ + --etcd-servers=https://127.0.0.1:$ETCD_CLIENT_PORT \ + --service-cluster-ip-range=10.96.0.0/12 \ + --bind-address=$BIND_ADDRESS \ + --secure-port=$KUBERNETES_PORT \ + --allow-privileged=true \ + --client-ca-file=$CERT_DIR/ca.crt \ + --kubelet-client-certificate=$CERT_DIR/apiserver-kubelet-client.crt \ + --kubelet-client-key=$CERT_DIR/apiserver-kubelet-client.key \ + --service-account-issuer=https://kubernetes.default.svc.cluster.local \ + --service-account-key-file=$CERT_DIR/sa.pub \ + --service-account-signing-key-file=$CERT_DIR/sa.key \ + --tls-cert-file=$CERT_DIR/apiserver.crt \ + --tls-private-key-file=$CERT_DIR/apiserver.key + ${if cfg.debug then "echo $command" else ""} + $command ${debugpipe} & + set APISERVER_PID $last_pid + + retry \ + --times=10 \ + --delay=0 \ + --jitter=1 \ + -- \ + kubectl \ + get \ + --raw \ + /healthz >/dev/null ${debugpipe} || exit 1 + + echo "kube-apiserver ready; applying manifest(s)" + + ${lib.getExe config.kluctl.script} --yes --no-wait && echo "Great success!" + + exit $status + ''; + }; +}