easykubenix/validation.nix
2025-10-21 16:09:15 +02:00

169 lines
6.5 KiB
Nix

{
config,
pkgs,
lib,
...
}:
let
cfg = config.validation;
settingsFormat = pkgs.formats.json { };
debugpipe = if (!cfg.debug) then "&>/dev/null" else "";
in
{
options.validation = {
debug = lib.mkEnableOption "validation debugging";
etcdPackage = lib.mkPackageOption pkgs "etcd" { };
kubeadmConfig = lib.mkOption {
type = settingsFormat.type;
};
podSubnet = lib.mkOption {
type = lib.types.str;
default = "10.97.0.0/16";
};
serviceSubnet = lib.mkOption {
type = lib.types.str;
default = "10.96.0.0/16";
};
script = lib.mkOption {
type = lib.types.package;
internal = true;
};
};
config.validation =
let
cfgFile = pkgs.writeText "ClusterConfiguration.json" (builtins.toJSON cfg.kubeadmConfig);
in
{
# Set kubeadmConfig sane defaults
kubeadmConfig = lib.mapAttrsRecursive (n: v: lib.mkDefault v) {
apiVersion = "kubeadm.k8s.io/v1beta4";
kind = "ClusterConfiguration";
kubernetesVersion = "v${config.kubernetes.package.version}";
controlPlaneEndpoint = "$BIND_ADDRESS:$KUBERNETES_PORT";
certificatesDir = "$CERT_DIR";
networking = {
podSubnet = cfg.podSubnet;
serviceSubnet = cfg.serviceSubnet;
};
};
script =
pkgs.writeScriptBin "kubeval" # fish
''
#! ${lib.getExe pkgs.fishMinimal}
set --prepend PATH ${cfg.etcdPackage}/bin
set --prepend PATH ${config.kubernetes.package}/bin
set --prepend PATH ${pkgs.retry}/bin
set TMPDIR (mktemp --directory --suffix=eknvalidation)
${if cfg.debug then "echo TMPDIR: $TMPDIR" else ""}
set --export CERT_DIR $TMPDIR/pki
set KUBEADM_CONFIG $TMPDIR/kubeadm-config.json
set --export KUBECONFIG $TMPDIR/admin.conf
function cleanup --on-event fish_exit
kill -9 $ETCD_PID 2>/dev/null || true
kill -9 $APISERVER_PID 2>/dev/null || true
${if cfg.debug then "" else "rm -rf $TMPDIR"}
end
function get_free_port
${lib.getExe pkgs.python3Minimal} -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'
end
set --export BIND_ADDRESS 127.0.0.1
set --export KUBERNETES_PORT $(get_free_port)
set ETCD_CLIENT_PORT $(get_free_port)
set ETCD_PEER_PORT $(get_free_port)
${lib.getExe pkgs.envsubst} -i ${cfgFile} > $KUBEADM_CONFIG
echo initializing with kubeadm
kubeadm init phase certs all --config=$KUBEADM_CONFIG ${debugpipe} || exit 1
kubeadm init phase kubeconfig admin --config=$KUBEADM_CONFIG --kubeconfig-dir=$TMPDIR ${debugpipe} || exit 1
echo "kubeadm initialized; starting etcd"
set command etcd --data-dir=$TMPDIR/etcd-data \
--name=default \
--listen-client-urls=https://127.0.0.1:$ETCD_CLIENT_PORT \
--advertise-client-urls=https://127.0.0.1:$ETCD_CLIENT_PORT \
--listen-peer-urls=https://127.0.0.1:$ETCD_PEER_PORT \
--initial-advertise-peer-urls=https://127.0.0.1:$ETCD_PEER_PORT \
--initial-cluster=default=https://127.0.0.1:$ETCD_PEER_PORT \
--client-cert-auth=true \
--trusted-ca-file=$CERT_DIR/etcd/ca.crt \
--cert-file=$CERT_DIR/etcd/server.crt \
--key-file=$CERT_DIR/etcd/server.key \
--peer-client-cert-auth=true \
--peer-trusted-ca-file=$CERT_DIR/etcd/ca.crt \
--peer-cert-file=$CERT_DIR/etcd/peer.crt \
--peer-key-file=$CERT_DIR/etcd/peer.key \
--log-level=error
${if cfg.debug then "echo $command" else ""}
$command ${debugpipe} &
set ETCD_PID $last_pid
retry \
--times=10 \
--delay=0 \
--jitter=1 \
-- \
etcdctl \
--endpoints=https://127.0.0.1:$ETCD_CLIENT_PORT \
--cacert=$CERT_DIR/etcd/ca.crt \
--cert=$CERT_DIR/etcd/healthcheck-client.crt \
--key=$CERT_DIR/etcd/healthcheck-client.key \
endpoint health ${debugpipe} || exit 1
echo "etcd is ready; starting kube-apiserver"
set command kube-apiserver \
--watch-cache=false \
--anonymous-auth=false \
--etcd-cafile=$CERT_DIR/etcd/ca.crt \
--etcd-certfile=$CERT_DIR/apiserver-etcd-client.crt \
--etcd-keyfile=$CERT_DIR/apiserver-etcd-client.key \
--etcd-servers=https://127.0.0.1:$ETCD_CLIENT_PORT \
--service-cluster-ip-range=10.96.0.0/12 \
--bind-address=$BIND_ADDRESS \
--secure-port=$KUBERNETES_PORT \
--allow-privileged=true \
--client-ca-file=$CERT_DIR/ca.crt \
--kubelet-client-certificate=$CERT_DIR/apiserver-kubelet-client.crt \
--kubelet-client-key=$CERT_DIR/apiserver-kubelet-client.key \
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
--service-account-key-file=$CERT_DIR/sa.pub \
--service-account-signing-key-file=$CERT_DIR/sa.key \
--tls-cert-file=$CERT_DIR/apiserver.crt \
--tls-private-key-file=$CERT_DIR/apiserver.key
${if cfg.debug then "echo $command" else ""}
$command ${debugpipe} &
set APISERVER_PID $last_pid
retry \
--times=10 \
--delay=0 \
--jitter=1 \
-- \
kubectl \
get \
--raw \
/healthz >/dev/null ${debugpipe} || exit 1
echo "kube-apiserver ready; applying manifest(s)"
${lib.getExe config.kluctl.script} --yes --no-wait || begin
echo "kluctl deploy failed"
exit 1
end
echo "dumping openapiv2 schema from apiserver"
kubectl get --raw /openapi/v2 > $TMPDIR/k8s-schema.json
echo "running kubeconform"
${lib.getExe pkgs.kubeconform} -schema-location $TMPDIR/k8s-schema.json -summary < ${config.internal.manifestJSONFile} || begin
echo "kubeconform verification failed"
exit 1
end
echo "Your manifests are as valid as they can be against Kubernetes ${config.kubernetes.package.version}"
'';
};
}