Initial commit
This commit is contained in:
commit
3103a2ad4c
12 changed files with 2099 additions and 0 deletions
80
README.md
Normal file
80
README.md
Normal file
|
|
@ -0,0 +1,80 @@
|
|||
# easykubenix
|
||||
|
||||
`easykubenix` uses the NixOS module system to generate Kubernetes manifests. It
|
||||
avoids generating Nix types for the entire Kubernetes API, resulting in faster
|
||||
evaluations and a simpler user experience compared to alternatives.
|
||||
|
||||
Manifest validation is performed by a script that applies the generated
|
||||
resources against an ephemeral `etcd` and `kube-apiserver` instance. This
|
||||
approach uses the Kubernetes API server as the single source of truth for
|
||||
validation.
|
||||
|
||||
## Usage
|
||||
|
||||
Define your resources using the NixOS module system. The top-level attribute
|
||||
is `kubernetes`, followed by the resource `kind`, then the resource name.
|
||||
|
||||
```nix
|
||||
# my-manifests.nix
|
||||
{
|
||||
kubernetes.ConfigMap.my-awesome-configmap = {
|
||||
metadata.namespace = "mynamespace123";
|
||||
data."config.json" = builtins.toJSON { key = "value"; };
|
||||
};
|
||||
|
||||
kubernetes.Deployment.my-app = {
|
||||
metadata.name = "my-app";
|
||||
spec.replicas = 3;
|
||||
# ...
|
||||
};
|
||||
}
|
||||
```
|
||||
|
||||
To generate the final YAML manifests, import your modules into the provided
|
||||
`eval` function.
|
||||
|
||||
```nix
|
||||
# default.nix
|
||||
{ pkgs ? import <nixpkgs> {} }:
|
||||
|
||||
(import <easykubenix> {
|
||||
inherit pkgs;
|
||||
modules = [ ./my-manifests.nix ];
|
||||
}).eval
|
||||
```
|
||||
|
||||
## Features
|
||||
|
||||
### Manifest Validation
|
||||
|
||||
To validate your manifests against a real Kubernetes API server without
|
||||
affecting a live cluster, run the validation script.
|
||||
|
||||
```sh
|
||||
nix run --file $youreasynix easykubenix.validator
|
||||
```
|
||||
|
||||
This command builds your manifests, spins up a temporary API server, and
|
||||
applies the configuration to it, reporting any errors from `kubectl`.
|
||||
|
||||
### Helm Chart Rendering
|
||||
|
||||
`easykubenix` can render Helm charts and import their resources into the NixOS
|
||||
module system. This allows you to override values from rendered charts using
|
||||
standard module system functions like `lib.mkForce`.
|
||||
|
||||
The import is performed via Import From Derivation (IFD), which is necessary
|
||||
as it requires running `helm template` during Nix evaluation.
|
||||
|
||||
### Kluctl Integration
|
||||
|
||||
`kluctl` is a CLI and GitOps tool that deploys manifests. It's distinctive
|
||||
feature is that it adds a label (discriminator) to every resource deployed which
|
||||
means when applying manifests with --prune it scans the watch-cache for resources
|
||||
with this label and removes them if they're not in the manifest we're applying.
|
||||
|
||||
Essentially kubectl apply --prune -l on steroids. It also integrates SOPS
|
||||
which can be used to keep secrets out of the Nix store.
|
||||
|
||||
`easykubenix` supports generating a minimal kluctl project and deployment script.
|
||||
|
||||
1150
apiResources/v1.33.json
Normal file
1150
apiResources/v1.33.json
Normal file
File diff suppressed because it is too large
Load diff
51
default.nix
Normal file
51
default.nix
Normal file
|
|
@ -0,0 +1,51 @@
|
|||
# default.nix
|
||||
{
|
||||
pkgs ? import <nixpkgs> { },
|
||||
modules ? [ ./demo ],
|
||||
specialArgs ? { },
|
||||
debug ? true,
|
||||
}:
|
||||
let
|
||||
inherit (pkgs) lib;
|
||||
attrIf = condition: content: if condition then content else { };
|
||||
|
||||
kubenix = builtins.fetchTree {
|
||||
type = "github";
|
||||
owner = "hall";
|
||||
repo = "kubenix";
|
||||
ref = "e8577e661f3286624f79ce5ca3240a0ffcea9a7d";
|
||||
};
|
||||
|
||||
eval = lib.evalModules {
|
||||
modules = [
|
||||
./internal.nix
|
||||
./kubernetes.nix
|
||||
./helm.nix
|
||||
./kluctl.nix
|
||||
./validation.nix
|
||||
./importyaml.nix
|
||||
]
|
||||
++ modules;
|
||||
specialArgs = {
|
||||
inherit pkgs;
|
||||
inherit (pkgs) lib;
|
||||
helm = import "${kubenix}/lib/helm" { inherit pkgs; };
|
||||
}
|
||||
// specialArgs;
|
||||
};
|
||||
in
|
||||
{
|
||||
inherit (eval.config.internal)
|
||||
manifestAttrs
|
||||
manifestJSON
|
||||
manifestJSONFile
|
||||
manifestYAML
|
||||
manifestYAMLFile
|
||||
;
|
||||
deploymentScript = eval.config.kluctl.script;
|
||||
validationScript = eval.config.validation.script;
|
||||
}
|
||||
# Add debug attributes if debug is set
|
||||
// (attrIf debug {
|
||||
inherit pkgs lib eval;
|
||||
})
|
||||
6
demo/default.nix
Normal file
6
demo/default.nix
Normal file
|
|
@ -0,0 +1,6 @@
|
|||
{ ... }:
|
||||
{
|
||||
imports = [
|
||||
./helm.nix
|
||||
];
|
||||
}
|
||||
27
demo/helm.nix
Normal file
27
demo/helm.nix
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
# Helm example
|
||||
{ helm, ... }:
|
||||
{
|
||||
config = {
|
||||
kubernetes.resources.kube-system.Secret.hcloud.data.token = "";
|
||||
helm.releases.hccm = {
|
||||
namespace = "hccm-system";
|
||||
|
||||
chart = helm.fetch {
|
||||
repo = "https://charts.hetzner.cloud";
|
||||
chart = "hcloud-cloud-controller-manager";
|
||||
version = "1.27.0";
|
||||
sha256 = "sha256-mzW5gQaRSPPKixaaJTTO+z7eQZcYhCRGulPnv9k/3Hg=";
|
||||
};
|
||||
|
||||
values = {
|
||||
env.HCLOUD_INSTANCES_ADDRESS_FAMILY.value = "dualstack";
|
||||
additionalTolerations = [
|
||||
{
|
||||
key = "node.cilium.io/agent-not-ready";
|
||||
operator = "Exists";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
15
demo/resources.nix
Normal file
15
demo/resources.nix
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
{ pkgs, ... }:
|
||||
{
|
||||
config = {
|
||||
kubernetes.resources.default.ConfigMap."my-app-config" = {
|
||||
# No need to set namespace or name here
|
||||
metadata = {
|
||||
labels.app = "my-app";
|
||||
};
|
||||
stringData = {
|
||||
"config.yaml" = "some-value";
|
||||
"storePath" = toString pkgs.fishMinimal;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
||||
151
helm.nix
Normal file
151
helm.nix
Normal file
|
|
@ -0,0 +1,151 @@
|
|||
# helm defines kubenix module with options for using helm charts with kubenix
|
||||
# Copied from hall/kubenix
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
helm,
|
||||
...
|
||||
}:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.helm;
|
||||
in
|
||||
{
|
||||
options.helm = {
|
||||
releases = mkOption {
|
||||
description = "Attribute set of helm releases";
|
||||
type = types.attrsOf (
|
||||
types.submodule (
|
||||
{ config, name, ... }:
|
||||
{
|
||||
options = {
|
||||
name = mkOption {
|
||||
description = "Helm release name";
|
||||
type = types.str;
|
||||
default = name;
|
||||
};
|
||||
|
||||
chart = mkOption {
|
||||
description = "Helm chart to use";
|
||||
type = types.package;
|
||||
};
|
||||
|
||||
namespace = mkOption {
|
||||
description = "Namespace to install helm chart to";
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
|
||||
values = mkOption {
|
||||
description = "Values to pass to chart";
|
||||
type = lib.types.anything;
|
||||
default = { };
|
||||
};
|
||||
|
||||
kubeVersion = mkOption {
|
||||
description = "Kubernetes version to build chart for";
|
||||
type = types.str;
|
||||
# default = globalConfig.kubernetes.version;
|
||||
default = "1.34.1";
|
||||
};
|
||||
|
||||
overrides = mkOption {
|
||||
description = "Overrides to apply to all chart resources";
|
||||
type = types.listOf types.unspecified;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
overrideNamespace = mkOption {
|
||||
description = "Whether to apply namespace override";
|
||||
type = types.bool;
|
||||
default = true;
|
||||
};
|
||||
|
||||
includeCRDs = mkOption {
|
||||
description = ''
|
||||
Whether to include CRDs.
|
||||
|
||||
Warning: Always including CRDs here is dangerous and can break CRs in your cluster as CRDs may be updated unintentionally.
|
||||
An interactive `helm install` NEVER updates CRDs, only installs them when they are not existing.
|
||||
See https://github.com/helm/community/blob/aa8e13054d91ee69857b13149a9652be09133a61/hips/hip-0011.md
|
||||
|
||||
Only set this to true if you know what you are doing and are manually checking the included CRDs for breaking changes whenever updating the Helm chart.
|
||||
'';
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
|
||||
noHooks = mkOption {
|
||||
description = ''
|
||||
Wether to include Helm hooks.
|
||||
|
||||
Without this all hooks run immediately on apply since we are bypassing the Helm CLI.
|
||||
However, some charts only have minor validation hooks (e.g., upgrade version skew validation) and are safe to ignore.
|
||||
'';
|
||||
type = types.bool;
|
||||
default = false;
|
||||
};
|
||||
|
||||
apiVersions = mkOption {
|
||||
description = ''
|
||||
Inform Helm about which CRDs are available in the cluster (`--api-versions` option).
|
||||
This is useful for charts which contain `.Capabilities.APIVersions.Has` checks.
|
||||
If you use `kubernetes.customTypes` to make kubenix aware of CRDs, it will include those as well by default.
|
||||
'';
|
||||
type = types.listOf types.str;
|
||||
default = [ ];
|
||||
};
|
||||
|
||||
objects = mkOption {
|
||||
description = "Generated kubernetes objects";
|
||||
type = types.listOf types.attrs;
|
||||
default = [ ];
|
||||
};
|
||||
};
|
||||
|
||||
config.overrides = mkIf (config.overrideNamespace && config.namespace != null) [
|
||||
{
|
||||
metadata.namespace = config.namespace;
|
||||
}
|
||||
];
|
||||
|
||||
config.objects = importJSON (
|
||||
helm.chart2json {
|
||||
inherit (config)
|
||||
chart
|
||||
name
|
||||
namespace
|
||||
values
|
||||
kubeVersion
|
||||
includeCRDs
|
||||
noHooks
|
||||
apiVersions
|
||||
;
|
||||
}
|
||||
);
|
||||
}
|
||||
)
|
||||
);
|
||||
default = { };
|
||||
};
|
||||
};
|
||||
|
||||
config = {
|
||||
kubernetes.resources = mkMerge (
|
||||
flatten (
|
||||
mapAttrsToList (
|
||||
_: release:
|
||||
map (object: {
|
||||
${object.metadata.namespace or "none"}.${object.kind}."${object.metadata.name}" = mkMerge (
|
||||
[
|
||||
object
|
||||
]
|
||||
++ release.overrides
|
||||
);
|
||||
}) release.objects
|
||||
) cfg.releases
|
||||
)
|
||||
);
|
||||
};
|
||||
}
|
||||
132
importyaml.nix
Normal file
132
importyaml.nix
Normal file
|
|
@ -0,0 +1,132 @@
|
|||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
with lib;
|
||||
let
|
||||
cfg = config.importyaml;
|
||||
/*
|
||||
Recursively applies a function `f` to every non-list, non-attrset value
|
||||
in a nested structure. The function `f` receives two arguments:
|
||||
`f key value`, where `key` is the attribute name. For list elements,
|
||||
`key` is null.
|
||||
*/
|
||||
deepMapWithKey =
|
||||
f:
|
||||
let
|
||||
# Recursive helper function
|
||||
recurse =
|
||||
key: value:
|
||||
if lib.isAttrs value then
|
||||
lib.mapAttrs (name: val: recurse name val) value
|
||||
else if lib.isList value then
|
||||
# List elements don't have a key, so pass null
|
||||
lib.map (elem: recurse null elem) value
|
||||
else
|
||||
# At a leaf, apply the user's function with the key and value
|
||||
f key value;
|
||||
in
|
||||
# Start the recursion; the top-level value has no parent key
|
||||
value: recurse null value;
|
||||
|
||||
importyaml = types.submodule (
|
||||
{ config, ... }:
|
||||
{
|
||||
options = {
|
||||
src = mkOption {
|
||||
description = "Should be either a derivation or URL for builtins.fetchTree";
|
||||
type = types.either types.package types.str;
|
||||
};
|
||||
overrideNamespace = lib.mkOption {
|
||||
description = "Override any attribute with name namespace DEEPLY";
|
||||
type = types.nullOr types.str;
|
||||
default = null;
|
||||
};
|
||||
overrides = mkOption {
|
||||
description = "Overrides to apply to all resources";
|
||||
type = types.listOf types.anything;
|
||||
default = [ ];
|
||||
};
|
||||
manifests = lib.mkOption {
|
||||
type = types.listOf types.anything;
|
||||
internal = true;
|
||||
};
|
||||
};
|
||||
config = {
|
||||
manifests =
|
||||
let
|
||||
src =
|
||||
if isDerivation config.src then
|
||||
config.src
|
||||
else
|
||||
builtins.fetchTree {
|
||||
type = "file";
|
||||
url = config.src;
|
||||
};
|
||||
|
||||
# Thanks kubenix
|
||||
jsonFile =
|
||||
pkgs.runCommandNoCCLocal "yaml2json" { } # bash
|
||||
''
|
||||
# Remove null values
|
||||
${lib.getExe pkgs.yq} -Scs 'walk(
|
||||
if type == "object" then
|
||||
with_entries(select(.value != null))
|
||||
elif type == "array" then
|
||||
map(select(. != null))
|
||||
else
|
||||
.
|
||||
end)' ${toString src} >$out
|
||||
'';
|
||||
jsonStr = builtins.readFile jsonFile;
|
||||
list = builtins.fromJSON jsonStr;
|
||||
in
|
||||
if config.overrideNamespace != null then
|
||||
# Update attribute with name namespace through the entire manifest
|
||||
lib.pipe list [
|
||||
# Special case for updating namespace resources
|
||||
(lib.map (
|
||||
v:
|
||||
if v.kind or null == "Namespace" then
|
||||
lib.recursiveUpdate v {
|
||||
metadata.name = config.overrideNamespace;
|
||||
}
|
||||
else
|
||||
v
|
||||
))
|
||||
# Recursively update anything called "namespace" with the new value
|
||||
# There's no way to import a YAML with multiple namespace specs correctly.
|
||||
(deepMapWithKey (n: v: if n == "namespace" then config.overrideNamespace else v))
|
||||
]
|
||||
else
|
||||
list;
|
||||
};
|
||||
}
|
||||
);
|
||||
in
|
||||
{
|
||||
options.importyaml = mkOption {
|
||||
type = types.attrsOf importyaml;
|
||||
default = { };
|
||||
};
|
||||
config = {
|
||||
# Thanks kubenix for this voodoo magic function
|
||||
kubernetes.resources = mkMerge (
|
||||
flatten (
|
||||
mapAttrsToList (
|
||||
_: yaml:
|
||||
map (object: {
|
||||
${object.metadata.namespace or "none"}.${object.kind}."${object.metadata.name}" = mkMerge (
|
||||
[
|
||||
object
|
||||
]
|
||||
++ yaml.overrides
|
||||
);
|
||||
}) yaml.manifests
|
||||
) cfg
|
||||
)
|
||||
);
|
||||
};
|
||||
}
|
||||
27
internal.nix
Normal file
27
internal.nix
Normal file
|
|
@ -0,0 +1,27 @@
|
|||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
{
|
||||
options.internal = lib.mkOption {
|
||||
type = lib.types.anything;
|
||||
};
|
||||
config.internal = rec {
|
||||
manifestAttrs = {
|
||||
apiVersion = "v1";
|
||||
kind = "List";
|
||||
items = config.kubernetes.generated;
|
||||
};
|
||||
manifestJSON = builtins.toJSON manifestAttrs;
|
||||
manifestJSONFile = pkgs.writeText "manifest.json" manifestJSON;
|
||||
# Beware that YAML rendering requires IFD
|
||||
manifestYAML = builtins.readFile manifestYAMLFile;
|
||||
manifestYAMLFile =
|
||||
pkgs.runCommandNoCCLocal "manifest.yaml" { } # bash
|
||||
''
|
||||
${lib.getExe pkgs.yq} --yaml-output '.' ${manifestJSONFile} > $out
|
||||
'';
|
||||
};
|
||||
}
|
||||
169
kluctl.nix
Normal file
169
kluctl.nix
Normal file
|
|
@ -0,0 +1,169 @@
|
|||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.kluctl;
|
||||
|
||||
writeMultipleFiles =
|
||||
{
|
||||
name,
|
||||
files,
|
||||
extraCommands ? "",
|
||||
}:
|
||||
let
|
||||
fileList = lib.mapAttrsToList (path: file: {
|
||||
inherit path;
|
||||
content = file.content or file;
|
||||
mode = if file.executable or false then "755" else file.mode or "644";
|
||||
}) files;
|
||||
|
||||
# Create attribute names for passAsFile
|
||||
passAsFileAttrs = builtins.listToAttrs (
|
||||
lib.imap0 (i: file: {
|
||||
name = "file${toString i}";
|
||||
value = file.content;
|
||||
}) fileList
|
||||
);
|
||||
|
||||
passAsFileNames = builtins.attrNames passAsFileAttrs;
|
||||
|
||||
commands =
|
||||
(lib.imap0 (i: file: ''
|
||||
mkdir -p $out/$(dirname "${file.path}")
|
||||
cp "$file${toString i}Path" $out/${file.path}
|
||||
chmod ${file.mode} $out/${file.path}
|
||||
'') fileList)
|
||||
++ (lib.toList extraCommands);
|
||||
|
||||
in
|
||||
pkgs.runCommand name (
|
||||
passAsFileAttrs
|
||||
// {
|
||||
passAsFile = passAsFileNames;
|
||||
}
|
||||
) (builtins.concatStringsSep "\n" commands);
|
||||
in
|
||||
{
|
||||
options = {
|
||||
kluctl = {
|
||||
package = lib.mkPackageOption pkgs "kluctl" { };
|
||||
discriminator = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
description = "kluctl deployment label discriminator, AKA K8s label conformant project name";
|
||||
default = "easykubenix";
|
||||
};
|
||||
project = lib.mkOption {
|
||||
type = lib.types.anything;
|
||||
description = "Anything to be rendered into .kluctl.yaml";
|
||||
default = {
|
||||
targets = [ { name = "local"; } ];
|
||||
};
|
||||
};
|
||||
resourcePriority = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.int;
|
||||
description = "Priority of which order to apply resource types in";
|
||||
default = {
|
||||
CustomResourceDefinition = 10;
|
||||
Namespace = 20;
|
||||
};
|
||||
};
|
||||
deployment = lib.mkOption {
|
||||
type = lib.types.anything;
|
||||
description = "Anything to be rendered into deployment.yaml";
|
||||
default = { };
|
||||
};
|
||||
files = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
description = "Attribute set where name is filename and value is string to be put into the file";
|
||||
default = { };
|
||||
};
|
||||
retryCount = lib.mkOption {
|
||||
type = lib.types.int;
|
||||
default = 3;
|
||||
apply = toString;
|
||||
};
|
||||
projectDir = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
internal = true;
|
||||
};
|
||||
script = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
internal = true;
|
||||
};
|
||||
};
|
||||
};
|
||||
config = {
|
||||
kluctl.deployment = {
|
||||
deployments =
|
||||
(lib.pipe cfg.resourcePriority [
|
||||
lib.attrsToList
|
||||
(lib.sort (a: b: a.value < b.value))
|
||||
(lib.map (v: {
|
||||
path = v.name;
|
||||
barrier = true;
|
||||
}))
|
||||
])
|
||||
++ [
|
||||
{
|
||||
path = "default";
|
||||
}
|
||||
];
|
||||
};
|
||||
kluctl.projectDir = writeMultipleFiles {
|
||||
name = "kluctlProject";
|
||||
files = {
|
||||
".kluctl.yaml" = {
|
||||
content = builtins.toJSON config.kluctl.project;
|
||||
};
|
||||
"deployment.yaml" = {
|
||||
content = builtins.toJSON config.kluctl.deployment;
|
||||
};
|
||||
# Don't apply prioritized resources again.
|
||||
"default/easykubenix.yaml" = {
|
||||
content = builtins.toJSON {
|
||||
apiVersion = "v1";
|
||||
kind = "List";
|
||||
items = lib.filter (
|
||||
v: !lib.elem (v.kind or null) (lib.attrNames cfg.resourcePriority)
|
||||
) config.kubernetes.generated;
|
||||
};
|
||||
};
|
||||
}
|
||||
# Prioritized resources
|
||||
// (lib.mapAttrs' (n: v: {
|
||||
name = "${n}/easykubenix.yaml";
|
||||
value = builtins.toJSON {
|
||||
apiVersion = "v1";
|
||||
kind = "List";
|
||||
items = lib.filter (v: v.kind or null == n) config.kubernetes.generated;
|
||||
};
|
||||
}) cfg.resourcePriority)
|
||||
# Other user-supplied files
|
||||
// cfg.files;
|
||||
};
|
||||
kluctl.script =
|
||||
pkgs.writeScriptBin "kubenixDeploy" # fish
|
||||
''
|
||||
#! ${lib.getExe pkgs.fishMinimal}
|
||||
echo "Calling kluctl with ${cfg.retryCount} retries."
|
||||
set command ${lib.getExe cfg.package} deploy \
|
||||
--no-update-check \
|
||||
--target local \
|
||||
--discriminator ${cfg.discriminator} \
|
||||
--project-dir ${cfg.projectDir} \
|
||||
$argv # --dry-run? --yes? --prune!
|
||||
echo $command
|
||||
|
||||
for i in (seq ${cfg.retryCount})
|
||||
$command && begin
|
||||
echo "Success!"
|
||||
exit 0
|
||||
end
|
||||
end
|
||||
exit $status
|
||||
'';
|
||||
};
|
||||
}
|
||||
131
kubernetes.nix
Normal file
131
kubernetes.nix
Normal file
|
|
@ -0,0 +1,131 @@
|
|||
{ config, lib, ... }:
|
||||
let
|
||||
cfg = config.kubernetes;
|
||||
|
||||
parseApiResources =
|
||||
file:
|
||||
let
|
||||
data = builtins.fromJSON (builtins.readFile file);
|
||||
resourceToAttr = resource: {
|
||||
name = resource.kind;
|
||||
value =
|
||||
if !(resource ? group) || resource.group == "" then
|
||||
resource.version
|
||||
else
|
||||
"${resource.group}/${resource.version}";
|
||||
};
|
||||
in
|
||||
lib.listToAttrs (map resourceToAttr data.resources);
|
||||
|
||||
resourceBody = lib.types.submodule (
|
||||
{ name, ... }:
|
||||
{
|
||||
freeformType = lib.types.anything;
|
||||
options = {
|
||||
apiVersion = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Kubernetes API version. If null, it's inferred from `apiMappings`.";
|
||||
};
|
||||
kind = lib.mkOption {
|
||||
type = lib.types.nullOr lib.types.str;
|
||||
default = null;
|
||||
description = "Kubernetes resource kind. If null, it's inferred from the attribute path.";
|
||||
};
|
||||
metadata = lib.mkOption {
|
||||
type = lib.types.submodule {
|
||||
freeformType = lib.types.anything;
|
||||
options.name = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = name;
|
||||
};
|
||||
};
|
||||
default = { };
|
||||
description = "Kubernetes resource metadata.";
|
||||
};
|
||||
};
|
||||
}
|
||||
);
|
||||
in
|
||||
{
|
||||
options.kubernetes = {
|
||||
# In your options definition
|
||||
resources = lib.mkOption {
|
||||
type = lib.types.attrsOf (lib.types.attrsOf (lib.types.attrsOf resourceBody));
|
||||
default = { };
|
||||
description = "Kubernetes resources, grouped by namespace, then kind.";
|
||||
};
|
||||
|
||||
apiMappings = lib.mkOption {
|
||||
type = lib.types.attrsOf lib.types.str;
|
||||
default = { };
|
||||
example = {
|
||||
Cluster = "cluster.x-k8s.io/v1beta1";
|
||||
HCloudMachineTemplate = "infrastructure.cluster.x-k8s.io/v1beta1";
|
||||
HCloudRemediationTemplate = "infrastructure.cluster.x-k8s.io/v1beta1";
|
||||
HelmChartProxy = "addons.cluster.x-k8s.io/v1alpha1";
|
||||
HelmReleaseProxy = "addons.cluster.x-k8s.io/v1alpha1";
|
||||
HetznerCluster = "infrastructure.cluster.x-k8s.io/v1beta1";
|
||||
KubeadmConfigTemplate = "bootstrap.cluster.x-k8s.io/v1beta1";
|
||||
KubeadmControlPlane = "controlplane.cluster.x-k8s.io/v1beta1";
|
||||
MachineDeployment = "cluster.x-k8s.io/v1beta1";
|
||||
MachineHealthCheck = "cluster.x-k8s.io/v1beta1";
|
||||
};
|
||||
description = "Map of Kind to apiVersion. Merged with mappings from `apiMappingFile`.";
|
||||
};
|
||||
|
||||
apiMappingFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = ./apiResources/v1.33.json;
|
||||
description = ''
|
||||
A JSON file to extend apiMappings.
|
||||
Generated by calling `kubectl api-resources --output=json > mappings.json`
|
||||
'';
|
||||
};
|
||||
|
||||
generated = lib.mkOption {
|
||||
type = lib.types.anything;
|
||||
internal = true;
|
||||
description = "The final, generated Kubernetes list object.";
|
||||
};
|
||||
};
|
||||
|
||||
config.kubernetes = {
|
||||
# The module system merges this definition with the default and any user-provided values.
|
||||
# Values from the file will override the defaults.
|
||||
apiMappings = parseApiResources cfg.apiMappingFile;
|
||||
|
||||
generated =
|
||||
let
|
||||
cleanedResources = lib.filterAttrsRecursive (n: v: v != null) cfg.resources;
|
||||
mappedResources = lib.mapAttrs (
|
||||
namespace: resourcesByKind:
|
||||
lib.mapAttrs (
|
||||
kind: resourcesByName:
|
||||
lib.mapAttrs (
|
||||
name: attrs:
|
||||
# Provide a base set of attributes. `recursiveUpdate` will merge `attrs` on top,
|
||||
# so any user-defined values for these fields will take precedence.
|
||||
lib.recursiveUpdate (
|
||||
{
|
||||
apiVersion = cfg.apiMappings.${kind};
|
||||
kind = kind;
|
||||
metadata.name = name;
|
||||
}
|
||||
// lib.optionalAttrs (namespace != "none") {
|
||||
metadata.namespace = namespace;
|
||||
}
|
||||
) attrs
|
||||
) resourcesByName
|
||||
) resourcesByKind
|
||||
) cleanedResources;
|
||||
|
||||
resourceList = lib.flatten (
|
||||
lib.mapAttrsToList (
|
||||
nsName: kinds: lib.mapAttrsToList (kindName: names: lib.attrValues names) kinds
|
||||
) mappedResources
|
||||
);
|
||||
in
|
||||
resourceList;
|
||||
};
|
||||
}
|
||||
160
validation.nix
Normal file
160
validation.nix
Normal file
|
|
@ -0,0 +1,160 @@
|
|||
{
|
||||
config,
|
||||
pkgs,
|
||||
lib,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.validation;
|
||||
debugpipe = if (!cfg.debug) then "&>/dev/null" else "";
|
||||
in
|
||||
{
|
||||
options.validation = {
|
||||
debug = lib.mkEnableOption "validation debugging";
|
||||
etcdPackage = lib.mkPackageOption pkgs "etcd" { };
|
||||
kubernetesPackage = lib.mkPackageOption pkgs "kubernetes" { };
|
||||
kubeadmConfig = lib.mkOption {
|
||||
type = lib.types.anything;
|
||||
};
|
||||
podSubnet = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.97.0.0/16";
|
||||
};
|
||||
serviceSubnet = lib.mkOption {
|
||||
type = lib.types.str;
|
||||
default = "10.96.0.0/16";
|
||||
};
|
||||
script = lib.mkOption {
|
||||
type = lib.types.package;
|
||||
internal = true;
|
||||
};
|
||||
};
|
||||
config.validation =
|
||||
let
|
||||
cfgFile = pkgs.writeText "ClusterConfiguration.json" (builtins.toJSON cfg.kubeadmConfig);
|
||||
in
|
||||
{
|
||||
# Set kubeadmConfig sane defaults
|
||||
kubeadmConfig = lib.mapAttrsRecursive (n: v: lib.mkDefault v) {
|
||||
apiVersion = "kubeadm.k8s.io/v1beta4";
|
||||
kind = "ClusterConfiguration";
|
||||
kubernetesVersion = "v${cfg.kubernetesPackage.version}";
|
||||
controlPlaneEndpoint = "$BIND_ADDRESS:$KUBERNETES_PORT";
|
||||
certificatesDir = "$CERT_DIR";
|
||||
networking = {
|
||||
podSubnet = cfg.podSubnet;
|
||||
serviceSubnet = cfg.serviceSubnet;
|
||||
};
|
||||
};
|
||||
|
||||
script =
|
||||
pkgs.writeScriptBin "kubeval" # fish
|
||||
''
|
||||
#! ${pkgs.fishMinimal}/bin/fish
|
||||
set --prepend PATH ${cfg.etcdPackage}/bin
|
||||
set --prepend PATH ${cfg.kubernetesPackage}/bin
|
||||
set --prepend PATH ${pkgs.retry}/bin
|
||||
set TMPDIR (mktemp --directory --suffix=eknvalidation)
|
||||
${if cfg.debug then "echo TMPDIR: $TMPDIR" else ""}
|
||||
set --export CERT_DIR $TMPDIR/pki
|
||||
set KUBEADM_CONFIG $TMPDIR/kubeadm-config.json
|
||||
set --export KUBECONFIG $TMPDIR/admin.conf
|
||||
|
||||
function cleanup --on-event fish_exit
|
||||
kill -9 $ETCD_PID 2>/dev/null || true
|
||||
kill -9 $APISERVER_PID 2>/dev/null || true
|
||||
${if cfg.debug then "" else "rm -rf $TMPDIR"}
|
||||
end
|
||||
|
||||
function get_free_port
|
||||
${lib.getExe pkgs.python3Minimal} -c 'import socket; s=socket.socket(); s.bind(("", 0)); print(s.getsockname()[1]); s.close()'
|
||||
end
|
||||
|
||||
set --export BIND_ADDRESS 127.0.0.1
|
||||
set --export KUBERNETES_PORT $(get_free_port)
|
||||
set ETCD_CLIENT_PORT $(get_free_port)
|
||||
set ETCD_PEER_PORT $(get_free_port)
|
||||
|
||||
${lib.getExe pkgs.envsubst} -i ${cfgFile} > $KUBEADM_CONFIG
|
||||
|
||||
echo initializing with kubeadm
|
||||
kubeadm init phase certs all --config=$KUBEADM_CONFIG ${debugpipe} || exit 1
|
||||
kubeadm init phase kubeconfig admin --config=$KUBEADM_CONFIG --kubeconfig-dir=$TMPDIR ${debugpipe} || exit 1
|
||||
|
||||
echo "kubeadm initialized; starting etcd"
|
||||
|
||||
set command etcd --data-dir=$TMPDIR/etcd-data \
|
||||
--name=default \
|
||||
--listen-client-urls=https://127.0.0.1:$ETCD_CLIENT_PORT \
|
||||
--advertise-client-urls=https://127.0.0.1:$ETCD_CLIENT_PORT \
|
||||
--listen-peer-urls=https://127.0.0.1:$ETCD_PEER_PORT \
|
||||
--initial-advertise-peer-urls=https://127.0.0.1:$ETCD_PEER_PORT \
|
||||
--initial-cluster=default=https://127.0.0.1:$ETCD_PEER_PORT \
|
||||
--client-cert-auth=true \
|
||||
--trusted-ca-file=$CERT_DIR/etcd/ca.crt \
|
||||
--cert-file=$CERT_DIR/etcd/server.crt \
|
||||
--key-file=$CERT_DIR/etcd/server.key \
|
||||
--peer-client-cert-auth=true \
|
||||
--peer-trusted-ca-file=$CERT_DIR/etcd/ca.crt \
|
||||
--peer-cert-file=$CERT_DIR/etcd/peer.crt \
|
||||
--peer-key-file=$CERT_DIR/etcd/peer.key \
|
||||
--log-level=error
|
||||
${if cfg.debug then "echo $command" else ""}
|
||||
$command ${debugpipe} &
|
||||
set ETCD_PID $last_pid
|
||||
|
||||
retry \
|
||||
--times=10 \
|
||||
--delay=0 \
|
||||
--jitter=1 \
|
||||
-- \
|
||||
etcdctl \
|
||||
--endpoints=https://127.0.0.1:$ETCD_CLIENT_PORT \
|
||||
--cacert=$CERT_DIR/etcd/ca.crt \
|
||||
--cert=$CERT_DIR/etcd/healthcheck-client.crt \
|
||||
--key=$CERT_DIR/etcd/healthcheck-client.key \
|
||||
endpoint health ${debugpipe} || exit 1
|
||||
|
||||
echo "etcd is ready; starting kube-apiserver"
|
||||
|
||||
set command kube-apiserver \
|
||||
--watch-cache=false \
|
||||
--anonymous-auth=false \
|
||||
--etcd-cafile=$CERT_DIR/etcd/ca.crt \
|
||||
--etcd-certfile=$CERT_DIR/apiserver-etcd-client.crt \
|
||||
--etcd-keyfile=$CERT_DIR/apiserver-etcd-client.key \
|
||||
--etcd-servers=https://127.0.0.1:$ETCD_CLIENT_PORT \
|
||||
--service-cluster-ip-range=10.96.0.0/12 \
|
||||
--bind-address=$BIND_ADDRESS \
|
||||
--secure-port=$KUBERNETES_PORT \
|
||||
--allow-privileged=true \
|
||||
--client-ca-file=$CERT_DIR/ca.crt \
|
||||
--kubelet-client-certificate=$CERT_DIR/apiserver-kubelet-client.crt \
|
||||
--kubelet-client-key=$CERT_DIR/apiserver-kubelet-client.key \
|
||||
--service-account-issuer=https://kubernetes.default.svc.cluster.local \
|
||||
--service-account-key-file=$CERT_DIR/sa.pub \
|
||||
--service-account-signing-key-file=$CERT_DIR/sa.key \
|
||||
--tls-cert-file=$CERT_DIR/apiserver.crt \
|
||||
--tls-private-key-file=$CERT_DIR/apiserver.key
|
||||
${if cfg.debug then "echo $command" else ""}
|
||||
$command ${debugpipe} &
|
||||
set APISERVER_PID $last_pid
|
||||
|
||||
retry \
|
||||
--times=10 \
|
||||
--delay=0 \
|
||||
--jitter=1 \
|
||||
-- \
|
||||
kubectl \
|
||||
get \
|
||||
--raw \
|
||||
/healthz >/dev/null ${debugpipe} || exit 1
|
||||
|
||||
echo "kube-apiserver ready; applying manifest(s)"
|
||||
|
||||
${lib.getExe config.kluctl.script} --yes --no-wait && echo "Great success!"
|
||||
|
||||
exit $status
|
||||
'';
|
||||
};
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue