diff --git a/playbook/playbook.yaml b/playbook/playbook.yaml index a1aca2f4..aaf153c8 100644 --- a/playbook/playbook.yaml +++ b/playbook/playbook.yaml @@ -33,3 +33,4 @@ with_items: "{{ repositories }}" loop_control: loop_var: operator + index_var: operator_index diff --git a/template/Makefile.j2 b/template/Makefile.j2 index aad3ff34..4609a35a 100644 --- a/template/Makefile.j2 +++ b/template/Makefile.j2 @@ -73,3 +73,6 @@ regenerate-charts: chart-clean compile-chart build: regenerate-charts helm-package docker-build publish: build docker-publish helm-publish + +run-dev: + nix run -f. tilt -- up --port {[5430 + operator_index}] \ No newline at end of file diff --git a/template/Tiltfile b/template/Tiltfile new file mode 100644 index 00000000..e10821a0 --- /dev/null +++ b/template/Tiltfile @@ -0,0 +1,31 @@ +default_registry("docker.stackable.tech/sandbox") + +meta = read_json('nix/meta.json') +operator_name = meta['operator']['name'] + +custom_build( + 'docker.stackable.tech/sandbox/' + operator_name, + 'nix shell -f . crate2nix -c crate2nix generate && nix-build . -A docker --argstr dockerName "${EXPECTED_REGISTRY}/' + operator_name + '" && ./result/load-image | docker load', + deps=['rust', 'Cargo.toml', 'Cargo.lock', 'default.nix', "nix", 'build.rs', 'vendor'], + # ignore=['result*', 'Cargo.nix', 'target', *.yaml], + outputs_image_ref_to='result/ref', +) + +# Load the latest CRDs from Nix +watch_file('result') +if os.path.exists('result'): + k8s_yaml('result/crds.yaml') + +# Exclude stale CRDs from Helm chart, and apply the rest +helm_crds, helm_non_crds = filter_yaml( + helm( + 'deploy/helm/' + operator_name, + name=operator_name, + set=[ + 'image.repository=docker.stackable.tech/sandbox/' + operator_name, + ], + ), + api_version = "^apiextensions\\.k8s\\.io/.*$", + kind = "^CustomResourceDefinition$", +) +k8s_yaml(helm_non_crds) diff --git a/template/default.nix b/template/default.nix new file mode 100644 index 00000000..3d60d36a --- /dev/null +++ b/template/default.nix @@ -0,0 +1,68 @@ +{ sources ? import ./nix/sources.nix # managed by https://github.com/nmattia/niv +, nixpkgs ? sources.nixpkgs +, pkgs ? import nixpkgs {} +, cargo ? import ./Cargo.nix { + inherit nixpkgs pkgs; release = false; + defaultCrateOverrides = pkgs.defaultCrateOverrides // { + prost-build = attrs: { + buildInputs = [ pkgs.protobuf ]; + }; + tonic-reflection = attrs: { + buildInputs = [ pkgs.rustfmt ]; + }; + }; + } +, meta ? pkgs.lib.importJSON ./nix/meta.json +, dockerName ? "docker.stackable.tech/sandbox/${meta.operator.name}" +, dockerTag ? null +}: +rec { + build = cargo.allWorkspaceMembers; + entrypoint = build+"/bin/stackable-${meta.operator.name}"; + crds = pkgs.runCommand "${meta.operator.name}-crds.yaml" {} + '' + ${entrypoint} crd > $out + ''; + + dockerImage = pkgs.dockerTools.streamLayeredImage { + name = dockerName; + tag = dockerTag; + contents = [ pkgs.bashInteractive pkgs.coreutils pkgs.util-linuxMinimal ]; + config = { + Env = + let + fileRefVars = { + PRODUCT_CONFIG = deploy/config-spec/properties.yaml; + }; + in lib.concatLists (lib.mapAttrsToList (env: path: lib.optional (lib.pathExists path) "${env}=${path}") fileRefVars); + Entrypoint = [ entrypoint ]; + Cmd = [ "run" ]; + }; + }; + docker = pkgs.linkFarm "listener-operator-docker" [ + { + name = "load-image"; + path = dockerImage; + } + { + name = "ref"; + path = pkgs.writeText "${dockerImage.name}-image-tag" "${dockerImage.imageName}:${dockerImage.imageTag}"; + } + { + name = "image-repo"; + path = pkgs.writeText "${dockerImage.name}-repo" dockerImage.imageName; + } + { + name = "image-tag"; + path = pkgs.writeText "${dockerImage.name}-tag" dockerImage.imageTag; + } + { + name = "crds.yaml"; + path = crds; + } + ]; + + # need to use vendored crate2nix because of https://github.com/kolloch/crate2nix/issues/264 + crate2nix = import sources.crate2nix {}; + tilt = pkgs.tilt; +} diff --git a/template/nix/meta.json.j2 b/template/nix/meta.json.j2 new file mode 100644 index 00000000..f93bdbf8 --- /dev/null +++ b/template/nix/meta.json.j2 @@ -0,0 +1 @@ +{[ {'operator': operator} | to_json }] diff --git a/template/nix/sources.json b/template/nix/sources.json new file mode 100644 index 00000000..6dc1f0e6 --- /dev/null +++ b/template/nix/sources.json @@ -0,0 +1,26 @@ +{ + "crate2nix": { + "branch": "master", + "description": "nix build file generator for rust crates", + "homepage": "", + "owner": "kolloch", + "repo": "crate2nix", + "rev": "45fc83132c8c91c77a1cd61fe0c945411d1edba8", + "sha256": "00w9qza56xh9m98308wzfbjf135gpz6sf7b5yfharvmax1k46c5q", + "type": "tarball", + "url": "https://github.com/kolloch/crate2nix/archive/45fc83132c8c91c77a1cd61fe0c945411d1edba8.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + }, + "nixpkgs": { + "branch": "nixpkgs-unstable", + "description": "Nix Packages collection", + "homepage": "", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "747927516efcb5e31ba03b7ff32f61f6d47e7d87", + "sha256": "1s4xabv59r99z8vd74w3r84kkxwqggqir3b0nh3ma04mni0m40gf", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/747927516efcb5e31ba03b7ff32f61f6d47e7d87.tar.gz", + "url_template": "https://github.com///archive/.tar.gz" + } +} diff --git a/template/nix/sources.nix b/template/nix/sources.nix new file mode 100644 index 00000000..9a01c8ac --- /dev/null +++ b/template/nix/sources.nix @@ -0,0 +1,194 @@ +# This file has been generated by Niv. + +let + + # + # The fetchers. fetch_ fetches specs of type . + # + + fetch_file = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchurl { inherit (spec) url sha256; name = name'; } + else + pkgs.fetchurl { inherit (spec) url sha256; name = name'; }; + + fetch_tarball = pkgs: name: spec: + let + name' = sanitizeName name + "-src"; + in + if spec.builtin or true then + builtins_fetchTarball { name = name'; inherit (spec) url sha256; } + else + pkgs.fetchzip { name = name'; inherit (spec) url sha256; }; + + fetch_git = name: spec: + let + ref = + if spec ? ref then spec.ref else + if spec ? branch then "refs/heads/${spec.branch}" else + if spec ? tag then "refs/tags/${spec.tag}" else + abort "In git source '${name}': Please specify `ref`, `tag` or `branch`!"; + submodules = if spec ? submodules then spec.submodules else false; + submoduleArg = + let + nixSupportsSubmodules = builtins.compareVersions builtins.nixVersion "2.4" >= 0; + emptyArgWithWarning = + if submodules == true + then + builtins.trace + ( + "The niv input \"${name}\" uses submodules " + + "but your nix's (${builtins.nixVersion}) builtins.fetchGit " + + "does not support them" + ) + {} + else {}; + in + if nixSupportsSubmodules + then { inherit submodules; } + else emptyArgWithWarning; + in + builtins.fetchGit + ({ url = spec.repo; inherit (spec) rev; inherit ref; } // submoduleArg); + + fetch_local = spec: spec.path; + + fetch_builtin-tarball = name: throw + ''[${name}] The niv type "builtin-tarball" is deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=tarball -a builtin=true''; + + fetch_builtin-url = name: throw + ''[${name}] The niv type "builtin-url" will soon be deprecated. You should instead use `builtin = true`. + $ niv modify ${name} -a type=file -a builtin=true''; + + # + # Various helpers + # + + # https://github.com/NixOS/nixpkgs/pull/83241/files#diff-c6f540a4f3bfa4b0e8b6bafd4cd54e8bR695 + sanitizeName = name: + ( + concatMapStrings (s: if builtins.isList s then "-" else s) + ( + builtins.split "[^[:alnum:]+._?=-]+" + ((x: builtins.elemAt (builtins.match "\\.*(.*)" x) 0) name) + ) + ); + + # The set of packages used when specs are fetched using non-builtins. + mkPkgs = sources: system: + let + sourcesNixpkgs = + import (builtins_fetchTarball { inherit (sources.nixpkgs) url sha256; }) { inherit system; }; + hasNixpkgsPath = builtins.any (x: x.prefix == "nixpkgs") builtins.nixPath; + hasThisAsNixpkgsPath = == ./.; + in + if builtins.hasAttr "nixpkgs" sources + then sourcesNixpkgs + else if hasNixpkgsPath && ! hasThisAsNixpkgsPath then + import {} + else + abort + '' + Please specify either (through -I or NIX_PATH=nixpkgs=...) or + add a package called "nixpkgs" to your sources.json. + ''; + + # The actual fetching function. + fetch = pkgs: name: spec: + + if ! builtins.hasAttr "type" spec then + abort "ERROR: niv spec ${name} does not have a 'type' attribute" + else if spec.type == "file" then fetch_file pkgs name spec + else if spec.type == "tarball" then fetch_tarball pkgs name spec + else if spec.type == "git" then fetch_git name spec + else if spec.type == "local" then fetch_local spec + else if spec.type == "builtin-tarball" then fetch_builtin-tarball name + else if spec.type == "builtin-url" then fetch_builtin-url name + else + abort "ERROR: niv spec ${name} has unknown type ${builtins.toJSON spec.type}"; + + # If the environment variable NIV_OVERRIDE_${name} is set, then use + # the path directly as opposed to the fetched source. + replace = name: drv: + let + saneName = stringAsChars (c: if isNull (builtins.match "[a-zA-Z0-9]" c) then "_" else c) name; + ersatz = builtins.getEnv "NIV_OVERRIDE_${saneName}"; + in + if ersatz == "" then drv else + # this turns the string into an actual Nix path (for both absolute and + # relative paths) + if builtins.substring 0 1 ersatz == "/" then /. + ersatz else /. + builtins.getEnv "PWD" + "/${ersatz}"; + + # Ports of functions for older nix versions + + # a Nix version of mapAttrs if the built-in doesn't exist + mapAttrs = builtins.mapAttrs or ( + f: set: with builtins; + listToAttrs (map (attr: { name = attr; value = f attr set.${attr}; }) (attrNames set)) + ); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/lists.nix#L295 + range = first: last: if first > last then [] else builtins.genList (n: first + n) (last - first + 1); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L257 + stringToCharacters = s: map (p: builtins.substring p 1 s) (range 0 (builtins.stringLength s - 1)); + + # https://github.com/NixOS/nixpkgs/blob/0258808f5744ca980b9a1f24fe0b1e6f0fecee9c/lib/strings.nix#L269 + stringAsChars = f: s: concatStrings (map f (stringToCharacters s)); + concatMapStrings = f: list: concatStrings (map f list); + concatStrings = builtins.concatStringsSep ""; + + # https://github.com/NixOS/nixpkgs/blob/8a9f58a375c401b96da862d969f66429def1d118/lib/attrsets.nix#L331 + optionalAttrs = cond: as: if cond then as else {}; + + # fetchTarball version that is compatible between all the versions of Nix + builtins_fetchTarball = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchTarball; + in + if lessThan nixVersion "1.12" then + fetchTarball ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchTarball attrs; + + # fetchurl version that is compatible between all the versions of Nix + builtins_fetchurl = { url, name ? null, sha256 }@attrs: + let + inherit (builtins) lessThan nixVersion fetchurl; + in + if lessThan nixVersion "1.12" then + fetchurl ({ inherit url; } // (optionalAttrs (!isNull name) { inherit name; })) + else + fetchurl attrs; + + # Create the final "sources" from the config + mkSources = config: + mapAttrs ( + name: spec: + if builtins.hasAttr "outPath" spec + then abort + "The values in sources.json should not have an 'outPath' attribute" + else + spec // { outPath = replace name (fetch config.pkgs name spec); } + ) config.sources; + + # The "config" used by the fetchers + mkConfig = + { sourcesFile ? if builtins.pathExists ./sources.json then ./sources.json else null + , sources ? if isNull sourcesFile then {} else builtins.fromJSON (builtins.readFile sourcesFile) + , system ? builtins.currentSystem + , pkgs ? mkPkgs sources system + }: rec { + # The sources, i.e. the attribute set of spec name to spec + inherit sources; + + # The "pkgs" (evaluated nixpkgs) to use for e.g. non-builtin fetchers + inherit pkgs; + }; + +in +mkSources (mkConfig {}) // { __functor = _: settings: mkSources (mkConfig settings); }