treewide cleanups and refactoring for initial tests (#157)

- [x] refactor lib into separate files, similar to NixOS/nixpkgs/lib.
- [x] refactor ci to automatically generate derivations from flake outputs
- [x] remove cluttered indirection statements throughout the codebase
- [x] refactor hosts to allow for upcoming integration tests
- [x] improve ambiguity in the existing docs 
- [x] add [BORS](https://bors.tech) support
- [x] add initial integration test
- [x] write tests documentation
- [x] test lib
- [x] improve version string generation, and do so automatically for pkgs/flake.nix sources

Clean up the codebase as best we can in preparation for #152 and add tests. From now on, all PRs will be merged with BORS.

.envrc

@@ -1,2 +1,2 @@
-watch_file **/*.nix
+watch_file shell/* flake.nix
 use flake || use nix

README.md

@@ -1,29 +1,31 @@
-[![Build](https://img.shields.io/github/checks-status/divnix/devos/core)](https://hercules-ci.com/github/divnix/devos/jobs)
+[![Bors enabled](https://bors.tech/images/badge_small.svg)](https://app.bors.tech/repositories/32678)
 [![MIT License](https://img.shields.io/github/license/divnix/devos)][mit]
 [![NixOS 20.09](https://img.shields.io/badge/NixOS-v20.09-blue.svg?style=flat&logo=NixOS&logoColor=white)](https://nixos.org)
 
 > #### ⚠ Advisory ⚠
-> DevOS leverages the [flakes][flakes] feature available via an _experimental_
-> branch of [nix][nix]. Until nix 3.0 is released, this project should be
-> considered unstable, though quite usable as flakes have been maturing
-> _well_
-> [for a while](https://github.com/divnix/devos/tree/17713c22d07c54525c728c62060a0428b76dee3b).
+> DevOS requires the [flakes][flakes] feature available via an _experimental_
+> branch of [nix][nix]. Until nix 3.0 is released, this project
+> should be considered unstable, though quite usable as flakes have been
+> maturing _well_ [for a while](https://github.com/divnix/devos/tree/17713c22d07c54525c728c62060a0428b76dee3b).
 
 # Introduction
 DevOS grants a simple way to use, deploy and manage [NixOS][nixos] systems for
-personal and productive use. It does this by providing a convenient repository
-structure, integrating several popular projects like
-[home-manager][home-manager], and [devshell][devshell], and offering useful
-conveniences like
+personal and productive use. A sane repository structure is provided,
+integrating several popular projects like [home-manager][home-manager],
+[devshell][devshell], and [more](./doc/integrations).
+
+Stiving for ___nix first™___ solutions with unobstrusive implementations,
+a [flake centric][flake-doc] approach is taken for useful conveniences such as
 [automatic source updates](./pkgs#automatic-source-updates).
 
-Skip the indeterminate nature of other systems, _and_ the perceived difficulty
-of Nix. It's easier than you think!
+Skip the indeterminate nature of other systems, _and_ the perceived
+tedium of bootstrapping Nix. It's easier than you think!
 
-### Status
-Alpha. A lot of the implementation is less than perfect, and huge redesigns
-_will_ happen. There are unstable versions (0._x_._x_) to help users keep
-track of changes and progress.
+### Status: Alpha
+A lot of the implementation is less than perfect, and huge
+[redesigns](https://github.com/divnix/devos/issues/152) _will_ happen. There
+are unstable versions (0._x_._x_) to help users keep track of changes and
+progress.
 
 ## Getting Started
 Check out the [guide](https://devos.divnix.com/doc/start) to get up and running.
@@ -36,8 +38,8 @@ make critical comments about the [code][please]. 😜
 NixOS provides an amazing abstraction to manage our environment, but that new
 power can sometimes bring feelings of overwhelm and confusion. Having a turing
 complete system can easily lead to unlimited complexity if we do it wrong.
-Instead, we should have a community consensus on how to manage a NixOS system.
-Help us reach that goal!
+Instead, we should have a community consensus on how to manage a NixOS system
+and its satellite projects, from which best practices can evolve.
 
 ___The future is declarative! 🎉___
 
@@ -73,6 +75,7 @@ DevOS is licensed under the [MIT License][mit].
 [nixos]: https://nixos.org/manual/nixos/stable
 [home-manager]: https://nix-community.github.io/home-manager
 [flakes]: https://nixos.wiki/wiki/Flakes
+[flake-doc]: https://github.com/NixOS/nix/blob/master/src/nix/flake.md
 [core]: https://github.com/divnix/devos
 [community]: https://github.com/divnix/devos/tree/community
 [dotfiles]: https://github.com/hlissner/dotfiles

SUMMARY.md

@@ -16,6 +16,7 @@
   - [Profiles](./profiles/README.md)
   - [Secrets](./secrets/README.md)
   - [Suites](./suites/README.md)
+  - [Tests](./tests/README.md)
   - [Users](./users/README.md)
 - [flk](./doc/flk/index.md)
   - [up](./doc/flk/up.md)
@@ -25,6 +26,6 @@
   - [install](./doc/flk/install.md)
   - [home](./doc/flk/home.md)
 - [Integrations](doc/integrations/index.md)
-  - [deploy-rs](./doc/integrations/deploy.md)
-  - [hercules-ci](./doc/integrations/hercules.md)
+  - [Deploy RS](./doc/integrations/deploy.md)
+  - [Hercules CI](./doc/integrations/hercules.md)
 - [Contributing](./doc/README.md)

bors.toml

@@ -0,0 +1,12 @@
+status = [
+  "ci/hercules/evaluation",
+  "ci/hercules/derivations"
+]
+
+required_approvals = 1
+
+up_to_date_approvals = true
+
+delete_merged_branches = true
+
+use_squash_merge = true

default.nix

@@ -1,8 +1,8 @@
 let
-  inherit (default.inputs.nixos.lib) recurseIntoAttrs;
+  inherit (default.inputs.nixos) lib;
 
   default = (import ./compat).defaultNix;
 in
-builtins.mapAttrs (_: v: recurseIntoAttrs v) default.packages // {
+builtins.mapAttrs (_: v: lib.recurseIntoAttrs v) default.packages // {
   shell = import ./shell.nix;
 }

doc/CONTRIBUTING.md

@@ -1 +0,0 @@
-# Contributing

doc/README.md

@@ -4,6 +4,8 @@ relevant docs. Each directory contains its own README.md, which will
 automatically be pulled into the [mdbook](https://devos.divnix.com). The book is
 rendered on every change, so the docs should always be up to date.
 
+We also use [BORS](https://bors.tech) to ensure that all pull requests pass the
+test suite once at least one review is completed.
 
 ## Community PRs
 While much of your work in this template may be idiosyncratic in nature. Anything

flake.nix

@@ -27,28 +27,14 @@
       srcs.url = "path:./pkgs";
     };
 
-  outputs =
-    inputs@{ ci-agent
-    , deploy
-    , devshell
-    , home
-    , nixos
-    , nixos-hardware
-    , nur
-    , override
-    , self
-    , utils
-    , ...
-    }:
+  outputs = inputs@{ deploy, nixos, nur, self, utils, ... }:
     let
-      inherit (utils.lib) eachDefaultSystem flattenTreeSystem;
-      inherit (nixos.lib) recursiveUpdate;
-      inherit (self.lib) overlays nixosModules genPackages genPkgs
-        genHomeActivationPackages mkNodes;
+      inherit (self) lib;
+      inherit (lib) os;
 
       extern = import ./extern { inherit inputs; };
 
-      pkgs' = genPkgs { inherit self; };
+      pkgs' = os.mkPkgs { inherit self; };
 
       outputs =
         let
@@ -56,36 +42,42 @@
           pkgs = pkgs'.${system};
         in
         {
-          inherit nixosModules overlays;
-
           nixosConfigurations =
-            import ./hosts (recursiveUpdate inputs {
+            import ./hosts (nixos.lib.recursiveUpdate inputs {
               inherit pkgs system extern;
               inherit (pkgs) lib;
             });
 
+          nixosModules =
+            let moduleList = import ./modules/module-list.nix;
+            in lib.pathsToImportedAttrs moduleList;
+
           overlay = import ./pkgs;
+          overlays = lib.pathsToImportedAttrs (lib.pathsIn ./overlays);
 
           lib = import ./lib { inherit nixos pkgs; };
 
           templates.flk.path = ./.;
-
           templates.flk.description = "flk template";
-
           defaultTemplate = self.templates.flk;
 
-          deploy.nodes = mkNodes deploy self.nixosConfigurations;
+          deploy.nodes = os.mkNodes deploy self.nixosConfigurations;
 
-          checks = builtins.mapAttrs
+          checks =
+            let
+              tests = import ./tests { inherit self pkgs; };
+              deployChecks = builtins.mapAttrs
                 (system: deployLib: deployLib.deployChecks self.deploy)
                 deploy.lib;
+            in
+            nixos.lib.recursiveUpdate tests deployChecks;
         };
 
-      systemOutputs = eachDefaultSystem (system:
+      systemOutputs = utils.lib.eachDefaultSystem (system:
         let pkgs = pkgs'.${system}; in
         {
-          packages = flattenTreeSystem system
-            (genPackages {
+          packages = utils.lib.flattenTreeSystem system
+            (os.mkPackages {
               inherit self pkgs;
             });
 
@@ -94,9 +86,9 @@
           };
 
           legacyPackages.hmActivationPackages =
-            genHomeActivationPackages { inherit self; };
+            os.mkHomeActivation { inherit self; };
         }
       );
     in
-    recursiveUpdate outputs systemOutputs;
+    nixos.lib.recursiveUpdate outputs systemOutputs;
 }

hosts/default.nix

@@ -2,7 +2,6 @@
 , home
 , lib
 , nixos
-, nixos-hardware
 , override
 , pkgs
 , self
@@ -10,21 +9,13 @@
 , ...
 }:
 let
-  inherit (lib.flk) recImport nixosSystemExtended defaultImports;
-  inherit (builtins) attrValues removeAttrs;
+  inherit (lib) dev;
 
   suites = import ../suites { inherit lib; };
 
-  config = hostName:
-    nixosSystemExtended {
-      inherit system;
-
-      specialArgs = extern.specialArgs // { inherit suites; };
-
   modules =
     let
       core = ../profiles/core;
-
       modOverrides = { config, overrideModulesPath, ... }:
         let
           overrides = import ../overrides;
@@ -43,8 +34,6 @@ let
 
         hardware.enableRedistributableFirmware = lib.mkDefault true;
 
-            networking.hostName = hostName;
-
         nix.nixPath = [
           "nixpkgs=${nixos}"
           "nixos-config=${self}/compat/nixos"
@@ -62,29 +51,47 @@ let
         system.configurationRevision = lib.mkIf (self ? rev) self.rev;
       };
 
-          local = {
-            require = [
-              "${toString ./.}/${hostName}.nix"
-            ];
-          };
-
       # Everything in `./modules/list.nix`.
       flakeModules =
-            attrValues self.nixosModules;
+        builtins.attrValues self.nixosModules;
 
     in
     flakeModules ++ [
       core
       global
-          local
       modOverrides
     ] ++ extern.modules;
 
+  specialArgs = extern.specialArgs // { inherit suites; };
+
+  mkHostConfig = hostName:
+    let
+      local = {
+        require = [
+          "${toString ./.}/${hostName}.nix"
+        ];
+
+        networking = { inherit hostName; };
+      };
+    in
+    dev.os.devosSystem {
+      inherit system specialArgs;
+
+      modules = modules ++ [
+        local
+        {
+          lib = { inherit specialArgs; };
+          lib.testModule = {
+            imports = modules;
+          };
+        }
+      ];
     };
 
-  hosts = recImport {
+  hosts = dev.os.recImport
+    {
       dir = ./.;
-    _import = config;
+      _import = mkHostConfig;
     };
 in
 hosts

lib/attrs.nix

@@ -0,0 +1,29 @@
+{ lib, ... }:
+rec {
+  # mapFilterAttrs ::
+  #   (name -> value -> bool )
+  #   (name -> value -> { name = any; value = any; })
+  #   attrs
+  mapFilterAttrs = seive: f: attrs:
+    lib.filterAttrs
+      seive
+      (lib.mapAttrs' f attrs);
+
+  # Generate an attribute set by mapping a function over a list of values.
+  genAttrs' = values: f: lib.listToAttrs (map f values);
+
+  # Convert a list of file paths to attribute set
+  # that has the filenames stripped of nix extension as keys
+  # and imported content of the file as value.
+  #
+  pathsToImportedAttrs = paths:
+    let
+      paths' = lib.filter (lib.hasSuffix ".nix") paths;
+    in
+    genAttrs' paths' (path: {
+      name = lib.removeSuffix ".nix" (baseNameOf path);
+      value = import path;
+    });
+
+  concatAttrs = lib.fold (attr: sum: lib.recursiveUpdate sum attr) { };
+}

lib/default.nix

@@ -1,232 +1,21 @@
-{ nixos, pkgs, ... }:
-let
-  inherit (builtins) attrNames attrValues isAttrs readDir listToAttrs mapAttrs
-    pathExists filter;
-
-  inherit (nixos.lib) fold filterAttrs hasSuffix mapAttrs' nameValuePair removeSuffix
-    recursiveUpdate genAttrs nixosSystem mkForce substring optionalAttrs;
-
-  # mapFilterAttrs ::
-  #   (name -> value -> bool )
-  #   (name -> value -> { name = any; value = any; })
-  #   attrs
-  mapFilterAttrs = seive: f: attrs: filterAttrs seive (mapAttrs' f attrs);
-
-  # Generate an attribute set by mapping a function over a list of values.
-  genAttrs' = values: f: listToAttrs (map f values);
-
-  # pkgImport :: Nixpkgs -> Overlays -> System -> Pkgs
-  pkgImport = nixpkgs: overlays: system:
-    import nixpkgs {
-      inherit system overlays;
-      config = { allowUnfree = true; };
-    };
-
-  # Convert a list to file paths to attribute set
-  # that has the filenames stripped of nix extension as keys
-  # and imported content of the file as value.
-  #
-  pathsToImportedAttrs = paths:
-    let
-      paths' = filter (hasSuffix ".nix") paths;
-    in
-    genAttrs' paths' (path: {
-      name = removeSuffix ".nix" (baseNameOf path);
-      value = import path;
-    });
-
-  overlayPaths =
-    let
-      overlayDir = ../overlays;
-      fullPath = name: overlayDir + "/${name}";
-    in
-    map fullPath (attrNames (readDir overlayDir));
-
-  /**
-  Synopsis: mkNodes _nixosConfigurations_
-
-  Generate the `nodes` attribute expected by deploy-rs
-  where _nixosConfigurations_ are `nodes`.
-  **/
-  mkNodes = deploy: mapAttrs (_: config: {
-    hostname = config.config.networking.hostName;
-
-    profiles.system = {
-      user = "root";
-      path = deploy.lib.x86_64-linux.activate.nixos config;
-    };
-  });
-
-  /**
-  Synopsis: mkProfileAttrs _path_
-
-  Recursively import the subdirs of _path_ containing a default.nix.
-
-  Example:
-  let profiles = mkProfileAttrs ./profiles; in
-  assert profiles ? core.default; 0
-  **/
-  mkProfileAttrs = dir:
-    let
-      imports =
-        let
-          files = readDir dir;
-
-          p = n: v:
-            v == "directory"
-            && n != "profiles";
-        in
-        filterAttrs p files;
-
-      f = n: _:
-        optionalAttrs
-          (pathExists "${dir}/${n}/default.nix")
-          { default = "${dir}/${n}"; }
-        // mkProfileAttrs "${dir}/${n}";
-    in
-    mapAttrs f imports;
+args@{ nixos, pkgs, ... }:
+let inherit (nixos) lib; in
+lib.makeExtensible (self:
+  let callLibs = file: import file
+    ({
+      inherit lib;
 
+      dev = self;
+    } // args);
   in
+  with self;
   {
-  inherit mkProfileAttrs mapFilterAttrs genAttrs' pkgImport
-    pathsToImportedAttrs mkNodes;
+    inherit callLibs;
 
-  overlays = pathsToImportedAttrs overlayPaths;
+    attrs = callLibs ./attrs.nix;
+    os = callLibs ./devos;
+    lists = callLibs ./lists.nix;
 
-  mkVersion = src: "${substring 0 8 src.lastModifiedDate}_${src.shortRev}";
-
-  genPkgs = { self }:
-    let inherit (self) inputs;
-    in
-    (inputs.utils.lib.eachDefaultSystem
-      (system:
-        let
-          extern = import ../extern { inherit inputs; };
-          overridePkgs = pkgImport inputs.override [ ] system;
-          overridesOverlay = (import ../overrides).packages;
-
-          overlays = [
-            (overridesOverlay overridePkgs)
-            self.overlay
-            (final: prev: {
-              srcs = self.inputs.srcs.inputs;
-              lib = (prev.lib or { }) // {
-                inherit (nixos.lib) nixosSystem;
-                flk = self.lib;
-                utils = inputs.utils.lib;
-              };
+    inherit (attrs) mapFilterAttrs genAttrs' pathsToImportedAttrs concatAttrs;
+    inherit (lists) pathsIn;
   })
-          ]
-          ++ extern.overlays
-          ++ (attrValues self.overlays);
-        in
-        { pkgs = pkgImport nixos overlays system; }
-      )
-    ).pkgs;
-
-  profileMap = map (profile: profile.default);
-
-  recImport = { dir, _import ? base: import "${dir}/${base}.nix" }:
-    mapFilterAttrs
-      (_: v: v != null)
-      (n: v:
-        if n != "default.nix" && hasSuffix ".nix" n && v == "regular"
-        then
-          let name = removeSuffix ".nix" n; in nameValuePair (name) (_import name)
-        else
-          nameValuePair ("") (null))
-      (readDir dir);
-
-  nixosSystemExtended = { modules, ... } @ args:
-    nixosSystem (args // {
-      modules =
-        let
-          modpath = "nixos/modules";
-          cd = "installer/cd-dvd/installation-cd-minimal-new-kernel.nix";
-          ciConfig =
-            (nixosSystem (args // {
-              modules =
-                let
-                  # remove host module
-                  modules' = filter (x: ! x ? require) modules;
-                in
-                modules' ++ [
-                  ({ suites, ... }: {
-                    imports = with suites;
-                      allProfiles ++ allUsers;
-
-                    boot.loader.systemd-boot.enable = true;
-                    boot.loader.efi.canTouchEfiVariables = true;
-
-                    fileSystems."/" = { device = "/dev/disk/by-label/nixos"; };
-                  })
-                ];
-            })).config;
-
-          isoConfig = (nixosSystem
-            (args // {
-              modules = modules ++ [
-                "${nixos}/${modpath}/${cd}"
-                ({ config, ... }: {
-                  isoImage.isoBaseName = "nixos-" + config.networking.hostName;
-                  # confilcts with networking.wireless which might be slightly
-                  # more useful on a stick
-                  networking.networkmanager.enable = mkForce false;
-                  # confilcts with networking.wireless
-                  networking.wireless.iwd.enable = mkForce false;
-                })
-              ];
-            })).config;
-        in
-        modules ++ [{
-          system.build = {
-            iso = isoConfig.system.build.isoImage;
-            ci = ciConfig.system.build.toplevel;
-          };
-        }];
-    });
-
-  nixosModules =
-    let
-      # binary cache
-      cachix = import ../cachix.nix;
-      cachixAttrs = { inherit cachix; };
-
-      # modules
-      moduleList = import ../modules/module-list.nix;
-      modulesAttrs = pathsToImportedAttrs moduleList;
-
-    in
-    recursiveUpdate cachixAttrs modulesAttrs;
-
-  genHomeActivationPackages = { self }:
-    let hmConfigs =
-      builtins.mapAttrs
-        (_: config: config.config.home-manager.users)
-        self.nixosConfigurations;
-    in
-    mapAttrs
-      (_: x: mapAttrs
-        (_: cfg: cfg.home.activationPackage)
-        x)
-      hmConfigs;
-
-  genPackages = { self, pkgs }:
-    let
-      inherit (self) overlay overlays;
-      packagesNames = attrNames (overlay null null)
-        ++ attrNames (fold
-        (attr: sum: recursiveUpdate sum attr)
-        { }
-        (attrValues
-          (mapAttrs (_: v: v null null) overlays)
-        )
-      );
-    in
-    fold
-      (key: sum: recursiveUpdate sum {
-        ${key} = pkgs.${key};
-      })
-      { }
-      packagesNames;
-}

lib/devos/default.nix

@@ -0,0 +1,26 @@
+{ lib, nixos, dev, ... }:
+{
+  # pkgImport :: Nixpkgs -> Overlays -> System -> Pkgs
+  pkgImport = nixpkgs: overlays: system:
+    import nixpkgs {
+      inherit system overlays;
+      config = { allowUnfree = true; };
+    };
+
+  profileMap = map (profile: profile.default);
+
+  mkNodes = dev.callLibs ./mkNodes.nix;
+
+  mkProfileAttrs = dev.callLibs ./mkProfileAttrs.nix;
+
+  mkPkgs = dev.callLibs ./mkPkgs.nix;
+
+  recImport = dev.callLibs ./recImport.nix;
+
+  devosSystem = dev.callLibs ./devosSystem.nix;
+
+  mkHomeActivation = dev.callLibs ./mkHomeActivation.nix;
+
+  mkPackages = dev.callLibs ./mkPackages.nix;
+}
+

lib/devos/devosSystem.nix

@@ -0,0 +1,30 @@
+{ lib, nixos, ... }:
+
+{ modules, ... } @ args:
+lib.nixosSystem (args // {
+  modules =
+    let
+      modpath = "nixos/modules";
+      cd = "installer/cd-dvd/installation-cd-minimal-new-kernel.nix";
+
+      isoConfig = (lib.nixosSystem
+        (args // {
+          modules = modules ++ [
+            "${nixos}/${modpath}/${cd}"
+            ({ config, ... }: {
+              isoImage.isoBaseName = "nixos-" + config.networking.hostName;
+              # confilcts with networking.wireless which might be slightly
+              # more useful on a stick
+              networking.networkmanager.enable = lib.mkForce false;
+              # confilcts with networking.wireless
+              networking.wireless.iwd.enable = lib.mkForce false;
+            })
+          ];
+        })).config;
+    in
+    modules ++ [{
+      system.build = {
+        iso = isoConfig.system.build.isoImage;
+      };
+    }];
+})

lib/devos/mkHomeActivation.nix

@@ -0,0 +1,13 @@
+{ lib, ... }:
+
+{ self }:
+let hmConfigs =
+  lib.mapAttrs
+    (_: config: config.config.home-manager.users)
+    self.nixosConfigurations;
+in
+lib.mapAttrs
+  (_: x: lib.mapAttrs
+    (_: cfg: cfg.home.activationPackage)
+    x)
+  hmConfigs

lib/devos/mkNodes.nix

@@ -0,0 +1,16 @@
+{ lib, ... }:
+
+/**
+  Synopsis: mkNodes _nixosConfigurations_
+
+  Generate the `nodes` attribute expected by deploy-rs
+  where _nixosConfigurations_ are `nodes`.
+  **/
+deploy: lib.mapAttrs (_: config: {
+  hostname = config.config.networking.hostName;
+
+  profiles.system = {
+    user = "root";
+    path = deploy.lib.x86_64-linux.activate.nixos config;
+  };
+})

lib/devos/mkPackages.nix

@@ -0,0 +1,18 @@
+{ lib, dev, ... }:
+
+{ self, pkgs }:
+let
+  inherit (self) overlay overlays;
+  packagesNames = lib.attrNames (overlay null null)
+    ++ lib.attrNames (dev.concatAttrs
+    (lib.attrValues
+      (lib.mapAttrs (_: v: v null null) overlays)
+    )
+  );
+in
+lib.fold
+  (key: sum: lib.recursiveUpdate sum {
+    ${key} = pkgs.${key};
+  })
+{ }
+  packagesNames

lib/devos/mkPkgs.nix

@@ -0,0 +1,66 @@
+{ lib, dev, nixos, ... }:
+
+{ self }:
+let inherit (self) inputs;
+in
+(inputs.utils.lib.eachDefaultSystem
+  (system:
+    let
+      extern = import ../../extern { inherit inputs; };
+      overridePkgs = dev.os.pkgImport inputs.override [ ] system;
+      overridesOverlay = (import ../../overrides).packages;
+
+      overlays = [
+        (overridesOverlay overridePkgs)
+        self.overlay
+        (final: prev: {
+          srcs =
+            let
+              mkVersion = name: input:
+                let
+                  inputs = (builtins.fromJSON
+                    (builtins.readFile ../../flake.lock)).nodes;
+
+                  ref =
+                    if lib.hasAttrByPath [ name "original" "ref" ] inputs
+                    then inputs.${name}.original.ref
+                    else "";
+
+                  version =
+                    let version' = builtins.match
+                      "[[:alpha:]]*[-._]?([0-9]+(\.[0-9]+)*)+"
+                      ref;
+                    in
+                    if lib.isList version'
+                    then lib.head version'
+                    else if input ? lastModifiedDate && input ? shortRev
+                    then "${lib.substring 0 8 input.lastModifiedDate}_${input.shortRev}"
+                    else null;
+                in
+                version;
+            in
+            lib.mapAttrs
+              (name: input:
+                let
+                  version = mkVersion name input;
+                in
+                input // lib.optionalAttrs (! isNull version)
+                  {
+                    inherit version;
+                  }
+              )
+              self.inputs.srcs.inputs;
+          lib = prev.lib.extend (lfinal: lprev: {
+            inherit dev;
+            inherit (lib) nixosSystem;
+
+            utils = inputs.utils.lib;
+          });
+        })
+      ]
+      ++ extern.overlays
+      ++ (lib.attrValues self.overlays);
+    in
+    { pkgs = dev.os.pkgImport nixos overlays system; }
+  )
+).pkgs

lib/devos/mkProfileAttrs.nix

@@ -0,0 +1,35 @@
+{ lib, ... }:
+
+let mkProfileAttrs =
+  /**
+  Synopsis: mkProfileAttrs _path_
+
+  Recursively collect the subdirs of _path_ containing a default.nix into attrs.
+  This sets a contract, eliminating ambiguity for _default.nix_ living under the
+  profile directory.
+
+  Example:
+  let profiles = mkProfileAttrs ./profiles; in
+  assert profiles ? core.default; 0
+  **/
+  dir:
+  let
+    imports =
+      let
+        files = builtins.readDir dir;
+
+        p = n: v:
+          v == "directory"
+          && n != "profiles";
+      in
+      lib.filterAttrs p files;
+
+    f = n: _:
+      lib.optionalAttrs
+        (lib.pathExists "${dir}/${n}/default.nix")
+        { default = "${dir}/${n}"; }
+      // mkProfileAttrs "${dir}/${n}";
+  in
+  lib.mapAttrs f imports;
+in mkProfileAttrs
+

lib/devos/recImport.nix

@@ -0,0 +1,12 @@
+{ lib, dev, ... }:
+
+{ dir, _import ? base: import "${dir}/${base}.nix" }:
+dev.mapFilterAttrs
+  (_: v: v != null)
+  (n: v:
+    if n != "default.nix" && lib.hasSuffix ".nix" n && v == "regular"
+    then
+      let name = lib.removeSuffix ".nix" n; in lib.nameValuePair (name) (_import name)
+    else
+      lib.nameValuePair ("") (null))
+  (builtins.readDir dir)

lib/lists.nix

@@ -0,0 +1,8 @@
+{ lib, ... }:
+{
+  pathsIn = dir:
+    let
+      fullPath = name: "${toString dir}/${name}";
+    in
+    map fullPath (lib.attrNames (builtins.readDir dir));
+}

nix/ci.nix

@@ -1,32 +1,31 @@
 let
-  inherit (default.inputs.nixos.lib) mapAttrs recurseIntoAttrs;
+  inherit (default.inputs.nixos) lib;
 
   default = (import "${../.}/compat").defaultNix;
-  packages = import ../default.nix;
+
+  ciSystems = [
+    "aarch64-linux"
+    "i686-linux"
+    "x86_64-linux"
+  ];
+
+  filterSystems = lib.filterAttrs
+    (system: _: lib.elem system ciSystems);
+
+  recurseIntoAttrsRecursive = lib.mapAttrs (_: v:
+    if lib.isAttrs v
+    then recurseIntoAttrsRecursive (lib.recurseIntoAttrs v)
+    else v
+  );
+
+  systemOutputs = lib.filterAttrs
+    (_: set: lib.isAttrs set
+      && lib.any
+      (system: set ? ${system})
+      ciSystems
+    )
+    default.outputs;
+
+  ciDrvs = lib.mapAttrs (_: system: filterSystems system) systemOutputs;
 in
-{
-  checks = recurseIntoAttrs (mapAttrs (_: v: recurseIntoAttrs v) {
-    inherit (default.checks)
-      aarch64-linux
-      i686-linux
-      x86_64-linux
-      ;
-  });
-
-  # platforms supported by our hercules-ci agent
-  inherit (packages)
-    aarch64-linux
-    i686-linux
-    x86_64-linux
-    ;
-
-  devShell = recurseIntoAttrs {
-    inherit (default.devShell)
-      aarch64-linux
-      i686-linux
-      x86_64-linux
-      ;
-  };
-
-  nixos = default.nixosConfigurations.NixOS.config.system.build.ci;
-}
+recurseIntoAttrsRecursive ciDrvs

pkgs/README.md

@@ -16,38 +16,31 @@ And, as usual, every package in the overlay is also available to any NixOS
 
 ## Automatic Source Updates
 There is the added, but optional, convenience of declaring your sources in
-_pkgs/flake.nix_ as an input. This allows updates to be managed automatically
-by simply [updating](../doc/flk/update.md#updating-package-sources) the lock
-file. No more manually entering sha256 hashes!
+_pkgs/flake.nix_ as an input. You can then access them from the `srcs` package.
+This allows updates to be managed automatically by simply
+[updating](../doc/flk/update.md#updating-package-sources) the lock file. No
+more manually entering sha256 hashes!
 
+As an added bonus, version strings are also generated automatically from either
+the flake ref, or the date and git revision of the source. For examples,
+definitely checkout the [community branch](../#community-profiles).
 
 ## Example
 pkgs/development/libraries/libinih/default.nix:
 ```nix
 { stdenv, meson, ninja, lib, srcs, ... }:
-let version = "r53";
-in
+let inherit (srcs) libinih; in
 stdenv.mkDerivation {
   pname = "libinih";
-  inherit version;
 
-  src = srcs.libinih;
+  # version will resolve to 53, as specified in the final example below
+  inherit (libinih) version;
+
+  src = libinih;
 
   buildInputs = [ meson ninja ];
 
-  mesonFlags = ''
-    -Ddefault_library=shared
-    -Ddistro_install=true
-  '';
-
-  meta = with lib; {
-    description = "Simple .INI file parser in C";
-    homepage = "https://github.com/benhoyt/inih";
-    maintainers = [ maintainers.divnix ];
-    license = licenses.bsd3;
-    platforms = platforms.all;
-    inherit version;
-  };
+  # ...
 }
 ```
 

profiles/README.md

@@ -1,43 +1,34 @@
 # Profiles
-Profiles are simply NixOS modules which contain generic expressions suitable
-for any host. A good example is the configuration for a text editor, or
-window manager. If you need some concrete examples, just checkout the
-community [branch](https://github.com/divnix/devos/tree/community/profiles).
+
+Profiles are a convenient shorthand for the [_definition_][definition] of
+[options][options] in contrast to their [_declaration_][declaration]. They're
+built into the NixOS module system for a reason: to elegantly provide a clear
+separation of concerns.
+
+If you need guidance, a community [branch](https://github.com/divnix/devos/tree/community/profiles)
+is maintained to help get up to speed on their usage.
 
 ## Constraints
 For the sake of consistency, a profile should always be defined in a
-_default.nix_ containing a valid [nixos module](https://nixos.wiki/wiki/Module)
-which ___does not___ declare any new
-[module options](https://nixos.org/manual/nixos/stable/index.html#sec-option-declarations).
-If you need to do that, use the [modules directory](../modules).
+___default.nix___ containing a [nixos module config][config].
+A profile's directory is used for quick modularization of
+[interelated bits](./#subprofiles).
 
-> ##### _Note:_
+> ##### _Notes:_
+> * For _declaring_ module options, there's the [modules](../modules) directory.
+> * This directory takes inspiration from
+>   [upstream](https://github.com/NixOS/nixpkgs/tree/master/nixos/modules/profiles)
+>   .
+> * Sticking to a simple [spec][spec] has refreshing advantages.
 >   [hercules-ci](../doc/integrations/hercules.md) expects all profiles to be
-> defined in a _default.nix_. Similarly, [suites](../suites) expect a
-> _default.nix_ as well.
-
-### Example
-#### Correct ✔
-profiles/develop/default.nix:
-```nix
-{  ... }:
-{
-  programs.zsh.enable = true;
-}
-```
-
-#### Incorrect ❌
-profiles/develop.nix:
-```nix
-{
-  options = {};
-}
-```
+>   defined in a ___default.nix___, allowing them to be built automatically when
+>   added. Congruently, [suites](../suites) expect ___default.nix___ to avoid
+>   having to manage their paths manually.
 
 ## Subprofiles
 Profiles can also define subprofiles. They follow the same constraints outlined
-above. A good top level profile should be a high level concern, such a your
-personal development environment, and the subprofiles should be more concrete
+above. A good top level profile should be a high level concern, such as your
+personal development environment while the subprofiles should be more focused
 program configurations such as your text editor, and shell configs. This way,
 you can either pull in the whole development profile, or pick and choose
 individual programs.
@@ -62,8 +53,13 @@ profiles/develop/zsh/default.nix:
 ```
 
 ## Conclusion
-Profiles are the most important concept in devos. They allow us to keep our
-nix expressions self contained and modular. This way we can maximize reuse
-while minimizing boilerplate. Always strive to keep your profiles as generic
-and modular as possible. Anything machine specific belongs in your
-[host](../hosts) files.
+Profiles are the most important concept in DevOS. They allow us to keep our
+Nix expressions self contained and modular. This way we can maximize reuse
+across hosts while minimizing boilerplate. Remember, anything machine
+specific belongs in your [host](../hosts) files instead.
+
+[definition]: https://nixos.org/manual/nixos/stable/index.html#sec-option-definitions
+[declaration]: https://nixos.org/manual/nixos/stable/index.html#sec-option-declarations
+[options]: https://nixos.org/manual/nixos/stable/index.html#sec-writing-modules
+[spec]: https://github.com/divnix/devos/tree/core/lib/devos/mkProfileAttrs.nix
+[config]: https://nixos.wiki/wiki/Module#structure

shell/default.nix

@@ -2,7 +2,7 @@
 , system ? builtins.currentSystem
 }:
 let
-  pkgs = (self.lib.genPkgs { inherit self; }).${system};
+  pkgs = (self.lib.os.mkPkgs { inherit self; }).${system};
 
   inherit (pkgs) lib;
 

suites/default.nix

@@ -1,10 +1,9 @@
 { lib }:
 let
-  inherit (builtins) mapAttrs isFunction;
-  inherit (lib.flk) mkProfileAttrs profileMap;
+  inherit (lib) dev;
 
-  profiles = mkProfileAttrs (toString ../profiles);
-  users = mkProfileAttrs (toString ../users);
+  profiles = dev.os.mkProfileAttrs (toString ../profiles);
+  users = dev.os.mkProfileAttrs (toString ../users);
 
   allProfiles =
     let defaults = lib.collect (x: x ? default) profiles;
@@ -19,6 +18,6 @@ let
     base = [ users.nixos users.root ];
   };
 in
-mapAttrs (_: v: profileMap v) suites // {
+lib.mapAttrs (_: v: dev.os.profileMap v) suites // {
   inherit allProfiles allUsers;
 }

tests/0004-nixos-testing-Add-support-for-specialArgs.patch

@@ -0,0 +1,61 @@
+From 9f33ab62d99c98e3f5bddd64532f15f482cf01b2 Mon Sep 17 00:00:00 2001
+From: =?UTF-8?q?Janne=20He=C3=9F?= <janne@hess.ooo>
+Date: Tue, 2 Jun 2020 16:27:07 +0200
+Subject: [PATCH 04/22] nixos/testing: Add support for specialArgs
+
+Since using flakes disallows the usage of <unstable> (which I use in
+some tests), this adds an alternative. By setting specialArgs, all VMs
+can get the `unstable` flake input as an arg. This is not possible with
+extraConfigurations, as that would lead to infinite recursions.
+---
+ nixos/lib/build-vms.nix      | 8 +++++---
+ nixos/lib/testing-python.nix | 4 +++-
+ 2 files changed, 8 insertions(+), 4 deletions(-)
+
+diff --git a/nixos/lib/build-vms.nix b/nixos/lib/build-vms.nix
+index 1bad63b9194..b1575fc13bb 100644
+--- a/nixos/lib/build-vms.nix
++++ b/nixos/lib/build-vms.nix
+@@ -3,8 +3,10 @@
+   minimal ? false
+ , # Ignored
+   config ? null
+-  # Nixpkgs, for qemu, lib and more
+-, pkgs
++, # Nixpkgs, for qemu, lib and more
++  pkgs
++, # !!! See comment about args in lib/modules.nix
++  specialArgs ? {}
+ , # NixOS configuration to add to the VMs
+   extraConfigurations ? []
+ }:
+@@ -31,7 +33,7 @@ rec {
+     nodes: configurations:
+ 
+     import ./eval-config.nix {
+-      inherit system;
++      inherit system specialArgs;
+       modules = configurations ++ extraConfigurations;
+       baseModules =  (import ../modules/module-list.nix) ++
+         [ ../modules/virtualisation/qemu-vm.nix
+diff --git a/nixos/lib/testing-python.nix b/nixos/lib/testing-python.nix
+index 76a2022082c..498f97336c0 100644
+--- a/nixos/lib/testing-python.nix
++++ b/nixos/lib/testing-python.nix
+@@ -4,10 +4,12 @@
+ , minimal ? false
+   # Ignored
+ , config ? {}
++  # !!! See comment about args in lib/modules.nix
++, specialArgs ? {}
+   # Modules to add to each VM
+ , extraConfigurations ? [] }:
+ 
+-with import ./build-vms.nix { inherit system pkgs minimal extraConfigurations; };
++with import ./build-vms.nix { inherit system pkgs minimal specialArgs extraConfigurations; };
+ with pkgs;
+ 
+ rec {
+-- 
+2.29.2
+

tests/README.md

@@ -0,0 +1,36 @@
+# Testing
+
+Testing is always an important aspect of any software development project, and
+NixOS offers some incredibly powerful tools to write tests for your
+configuration, and, optionally, run them in
+[CI](../doc/integrations/hercules.md).
+
+## Lib Tests
+You can easily write tests for your own library functions in the
+___tests/lib.nix___ file and they will be run on every `nix flake check` or
+during a CI run.
+
+## Unit Tests
+Unit tests are can be created from regular derivations, and they can do
+almost anything you can imagine. By convention, it is best to test your
+packages during their [check phase][check]. All packages and their tests will
+be built during CI.
+
+## Integration Tests
+You can write integration tests for one or more NixOS VMs that can,
+optionally, be networked together, and yes, it's as awesome as it sounds!
+
+Be sure to use the `mkTest` function, in the [___tests/default.nix___][default]
+which wraps the official [testing-python][testing-python] function to ensure
+that the system is setup exactly as it is for a bare DevOS system. There are
+already great resources for learning how to use these tests effectively,
+including the official [docs][test-doc], a fantastic [blog post][test-blog],
+and the examples in [nixpkgs][nixos-tests].
+
+[test-doc]: https://nixos.org/manual/nixos/stable/index.html#sec-nixos-tests
+[test-blog]: https://www.haskellforall.com/2020/11/how-to-use-nixos-for-lightweight.html
+[default]: https://github.com/divnix/devos/tree/core/tests/default.nix
+[run-test]: https://github.com/NixOS/nixpkgs/blob/6571462647d7316aff8b8597ecdf5922547bf365/lib/debug.nix#L154-L166
+[nixos-tests]: https://github.com/NixOS/nixpkgs/tree/master/nixos/tests
+[testing-python]: https://github.com/NixOS/nixpkgs/tree/master/nixos/lib/testing-python.nix
+[check]: https://nixos.org/manual/nixpkgs/stable/#ssec-check-phase

tests/default.nix

@@ -0,0 +1,103 @@
+{ self, pkgs }:
+let
+  inherit (self.inputs) nixos;
+  inherit (self.nixosConfigurations.NixOS.config.lib) testModule specialArgs;
+
+  # current release 20.09 does not support the specialArgs required for us
+  # to use tests as we would normally use hosts. Using the "testing-python.nix"
+  # from the override flake would build the test-vm from an unstable os
+  # different than the one our systems are running. Instead simply patch nixpkgs
+  # to include the updated version. This can be removed in the next release
+  patchedNixpkgs =
+    pkgs.stdenv.mkDerivation {
+      name = "nixpkgs-patched";
+
+      src = nixos;
+      patches = [ ./0004-nixos-testing-Add-support-for-specialArgs.patch ];
+
+      dontBuild = true;
+      dontFixup = true;
+
+      versionSuffix = "pre${
+    if nixos ? lastModified
+    then builtins.substring 0 8 (nixos.lastModifiedDate or nixos.lastModified)
+    else toString nixos.revCount}.${nixos.shortRev or "dirty"}";
+
+      configurePhase = ''
+        echo -n $VERSION_SUFFIX > .version-suffix
+        echo -n ${nixos.rev or nixos.shortRev or "dirty"} > .git-revision
+      '';
+
+      installPhase = ''
+        cp -r $PWD $out
+      '';
+    };
+
+  mkTest =
+    let
+      nixosTesting =
+        (import "${patchedNixpkgs}/nixos/lib/testing-python.nix" {
+          inherit (pkgs.stdenv.hostPlatform) system;
+          inherit specialArgs;
+          inherit pkgs;
+          extraConfigurations = [
+            testModule
+          ];
+        });
+    in
+    test:
+    let
+      loadedTest =
+        if builtins.typeOf test == "path"
+        then import test
+        else test;
+      calledTest =
+        if pkgs.lib.isFunction loadedTest
+        then pkgs.callPackage loadedTest { }
+        else loadedTest;
+    in
+    nixosTesting.makeTest calledTest;
+in
+{
+  x86_64-linux = {
+    profilesTest = mkTest {
+      name = "profiles";
+
+      machine = { suites, ... }: {
+        imports = suites.allProfiles ++ suites.allUsers;
+      };
+
+      testScript = ''
+        machine.systemctl("is-system-running --wait")
+      '';
+    };
+
+    libTests = pkgs.runCommandNoCC "devos-lib-tests"
+      {
+        buildInputs = [
+          pkgs.nix
+          (
+            let tests = import ./lib.nix { inherit self pkgs; };
+            in
+            if tests == [ ]
+            then null
+            else throw (builtins.toJSON tests)
+          )
+        ];
+      } ''
+      datadir="${pkgs.nix}/share"
+      export TEST_ROOT=$(pwd)/test-tmp
+      export NIX_BUILD_HOOK=
+      export NIX_CONF_DIR=$TEST_ROOT/etc
+      export NIX_LOCALSTATE_DIR=$TEST_ROOT/var
+      export NIX_LOG_DIR=$TEST_ROOT/var/log/nix
+      export NIX_STATE_DIR=$TEST_ROOT/var/nix
+      export NIX_STORE_DIR=$TEST_ROOT/store
+      export PAGER=cat
+      cacheDir=$TEST_ROOT/binary-cache
+      nix-store --init
+
+      touch $out
+    '';
+  };
+}

tests/lib.nix

@@ -0,0 +1,62 @@
+{ self, pkgs }:
+let inherit (self.inputs.nixos) lib; in
+with self.lib;
+lib.runTests {
+  testConcatAttrs = {
+    expr = concatAttrs [{ foo = 1; } { bar = 2; } { baz = 3; }];
+
+    expected = { foo = 1; bar = 2; baz = 3; };
+  };
+
+  testGenAttrs' = {
+    expr = genAttrs'
+      [ "/foo/bar" "/baz/buzz" ]
+      (path: {
+        name = baseNameOf path;
+        value = "${path}/fizz";
+      });
+
+    expected = { bar = "/foo/bar/fizz"; buzz = "/baz/buzz/fizz"; };
+  };
+
+  testMapFilterAttrs = {
+    expr = mapFilterAttrs
+      (n: v: n == "foobar" && v == 1)
+      (n: v: lib.nameValuePair ("${n}bar") (v + 1))
+      { foo = 0; bar = 2; };
+
+    expected = { foobar = 1; };
+  };
+
+  testPathsIn =
+    let testPaths = pkgs.runCommandNoCC "test-paths-in" { } ''
+      mkdir -p $out/{foo,bar,baz}
+    '';
+    in
+    {
+      expr = pathsIn testPaths;
+
+      expected = [
+        "${testPaths}/bar"
+        "${testPaths}/baz"
+        "${testPaths}/foo"
+      ];
+    };
+
+  testPathsToImportedAttrs = {
+    expr =
+      pathsToImportedAttrs [
+        ./testPathsToImportedAttrs/foo.nix
+        ./testPathsToImportedAttrs/bar.nix
+        ./testPathsToImportedAttrs/t.nix
+        ./testPathsToImportedAttrs/f.nix
+      ];
+
+    expected = {
+      foo = { bar = 1; };
+      bar = { foo = 2; };
+      t = true;
+      f = false;
+    };
+  };
+}

tests/testPathsToImportedAttrs/bar.nix

@@ -0,0 +1 @@
+{ foo = 2; }

tests/testPathsToImportedAttrs/f.nix

@@ -0,0 +1 @@
+true && false

tests/testPathsToImportedAttrs/foo.nix

@@ -0,0 +1 @@
+{ bar = 1; }

tests/testPathsToImportedAttrs/t.nix

@@ -0,0 +1 @@
+true || false