datasets: create named default dataset

Prepare for allowing to register external datasets

Author: hsjobeki

pasta/default.nix

@@ -1,12 +1,12 @@
-{ pkgs, nixpkgs, noogle-plugin, ... }:
+{ pkgs, nixpkgs, noogle-plugin, dataset ? "nix", ... }:
 let nix = noogle-plugin.passthru.nix; in
 pkgs.stdenv.mkDerivation {
-  name = "pasta";
+  name = "pasta-${dataset}";
   src = ./src;
   nativeBuildInputs = [ nix ];
   buildPhase = ''
     nix-instantiate --extra-experimental-features 'nix-command flakes' --plugin-files ${noogle-plugin}/lib/libnoogle-plugin.so --eval --strict --json --store $PWD --show-trace \
-      eval.nix --arg 'pkgs' 'import ${nixpkgs} {}' --arg 'repo' '${nixpkgs.outPath}' -A all \
+      eval.nix --arg 'pkgs' 'import ${nixpkgs} {}' --arg 'repo' '${nixpkgs.outPath}' -A docs.${dataset} \
       > $out
   '';
 }

pasta/flake-module.nix

@@ -17,9 +17,11 @@
             cat ${metaFile} > $out
           '';
         };
+        # Default "nix" dataset (nixpkgs-master)
         pasta = pkgs.callPackage ./default.nix {
           inherit nixpkgs pkgs;
           inherit (self'.packages) noogle-plugin;
+          dataset = "nix";
         };
       };
       devShells.pastaMaker = pkgs.callPackage ./shell.nix { inherit pkgs; inherit (self'.packages) noogle-plugin; };

pasta/src/eval.nix

@@ -17,58 +17,74 @@ let
   tools = import ./tools.nix { inherit lib; };
   inherit (tools) getDocsFromSet collectFns toFile;
 
-  # Contains separate sets of metadata.
-  # which then allows running separate evaluations. Once at a time for better error tracing and memory management.
-
-  docs = {
-    ############# Recusive analysis sets
-    lib = collectFns lib { initialPath = [ "lib" ]; };
-    rustTools = collectFns pkgs.rustPackages {
-      initialPath = [ "pkgs" "rustPackages" ];
-    };
-    appimageTools = collectFns pkgs.appimageTools { initialPath = [ "pkgs" "appimageTools" ]; };
-    elmPackages = collectFns pkgs.elmPackages { initialPath = [ "pkgs" "elmPackages" ]; };
-
-    agdaPackages = collectFns pkgs.agdaPackages { initialPath = [ "pkgs" "agdaPackages" ]; ignores = [ "lib" ]; };
-    dhallPackages = collectFns pkgs.dhallPackages { initialPath = [ "pkgs" "dhallPackages" ]; ignores = [ "lib" ]; };
+  # Evaluate a single entry based on its mode
+  evalEntry = _name: entry:
+    if entry.mode == "recursive" then
+      collectFns entry.set {
+        initialPath = entry.path;
+        ignores = entry.ignores or [ ];
+      }
+    else if entry.mode == "shallow" then
+      getDocsFromSet entry.set entry.path
+    else
+      throw "unknown mode '${entry.mode}' for entry '${_name}'";
 
-    beamPackages = collectFns pkgs.beamPackages { initialPath = [ "pkgs" "beamPackages" ]; };
-    testers = collectFns pkgs.testers { initialPath = [ "pkgs" "testers" ]; };
+  # Named datasets, each with declarative entries.
+  # Each entry specifies:
+  #   - set: the attribute set to analyze
+  #   - path: display path in noogle
+  #   - mode: "recursive" (traverse nested sets) or "shallow" (top-level functions only)
+  #   - ignores: (optional, recursive only) attribute names to skip
 
-    dotnetCorePackages = collectFns pkgs.dotnetCorePackages { initialPath = [ "pkgs" "dotnetCorePackages" ]; };
-    emacsPackages = collectFns pkgs.emacsPackages { initialPath = [ "pkgs" "emacsPackages" ]; };
-    gradle-packages = collectFns pkgs.gradle-packages { initialPath = [ "pkgs" "gradle-packages" ]; };
+  datasets = {
+    nix = {
+      entries = {
+        ############# Recursive analysis
+        lib = { set = lib; path = [ "lib" ]; mode = "recursive"; };
+        rustPackages = { set = pkgs.rustPackages; path = [ "pkgs" "rustPackages" ]; mode = "recursive"; };
+        appimageTools = { set = pkgs.appimageTools; path = [ "pkgs" "appimageTools" ]; mode = "recursive"; };
+        elmPackages = { set = pkgs.elmPackages; path = [ "pkgs" "elmPackages" ]; mode = "recursive"; };
+        agdaPackages = { set = pkgs.agdaPackages; path = [ "pkgs" "agdaPackages" ]; mode = "recursive"; ignores = [ "lib" ]; };
+        dhallPackages = { set = pkgs.dhallPackages; path = [ "pkgs" "dhallPackages" ]; mode = "recursive"; ignores = [ "lib" ]; };
+        beamPackages = { set = pkgs.beamPackages; path = [ "pkgs" "beamPackages" ]; mode = "recursive"; };
+        testers = { set = pkgs.testers; path = [ "pkgs" "testers" ]; mode = "recursive"; };
+        dotnetCorePackages = { set = pkgs.dotnetCorePackages; path = [ "pkgs" "dotnetCorePackages" ]; mode = "recursive"; };
+        emacsPackages = { set = pkgs.emacsPackages; path = [ "pkgs" "emacsPackages" ]; mode = "recursive"; };
+        gradle-packages = { set = pkgs.gradle-packages; path = [ "pkgs" "gradle-packages" ]; mode = "recursive"; };
+        ociTools = { set = pkgs.ociTools; path = [ "pkgs" "ociTools" ]; mode = "recursive"; };
+        releaseTools = { set = pkgs.releaseTools; path = [ "pkgs" "releaseTools" ]; mode = "recursive"; };
+        vmTools = { set = pkgs.vmTools; path = [ "pkgs" "vmTools" ]; mode = "recursive"; };
 
-    ociTools = collectFns pkgs.ociTools { initialPath = [ "pkgs" "ociTools" ]; };
-    releaseTools = collectFns pkgs.releaseTools { initialPath = [ "pkgs" "releaseTools" ]; };
-    vmTools = collectFns pkgs.vmTools { initialPath = [ "pkgs" "vmTools" ]; };
+        ############# Shallow analysis
+        # pkgs cannot be analyzed recursively; nested items must be configured specifically
+        stdenv = { set = pkgs.stdenv; path = [ "pkgs" "stdenv" ]; mode = "shallow"; };
+        pkgs = { set = pkgs; path = [ "pkgs" ]; mode = "shallow"; };
+        dockerTools = { set = pkgs.dockerTools; path = [ "pkgs" "dockerTools" ]; mode = "shallow"; };
+        writers = { set = pkgs.writers; path = [ "pkgs" "writers" ]; mode = "shallow"; };
+        haskellLib = { set = pkgs.haskell.lib; path = [ "pkgs" "haskell" "lib" ]; mode = "shallow"; };
+        python3Packages = { set = pkgs.python3Packages; path = [ "pkgs" "python3Packages" ]; mode = "shallow"; };
+        builtins = { set = builtins; path = [ "builtins" ]; mode = "shallow"; };
+        make-disk-image = { set = { inherit make-disk-image; }; path = [ ]; mode = "shallow"; };
+      };
+    };
+  };
 
+  # Evaluate all entries in a dataset
+  evalDataset = dataset:
+    let
+      docs = lib.mapAttrs evalEntry dataset.entries;
+    in
+    builtins.foldl' (acc: name: acc ++ docs.${name}) [ ] (builtins.attrNames docs);
 
-    ############# Non-recursive analysis sets (pkgs.<nested>)
-    # pkgs cannot be analysed recursively
-    # nested documentation items must be configured specifically
-    stdenvTools = getDocsFromSet pkgs.stdenv [ "pkgs" "stdenv" ];
-    pkgs = getDocsFromSet pkgs [ "pkgs" ];
-    dockerTools = getDocsFromSet pkgs.dockerTools [ "pkgs" "dockerTools" ];
-    writers = getDocsFromSet pkgs.writers [ "pkgs" "writers" ];
-    haskellLib = getDocsFromSet pkgs.haskell.lib [ "pkgs" "haskell" "lib" ];
-    pythonTools =
-      getDocsFromSet pkgs.python3Packages [ "pkgs" "python3Packages" ];
-    builtins =
-      getDocsFromSet builtins [ "builtins" ];
-    mkDiskImage = [
-      {
-        path = [ "make-disk-image" ];
-        docs = tools.getDocs { inherit make-disk-image; } "make-disk-image";
-      }
-    ];
-  };
+  # Evaluate each dataset separately
+  docs = lib.mapAttrs (_: evalDataset) datasets;
 
-  all = builtins.foldl' (acc: name: acc ++ docs.${name}) [ ] (builtins.attrNames docs);
+  # For backwards compat: flatten the "nix" dataset
+  all = docs.nix;
 
   # generate test_data for pesto
   test_data = {
     attrsets = getDocsFromSet lib.attrsets [ "lib" "attrsets" ];
   };
 in
-{ inherit tools pkgs docs toFile getDocsFromSet collectFns all test_data; }
+{ inherit tools pkgs datasets docs evalDataset evalEntry toFile getDocsFromSet collectFns all test_data; }