only download packages which might be required

Package files are parsed to determine the dependency package
dependencies, and these are used to find the packages to download which
might be required.

This doesn't implement every detail of the opkg dependency resolution,
but instead tries to find all packages which might be required.
In particular, no attempt is made to resolve dependencies on virtual
packages (which may be provided by multiple packages) to a single
package (all providing packages are downloaded).

builder.nix

@@ -31,7 +31,13 @@ with pkgs;
 let
   inherit (import ./files.nix {
     inherit pkgs release target variant sha256 feedsSha256 packagesArch;
-  }) arch variantPackages variantFiles feedsPackages;
+  }) arch variantFiles profiles expandDeps allPackages;
+
+  requiredPackages = (
+    profiles.default_packages
+    ++ profiles.profiles.${profile}.device_packages
+    ++ packages);
+  allRequiredPackages = expandDeps allPackages requiredPackages;
 in
 
 stdenv.mkDerivation {
@@ -51,13 +57,17 @@ stdenv.mkDerivation {
   '';
 
   configurePhase = ''
-    cat >repositories.conf <<EOF
-    src imagebuilder file:${variantPackages}
-    ${lib.concatMapStrings (feed: ''
-      src openwrt_${feed} file:${feedsPackages.${feed}}
-    '') (builtins.attrNames feedsPackages)}
-    EOF
-    cat repositories.conf
+    ${lib.concatMapStrings (pname:
+      let
+        package = allPackages.${pname};
+      in
+        lib.optionalString
+          (package.type == "real")
+          "ln -s ${package.file} packages/${package.filename}\n"
+        )
+      allRequiredPackages}
+
+    echo "src imagebuilder file:packages" > repositories.conf
   '';
 
   nativeBuildInputs =

files.nix

@@ -46,9 +46,9 @@ let
         }
       ) filesSha256;
 
-  fetchPackages = url: packagesContent:
+  parsePackages = url: packagesContent:
     let
-      parsedPackages = map (section:
+      parsedRaw = map (section:
         builtins.foldl' (data: line:
           let
             m = builtins.match "(.+): (.+)" line;
@@ -60,34 +60,35 @@ let
         ) {} (lib.splitString "\n" section)
       ) (lib.splitString "\n\n" packagesContent);
 
+      parseDepends = depStr:
+        map (dep: builtins.elemAt (lib.splitString " " dep) 0)
+          (lib.splitString ", " depStr);
     in
-      builtins.foldl' (variantFiles: parsed:
+    builtins.foldl'
+      (variantFiles: parsed:
         if parsed ? Filename && parsed ? SHA256sum
         then
           variantFiles // {
-            ${parsed.Filename} = fetchurl {
-              url = "${url}/${parsed.Filename}";
-              name = sanitizeFilename parsed.Filename;
-              sha256 = parsed.SHA256sum;
+            ${parsed.Package} = {
+              filename = parsed.Filename;
+              file = fetchurl {
+                url = "${url}/${parsed.Filename}";
+                sha256 = parsed.SHA256sum;
+                name = sanitizeFilename parsed.Filename;
+              };
+              depends = if parsed ? Depends then parseDepends parsed.Depends else [];
+              provides = if parsed ? Provides then parsed.Provides else null;
+              type = "real";
             };
           }
         else
           variantFiles
-      ) {} parsedPackages;
+      ) {} parsedRaw;
 
   baseUrl = "https://downloads.openwrt.org/releases/${release}";
 
   variantFiles = fetchSums "${baseUrl}/targets/${target}/${variant}" sha256;
 
-  variantPackages = runCommandNoCC "openwrt-${release}-${target}-${variant}-packages" {} ''
-    mkdir packages
-    ${lib.concatMapStrings (file:
-      lib.optionalString (lib.hasPrefix "packages/" file) ''
-        ln -s ${variantFiles.${file}} ${file}
-      '') (builtins.attrNames variantFiles)}
-    mv packages $out
-  '';
-
   feedsPackagesFile = builtins.mapAttrs (feed: sha256:
     fetchurl {
       url = "${baseUrl}/packages/${arch}/${feed}/Packages";
@@ -95,19 +96,92 @@ let
     }
   ) feedsSha256;
 
-  feedsFiles = builtins.mapAttrs (feed: packagesFile:
-    fetchPackages "${baseUrl}/packages/${arch}/${feed}" (builtins.readFile packagesFile)
+  packagesByFeed = builtins.mapAttrs (feed: packagesFile:
+    parsePackages "${baseUrl}/packages/${arch}/${feed}" (builtins.readFile packagesFile)
   ) feedsPackagesFile;
 
-  feedsPackages = builtins.mapAttrs (feed: files:
-    runCommandNoCC "openwrt-${release}-${arch}-${feed}-packages" {} ''
-      mkdir $out
-      ln -s ${feedsPackagesFile.${feed}} $out/Packages
-      ${lib.concatMapStrings (file: ''
-        ln -s ${files.${file}} $out/${file}
-      '') (builtins.attrNames files)}
-    ''
-  ) feedsFiles;
+  corePackages =
+    parsePackages
+      "${baseUrl}/targets/${target}/${variant}/packages"
+      (builtins.readFile variantFiles."packages/Packages");
+
+  realPackages =
+    (builtins.foldl' (a: b: a // b) { } (builtins.attrValues packagesByFeed))
+    // corePackages;
+
+  # for each package that 'provides' something, register that package as a
+  # dependency of the provided package. if there is no real package with that
+  # name, then a 'virtual' one is created.  Using this, when a real package
+  # depends on something provided by several real packages, all possible
+  # providers will be downloaded
+  addVirtual = packages:
+    builtins.foldl'
+      (packages: pn:
+        let
+          p = packages.${pn};
+        in
+        if p.provides != null
+        then
+          let
+            vp = if packages ? ${p.provides} then packages.${p.provides} else {
+              type = "virtual";
+              depends = [ ];
+            };
+          in
+          packages // {
+            ${p.provides} = vp // { depends = vp.depends ++ [ pn ]; };
+          }
+        else
+          packages
+      )
+      packages
+      (builtins.attrNames packages);
+
+  # packages which aren't available in feeds but are provided by imagebuilders
+  dummyPackages =
+    let
+      dummyPackage = { depends = [ ]; provides = null; type = "dummy"; };
+    in
+    {
+      libc = dummyPackage;
+      kernel = dummyPackage;
+    };
+
+  # all packages, including dummy and virtual
+  allPackages = addVirtual (realPackages // dummyPackages);
+
+  # remove package names starting with '-' from deps
+  #
+  # this should be used on the dependencies before expanding their
+  # requirements, and assumes that '-' deps come after the non-'-' version in
+  # the list
+  applyMinusDeps = deps:
+    builtins.foldl'
+      (deps: dep:
+        if lib.hasPrefix "-" dep
+        then
+          lib.remove (lib.removePrefix "-" dep) deps
+        else
+          deps ++ [ dep ]
+      ) [ ]
+      deps;
+
+  # given a package set and a list of package names to install (including names
+  # starting with - to be removed), find all possible required package names
+  expandDeps = packages:
+    let
+      addDep = current_deps: new_dep:
+        if builtins.hasAttr new_dep current_deps
+        then
+          current_deps
+        else
+          let
+            with_new_dep = current_deps // { ${new_dep} = true; };
+            deps = packages.${new_dep}.depends;
+          in
+          builtins.foldl' addDep with_new_dep deps;
+    in
+    deps: builtins.attrNames (builtins.foldl' addDep { } (applyMinusDeps deps));
 
   profiles =
     if variantFiles ? "profiles.json"
@@ -123,6 +197,8 @@ let
          else packagesArch;
 
 in {
-  inherit variantFiles variantPackages feedsFiles feedsPackages;
+  inherit allPackages;
+  inherit expandDeps;
+  inherit variantFiles;
   inherit profiles arch;
 }