Merge pull request #281 from DavHau/dev

version 3.3.0
DavHau · May 23, 2021 · 773580c · 773580c
2 parents 97d7448 + 40e1302
commit 773580c
Show file tree

Hide file tree

Showing 13 changed files with 87 additions and 56 deletions.
diff --git a/Changelog.md b/Changelog.md
@@ -1,3 +1,34 @@
+# 3.3.0 (22 May 2021)
+bugfixes, flakes improvements
+
+### Changes
+ - The flakes cmdline api has been changed. New usage:
+   ```
+   nix (build|shell) mach-nix#gen.(python|docker).package1.package2...
+   ```
+   (Despite this changes being backward incompatible, I did not bump the major version since everything flakes related should be considered experimental anyways)
+
+### Improvements
+ - Mach-nix (used via flakes) will now throw an error if the selected nixpkgs version is newer than the dependency DB since this can cause conflicts in the resulting environment.
+ - When used via flakes, it was impossible to select the python version because the import function is not used anymore. Now `python` can be passed to `mkPython` alternatively.
+ - For the flakes cmdline api, collisions are now ignored by default
+ - The simplified override interface did not deal well with non-existent values. 
+    - Now the `.add` directive automatically assumes an empty list/set/string when the attribute to be extended doesn't exist.
+    - Now the `.mod` directive will pass `null` to the given function if the attribute to modify doesn't exist instead.
+
+### Fixes
+ - Generating an environment with a package named `overrides` failed due to a variable name collision in the resulting nix expression.
+ - When used via flakes, the pypiData was downloaded twice, because the legacy code path for fetching was still used instead of the flakes input.
+ - `nix flake show mach-nix` failed because it required IFD for foreign platforms.
+ - For environments generated via `mach-nix env ...` the `python` command referred to the wrong interpreter.
+ - When checking wheels for compatibility, the minor version for python was not respected which could lead to invalid environments.
+ - Some python modules in nixpkgs propagate unnecessary dependencies which could lead to collisions in the final environment. Now mach-nix recursively removes all python dependencies which are not strictly required.
+
+### Package Fixes
+ - cryptography: remove rust related hook when version < 3.4
+
+
+
 # 3.2.0 (11 Mar 2021)
 bugfixes, ignoreCollisions
 

diff --git a/Readme.md b/Readme.md
@@ -63,11 +63,11 @@ Table of Contents
 You can either install mach-nix via pip or by using nix in case you already have the nix package manager installed.
 #### Installing via pip
 ```shell
-pip install git+git://github.com/DavHau/mach-nix@3.2.0
+pip install git+git://github.com/DavHau/mach-nix@3.3.0
 ```
 #### Installing via nix
 ```shell
-nix-env -if https://github.com/DavHau/mach-nix/tarball/3.2.0 -A mach-nix
+nix-env -if https://github.com/DavHau/mach-nix/tarball/3.3.0 -A mach-nix
 ```
 or, if you prefer `nix-shell`:
 
@@ -80,7 +80,7 @@ or, if you prefer `nix-shell`:
 + otherwise:
 
   ```shell
-  nix-shell -p '(callPackage (fetchTarball https://github.com/DavHau/mach-nix/tarball/3.2.0) {}).mach-nix'
+  nix-shell -p '(callPackage (fetchTarball https://github.com/DavHau/mach-nix/tarball/3.3.0) {}).mach-nix'
   ```
 
 ---
@@ -112,7 +112,7 @@ You can call mach-nix directly from a nix expression
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "refs/tags/3.2.0";
+    ref = "refs/tags/3.3.0";
   }) {};
 in
 mach-nix.mkPython {

diff --git a/examples.md b/examples.md
@@ -45,7 +45,7 @@ every mach-nix expression should begin like this:
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "refs/tags/3.2.0";
+    ref = "refs/tags/3.3.0";
   }) {
     # optionally bring your own nixpkgs
     # pkgs = import <nixpkgs> {};
@@ -281,7 +281,7 @@ In this example, mach-nix is used to resolve our python dependencies and provide
 let
   mach-nix = import (builtins.fetchGit {
     url = "https://github.com/DavHau/mach-nix/";
-    ref = "refs/tags/3.2.0";  # update this version
+    ref = "refs/tags/3.3.0";  # update this version
   }) {
     python = "python37";
   };

diff --git a/interpreter.nix b/interpreter.nix
@@ -1,7 +1,7 @@
 # python interpreter for dev environment
 let
   pkgs = import (import ./mach_nix/nix/nixpkgs-src.nix) { config = {}; };
-  python = pkgs.python37;
+  python = pkgs.python38;
   deps = (pkgs.lib.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = pkgs.fetchurl; }));
 in
 python.withPackages (ps: deps ++ [

diff --git a/mach_nix/VERSION b/mach_nix/VERSION
@@ -1 +1 @@
-master
+3.3.0
diff --git a/mach_nix/fixes.nix b/mach_nix/fixes.nix
@@ -20,28 +20,36 @@ let
     result = parse('${ver1}') ${op} parse('${ver2}')
     print(json.dumps(result))
   '';
+
+  # prevent the normal fitler from being used
+  filter = raise "Error: only use filterSafe (doesn't crash on null input)";
+
+  # filter function that doesn't crash on 'objects = null'
+  # This is important because null will be returned whenever the attribute to modify doesn't exist
+  filterSafe = func: objects: if isNull objects then [] else builtins.filter func objects;
+
 in
 
 
 ### Put Fixes here
 rec {
 
-###  FORMAT  ############################################################################
-#                                                                                       #
-#   package-to-fix = {                                                                  #
-#     name-of-the-fix = {                                                               #
-#       # optionally limit the fix to a condtion                                        #
-#       _cond = {prov, ver, ... }: some boolean expression;                             #
-#                                                                                       #
-#       # define overrides                                                              #
-#       key-to-override = ...;                                 # replace                #
-#       key-to-override.add = ...;                             # append                 #
-#       key-to-override.mod = oldVal: ...;                     # modify                 #
-#       key-to-override.mod = pySelf: oldAttrs: oldVal: ...;   # modify (more args)     #
-#     };                                                                                #
-#   };                                                                                  #
-#                                                                                       #
-#########################################################################################
+###  FORMAT  ##################################################################################
+#                                                                                             #
+#   package-to-fix = {                                                                        #
+#     name-of-the-fix = {                                                                     #
+#       # optionally limit the fix to a condtion                                              #
+#       _cond = {prov, ver, ... }: some boolean expression;                                   #
+#                                                                                             #
+#       # define overrides                                                                    #
+#       key-to-override = ...;                                 # replace                      #
+#       key-to-override.add = ...;                             # append (list/attrs/string)   #
+#       key-to-override.mod = oldVal: ...;                     # modify                       #
+#       key-to-override.mod = pySelf: oldAttrs: oldVal: ...;   # modify (accessing all pkgs)  #
+#     };                                                                                      #
+#   };                                                                                        #
+#                                                                                             #
+###############################################################################################
 
 ### _cond ####################################
 #  possible arguments:                       #
@@ -57,7 +65,7 @@ rec {
 
   cryptography.no-rust-build = {
     _cond = { prov, ver, ... }: prov == "sdist" && comp_ver ver "<" "3.4";
-    nativeBuildInputs.mod = old: filter (inp: (inp.name or "") != "cargo-setup-hook.sh") old;
+    nativeBuildInputs.mod = old: filterSafe (inp: (inp.name or "") != "cargo-setup-hook.sh") old;
   };
 
   # remove if merged: https://github.com/NixOS/nixpkgs/pull/114384
@@ -83,15 +91,15 @@ rec {
 
   pip.remove-reproducible-patch = {
     _cond = { prov, ver, ... }: prov == "sdist" && comp_ver ver "<" "20.0";
-    patches.mod = oldPatches: filter (patch: ! hasSuffix "reproducible.patch" patch) oldPatches;
+    patches.mod = oldPatches: filterSafe (patch: ! hasSuffix "reproducible.patch" patch) oldPatches;
   };
 
   pyqt5 = {
     fix-build-inputs = {
       # fix mach-nix induced problem: mach-nix removes all previous python inputs from propagatedBuildInputs
       _cond = {prov, ... }: prov == "nixpkgs";
       propagatedBuildInputs.mod = pySelf: oldAttrs: oldVal:
-        (filter (p: p.pname != "pyqt5-sip") oldVal) ++ [ pySelf.sip pySelf.dbus-python ];
+        (filterSafe (p: p.pname != "pyqt5-sip") oldVal) ++ [ pySelf.sip pySelf.dbus-python ];
     };
     fix-wheel-inputs = {
       _cond = {prov, ... }: prov == "wheel";
@@ -106,7 +114,7 @@ rec {
       # https://github.com/rpy2/rpy2/commit/39e1cb6fca0d4107f1078727d8670c422e3c6f7f
       prov == "sdist"
       && comp_ver ver ">=" "3.2.6";
-    patches.mod = oldPatches: filter (p: ! hasSuffix "pandas-1.x.patch" p) oldPatches;
+    patches.mod = oldPatches: filterSafe (p: ! hasSuffix "pandas-1.x.patch" p) oldPatches;
   };
 
   tensorflow.rm-tensorboard = {

diff --git a/mach_nix/flake.lock b/mach_nix/flake.lock
diff --git a/mach_nix/nix/compileOverrides.nix b/mach_nix/nix/compileOverrides.nix
@@ -12,8 +12,8 @@ let
   nixpkgs_json = import ./nixpkgs-json.nix {
     inherit overrides pkgs python;
   };
-  builder_python = pkgs.pkgsBuildHost.python37.withPackages(ps:
-    (pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python37; fetchurl = pkgs.fetchurl; }))
+  builder_python = pkgs.pkgsBuildHost.python38.withPackages(ps:
+    (pkgs.lib.attrValues (import ./python-deps.nix {python = pkgs.python38; fetchurl = pkgs.fetchurl; }))
   );
   src = ./../../.;
   db_and_fetcher = import ./deps-db-and-fetcher.nix {

diff --git a/mach_nix/nix/deps-db-and-fetcher.nix b/mach_nix/nix/deps-db-and-fetcher.nix
@@ -21,18 +21,11 @@ let
     inherit pkgs;
   };
 in
-let
-  pypi_deps_db_src = pkgs.buildEnv {
-    name = "pypi-deps-db-src";
-    paths = [ deps_db_src ];
-  };
-in
-{ inherit
-    pypi_deps_db_src
+{
+  pypi_deps_db_src = deps_db_src;
+  inherit
     pypi_fetcher_src
-
     pypi_fetcher_commit
     pypi_fetcher_sha256
-
     pypi_fetcher;
 }
diff --git a/mach_nix/nix/lib.nix b/mach_nix/nix/lib.nix
@@ -91,9 +91,9 @@ rec {
   };
 
   combine = pname: key: val1: val2:
-    if isList val2 then val1 ++ val2
-    else if isAttrs val2 then val1 // val2
-    else if isString val2 then val1 + val2
+    if isList val2 then (if ! isNull val1 then val1 else []) ++ val2
+    else if isAttrs val2 then (if ! isNull val1 then val1 else {}) // val2
+    else if isString val2 then (if ! isNull val1 then val1 else "") + val2
     else throw "_.${pname}.${key}.add only accepts list or attrs or string.";
 
   fixes_to_overrides = fixes:
@@ -107,14 +107,14 @@ rec {
               mapAttrs (key: val:
                 trace "\napplying fix '${fix}' (${key}) for ${pkg}:${oa.version}\n" (
                   if isAttrs val && hasAttr "add" val then
-                    combine pkg key oa."${key}" val.add
+                    combine pkg key (oa."${key}" or null) val.add
                   else if isAttrs val && hasAttr "mod" val && isFunction val.mod then
-                    let result = val.mod oa."${key}"; in
+                    let result = val.mod (oa."${key}" or null); in
                       # if the mod function wants more argument, call with more arguments (alternative style)
                       if ! isFunction result then
                         result
                       else
-                          val.mod pySelf oa oa."${key}"
+                          val.mod pySelf oa (oa."${key}" or null)
                   else
                     val
                 )

diff --git a/mach_nix/nix/python.nix b/mach_nix/nix/python.nix
@@ -4,7 +4,7 @@
 }:
 with pkgs;
 let
-  python = python37;
+  python = python38;
   python_deps = (lib.attrValues (import ./python-deps.nix { inherit python; fetchurl = fetchurl; }));
 in
 python.withPackages ( ps: python_deps )
diff --git a/shell.nix b/shell.nix
@@ -4,7 +4,7 @@
 }:
 with pkgs;
 let
-  python = python37;
+  python = python38;
   machnixDeps = (lib.attrValues (import ./mach_nix/nix/python-deps.nix { inherit python; fetchurl = fetchurl; }));
 in
 mkShell {

diff --git a/tests/test_flakes.nix b/tests/test_flakes.nix
@@ -5,7 +5,6 @@
 with builtins;
 let
   pyEnv = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.python.requests;
-  pyEnvShell = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.shell.requests;
   pyEnvDockerImage = (builtins.getFlake (toString ../.)).packages.x86_64-linux.gen.docker.requests;
 in
 (map (p:
@@ -15,7 +14,7 @@ in
     throw "Error"
 ) [pyEnv pyEnvDockerImage])
 ++ [
-  (if ! pyEnvShell ? _passthru.python.pkgs.requests then
+  (if ! pyEnv ? _passthru.python.pkgs.requests then
     throw "Error with shell"
   else
     mach-nix.nixpkgs.hello)