Merge pull request #1813 from sjhloco/validate_tolerance_percentage

Add tolerance percentage to validate

docs/validate/index.rst

@@ -95,7 +95,9 @@ A few notes:
     * We can also use comparison on the conditions of numerical validate. For example, if you want 
       to validate there that the ``cpu``and ``memory`` into ``get_environment`` are ``15%`` or less.
       We can use writing comparison operators such as ``<15.0`` or ``>10.0`` in this case, or range
-      with the operator syntax of ``<->`` such as ``10.0<->20.0`` or ``10<->20``.
+      with the operator syntax of ``<->`` such as ``10.0<->20.0`` or ``10<->20``. In a similar vain
+      a percentage tolerance can be validated upon, for example ``10%20`` allows a 10% tolerance
+      either side of 20 (a range of 18 to 22).
     * Some methods require extra arguments, for example ``ping``. You can pass arguments to those
       methods using the magic keyword ``_kwargs``. In addition, an optional keyword ``_name`` can
       be specified to override the name in the report. Useful for having a more descriptive report

napalm/base/validate.py

@@ -6,6 +6,7 @@
 import yaml
 import copy
 import re
+from math import isclose
 from typing import Dict, List, Union, TypeVar, Optional, TYPE_CHECKING
 
 if TYPE_CHECKING:
@@ -16,6 +17,7 @@
 
 # We put it here to compile it only once
 numeric_compare_regex = re.compile(r"^(<|>|<=|>=|==|!=)(\d+(\.\d+){0,1})$")
+numeric_tolerance_regex = re.compile(r"^(\d+)%(\d+)$")
 
 
 def _get_validation_file(validation_file: str) -> Dict[str, Dict]:
@@ -155,6 +157,9 @@ def compare(
         elif "<->" in src and len(src.split("<->")) == 2:
             cmp_result = _compare_range(src, dst)
             return cmp_result
+        elif re.search(r"^\d+%\d+$", src):
+            cmp_result = _compare_tolerance(src, dst)
+            return cmp_result
         else:
             m = re.search(src, str(dst))
             if m:
@@ -214,6 +219,22 @@ def _compare_range(src_num: str, dst_num: str) -> bool:
         return False
 
 
+def _compare_tolerance(src_num: str, dst_num: str) -> bool:
+    """Compare against a tolerance percentage either side. You can use 't%%d'."""
+    dst_num = float(dst_num)
+
+    match = numeric_tolerance_regex.match(src_num)
+    if not match:
+        error = "Failed tolerance comparison. Collected: {}. Expected: {}".format(
+            dst_num, src_num
+        )
+        raise ValueError(error)
+
+    src_num = float(match.group(2))
+    max_diff = src_num * int(match.group(1)) / 100
+    return isclose(src_num, dst_num, abs_tol=max_diff)
+
+
 def empty_tree(input_list: List) -> bool:
     """Recursively iterate through values in nested lists."""
     for item in input_list:

test/base/validate/mocked_data/non_strict_fail/get_bgp_neighbors.json

@@ -38,6 +38,22 @@
             "received_prefixes": 0
           }
         }
+      },
+      "192.0.2.4": {
+        "is_enabled": true,
+        "uptime": -1,
+        "remote_as": 65020,
+        "description": "",
+        "remote_id": "192.168.0.4",
+        "local_as": 65000,
+        "is_up": false,
+        "address_family": {
+          "ipv4": {
+            "sent_prefixes": 19,
+            "accepted_prefixes": 0,
+            "received_prefixes": 12
+          }
+        }
       }
     }
   }

test/base/validate/mocked_data/non_strict_fail/report.yml

@@ -49,6 +49,34 @@ get_bgp_neighbors:
                       is_enabled: {complies: true, nested: false}
                   nested: true
                 192.0.2.3: {complies: true, nested: true}
+                192.0.2.4:
+                  complies: false
+                  diff:
+                    complies: false
+                    extra: []
+                    missing: []
+                    present:
+                      address_family:
+                        complies: false
+                        diff:
+                          complies: false
+                          extra: []
+                          missing: []
+                          present:
+                            ipv4:
+                              complies: false
+                              diff:
+                                complies: false
+                                extra: []
+                                missing: []
+                                present:
+                                  received_prefixes: {actual_value: 12, expected_value: "10%10", complies: False,
+                                  nested: false}
+                                  sent_prefixes: {complies: true, nested: False}
+                              nested: true
+                        nested: true
+                      is_enabled: {complies: true, nested: false}
+                  nested: true
             nested: true
           router_id: {actual_value: 192.0.2.2, expected_value: 192.6.6.6, complies: false, nested: false}
       nested: true

test/base/validate/mocked_data/non_strict_fail/validate.yml

@@ -23,6 +23,12 @@
               sent_prefixes: 5
             ipv6:
               sent_prefixes: 6
+        192.0.2.4:
+          is_enabled: true
+          address_family:
+            ipv4:
+              sent_prefixes: "5%20"
+              received_prefixes: "10%10"
 
 - get_interfaces_ip:
     Ethernet2/1:
@@ -35,9 +41,9 @@
       destination: 185.155.180.192/26
     "10.155.180.192/26":
       list:
-       - next_hop: 10.155.180.22
-         outgoing_interface: "irb.0"
-         protocol: "OSPF"
+        - next_hop: 10.155.180.22
+          outgoing_interface: "irb.0"
+          protocol: "OSPF"
 
 - get_environment:
     memory:

test/base/validate/mocked_data/non_strict_pass/get_bgp_neighbors.json

@@ -38,6 +38,22 @@
             "received_prefixes": 0
           }
         }
+      },
+      "192.0.2.4": {
+        "is_enabled": true,
+        "uptime": -1,
+        "remote_as": 65020,
+        "description": "",
+        "remote_id": "192.168.0.4",
+        "local_as": 65000,
+        "is_up": false,
+        "address_family": {
+          "ipv4": {
+            "sent_prefixes": 19,
+            "accepted_prefixes": 0,
+            "received_prefixes": 12
+          }
+        }
       }
     }
   }

test/base/validate/mocked_data/non_strict_pass/validate.yml

@@ -21,6 +21,12 @@
               sent_prefixes: 5
             ipv6:
               sent_prefixes: 2
+        192.0.2.4:
+          is_enabled: true
+          address_family:
+            ipv4:
+              sent_prefixes: "5%20"
+              received_prefixes: "20%10"
 
 - get_interfaces_ip:
     Ethernet2/1:
@@ -33,9 +39,9 @@
       destination: 185.155.180.192/26
     "10.155.180.192/26":
       list:
-       - next_hop: 10.155.180.22
-         outgoing_interface: "irb.0"
-         protocol: "BGP"
+        - next_hop: 10.155.180.22
+          outgoing_interface: "irb.0"
+          protocol: "BGP"
 
 - get_environment:
     memory:

test/base/validate/test_unit.py

@@ -414,6 +414,8 @@ def test_numeric_comparison(self):
         assert validate._compare_numeric("<=2", 2)
         assert validate._compare_numeric("<3", "2")
         assert validate._compare_numeric("!=3", "2")
+        assert validate._compare_tolerance("10%20", 18)
+        assert validate._compare_tolerance("10%20", 22)
         with pytest.raises(ValueError):
             assert validate._compare_numeric("a2a", 2)
         with pytest.raises(ValueError):