NAS-128003 / 24.10 / Update zvol collection method

src/freenas/usr/local/bin/snmp-agent.py

@@ -2,7 +2,7 @@
 import threading
 import time
 import contextlib
-import pathlib
+import os
 
 import libzfs
 import netsnmpagent
@@ -411,13 +411,10 @@ def report_zfs_info(prev_zpool_info):
 
 def get_list_of_zvols():
     zvols = set()
-    root_dir = '/dev/zvol/'
     with contextlib.suppress(FileNotFoundError):  # no zvols
-        for zpool in pathlib.Path(root_dir).iterdir():
-            for zvol in filter(lambda x: '@' not in x.name, zpool.iterdir()):
-                zvol_normalized = zvol.as_posix().removeprefix(root_dir)
-                zvol_normalized = zvol_normalized.replace('+', ' ')
-                zvols.add(zvol_normalized)
+        for root_dir, unused_dirs, files in os.walk('/dev/zvol/'):
+            for file in filter(lambda x: '@' not in x, files):
+                zvols.add(os.path.join(root_dir, file).removeprefix(root_dir).replace('+', ' '))
 
     return list(zvols)
 

src/freenas/usr/local/share/pysnmp/mibs/TRUENAS-MIB.py

@@ -1,6 +1,6 @@
 # PySNMP SMI module. Autogenerated from smidump -f python TRUENAS-MIB
-# by libsmi2pysnmp-0.1.3 at Fri Aug 18 13:42:49 2023,
-# Python version sys.version_info(major=2, minor=7, micro=17, releaselevel='final', serial=0)
+# by libsmi2pysnmp-0.1.3 at Wed Jul 24 12:51:26 2024,
+# Python version sys.version_info(major=3, minor=11, micro=2, releaselevel='final', serial=0)
 
 # Imports
 
@@ -13,7 +13,7 @@
 # Types
 
 class AlertLevelType(Integer):
-    subtypeSpec = Integer.subtypeSpec+SingleValueConstraint(1,2,3,5,7,4,6,)
+    subtypeSpec = Integer.subtypeSpec+SingleValueConstraint(1,2,3,4,5,6,7,)
     namedValues = NamedValues(("info", 1), ("notice", 2), ("warning", 3), ("error", 4), ("critical", 5), ("alert", 6), ("emergency", 7), )
 
 
@@ -80,11 +80,11 @@ class AlertLevelType(Integer):
 zfsArcC = MibScalar((1, 3, 6, 1, 4, 1, 50536, 1, 3, 6), Gauge32()).setMaxAccess("readonly")
 if mibBuilder.loadTexts: zfsArcC.setDescription("")
 zfsArcMissPercent = MibScalar((1, 3, 6, 1, 4, 1, 50536, 1, 3, 8), DisplayString()).setMaxAccess("readonly")
-if mibBuilder.loadTexts: zfsArcMissPercent.setDescription("Arc Miss Percentage.\n(Note: Floating precision sent across SNMP as a String")
+if mibBuilder.loadTexts: zfsArcMissPercent.setDescription("Arc Miss Percentage.\nNote: Floating precision sent across SNMP as a String")
 zfsArcCacheHitRatio = MibScalar((1, 3, 6, 1, 4, 1, 50536, 1, 3, 9), DisplayString()).setMaxAccess("readonly")
-if mibBuilder.loadTexts: zfsArcCacheHitRatio.setDescription("Arc Cache Hit Ration Percentage.\n(Note: Floating precision sent across SNMP as a String")
+if mibBuilder.loadTexts: zfsArcCacheHitRatio.setDescription("Arc Cache Hit Ration Percentage.\nNote: Floating precision sent across SNMP as a String")
 zfsArcCacheMissRatio = MibScalar((1, 3, 6, 1, 4, 1, 50536, 1, 3, 10), DisplayString()).setMaxAccess("readonly")
-if mibBuilder.loadTexts: zfsArcCacheMissRatio.setDescription("Arc Cache Miss Ration Percentage.\n(Note: Floating precision sent across SNMP as a String")
+if mibBuilder.loadTexts: zfsArcCacheMissRatio.setDescription("Arc Cache Miss Ration Percentage.\nNote: Floating precision sent across SNMP as a String")
 l2arc = MibIdentifier((1, 3, 6, 1, 4, 1, 50536, 1, 4))
 zfsL2ArcHits = MibScalar((1, 3, 6, 1, 4, 1, 50536, 1, 4, 1), Counter32()).setMaxAccess("readonly")
 if mibBuilder.loadTexts: zfsL2ArcHits.setDescription("")
@@ -127,7 +127,7 @@ class AlertLevelType(Integer):
 
 # Notifications
 
-alert = NotificationType((1, 3, 6, 1, 4, 1, 50536, 2, 1, 1)).setObjects(*(("TRUENAS-MIB", "alertMessage"), ("TRUENAS-MIB", "alertLevel"), ("TRUENAS-MIB", "alertId"), ) )
+alert = NotificationType((1, 3, 6, 1, 4, 1, 50536, 2, 1, 1)).setObjects(*(("TRUENAS-MIB", "alertId"), ("TRUENAS-MIB", "alertLevel"), ("TRUENAS-MIB", "alertMessage"), ) )
 if mibBuilder.loadTexts: alert.setDescription("An alert raised")
 alertCancellation = NotificationType((1, 3, 6, 1, 4, 1, 50536, 2, 1, 2)).setObjects(*(("TRUENAS-MIB", "alertId"), ) )
 if mibBuilder.loadTexts: alertCancellation.setDescription("An alert cancelled")

src/freenas/usr/local/share/snmp/mibs/TRUENAS-MIB.txt

@@ -293,7 +293,7 @@ zfsArcMissPercent OBJECT-TYPE
     STATUS     current
     DESCRIPTION
         "Arc Miss Percentage.
-        (Note: Floating precision sent across SNMP as a String"
+        Note: Floating precision sent across SNMP as a String"
     ::= { arc 8 }
 
 zfsArcCacheHitRatio OBJECT-TYPE
@@ -302,7 +302,7 @@ zfsArcCacheHitRatio OBJECT-TYPE
     STATUS     current
     DESCRIPTION
         "Arc Cache Hit Ration Percentage.
-        (Note: Floating precision sent across SNMP as a String"
+        Note: Floating precision sent across SNMP as a String"
     ::= { arc 9 }
 
 zfsArcCacheMissRatio OBJECT-TYPE
@@ -311,7 +311,7 @@ zfsArcCacheMissRatio OBJECT-TYPE
     STATUS     current
     DESCRIPTION
         "Arc Cache Miss Ration Percentage.
-        (Note: Floating precision sent across SNMP as a String"
+        Note: Floating precision sent across SNMP as a String"
     ::= { arc 10 }
 
 zfsL2ArcHits OBJECT-TYPE

src/middlewared/middlewared/test/integration/assets/filesystem.py

@@ -11,3 +11,26 @@ def directory(path, options=None):
         yield path
     finally:
         ssh(f'rm -rf {path}')
+
+
+@contextlib.contextmanager
+def mkfile(path, size=None):
+    """
+    Create a simple file
+    * path is the full-pathname. e.g. /mnt/tank/dataset/filename
+    * If size is None then use 'touch',
+      else create a random filled file of size bytes.
+      Creation will be faster if size is a power of 2, e.g. 1024 or 1048576
+    TODO: sparse files, owner, permissions
+    """
+    try:
+        if size is None:
+            ssh(f"touch {path}")
+        else:
+            t = 1048576
+            while t > 1 and size % t != 0:
+                t = t // 2
+            ssh(f"dd if=/dev/urandom of={path} bs={t} count={size // t}")
+        yield path
+    finally:
+        ssh(f"rm -rf {path}")

tests/api2/test_440_snmp.py

@@ -6,15 +6,18 @@
 
 from time import sleep
 
+from contextlib import ExitStack
 from middlewared.service_exception import ValidationErrors
+from middlewared.test.integration.assets.pool import dataset, snapshot
+from middlewared.test.integration.assets.filesystem import directory, mkfile
 from middlewared.test.integration.utils import call, ssh
 from middlewared.test.integration.utils.client import truenas_server
 from middlewared.test.integration.utils.system import reset_systemd_svcs
 from pysnmp.hlapi import (CommunityData, ContextData, ObjectIdentity,
                           ObjectType, SnmpEngine, UdpTransportTarget, getCmd)
 
 
-from auto_config import ha, interface, password, user
+from auto_config import ha, interface, password, user, pool_name
 from functions import async_SSH_done, async_SSH_start
 
 skip_ha_tests = pytest.mark.skipif(not (ha and "virtual_ip" in os.environ), reason="Skip HA tests")
@@ -97,6 +100,66 @@ def add_SNMPv3_user():
     yield
 
 
+@pytest.fixture(scope='function')
+def create_nested_structure():
+    """
+    Create the following structure:
+        tank -+-> dataset_1 -+-> dataset2 -+-> dataset_3
+              |-> zvol_1a    |-> zvol_2a   |-> zvol_3a
+              |-> zvol_1b    |-> zvol_2b   |-> zvol_3b
+              |-> file_1     |-> file_2    |-> file_3
+              |-> dir_1      |-> dir_2     |-> dir_3
+    TODO: Make this generic and move to assets
+    """
+    ds_path = ""
+    ds_list = []
+    zv_list = []
+    dir_list = []
+    file_list = []
+    with ExitStack() as es:
+
+        for i in range(1, 4):
+            preamble = f"{ds_path + '/' if i > 1 else ''}"
+            vol_path = f"{preamble}zvol_{i}"
+
+            # Create zvols
+            for c in crange('a', 'b'):
+                zv = es.enter_context(dataset(vol_path + c, {"type": "VOLUME", "volsize": 1048576}))
+                zv_list.append(zv)
+
+            # Create directories
+            d = es.enter_context(directory(f"/mnt/{pool_name}/{preamble}dir_{i}"))
+            dir_list.append(d)
+
+            # Create files
+            f = es.enter_context(mkfile(f"/mnt/{pool_name}/{preamble}file_{i}", 1048576))
+            file_list.append(f)
+
+            # Create datasets
+            ds_path += f"{'/' if i > 1 else ''}dataset_{i}"
+            ds = es.enter_context(dataset(ds_path))
+            ds_list.append(ds)
+
+        yield {'zv': zv_list, 'ds': ds_list, 'dir': dir_list, 'file': file_list}
+
+
+def crange(c1, c2):
+    """
+    Generates the characters from `c1` to `c2`, inclusive.
+    Simple lowercase ascii only.
+    NOTE: Not safe for runtime code
+    """
+    ord_a = 97
+    ord_z = 122
+    c1_ord = ord(c1)
+    c2_ord = ord(c2)
+    assert c1_ord < c2_ord, f"'{c1}' must be 'less than' '{c2}'"
+    assert ord_a <= c1_ord <= ord_z
+    assert ord_a <= c2_ord <= ord_z
+    for c in range(c1_ord, c2_ord + 1):
+        yield chr(c)
+
+
 def get_systemctl_status(service):
     """ Return 'RUNNING' or 'STOPPED' """
     try:
@@ -173,6 +236,20 @@ def user_list_users(snmp_config):
     return [x.split()[-1].strip('\"') for x in res.splitlines()]
 
 
+def v2c_snmpwalk(mib):
+    """
+    Run snmpwalk with v2c protocol
+    mib is the item to be gathered.  mib format examples:
+        iso.3.6.1.6.3.15.1.2.2.1.3
+        1.3.6.1.4.1.50536.1.2
+    """
+    cmd = f"snmpwalk -v2c -cpublic localhost {mib}"
+
+    # This call will timeout if SNMP is not running
+    res = ssh(cmd)
+    return [x.split()[-1].strip('\"') for x in res.splitlines()]
+
+
 # =====================================================================
 #                           Tests
 # =====================================================================
@@ -349,3 +426,18 @@ def test_SNMPv3_user_delete(self):
             with pytest.raises(Exception) as ve:
                 res = user_list_users(SNMP_USER_CONFIG)
             assert "Unknown user name" in str(ve.value)
+
+    def test_zvol_reporting(self, create_nested_structure):
+        """
+        The TrueNAS snmp agent should list all zvols.
+        TrueNAS zvols can be created on any ZFS pool or dataset.
+        The snmp agent should list them all.
+        snmpwalk -v2c -cpublic localhost 1.3.6.1.4.1.50536.1.2.1.1.2
+        """
+        # The expectation is that the snmp agent should list exactly the six zvols.
+        created_items = create_nested_structure
+
+        # Include a snapshot of one of the zvols
+        with snapshot(created_items['zv'][0], "snmpsnap01"):
+            snmp_res = v2c_snmpwalk('1.3.6.1.4.1.50536.1.2.1.1.2')
+            assert all(v in created_items['zv'] for v in snmp_res), f"expected {created_items['zv']}, but found {snmp_res}"