feat: Harvest should include sizing prediction via harvest-planner

harvest-planner/.dockerignore

@@ -0,0 +1,7 @@
+.dockerignore
+.git
+.idea
+.ruff_cache
+.venv
+objects.json
+ruff.toml

harvest-planner/.gitignore

@@ -0,0 +1,10 @@
+# python generated files
+__pycache__/
+*.py[oc]
+build/
+dist/
+wheels/
+*.egg-info
+
+# venv
+.venv

harvest-planner/.python-version

@@ -0,0 +1 @@
+3.11.9

harvest-planner/Dockerfile

@@ -0,0 +1,24 @@
+FROM python:3.11  AS build-env
+
+ARG version=3.11
+ARG APP=app
+
+COPY . ./$APP
+
+WORKDIR /$APP
+
+COPY requirements.lock /$APP/
+RUN sed '/-e/d' requirements.lock > requirements.txt
+RUN --mount=type=cache,target=/root/.cache \
+    pip install -r ./requirements.txt
+
+FROM gcr.io/distroless/python3-debian12:debug
+ARG version=3.11
+ARG APP=app
+
+COPY --from=build-env /$APP /$APP
+COPY --from=build-env /usr/local/lib/python${version}/site-packages /usr/local/lib/python${version}/site-packages
+
+WORKDIR /$APP
+ENV PYTHONPATH=/usr/local/lib/python${version}/site-packages
+ENTRYPOINT [ "python", "src/harvest_planner/main.py" ]

harvest-planner/README.md

@@ -0,0 +1,30 @@
+# Harvest Planner
+
+Harvest Planner estimates how much memory each poller needs to monitor ONTAP and StorageGRID clusters.
+Here's how to use it:
+
+1. Run the following Harvest command to gather object counts from you cluster(s)
+   `bin/harvest planner -p poller` # one cluster 
+   `bin/harvest planner`           # multiple clusters
+   `bin/harvest planner --docker`  # multiple clusters and run the following Docker command for you
+
+The planner command will create a `objects.json` file that contains the object counts for each cluster.
+
+2. Run the following Docker command to estimate how much memory each poller needs to monitor its cluster.
+
+```bash
+docker run --rm \
+  --volume "$(pwd)/objects.json:/objects.json" \
+  ghcr.io/netapp/harvest-planner \
+  estimate-memory -i /objects.json
+```
+
+# Development
+
+Harvest-planner is written in Python and uses [Rye](https://rye.astral.sh/) for development.
+
+To get started, install Rye, clone the repo, cd into `harvest-metrics/harvest-planner` and run the following command:
+
+```bash
+rye sync
+```

harvest-planner/objects.json.example

@@ -0,0 +1,116 @@
+[
+  {
+    "DiskConfig": 2040,
+    "DiskPerf": 6400,
+    "LunConfig": 0,
+    "LunPerf": 0,
+    "NFSClientsConfig": 0,
+    "Poller": "cluster-01",
+    "QtreeConfig": 1847,
+    "QtreePerf": 0,
+    "SVMConfig": 24,
+    "SensorConfig": 2400,
+    "SnapMirrorConfig": 0,
+    "SnapshotConfig": 180,
+    "StorageGridSG": 0,
+    "VolumeAnalyticsConfig": 0,
+    "VolumeConfig": 30492,
+    "VolumePerf": 9242,
+    "WorkloadDetailVolumePerf": 91296
+  },
+  {
+    "DiskConfig": 240,
+    "DiskPerf": 640,
+    "LunConfig": 0,
+    "LunPerf": 0,
+    "NFSClientsConfig": 0,
+    "Poller": "cluster-02",
+    "QtreeConfig": 0,
+    "QtreePerf": 0,
+    "SVMConfig": 24,
+    "SensorConfig": 0,
+    "SnapMirrorConfig": 0,
+    "SnapshotConfig": 0,
+    "StorageGridSG": 0,
+    "VolumeAnalyticsConfig": 1735,
+    "VolumeConfig": 30492,
+    "VolumePerf": 9242,
+    "WorkloadDetailVolumePerf": 91296
+  },
+  {
+    "DiskConfig": 23,
+    "DiskPerf": 64,
+    "LunConfig": 15,
+    "LunPerf": 6,
+    "NFSClientsConfig": 0,
+    "Poller": "sar",
+    "QtreeConfig": 935,
+    "QtreePerf": 938,
+    "SVMConfig": 87,
+    "SensorConfig": 186,
+    "SnapMirrorConfig": 684,
+    "SnapshotConfig": 1703,
+    "StorageGridSG": 0,
+    "VolumeAnalyticsConfig": 4,
+    "VolumeConfig": 940,
+    "VolumePerf": 922,
+    "WorkloadDetailVolumePerf": 0
+  },
+  {
+    "DiskConfig": 48,
+    "DiskPerf": 63,
+    "LunConfig": 15,
+    "LunPerf": 6,
+    "NFSClientsConfig": 0,
+    "Poller": "F2240-127-26",
+    "QtreeConfig": 862,
+    "QtreePerf": 17,
+    "SVMConfig": 273,
+    "SensorConfig": 0,
+    "SnapMirrorConfig": 89,
+    "SnapshotConfig": 0,
+    "StorageGridSG": 0,
+    "VolumeAnalyticsConfig": 0,
+    "VolumeConfig": 918,
+    "VolumePerf": 908,
+    "WorkloadDetailVolumePerf": 0
+  },
+  {
+    "DiskConfig": 24,
+    "DiskPerf": 66,
+    "LunConfig": 133,
+    "LunPerf": 133,
+    "NFSClientsConfig": 0,
+    "Poller": "nikhita",
+    "QtreeConfig": 0,
+    "QtreePerf": 86,
+    "SVMConfig": 12,
+    "SensorConfig": 242,
+    "SnapMirrorConfig": 1,
+    "SnapshotConfig": 2433,
+    "StorageGridSG": 0,
+    "VolumeAnalyticsConfig": 0,
+    "VolumeConfig": 90,
+    "VolumePerf": 89,
+    "WorkloadDetailVolumePerf": 73
+  },
+  {
+    "DiskConfig": 792,
+    "DiskPerf": 1144,
+    "LunConfig": 0,
+    "LunPerf": 0,
+    "NFSClientsConfig": 0,
+    "Poller": "nasclu01",
+    "QtreeConfig": 325,
+    "QtreePerf": 4228,
+    "SVMConfig": 92,
+    "SensorConfig": 1116,
+    "SnapMirrorConfig": 283,
+    "SnapshotConfig": 0,
+    "StorageGridSG": 0,
+    "VolumeAnalyticsConfig": 1,
+    "VolumeConfig": 4028,
+    "VolumePerf": 4020,
+    "WorkloadDetailVolumePerf": 0
+  }
+]

harvest-planner/pyproject.toml

@@ -0,0 +1,27 @@
+[project]
+name = "harvest-planner"
+version = "0.1.0"
+description = "Harvest Planner estimates how much memory each poller needs to monitor ONTAP and StorageGRID clusters"
+dependencies = [
+    "joblib>=1.4.2",
+    "numpy>=2.0.0",
+    "pandas>=2.2.2",
+    "scikit-learn>=1.5.0",
+    "scipy>=1.13.1",
+]
+readme = "README.md"
+requires-python = ">= 3.8"
+
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[tool.rye]
+managed = true
+dev-dependencies = []
+
+[tool.hatch.metadata]
+allow-direct-references = true
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/harvest_planner"]

harvest-planner/requirements-dev.lock

@@ -0,0 +1,36 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+
+-e file:.
+joblib==1.4.2
+    # via harvest-planner
+    # via scikit-learn
+numpy==2.0.0
+    # via harvest-planner
+    # via pandas
+    # via scikit-learn
+    # via scipy
+pandas==2.2.2
+    # via harvest-planner
+python-dateutil==2.9.0.post0
+    # via pandas
+pytz==2024.1
+    # via pandas
+scikit-learn==1.5.0
+    # via harvest-planner
+scipy==1.14.0
+    # via harvest-planner
+    # via scikit-learn
+six==1.16.0
+    # via python-dateutil
+threadpoolctl==3.5.0
+    # via scikit-learn
+tzdata==2024.1
+    # via pandas

harvest-planner/requirements.lock

@@ -0,0 +1,36 @@
+# generated by rye
+# use `rye lock` or `rye sync` to update this lockfile
+#
+# last locked with the following flags:
+#   pre: false
+#   features: []
+#   all-features: false
+#   with-sources: false
+#   generate-hashes: false
+
+-e file:.
+joblib==1.4.2
+    # via harvest-planner
+    # via scikit-learn
+numpy==2.0.0
+    # via harvest-planner
+    # via pandas
+    # via scikit-learn
+    # via scipy
+pandas==2.2.2
+    # via harvest-planner
+python-dateutil==2.9.0.post0
+    # via pandas
+pytz==2024.1
+    # via pandas
+scikit-learn==1.5.0
+    # via harvest-planner
+scipy==1.14.0
+    # via harvest-planner
+    # via scikit-learn
+six==1.16.0
+    # via python-dateutil
+threadpoolctl==3.5.0
+    # via scikit-learn
+tzdata==2024.1
+    # via pandas

harvest-planner/ruff.toml

@@ -0,0 +1,26 @@
+[lint]
+select = [
+    "ARG",
+    "B",
+    "E",
+    "ERA",
+    "F",
+    "I",
+    "ISC",
+    "NPY",
+    "PD",
+    "PERF",
+    "PIE",
+    "PL",
+    "PTH",
+    "PYI",
+    "Q",
+    "RET",
+    "RUF",
+    "S",
+    "SIM",
+    "UP",
+    "YTT",
+]
+
+ignore = ["ISC001"]