diff --git a/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_netcdf.ipynb b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_netcdf.ipynb new file mode 100644 index 000000000..8ad14d6e4 --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_netcdf.ipynb @@ -0,0 +1,480 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CDAT Migration Regression Testing Notebook (`.nc` files)\n", + "\n", + "This notebook is used to perform regression testing between the development and\n", + "production versions of a diagnostic set.\n", + "\n", + "## How it works\n", + "\n", + "It compares the relative differences (%) between ref and test variables between\n", + "the dev and `main` branches.\n", + "\n", + "## How to use\n", + "\n", + "PREREQUISITE: The diagnostic set's netCDF stored in `.json` files in two directories\n", + "(dev and `main` branches).\n", + "\n", + "1. Make a copy of this notebook under `auxiliary_tools/cdat_regression_testing/`.\n", + "2. Run `mamba create -n cdat_regression_test -y -c conda-forge \"python<3.12\" xarray netcdf4 dask pandas matplotlib-base ipykernel`\n", + "3. Run `mamba activate cdat_regression_test`\n", + "4. Update `SET_DIR` and `SET_NAME` in the copy of your notebook.\n", + "5. Run all cells IN ORDER.\n", + "6. Review results for any outstanding differences (>=1e-5 relative tolerance).\n", + " - Debug these differences (e.g., bug in metrics functions, incorrect variable references, etc.)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Code\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "\n", + "import numpy as np\n", + "import xarray as xr\n", + "from e3sm_diags.derivations.derivations import DERIVED_VARIABLES\n", + "\n", + "\n", + "# TODO: Update SET_NAME and SET_DIR\n", + "SET_NAME = \"tropical_subseasonal\"\n", + "SET_DIR = \"886-jjb\"\n", + "\n", + "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/{SET_DIR}/{SET_NAME}/wavenumber-frequency/\"\n", + "DEV_GLOB = sorted(glob.glob(DEV_PATH + \"/*.nc\"))\n", + "DEV_NUM_FILES = len(DEV_GLOB)\n", + "\n", + "MAIN_PATH = f\"/global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/{SET_NAME}/wavenumber-frequency/\"\n", + "MAIN_GLOB = sorted(glob.glob(MAIN_PATH + \"/*.nc\"))\n", + "MAIN_NUM_FILES = len(MAIN_GLOB)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def _check_if_matching_filecount():\n", + " if DEV_NUM_FILES != MAIN_NUM_FILES:\n", + " raise IOError(\n", + " \"Number of files do not match at DEV_PATH and MAIN_PATH \"\n", + " f\"({DEV_NUM_FILES} vs. {MAIN_NUM_FILES}).\"\n", + " )\n", + "\n", + " print(f\"Matching file count ({DEV_NUM_FILES} and {MAIN_NUM_FILES}).\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def _get_relative_diffs():\n", + " # We are mainly focusing on relative tolerance here (in percentage terms).\n", + " atol = 0\n", + " rtol = 1e-5\n", + "\n", + " for fp_main in MAIN_GLOB:\n", + " fp_dev = fp_main.replace(\"main\", SET_DIR)\n", + "\n", + " print(\"Comparing:\")\n", + " print(f\" * {fp_dev}\")\n", + " print(f\" * {fp_main}\")\n", + "\n", + " ds1 = xr.open_dataset(fp_dev)\n", + " ds2 = xr.open_dataset(fp_main)\n", + "\n", + " try:\n", + " xr.testing.assert_allclose(\n", + " ds1,\n", + " ds2,\n", + " atol=atol,\n", + " rtol=rtol,\n", + " )\n", + " except (KeyError, AssertionError) as e:\n", + " print(f\" {e}\")\n", + " else:\n", + " print(f\" * All close and within relative tolerance ({rtol})\")\n", + "\n", + "\n", + "def _get_var_data(ds: xr.Dataset, var_key: str) -> np.ndarray:\n", + " \"\"\"Get the variable data using a list of matching keys.\n", + "\n", + " The `main` branch saves the dataset using the original variable name,\n", + " while the dev branch saves the variable with the derived variable name.\n", + " The dev branch is performing the expected behavior here.\n", + "\n", + " Parameters\n", + " ----------\n", + " ds : xr.Dataset\n", + " _description_\n", + " var_key : str\n", + " _description_\n", + "\n", + " Returns\n", + " -------\n", + " np.ndarray\n", + " _description_\n", + " \"\"\"\n", + "\n", + " data = None\n", + "\n", + " var_keys = DERIVED_VARIABLES[var_key].keys()\n", + " var_keys = [var_key] + list(sum(var_keys, ()))\n", + "\n", + " for key in var_keys:\n", + " if key in ds.data_vars.keys():\n", + " data = ds[key].values\n", + " break\n", + "\n", + " return data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Check for matching and equal number of files\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Matching file count (63 and 63).\n" + ] + } + ], + "source": [ + "_check_if_matching_filecount()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 Compare the netCDF files between branches\n", + "\n", + "- Compare \"ref\" and \"test\" files\n", + "- \"diff\" files are ignored because getting relative diffs for these does not make sense (relative diff will be above tolerance)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S_0.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S_0.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S_1.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S_1.nc\n", + " * All close and within relative tolerance (1e-05)\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S_2.nc\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S_2.nc\n", + " * All close and within relative tolerance (1e-05)\n" + ] + } + ], + "source": [ + "_get_relative_diffs()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Results\n", + "\n", + "All files are within rtol 1e-5, so the changes should be good to go.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_png.ipynb b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_png.ipynb new file mode 100644 index 000000000..b06ebbe19 --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/886-jjb/regression_test_png.ipynb @@ -0,0 +1,240 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# CDAT Migration Regression Testing Notebook (`.png` files)\n", + "\n", + "This notebook is used to perform regression testing between the development and\n", + "production versions of a diagnostic set.\n", + "\n", + "## How to use\n", + "\n", + "PREREQUISITE: The diagnostic set's netCDF stored in `.json` files in two directories\n", + "(dev and `main` branches).\n", + "\n", + "1. Make a copy of this notebook under `auxiliary_tools/cdat_regression_testing/`.\n", + "2. Run `mamba create -n cdat_regression_test -y -c conda-forge \"python<3.12\" xarray netcdf4 dask pandas matplotlib-base ipykernel`\n", + "3. Run `mamba activate cdat_regression_test`\n", + "4. Update `SET_DIR` and `SET_NAME` in the copy of your notebook.\n", + "5. Run all cells IN ORDER.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup Code\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import glob\n", + "from auxiliary_tools.cdat_regression_testing.utils import get_image_diffs\n", + "\n", + "\n", + "SET_NAME = \"tropical_subseasonal\"\n", + "SET_DIR = \"886-jjb\"\n", + "\n", + "DEV_PATH = f\"/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/{SET_DIR}/{SET_NAME}/wavenumber-frequency/\"\n", + "DEV_GLOB = sorted(glob.glob(DEV_PATH + \"/*.png\"))\n", + "DEV_NUM_FILES = len(DEV_GLOB)\n", + "\n", + "MAIN_PATH = f\"/global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/{SET_NAME}/wavenumber-frequency/\"\n", + "MAIN_GLOB = sorted(glob.glob(MAIN_PATH + \"/*.png\"))\n", + "MAIN_NUM_FILES = len(MAIN_GLOB)" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "def _check_if_matching_filecount():\n", + " if DEV_NUM_FILES != MAIN_NUM_FILES:\n", + " raise IOError(\n", + " \"Number of files do not match at DEV_PATH and MAIN_PATH \"\n", + " f\"({DEV_NUM_FILES} vs. {MAIN_NUM_FILES}).\"\n", + " )\n", + "\n", + " print(f\"Matching file count ({DEV_NUM_FILES} and {MAIN_NUM_FILES}).\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Check for matching and equal number of files\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Matching file count (21 and 21).\n" + ] + } + ], + "source": [ + "_check_if_matching_filecount()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2 Compare the plots between branches\n", + "\n", + "- Compare \"ref\" and \"test\" files\n", + "- \"diff\" files are ignored because getting relative diffs for these does not make sense (relative diff will be above tolerance)\n" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/FLUT_background_15N-15S.png\n", + " * Plots are identical\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/FLUT_norm_asy_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/FLUT_norm_asy_zoom_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/FLUT_norm_asy_zoom_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/FLUT_norm_sym_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/FLUT_norm_sym_zoom_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/FLUT_norm_sym_zoom_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/FLUT_raw_asy_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/FLUT_raw_asy_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/FLUT_raw_sym_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/FLUT_raw_sym_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/PRECT_background_15N-15S.png\n", + " * Plots are identical\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/PRECT_norm_asy_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/PRECT_norm_asy_zoom_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/PRECT_norm_asy_zoom_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/PRECT_norm_sym_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/PRECT_norm_sym_zoom_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/PRECT_norm_sym_zoom_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/PRECT_raw_asy_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/PRECT_raw_asy_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/PRECT_raw_sym_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/PRECT_raw_sym_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/U850_background_15N-15S.png\n", + " * Plots are identical\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/U850_norm_asy_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/U850_norm_asy_zoom_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/U850_norm_asy_zoom_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/U850_norm_sym_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/U850_norm_sym_zoom_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/U850_norm_sym_zoom_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/U850_raw_asy_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/U850_raw_asy_15N-15S.png\n", + "Comparing:\n", + " * /global/cfs/cdirs/e3sm/www/chengzhu/tests/tropical_diags_subsetting/tropical_variability_model_obs_refine/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S.png\n", + " * /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency/U850_raw_sym_15N-15S.png\n", + " * Difference path /global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb/tropical_subseasonal/wavenumber-frequency_diff/U850_raw_sym_15N-15S.png\n" + ] + } + ], + "source": [ + "for main_path, dev_path in zip(MAIN_GLOB, DEV_GLOB):\n", + " print(\"Comparing:\")\n", + " print(f\" * {main_path}\")\n", + " print(f\" * {dev_path}\")\n", + "\n", + " get_image_diffs(dev_path, main_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Results\n", + "\n", + "All plots are virtually identical, only diff is a red outline around plots (positioning\n", + "diff probably due to different matplotlib version).\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/auxiliary_tools/cdat_regression_testing/886-jjb/run_script.py b/auxiliary_tools/cdat_regression_testing/886-jjb/run_script.py new file mode 100644 index 000000000..14570aaf7 --- /dev/null +++ b/auxiliary_tools/cdat_regression_testing/886-jjb/run_script.py @@ -0,0 +1,25 @@ +import os +from e3sm_diags.parameter.tropical_subseasonal_parameter import ( + TropicalSubseasonalParameter, +) +from e3sm_diags.run import runner + +param = TropicalSubseasonalParameter() + +param.reference_data_path = ( + "/global/cfs/cdirs/e3sm/e3sm_diags/obs_for_e3sm_diags/time-series" +) +# param.reference_data_path = '/global/cfs/cdirs/e3sm/chengzhu/e3sm_diags_zppy_test_complete_run_output/v2.LR.historical_0101_20240130/post/atm/180x360_aave/ts/daily/15yr' +param.test_data_path = "/global/cfs/cdirs/e3sm/chengzhu/e3sm_diags_zppy_test_complete_run_output/v2.LR.historical_0101_20240130/post/atm/180x360_aave/ts/daily/15yr" +param.test_name = "E3SMv2" +param.results_dir = "/global/cfs/cdirs/e3sm/www/cdat-migration-fy24/886-jjb" +# param.run_type = "model_vs_model" +# param.ref_name = 'E3SMv2' +param.test_start_yr = "2000" +param.test_end_yr = "2000" +param.ref_start_yr = "2001" +param.ref_end_yr = "2001" +param.save_netcdf = True + +runner.sets_to_run = ["tropical_subseasonal"] +runner.run_diags([param]) diff --git a/e3sm_diags/driver/tropical_subseasonal_driver.py b/e3sm_diags/driver/tropical_subseasonal_driver.py index f2c6e1bc1..acea8d54d 100644 --- a/e3sm_diags/driver/tropical_subseasonal_driver.py +++ b/e3sm_diags/driver/tropical_subseasonal_driver.py @@ -14,10 +14,11 @@ import numpy as np import xarray as xr -from e3sm_diags.driver import utils from e3sm_diags.driver.utils import zwf_functions as wf +from e3sm_diags.driver.utils.climo_xr import ClimoFreq +from e3sm_diags.driver.utils.dataset_xr import Dataset from e3sm_diags.logger import custom_logger -from e3sm_diags.plot.cartopy.tropical_subseasonal_plot import plot +from e3sm_diags.plot.tropical_subseasonal_plot import plot if TYPE_CHECKING: from e3sm_diags.parameter.tropical_subseasonal_parameter import ( @@ -38,11 +39,11 @@ def run_diag(parameter: TropicalSubseasonalParameter) -> TropicalSubseasonalPara :rtype: CoreParameter """ run_type = parameter.run_type - season = "ANN" + season: ClimoFreq = "ANN" - test_data = utils.dataset.Dataset(parameter, test=True) + test_data = Dataset(parameter, data_type="test") + ref_data = Dataset(parameter, data_type="ref") - ref_data = utils.dataset.Dataset(parameter, ref=True) for variable in parameter.variables: test, test_start, test_end = calculate_spectrum( parameter.test_data_path, @@ -52,9 +53,7 @@ def run_diag(parameter: TropicalSubseasonalParameter) -> TropicalSubseasonalPara ) parameter.test_start_yr = test_start parameter.test_end_yr = test_end - parameter.test_name_yrs = utils.general.get_name_and_yrs( - parameter, test_data, season - ) + parameter.test_name_yrs = test_data.get_name_yrs_attr(season) if run_type == "model_vs_model": ref, ref_start, ref_end = calculate_spectrum( parameter.reference_data_path, @@ -77,13 +76,9 @@ def run_diag(parameter: TropicalSubseasonalParameter) -> TropicalSubseasonalPara ) parameter.ref_start_yr = ref_start parameter.ref_end_yr = ref_end - parameter.ref_name_yrs = utils.general.get_name_and_yrs( - parameter, ref_data, season - ) - # test = xr.open_dataset(f"{parameter.results_dir}/full_spec_test.nc").load() - # ref = xr.open_dataset(f"{parameter.results_dir}/full_spec_ref_{parameter.ref_name}.nc").load() - + parameter.ref_name_yrs = ref_data.get_name_yrs_attr(season) parameter.var_id = variable + for diff_name in ["raw_sym", "raw_asy", "norm_sym", "norm_asy", "background"]: diff = ( 100 @@ -92,10 +87,12 @@ def run_diag(parameter: TropicalSubseasonalParameter) -> TropicalSubseasonalPara ) diff.name = f"spec_{diff_name}" diff.attrs.update(test[f"spec_{diff_name}"].attrs) + parameter.spec_type = diff_name parameter.output_file = f"{parameter.var_id}_{parameter.spec_type}_15N-15S" parameter.diff_title = "percent difference" plot(parameter, test[f"spec_{diff_name}"], ref[f"spec_{diff_name}"], diff) + if "norm" in diff_name: parameter.spec_type = f"{diff_name}_zoom" parameter.output_file = ( diff --git a/e3sm_diags/plot/cartopy/tropical_subseasonal_plot.py b/e3sm_diags/plot/tropical_subseasonal_plot.py similarity index 100% rename from e3sm_diags/plot/cartopy/tropical_subseasonal_plot.py rename to e3sm_diags/plot/tropical_subseasonal_plot.py