diff --git a/lollipop/cli/deconvolute.py b/lollipop/cli/deconvolute.py index b827b88..2362da9 100755 --- a/lollipop/cli/deconvolute.py +++ b/lollipop/cli/deconvolute.py @@ -14,6 +14,7 @@ import pandas as pd import numpy as np from tqdm import tqdm, trange +from threadpoolctl import ThreadpoolController import lollipop as ll @@ -147,6 +148,10 @@ def _deconvolute_bootstrap( The deconvolution results for the location and bootstrap iterations. """ + # monitor the number of threads, to prevent oversubscription on blas / cluster systmes + controller = ThreadpoolController() + logging.info(f"Threading configuration:\n {controller.info()}") + # deconvolution results deconv = [] @@ -232,7 +237,7 @@ def _deconvolute_bootstrap( # just run one on everything weights = {} - # deconvolution + # define deconvolution kernel t_kdec = ll.KernelDeconv( temp_df2[var_dates["var_dates"][mindate] + ["undetermined"]], temp_df2["frac"], @@ -242,7 +247,11 @@ def _deconvolute_bootstrap( confint=confint(**confint_params), **weights, ) - t_kdec = t_kdec.deconv_all(**deconv_params) + # limit the number of threads, to prevent oversubscription on blas / cluster systmes + with controller.limit(limits=1, user_api='blas'): + # do the deconvolution + t_kdec = t_kdec.deconv_all(**deconv_params) + if have_confint: # with conf int res = t_kdec.fitted.copy() @@ -389,6 +398,7 @@ def deconvolute( tally_data, namefield, ): + # load data yaml = ruamel.yaml.YAML(typ="rt") print("load data") diff --git a/pyproject.toml b/pyproject.toml index 20a3226..a7b4634 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,6 +23,7 @@ strictyaml = { version = ">=1.7", optional = true } tqdm = { version = ">=4.64", optional = true } click = { version = "^8.0", optional = true } click-option-group = { version = "^0.5", optional = true } +threadpoolctl = "^3.5.0" [tool.poetry.extras] cli = [ "zstandard", "ruamel.yaml", "strictyaml", "tqdm", "click", "click-option-group" ]