Fixes for timeouts in tests (#5598)

Timeouts in CI were happening due to CUDA context creation happening in some dask estimator docstrings, this PR fixes that by avoiding that context creation. 

Also fixes a typo in the coordinate descent solver that caused the intercept to be calculated wrong in some cases

Authors:
  - Dante Gama Dessavre (https://github.com/dantegd)

Approvers:
  - Simon Adorf (https://github.com/csadorf)

URL: #5598

python/cuml/internals/available_devices.py

@@ -24,12 +24,22 @@
     cache = lru_cache(maxsize=None)
 
 
-get_cuda_count = gpu_only_import_from("rmm._cuda.gpu", "getDeviceCount")
+def gpu_available_no_context_creation():
+    """
+    Function tries to check if GPUs are available in the system without
+    creating a CUDA context. We check for CuPy presence as a proxy of that.
+    """
+    try:
+        import cupy
+
+        return True
+    except ImportError:
+        return False
 
 
 @cache
 def is_cuda_available():
     try:
-        return GPU_ENABLED and get_cuda_count() >= 1
+        return GPU_ENABLED and gpu_available_no_context_creation()
     except UnavailableError:
         return False

python/cuml/solvers/cd.pyx

@@ -296,7 +296,7 @@ class CD(Base,
                       <double>self.tol,
                       <double*>sample_weight_ptr)
 
-            self.intercept_ = _c_intercept2_f64
+                self.intercept_ = _c_intercept2_f64
 
         self.handle.sync()
         del X_m

python/cuml/tests/test_nearest_neighbors.py

@@ -256,6 +256,8 @@ def test_ivfflat_pred(nrows, ncols, n_neighbors, nlist):
 def test_ivfpq_pred(
     nrows, ncols, n_neighbors, nlist, M, n_bits, usePrecomputedTables
 ):
+    if ncols == 512 and usePrecomputedTables is True:
+        pytest.skip("https://github.com/rapidsai/cuml/issues/5603")
     algo_params = {
         "nlist": nlist,
         "nprobe": int(nlist * 0.2),

python/cuml/tests/test_no_cuinit.py

@@ -0,0 +1,109 @@
+# Copyright (c) 2023, NVIDIA CORPORATION.
+
+import os
+import subprocess
+import sys
+from shutil import which
+
+import pytest
+
+GDB_COMMANDS = """
+set confirm off
+set breakpoint pending on
+break cuInit
+run
+exit
+"""
+
+
+@pytest.fixture(scope="module")
+def cuda_gdb(request):
+    gdb = which("cuda-gdb")
+    if gdb is None:
+        request.applymarker(
+            pytest.mark.xfail(reason="No cuda-gdb found, can't detect cuInit"),
+        )
+        return gdb
+    else:
+        output = subprocess.run(
+            [gdb, "--version"], capture_output=True, text=True
+        )
+        if output.returncode != 0:
+            request.applymarker(
+                pytest.mark.xfail(
+                    reason=(
+                        "cuda-gdb not working on this platform, "
+                        f"can't detect cuInit: {output.stderr}"
+                    )
+                ),
+            )
+        return gdb
+
+
+def test_cuml_import_no_cuinit(cuda_gdb):
+    # When RAPIDS_NO_INITIALIZE is set, importing cuml should _not_
+    # create a CUDA context (i.e. cuInit should not be called).
+    # Intercepting the call to cuInit programmatically is tricky since
+    # the way it is resolved from dynamic libraries by
+    # cuda-python/numba/cupy is multitudinous (see discussion at
+    # https://github.com/rapidsai/cuml/pull/12361 which does this, but
+    # needs provide hooks that override dlsym, cuGetProcAddress, and
+    # cuInit.
+    # Instead, we just run under GDB and see if we hit a breakpoint
+    env = os.environ.copy()
+    env["RAPIDS_NO_INITIALIZE"] = "1"
+    output = subprocess.run(
+        [
+            cuda_gdb,
+            "-x",
+            "-",
+            "--args",
+            sys.executable,
+            "-c",
+            "import cuml",
+        ],
+        input=GDB_COMMANDS,
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+    cuInit_called = output.stdout.find("in cuInit ()")
+    print("Command output:\n")
+    print("*** STDOUT ***")
+    print(output.stdout)
+    print("*** STDERR ***")
+    print(output.stderr)
+    assert output.returncode == 0
+    assert cuInit_called < 0
+
+
+def test_cuml_create_estimator_cuinit(cuda_gdb):
+    # This tests that our gdb scripting correctly identifies cuInit
+    # when it definitely should have been called.
+    env = os.environ.copy()
+    env["RAPIDS_NO_INITIALIZE"] = "1"
+    output = subprocess.run(
+        [
+            cuda_gdb,
+            "-x",
+            "-",
+            "--args",
+            sys.executable,
+            "-c",
+            "import cupy as cp; a = cp.ones(10)",
+        ],
+        input=GDB_COMMANDS,
+        env=env,
+        capture_output=True,
+        text=True,
+    )
+
+    cuInit_called = output.stdout.find("in cuInit ()")
+    print("Command output:\n")
+    print("*** STDOUT ***")
+    print(output.stdout)
+    print("*** STDERR ***")
+    print(output.stderr)
+    assert output.returncode == 0
+    assert cuInit_called >= 0