[MNT] Diagnose and address long test runtimes (#1633)

.github/workflows/test.yml

@@ -119,7 +119,7 @@ jobs:
           marks="not production_server and not test_server"
         fi
 
-        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
+        pytest -n 4 --durations=0 --timeout=600 --dist load -sv $codecov -o log_cli=true -m "$marks"
 
     - name: Run tests on Ubuntu Production
       if: matrix.os == 'ubuntu-latest'
@@ -136,14 +136,14 @@ jobs:
           marks="production_server and not test_server"
         fi
 
-        pytest -n 4 --durations=20 --dist load -sv $codecov -o log_cli=true -m "$marks"
+        pytest -n 4 --durations=0 --timeout=600 --dist load -sv $codecov -o log_cli=true -m "$marks"
 
     - name: Run tests on Windows
       if: matrix.os == 'windows-latest'
       env:
         OPENML_TEST_SERVER_ADMIN_KEY: ${{ secrets.OPENML_TEST_SERVER_ADMIN_KEY }}
       run: |  # we need a separate step because of the bash-specific if-statement in the previous one.
-        pytest -n 4 --durations=20 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server"
+        pytest -n 4 --durations=0 --timeout=600 --dist load -sv --reruns 5 --reruns-delay 1 -m "not test_server"
 
     - name: Check for files left behind by test
       if: matrix.os != 'windows-latest' && always()

pyproject.toml

@@ -129,6 +129,7 @@ version = {attr = "openml.__version__.__version__"}
 testpaths = ["tests"]
 minversion = "7.0"
 xfail_strict = true
+timeout = 600
 filterwarnings=[
     "ignore:the matrix subclass:PendingDeprecationWarning"
 ]

scripts/profile_tests.sh

@@ -0,0 +1,27 @@
+#!/bin/bash
+# Profile test durations to diagnose slow tests (Issue #1633)
+# Usage: ./scripts/profile_tests.sh [marker_filter]
+#
+# Examples:
+#   ./scripts/profile_tests.sh                               # non-server tests
+#   ./scripts/profile_tests.sh "production_server"            # production server tests only
+#   ./scripts/profile_tests.sh "sklearn"                      # sklearn tests only
+
+set -euo pipefail
+
+MARKER_FILTER="${1:-not production_server and not test_server}"
+
+echo "=== OpenML Test Duration Profiler ==="
+echo "Marker filter: $MARKER_FILTER"
+echo "Timeout per test: 300s"
+echo ""
+
+pytest \
+  --durations=0 \
+  --timeout=300 \
+  -q \
+  -m "$MARKER_FILTER" \
+  2>&1 | tee test_durations_report.txt
+
+echo ""
+echo "=== Report saved to test_durations_report.txt ==="

tests/conftest.py

@@ -105,7 +105,8 @@ def delete_remote_files(tracker, flow_names) -> None:
     if "flow" in tracker:
         to_sort = list(zip(tracker["flow"], flow_names))
         flow_deletion_order = [
-            entity_id for entity_id, _ in sorted(to_sort, key=lambda x: len(x[1]), reverse=True)
+            entity_id
+            for entity_id, _ in sorted(to_sort, key=lambda x: len(x[1]), reverse=True)
         ]
         tracker["flow"] = [flow_deletion_order[1] for flow_id, _ in flow_deletion_order]
 
@@ -254,7 +255,7 @@ def test_api_key() -> str:
     return TestBase.user_key
 
 
-@pytest.fixture(autouse=True, scope="function")
+@pytest.fixture(autouse=True, scope="module")
 def verify_cache_state(test_files_directory) -> Iterator[None]:
     assert_static_test_cache_correct(test_files_directory)
     yield
@@ -295,11 +296,12 @@ def with_test_cache(test_files_directory, request):
     openml.config.set_root_cache_directory(_root_cache_directory)
     if tmp_cache.exists():
         shutil.rmtree(tmp_cache)
-        
+
 
 @pytest.fixture
 def static_cache_dir():
-    return Path(__file__).parent / "files" 
+    return Path(__file__).parent / "files"
+
 
 @pytest.fixture
 def workdir(tmp_path):