diff --git a/setup.py b/setup.py
index 3b271badd..52de238aa 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,8 @@
                          'pytest-xdist',
                          'pytest-timeout',
                          'nbformat',
-                         'oslo.concurrency'
+                         'oslo.concurrency',
+                         'flaky',
                      ],
                      'examples': [
                          'matplotlib',
diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py
index 88fe8d6ef..dacade858 100644
--- a/tests/test_runs/test_run.py
+++ b/tests/test_runs/test_run.py
@@ -134,7 +134,7 @@ def test_to_from_filesystem_vanilla(self):
         TestBase.logger.info("collected from {}: {}".format(__file__.split('/')[-1],
                                                             run_prime.run_id))
 
-    @pytest.mark.flaky(reruns=3)
+    @pytest.mark.flaky()
     def test_to_from_filesystem_search(self):
 
         model = Pipeline([
diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py
index dc35d1f01..98df7dae8 100644
--- a/tests/test_runs/test_run_functions.py
+++ b/tests/test_runs/test_run_functions.py
@@ -7,6 +7,7 @@
 import unittest.mock
 
 import numpy as np
+import pytest
 
 import openml
 import openml.exceptions
@@ -826,6 +827,7 @@ def test_initialize_model_from_run(self):
         self.assertEqual(flowS.components['VarianceThreshold'].
                          parameters['threshold'], '0.05')
 
+    @pytest.mark.flaky()
     def test_get_run_trace(self):
         # get_run_trace is already tested implicitly in test_run_and_publish
         # this test is a bit additional.