From d04e5ca3793c5763cf49aea4a8b75ea1a167cd08 Mon Sep 17 00:00:00 2001
From: Matthias Feurer <feurerm@informatik.uni-freiburg.de>
Date: Tue, 20 Aug 2019 11:06:42 +0200
Subject: [PATCH 1/2] add flaky decorator and install flaky

---
 setup.py                              | 3 ++-
 tests/test_runs/test_run_functions.py | 2 ++
 2 files changed, 4 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 3b271badd..52de238aa 100644
--- a/setup.py
+++ b/setup.py
@@ -48,7 +48,8 @@
                          'pytest-xdist',
                          'pytest-timeout',
                          'nbformat',
-                         'oslo.concurrency'
+                         'oslo.concurrency',
+                         'flaky',
                      ],
                      'examples': [
                          'matplotlib',
diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py
index dc35d1f01..924a14c6f 100644
--- a/tests/test_runs/test_run_functions.py
+++ b/tests/test_runs/test_run_functions.py
@@ -7,6 +7,7 @@
 import unittest.mock
 
 import numpy as np
+import pytest
 
 import openml
 import openml.exceptions
@@ -826,6 +827,7 @@ def test_initialize_model_from_run(self):
         self.assertEqual(flowS.components['VarianceThreshold'].
                          parameters['threshold'], '0.05')
 
+    @pytest.mark.flaky(reruns=3)
     def test_get_run_trace(self):
         # get_run_trace is already tested implicitly in test_run_and_publish
         # this test is a bit additional.

From f7f8e6f91ebf31c8bb303ca580094d35272fe7ab Mon Sep 17 00:00:00 2001
From: Matthias Feurer <feurerm@informatik.uni-freiburg.de>
Date: Tue, 20 Aug 2019 11:21:51 +0200
Subject: [PATCH 2/2] remove reruns flag

---
 tests/test_runs/test_run.py           | 2 +-
 tests/test_runs/test_run_functions.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/test_runs/test_run.py b/tests/test_runs/test_run.py
index 88fe8d6ef..dacade858 100644
--- a/tests/test_runs/test_run.py
+++ b/tests/test_runs/test_run.py
@@ -134,7 +134,7 @@ def test_to_from_filesystem_vanilla(self):
         TestBase.logger.info("collected from {}: {}".format(__file__.split('/')[-1],
                                                             run_prime.run_id))
 
-    @pytest.mark.flaky(reruns=3)
+    @pytest.mark.flaky()
     def test_to_from_filesystem_search(self):
 
         model = Pipeline([
diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py
index 924a14c6f..98df7dae8 100644
--- a/tests/test_runs/test_run_functions.py
+++ b/tests/test_runs/test_run_functions.py
@@ -827,7 +827,7 @@ def test_initialize_model_from_run(self):
         self.assertEqual(flowS.components['VarianceThreshold'].
                          parameters['threshold'], '0.05')
 
-    @pytest.mark.flaky(reruns=3)
+    @pytest.mark.flaky()
     def test_get_run_trace(self):
         # get_run_trace is already tested implicitly in test_run_and_publish
         # this test is a bit additional.