From b4a5576d9bf577a068323b353698d41fbc0e4c55 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:25:22 +0100 Subject: [PATCH 01/33] added additional task agnostic local result to print of run --- openml/runs/run.py | 63 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 58367179e..2ef875836 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -139,8 +139,37 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id + def evaluation_summary(self, metric: str): + """Summarizes the evaluation of a metric over all folds. + + The fold scores for the metric must exist already. During run creation, + by default, the MAE for OpenMLRegressionTask and the accuracy for + OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed. + + If repetition exist, we take the mean over all repetitions. + + Parameters + ---------- + metric: str + Name of an evaluation metric that was used to compute fold scores. + + Returns + ------- + metric_summary: str + A formatted string that displays the metric's evaluation summary. + The summary consists of the mean and std. + """ + fold_score_lists = self.fold_evaluations[metric].values() + + # Get the mean and std over all repetitions + rep_means = [np.mean(list(x.values())) for x in fold_score_lists] + rep_stds = [np.std(list(x.values())) for x in fold_score_lists] + + return "{:.4f} +- {:.4f}".format(np.mean(rep_means), np.mean(rep_stds)) + def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: """Collect all information to display in the __repr__ body.""" + # Set up fields fields = { "Uploader Name": self.uploader_name, "Metric": self.task_evaluation_measure, @@ -156,6 +185,10 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: "Dataset ID": self.dataset_id, "Dataset URL": openml.datasets.OpenMLDataset.url_for_id(self.dataset_id), } + + # determines the order of the initial fields in which the information will be printed + order = ["Uploader Name", "Uploader Profile", "Metric", "Result"] + if self.uploader is not None: fields["Uploader Profile"] = "{}/u/{}".format( openml.config.get_server_base_url(), self.uploader @@ -164,13 +197,29 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: fields["Run URL"] = self.openml_url if self.evaluations is not None and self.task_evaluation_measure in self.evaluations: fields["Result"] = self.evaluations[self.task_evaluation_measure] - - # determines the order in which the information will be printed - order = [ - "Uploader Name", - "Uploader Profile", - "Metric", - "Result", + elif self.fold_evaluations is not None: + # -- Add Locally computed summary values to if possible + if "predictive_accuracy" in self.fold_evaluations: + # OpenMLClassificationTask; OpenMLLearningCurveTask + # default: predictive_accuracy + result_field = "Local Result - Accuracy (+- STD)" + fields[result_field] = self.evaluation_summary("predictive_accuracy") + order.append(result_field) + elif "mean_absolute_error" in self.fold_evaluations: + # OpenMLRegressionTask + # default: mean_absolute_error + result_field = "Local Result - MAE (+- STD)" + fields[result_field] = self.evaluation_summary("mean_absolute_error") + order.append(result_field) + + rt_field = "Local Runtime - ms (+- STD)" + fields[rt_field] = self.evaluation_summary("usercpu_time_millis") + + # Add to order to be below / same as results + order.append(rt_field) + + # determines the remaining order + order += [ "Run ID", "Run URL", "Task ID", From ece79f04e267a4e9333ed90c9f7bdc77088a751a Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:38:51 +0100 Subject: [PATCH 02/33] add PR to progress.rst --- doc/progress.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/progress.rst b/doc/progress.rst index 6b42e851f..e2e3610fb 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -10,7 +10,7 @@ Changelog ~~~~~~ * Add new contributions here. - + * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. 0.13.0 ~~~~~~ From a09c850113e96fca04f07c7d9c2b4060eb3ba428 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:43:21 +0100 Subject: [PATCH 03/33] fix comment typo --- openml/runs/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 2ef875836..b9db83061 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -198,7 +198,7 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: if self.evaluations is not None and self.task_evaluation_measure in self.evaluations: fields["Result"] = self.evaluations[self.task_evaluation_measure] elif self.fold_evaluations is not None: - # -- Add Locally computed summary values to if possible + # -- Add locally computed summary values if possible if "predictive_accuracy" in self.fold_evaluations: # OpenMLClassificationTask; OpenMLLearningCurveTask # default: predictive_accuracy @@ -212,6 +212,7 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: fields[result_field] = self.evaluation_summary("mean_absolute_error") order.append(result_field) + # Runtime should be available for any task type rt_field = "Local Runtime - ms (+- STD)" fields[rt_field] = self.evaluation_summary("usercpu_time_millis") From c8724b64866c8ee4f4ba1b532d83e3a608fe7ba3 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Wed, 22 Feb 2023 12:12:18 +0100 Subject: [PATCH 04/33] Update openml/runs/run.py Co-authored-by: Matthias Feurer --- openml/runs/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index b9db83061..09e3c3ee6 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -139,7 +139,7 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id - def evaluation_summary(self, metric: str): + def evaluation_summary(self, metric: str) -> str: """Summarizes the evaluation of a metric over all folds. The fold scores for the metric must exist already. During run creation, From b6af420f27933746d331afdf99f8666c4a664cd7 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:06:40 +0100 Subject: [PATCH 05/33] add a function to list available estimation procedures --- openml/evaluations/functions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 30d376c04..16b51b8a0 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -275,6 +275,39 @@ def list_evaluation_measures() -> List[str]: return qualities +def list_estimation_procedures(): + """Return list of evaluation procedures available. + + The function performs an API call to retrieve the entire list of + evaluation procedures' names that are available. + + Returns + ------- + list + """ + + api_call = "estimationprocedure/list" + xml_string = openml._api_calls._perform_api_call(api_call, "get") + api_results = xmltodict.parse(xml_string) + + # Minimalistic check if the XML is useful + if "oml:estimationprocedures" not in api_results: + raise ValueError("Error in return XML, does not contain " '"oml:estimationprocedures"') + if "oml:estimationprocedure" not in api_results["oml:estimationprocedures"]: + raise ValueError("Error in return XML, does not contain " '"oml:estimationprocedure"') + + if not isinstance(api_results["oml:estimationprocedures"]["oml:estimationprocedure"], list): + raise TypeError( + "Error in return XML, does not contain " '"oml:estimationprocedure" as a list' + ) + + prods = [ + prod["oml:name"] + for prod in api_results["oml:estimationprocedures"]["oml:estimationprocedure"] + ] + return prods + + def list_evaluations_setups( function: str, offset: Optional[int] = None, From 5258ded6875e5dd101bc7a444c72870c8c025eb2 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:07:43 +0100 Subject: [PATCH 06/33] refactor print to only work for supported task types and local measures --- openml/runs/run.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 09e3c3ee6..d30c6e41c 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -139,7 +139,7 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id - def evaluation_summary(self, metric: str) -> str: + def _evaluation_summary(self, metric: str) -> str: """Summarizes the evaluation of a metric over all folds. The fold scores for the metric must exist already. During run creation, @@ -203,21 +203,20 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: # OpenMLClassificationTask; OpenMLLearningCurveTask # default: predictive_accuracy result_field = "Local Result - Accuracy (+- STD)" - fields[result_field] = self.evaluation_summary("predictive_accuracy") + fields[result_field] = self._evaluation_summary("predictive_accuracy") order.append(result_field) elif "mean_absolute_error" in self.fold_evaluations: # OpenMLRegressionTask # default: mean_absolute_error result_field = "Local Result - MAE (+- STD)" - fields[result_field] = self.evaluation_summary("mean_absolute_error") + fields[result_field] = self._evaluation_summary("mean_absolute_error") order.append(result_field) - # Runtime should be available for any task type - rt_field = "Local Runtime - ms (+- STD)" - fields[rt_field] = self.evaluation_summary("usercpu_time_millis") - - # Add to order to be below / same as results - order.append(rt_field) + if "usercpu_time_millis" in self.fold_evaluations: + # Runtime should be available for most tasks types + rt_field = "Local Runtime - ms (+- STD)" + fields[rt_field] = self._evaluation_summary("usercpu_time_millis") + order.append(rt_field) # determines the remaining order order += [ From f15a1d8bf7bbe712a3c1ce58fa2c07815cc13c1f Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:33:47 +0100 Subject: [PATCH 07/33] add test for pint out and update progress --- doc/progress.rst | 2 +- tests/test_runs/test_run_functions.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/doc/progress.rst b/doc/progress.rst index e2e3610fb..8ace81100 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -10,7 +10,7 @@ Changelog ~~~~~~ * Add new contributions here. - * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. + * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's string and print representation for `SUPERVISED_CLASSIFICATION, LEARNING_CURVE, SUPERVISED_REGRESSION`. 0.13.0 ~~~~~~ diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 1e92613c3..4dcbfc24d 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -530,6 +530,14 @@ def determine_grid_size(param_grid): # todo: check if runtime is present self._check_fold_timing_evaluations(run.fold_evaluations, 1, num_folds, task_type=task_type) + + # Check if run string and print representation do not run into an error + # The above check already verifies that all columns needed for supported + # representations are present. + # Supported: SUPERVISED_CLASSIFICATION, LEARNING_CURVE, SUPERVISED_REGRESSION + str(run) + self.logger.info(run) + return run def _run_and_upload_classification( From ca1c5a8193345c67e521eb787b443208b244fd2b Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:25:22 +0100 Subject: [PATCH 08/33] added additional task agnostic local result to print of run --- openml/runs/run.py | 63 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 58367179e..2ef875836 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -139,8 +139,37 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id + def evaluation_summary(self, metric: str): + """Summarizes the evaluation of a metric over all folds. + + The fold scores for the metric must exist already. During run creation, + by default, the MAE for OpenMLRegressionTask and the accuracy for + OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed. + + If repetition exist, we take the mean over all repetitions. + + Parameters + ---------- + metric: str + Name of an evaluation metric that was used to compute fold scores. + + Returns + ------- + metric_summary: str + A formatted string that displays the metric's evaluation summary. + The summary consists of the mean and std. + """ + fold_score_lists = self.fold_evaluations[metric].values() + + # Get the mean and std over all repetitions + rep_means = [np.mean(list(x.values())) for x in fold_score_lists] + rep_stds = [np.std(list(x.values())) for x in fold_score_lists] + + return "{:.4f} +- {:.4f}".format(np.mean(rep_means), np.mean(rep_stds)) + def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: """Collect all information to display in the __repr__ body.""" + # Set up fields fields = { "Uploader Name": self.uploader_name, "Metric": self.task_evaluation_measure, @@ -156,6 +185,10 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: "Dataset ID": self.dataset_id, "Dataset URL": openml.datasets.OpenMLDataset.url_for_id(self.dataset_id), } + + # determines the order of the initial fields in which the information will be printed + order = ["Uploader Name", "Uploader Profile", "Metric", "Result"] + if self.uploader is not None: fields["Uploader Profile"] = "{}/u/{}".format( openml.config.get_server_base_url(), self.uploader @@ -164,13 +197,29 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: fields["Run URL"] = self.openml_url if self.evaluations is not None and self.task_evaluation_measure in self.evaluations: fields["Result"] = self.evaluations[self.task_evaluation_measure] - - # determines the order in which the information will be printed - order = [ - "Uploader Name", - "Uploader Profile", - "Metric", - "Result", + elif self.fold_evaluations is not None: + # -- Add Locally computed summary values to if possible + if "predictive_accuracy" in self.fold_evaluations: + # OpenMLClassificationTask; OpenMLLearningCurveTask + # default: predictive_accuracy + result_field = "Local Result - Accuracy (+- STD)" + fields[result_field] = self.evaluation_summary("predictive_accuracy") + order.append(result_field) + elif "mean_absolute_error" in self.fold_evaluations: + # OpenMLRegressionTask + # default: mean_absolute_error + result_field = "Local Result - MAE (+- STD)" + fields[result_field] = self.evaluation_summary("mean_absolute_error") + order.append(result_field) + + rt_field = "Local Runtime - ms (+- STD)" + fields[rt_field] = self.evaluation_summary("usercpu_time_millis") + + # Add to order to be below / same as results + order.append(rt_field) + + # determines the remaining order + order += [ "Run ID", "Run URL", "Task ID", From 8a572fe1ba62649bd0055e4e7a59afcce47fd1ea Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:38:51 +0100 Subject: [PATCH 09/33] add PR to progress.rst --- doc/progress.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/progress.rst b/doc/progress.rst index 344a0e3dd..f4b3aa42b 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -10,7 +10,7 @@ Changelog ~~~~~~ * FIX #1198: Support numpy 1.24 and higher. - + * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. 0.13.0 ~~~~~~ From 27c2c15b070935ac4961e43777def2b62d02ab16 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:43:21 +0100 Subject: [PATCH 10/33] fix comment typo --- openml/runs/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 2ef875836..b9db83061 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -198,7 +198,7 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: if self.evaluations is not None and self.task_evaluation_measure in self.evaluations: fields["Result"] = self.evaluations[self.task_evaluation_measure] elif self.fold_evaluations is not None: - # -- Add Locally computed summary values to if possible + # -- Add locally computed summary values if possible if "predictive_accuracy" in self.fold_evaluations: # OpenMLClassificationTask; OpenMLLearningCurveTask # default: predictive_accuracy @@ -212,6 +212,7 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: fields[result_field] = self.evaluation_summary("mean_absolute_error") order.append(result_field) + # Runtime should be available for any task type rt_field = "Local Runtime - ms (+- STD)" fields[rt_field] = self.evaluation_summary("usercpu_time_millis") From 454364ee3eca296c2038fcb721df3df8a8974517 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Wed, 22 Feb 2023 12:12:18 +0100 Subject: [PATCH 11/33] Update openml/runs/run.py Co-authored-by: Matthias Feurer --- openml/runs/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index b9db83061..09e3c3ee6 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -139,7 +139,7 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id - def evaluation_summary(self, metric: str): + def evaluation_summary(self, metric: str) -> str: """Summarizes the evaluation of a metric over all folds. The fold scores for the metric must exist already. During run creation, From bbb849d6d73443bba48be861c0d47f90d3c3b53a Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:06:40 +0100 Subject: [PATCH 12/33] add a function to list available estimation procedures --- openml/evaluations/functions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 30d376c04..16b51b8a0 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -275,6 +275,39 @@ def list_evaluation_measures() -> List[str]: return qualities +def list_estimation_procedures(): + """Return list of evaluation procedures available. + + The function performs an API call to retrieve the entire list of + evaluation procedures' names that are available. + + Returns + ------- + list + """ + + api_call = "estimationprocedure/list" + xml_string = openml._api_calls._perform_api_call(api_call, "get") + api_results = xmltodict.parse(xml_string) + + # Minimalistic check if the XML is useful + if "oml:estimationprocedures" not in api_results: + raise ValueError("Error in return XML, does not contain " '"oml:estimationprocedures"') + if "oml:estimationprocedure" not in api_results["oml:estimationprocedures"]: + raise ValueError("Error in return XML, does not contain " '"oml:estimationprocedure"') + + if not isinstance(api_results["oml:estimationprocedures"]["oml:estimationprocedure"], list): + raise TypeError( + "Error in return XML, does not contain " '"oml:estimationprocedure" as a list' + ) + + prods = [ + prod["oml:name"] + for prod in api_results["oml:estimationprocedures"]["oml:estimationprocedure"] + ] + return prods + + def list_evaluations_setups( function: str, offset: Optional[int] = None, From 47f434668c373f94c84d920aa93c18ea5951fd40 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:07:43 +0100 Subject: [PATCH 13/33] refactor print to only work for supported task types and local measures --- openml/runs/run.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 09e3c3ee6..d30c6e41c 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -139,7 +139,7 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id - def evaluation_summary(self, metric: str) -> str: + def _evaluation_summary(self, metric: str) -> str: """Summarizes the evaluation of a metric over all folds. The fold scores for the metric must exist already. During run creation, @@ -203,21 +203,20 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: # OpenMLClassificationTask; OpenMLLearningCurveTask # default: predictive_accuracy result_field = "Local Result - Accuracy (+- STD)" - fields[result_field] = self.evaluation_summary("predictive_accuracy") + fields[result_field] = self._evaluation_summary("predictive_accuracy") order.append(result_field) elif "mean_absolute_error" in self.fold_evaluations: # OpenMLRegressionTask # default: mean_absolute_error result_field = "Local Result - MAE (+- STD)" - fields[result_field] = self.evaluation_summary("mean_absolute_error") + fields[result_field] = self._evaluation_summary("mean_absolute_error") order.append(result_field) - # Runtime should be available for any task type - rt_field = "Local Runtime - ms (+- STD)" - fields[rt_field] = self.evaluation_summary("usercpu_time_millis") - - # Add to order to be below / same as results - order.append(rt_field) + if "usercpu_time_millis" in self.fold_evaluations: + # Runtime should be available for most tasks types + rt_field = "Local Runtime - ms (+- STD)" + fields[rt_field] = self._evaluation_summary("usercpu_time_millis") + order.append(rt_field) # determines the remaining order order += [ From 992dc52da8bda2c3addb8f4c8ff3ce0a620d7235 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:33:47 +0100 Subject: [PATCH 14/33] add test for pint out and update progress --- tests/test_runs/test_run_functions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index ca38750d8..8d5ee97a4 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -533,6 +533,14 @@ def determine_grid_size(param_grid): # todo: check if runtime is present self._check_fold_timing_evaluations(run.fold_evaluations, 1, num_folds, task_type=task_type) + + # Check if run string and print representation do not run into an error + # The above check already verifies that all columns needed for supported + # representations are present. + # Supported: SUPERVISED_CLASSIFICATION, LEARNING_CURVE, SUPERVISED_REGRESSION + str(run) + self.logger.info(run) + return run def _run_and_upload_classification( From 5730669fadbb6ddd69e4497cca4491ca23b7700b Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Fri, 24 Feb 2023 11:23:41 +0100 Subject: [PATCH 15/33] Fix CI Python 3.6 (#1218) * Try Ubunte 20.04 for Python 3.6 * use old ubuntu for python 3.6 --- .github/workflows/test.yml | 24 ++++++++++++++++++++---- 1 file changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 7241f7990..782b6e0a3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -8,7 +8,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - python-version: [3.6, 3.7, 3.8] + python-version: [3.7, 3.8] scikit-learn: [0.21.2, 0.22.2, 0.23.1, 0.24] os: [ubuntu-latest] sklearn-only: ['true'] @@ -19,15 +19,31 @@ jobs: - python-version: 3.6 scikit-learn: 0.18.2 scipy: 1.2.0 - os: ubuntu-latest + os: ubuntu-20.04 sklearn-only: 'true' - python-version: 3.6 scikit-learn: 0.19.2 - os: ubuntu-latest + os: ubuntu-20.04 sklearn-only: 'true' - python-version: 3.6 scikit-learn: 0.20.2 - os: ubuntu-latest + os: ubuntu-20.04 + sklearn-only: 'true' + - python-version: 3.6 + scikit-learn: 0.21.2 + os: ubuntu-20.04 + sklearn-only: 'true' + - python-version: 3.6 + scikit-learn: 0.22.2 + os: ubuntu-20.04 + sklearn-only: 'true' + - python-version: 3.6 + scikit-learn: 0.23.1 + os: ubuntu-20.04 + sklearn-only: 'true' + - python-version: 3.6 + scikit-learn: 0.24 + os: ubuntu-20.04 sklearn-only: 'true' - python-version: 3.8 scikit-learn: 0.23.1 From 5b2ac461da654b021e1ee050d850990d99798558 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 24 Feb 2023 15:26:02 +0100 Subject: [PATCH 16/33] Bump docker/setup-buildx-action from 1 to 2 (#1221) Bumps [docker/setup-buildx-action](https://github.com/docker/setup-buildx-action) from 1 to 2. - [Release notes](https://github.com/docker/setup-buildx-action/releases) - [Commits](https://github.com/docker/setup-buildx-action/compare/v1...v2) --- updated-dependencies: - dependency-name: docker/setup-buildx-action dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/release_docker.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/release_docker.yaml b/.github/workflows/release_docker.yaml index 3df6cdf4c..6ceb1d060 100644 --- a/.github/workflows/release_docker.yaml +++ b/.github/workflows/release_docker.yaml @@ -18,7 +18,7 @@ jobs: uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 + uses: docker/setup-buildx-action@v2 - name: Login to DockerHub uses: docker/login-action@v2 From 5dcb7a319c687befe6faf86404780d5c574496f8 Mon Sep 17 00:00:00 2001 From: Vishal Parmar Date: Fri, 24 Feb 2023 21:39:52 +0530 Subject: [PATCH 17/33] Update run.py (#1194) * Update run.py * Update run.py updated description to not contain duplicate information. * Update run.py --- openml/runs/run.py | 29 ++++++++++++++++++++++++----- 1 file changed, 24 insertions(+), 5 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 804c0f484..90e7a4b0b 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -31,36 +31,55 @@ class OpenMLRun(OpenMLBase): Parameters ---------- task_id: int + The ID of the OpenML task associated with the run. flow_id: int + The ID of the OpenML flow associated with the run. dataset_id: int + The ID of the OpenML dataset used for the run. setup_string: str + The setup string of the run. output_files: Dict[str, str] - A dictionary that specifies where each related file can be found. + Specifies where each related file can be found. setup_id: int + An integer representing the ID of the setup used for the run. tags: List[str] + Representing the tags associated with the run. uploader: int - User ID of the uploader. + User ID of the uploader. uploader_name: str + The name of the person who uploaded the run. evaluations: Dict + Representing the evaluations of the run. fold_evaluations: Dict + The evaluations of the run for each fold. sample_evaluations: Dict + The evaluations of the run for each sample. data_content: List[List] The predictions generated from executing this run. trace: OpenMLRunTrace + The trace containing information on internal model evaluations of this run. model: object + The untrained model that was evaluated in the run. task_type: str + The type of the OpenML task associated with the run. task_evaluation_measure: str + The evaluation measure used for the task. flow_name: str + The name of the OpenML flow associated with the run. parameter_settings: List[OrderedDict] + Representing the parameter settings used for the run. predictions_url: str + The URL of the predictions file. task: OpenMLTask + An instance of the OpenMLTask class, representing the OpenML task associated with the run. flow: OpenMLFlow + An instance of the OpenMLFlow class, representing the OpenML flow associated with the run. run_id: int + The ID of the run. description_text: str, optional - Description text to add to the predictions file. - If left None, is set to the time the arff file is generated. + Description text to add to the predictions file. If left None, is set to the time the arff file is generated. run_details: str, optional (default=None) - Description of the run stored in the run meta-data. + Description of the run stored in the run meta-data. """ def __init__( From fb5841199c0ab53c4817229e42727f10ad977451 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Fri, 24 Feb 2023 18:13:43 +0100 Subject: [PATCH 18/33] add type hint for new function --- openml/evaluations/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 16b51b8a0..693ec06cf 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -275,7 +275,7 @@ def list_evaluation_measures() -> List[str]: return qualities -def list_estimation_procedures(): +def list_estimation_procedures() -> List[str]: """Return list of evaluation procedures available. The function performs an API call to retrieve the entire list of From 6dbb498eaadec449d1539489cea55b6062aa55ec Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Fri, 24 Feb 2023 18:20:12 +0100 Subject: [PATCH 19/33] update add description --- doc/progress.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/progress.rst b/doc/progress.rst index c47644ce1..2fce2bd8f 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -11,7 +11,7 @@ Changelog * FIX #1197 #559 #1131: Fix the order of ground truth and predictions in the ``OpenMLRun`` object and in ``format_prediction``. * FIX #1198: Support numpy 1.24 and higher. - * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. + * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation if the run was created locally and not downloaded from the server. 0.13.0 ~~~~~~ From 687a0f11e7eead5a26135ad4a1c826acc0aa1503 Mon Sep 17 00:00:00 2001 From: Pieter Gijsbers Date: Wed, 1 Mar 2023 08:41:17 +0100 Subject: [PATCH 20/33] Refactor if-statements (#1219) * Refactor if-statements * Add explicit names to conditional expression * Add 'dependencies' to better mimic OpenMLFlow --- openml/_api_calls.py | 4 +-- openml/datasets/dataset.py | 12 +++---- openml/extensions/sklearn/extension.py | 47 ++++++++++--------------- openml/flows/functions.py | 5 +-- openml/setups/functions.py | 5 +-- openml/tasks/split.py | 10 +++--- openml/utils.py | 5 +-- tests/test_extensions/test_functions.py | 9 ++--- tests/test_runs/test_run_functions.py | 6 ++-- 9 files changed, 37 insertions(+), 66 deletions(-) diff --git a/openml/_api_calls.py b/openml/_api_calls.py index f3c3306fc..c22f82840 100644 --- a/openml/_api_calls.py +++ b/openml/_api_calls.py @@ -303,9 +303,7 @@ def __is_checksum_equal(downloaded_file, md5_checksum=None): md5 = hashlib.md5() md5.update(downloaded_file.encode("utf-8")) md5_checksum_download = md5.hexdigest() - if md5_checksum == md5_checksum_download: - return True - return False + return md5_checksum == md5_checksum_download def _send_request(request_method, url, data, files=None, md5_checksum=None): diff --git a/openml/datasets/dataset.py b/openml/datasets/dataset.py index 6f3f66853..1644ff177 100644 --- a/openml/datasets/dataset.py +++ b/openml/datasets/dataset.py @@ -275,7 +275,7 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: def __eq__(self, other): - if type(other) != OpenMLDataset: + if not isinstance(other, OpenMLDataset): return False server_fields = { @@ -287,14 +287,12 @@ def __eq__(self, other): "data_file", } - # check that the keys are identical + # check that common keys and values are identical self_keys = set(self.__dict__.keys()) - server_fields other_keys = set(other.__dict__.keys()) - server_fields - if self_keys != other_keys: - return False - - # check that values of the common keys are identical - return all(self.__dict__[key] == other.__dict__[key] for key in self_keys) + return self_keys == other_keys and all( + self.__dict__[key] == other.__dict__[key] for key in self_keys + ) def _download_data(self) -> None: """Download ARFF data file to standard cache directory. Set `self.data_file`.""" diff --git a/openml/extensions/sklearn/extension.py b/openml/extensions/sklearn/extension.py index 28ecd217f..997a9b8ea 100644 --- a/openml/extensions/sklearn/extension.py +++ b/openml/extensions/sklearn/extension.py @@ -38,19 +38,16 @@ logger = logging.getLogger(__name__) - if sys.version_info >= (3, 5): from json.decoder import JSONDecodeError else: JSONDecodeError = ValueError - DEPENDENCIES_PATTERN = re.compile( r"^(?P[\w\-]+)((?P==|>=|>)" r"(?P(\d+\.)?(\d+\.)?(\d+)?(dev)?[0-9]*))?$" ) - SIMPLE_NUMPY_TYPES = [ nptype for type_cat, nptypes in np.sctypes.items() @@ -580,15 +577,11 @@ def _is_cross_validator(self, o: Any) -> bool: @classmethod def _is_sklearn_flow(cls, flow: OpenMLFlow) -> bool: - if getattr(flow, "dependencies", None) is not None and "sklearn" in flow.dependencies: - return True - if flow.external_version is None: - return False - else: - return ( - flow.external_version.startswith("sklearn==") - or ",sklearn==" in flow.external_version - ) + sklearn_dependency = isinstance(flow.dependencies, str) and "sklearn" in flow.dependencies + sklearn_as_external = isinstance(flow.external_version, str) and ( + flow.external_version.startswith("sklearn==") or ",sklearn==" in flow.external_version + ) + return sklearn_dependency or sklearn_as_external def _get_sklearn_description(self, model: Any, char_lim: int = 1024) -> str: """Fetches the sklearn function docstring for the flow description @@ -1867,24 +1860,22 @@ def is_subcomponent_specification(values): # checks whether the current value can be a specification of # subcomponents, as for example the value for steps parameter # (in Pipeline) or transformers parameter (in - # ColumnTransformer). These are always lists/tuples of lists/ - # tuples, size bigger than 2 and an OpenMLFlow item involved. - if not isinstance(values, (tuple, list)): - return False - for item in values: - if not isinstance(item, (tuple, list)): - return False - if len(item) < 2: - return False - if not isinstance(item[1], (openml.flows.OpenMLFlow, str)): - if ( + # ColumnTransformer). + return ( + # Specification requires list/tuple of list/tuple with + # at least length 2. + isinstance(values, (tuple, list)) + and all(isinstance(item, (tuple, list)) and len(item) > 1 for item in values) + # And each component needs to be a flow or interpretable string + and all( + isinstance(item[1], openml.flows.OpenMLFlow) + or ( isinstance(item[1], str) and item[1] in SKLEARN_PIPELINE_STRING_COMPONENTS - ): - pass - else: - return False - return True + ) + for item in values + ) + ) # _flow is openml flow object, _param dict maps from flow name to flow # id for the main call, the param dict can be overridden (useful for diff --git a/openml/flows/functions.py b/openml/flows/functions.py index 43cb453fa..99525c3e4 100644 --- a/openml/flows/functions.py +++ b/openml/flows/functions.py @@ -261,10 +261,7 @@ def flow_exists(name: str, external_version: str) -> Union[int, bool]: result_dict = xmltodict.parse(xml_response) flow_id = int(result_dict["oml:flow_exists"]["oml:id"]) - if flow_id > 0: - return flow_id - else: - return False + return flow_id if flow_id > 0 else False def get_flow_id( diff --git a/openml/setups/functions.py b/openml/setups/functions.py index 1ce0ed005..f4fab3219 100644 --- a/openml/setups/functions.py +++ b/openml/setups/functions.py @@ -55,10 +55,7 @@ def setup_exists(flow) -> int: ) result_dict = xmltodict.parse(result) setup_id = int(result_dict["oml:setup_exists"]["oml:id"]) - if setup_id > 0: - return setup_id - else: - return False + return setup_id if setup_id > 0 else False def _get_cached_setup(setup_id): diff --git a/openml/tasks/split.py b/openml/tasks/split.py index e5fafedc5..dc496ef7d 100644 --- a/openml/tasks/split.py +++ b/openml/tasks/split.py @@ -47,12 +47,10 @@ def __eq__(self, other): or self.name != other.name or self.description != other.description or self.split.keys() != other.split.keys() - ): - return False - - if any( - self.split[repetition].keys() != other.split[repetition].keys() - for repetition in self.split + or any( + self.split[repetition].keys() != other.split[repetition].keys() + for repetition in self.split + ) ): return False diff --git a/openml/utils.py b/openml/utils.py index 8ab238463..0f60f2bb8 100644 --- a/openml/utils.py +++ b/openml/utils.py @@ -174,10 +174,7 @@ def _delete_entity(entity_type, entity_id): url_suffix = "%s/%d" % (entity_type, entity_id) result_xml = openml._api_calls._perform_api_call(url_suffix, "delete") result = xmltodict.parse(result_xml) - if "oml:%s_delete" % entity_type in result: - return True - else: - return False + return "oml:%s_delete" % entity_type in result def _list_all(listing_call, output_format="dict", *args, **filters): diff --git a/tests/test_extensions/test_functions.py b/tests/test_extensions/test_functions.py index 791e815e1..36bb06061 100644 --- a/tests/test_extensions/test_functions.py +++ b/tests/test_extensions/test_functions.py @@ -9,6 +9,7 @@ class DummyFlow: external_version = "DummyFlow==0.1" + dependencies = None class DummyModel: @@ -18,15 +19,11 @@ class DummyModel: class DummyExtension1: @staticmethod def can_handle_flow(flow): - if not inspect.stack()[2].filename.endswith("test_functions.py"): - return False - return True + return inspect.stack()[2].filename.endswith("test_functions.py") @staticmethod def can_handle_model(model): - if not inspect.stack()[2].filename.endswith("test_functions.py"): - return False - return True + return inspect.stack()[2].filename.endswith("test_functions.py") class DummyExtension2: diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 14e6d7298..786ab2291 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -127,7 +127,7 @@ def _wait_for_processed_run(self, run_id, max_waiting_time_seconds): "evaluated correctly on the server".format(run_id) ) - def _compare_predictions(self, predictions, predictions_prime): + def _assert_predictions_equal(self, predictions, predictions_prime): self.assertEqual( np.array(predictions_prime["data"]).shape, np.array(predictions["data"]).shape ) @@ -151,8 +151,6 @@ def _compare_predictions(self, predictions, predictions_prime): else: self.assertEqual(val_1, val_2) - return True - def _rerun_model_and_compare_predictions(self, run_id, model_prime, seed, create_task_obj): run = openml.runs.get_run(run_id) @@ -183,7 +181,7 @@ def _rerun_model_and_compare_predictions(self, run_id, model_prime, seed, create predictions_prime = run_prime._generate_arff_dict() - self._compare_predictions(predictions, predictions_prime) + self._assert_predictions_equal(predictions, predictions_prime) pd.testing.assert_frame_equal( run.predictions, run_prime.predictions, From c0a75bdd0d30dc1b038a56cfa51ca51e5ba5f5b1 Mon Sep 17 00:00:00 2001 From: Matthias Feurer Date: Wed, 1 Mar 2023 09:26:54 +0100 Subject: [PATCH 21/33] Ci python 38 (#1220) * Install custom numpy version for specific combination of Python3.8 and numpy * Debug output * Change syntax * move to coverage action v3 * Remove test output --- .github/workflows/test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 782b6e0a3..974147ed3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -72,6 +72,11 @@ jobs: - name: Install scikit-learn ${{ matrix.scikit-learn }} run: | pip install scikit-learn==${{ matrix.scikit-learn }} + - name: Install numpy for Python 3.8 + # Python 3.8 & scikit-learn<0.24 requires numpy<=1.23.5 + if: ${{ matrix.python-version == '3.8' && contains(fromJSON('["0.23.1", "0.22.2", "0.21.2"]'), matrix.scikit-learn) }} + run: | + pip install numpy==1.23.5 - name: Install scipy ${{ matrix.scipy }} if: ${{ matrix.scipy }} run: | @@ -105,7 +110,7 @@ jobs: fi - name: Upload coverage if: matrix.code-cov && always() - uses: codecov/codecov-action@v1 + uses: codecov/codecov-action@v3 with: files: coverage.xml fail_ci_if_error: true From 502988b4723075454601413f7b69de62d541a493 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:25:22 +0100 Subject: [PATCH 22/33] added additional task agnostic local result to print of run --- openml/runs/run.py | 63 ++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 56 insertions(+), 7 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 90e7a4b0b..cc30aba85 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -158,8 +158,37 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id + def evaluation_summary(self, metric: str): + """Summarizes the evaluation of a metric over all folds. + + The fold scores for the metric must exist already. During run creation, + by default, the MAE for OpenMLRegressionTask and the accuracy for + OpenMLClassificationTask/OpenMLLearningCurveTasktasks are computed. + + If repetition exist, we take the mean over all repetitions. + + Parameters + ---------- + metric: str + Name of an evaluation metric that was used to compute fold scores. + + Returns + ------- + metric_summary: str + A formatted string that displays the metric's evaluation summary. + The summary consists of the mean and std. + """ + fold_score_lists = self.fold_evaluations[metric].values() + + # Get the mean and std over all repetitions + rep_means = [np.mean(list(x.values())) for x in fold_score_lists] + rep_stds = [np.std(list(x.values())) for x in fold_score_lists] + + return "{:.4f} +- {:.4f}".format(np.mean(rep_means), np.mean(rep_stds)) + def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: """Collect all information to display in the __repr__ body.""" + # Set up fields fields = { "Uploader Name": self.uploader_name, "Metric": self.task_evaluation_measure, @@ -175,6 +204,10 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: "Dataset ID": self.dataset_id, "Dataset URL": openml.datasets.OpenMLDataset.url_for_id(self.dataset_id), } + + # determines the order of the initial fields in which the information will be printed + order = ["Uploader Name", "Uploader Profile", "Metric", "Result"] + if self.uploader is not None: fields["Uploader Profile"] = "{}/u/{}".format( openml.config.get_server_base_url(), self.uploader @@ -183,13 +216,29 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: fields["Run URL"] = self.openml_url if self.evaluations is not None and self.task_evaluation_measure in self.evaluations: fields["Result"] = self.evaluations[self.task_evaluation_measure] - - # determines the order in which the information will be printed - order = [ - "Uploader Name", - "Uploader Profile", - "Metric", - "Result", + elif self.fold_evaluations is not None: + # -- Add Locally computed summary values to if possible + if "predictive_accuracy" in self.fold_evaluations: + # OpenMLClassificationTask; OpenMLLearningCurveTask + # default: predictive_accuracy + result_field = "Local Result - Accuracy (+- STD)" + fields[result_field] = self.evaluation_summary("predictive_accuracy") + order.append(result_field) + elif "mean_absolute_error" in self.fold_evaluations: + # OpenMLRegressionTask + # default: mean_absolute_error + result_field = "Local Result - MAE (+- STD)" + fields[result_field] = self.evaluation_summary("mean_absolute_error") + order.append(result_field) + + rt_field = "Local Runtime - ms (+- STD)" + fields[rt_field] = self.evaluation_summary("usercpu_time_millis") + + # Add to order to be below / same as results + order.append(rt_field) + + # determines the remaining order + order += [ "Run ID", "Run URL", "Task ID", From 1bb2b1ff21770ff1971a7054cfabdef9177dc3b6 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:38:51 +0100 Subject: [PATCH 23/33] add PR to progress.rst --- doc/progress.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/progress.rst b/doc/progress.rst index 46c34c03c..c47644ce1 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -11,6 +11,7 @@ Changelog * FIX #1197 #559 #1131: Fix the order of ground truth and predictions in the ``OpenMLRun`` object and in ``format_prediction``. * FIX #1198: Support numpy 1.24 and higher. + * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. 0.13.0 ~~~~~~ From 741f766e70a6aa0d0378ef9de406c28e913362c9 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:43:21 +0100 Subject: [PATCH 24/33] fix comment typo --- openml/runs/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index cc30aba85..09a049c42 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -217,7 +217,7 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: if self.evaluations is not None and self.task_evaluation_measure in self.evaluations: fields["Result"] = self.evaluations[self.task_evaluation_measure] elif self.fold_evaluations is not None: - # -- Add Locally computed summary values to if possible + # -- Add locally computed summary values if possible if "predictive_accuracy" in self.fold_evaluations: # OpenMLClassificationTask; OpenMLLearningCurveTask # default: predictive_accuracy @@ -231,6 +231,7 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: fields[result_field] = self.evaluation_summary("mean_absolute_error") order.append(result_field) + # Runtime should be available for any task type rt_field = "Local Runtime - ms (+- STD)" fields[rt_field] = self.evaluation_summary("usercpu_time_millis") From c5b0789bbd5bef5db77072b58861a76931b4989b Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Wed, 22 Feb 2023 12:12:18 +0100 Subject: [PATCH 25/33] Update openml/runs/run.py Co-authored-by: Matthias Feurer --- openml/runs/run.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 09a049c42..6c22832ea 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -158,7 +158,7 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id - def evaluation_summary(self, metric: str): + def evaluation_summary(self, metric: str) -> str: """Summarizes the evaluation of a metric over all folds. The fold scores for the metric must exist already. During run creation, From 9706132c109a6e62b93c134f3074ebef89ad0ef8 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:06:40 +0100 Subject: [PATCH 26/33] add a function to list available estimation procedures --- openml/evaluations/functions.py | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 30d376c04..16b51b8a0 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -275,6 +275,39 @@ def list_evaluation_measures() -> List[str]: return qualities +def list_estimation_procedures(): + """Return list of evaluation procedures available. + + The function performs an API call to retrieve the entire list of + evaluation procedures' names that are available. + + Returns + ------- + list + """ + + api_call = "estimationprocedure/list" + xml_string = openml._api_calls._perform_api_call(api_call, "get") + api_results = xmltodict.parse(xml_string) + + # Minimalistic check if the XML is useful + if "oml:estimationprocedures" not in api_results: + raise ValueError("Error in return XML, does not contain " '"oml:estimationprocedures"') + if "oml:estimationprocedure" not in api_results["oml:estimationprocedures"]: + raise ValueError("Error in return XML, does not contain " '"oml:estimationprocedure"') + + if not isinstance(api_results["oml:estimationprocedures"]["oml:estimationprocedure"], list): + raise TypeError( + "Error in return XML, does not contain " '"oml:estimationprocedure" as a list' + ) + + prods = [ + prod["oml:name"] + for prod in api_results["oml:estimationprocedures"]["oml:estimationprocedure"] + ] + return prods + + def list_evaluations_setups( function: str, offset: Optional[int] = None, From 4493b12772bc50616e1e1e6f9b7cc7f866a8069c Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:07:43 +0100 Subject: [PATCH 27/33] refactor print to only work for supported task types and local measures --- openml/runs/run.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 6c22832ea..5f0b72927 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -158,7 +158,7 @@ def predictions(self) -> pd.DataFrame: def id(self) -> Optional[int]: return self.run_id - def evaluation_summary(self, metric: str) -> str: + def _evaluation_summary(self, metric: str) -> str: """Summarizes the evaluation of a metric over all folds. The fold scores for the metric must exist already. During run creation, @@ -222,21 +222,20 @@ def _get_repr_body_fields(self) -> List[Tuple[str, Union[str, int, List[str]]]]: # OpenMLClassificationTask; OpenMLLearningCurveTask # default: predictive_accuracy result_field = "Local Result - Accuracy (+- STD)" - fields[result_field] = self.evaluation_summary("predictive_accuracy") + fields[result_field] = self._evaluation_summary("predictive_accuracy") order.append(result_field) elif "mean_absolute_error" in self.fold_evaluations: # OpenMLRegressionTask # default: mean_absolute_error result_field = "Local Result - MAE (+- STD)" - fields[result_field] = self.evaluation_summary("mean_absolute_error") + fields[result_field] = self._evaluation_summary("mean_absolute_error") order.append(result_field) - # Runtime should be available for any task type - rt_field = "Local Runtime - ms (+- STD)" - fields[rt_field] = self.evaluation_summary("usercpu_time_millis") - - # Add to order to be below / same as results - order.append(rt_field) + if "usercpu_time_millis" in self.fold_evaluations: + # Runtime should be available for most tasks types + rt_field = "Local Runtime - ms (+- STD)" + fields[rt_field] = self._evaluation_summary("usercpu_time_millis") + order.append(rt_field) # determines the remaining order order += [ From f3d5753b152a6ba5bc938e9d8ed32f3fcf085b66 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Thu, 23 Feb 2023 14:33:47 +0100 Subject: [PATCH 28/33] add test for pint out and update progress --- tests/test_runs/test_run_functions.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/test_runs/test_run_functions.py b/tests/test_runs/test_run_functions.py index 786ab2291..520b7c0bc 100644 --- a/tests/test_runs/test_run_functions.py +++ b/tests/test_runs/test_run_functions.py @@ -531,6 +531,14 @@ def determine_grid_size(param_grid): # todo: check if runtime is present self._check_fold_timing_evaluations(run.fold_evaluations, 1, num_folds, task_type=task_type) + + # Check if run string and print representation do not run into an error + # The above check already verifies that all columns needed for supported + # representations are present. + # Supported: SUPERVISED_CLASSIFICATION, LEARNING_CURVE, SUPERVISED_REGRESSION + str(run) + self.logger.info(run) + return run def _run_and_upload_classification( From cd91ba0e5de54c0b8fc73bb777659d72488662d6 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:25:22 +0100 Subject: [PATCH 29/33] added additional task agnostic local result to print of run --- openml/runs/run.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index 5f0b72927..c3448f16b 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -39,13 +39,13 @@ class OpenMLRun(OpenMLBase): setup_string: str The setup string of the run. output_files: Dict[str, str] - Specifies where each related file can be found. + Specifies where each related file can be found. setup_id: int An integer representing the ID of the setup used for the run. tags: List[str] Representing the tags associated with the run. uploader: int - User ID of the uploader. + User ID of the uploader. uploader_name: str The name of the person who uploaded the run. evaluations: Dict @@ -79,7 +79,7 @@ class OpenMLRun(OpenMLBase): description_text: str, optional Description text to add to the predictions file. If left None, is set to the time the arff file is generated. run_details: str, optional (default=None) - Description of the run stored in the run meta-data. + Description of the run stored in the run meta-data. """ def __init__( From dea48f4f1f3c05cffc328e72272263c42f454633 Mon Sep 17 00:00:00 2001 From: LennartPurucker Date: Wed, 22 Feb 2023 11:38:51 +0100 Subject: [PATCH 30/33] add PR to progress.rst --- doc/progress.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/doc/progress.rst b/doc/progress.rst index c47644ce1..618e15185 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -9,6 +9,8 @@ Changelog 0.13.1 ~~~~~~ + * Add new contributions here. + * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. * FIX #1197 #559 #1131: Fix the order of ground truth and predictions in the ``OpenMLRun`` object and in ``format_prediction``. * FIX #1198: Support numpy 1.24 and higher. * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. From 8b868163a9e1635d179d7671b104d429dd5513bc Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Fri, 24 Feb 2023 18:13:43 +0100 Subject: [PATCH 31/33] add type hint for new function --- openml/evaluations/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openml/evaluations/functions.py b/openml/evaluations/functions.py index 16b51b8a0..693ec06cf 100644 --- a/openml/evaluations/functions.py +++ b/openml/evaluations/functions.py @@ -275,7 +275,7 @@ def list_evaluation_measures() -> List[str]: return qualities -def list_estimation_procedures(): +def list_estimation_procedures() -> List[str]: """Return list of evaluation procedures available. The function performs an API call to retrieve the entire list of From b3b4447dbdb0f5b3aa11de5ec0472dafa9a68af1 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Fri, 24 Feb 2023 18:20:12 +0100 Subject: [PATCH 32/33] update add description --- doc/progress.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/progress.rst b/doc/progress.rst index 618e15185..48dc2a1a3 100644 --- a/doc/progress.rst +++ b/doc/progress.rst @@ -13,7 +13,7 @@ Changelog * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. * FIX #1197 #559 #1131: Fix the order of ground truth and predictions in the ``OpenMLRun`` object and in ``format_prediction``. * FIX #1198: Support numpy 1.24 and higher. - * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation. + * ADD#1144: Add locally computed results to the ``OpenMLRun`` object's representation if the run was created locally and not downloaded from the server. 0.13.0 ~~~~~~ From 4ba2dc0463c07f0173660690b0454e2426973d69 Mon Sep 17 00:00:00 2001 From: Lennart Purucker Date: Wed, 1 Mar 2023 10:53:19 +0100 Subject: [PATCH 33/33] fix run doc string --- openml/runs/run.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/openml/runs/run.py b/openml/runs/run.py index c3448f16b..5528c8a67 100644 --- a/openml/runs/run.py +++ b/openml/runs/run.py @@ -26,7 +26,7 @@ class OpenMLRun(OpenMLBase): - """OpenML Run: result of running a model on an openml dataset. + """OpenML Run: result of running a model on an OpenML dataset. Parameters ---------- @@ -71,13 +71,16 @@ class OpenMLRun(OpenMLBase): predictions_url: str The URL of the predictions file. task: OpenMLTask - An instance of the OpenMLTask class, representing the OpenML task associated with the run. + An instance of the OpenMLTask class, representing the OpenML task associated + with the run. flow: OpenMLFlow - An instance of the OpenMLFlow class, representing the OpenML flow associated with the run. + An instance of the OpenMLFlow class, representing the OpenML flow associated + with the run. run_id: int The ID of the run. description_text: str, optional - Description text to add to the predictions file. If left None, is set to the time the arff file is generated. + Description text to add to the predictions file. If left None, is set to the + time the arff file is generated. run_details: str, optional (default=None) Description of the run stored in the run meta-data. """