Uh oh!
There was an error while loading.Please reload this page.
- Notifications
You must be signed in to change notification settings - Fork8.1k
AUC performance comparison between two methods using multiple iterati…#30891
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to ourterms of service andprivacy statement. We’ll occasionally send you account related emails.
Already on GitHub?Sign in to your account
Open
ZhenjiangFan wants to merge1 commit intomatplotlib:mainChoose a base branch
base:main
Could not load branches
Branch not found:{{ refName }}
Loading
Could not load tags
Nothing to show
Loading
Are you sure you want to change the base?
Some commits from the old base branch may be removed from the timeline, and old review comments may become outdated.
+170 −0
Open
Changes fromall commits
Commits
File filter
Filter by extension
Conversations
Failed to load comments.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Jump to
Jump to file
Failed to load files.
Loading
Uh oh!
There was an error while loading.Please reload this page.
Diff view
Diff view
There are no files selected for viewing
170 changes: 170 additions & 0 deletionsgalleries/examples/statistics/auc_performance_comparison_between_two_methods.py
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,170 @@ | ||
| ### AUC performance comparison between two methods using multiple iterations | ||
Check warning on line 1 in galleries/examples/statistics/auc_performance_comparison_between_two_methods.py
| ||
| #of true labels and corresponding predicted labels. | ||
| import random | ||
| import numpy as np | ||
| import matplotlib.pyplot as plt | ||
| import sklearn.metrics as metrics | ||
| ''' | ||
| Generate synthetic label data. | ||
| ''' | ||
| def generate_synthetic_label_data(): | ||
Check warning on line 14 in galleries/examples/statistics/auc_performance_comparison_between_two_methods.py
| ||
| predicted_labels_existing = [] | ||
| predicted_labels_proposed = [] | ||
| #Generate synthetic predicted labels from an existing method. | ||
| #20 runs and each run has 20 labels. | ||
| #The synthetic data is stored in a 2D list. | ||
| for i in range(20): | ||
| predicted_labels_p1 = [] | ||
| predicted_labels_p2 = [] | ||
| for idx in range(20): | ||
| probability_of_true = 0.5 | ||
| probability_of_false = 1.0 - probability_of_true | ||
| ran_bool = random.choices([True, False], weights=[probability_of_true | ||
| , probability_of_false])[0] | ||
Check warning on line 31 in galleries/examples/statistics/auc_performance_comparison_between_two_methods.py
| ||
| temp_num = 0 | ||
| if ran_bool: | ||
| temp_num = random.uniform(0.9, 1.0) | ||
| else: | ||
| temp_num = random.uniform(0.0, 0.1) | ||
| predicted_labels_p1.append(temp_num) | ||
| ran_bool = random.choices([True, False], weights=[probability_of_true | ||
| , probability_of_false])[0] | ||
Check warning on line 40 in galleries/examples/statistics/auc_performance_comparison_between_two_methods.py
| ||
| if ran_bool: | ||
| temp_num = random.uniform(0.0, 0.1) | ||
| else: | ||
| temp_num = random.uniform(0.9, 1.0) | ||
| predicted_labels_p2.append(temp_num) | ||
| predicted_labels_temp = predicted_labels_p1 + predicted_labels_p2 | ||
| predicted_labels_existing.append(predicted_labels_temp) | ||
| #Generate synthetic predicted labels from the proposed method. | ||
| #20 runs and each run has 20 labels. | ||
| #The synthetic data is stored in a 2D list. | ||
| for i in range(20): | ||
| predicted_labels_p1 = [] | ||
| predicted_labels_p2 = [] | ||
| for idx in range(20): | ||
| probability_of_true = 0.9 | ||
| probability_of_false = 1.0 - probability_of_true | ||
| ran_bool = random.choices([True, False], weights=[probability_of_true | ||
| , probability_of_false])[0] | ||
Check warning on line 64 in galleries/examples/statistics/auc_performance_comparison_between_two_methods.py
| ||
| temp_num = 0 | ||
| if ran_bool: | ||
| temp_num = random.uniform(0.9, 1.0) | ||
| else: | ||
| temp_num = random.uniform(0.0, 0.1) | ||
| predicted_labels_p1.append(temp_num) | ||
| ran_bool = random.choices([True, False], weights=[probability_of_true | ||
| , probability_of_false])[0] | ||
Check warning on line 73 in galleries/examples/statistics/auc_performance_comparison_between_two_methods.py
| ||
| if ran_bool: | ||
| temp_num = random.uniform(0.0, 0.1) | ||
| else: | ||
| temp_num = random.uniform(0.9, 1.0) | ||
| predicted_labels_p2.append(temp_num) | ||
| predicted_labels_temp = predicted_labels_p1 + predicted_labels_p2 | ||
| predicted_labels_proposed.append(predicted_labels_temp) | ||
| return predicted_labels_existing, predicted_labels_proposed | ||
| ''' | ||
| Plot an average AUC figure. | ||
| ''' | ||
| def plot_average_auc_from_multiple_runs(ax, true_labels, predicted_labels_mat | ||
Check warning on line 89 in galleries/examples/statistics/auc_performance_comparison_between_two_methods.py
| ||
| , method_name="existing method", color_to_method="red"): | ||
| tprs = [] | ||
| aucs = [] | ||
| mean_fpr = np.linspace(0, 1, 100) | ||
| for index,predicted_labels in enumerate(predicted_labels_mat): | ||
| curve_parameters = {'color': color_to_method, 'label': None, 'alpha':0.3 | ||
| , 'lw':1} | ||
| viz = metrics.RocCurveDisplay.from_predictions(true_labels, predicted_labels | ||
| ,ax=ax, curve_kwargs=curve_parameters) | ||
| interp_tpr = np.interp(mean_fpr, viz.fpr, viz.tpr) | ||
| interp_tpr[0] = 0.0 | ||
| tprs.append(interp_tpr) | ||
| aucs.append(viz.roc_auc) | ||
| mean_tpr = np.mean(tprs, axis=0) | ||
| mean_tpr[-1] = 1.0 | ||
| mean_auc = metrics.auc(mean_fpr, mean_tpr) | ||
| ax.plot( | ||
| mean_fpr, | ||
| mean_tpr, | ||
| color=color_to_method, | ||
| label=f"Average ROC (AUC = {mean_auc:.2f}) from {method_name}", | ||
| lw=2, | ||
| ) | ||
| std_tpr = np.std(tprs, axis=0) | ||
| tprs_upper = np.minimum(mean_tpr + std_tpr, 1) | ||
| tprs_lower = np.maximum(mean_tpr - std_tpr, 0) | ||
| ax.fill_between( | ||
| mean_fpr, | ||
| tprs_lower, | ||
| tprs_upper, | ||
| color=color_to_method, | ||
| alpha=0.1, | ||
| ) | ||
| ax.plot([0, 0], [0, 0], lw=1, color=color_to_method | ||
| , label=f"ROCs from {method_name}", alpha=0.2) | ||
| predicted_labels_existing, predicted_labels_proposed = generate_synthetic_label_data() | ||
| fig, ax = plt.subplots(figsize=(10,12)) | ||
| #True label list | ||
| true_labels = [1,1,1,1,1,1,1,1,1,1 | ||
| ,1,1,1,1,1,1,1,1,1,1 | ||
| ,0,0,0,0,0,0,0,0,0,0 | ||
| ,0,0,0,0,0,0,0,0,0,0 | ||
| ] | ||
| #Plot an average AUC line and other stats from multiple AUCs for an existing method. | ||
| method_name = "existing method" | ||
| color="grey" | ||
| plot_average_auc_from_multiple_runs(ax, true_labels, predicted_labels_existing | ||
| , method_name=method_name, color_to_method=color) | ||
| #Plot an average AUC line and other stats from multiple AUCs for the proposed method. | ||
| method_name = "proposed method" | ||
| color="red" | ||
| plot_average_auc_from_multiple_runs(ax, true_labels, predicted_labels_proposed | ||
| , method_name=method_name, color_to_method=color) | ||
| #Plot a line for random guess | ||
| ax.plot([0, 1], [0, 1], linestyle="--", lw=2, color="black", label="Random guess" | ||
| , alpha=0.8) | ||
| ax.set(xlim=[-0.05, 1.05],ylim=[-0.05, 1.05],) | ||
| plt.xlabel("False positive rate",fontsize=20) | ||
| plt.ylabel("True positive rate",fontsize=20) | ||
| plt.xticks(fontsize=20) | ||
| plt.yticks(fontsize=20) | ||
| #Setrings for legends | ||
| ax.legend(loc="lower right",fontsize=12,frameon=False) | ||
| #Display the figure | ||
| plt.show() | ||
Oops, something went wrong.
Uh oh!
There was an error while loading.Please reload this page.
Add this suggestion to a batch that can be applied as a single commit.This suggestion is invalid because no changes were made to the code.Suggestions cannot be applied while the pull request is closed.Suggestions cannot be applied while viewing a subset of changes.Only one suggestion per line can be applied in a batch.Add this suggestion to a batch that can be applied as a single commit.Applying suggestions on deleted lines is not supported.You must change the existing code in this line in order to create a valid suggestion.Outdated suggestions cannot be applied.This suggestion has been applied or marked resolved.Suggestions cannot be applied from pending reviews.Suggestions cannot be applied on multi-line comments.Suggestions cannot be applied while the pull request is queued to merge.Suggestion cannot be applied right now. Please check back later.