test_data_drift_table.py
1 import json 2 3 import numpy as np 4 import pandas as pd 5 import pytest 6 from pytest import approx 7 8 from evidently.legacy.metrics.data_drift.data_drift_table import DataDriftTable 9 from evidently.legacy.pipeline.column_mapping import ColumnMapping 10 from evidently.legacy.report import Report 11 12 13 @pytest.mark.parametrize( 14 "current_dataset, reference_dataset, data_mapping", 15 ( 16 ( 17 pd.DataFrame( 18 { 19 "category_feature": ["1", "2", "3"], 20 "numerical_feature": [3, 2, 1], 21 "target": [None, np.nan, 1], 22 "prediction": [1, np.nan, 1], 23 } 24 ), 25 pd.DataFrame( 26 { 27 "category_feature": ["1", "2", "3"], 28 "numerical_feature": [3, 2, 1], 29 "target": [None, np.nan, 1], 30 "prediction": [1, np.nan, 1], 31 } 32 ), 33 ColumnMapping(), 34 ), 35 ( 36 pd.DataFrame( 37 { 38 "category_feature": ["1", "2", "3"], 39 "numerical_feature": [3, 2, 1], 40 "target": [None, np.nan, 1], 41 "prediction": [1, np.nan, 1], 42 } 43 ), 44 pd.DataFrame( 45 { 46 "category_feature": ["a", "b", "c", "a", "b", "c"], 47 "numerical_feature": [6, 6, 6, 9, 9, 9], 48 "target": [5, 4, 3, 2, 1, 0], 49 "prediction": [1, 2, 3, 4, 5, 6], 50 } 51 ), 52 ColumnMapping(), 53 ), 54 # binary classification 55 ( 56 pd.DataFrame( 57 { 58 "category_feature": ["a", "b", "c"], 59 "numerical_feature": [6, 6, 6], 60 "label_a": [0.3, 0.2, 0.1], 61 "label_b": [0.5, 0.5, 1], 62 "target": [1, 1, 1], 63 } 64 ), 65 pd.DataFrame( 66 { 67 "category_feature": ["a", "b", "c"], 68 "numerical_feature": [6, 6, 6], 69 "label_a": [0.9, 0.5, 0.3], 70 "label_b": [0.2, 0.5, 0.7], 71 "target": [0, 0, 0], 72 } 73 ), 74 ColumnMapping(prediction=["label_a", "label_b"]), 75 ), 76 # multi classification 77 ( 78 pd.DataFrame( 79 { 80 "category_feature": ["a", "b", "c"], 81 "numerical_feature": [6, 6, 6], 82 "label_a": [0.3, 0.2, 0.1], 83 "label_b": [0.5, 0.5, 1], 84 "label_c": [0.2, 0.5, 0.7], 85 "my_target": [1, 1, 1], 86 } 87 ), 88 pd.DataFrame( 89 { 90 "category_feature": ["a", "b", "c"], 91 "numerical_feature": [6, 6, 6], 92 "label_a": [0.9, 0.5, 0.3], 93 "label_b": [0.2, 0.5, 0.7], 94 "label_c": [0.3, 0.2, 0.1], 95 "my_target": [0, 0, 0], 96 } 97 ), 98 ColumnMapping(target="my_target", prediction=["label_a", "label_b", "label_c"]), 99 ), 100 ), 101 ) 102 def test_data_drift_metrics_no_errors( 103 current_dataset: pd.DataFrame, reference_dataset: pd.DataFrame, data_mapping: ColumnMapping 104 ) -> None: 105 report = Report(metrics=[DataDriftTable()]) 106 report.run(current_data=current_dataset, reference_data=reference_dataset, column_mapping=data_mapping) 107 assert report.show() 108 assert report.json() 109 110 111 def test_data_drift_metrics_value_error() -> None: 112 test_data = pd.DataFrame( 113 { 114 "category_feature": ["1", "2", "3"], 115 "numerical_feature": [3, 2, 1], 116 "target": [None, np.nan, 1], 117 "prediction": [1, np.nan, 1], 118 } 119 ) 120 data_mapping = ColumnMapping() 121 report = Report(metrics=[DataDriftTable()]) 122 123 with pytest.raises(ValueError): 124 report.run(current_data=test_data, reference_data=None, column_mapping=data_mapping) 125 report.json() 126 127 with pytest.raises(ValueError): 128 # noinspection PyTypeChecker 129 report.run(current_data=None, reference_data=test_data, column_mapping=data_mapping) 130 report.json() 131 132 133 def test_data_drift_metrics_with_options() -> None: 134 current_dataset = pd.DataFrame( 135 { 136 "category_feature": ["a", "b", "a"], 137 "target": [1, 2, 3], 138 "prediction": [1, 0, 1], 139 } 140 ) 141 reference_dataset = pd.DataFrame( 142 { 143 "category_feature": ["a", "a", "b"], 144 "target": [1, 4, 5], 145 "prediction": [1, 0, 1], 146 } 147 ) 148 report = Report(metrics=[DataDriftTable(stattest_threshold=0.7)]) 149 report.run(current_data=current_dataset, reference_data=reference_dataset) 150 assert report.show() 151 result_json = report.json() 152 result = json.loads(result_json) 153 assert result["metrics"][0]["metric"] == "DataDriftTable" 154 assert result["metrics"][0]["result"] == { 155 "current_fi": None, 156 "dataset_drift": False, 157 "drift_by_columns": { 158 "category_feature": { 159 "column_name": "category_feature", 160 "column_type": "cat", 161 "drift_detected": False, 162 "drift_score": 1.0, 163 "stattest_name": "Z-test p_value", 164 "stattest_threshold": 0.7, 165 "current": {"small_distribution": {"x": ["a", "b"], "y": [2, 1]}}, 166 "reference": {"small_distribution": {"x": ["a", "b"], "y": [2, 1]}}, 167 }, 168 "prediction": { 169 "column_name": "prediction", 170 "column_type": "cat", 171 "drift_detected": False, 172 "drift_score": 1.0, 173 "stattest_name": "Z-test p_value", 174 "stattest_threshold": 0.7, 175 "current": {"small_distribution": {"x": [0, 1], "y": [1, 2]}}, 176 "reference": {"small_distribution": {"x": [0, 1], "y": [1, 2]}}, 177 }, 178 "target": { 179 "column_name": "target", 180 "column_type": "cat", 181 "drift_detected": True, 182 "drift_score": 0.0, 183 "stattest_name": "chi-square p_value", 184 "stattest_threshold": 0.7, 185 "current": {"small_distribution": {"x": [1, 2, 3, 4, 5], "y": [1, 1, 1, 0, 0]}}, 186 "reference": {"small_distribution": {"x": [1, 2, 3, 4, 5], "y": [1, 0, 0, 1, 1]}}, 187 }, 188 }, 189 "number_of_columns": 3, 190 "number_of_drifted_columns": 1, 191 "reference_fi": None, 192 "share_of_drifted_columns": 0.3333333333333333, 193 } 194 195 196 def test_data_drift_metrics_json_output() -> None: 197 current_dataset = pd.DataFrame( 198 { 199 "category_feature": ["a", "b", "a", np.nan], 200 "target": [np.nan, np.nan, 3, 4], 201 "prediction": [1, 0, np.nan, 5], 202 } 203 ) 204 reference_dataset = pd.DataFrame( 205 { 206 "category_feature": ["a", "a", "b", "b"], 207 "target": [1, 4, 5, 5], 208 "prediction": [1, 5, 4, 4], 209 } 210 ) 211 report = Report(metrics=[DataDriftTable(stattest_threshold=0.7)]) 212 report.run(current_data=current_dataset, reference_data=reference_dataset) 213 result_json = report.json() 214 result = json.loads(result_json) 215 assert result["metrics"][0]["metric"] == "DataDriftTable" 216 assert result["metrics"][0]["result"] == { 217 "current_fi": None, 218 "dataset_drift": True, 219 "drift_by_columns": { 220 "category_feature": { 221 "column_name": "category_feature", 222 "column_type": "cat", 223 "drift_detected": True, 224 "drift_score": approx(0.66, abs=0.01), 225 "stattest_name": "Z-test p_value", 226 "stattest_threshold": 0.7, 227 "current": {"small_distribution": {"x": ["a", "b"], "y": [2, 1]}}, 228 "reference": {"small_distribution": {"x": ["a", "b"], "y": [2, 2]}}, 229 }, 230 "prediction": { 231 "column_name": "prediction", 232 "column_type": "cat", 233 "drift_detected": True, 234 "drift_score": 0.0, 235 "stattest_name": "chi-square p_value", 236 "stattest_threshold": 0.7, 237 "current": {"small_distribution": {"x": [0.0, 1.0, 4.0, 5.0], "y": [1, 1, 0, 1]}}, 238 "reference": {"small_distribution": {"x": [0, 1, 4, 5], "y": [0, 1, 2, 1]}}, 239 }, 240 "target": { 241 "column_name": "target", 242 "column_type": "cat", 243 "drift_detected": True, 244 "drift_score": 0.0, 245 "stattest_name": "chi-square p_value", 246 "stattest_threshold": 0.7, 247 "current": {"small_distribution": {"x": [1.0, 3.0, 4.0, 5.0], "y": [0, 1, 1, 0]}}, 248 "reference": {"small_distribution": {"x": [1, 3, 4, 5], "y": [1, 0, 1, 2]}}, 249 }, 250 }, 251 "number_of_columns": 3, 252 "number_of_drifted_columns": 3, 253 "reference_fi": None, 254 "share_of_drifted_columns": 1, 255 }