1  import json
  2  
  3  import pandas as pd
  4  
  5  from evidently.legacy.pipeline.column_mapping import ColumnMapping
  6  from evidently.legacy.test_suite import TestSuite
  7  from evidently.legacy.tests import TestValueAbsMaxError
  8  from evidently.legacy.tests import TestValueMAE
  9  from evidently.legacy.tests import TestValueMAPE
 10  from evidently.legacy.tests import TestValueMeanError
 11  from evidently.legacy.tests import TestValueR2Score
 12  from evidently.legacy.tests import TestValueRMSE
 13  
 14  
 15  def test_value_mae_test() -> None:
 16      test_dataset = pd.DataFrame(
 17          {
 18              "category_feature": ["n", "d", "p", "n"],
 19              "numerical_feature": [0, 1, 2, 5],
 20              "target": [0, 0, 0, 1],
 21              "prediction": [0, 1, 0, 0],
 22          }
 23      )
 24      suite = TestSuite(tests=[TestValueMAE(gte=10)])
 25      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping())
 26      assert not suite
 27  
 28      suite = TestSuite(tests=[TestValueMAE(eq=0.5)])
 29      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping())
 30      assert suite
 31      assert suite.show()
 32      assert suite.json()
 33  
 34  
 35  def test_value_mae_test_render_json() -> None:
 36      test_dataset = pd.DataFrame(
 37          {
 38              "category_feature": ["n", "d", "p", "n"],
 39              "numerical_feature": [0, 1, 2, 5],
 40              "target": [0, 0, 0, 1],
 41              "prediction": [0, 1, 0, 0],
 42          }
 43      )
 44      suite = TestSuite(tests=[TestValueMAE()])
 45      suite.run(current_data=test_dataset, reference_data=test_dataset, column_mapping=ColumnMapping())
 46      result_json = suite.json()
 47      assert isinstance(result_json, str)
 48  
 49      result = json.loads(result_json)["tests"][0]
 50      assert result == {
 51          "description": "The MAE is 0.5. The test threshold is eq=0.5 ± 0.05",
 52          "group": "regression",
 53          "name": "Mean Absolute Error (MAE)",
 54          "parameters": {"condition": {"eq": {"absolute": 1e-12, "relative": 0.1, "value": 0.5}}, "value": 0.5},
 55          "status": "SUCCESS",
 56      }
 57  
 58  
 59  def test_value_mape_test() -> None:
 60      test_dataset = pd.DataFrame(
 61          {
 62              "category_feature": ["n", "d", "p", "n"],
 63              "numerical_feature": [0, 1, 2, 5],
 64              "target": [0, 0, 0, 1],
 65              "prediction": [0, 0, 0, 0],
 66          }
 67      )
 68      suite = TestSuite(tests=[TestValueMAPE(lt=25)])
 69      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping())
 70      assert not suite
 71  
 72      suite = TestSuite(tests=[TestValueMAPE(eq=120)])
 73      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping())
 74      assert not suite
 75      assert suite.show()
 76      assert suite.json()
 77  
 78  
 79  def test_value_mape_test_render_json() -> None:
 80      test_dataset = pd.DataFrame(
 81          {
 82              "category_feature": ["n", "d", "p", "n"],
 83              "numerical_feature": [0, 1, 2, 5],
 84              "target": [0, 0, 0, 1],
 85              "prediction": [0, 0, 0, 0],
 86          }
 87      )
 88      suite = TestSuite(tests=[TestValueMAPE()])
 89      suite.run(current_data=test_dataset, reference_data=test_dataset, column_mapping=ColumnMapping())
 90      result_json = suite.json()
 91      assert isinstance(result_json, str)
 92  
 93      result = json.loads(result_json)["tests"][0]
 94      assert result == {
 95          "description": "The MAPE is 1e+02. The test threshold is eq=100 ± 10.",
 96          "group": "regression",
 97          "name": "Mean Absolute Percentage Error (MAPE)",
 98          "parameters": {"condition": {"eq": {"absolute": 1e-12, "relative": 0.1, "value": 100.0}}, "value": 100.0},
 99          "status": "SUCCESS",
100      }
101  
102  
103  def test_value_mean_error_test() -> None:
104      test_dataset = pd.DataFrame(
105          {
106              "category_feature": ["n", "d", "p", "n"],
107              "numerical_feature": [0, 1, 2, 5],
108              "target": [0, 0, 0, 1],
109              "prediction": [0, 1, 0, 0],
110          }
111      )
112      suite = TestSuite(tests=[TestValueMeanError(gt=1)])
113      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping())
114      assert not suite
115  
116      suite = TestSuite(tests=[TestValueMeanError(eq=0.0)])
117      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping())
118      assert suite
119      assert suite.show()
120      assert suite.json()
121  
122  
123  def test_value_mean_error_test_render_json() -> None:
124      test_dataset = pd.DataFrame(
125          {
126              "category_feature": ["n", "d", "p", "n"],
127              "numerical_feature": [0, 1, 2, 5],
128              "target": [0, 0, 0, 1],
129              "prediction": [0, 1, 0, 0],
130          }
131      )
132      suite = TestSuite(tests=[TestValueMeanError()])
133      suite.run(current_data=test_dataset, reference_data=test_dataset, column_mapping=ColumnMapping())
134      result_json = suite.json()
135      assert isinstance(result_json, str)
136  
137      result = json.loads(result_json)["tests"][0]
138      assert result == {
139          "description": "The ME is 0.0. The test threshold is eq=0 ± 0.0816.",
140          "group": "regression",
141          "name": "Mean Error (ME)",
142          "parameters": {
143              "condition": {"eq": {"absolute": 0.08164965809277261, "relative": 1e-06, "value": 0}},
144              "value": 0.0,
145          },
146          "status": "SUCCESS",
147      }
148  
149  
150  def test_abs_max_error_test() -> None:
151      test_dataset = pd.DataFrame(
152          {
153              "category_feature": ["n", "d", "p", "n"],
154              "numerical_feature": [0, 1, 2, 5],
155              "target": [1, 2, 3, 4],
156              "preds": [1.0, 2.0, 3.0, 4.0],
157          }
158      )
159      suite = TestSuite(tests=[TestValueAbsMaxError(lt=1)])
160      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping(prediction="preds"))
161      assert suite
162      assert suite.show()
163      assert suite.json()
164  
165  
166  def test_abs_max_error_test_render_json() -> None:
167      test_dataset = pd.DataFrame(
168          {
169              "category_feature": ["n", "d", "p", "n"],
170              "numerical_feature": [0, 1, 2, 5],
171              "target": [1, 2, 3, 4],
172              "prediction": [1.0, 2.0, 3.0, 4.0],
173          }
174      )
175      suite = TestSuite(tests=[TestValueAbsMaxError()])
176      suite.run(current_data=test_dataset, reference_data=test_dataset, column_mapping=ColumnMapping())
177  
178      result_json = suite.json()
179      assert isinstance(result_json, str)
180  
181      result = json.loads(result_json)["tests"][0]
182      assert result == {
183          "description": "The Max Absolute Error is 0.0. The test threshold is lte=0 ± 1e-12.",
184          "group": "regression",
185          "name": "Max Absolute Error",
186          "parameters": {
187              "value": 0.0,
188              "condition": {"lte": {"absolute": 1e-12, "relative": 0.1, "value": 0.0}},
189          },
190          "status": "SUCCESS",
191      }
192  
193  
194  def test_r2_score_test() -> None:
195      test_dataset = pd.DataFrame(
196          {
197              "category_feature": ["n", "d", "p", "n"],
198              "numerical_feature": [0, 1, 2, 5],
199              "target": [1, 2, 3, 4],
200              "preds": [1, 2, 3, 3],
201          }
202      )
203      suite = TestSuite(tests=[TestValueR2Score(lt=1)])
204      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping(prediction="preds"))
205      assert suite
206      assert suite.show()
207      assert suite.json()
208  
209  
210  def test_r2_score_test_render_json() -> None:
211      test_dataset = pd.DataFrame(
212          {
213              "category_feature": ["n", "d", "p", "n"],
214              "numerical_feature": [0, 1, 2, 5],
215              "target": [1, 2, 3, 4],
216              "prediction": [1, 2, 3, 3],
217          }
218      )
219      suite = TestSuite(tests=[TestValueR2Score()])
220      suite.run(current_data=test_dataset, reference_data=test_dataset, column_mapping=ColumnMapping())
221      assert suite
222  
223      result_json = suite.json()
224      assert isinstance(result_json, str)
225  
226      result = json.loads(result_json)["tests"][0]
227      assert result == {
228          "description": "The R2 score is 0.8. The test threshold is eq=0.8 ± 0.08.",
229          "group": "regression",
230          "name": "R2 Score",
231          "parameters": {"condition": {"eq": {"absolute": 1e-12, "relative": 0.1, "value": 0.8}}, "value": 0.8},
232          "status": "SUCCESS",
233      }
234  
235  
236  def test_rmse_score_test() -> None:
237      test_dataset = pd.DataFrame(
238          {
239              "category_feature": ["n", "d", "p", "n"],
240              "numerical_feature": [0, 1, 2, 5],
241              "target": [1, 2, 3, 4],
242              "preds": [1, 2, 3, 3],
243          }
244      )
245      suite = TestSuite(tests=[TestValueRMSE(lt=1)])
246      suite.run(current_data=test_dataset, reference_data=None, column_mapping=ColumnMapping(prediction="preds"))
247      assert suite
248      assert suite.show()
249      assert suite.json()
250  
251  
252  def test_rmse_score_test_render_json() -> None:
253      test_dataset = pd.DataFrame(
254          {
255              "category_feature": ["n", "d", "p", "n"],
256              "numerical_feature": [0, 1, 2, 5],
257              "target": [1, 2, 3, 4],
258              "prediction": [1, 2, 3, 3],
259          }
260      )
261      suite = TestSuite(tests=[TestValueRMSE()])
262      suite.run(current_data=test_dataset, reference_data=test_dataset, column_mapping=ColumnMapping())
263      assert suite
264  
265      result_json = suite.json()
266      assert isinstance(result_json, str)
267  
268      result = json.loads(result_json)["tests"][0]
269      assert result == {
270          "description": "The RMSE is 0.5. The test threshold is eq=0.5 ± 0.05.",
271          "group": "regression",
272          "name": "Root Mean Square Error (RMSE)",
273          "parameters": {"condition": {"eq": {"absolute": 1e-12, "relative": 0.1, "value": 0.5}}, "value": 0.5},
274          "status": "SUCCESS",
275      }