Skip to content

Evaluation

Results of an experiment evaluation.

Source code in src/epstats/toolkit/experiment.py 
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
class Evaluation:
    """
    Results of an experiment evaluation.
    """

    def __init__(self, metrics: pd.DataFrame, checks: pd.DataFrame, exposures: pd.DataFrame):
        self.metrics = metrics
        self.checks = checks
        self.exposures = exposures

    @classmethod
    def metric_columns(cls) -> List[str]:
        """
        `metrics` dataframe with columns:

        1. `timestamp` - timestamp of evaluation
        1. `exp_id` - experiment id
        1. `metric_id` - metric id as in [`Experiment`][epstats.toolkit.experiment.Experiment] definition
        1. `metric_name` - metric name as in [`Experiment`][epstats.toolkit.experiment.Experiment] definition
        1. `exp_variant_id` - variant id
        1. `count` - number of exposures, value of metric denominator
        1. `mean` - `sum_value` / `count`
        1. `std` - sample standard deviation
        1. `sum_value` - value of goals, value of metric nominator
        1. `confidence_level` - current confidence level used to calculate `p_value` and `confidence_interval`
        1. `diff` - relative diff between sample means of this and control variant
        1. `test_stat` - value of test statistic of the relative difference in means
        1. `p_value` - p-value of the test statistic under current `confidence_level`
        1. `confidence_interval` - confidence interval of the `diff` under current `confidence_level`
        1. `standard_error` - standard error of the `diff`
        1. `degrees_of_freedom` - degrees of freedom of this variant mean
        1. `sample_size` - current sample size
        1. `required_sample_size` - size of the sample required to reach the required power
        1. `power` - power based on the collected `sample_size`
        1. `false_positive_risk` - false positive risk of a significant metric
        """
        return [
            "timestamp",
            "exp_id",
            "metric_id",
            "metric_name",
            "exp_variant_id",
            "count",
            "mean",
            "std",
            "sum_value",
            "confidence_level",
            "diff",
            "test_stat",
            "p_value",
            "confidence_interval",
            "standard_error",
            "degrees_of_freedom",
            "minimum_effect",
            "sample_size",
            "required_sample_size",
            "power",
            "false_positive_risk",
        ]

    @classmethod
    def check_columns(cls) -> List[str]:
        """
        `checks` dataframe with columns:

        1. `timestamp` - timestamp of evaluation
        1. `exp_id` - experiment id
        1. `check_id` - check id as in [`Experiment`][epstats.toolkit.experiment.Experiment] definition
        1. `variable_id` - name of the variable in check evaluation, SRM check has following variables `p_value`,
        `test_stat`, `confidence_level`
        1. `value` - value of the variable
        """
        return ["timestamp", "exp_id", "check_id", "check_name", "variable_id", "value"]

    @classmethod
    def exposure_columns(cls) -> List[str]:
        """
        `exposures` dataframe with columns:

        1. `timestamp` - timestamp of evaluation
        1. `exp_id` - experiment id
        1. `exp_variant_id` - variant id
        1. `exposures` - number of exposures of this variant
        """
        return ["exp_variant_id", "exposures"]

check_columns() classmethod

checks dataframe with columns:

  1. timestamp - timestamp of evaluation
  2. exp_id - experiment id
  3. check_id - check id as in Experiment definition
  4. variable_id - name of the variable in check evaluation, SRM check has following variables p_value, test_stat, confidence_level
  5. value - value of the variable
Source code in src/epstats/toolkit/experiment.py 
83
84
85
86
87
88
89
90
91
92
93
94
95
@classmethod
def check_columns(cls) -> List[str]:
    """
    `checks` dataframe with columns:

    1. `timestamp` - timestamp of evaluation
    1. `exp_id` - experiment id
    1. `check_id` - check id as in [`Experiment`][epstats.toolkit.experiment.Experiment] definition
    1. `variable_id` - name of the variable in check evaluation, SRM check has following variables `p_value`,
    `test_stat`, `confidence_level`
    1. `value` - value of the variable
    """
    return ["timestamp", "exp_id", "check_id", "check_name", "variable_id", "value"]

exposure_columns() classmethod

exposures dataframe with columns:

  1. timestamp - timestamp of evaluation
  2. exp_id - experiment id
  3. exp_variant_id - variant id
  4. exposures - number of exposures of this variant
Source code in src/epstats/toolkit/experiment.py 
 97
 98
 99
100
101
102
103
104
105
106
107
@classmethod
def exposure_columns(cls) -> List[str]:
    """
    `exposures` dataframe with columns:

    1. `timestamp` - timestamp of evaluation
    1. `exp_id` - experiment id
    1. `exp_variant_id` - variant id
    1. `exposures` - number of exposures of this variant
    """
    return ["exp_variant_id", "exposures"]

metric_columns() classmethod

metrics dataframe with columns:

  1. timestamp - timestamp of evaluation
  2. exp_id - experiment id
  3. metric_id - metric id as in Experiment definition
  4. metric_name - metric name as in Experiment definition
  5. exp_variant_id - variant id
  6. count - number of exposures, value of metric denominator
  7. mean - sum_value / count
  8. std - sample standard deviation
  9. sum_value - value of goals, value of metric nominator
  10. confidence_level - current confidence level used to calculate p_value and confidence_interval
  11. diff - relative diff between sample means of this and control variant
  12. test_stat - value of test statistic of the relative difference in means
  13. p_value - p-value of the test statistic under current confidence_level
  14. confidence_interval - confidence interval of the diff under current confidence_level
  15. standard_error - standard error of the diff
  16. degrees_of_freedom - degrees of freedom of this variant mean
  17. sample_size - current sample size
  18. required_sample_size - size of the sample required to reach the required power
  19. power - power based on the collected sample_size
  20. false_positive_risk - false positive risk of a significant metric
Source code in src/epstats/toolkit/experiment.py 
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
@classmethod
def metric_columns(cls) -> List[str]:
    """
    `metrics` dataframe with columns:

    1. `timestamp` - timestamp of evaluation
    1. `exp_id` - experiment id
    1. `metric_id` - metric id as in [`Experiment`][epstats.toolkit.experiment.Experiment] definition
    1. `metric_name` - metric name as in [`Experiment`][epstats.toolkit.experiment.Experiment] definition
    1. `exp_variant_id` - variant id
    1. `count` - number of exposures, value of metric denominator
    1. `mean` - `sum_value` / `count`
    1. `std` - sample standard deviation
    1. `sum_value` - value of goals, value of metric nominator
    1. `confidence_level` - current confidence level used to calculate `p_value` and `confidence_interval`
    1. `diff` - relative diff between sample means of this and control variant
    1. `test_stat` - value of test statistic of the relative difference in means
    1. `p_value` - p-value of the test statistic under current `confidence_level`
    1. `confidence_interval` - confidence interval of the `diff` under current `confidence_level`
    1. `standard_error` - standard error of the `diff`
    1. `degrees_of_freedom` - degrees of freedom of this variant mean
    1. `sample_size` - current sample size
    1. `required_sample_size` - size of the sample required to reach the required power
    1. `power` - power based on the collected `sample_size`
    1. `false_positive_risk` - false positive risk of a significant metric
    """
    return [
        "timestamp",
        "exp_id",
        "metric_id",
        "metric_name",
        "exp_variant_id",
        "count",
        "mean",
        "std",
        "sum_value",
        "confidence_level",
        "diff",
        "test_stat",
        "p_value",
        "confidence_interval",
        "standard_error",
        "degrees_of_freedom",
        "minimum_effect",
        "sample_size",
        "required_sample_size",
        "power",
        "false_positive_risk",
    ]