Skip to content

Metric

Definition of a metric to evaluate in an experiment.

Source code in src/epstats/toolkit/metric.py 
  9
 10
 11
 12
 13
 14
 15
 16
 17
 18
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
class Metric:
    """
    Definition of a metric to evaluate in an experiment.
    """

    def __init__(
        self,
        id: int,
        name: str,
        nominator: str,
        denominator: str,
        metric_format: str = "{:.2%}",
        metric_value_multiplier: int = 1,
        minimum_effect: Optional[float] = None,
    ):
        """
        Constructor of the general metric definition.

        Parameters `nominator` and `denominator` specify exactly type of data and aggregation of the metric nominator
        and denominator.

        Parameters `format` and `multiplier` does not play any role in metric evaluation. They are used independently
        after the metric evaluation.

        Arguments:
            id: metric (order) id
            name: metric name
            nominator: definition of nominator
            denominator: definition of denominator
            metric_format: specify format of the metric, e.g. '${:,.1f}' for RPM
            metric_value_multiplier: specify multiplier, e.g. 1000 for RPM

        Usage:

        ```python
        Metric(
            1,
            'Click-through Rate',
            'count(test_unit_type.unit.click)',
            'count(test_unit_type.global.exposure)')
        ```
        """

        self.id = id
        self.name = name
        self.nominator = nominator
        self.denominator = denominator
        self._parser = Parser(nominator, denominator)
        self._goals = self._parser.get_goals()
        self.metric_format = metric_format
        self.metric_value_multiplier = metric_value_multiplier
        self.minimum_effect = minimum_effect

    def get_goals(self) -> Set:
        """
        Get all goals needed to evaluate the metric.
        """
        return self._goals

    def get_evaluate_columns_agg(self, goals: pd.DataFrame) -> np.array:
        """
        Get `count`, `sum_value`, `sum_value_sqr` numpy array of shape (variants, metrics) after
        evaluating nominator and denominator expressions from pre-aggregated goals.

        Arguments:
            goals: one row per experiment variant

        See [`Experiment.evaluate_agg`][epstats.toolkit.experiment.Experiment.evaluate_agg] for details
        on `goals` at input.

        Returns:
            numpy array of shape (variants, metrics) where metrics are in order of
            (count, sum_value, sum_sqr_value)
        """
        return self._parser.evaluate_agg(goals)

    def get_evaluate_columns_by_unit(self, goals: pd.DataFrame) -> np.array:
        """
        Get `count`, `sum_value`, `sum_value_sqr` numpy array of shape (variants, metrics) after
        evaluating nominator and denominator expressions from goals aggregated by unit.

        Arguments:
            goals: one row per experiment variant

        See [`Experiment.evaluate_agg`][epstats.toolkit.experiment.Experiment.evaluate_by_unit] for details
        on `goals` at input.

        Returns:
            numpy array of shape (variants, metrics) where metrics are in order of
            (count, sum_value, sum_sqr_value)
        """
        return self._parser.evaluate_by_unit(goals)

__init__(id, name, nominator, denominator, metric_format='{:.2%}', metric_value_multiplier=1, minimum_effect=None)

Constructor of the general metric definition.

Parameters nominator and denominator specify exactly type of data and aggregation of the metric nominator and denominator.

Parameters format and multiplier does not play any role in metric evaluation. They are used independently after the metric evaluation.

Parameters:

Name Type Description Default
id int

metric (order) id

 required
name str

metric name

 required
nominator str

definition of nominator

 required
denominator str

definition of denominator

 required
metric_format str

specify format of the metric, e.g. '${:,.1f}' for RPM

 '{:.2%}'
metric_value_multiplier int

specify multiplier, e.g. 1000 for RPM

 1

Usage:

Metric(
    1,
    'Click-through Rate',
    'count(test_unit_type.unit.click)',
    'count(test_unit_type.global.exposure)')
Source code in src/epstats/toolkit/metric.py 
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
def __init__(
    self,
    id: int,
    name: str,
    nominator: str,
    denominator: str,
    metric_format: str = "{:.2%}",
    metric_value_multiplier: int = 1,
    minimum_effect: Optional[float] = None,
):
    """
    Constructor of the general metric definition.

    Parameters `nominator` and `denominator` specify exactly type of data and aggregation of the metric nominator
    and denominator.

    Parameters `format` and `multiplier` does not play any role in metric evaluation. They are used independently
    after the metric evaluation.

    Arguments:
        id: metric (order) id
        name: metric name
        nominator: definition of nominator
        denominator: definition of denominator
        metric_format: specify format of the metric, e.g. '${:,.1f}' for RPM
        metric_value_multiplier: specify multiplier, e.g. 1000 for RPM

    Usage:

    ```python
    Metric(
        1,
        'Click-through Rate',
        'count(test_unit_type.unit.click)',
        'count(test_unit_type.global.exposure)')
    ```
    """

    self.id = id
    self.name = name
    self.nominator = nominator
    self.denominator = denominator
    self._parser = Parser(nominator, denominator)
    self._goals = self._parser.get_goals()
    self.metric_format = metric_format
    self.metric_value_multiplier = metric_value_multiplier
    self.minimum_effect = minimum_effect

get_evaluate_columns_agg(goals)

Get count, sum_value, sum_value_sqr numpy array of shape (variants, metrics) after evaluating nominator and denominator expressions from pre-aggregated goals.

Parameters:

Name Type Description Default
goals DataFrame

one row per experiment variant

 required

See Experiment.evaluate_agg for details on goals at input.

Returns:

Type Description
array

numpy array of shape (variants, metrics) where metrics are in order of
array

(count, sum_value, sum_sqr_value)
Source code in src/epstats/toolkit/metric.py 
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
def get_evaluate_columns_agg(self, goals: pd.DataFrame) -> np.array:
    """
    Get `count`, `sum_value`, `sum_value_sqr` numpy array of shape (variants, metrics) after
    evaluating nominator and denominator expressions from pre-aggregated goals.

    Arguments:
        goals: one row per experiment variant

    See [`Experiment.evaluate_agg`][epstats.toolkit.experiment.Experiment.evaluate_agg] for details
    on `goals` at input.

    Returns:
        numpy array of shape (variants, metrics) where metrics are in order of
        (count, sum_value, sum_sqr_value)
    """
    return self._parser.evaluate_agg(goals)

get_evaluate_columns_by_unit(goals)

Get count, sum_value, sum_value_sqr numpy array of shape (variants, metrics) after evaluating nominator and denominator expressions from goals aggregated by unit.

Parameters:

Name Type Description Default
goals DataFrame

one row per experiment variant

 required

See Experiment.evaluate_agg for details on goals at input.

Returns:

Type Description
array

numpy array of shape (variants, metrics) where metrics are in order of
array

(count, sum_value, sum_sqr_value)
Source code in src/epstats/toolkit/metric.py 
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
def get_evaluate_columns_by_unit(self, goals: pd.DataFrame) -> np.array:
    """
    Get `count`, `sum_value`, `sum_value_sqr` numpy array of shape (variants, metrics) after
    evaluating nominator and denominator expressions from goals aggregated by unit.

    Arguments:
        goals: one row per experiment variant

    See [`Experiment.evaluate_agg`][epstats.toolkit.experiment.Experiment.evaluate_by_unit] for details
    on `goals` at input.

    Returns:
        numpy array of shape (variants, metrics) where metrics are in order of
        (count, sum_value, sum_sqr_value)
    """
    return self._parser.evaluate_by_unit(goals)

get_goals()

Get all goals needed to evaluate the metric.

Source code in src/epstats/toolkit/metric.py 
62
63
64
65
66
def get_goals(self) -> Set:
    """
    Get all goals needed to evaluate the metric.
    """
    return self._goals

Simple Metric

Bases: Metric

Simplified metric definition to evaluate in an experiment.

Source code in src/epstats/toolkit/metric.py 
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
class SimpleMetric(Metric):
    """
    Simplified metric definition to evaluate in an experiment.
    """

    def __init__(
        self,
        id: int,
        name: str,
        numerator: str,
        denominator: str,
        unit_type: str = "test_unit_type",
        metric_format: str = "{:.2%}",
        metric_value_multiplier: int = 1,
        minimum_effect: Optional[float] = None,
    ):
        """
        Constructor of the simplified metric definition.

        It modifies parameters numerator and denominator in a way that it is in line with general Metric definition.
        It adds all the niceties necessary for proper Metric format. Finally it calls constructor of the parent Metric
        class.

        Arguments:
            id: metric (order) id
            name: metric name
            numerator: value (column) of the numerator
            denominator: value (column) of the denominator
            unit_type: unit type
            metric_format: specify format of the metric, e.g. '${:,.1f}' for RPM
            metric_value_multiplier: specify multiplier, e.g. 1000 for RPM

        Usage:

        ```python
        SimpleMetric(
            1,
            'Click-through Rate',
            'clicks',
            'views',
            unit_type='test_unit_type',
            metric_format='{:.2%}',
            metric_value_multiplier=1)
        ```
        """
        agg_type = "global"  # technical parameter; it has no impact
        num = "value" + "(" + unit_type + "." + agg_type + "." + numerator + ")"
        den = "value" + "(" + unit_type + "." + agg_type + "." + denominator + ")"

        super().__init__(id, name, num, den, metric_format, metric_value_multiplier, minimum_effect)

__init__(id, name, numerator, denominator, unit_type='test_unit_type', metric_format='{:.2%}', metric_value_multiplier=1, minimum_effect=None)

Constructor of the simplified metric definition.

It modifies parameters numerator and denominator in a way that it is in line with general Metric definition. It adds all the niceties necessary for proper Metric format. Finally it calls constructor of the parent Metric class.

Parameters:

Name Type Description Default
id int

metric (order) id

 required
name str

metric name

 required
numerator str

value (column) of the numerator

 required
denominator str

value (column) of the denominator

 required
unit_type str

unit type

 'test_unit_type'
metric_format str

specify format of the metric, e.g. '${:,.1f}' for RPM

 '{:.2%}'
metric_value_multiplier int

specify multiplier, e.g. 1000 for RPM

 1

Usage:

SimpleMetric(
    1,
    'Click-through Rate',
    'clicks',
    'views',
    unit_type='test_unit_type',
    metric_format='{:.2%}',
    metric_value_multiplier=1)
Source code in src/epstats/toolkit/metric.py 
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
def __init__(
    self,
    id: int,
    name: str,
    numerator: str,
    denominator: str,
    unit_type: str = "test_unit_type",
    metric_format: str = "{:.2%}",
    metric_value_multiplier: int = 1,
    minimum_effect: Optional[float] = None,
):
    """
    Constructor of the simplified metric definition.

    It modifies parameters numerator and denominator in a way that it is in line with general Metric definition.
    It adds all the niceties necessary for proper Metric format. Finally it calls constructor of the parent Metric
    class.

    Arguments:
        id: metric (order) id
        name: metric name
        numerator: value (column) of the numerator
        denominator: value (column) of the denominator
        unit_type: unit type
        metric_format: specify format of the metric, e.g. '${:,.1f}' for RPM
        metric_value_multiplier: specify multiplier, e.g. 1000 for RPM

    Usage:

    ```python
    SimpleMetric(
        1,
        'Click-through Rate',
        'clicks',
        'views',
        unit_type='test_unit_type',
        metric_format='{:.2%}',
        metric_value_multiplier=1)
    ```
    """
    agg_type = "global"  # technical parameter; it has no impact
    num = "value" + "(" + unit_type + "." + agg_type + "." + numerator + ")"
    den = "value" + "(" + unit_type + "." + agg_type + "." + denominator + ")"

    super().__init__(id, name, num, den, metric_format, metric_value_multiplier, minimum_effect)