Skip to content

Pandas Util pend_props

pend_props

Prepend model properties as index levels to data.

is_dataframe

is_dataframe(obj: Series | DataFrame) -> TypeGuard[DataFrame]

Check if object is a pandas DataFrame.

Parameters:

Name Type Description Default
obj Series | DataFrame

Object to check, expected to be either Series or DataFrame.

required

Returns:

Type Description
TypeGuard[DataFrame]

True if obj is a DataFrame, False otherwise.

Source code in src/mesqual/utils/pandas_utils/pend_props.py
 8
 9
10
11
12
13
14
15
16
17
def is_dataframe(obj: pd.Series | pd.DataFrame) -> TypeGuard[pd.DataFrame]:
    """Check if object is a pandas DataFrame.

    Args:
        obj: Object to check, expected to be either Series or DataFrame.

    Returns:
        True if obj is a DataFrame, False otherwise.
    """
    return isinstance(obj, pd.DataFrame)

is_series

is_series(obj: Series | DataFrame) -> TypeGuard[Series]

Check if object is a pandas Series.

Parameters:

Name Type Description Default
obj Series | DataFrame

Object to check, expected to be either Series or DataFrame.

required

Returns:

Type Description
TypeGuard[Series]

True if obj is a Series, False otherwise.

Source code in src/mesqual/utils/pandas_utils/pend_props.py
20
21
22
23
24
25
26
27
28
29
def is_series(obj: pd.Series | pd.DataFrame) -> TypeGuard[pd.Series]:
    """Check if object is a pandas Series.

    Args:
        obj: Object to check, expected to be either Series or DataFrame.

    Returns:
        True if obj is a Series, False otherwise.
    """
    return isinstance(obj, pd.Series)

get_matching_axis_and_level

get_matching_axis_and_level(data: Series | DataFrame, match_index_level: Index, match_on_level: int | str = None) -> tuple[int, int]

Find the axis and level in data that matches the given index level.

Searches through all axes and levels of the data to find where the index values match the provided match_index_level. Returns the first match found.

Parameters:

Name Type Description Default
data Series | DataFrame

The pandas object to search within.

required
match_index_level Index

Index to match against data's axes.

required
match_on_level int | str

Optional constraint to match on specific level (by position or name).

None

Returns:

Type Description
tuple[int, int]

Tuple of (axis, level) indicating where the match was found.

Raises:

Type Description
ValueError

If match_index_level is a MultiIndex, or if no match is found.

Warns:

Type Description
UserWarning

If multiple matches found and no explicit match_on_level provided.

Source code in src/mesqual/utils/pandas_utils/pend_props.py
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
def get_matching_axis_and_level(
    data: pd.Series | pd.DataFrame, match_index_level: pd.Index, match_on_level: int | str = None
) -> tuple[int, int]:
    """Find the axis and level in data that matches the given index level.

    Searches through all axes and levels of the data to find where the index values
    match the provided match_index_level. Returns the first match found.

    Args:
        data: The pandas object to search within.
        match_index_level: Index to match against data's axes.
        match_on_level: Optional constraint to match on specific level (by position or name).

    Returns:
        Tuple of (axis, level) indicating where the match was found.

    Raises:
        ValueError: If match_index_level is a MultiIndex, or if no match is found.

    Warnings:
        UserWarning: If multiple matches found and no explicit match_on_level provided.
    """
    if isinstance(match_index_level, pd.MultiIndex):
        raise ValueError('Method only works for single index level.')

    missing_values = dict()
    matches = []
    for axis in range(len(data.axes)):
        for level in range(data.axes[axis].nlevels):
            missing = set(data.axes[axis].get_level_values(level)).difference(match_index_level)
            if len(missing) == 0:
                if match_on_level is None:
                    matches.append((axis, level))
                elif isinstance(match_on_level, int) and (level == match_on_level):
                    matches.append((axis, level))
                elif isinstance(match_on_level, str) and (data.axes[axis].names[level] == match_on_level):
                    matches.append((axis, level))
            missing_values[(axis, level)] = len(missing)
    if len(matches) == 1:
        return matches[0]
    if len(matches) > 1:
        warnings.warn(
            f'Multiple levels match your index, but no explicit match_on_level was provided. '
            f'Auto fall-back first match; axis, level: {matches[0]}'
        )
        return matches[0]

    raise ValueError(f'No Index Match Found. Missing values: {missing_values}')

prepend_model_prop_levels

prepend_model_prop_levels(data: Series | DataFrame, model: DataFrame, *properties: Hashable, prepend_to_top: bool = True, match_on_level: str = None) -> Series | DataFrame

Prepend model properties as new index levels to data.

Searches for an index level in data that matches the model's index, then prepends specified properties from the model as new index levels.

Parameters:

Name Type Description Default
data Series | DataFrame

The pandas object to add properties to.

required
model DataFrame

DataFrame containing properties to prepend, with matching index.

required
*properties Hashable

Column names from model to use as new index levels.

()
prepend_to_top bool

If True, add properties at the beginning of index levels. If False, add at the end.

True
match_on_level str

Optional level name to constrain matching to specific level. Useful in case the there are multiple index levels in data that match the model's index

None

Returns:

Type Description
Series | DataFrame

Copy of data with properties prepended as new index levels.

Raises:

Type Description
ValueError

If any property is not found in model columns.

Use it before a groupby to aggregate by a model property, for example the generation per node or per fuel.

Examples:

>>> generation = dataset.fetch('Generator.Results.generation')
>>> generators = dataset.fetch('Generator.Model')
>>> with_props = prepend_model_prop_levels(generation, generators, 'node', 'fuel')
>>> with_props.columns.tolist()
[('nuclear', 'N1', 'G1'), ('coal', 'N1', 'G2'), ('gas', 'N2', 'G3'), ('wind', 'N2', 'G4'), ('solar', 'N3', 'G5')]
>>> with_props.T.groupby(level='node').sum().T.iloc[:2].round(1)
node                           N1     N2   N3
time
2024-03-28 00:00:00+01:00  1218.4  891.8  0.0
2024-03-28 01:00:00+01:00  1247.7  825.7  0.0
Source code in src/mesqual/utils/pandas_utils/pend_props.py
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
def prepend_model_prop_levels(
    data: pd.Series | pd.DataFrame,
    model: pd.DataFrame,
    *properties: Hashable,
    prepend_to_top: bool = True,
    match_on_level: str = None,
) -> pd.Series | pd.DataFrame:
    """Prepend model properties as new index levels to data.

    Searches for an index level in data that matches the model's index, then
    prepends specified properties from the model as new index levels.

    Args:
        data: The pandas object to add properties to.
        model: DataFrame containing properties to prepend, with matching index.
        *properties: Column names from model to use as new index levels.
        prepend_to_top: If True, add properties at the beginning of index levels.
            If False, add at the end.
        match_on_level: Optional level name to constrain matching to specific level.
            Useful in case the there are multiple index levels in data that match
            the model's index

    Returns:
        Copy of data with properties prepended as new index levels.

    Raises:
        ValueError: If any property is not found in model columns.

    Use it before a groupby to aggregate by a model property, for example
    the generation per node or per fuel.

    Examples:
        >>> generation = dataset.fetch('Generator.Results.generation')
        >>> generators = dataset.fetch('Generator.Model')
        >>> with_props = prepend_model_prop_levels(generation, generators, 'node', 'fuel')
        >>> with_props.columns.tolist()
        [('nuclear', 'N1', 'G1'), ('coal', 'N1', 'G2'), ('gas', 'N2', 'G3'), ('wind', 'N2', 'G4'), ('solar', 'N3', 'G5')]
        >>> with_props.T.groupby(level='node').sum().T.iloc[:2].round(1)
        node                           N1     N2   N3
        time
        2024-03-28 00:00:00+01:00  1218.4  891.8  0.0
        2024-03-28 01:00:00+01:00  1247.7  825.7  0.0
    """
    tmp = data.copy()
    properties = [p for p in properties if not ((p is None) or (p == ''))]

    if not properties:
        return tmp

    for prop in properties:
        if prop not in model.columns.tolist():
            raise ValueError(f'Property unavailable: {prop} was not found in your model_df.')
    axis, level = get_matching_axis_and_level(data, model.index, match_on_level)

    match_keys = tmp.axes[axis].get_level_values(level)
    new_index = tmp.axes[axis].to_frame(index=False)
    for prop in properties:
        if prop not in new_index:
            loc = 0 if prepend_to_top else len(new_index.columns)
            new_index.insert(loc, prop, model.loc[match_keys, prop].values)
    new_index = pd.MultiIndex.from_frame(new_index)
    if axis == 0:
        tmp.index = new_index
    else:
        tmp.columns = new_index

    if is_series(data):
        tmp: pd.Series = tmp
        return tmp
    elif is_dataframe(data):
        tmp: pd.DataFrame = tmp
        return tmp
    else:
        return tmp