Skip to content

Other Pandas Utils

add_index_as_column

Add the index of a frame as a column.

add_index_as_column

add_index_as_column(df: DataFrame) -> DataFrame

Adds the DataFrame's index to a new column while preserving the original index.

This function handles single-level, unnamed, and multi-level indices:

  • For a single-level index, a new column with the index's name is created. If the index is unnamed, the new column will be named 'index'.
  • For a MultiIndex, a new column is created for each index level, and an additional column named 'combined_index' holds the full index tuple.

Parameters:

Name Type Description Default
df DataFrame

The pandas DataFrame to process.

required

Returns:

Type Description
DataFrame

A new DataFrame with the index added as a column. The original DataFrame is not modified.

Examples:

A named index:

>>> df = pd.DataFrame({'A': [10, 20]}, index=pd.Index(['X', 'Y'], name='label'))
>>> add_index_as_column(df).columns.tolist()
['A', 'label']

A MultiIndex:

>>> df_multi = pd.DataFrame(
...     {'value': [100, 200]}, index=pd.MultiIndex.from_tuples([('A', 1), ('B', 2)], names=['cat', 'id']),
... )
>>> add_index_as_column(df_multi).columns.tolist()
['value', 'cat', 'id', 'combined_index']
Source code in src/mesqual/utils/pandas_utils/add_index_as_column.py
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
def add_index_as_column(df: pd.DataFrame) -> pd.DataFrame:
    """Adds the DataFrame's index to a new column while preserving the original index.

    This function handles single-level, unnamed, and multi-level indices:

    - For a single-level index, a new column with the index's name is created.
      If the index is unnamed, the new column will be named 'index'.
    - For a MultiIndex, a new column is created for each index level, and
      an additional column named 'combined_index' holds the full index tuple.

    Args:
        df: The pandas DataFrame to process.

    Returns:
        A new DataFrame with the index added as a column. The original DataFrame
            is not modified.

    Examples:
        A named index:

        >>> df = pd.DataFrame({'A': [10, 20]}, index=pd.Index(['X', 'Y'], name='label'))
        >>> add_index_as_column(df).columns.tolist()
        ['A', 'label']

        A MultiIndex:

        >>> df_multi = pd.DataFrame(
        ...     {'value': [100, 200]}, index=pd.MultiIndex.from_tuples([('A', 1), ('B', 2)], names=['cat', 'id']),
        ... )
        >>> add_index_as_column(df_multi).columns.tolist()
        ['value', 'cat', 'id', 'combined_index']
    """
    # Create a copy to avoid modifying the original DataFrame
    df_copy = df.copy()

    # Check if the index is a MultiIndex
    if isinstance(df_copy.index, pd.MultiIndex):
        # Get the names of the index levels
        level_names = df_copy.index.names

        # Add columns for each level of the MultiIndex
        for i, name in enumerate(level_names):
            if name not in df_copy.columns:
                df_copy[name] = df_copy.index.get_level_values(i)

        # Add a column for the combined index tuple
        df_copy['combined_index'] = df_copy.index.to_list()

    else:
        # Get the index name, defaulting to 'index' if unnamed
        index_name = df_copy.index.name
        if index_name is None:
            index_name = 'index'

        # Add the index as a new column
        if index_name not in df_copy.columns:
            df_copy[index_name] = df_copy.index

    return df_copy

move_level

Move the top level of an axis to the last position.

move_top_level_last

move_top_level_last(data: Series | DataFrame, axis: int = 0) -> Series | DataFrame

Return data with the top level of an axis moved to the last position.

The other levels keep their order. An axis with one level is unchanged.

Parameters:

Name Type Description Default
data Series | DataFrame

A Series or DataFrame.

required
axis int

0 for the rows, 1 for the columns.

0

Examples:

>>> prices = study.scen.fetch('Node.Results.price')
>>> prices.columns.names
FrozenList(['dataset', 'Node'])
>>> move_top_level_last(prices, axis=1).columns.names
FrozenList(['Node', 'dataset'])
Source code in src/mesqual/utils/pandas_utils/move_level.py
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
def move_top_level_last(data: pd.Series | pd.DataFrame, axis: int = 0) -> pd.Series | pd.DataFrame:
    """Return `data` with the top level of an axis moved to the last position.

    The other levels keep their order. An axis with one level is unchanged.

    Args:
        data: A Series or DataFrame.
        axis: `0` for the rows, `1` for the columns.

    Examples:
        >>> prices = study.scen.fetch('Node.Results.price')
        >>> prices.columns.names
        FrozenList(['dataset', 'Node'])
        >>> move_top_level_last(prices, axis=1).columns.names
        FrozenList(['Node', 'dataset'])
    """
    ax = data.axes[axis]
    return data.set_axis(ax.reorder_levels(list(range(1, ax.nlevels)) + [0]), axis=axis)

dtype_processing

Conversion of True/False columns to bool dtype.

identify_and_convert_boolean_columns

identify_and_convert_boolean_columns(df: DataFrame) -> DataFrame

Identifies columns that contain only True / False / empty values and converts them to boolean dtype.

Source code in src/mesqual/utils/pandas_utils/dtype_processing.py
 6
 7
 8
 9
10
11
12
13
14
15
def identify_and_convert_boolean_columns(df: pd.DataFrame) -> pd.DataFrame:
    """
    Identifies columns that contain only True / False / empty values and converts them to boolean dtype.
    """
    for column in df:
        unique_values = df[column].dropna().unique()
        if len(set(unique_values).difference({True, False})) == 0:
            df.loc[df[column].isna(), column] = 0
            df = df.astype({column: bool})
    return df

is_numeric

Check whether a frame or series has numeric dtypes only.

pd_is_numeric

pd_is_numeric(df: DataFrame | Series) -> bool

True if every column has a numeric dtype that is not boolean.

The check reads the dtypes, not the values. So a column of numbers with dtype object is not numeric.

Source code in src/mesqual/utils/pandas_utils/is_numeric.py
 6
 7
 8
 9
10
11
12
13
def pd_is_numeric(df: pd.DataFrame | pd.Series) -> bool:
    """`True` if every column has a numeric dtype that is not boolean.

    The check reads the dtypes, not the values. So a column of numbers with
    dtype `object` is not numeric.
    """
    dtypes = [df.dtype] if isinstance(df, pd.Series) else df.dtypes
    return all(pd.api.types.is_numeric_dtype(d) and not pd.api.types.is_bool_dtype(d) for d in dtypes)