Skip to content

Index

frequenz.data.microgrid ¤

Initialize the microgrid data module.

Classes¤

frequenz.data.microgrid.MicrogridConfig ¤

Configuration of a microgrid.

Source code in frequenz/gridpool/config/_microgrid.py
@dataclass
class MicrogridConfig:
    """Configuration of a microgrid."""

    microgrid_id: int
    """ID of the microgrid."""

    name: str | None = None
    """Name of the microgrid."""

    enterprise_id: int | None = None
    """Enterprise ID of the microgrid."""

    gid: int | None = None
    """Legacy ID; if set, all gridpool relations must name it."""

    latitude: float | None = None
    """Geographic latitude of the microgrid."""

    longitude: float | None = None
    """Geographic longitude of the microgrid."""

    altitude: float | None = None
    """Geographic altitude of the microgrid."""

    start_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
    """Start time of the microgrid operation."""

    end_time: datetime | None = field(default=None, metadata=toml_datetime_metadata())
    """End time of the microgrid operation."""

    pv: dict[str, PVConfig] | None = None
    """Configuration of the PV system."""

    wind: dict[str, WindConfig] | None = None
    """Configuration of the wind turbines."""

    battery: dict[str, BatteryConfig] | None = None
    """Configuration of the batteries."""

    ctype: dict[str, ComponentTypeConfig] = field(default_factory=dict)
    """Mapping of component category types to ac power component config."""

    def component_types(self) -> list[str]:
        """Get a list of all component types in the configuration."""
        return list(self.ctype.keys())

    def component_type_ids(
        self,
        component_type: str,
        component_category: str | None = None,
        metric: str = "",
    ) -> list[int]:
        """Get a list of all component IDs for a component type.

        Args:
            component_type: Component type to be aggregated.
            component_category: Specific category of component IDs to retrieve
                (e.g., "meter", "inverter", or "component"). If not provided,
                the default logic is used.
            metric: Metric name of the formula if CIDs should be extracted from the formula.

        Returns:
            List of component IDs for this component type.

        Raises:
            ValueError: If the component type is unknown.
            KeyError: If `component_category` is invalid.
        """
        cfg = self.ctype.get(component_type)
        if not cfg:
            raise ValueError(f"{component_type} not found in config.")

        if component_category:
            valid_categories = get_args(ComponentCategory)
            if component_category not in valid_categories:
                raise KeyError(
                    f"Invalid component category: {component_category}. "
                    f"Valid categories are {valid_categories}"
                )
            category_ids = cast(list[int], getattr(cfg, component_category, []))
            return category_ids

        return cfg.cids(metric)

    def formula(self, component_type: str, metric: str) -> str:
        """Get the formula for a component type.

        Args:
            component_type: Component type to be aggregated.
            metric: Metric to be aggregated.

        Returns:
            Formula to be used for this aggregated component as string.

        Raises:
            ValueError: If the component type is unknown or formula is missing.
        """
        cfg = self.ctype.get(component_type)
        if not cfg:
            raise ValueError(f"{component_type} not found in config.")
        if cfg.formula is None:
            raise ValueError(f"No formula set for {component_type}")
        formula = cfg.formula.get(metric)
        if not formula:
            raise ValueError(f"{component_type} is missing formula for {metric}")

        return formula

    Schema: ClassVar[Type[Schema]] = Schema

    @classmethod
    def _load_table_entries(cls, data: dict[str, Any]) -> dict[int, Self]:
        """Load microgrid configurations from table entries.

        Args:
            data: The table mapping microgrid IDs to their entries.

        Returns:
            A dict mapping microgrid IDs to MicrogridConfig instances.

        Raises:
            ValueError: If the keys are not numeric microgrid IDs
                or if there is a microgrid ID mismatch.
            TypeError: If microgrid data is not a dict.
        """
        if not all(str(k).isdigit() for k in data.keys()):
            raise ValueError("All microgrid keys must be numeric microgrid IDs.")

        mgrids = {}
        for mid, entry in data.items():
            if not mid.isdigit():
                raise ValueError(
                    f"Table reader: Microgrid ID key must be numeric, got {mid}"
                )
            if not isinstance(entry, dict):
                raise TypeError("Table reader: Each microgrid entry must be a dict")

            mgrid = cls.Schema().load(entry)
            if int(mgrid.microgrid_id) != int(mid):
                raise ValueError(
                    f"Table reader: Microgrid ID mismatch: key {mid} != {mgrid.microgrid_id}"
                )

            mgrids[int(mid)] = mgrid

        return mgrids
Attributes¤
altitude class-attribute instance-attribute ¤
altitude: float | None = None

Geographic altitude of the microgrid.

battery class-attribute instance-attribute ¤
battery: dict[str, BatteryConfig] | None = None

Configuration of the batteries.

ctype class-attribute instance-attribute ¤
ctype: dict[str, ComponentTypeConfig] = field(
    default_factory=dict
)

Mapping of component category types to ac power component config.

end_time class-attribute instance-attribute ¤
end_time: datetime | None = field(
    default=None, metadata=toml_datetime_metadata()
)

End time of the microgrid operation.

enterprise_id class-attribute instance-attribute ¤
enterprise_id: int | None = None

Enterprise ID of the microgrid.

gid class-attribute instance-attribute ¤
gid: int | None = None

Legacy ID; if set, all gridpool relations must name it.

latitude class-attribute instance-attribute ¤
latitude: float | None = None

Geographic latitude of the microgrid.

longitude class-attribute instance-attribute ¤
longitude: float | None = None

Geographic longitude of the microgrid.

microgrid_id instance-attribute ¤
microgrid_id: int

ID of the microgrid.

name class-attribute instance-attribute ¤
name: str | None = None

Name of the microgrid.

pv class-attribute instance-attribute ¤
pv: dict[str, PVConfig] | None = None

Configuration of the PV system.

start_time class-attribute instance-attribute ¤
start_time: datetime | None = field(
    default=None, metadata=toml_datetime_metadata()
)

Start time of the microgrid operation.

wind class-attribute instance-attribute ¤
wind: dict[str, WindConfig] | None = None

Configuration of the wind turbines.

Methods:¤
component_type_ids ¤
component_type_ids(
    component_type: str,
    component_category: str | None = None,
    metric: str = "",
) -> list[int]

Get a list of all component IDs for a component type.

PARAMETER DESCRIPTION
component_type

Component type to be aggregated.

TYPE: str

component_category

Specific category of component IDs to retrieve (e.g., "meter", "inverter", or "component"). If not provided, the default logic is used.

TYPE: str | None DEFAULT: None

metric

Metric name of the formula if CIDs should be extracted from the formula.

TYPE: str DEFAULT: ''

RETURNS DESCRIPTION
list[int]

List of component IDs for this component type.

RAISES DESCRIPTION
ValueError

If the component type is unknown.

KeyError

If component_category is invalid.

Source code in frequenz/gridpool/config/_microgrid.py
def component_type_ids(
    self,
    component_type: str,
    component_category: str | None = None,
    metric: str = "",
) -> list[int]:
    """Get a list of all component IDs for a component type.

    Args:
        component_type: Component type to be aggregated.
        component_category: Specific category of component IDs to retrieve
            (e.g., "meter", "inverter", or "component"). If not provided,
            the default logic is used.
        metric: Metric name of the formula if CIDs should be extracted from the formula.

    Returns:
        List of component IDs for this component type.

    Raises:
        ValueError: If the component type is unknown.
        KeyError: If `component_category` is invalid.
    """
    cfg = self.ctype.get(component_type)
    if not cfg:
        raise ValueError(f"{component_type} not found in config.")

    if component_category:
        valid_categories = get_args(ComponentCategory)
        if component_category not in valid_categories:
            raise KeyError(
                f"Invalid component category: {component_category}. "
                f"Valid categories are {valid_categories}"
            )
        category_ids = cast(list[int], getattr(cfg, component_category, []))
        return category_ids

    return cfg.cids(metric)
component_types ¤
component_types() -> list[str]

Get a list of all component types in the configuration.

Source code in frequenz/gridpool/config/_microgrid.py
def component_types(self) -> list[str]:
    """Get a list of all component types in the configuration."""
    return list(self.ctype.keys())
formula ¤
formula(component_type: str, metric: str) -> str

Get the formula for a component type.

PARAMETER DESCRIPTION
component_type

Component type to be aggregated.

TYPE: str

metric

Metric to be aggregated.

TYPE: str

RETURNS DESCRIPTION
str

Formula to be used for this aggregated component as string.

RAISES DESCRIPTION
ValueError

If the component type is unknown or formula is missing.

Source code in frequenz/gridpool/config/_microgrid.py
def formula(self, component_type: str, metric: str) -> str:
    """Get the formula for a component type.

    Args:
        component_type: Component type to be aggregated.
        metric: Metric to be aggregated.

    Returns:
        Formula to be used for this aggregated component as string.

    Raises:
        ValueError: If the component type is unknown or formula is missing.
    """
    cfg = self.ctype.get(component_type)
    if not cfg:
        raise ValueError(f"{component_type} not found in config.")
    if cfg.formula is None:
        raise ValueError(f"No formula set for {component_type}")
    formula = cfg.formula.get(metric)
    if not formula:
        raise ValueError(f"{component_type} is missing formula for {metric}")

    return formula

frequenz.data.microgrid.MicrogridData ¤

Fetch metric data for component types of a microgrid.

Source code in src/frequenz/data/microgrid/component_data.py
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
class MicrogridData:
    """Fetch metric data for component types of a microgrid."""

    _AC_ACTIVE_ENERGY_METRICS = {
        "energy_net": "AC_ENERGY_ACTIVE",
        "energy_consumed": "AC_ENERGY_ACTIVE_CONSUMED",
        "energy_delivered": "AC_ENERGY_ACTIVE_DELIVERED",
    }

    def __init__(
        self,
        server_url: str,
        auth_key: str,
        sign_secret: str,
        microgrid_configs: (
            _MicrogridConfigMapping | _MicrogridConfigsContainer | None
        ) = None,
        assets_client: AssetsApiClient | None = None,
    ) -> None:
        """Initialize microgrid data.

        Args:
            server_url: URL of the reporting service.
            auth_key: Authentication key to the service.
            sign_secret: Secret for signing requests.
            microgrid_configs: A mapping of microgrid IDs to configurations, or a
                gridpool configuration document containing that mapping.
            assets_client: Optional Assets API client used to fetch static battery
                SOC rated bounds.
        """
        self._microgrid_configs = (
            None
            if microgrid_configs is None
            else _normalize_microgrid_configs(microgrid_configs)
        )
        self._client = ReportingApiClient(
            server_url=server_url, auth_key=auth_key, sign_secret=sign_secret
        )
        self._assets_client = assets_client

    @property
    def microgrid_ids(self) -> list[int]:
        """Get the microgrid IDs.

        Returns:
            List of microgrid IDs.
        """
        if self._microgrid_configs is None:
            return []
        return list(self._microgrid_configs.keys())

    @property
    def microgrid_configs(self) -> dict[int, MicrogridConfig] | None:
        """Return the microgrid configurations."""
        return self._microgrid_configs

    @staticmethod
    def _convert_units(
        df: pd.DataFrame,
        *,
        unit: str,
        scale_by_unit: dict[str, float],
    ) -> pd.DataFrame:
        """Convert a dataframe with values expressed in a base unit."""
        if unit not in scale_by_unit:
            raise ValueError(f"Unknown unit: {unit}")
        return df / scale_by_unit[unit]

    @staticmethod
    def _add_split_pos_neg_cols(df: pd.DataFrame) -> pd.DataFrame:
        """Add positive and negative split columns for each column."""
        cols = df.columns
        pos_cols = [f"{col}_pos" for col in cols]
        neg_cols = [f"{col}_neg" for col in cols]
        df[pos_cols] = df[cols].clip(lower=0)
        df[neg_cols] = df[cols].clip(upper=0)
        return df

    # pylint: disable=too-many-locals,too-many-branches,too-many-statements
    async def metric_data(  # pylint: disable=too-many-arguments
        self,
        *,
        microgrid_id: int | str,
        start: datetime,
        end: datetime,
        component_types: tuple[str, ...] = (
            "grid",
            "pv",
            "battery",
            "wind",
            "chp",
            "consumption",
        ),
        resampling_period: timedelta = timedelta(seconds=10),
        metric: str = "AC_POWER_ACTIVE",
        keep_components: bool = False,
        splits: bool = False,
    ) -> pd.DataFrame | None:
        """Fetch aggregated data for an arbitrary metric across component types.

        Args:
            microgrid_id: Microgrid ID. Numeric strings are accepted for notebook
                compatibility.
            start: Start timestamp.
            end: End timestamp.
            component_types: Component types whose aggregation formulas should be
                queried for the requested metric.
            resampling_period: Sampling period used for the reporting query.
            metric: Reporting metric name to fetch.
            keep_components: Whether to include individual component IDs alongside
                the aggregated component-type columns.
            splits: Whether to append positive and negative split columns for each
                returned column.
        Returns:
            DataFrame with power data of aggregated components
            or None if no data is available

        Raises:
            ValueError: If microgrid configurations are not loaded.
        """
        if self._microgrid_configs is None:
            raise ValueError("Microgrid configurations are not loaded.")
        microgrid_id = int(microgrid_id)
        mcfg = self._microgrid_configs[microgrid_id]
        metric = metric.upper()

        formulas = {
            ctype: mcfg.formula(ctype, metric.upper()) for ctype in component_types
        }

        logging.debug("Formulas: %s", formulas)

        metric_enum = Metric[metric.upper()]

        async def _collect(formula: str) -> list:
            return [
                sample
                async for sample in self._client.receive_aggregated_data(
                    microgrid_id=microgrid_id,
                    metric=metric_enum,
                    aggregation_formula=formula,
                    start_time=start,
                    end_time=end,
                    resampling_period=resampling_period,
                )
            ]

        all_cids: list[int] = []
        if keep_components:
            # IDs are read from the formulas: derived types (e.g. consumption)
            # mention the IDs of other types and a formula can repeat an ID.
            # Request each ID once to avoid streaming identical samples.
            all_cids = list(
                dict.fromkeys(
                    cid
                    for ctype in component_types
                    for cid in mcfg.component_type_ids(ctype, metric=metric)
                )
            )
            _logger.debug("CIDs: %s", all_cids)

        async def _components() -> pd.DataFrame | None:
            if not all_cids:
                return None
            return await self._component_metric_data(
                microgrid_id=microgrid_id,
                component_ids=all_cids,
                metric=metric_enum,
                start=start,
                end=end,
                resampling_period=resampling_period,
            )

        # The formula streams and the per-component query are independent.
        *formula_results, component_df = await asyncio.gather(
            *(_collect(formula) for formula in formulas.values()), _components()
        )
        data = [sample for samples in formula_results for sample in samples]

        if len(data) == 0 and component_df is None:
            _logger.warning("No data found")
            return None

        df = None
        if data:
            df = pd.DataFrame(data)
            df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True)
            assert df["timestamp"].dt.tz is not None, "Timestamps are not tz-aware"

            # Remove duplicates
            dup_mask = df.duplicated(keep="first")
            if not dup_mask.empty:
                _logger.info("Found %s rows that have duplicates", dup_mask.sum())
            df = df[~dup_mask]

            # Pivot table
            df = df.pivot_table(
                index="timestamp", columns="component_id", values="value"
            )
        # Rename formula columns
        rename_cols: dict[str, str] = {}
        for ctype, formula in formulas.items():
            if formula in rename_cols:
                _logger.warning(
                    "Ignoring %s since formula %s exists already for %s",
                    ctype,
                    formula,
                    rename_cols[formula],
                )
                continue
            rename_cols[formula] = ctype

        if df is not None:
            df = df.rename(columns=rename_cols)
        if component_df is not None:
            df = component_df if df is None else df.join(component_df, how="outer")
        elif df is not None and keep_components:
            for component_id in all_cids:
                if component_id not in df.columns:
                    _logger.warning(
                        "Component ID %s not found in data, setting zero", component_id
                    )
                    df.loc[:, component_id] = np.nan

        assert df is not None

        # Make string columns
        df.columns = [str(e) for e in df.columns]

        if splits:
            df = self._add_split_pos_neg_cols(df)

        ctypes = list(rename_cols.values())
        new_cols = [e for e in ctypes if e in df.columns] + sorted(
            [e for e in df.columns if e not in ctypes]
        )
        return df[new_cols]

    async def ac_active_power(  # pylint: disable=too-many-arguments
        self,
        *,
        microgrid_id: int | str,
        start: datetime,
        end: datetime,
        component_types: tuple[str, ...] = (
            "grid",
            "pv",
            "wind",
            "battery",
            "chp",
            "consumption",
        ),
        resampling_period: timedelta = timedelta(seconds=10),
        keep_components: bool = False,
        splits: bool = False,
        unit: str = "kW",
        from_energy: bool = False,
    ) -> pd.DataFrame | None:
        """Power data for component types of a microgrid.

        Set `from_energy` to derive power from net AC active energy instead of
        fetching the instantaneous power metric.
        """
        if from_energy:
            df = await self.ac_active_energy(
                microgrid_id=microgrid_id,
                start=start,
                end=end,
                component_types=component_types,
                resampling_period=resampling_period,
                keep_components=keep_components,
                unit="Wh",
            )
            if df is not None:
                df = self._convert_cumulative_energy_to_power(
                    df, resampling_period=resampling_period
                )
                if splits:
                    df = self._add_split_pos_neg_cols(df)
        else:
            df = await self.metric_data(
                microgrid_id=microgrid_id,
                start=start,
                end=end,
                component_types=component_types,
                resampling_period=resampling_period,
                metric="AC_POWER_ACTIVE",
                keep_components=keep_components,
                splits=splits,
            )
        if df is None:
            return df

        return self._convert_units(
            df, unit=unit, scale_by_unit={"W": 1, "kW": 1000, "MW": 1e6}
        )

    async def soc(  # pylint: disable=too-many-arguments
        self,
        *,
        microgrid_id: int | str,
        start: datetime,
        end: datetime,
        resampling_period: timedelta = timedelta(seconds=10),
        keep_components: bool = False,
    ) -> pd.DataFrame | None:
        """Fetch and aggregate SOC for the configured battery components.

        Battery component IDs are read from the microgrid configuration.  Their
        SOC and capacity values are fetched directly per component, without
        requiring aggregation formulas in the microgrid configuration. The
        aggregate SOC is a capacity-weighted average, using each component's
        own capacity reading at each timestamp so that capacity changes are
        reflected over time. SOC bounds are aggregated using those same
        capacity weights.

        Args:
            microgrid_id: The ID of the microgrid.
            start: The start time for the data fetch.
            end: The end time for the data fetch.
            resampling_period: The period for resampling the data.
            keep_components: Whether to keep individual component data.

        Returns:
            A DataFrame containing the aggregated SOC data, or None if no data is available.

        Raises:
            ValueError: If microgrid configurations are not loaded.
        """
        if self._microgrid_configs is None:
            raise ValueError("Microgrid configurations are not loaded.")

        mg_id = int(microgrid_id)
        config = self._microgrid_configs[mg_id]
        component_ids = config.component_type_ids(
            "battery", component_category="component"
        )
        if not component_ids:
            _logger.warning(
                "No battery components are configured for microgrid %s.", mg_id
            )
            return None

        # SOC, capacity and static bounds are independent requests.
        df, capacity_df, bounds = await asyncio.gather(
            self._component_metric_data(
                microgrid_id=mg_id,
                component_ids=component_ids,
                metric=Metric.BATTERY_SOC_PCT,
                start=start,
                end=end,
                resampling_period=resampling_period,
            ),
            self._component_metric_data(
                microgrid_id=mg_id,
                component_ids=component_ids,
                metric=Metric.BATTERY_CAPACITY,
                start=start,
                end=end,
                resampling_period=resampling_period,
            ),
            self._battery_soc_asset_bounds(mg_id, component_ids),
        )
        if df is None:
            _logger.warning("No battery SOC data found for microgrid %s.", mg_id)
            return None

        if capacity_df is None:
            _logger.warning("No battery capacity data found for microgrid %s.", mg_id)
        component_columns = [str(component_id) for component_id in component_ids]
        weighted_soc = None
        if capacity_df is not None:
            capacity_aligned = self._align_forward_filled(
                capacity_df.reindex(columns=component_columns), df.index
            )
            weighted_soc = self._capacity_weighted_soc(
                df.reindex(columns=component_columns), capacity_aligned
            )
        if weighted_soc is None:
            return df if keep_components else None
        df["battery"] = weighted_soc

        if not keep_components:
            component_cols = [col for col in df.columns if col.isdigit()]
            df = df.drop(columns=component_cols)

        if bounds is not None and capacity_df is not None:
            capacity_aligned = self._align_forward_filled(capacity_df, df.index)
            weighted_bounds = self._capacity_weighted_bounds(bounds, capacity_aligned)
            if weighted_bounds["lower"] is not None:
                df["battery_soc_lower_bound_pct"] = weighted_bounds["lower"]
            if weighted_bounds["upper"] is not None:
                df["battery_soc_upper_bound_pct"] = weighted_bounds["upper"]
        return df

    async def ac_active_energy(  # pylint: disable=too-many-arguments
        self,
        *,
        microgrid_id: int,
        start: datetime,
        end: datetime,
        component_types: tuple[str, ...] = (
            "grid",
            "pv",
            "wind",
            "battery",
            "chp",
            "consumption",
        ),
        resampling_period: timedelta = timedelta(seconds=10),
        keep_components: bool = False,
        unit: str = "Wh",
        direction: Literal["net", "consumed", "delivered"] = "net",
    ) -> pd.DataFrame | None:
        """Fetch AC active energy, optionally for one direction only.

        direction="net" fetches the net cumulative-energy metric.  The raw
        readings are cumulative meter values.
        """
        if direction not in ("net", "consumed", "delivered"):
            raise ValueError(f"Unknown energy direction: {direction}")

        energy = await self.metric_data(
            microgrid_id=microgrid_id,
            start=start,
            end=end,
            component_types=component_types,
            resampling_period=resampling_period,
            metric=self._AC_ACTIVE_ENERGY_METRICS[f"energy_{direction}"],
            keep_components=keep_components,
        )
        if energy is None:
            return None
        energy = self._convert_units(
            energy, unit=unit, scale_by_unit={"Wh": 1, "kWh": 1000, "MWh": 1e6}
        )
        return energy

    @staticmethod
    def _convert_cumulative_energy_to_power(
        df: pd.DataFrame, *, resampling_period: timedelta
    ) -> pd.DataFrame:
        """Convert cumulative energy meter readings into power values."""
        energy_delta = df.diff().shift(-1)
        negative_deltas = energy_delta < 0
        if negative_deltas.to_numpy().any():
            _logger.warning(
                "Ignoring %s negative cumulative-energy deltas while converting "
                "to power, likely due to meter resets.",
                int(negative_deltas.sum().sum()),
            )
            energy_delta = energy_delta.mask(negative_deltas)
        return energy_delta * (timedelta(hours=1) / resampling_period)

    async def _component_metric_data(  # pylint: disable=too-many-arguments
        self,
        *,
        microgrid_id: int,
        component_ids: list[int],
        metric: Metric,
        start: datetime,
        end: datetime,
        resampling_period: timedelta,
    ) -> pd.DataFrame | None:
        """Fetch per-component metric data as a timestamp-indexed table."""
        data = [
            sample
            async for sample in self._client.receive_microgrid_components_data(
                microgrid_components=[(microgrid_id, component_ids)],
                metrics=metric,
                start_time=start,
                end_time=end,
                resampling_period=resampling_period,
            )
        ]
        if not data:
            return None

        df = pd.DataFrame(data)
        df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True)
        df = df.drop_duplicates(keep="first").pivot_table(
            index="timestamp", columns="component_id", values="value"
        )
        for component_id in component_ids:
            if component_id not in df.columns:
                df.loc[:, component_id] = np.nan
        df = df.rename(columns=str)
        return df.reindex(columns=sorted(df.columns))

    @staticmethod
    def _capacity_weighted_soc(
        soc_df: pd.DataFrame, capacity_df: pd.DataFrame
    ) -> pd.Series | None:
        """Calculate aggregate battery SOC weighted by per-timestamp capacities."""
        component_cols = [col for col in capacity_df.columns if col in soc_df.columns]
        if not component_cols:
            _logger.warning(
                "Cannot calculate capacity-weighted battery SOC: no component "
                "SOC columns match battery capacity columns."
            )
            return None

        soc = soc_df[component_cols].apply(pd.to_numeric, errors="coerce")
        capacity = capacity_df[component_cols].apply(pd.to_numeric, errors="coerce")
        valid = soc.notna() & capacity.notna() & (capacity > 0)

        weighted_capacity = capacity.where(valid)
        available_capacity = weighted_capacity.sum(axis=1)
        weighted_sum = (soc.where(valid) * weighted_capacity).sum(axis=1, min_count=1)

        return cast(
            pd.Series,
            weighted_sum.div(available_capacity).where(available_capacity > 0),
        )

    @classmethod
    def _capacity_weighted_bounds(
        cls, bounds: pd.DataFrame, capacity_df: pd.DataFrame
    ) -> dict[str, pd.Series | None]:
        """Calculate capacity-weighted static SOC bounds for a battery pool."""
        weighted_bounds: dict[str, pd.Series | None] = {}
        for bound_name in ("lower", "upper"):
            bound_values = bounds[bound_name].dropna()
            if bound_values.empty:
                weighted_bounds[bound_name] = None
                continue

            component_ids = [
                component_id
                for component_id in bound_values.index
                if component_id in capacity_df.columns
            ]
            if not component_ids:
                weighted_bounds[bound_name] = None
                continue

            bound_df = pd.DataFrame(
                {
                    component_id: bound_values[component_id]
                    for component_id in component_ids
                },
                index=capacity_df.index,
            )
            weighted_bounds[bound_name] = cls._capacity_weighted_soc(
                bound_df, capacity_df[component_ids]
            )
        return weighted_bounds

    @staticmethod
    def _align_forward_filled(df: pd.DataFrame, index: pd.Index) -> pd.DataFrame:
        """Reindex a DataFrame to `index`, forward-filling from its own history."""
        combined_index = df.index.union(index)
        return df.reindex(combined_index).ffill().reindex(index)

    async def _battery_soc_asset_bounds(
        self,
        microgrid_id: int | str,
        component_ids: list[int],
    ) -> pd.DataFrame | None:
        """Fetch static per-component SOC bounds from battery asset metadata."""
        if self._assets_client is None:
            _logger.warning(
                "Cannot fetch battery SOC bounds for microgrid %s: no Assets API client.",
                microgrid_id,
            )
            return None

        batteries = await self._assets_client.list_microgrid_electrical_components(
            MicrogridId(int(microgrid_id)),
            categories=[ElectricalComponentCategory.BATTERY],
        )
        configured_ids = set(component_ids)
        bounds = {
            str(int(battery.id)): battery.rated_bounds.get(AssetsMetric.BATTERY_SOC_PCT)
            for battery in batteries
            if int(battery.id) in configured_ids
        }
        bounds_df = pd.DataFrame.from_dict(
            {
                component_id: {
                    "lower": bound.lower if bound is not None else None,
                    "upper": bound.upper if bound is not None else None,
                }
                for component_id, bound in bounds.items()
            },
            orient="index",
        )
        if bounds_df.empty or bounds_df.isna().all().all():
            _logger.warning(
                "Assets API returned no battery SOC rated bounds for microgrid %s.",
                microgrid_id,
            )
            return None
        return bounds_df
Attributes¤
microgrid_configs property ¤
microgrid_configs: dict[int, MicrogridConfig] | None

Return the microgrid configurations.

microgrid_ids property ¤
microgrid_ids: list[int]

Get the microgrid IDs.

RETURNS DESCRIPTION
list[int]

List of microgrid IDs.

Methods:¤
__init__ ¤
__init__(
    server_url: str,
    auth_key: str,
    sign_secret: str,
    microgrid_configs: (
        _MicrogridConfigMapping
        | _MicrogridConfigsContainer
        | None
    ) = None,
    assets_client: AssetsApiClient | None = None,
) -> None

Initialize microgrid data.

PARAMETER DESCRIPTION
server_url

URL of the reporting service.

TYPE: str

auth_key

Authentication key to the service.

TYPE: str

sign_secret

Secret for signing requests.

TYPE: str

microgrid_configs

A mapping of microgrid IDs to configurations, or a gridpool configuration document containing that mapping.

TYPE: _MicrogridConfigMapping | _MicrogridConfigsContainer | None DEFAULT: None

assets_client

Optional Assets API client used to fetch static battery SOC rated bounds.

TYPE: AssetsApiClient | None DEFAULT: None

Source code in src/frequenz/data/microgrid/component_data.py
def __init__(
    self,
    server_url: str,
    auth_key: str,
    sign_secret: str,
    microgrid_configs: (
        _MicrogridConfigMapping | _MicrogridConfigsContainer | None
    ) = None,
    assets_client: AssetsApiClient | None = None,
) -> None:
    """Initialize microgrid data.

    Args:
        server_url: URL of the reporting service.
        auth_key: Authentication key to the service.
        sign_secret: Secret for signing requests.
        microgrid_configs: A mapping of microgrid IDs to configurations, or a
            gridpool configuration document containing that mapping.
        assets_client: Optional Assets API client used to fetch static battery
            SOC rated bounds.
    """
    self._microgrid_configs = (
        None
        if microgrid_configs is None
        else _normalize_microgrid_configs(microgrid_configs)
    )
    self._client = ReportingApiClient(
        server_url=server_url, auth_key=auth_key, sign_secret=sign_secret
    )
    self._assets_client = assets_client
ac_active_energy async ¤
ac_active_energy(
    *,
    microgrid_id: int,
    start: datetime,
    end: datetime,
    component_types: tuple[str, ...] = (
        "grid",
        "pv",
        "wind",
        "battery",
        "chp",
        "consumption",
    ),
    resampling_period: timedelta = timedelta(seconds=10),
    keep_components: bool = False,
    unit: str = "Wh",
    direction: Literal[
        "net", "consumed", "delivered"
    ] = "net"
) -> DataFrame | None

Fetch AC active energy, optionally for one direction only.

direction="net" fetches the net cumulative-energy metric. The raw readings are cumulative meter values.

Source code in src/frequenz/data/microgrid/component_data.py
async def ac_active_energy(  # pylint: disable=too-many-arguments
    self,
    *,
    microgrid_id: int,
    start: datetime,
    end: datetime,
    component_types: tuple[str, ...] = (
        "grid",
        "pv",
        "wind",
        "battery",
        "chp",
        "consumption",
    ),
    resampling_period: timedelta = timedelta(seconds=10),
    keep_components: bool = False,
    unit: str = "Wh",
    direction: Literal["net", "consumed", "delivered"] = "net",
) -> pd.DataFrame | None:
    """Fetch AC active energy, optionally for one direction only.

    direction="net" fetches the net cumulative-energy metric.  The raw
    readings are cumulative meter values.
    """
    if direction not in ("net", "consumed", "delivered"):
        raise ValueError(f"Unknown energy direction: {direction}")

    energy = await self.metric_data(
        microgrid_id=microgrid_id,
        start=start,
        end=end,
        component_types=component_types,
        resampling_period=resampling_period,
        metric=self._AC_ACTIVE_ENERGY_METRICS[f"energy_{direction}"],
        keep_components=keep_components,
    )
    if energy is None:
        return None
    energy = self._convert_units(
        energy, unit=unit, scale_by_unit={"Wh": 1, "kWh": 1000, "MWh": 1e6}
    )
    return energy
ac_active_power async ¤
ac_active_power(
    *,
    microgrid_id: int | str,
    start: datetime,
    end: datetime,
    component_types: tuple[str, ...] = (
        "grid",
        "pv",
        "wind",
        "battery",
        "chp",
        "consumption",
    ),
    resampling_period: timedelta = timedelta(seconds=10),
    keep_components: bool = False,
    splits: bool = False,
    unit: str = "kW",
    from_energy: bool = False
) -> DataFrame | None

Power data for component types of a microgrid.

Set from_energy to derive power from net AC active energy instead of fetching the instantaneous power metric.

Source code in src/frequenz/data/microgrid/component_data.py
async def ac_active_power(  # pylint: disable=too-many-arguments
    self,
    *,
    microgrid_id: int | str,
    start: datetime,
    end: datetime,
    component_types: tuple[str, ...] = (
        "grid",
        "pv",
        "wind",
        "battery",
        "chp",
        "consumption",
    ),
    resampling_period: timedelta = timedelta(seconds=10),
    keep_components: bool = False,
    splits: bool = False,
    unit: str = "kW",
    from_energy: bool = False,
) -> pd.DataFrame | None:
    """Power data for component types of a microgrid.

    Set `from_energy` to derive power from net AC active energy instead of
    fetching the instantaneous power metric.
    """
    if from_energy:
        df = await self.ac_active_energy(
            microgrid_id=microgrid_id,
            start=start,
            end=end,
            component_types=component_types,
            resampling_period=resampling_period,
            keep_components=keep_components,
            unit="Wh",
        )
        if df is not None:
            df = self._convert_cumulative_energy_to_power(
                df, resampling_period=resampling_period
            )
            if splits:
                df = self._add_split_pos_neg_cols(df)
    else:
        df = await self.metric_data(
            microgrid_id=microgrid_id,
            start=start,
            end=end,
            component_types=component_types,
            resampling_period=resampling_period,
            metric="AC_POWER_ACTIVE",
            keep_components=keep_components,
            splits=splits,
        )
    if df is None:
        return df

    return self._convert_units(
        df, unit=unit, scale_by_unit={"W": 1, "kW": 1000, "MW": 1e6}
    )
metric_data async ¤
metric_data(
    *,
    microgrid_id: int | str,
    start: datetime,
    end: datetime,
    component_types: tuple[str, ...] = (
        "grid",
        "pv",
        "battery",
        "wind",
        "chp",
        "consumption",
    ),
    resampling_period: timedelta = timedelta(seconds=10),
    metric: str = "AC_POWER_ACTIVE",
    keep_components: bool = False,
    splits: bool = False
) -> DataFrame | None

Fetch aggregated data for an arbitrary metric across component types.

PARAMETER DESCRIPTION
microgrid_id

Microgrid ID. Numeric strings are accepted for notebook compatibility.

TYPE: int | str

start

Start timestamp.

TYPE: datetime

end

End timestamp.

TYPE: datetime

component_types

Component types whose aggregation formulas should be queried for the requested metric.

TYPE: tuple[str, ...] DEFAULT: ('grid', 'pv', 'battery', 'wind', 'chp', 'consumption')

resampling_period

Sampling period used for the reporting query.

TYPE: timedelta DEFAULT: timedelta(seconds=10)

metric

Reporting metric name to fetch.

TYPE: str DEFAULT: 'AC_POWER_ACTIVE'

keep_components

Whether to include individual component IDs alongside the aggregated component-type columns.

TYPE: bool DEFAULT: False

splits

Whether to append positive and negative split columns for each returned column.

TYPE: bool DEFAULT: False

Returns: DataFrame with power data of aggregated components or None if no data is available

RAISES DESCRIPTION
ValueError

If microgrid configurations are not loaded.

Source code in src/frequenz/data/microgrid/component_data.py
async def metric_data(  # pylint: disable=too-many-arguments
    self,
    *,
    microgrid_id: int | str,
    start: datetime,
    end: datetime,
    component_types: tuple[str, ...] = (
        "grid",
        "pv",
        "battery",
        "wind",
        "chp",
        "consumption",
    ),
    resampling_period: timedelta = timedelta(seconds=10),
    metric: str = "AC_POWER_ACTIVE",
    keep_components: bool = False,
    splits: bool = False,
) -> pd.DataFrame | None:
    """Fetch aggregated data for an arbitrary metric across component types.

    Args:
        microgrid_id: Microgrid ID. Numeric strings are accepted for notebook
            compatibility.
        start: Start timestamp.
        end: End timestamp.
        component_types: Component types whose aggregation formulas should be
            queried for the requested metric.
        resampling_period: Sampling period used for the reporting query.
        metric: Reporting metric name to fetch.
        keep_components: Whether to include individual component IDs alongside
            the aggregated component-type columns.
        splits: Whether to append positive and negative split columns for each
            returned column.
    Returns:
        DataFrame with power data of aggregated components
        or None if no data is available

    Raises:
        ValueError: If microgrid configurations are not loaded.
    """
    if self._microgrid_configs is None:
        raise ValueError("Microgrid configurations are not loaded.")
    microgrid_id = int(microgrid_id)
    mcfg = self._microgrid_configs[microgrid_id]
    metric = metric.upper()

    formulas = {
        ctype: mcfg.formula(ctype, metric.upper()) for ctype in component_types
    }

    logging.debug("Formulas: %s", formulas)

    metric_enum = Metric[metric.upper()]

    async def _collect(formula: str) -> list:
        return [
            sample
            async for sample in self._client.receive_aggregated_data(
                microgrid_id=microgrid_id,
                metric=metric_enum,
                aggregation_formula=formula,
                start_time=start,
                end_time=end,
                resampling_period=resampling_period,
            )
        ]

    all_cids: list[int] = []
    if keep_components:
        # IDs are read from the formulas: derived types (e.g. consumption)
        # mention the IDs of other types and a formula can repeat an ID.
        # Request each ID once to avoid streaming identical samples.
        all_cids = list(
            dict.fromkeys(
                cid
                for ctype in component_types
                for cid in mcfg.component_type_ids(ctype, metric=metric)
            )
        )
        _logger.debug("CIDs: %s", all_cids)

    async def _components() -> pd.DataFrame | None:
        if not all_cids:
            return None
        return await self._component_metric_data(
            microgrid_id=microgrid_id,
            component_ids=all_cids,
            metric=metric_enum,
            start=start,
            end=end,
            resampling_period=resampling_period,
        )

    # The formula streams and the per-component query are independent.
    *formula_results, component_df = await asyncio.gather(
        *(_collect(formula) for formula in formulas.values()), _components()
    )
    data = [sample for samples in formula_results for sample in samples]

    if len(data) == 0 and component_df is None:
        _logger.warning("No data found")
        return None

    df = None
    if data:
        df = pd.DataFrame(data)
        df["timestamp"] = pd.to_datetime(df["timestamp"], utc=True)
        assert df["timestamp"].dt.tz is not None, "Timestamps are not tz-aware"

        # Remove duplicates
        dup_mask = df.duplicated(keep="first")
        if not dup_mask.empty:
            _logger.info("Found %s rows that have duplicates", dup_mask.sum())
        df = df[~dup_mask]

        # Pivot table
        df = df.pivot_table(
            index="timestamp", columns="component_id", values="value"
        )
    # Rename formula columns
    rename_cols: dict[str, str] = {}
    for ctype, formula in formulas.items():
        if formula in rename_cols:
            _logger.warning(
                "Ignoring %s since formula %s exists already for %s",
                ctype,
                formula,
                rename_cols[formula],
            )
            continue
        rename_cols[formula] = ctype

    if df is not None:
        df = df.rename(columns=rename_cols)
    if component_df is not None:
        df = component_df if df is None else df.join(component_df, how="outer")
    elif df is not None and keep_components:
        for component_id in all_cids:
            if component_id not in df.columns:
                _logger.warning(
                    "Component ID %s not found in data, setting zero", component_id
                )
                df.loc[:, component_id] = np.nan

    assert df is not None

    # Make string columns
    df.columns = [str(e) for e in df.columns]

    if splits:
        df = self._add_split_pos_neg_cols(df)

    ctypes = list(rename_cols.values())
    new_cols = [e for e in ctypes if e in df.columns] + sorted(
        [e for e in df.columns if e not in ctypes]
    )
    return df[new_cols]
soc async ¤
soc(
    *,
    microgrid_id: int | str,
    start: datetime,
    end: datetime,
    resampling_period: timedelta = timedelta(seconds=10),
    keep_components: bool = False
) -> DataFrame | None

Fetch and aggregate SOC for the configured battery components.

Battery component IDs are read from the microgrid configuration. Their SOC and capacity values are fetched directly per component, without requiring aggregation formulas in the microgrid configuration. The aggregate SOC is a capacity-weighted average, using each component's own capacity reading at each timestamp so that capacity changes are reflected over time. SOC bounds are aggregated using those same capacity weights.

PARAMETER DESCRIPTION
microgrid_id

The ID of the microgrid.

TYPE: int | str

start

The start time for the data fetch.

TYPE: datetime

end

The end time for the data fetch.

TYPE: datetime

resampling_period

The period for resampling the data.

TYPE: timedelta DEFAULT: timedelta(seconds=10)

keep_components

Whether to keep individual component data.

TYPE: bool DEFAULT: False

RETURNS DESCRIPTION
DataFrame | None

A DataFrame containing the aggregated SOC data, or None if no data is available.

RAISES DESCRIPTION
ValueError

If microgrid configurations are not loaded.

Source code in src/frequenz/data/microgrid/component_data.py
async def soc(  # pylint: disable=too-many-arguments
    self,
    *,
    microgrid_id: int | str,
    start: datetime,
    end: datetime,
    resampling_period: timedelta = timedelta(seconds=10),
    keep_components: bool = False,
) -> pd.DataFrame | None:
    """Fetch and aggregate SOC for the configured battery components.

    Battery component IDs are read from the microgrid configuration.  Their
    SOC and capacity values are fetched directly per component, without
    requiring aggregation formulas in the microgrid configuration. The
    aggregate SOC is a capacity-weighted average, using each component's
    own capacity reading at each timestamp so that capacity changes are
    reflected over time. SOC bounds are aggregated using those same
    capacity weights.

    Args:
        microgrid_id: The ID of the microgrid.
        start: The start time for the data fetch.
        end: The end time for the data fetch.
        resampling_period: The period for resampling the data.
        keep_components: Whether to keep individual component data.

    Returns:
        A DataFrame containing the aggregated SOC data, or None if no data is available.

    Raises:
        ValueError: If microgrid configurations are not loaded.
    """
    if self._microgrid_configs is None:
        raise ValueError("Microgrid configurations are not loaded.")

    mg_id = int(microgrid_id)
    config = self._microgrid_configs[mg_id]
    component_ids = config.component_type_ids(
        "battery", component_category="component"
    )
    if not component_ids:
        _logger.warning(
            "No battery components are configured for microgrid %s.", mg_id
        )
        return None

    # SOC, capacity and static bounds are independent requests.
    df, capacity_df, bounds = await asyncio.gather(
        self._component_metric_data(
            microgrid_id=mg_id,
            component_ids=component_ids,
            metric=Metric.BATTERY_SOC_PCT,
            start=start,
            end=end,
            resampling_period=resampling_period,
        ),
        self._component_metric_data(
            microgrid_id=mg_id,
            component_ids=component_ids,
            metric=Metric.BATTERY_CAPACITY,
            start=start,
            end=end,
            resampling_period=resampling_period,
        ),
        self._battery_soc_asset_bounds(mg_id, component_ids),
    )
    if df is None:
        _logger.warning("No battery SOC data found for microgrid %s.", mg_id)
        return None

    if capacity_df is None:
        _logger.warning("No battery capacity data found for microgrid %s.", mg_id)
    component_columns = [str(component_id) for component_id in component_ids]
    weighted_soc = None
    if capacity_df is not None:
        capacity_aligned = self._align_forward_filled(
            capacity_df.reindex(columns=component_columns), df.index
        )
        weighted_soc = self._capacity_weighted_soc(
            df.reindex(columns=component_columns), capacity_aligned
        )
    if weighted_soc is None:
        return df if keep_components else None
    df["battery"] = weighted_soc

    if not keep_components:
        component_cols = [col for col in df.columns if col.isdigit()]
        df = df.drop(columns=component_cols)

    if bounds is not None and capacity_df is not None:
        capacity_aligned = self._align_forward_filled(capacity_df, df.index)
        weighted_bounds = self._capacity_weighted_bounds(bounds, capacity_aligned)
        if weighted_bounds["lower"] is not None:
            df["battery_soc_lower_bound_pct"] = weighted_bounds["lower"]
        if weighted_bounds["upper"] is not None:
            df["battery_soc_upper_bound_pct"] = weighted_bounds["upper"]
    return df

frequenz.data.microgrid.StatefulDataFetcher ¤

A helper class to handle fetching of microgrid component data.

This class provides methods to query new data for a specific microgrid and its components, and to write the updated data to a temporary file. The file is written only temporarily and needs to be committed or rolled back in case of success or failure, respectively.

Source code in src/frequenz/data/microgrid/_stateful_data_fetcher.py
class StatefulDataFetcher:
    """A helper class to handle fetching of microgrid component data.

    This class provides methods to query new data for a specific microgrid and
    its components, and to write the updated data to a temporary file.
    The file is written only temporarily and needs to be committed or rolled back
    in case of success or failure, respectively.
    """

    def __init__(  # pylint: disable=too-many-arguments, too-many-positional-arguments
        self,
        microgrid_data: MicrogridData,
        data_buffer_dir: Path,
        resampling_period: timedelta,
        end_time_delta: timedelta = END_TIME_DELTA,
        initial_period: timedelta = timedelta(hours=1),
    ) -> None:
        """Initialize the TransactionalDataExport class.

        Args:
            microgrid_data: An instance of MicrogridData to query data from.
            data_buffer_dir: The path to the directory for buffering data.
            resampling_period: The period to resample the data.
            end_time_delta: The time subtracted from now to be passed as
                            end_time to the API.
            initial_period: The initial period to use for the first data fetch.
        """
        _logger.debug("Initializing StatefulDataFetcher instance.")
        self._microgrid_data = microgrid_data
        self._data_buffer = data_buffer_dir
        self._resampling_period = resampling_period
        self._end_time_delta = end_time_delta
        self._initial_period = initial_period

        self._data_buffer.mkdir(parents=True, exist_ok=True)

        # Maps temp_path -> final_path for files that need to be committed
        self._temp_files_to_commit: dict[Path, Path] = {}

    @staticmethod
    def _generate_filename(
        microgrid_id: int, components: tuple[str, ...], metric: str
    ) -> str:
        """Generate a unique, sanitized filename.

        Args:
            microgrid_id: The ID of the microgrid.
            components: The component types to include in the filename.
            metric: The name of the metric to include in the filename.

        Returns:
            A sanitized string suitable for use as a filename.
        """
        sorted_components = sorted([c.lower().replace(" ", "_") for c in components])
        return (
            f"mid{microgrid_id}_{'_'.join(sorted_components)}_{metric.lower()}.parquet"
        )

    async def receive_microgrid_data(
        self,
        microgrid_id: int,
        components: tuple[str, ...],
        metric: str,
    ) -> pd.DataFrame | None:
        """Query new microgrid data and write the updated buffer to a temp file.

        Reads the last timestamp from the main buffer file, queries for new data,
        and writes the combined data to a new temporary buffer file.

        Args:
            microgrid_id: The ID of the microgrid to query.
            components: The component types to include in the query.
            metric: The metric to query.

        Returns:
            A pandas DataFrame with only the new data points, or None.
        """
        _logger.debug(
            "Processing data for microgrid ID %s, components %s",
            microgrid_id,
            components,
        )

        final_parquet_file = self._data_buffer / self._generate_filename(
            microgrid_id, components, metric
        )
        temp_parquet_file = final_parquet_file.with_suffix(
            final_parquet_file.suffix + ".tmp"
        )

        now = datetime.now(timezone.utc)
        end_time = now - self._end_time_delta
        on_disk_buffer = pd.DataFrame()
        start_time: datetime

        try:
            on_disk_buffer = pd.read_parquet(final_parquet_file)
            if not on_disk_buffer.empty and isinstance(
                on_disk_buffer.index, pd.DatetimeIndex
            ):
                last_timestamp = on_disk_buffer.index.max().to_pydatetime()
                start_time = last_timestamp + timedelta(microseconds=1)
                _logger.info("Read buffer. Last timestamp: %s.", last_timestamp)
            else:
                _logger.info(
                    "Buffer file is empty. Defaulting to: %s.", self._initial_period
                )
                start_time = now - self._initial_period
        except FileNotFoundError:
            _logger.info(
                "No buffer file at '%s'. Defaulting to %s.",
                final_parquet_file,
                self._initial_period,
            )
            start_time = now - self._initial_period

        if start_time >= end_time:
            _logger.info("Buffer is up to date. No new data to fetch.")
            return None

        _logger.info("Fetching new data from %s to %s...", start_time, end_time)

        new_df = await self._microgrid_data.metric_data(
            microgrid_id=microgrid_id,
            start=start_time,
            end=end_time,
            component_types=components,
            resampling_period=self._resampling_period,
            metric=metric,
        )

        if new_df is not None and not new_df.empty:
            _logger.info("Fetched %d new data points.", len(new_df))
            combined_df = pd.concat([on_disk_buffer, new_df])
            combined_df = combined_df[~combined_df.index.duplicated(keep="last")]
            combined_df.sort_index(inplace=True)
            updated_buffer = combined_df.tail(BUFFER_SIZE)

            # Write to temporary file instead of final destination
            updated_buffer.to_parquet(temp_parquet_file, engine="pyarrow")

            _logger.info(
                "Wrote updated buffer with %d points to temporary file '%s'.",
                len(updated_buffer),
                temp_parquet_file,
            )

            self._temp_files_to_commit[temp_parquet_file] = final_parquet_file
        else:
            _logger.info("No new data was returned from the API.")

        return new_df

    def commit(self) -> None:
        """Commit the temporary files to their final locations."""
        _logger.info("Reports sent successfully. Committing buffer updates.")
        for temp_path, final_path in self._temp_files_to_commit.items():
            try:
                os.replace(temp_path, final_path)
                _logger.debug(
                    "Committed buffer update: %s -> %s", temp_path, final_path
                )
            except OSError as e:
                _logger.error(
                    "Failed to commit buffer update from %s to %s: %s",
                    temp_path,
                    final_path,
                    e,
                )

        self._temp_files_to_commit.clear()

    def rollback(self) -> None:
        """Rollback the temporary files if the transaction fails."""
        for temp_path in self._temp_files_to_commit:
            try:
                temp_path.unlink()
                _logger.debug("Rolled back by deleting temp file: %s", temp_path)
            except OSError as unlink_e:
                _logger.error(
                    "Failed to clean up temporary buffer file %s during rollback: %s",
                    temp_path,
                    unlink_e,
                )

        self._temp_files_to_commit.clear()
Methods:¤
__init__ ¤
__init__(
    microgrid_data: MicrogridData,
    data_buffer_dir: Path,
    resampling_period: timedelta,
    end_time_delta: timedelta = END_TIME_DELTA,
    initial_period: timedelta = timedelta(hours=1),
) -> None

Initialize the TransactionalDataExport class.

PARAMETER DESCRIPTION
microgrid_data

An instance of MicrogridData to query data from.

TYPE: MicrogridData

data_buffer_dir

The path to the directory for buffering data.

TYPE: Path

resampling_period

The period to resample the data.

TYPE: timedelta

end_time_delta

The time subtracted from now to be passed as end_time to the API.

TYPE: timedelta DEFAULT: END_TIME_DELTA

initial_period

The initial period to use for the first data fetch.

TYPE: timedelta DEFAULT: timedelta(hours=1)

Source code in src/frequenz/data/microgrid/_stateful_data_fetcher.py
def __init__(  # pylint: disable=too-many-arguments, too-many-positional-arguments
    self,
    microgrid_data: MicrogridData,
    data_buffer_dir: Path,
    resampling_period: timedelta,
    end_time_delta: timedelta = END_TIME_DELTA,
    initial_period: timedelta = timedelta(hours=1),
) -> None:
    """Initialize the TransactionalDataExport class.

    Args:
        microgrid_data: An instance of MicrogridData to query data from.
        data_buffer_dir: The path to the directory for buffering data.
        resampling_period: The period to resample the data.
        end_time_delta: The time subtracted from now to be passed as
                        end_time to the API.
        initial_period: The initial period to use for the first data fetch.
    """
    _logger.debug("Initializing StatefulDataFetcher instance.")
    self._microgrid_data = microgrid_data
    self._data_buffer = data_buffer_dir
    self._resampling_period = resampling_period
    self._end_time_delta = end_time_delta
    self._initial_period = initial_period

    self._data_buffer.mkdir(parents=True, exist_ok=True)

    # Maps temp_path -> final_path for files that need to be committed
    self._temp_files_to_commit: dict[Path, Path] = {}
commit ¤
commit() -> None

Commit the temporary files to their final locations.

Source code in src/frequenz/data/microgrid/_stateful_data_fetcher.py
def commit(self) -> None:
    """Commit the temporary files to their final locations."""
    _logger.info("Reports sent successfully. Committing buffer updates.")
    for temp_path, final_path in self._temp_files_to_commit.items():
        try:
            os.replace(temp_path, final_path)
            _logger.debug(
                "Committed buffer update: %s -> %s", temp_path, final_path
            )
        except OSError as e:
            _logger.error(
                "Failed to commit buffer update from %s to %s: %s",
                temp_path,
                final_path,
                e,
            )

    self._temp_files_to_commit.clear()
receive_microgrid_data async ¤
receive_microgrid_data(
    microgrid_id: int,
    components: tuple[str, ...],
    metric: str,
) -> DataFrame | None

Query new microgrid data and write the updated buffer to a temp file.

Reads the last timestamp from the main buffer file, queries for new data, and writes the combined data to a new temporary buffer file.

PARAMETER DESCRIPTION
microgrid_id

The ID of the microgrid to query.

TYPE: int

components

The component types to include in the query.

TYPE: tuple[str, ...]

metric

The metric to query.

TYPE: str

RETURNS DESCRIPTION
DataFrame | None

A pandas DataFrame with only the new data points, or None.

Source code in src/frequenz/data/microgrid/_stateful_data_fetcher.py
async def receive_microgrid_data(
    self,
    microgrid_id: int,
    components: tuple[str, ...],
    metric: str,
) -> pd.DataFrame | None:
    """Query new microgrid data and write the updated buffer to a temp file.

    Reads the last timestamp from the main buffer file, queries for new data,
    and writes the combined data to a new temporary buffer file.

    Args:
        microgrid_id: The ID of the microgrid to query.
        components: The component types to include in the query.
        metric: The metric to query.

    Returns:
        A pandas DataFrame with only the new data points, or None.
    """
    _logger.debug(
        "Processing data for microgrid ID %s, components %s",
        microgrid_id,
        components,
    )

    final_parquet_file = self._data_buffer / self._generate_filename(
        microgrid_id, components, metric
    )
    temp_parquet_file = final_parquet_file.with_suffix(
        final_parquet_file.suffix + ".tmp"
    )

    now = datetime.now(timezone.utc)
    end_time = now - self._end_time_delta
    on_disk_buffer = pd.DataFrame()
    start_time: datetime

    try:
        on_disk_buffer = pd.read_parquet(final_parquet_file)
        if not on_disk_buffer.empty and isinstance(
            on_disk_buffer.index, pd.DatetimeIndex
        ):
            last_timestamp = on_disk_buffer.index.max().to_pydatetime()
            start_time = last_timestamp + timedelta(microseconds=1)
            _logger.info("Read buffer. Last timestamp: %s.", last_timestamp)
        else:
            _logger.info(
                "Buffer file is empty. Defaulting to: %s.", self._initial_period
            )
            start_time = now - self._initial_period
    except FileNotFoundError:
        _logger.info(
            "No buffer file at '%s'. Defaulting to %s.",
            final_parquet_file,
            self._initial_period,
        )
        start_time = now - self._initial_period

    if start_time >= end_time:
        _logger.info("Buffer is up to date. No new data to fetch.")
        return None

    _logger.info("Fetching new data from %s to %s...", start_time, end_time)

    new_df = await self._microgrid_data.metric_data(
        microgrid_id=microgrid_id,
        start=start_time,
        end=end_time,
        component_types=components,
        resampling_period=self._resampling_period,
        metric=metric,
    )

    if new_df is not None and not new_df.empty:
        _logger.info("Fetched %d new data points.", len(new_df))
        combined_df = pd.concat([on_disk_buffer, new_df])
        combined_df = combined_df[~combined_df.index.duplicated(keep="last")]
        combined_df.sort_index(inplace=True)
        updated_buffer = combined_df.tail(BUFFER_SIZE)

        # Write to temporary file instead of final destination
        updated_buffer.to_parquet(temp_parquet_file, engine="pyarrow")

        _logger.info(
            "Wrote updated buffer with %d points to temporary file '%s'.",
            len(updated_buffer),
            temp_parquet_file,
        )

        self._temp_files_to_commit[temp_parquet_file] = final_parquet_file
    else:
        _logger.info("No new data was returned from the API.")

    return new_df
rollback ¤
rollback() -> None

Rollback the temporary files if the transaction fails.

Source code in src/frequenz/data/microgrid/_stateful_data_fetcher.py
def rollback(self) -> None:
    """Rollback the temporary files if the transaction fails."""
    for temp_path in self._temp_files_to_commit:
        try:
            temp_path.unlink()
            _logger.debug("Rolled back by deleting temp file: %s", temp_path)
        except OSError as unlink_e:
            _logger.error(
                "Failed to clean up temporary buffer file %s during rollback: %s",
                temp_path,
                unlink_e,
            )

    self._temp_files_to_commit.clear()

Functions:¤

frequenz.data.microgrid.load_configs async ¤

load_configs(
    default_files: (
        str | Path | list[str | Path] | None
    ) = None,
    assets_client: AssetsApiClient | None = None,
    override_files: (
        str | Path | list[str | Path] | None
    ) = None,
    microgrid_ids: list[int] | None = None,
    component_graph_config: (
        ComponentGraphConfig | None
    ) = None,
) -> AssetsConfig

Load configs from up to three sources and merge them in layers.

Combines up to three sources, listed here from lowest to highest precedence: a default config file layer, the Assets API, and an override config file layer. Higher layers win on conflicts, while lower layers fill in anything the higher ones leave unset. This lets callers pick a strategy by choosing which sources to pass, for example:

  • default_files + assets_client: files provide defaults that the Assets API overrides.
  • assets_client + override_files: the Assets API provides the base that files override.
  • all three: the Assets API sits between a default and an override file layer.

The microgrid IDs fetched from the Assets API are microgrid_ids when given, otherwise the IDs found in the default and override files. This lets the Assets API layer be used even when no files are given.

PARAMETER DESCRIPTION
default_files

Optional path or list of paths to config files forming the lowest-precedence layer.

TYPE: str | Path | list[str | Path] | None DEFAULT: None

assets_client

Optional Assets API client. When given, microgrid metadata and formulas are fetched and layered above the default files.

TYPE: AssetsApiClient | None DEFAULT: None

override_files

Optional path or list of paths to config files forming the highest-precedence layer.

TYPE: str | Path | list[str | Path] | None DEFAULT: None

microgrid_ids

Optional explicit microgrid IDs to fetch from the Assets API. When given, these replace the IDs derived from the files, so the Assets API layer can be used without any files.

TYPE: list[int] | None DEFAULT: None

component_graph_config

How to build the component graph and generate its formulas. See ComponentGraphConfig. Defaults to that class's own defaults. Requires an assets_client.

TYPE: ComponentGraphConfig | None DEFAULT: None

RETURNS DESCRIPTION
AssetsConfig

The merged document. Use .microgrids for just the microgrid map; the

AssetsConfig

file layers may also contribute relations and market_locations,

AssetsConfig

which the Assets API layer does not provide.

RAISES DESCRIPTION
ValueError

If none of the three sources is provided, if microgrid_ids or component_graph_config is given without an assets_client, or if a file's assets.microgrids is not a table.

Source code in frequenz/gridpool/config/_load.py
async def load_configs(
    default_files: str | Path | list[str | Path] | None = None,
    assets_client: AssetsApiClient | None = None,
    override_files: str | Path | list[str | Path] | None = None,
    microgrid_ids: list[int] | None = None,
    component_graph_config: ComponentGraphConfig | None = None,
) -> AssetsConfig:
    """Load configs from up to three sources and merge them in layers.

    Combines up to three sources, listed here from lowest to highest
    precedence: a *default* config file layer, the Assets API, and an
    *override* config file layer.  Higher layers win on conflicts, while
    lower layers fill in anything the higher ones leave unset.  This lets
    callers pick a strategy by choosing which sources to pass, for example:

    - `default_files` + `assets_client`: files provide defaults that the
      Assets API overrides.
    - `assets_client` + `override_files`: the Assets API provides the base
      that files override.
    - all three: the Assets API sits between a default and an override file
      layer.

    The microgrid IDs fetched from the Assets API are `microgrid_ids` when
    given, otherwise the IDs found in the default and override files.  This
    lets the Assets API layer be used even when no files are given.

    Args:
        default_files:
            Optional path or list of paths to config files forming the
            lowest-precedence layer.
        assets_client:
            Optional Assets API client.  When given, microgrid metadata and
            formulas are fetched and layered above the default files.
        override_files:
            Optional path or list of paths to config files forming the
            highest-precedence layer.
        microgrid_ids:
            Optional explicit microgrid IDs to fetch from the Assets API.
            When given, these replace the IDs derived from the files, so the
            Assets API layer can be used without any files.
        component_graph_config:
            How to build the component graph and generate its formulas.  See
            `ComponentGraphConfig`.  Defaults to that class's own defaults.
            Requires an `assets_client`.

    Returns:
        The merged document. Use `.microgrids` for just the microgrid map; the
        file layers may also contribute `relations` and `market_locations`,
        which the Assets API layer does not provide.

    Raises:
        ValueError: If none of the three sources is provided, if `microgrid_ids`
            or `component_graph_config` is given without an `assets_client`, or
            if a file's `assets.microgrids` is not a table.
    """
    if default_files is None and assets_client is None and override_files is None:
        raise ValueError("At least one config source must be provided.")

    if microgrid_ids is not None and assets_client is None:
        raise ValueError("microgrid_ids requires an assets_client.")

    if component_graph_config is not None and assets_client is None:
        raise ValueError("component_graph_config requires an assets_client.")

    default_table: dict[str, Any] = {}
    if default_files is not None:
        default_table = _merge_file_tables(default_files)

    override_table: dict[str, Any] = {}
    if override_files is not None:
        override_table = _merge_file_tables(override_files)

    merged = default_table
    if assets_client is not None:
        if microgrid_ids is None:
            file_ids: set[str] = set()
            for table in (default_table, override_table):
                microgrids = table.get("microgrids", {})
                if not isinstance(microgrids, dict):
                    raise ValueError(
                        f"`assets.microgrids` must be a table, got {type(microgrids)}"
                    )
                file_ids |= set(microgrids)
            microgrid_ids = sorted(int(mid) for mid in file_ids)
        assets_configs = await _load_microgrids_from_api(
            assets_client=assets_client,
            microgrid_ids=microgrid_ids,
            component_graph_config=component_graph_config,
        )
        schema = MicrogridConfig.Schema()
        api_table: dict[str, Any] = {
            "microgrids": {
                str(mid): schema.dump(cfg) for mid, cfg in assets_configs.items()
            }
        }
        merged = _deep_merge(merged, api_table)

    merged = _deep_merge(merged, override_table)

    loaded = AssetsConfig.Schema().load(merged)
    assert isinstance(loaded, AssetsConfig)
    loaded.check()
    return loaded