Clustering Internals¶

from tsam import ExtremeConfig

fs_clustered = flow_system.transform.cluster(
    n_clusters=8,
    cluster_duration='1D',
    extremes=ExtremeConfig(
        method='new_cluster', max_value=['HeatDemand(Q_th)|fixed_relative_profile'], preserve_n_clusters=True
    ),
)

fs_clustered.clustering

# Cluster order shows which cluster each original period maps to
fs_clustered.clustering.cluster_assignments

# Cluster occurrences shows how many original periods each cluster represents
fs_clustered.clustering.cluster_occurrences

# Compare original vs aggregated data as timeseries
# By default, plots all time-varying variables
fs_clustered.clustering.plot.compare()

# Use a different approach of visualizing the data using normalize heatmaps
ds = fs_clustered.clustering.plot.compare(data_only=True).data

ds_normalized = (ds - ds.min()) / (ds.max() - ds.min())
ds_normalized.to_array().plotly.imshow(
    x='time',
    animation_frame='representation',
    zmin=0,
    zmax=1,
    color_continuous_scale='viridis',
    title='Normalized Comparison',
)

# Compare specific variables only
fs_clustered.clustering.plot.compare(variables='HeatDemand(Q_th)|fixed_relative_profile')

# Duration curves show how well the aggregated data preserves the distribution
fs_clustered.clustering.plot.compare(kind='duration_curve').data

# View typical period profiles for each cluster
# Each line represents a cluster's representative day
fs_clustered.clustering.plot.clusters(variables='HeatDemand(Q_th)|fixed_relative_profile', color='cluster')

# Heatmap shows cluster assignments for each original period
fs_clustered.clustering.plot.heatmap()

# Get original and aggregated data
clustering = fs_clustered.clustering
original = clustering.original_data['HeatDemand(Q_th)|fixed_relative_profile']
aggregated = clustering.aggregated_data['HeatDemand(Q_th)|fixed_relative_profile']

# Expand aggregated data back to original timesteps
expanded = clustering.expand_data(aggregated)

print(f'Original:   {len(original.time)} timesteps')
print(f'Aggregated: {len(aggregated.time)} timesteps')
print(f'Expanded:   {len(expanded.time)} timesteps')

print(f'Sum of weights: {fs_clustered.cluster_weight.sum().item():.0f}')
print(f'Original timesteps: {len(flow_system.timesteps)}')

solver = fx.solvers.HighsSolver(mip_gap=0.01, log_to_console=False)
fs_clustered.optimize(solver)

fs_expanded = fs_clustered.transform.expand()

print(f'Clustered: {len(fs_clustered.timesteps)} timesteps')
print(f'Expanded:  {len(fs_expanded.timesteps)} timesteps')

# Before IO: Full tsam access is available
result = fs_clustered.clustering.sel()  # Get the AggregationResult
print(f'Before IO - AggregationResult available: {type(result).__name__}')
print(f'  - n_clusters: {result.n_clusters}')
print(f'  - accuracy.rmse (mean): {result.accuracy.rmse.mean():.4f}')

# Save and load the clustered system
import tempfile
from pathlib import Path

try:
    with tempfile.TemporaryDirectory() as tmpdir:
        path = Path(tmpdir) / 'clustered_system.nc'
        fs_clustered.to_netcdf(path)
        fs_loaded = fx.FlowSystem.from_netcdf(path)

    # Structure is preserved
    print('After IO - Structure preserved:')
    print(f'  - n_clusters: {fs_loaded.clustering.n_clusters}')
    print(f'  - dims: {fs_loaded.clustering.dims}')
    print(f'  - original_data variables: {list(fs_loaded.clustering.original_data.data_vars)[:3]}...')
except OSError as e:
    print(f'Note: NetCDF save/load skipped due to environment issue: {type(e).__name__}')
    print('This can happen in some CI environments. The functionality works locally.')
    fs_loaded = fs_clustered  # Use original for subsequent cells

# Key operations still work after IO:
# - Optimization
# - Expansion back to full resolution
# - Accessing original_data and aggregated_data

fs_loaded.optimize(solver)
fs_loaded_expanded = fs_loaded.transform.expand()

print('Loaded system can still be:')
print(f'  - Optimized: {fs_loaded.solution is not None}')
print(f'  - Expanded: {len(fs_loaded_expanded.timesteps)} timesteps')

import tempfile
from pathlib import Path

# Compare file sizes with and without original_data
try:
    with tempfile.TemporaryDirectory() as tmpdir:
        path_full = Path(tmpdir) / 'full.nc'
        path_small = Path(tmpdir) / 'small.nc'

        fs_clustered.to_netcdf(path_full, include_original_data=True)
        fs_clustered.to_netcdf(path_small, include_original_data=False)

        size_full = path_full.stat().st_size / 1024
        size_small = path_small.stat().st_size / 1024

    print(f'With original_data:    {size_full:.1f} KB')
    print(f'Without original_data: {size_small:.1f} KB')
    print(f'Size reduction: {(1 - size_small / size_full) * 100:.0f}%')
except OSError as e:
    print(f'Note: File size comparison skipped due to environment issue: {type(e).__name__}')