Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· openaiHugging Face Suzhen/CodeChat

my ensemble code from statsforecast import StatsForecast from statsforecast.models…

  1. User

    my ensemble code from statsforecast import StatsForecast from statsforecast.models import AutoARIMA, AutoETS, DynamicOptimizedTheta from statsforecast.utils import ConformalIntervals import numpy as np import polars as pl

    Polars option to display all rows

    pl.Config.set_tbl_rows(None)

    Initialize the models

    models = [ AutoARIMA(season_length=12), AutoETS(season_length=12), DynamicOptimizedTheta(season_length=12) ]

    Initialize the StatsForecast model

    sf = StatsForecast(models=models, freq='1w', n_jobs=-1)

    Perform cross-validation with a step size of 1 to mimic an expanding window

    crossvalidation_df = sf.cross_validation(df=y_cl4_filtered, h=2, step_size=1, n_windows=8, sort_df=True)

    Calculate the ensemble mean

    ensemble = crossvalidation_df[['AutoARIMA', 'AutoETS', 'DynamicOptimizedTheta']].mean(axis=1)

    Create a Series for the ensemble mean

    ensemble_series = pl.Series('Ensemble', ensemble)

    Add the ensemble mean as a new column to the DataFrame

    crossvalidation_df = crossvalidation_df.with_columns(ensemble_series)

    def wmape(y_true, y_pred): return np.abs(y_true - y_pred).sum() / np.abs(y_true).sum()

    Calculate the WMAPE for the ensemble model

    wmape_value = wmape(crossvalidation_df['y'], crossvalidation_df['Ensemble']) print('Average WMAPE for Ensemble: ', round(wmape_value, 4))

    Calculate the errors for the ensemble model

    errors = crossvalidation_df['y'] - crossvalidation_df['Ensemble']

    For an individual forecast

    individual_accuracy = 1 - (abs(crossvalidation_df['y'] - crossvalidation_df['Ensemble']) / crossvalidation_df['y']) individual_bias = (crossvalidation_df['Ensemble'] / crossvalidation_df['y']) - 1

    Add these calculations as new columns to DataFrame

    crossvalidation_df = crossvalidation_df.with_columns([ individual_accuracy.alias("individual_accuracy"), individual_bias.alias("individual_bias") ])

    Print the individual accuracy and bias for each week

    for row in crossvalidation_df.to_dicts(): id = row['unique_id'] date = row['ds'] accuracy = row['individual_accuracy'] bias = row['individual_bias'] print(f"{id}, {date}, Individual Accuracy: {accuracy:.4f}, Individual Bias: {bias:.4f}")

    For groups of forecasts

    group_accuracy = 1 - (errors.abs().sum() / crossvalidation_df['y'].sum()) group_bias = (crossvalidation_df['Ensemble'].sum() / crossvalidation_df['y'].sum()) - 1

    Print the average group accuracy and group bias over all folds for the ensemble model

    print('Average Group Accuracy: ', round(group_accuracy, 4)) print('Average Group Bias: ', round(group_bias, 4))

    Fit the models on the entire dataset

    sf.fit(y_cl4_fit_1)

    Instantiate the ConformalIntervals class

    prediction_intervals = ConformalIntervals()

    Generate 24 months forecasts

    forecasts_df = sf.forecast(h=52*2, prediction_intervals=prediction_intervals, level=[95], id_col='unique_id', sort_df=True)

    Apply the non-negative constraint to the forecasts of individual models

    forecasts_df = forecasts_df.with_columns([ pl.when(pl.col('AutoARIMA') < 0).then(0).otherwise(pl.col('AutoARIMA')).alias('AutoARIMA'), pl.when(pl.col('AutoETS') < 0).then(0).otherwise(pl.col('AutoETS')).alias('AutoETS'), pl.when(pl.col('DynamicOptimizedTheta') < 0).then(0).otherwise(pl.col('DynamicOptimizedTheta')).alias('DynamicOptimizedTheta'), ])

    Calculate the ensemble forecast

    ensemble_forecast = forecasts_df[['AutoARIMA', 'AutoETS', 'DynamicOptimizedTheta']].mean(axis=1)

    Calculate the lower and upper prediction intervals for the ensemble forecast

    ensemble_lo_95 = forecasts_df.select( [ pl.when(pl.col('AutoARIMA-lo-95') < 0).then(0).otherwise(pl.col('AutoARIMA-lo-95')).alias('AutoARIMA-lo-95'), pl.when(pl.col('AutoETS-lo-95') < 0).then(0).otherwise(pl.col('AutoETS-lo-95')).alias('AutoETS-lo-95'), pl.when(pl.col('DynamicOptimizedTheta-lo-95') < 0).then(0).otherwise(pl.col('DynamicOptimizedTheta-lo-95')).alias('DynamicOptimizedTheta-lo-95'), ] ).mean(axis=1) ensemble_hi_95 = forecasts_df[['AutoARIMA-hi-95', 'AutoETS-hi-95', 'DynamicOptimizedTheta-hi-95']].mean(axis=1)

    Create Series for the ensemble forecast and its prediction intervals

    ensemble_forecast_series = pl.Series('EnsembleForecast', ensemble_forecast) ensemble_lo_95_series = pl.Series('Ensemble-lo-95', ensemble_lo_95) ensemble_hi_95_series = pl.Series('Ensemble-hi-95', ensemble_hi_95)

    Add the ensemble forecast and its prediction intervals as new columns to the DataFrame

    forecasts_df = forecasts_df.with_columns([ensemble_forecast_series, ensemble_lo_95_series, ensemble_hi_95_series])

    Round the ensemble forecast and prediction intervals and convert to integer

    forecasts_df = forecasts_df.with_columns([ pl.col("EnsembleForecast").round().cast(pl.Int32), pl.col("Ensemble-lo-95").round().cast(pl.Int32), pl.col("Ensemble-hi-95").round().cast(pl.Int32) ])

    Split the unique_id concat into the original columns

    def split_unique_id(unique_id): parts = unique_id.split('_') return parts if len(parts) >= 4 else (parts + [None] * (4 - len(parts)))

    forecasts_df = ( forecasts_df .with_columns([ pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[0]).alias('MaterialID'), pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[1]).alias('SalesOrg'), pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[2]).alias('DistrChan'), pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[3]).alias('CL4'), ]) .drop('unique_id') )

    Rename ‘ds’ to ‘WeekDate’

    forecasts_df = forecasts_df.rename({'ds': 'WeekDate'})

    Reorder the columns

    forecasts_df = forecasts_df.select([ "MaterialID", "SalesOrg", "DistrChan", "CL4", "WeekDate", "EnsembleForecast", "Ensemble-lo-95", "Ensemble-hi-95", "AutoARIMA", "AutoARIMA-lo-95", "AutoARIMA-hi-95", "AutoETS", "AutoETS-lo-95", "AutoETS-hi-95", "DynamicOptimizedTheta", "DynamicOptimizedTheta-lo-95", "DynamicOptimizedTheta-hi-95" ])

    Create an empty list

    forecasts_list = []

    Append each row to the list

    for row in forecasts_df.to_dicts(): forecasts_list.append(row)

    Print the list

    for forecast in forecasts_list: print(forecast) you could see I set season_length = 12 for all 3 models which not good since these are the length of weeks for the series, length ┆ count │ │ --- ┆ --- │ │ u32 ┆ u32 │ ╞════════╪═══════╡ │ 1 ┆ 1942 │ │ 2 ┆ 357 │ │ 3 ┆ 157 │ │ 4 ┆ 107 │ │ 5 ┆ 74 │ │ 6 ┆ 40 │ │ 7 ┆ 48 │ │ 8 ┆ 37 │ │ 9 ┆ 39 │ │ 10 ┆ 47 │ │ 11 ┆ 54 │ │ 12 ┆ 36 │ │ 13 ┆ 35 │ │ 14 ┆ 43 │ │ 15 ┆ 47 │ │ 16 ┆ 45 │ │ 17 ┆ 36 │ │ 18 ┆ 37 │ │ 19 ┆ 51 │ │ 20 ┆ 35 │ │ 21 ┆ 41 │ │ 22 ┆ 29 │ │ 23 ┆ 26 │ │ 24 ┆ 33 │ │ 25 ┆ 35 │ │ 26 ┆ 41 │ │ 27 ┆ 39 │ │ 28 ┆ 34 │ │ 29 ┆ 37 │ │ 30 ┆ 31 │ │ 31 ┆ 32 │ │ 32 ┆ 26 │ │ 33 ┆ 30 │ │ 34 ┆ 22 │ │ 35 ┆ 39 │ │ 36 ┆ 32 │ │ 37 ┆ 32 │ │ 38 ┆ 33 │ │ 39 ┆ 37 │ │ 40 ┆ 34 │ │ 41 ┆ 24 │ │ 42 ┆ 22 │ │ 43 ┆ 17 │ │ 44 ┆ 18 │ │ 45 ┆ 13 │ │ 46 ┆ 9 │ │ 47 ┆ 18 │ │ 48 ┆ 15 │ │ 49 ┆ 17 │ │ 50 ┆ 12 │ │ 51 ┆ 15 │ │ 52 ┆ 10 │ │ 53 ┆ 11 │ │ 54 ┆ 6 │ │ 55 ┆ 9 │ │ 56 ┆ 7 │ │ 57 ┆ 11 │ │ 58 ┆ 11 │ │ 59 ┆ 9 │ │ 60 ┆ 13 │ │ 61 ┆ 14 │ │ 62 ┆ 7 │ │ 63 ┆ 5 │ │ 64 ┆ 3 │ │ 65 ┆ 6 │ │ 66 ┆ 6 │ │ 67 ┆ 5 │ │ 68 ┆ 11 │ │ 69 ┆ 7 │ │ 70 ┆ 4 │ │ 71 ┆ 2 │ │ 72 ┆ 4 │ │ 73 ┆ 3 │ │ 74 ┆ 1 │ └────────┴───────┘ data is weekly, we need to be able to follow the example here https://nixtlaverse.nixtla.io/statsforecast/docs/how-to-guides/generating_features.html and here https://nixtlaverse.nixtla.io/statsforecast/src/feature_engineering.html to be able to extract seasonality since right now ensemble accuracy is 50%, we need at least 60%, I'm thinking if I have seasonal column, I could feed that into the season_length for the model unique_id ds y trend seasonal 0 H1 1 605.0 501.350550 124.683643 1 H1 2 586.0 506.424549 87.115039 2 H1 3 586.0 511.453736 79.479564 3 H1 4 559.0 516.434474 45.616992 4 H1 5 511.0 521.362991 -10.940498

  2. Assistant

    To ensure accurate seasonality detection in your weekly data, you need to provide an appropriate season_length value to the models in StatsForecast. This value informs the models about the periodicity of your seasonality; for instance, a value of 52 would be used for weekly data that exhibits annual seasonality.

    You can leverage the feature engineering capabilities in StatsForecast to extract seasonality from your data. This process typically involves fitting models to individual time series and decomposing them into trend, seasonality, and residual components.

    The fourier_series() function in StatsForecast is particularly useful for generating periodic features that can be used by models to capture seasonality. The fourier_series() function requires two parameters: dates, which is the column representing time, and K, which is the number of Fourier pairs used to approximate the seasonality.

    To apply this in your code, you need to:

    • Determine the periodicity of your data (which seems to be weekly with potential annual seasonality).
    • Generate Fourier features that capture this seasonality.

    Here's how you might modify your code to include Fourier features for weekly data with an assumed annual seasonality. This is based on the assumption that you have a DataFrame or Series with dates from which you can extract these features.

    from statsforecast import fourier_series
    import pandas as pd
    
    # Assuming 'y_cl4_filtered' is your DataFrame and 'ds' represents the dates
    # Convert 'ds' to pandas datetime if not already
    y_cl4_filtered['ds'] = pd.to_datetime(y_cl4_filtered['ds'])
    
    # Generate Fourier features assuming annual seasonality for weekly data (K=5)
    # We use 5 pairs of sine/cosine functions as an example
    fourier_features = fourier_series(dates=y_cl4_filtered['ds'], K=5, period=365.25/7)
    
    # Add Fourier features back to the DataFrame for model training
    for i, (sin_col, cos_col) in enumerate(zip(fourier_features[::2], fourier_features[1::2]), start=1):
        y_cl4_filtered[f'fourier_sin_{i}'] = sin_col
        y_cl4_filtered[f'fourier_cos_{i}'] = cos_col
    
    # Now, 'y_cl4_filtered' includes the Fourier features as columns
    # ...
    
    # Fit your models using this DataFrame with added features
    # You might need to modify the model initialization and fitting procedures
    # to account for exogenous variables (the Fourier features)
    
    # Note: Depending on the StatsForecast version and changes, modifications might be needed.
    

    This example adds the generated Fourier features to your existing DataFrame. You would then need to adapt the subsequent fitting and forecasting steps to include these features.

    Please ensure you read the StatsForecast documentation for the specific implementations and requirements for using exogenous variables and feature generation functions.

    Also, the ideal Fourier series parameters may differ based on your specific data, so you may need to experiment with different values for K.

    After including these features, you'd pass them into your models' fitting and forecasting functions as exogenous variables (X), providing the models with additional information about the seasonal patterns. However, make sure your chosen models support exogenous variables; some automated forecasting models may handle seasonality internally and not require additional features.

  3. User

    I dont know where you get that from but I cant find fourier series in statsforecast, look at my message again, I was showing you this from statsforecast.feature_engineering import mstl_decomposition

  4. Assistant

    Apologies for the confusion. In the StatsForecast library, if fourier_series is not directly available or if you would like to use a different approach, you can use mstl_decomposition for seasonal decomposition which is a part of the utility functions for feature engineering in StatsForecast.

    The mstl_decomposition function can be used to decompose a time series into trend, seasonal, and remainder components. Using the seasonal component, you can identify the dominant seasonality in your data which can then be used to set the season_length parameter in your models.

    Here's how you could use the mstl_decomposition to extract the seasonal pattern and then compute the dominant seasonality from your data.

    First, you would need to apply the mstl_decomposition to each time series:

    from statsforecast.feature_engineering import mstl_decomposition
    import pandas as pd
    
    # Assuming `y_cl4_filtered` is your DataFrame and `ds` represents the dates
    # Convert 'ds' to pandas datetime if not already
    y_cl4_filtered['ds'] = pd.to_datetime(y_cl4_filtered['ds'])
    
    # Assuming `unique_id` is the column that identifies each time series
    # Decompose each time series to extract the seasonal part
    y_cl4_filtered['seasonal'] = y_cl4_filtered.groupby('unique_id').apply(
        lambda df: mstl_decomposition(df['y'], seasonal_period=52)
    )['seasonal']
    
    # Now you have the seasonal component in your data.
    

    Here, seasonal_period=52 is used since the data is weekly and you are considering annual seasonality, which means you believe the seasonality repeats every 52 weeks.

    After you have the seasonal component, you could visually inspect it, use a Fourier transform, or apply other methods to establish the strongest seasonal frequency. This would then inform the choice of season_length for your models.

    Please note that you may need to adjust the seasonal_period parameter based on what you observe in the seasonal component of your data, and you should ensure that this parameter matches the expected periodicity of your data.

    Finally, you'd use the obtained seasonal length in your models initialization, like so:

    # Assuming you have obtained an appropriate season_length
    season_length = 52  # or any number that you found as the dominant seasonality
    
    # Initialize the models with the discovered season_length
    models = [
        AutoARIMA(season_length=season_length),
        AutoETS(season_length=season_length),
        DynamicOptimizedTheta(season_length=season_length)
    ]
    
    # ... continue with model fitting and forecasting as before.
    

    Remember that extracting the dominant seasonality is an iterative process that might require domain knowledge and visual analysis to determine the appropriate season_length. Adjustments to this procedure might be needed to fit the specifics of your case and the features provided by the specific version of the StatsForecast library you are working with.

  5. User

    where are you reading those code? I gave you the website that has the example code freq = 1 season_length = 24 horizon = 2 * season_length valid = df.groupby('unique_id').tail(horizon) train = df.drop(valid.index) model = MSTL(season_length=24) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon) https://nixtlaverse.nixtla.io/statsforecast/docs/how-to-guides/generating_features.html

  6. Assistant

    You are correct, and I apologize for the oversight. The mstl_decomposition from the statsforecast library lets you decompose a time series and extract seasonal features that can be used for training forecasting models.

    Here is an example of how you can use mstl_decomposition based on the example you provided from the StatsForecast documentation. This approach involves using the MSTL class to decompose the time series and generate the relevant features.

    from statsforecast.feature_engineering import MSTL, mstl_decomposition
    import pandas as pd
    
    # Assuming df is your initial DataFrame with columns 'unique_id', 'ds' (for date), and 'y' (for target variable)
    # Make sure that the date column 'ds' is in a datetime format
    df['ds'] = pd.to_datetime(df['ds'])
    
    # Define the seasonal period and other parameters
    season_length = 52
    freq = 1 
    horizon = 2 * season_length
    
    # Split the data into training and validation sets based on the horizon
    valid = df.groupby('unique_id').tail(horizon)
    train = df.drop(valid.index)
    
    # Create the MSTL model with the appropriate season_length
    model = MSTL(season_length=season_length)
    
    # Generate features using the mstl_decomposition function
    transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)
    
    # X_df will contain the decomposed features including seasonal components that can be used for fitting your forecast models
    # transformed_df will contain the original data with the trend and seasonal components removed
    
    # Continue with your forecasting models using the generated features in X_df
    ...
    

    In the code snippet above, mstl_decomposition is used to decompose train, which is the training data. The MSTL model is initialized with the indicated season_length. The transformed_df contains the decomposed series where the trend and seasonal components are removed, and X_df contains the features extracted by the decomposition which can be used as regressors in your forecasting model.

    You can then train your forecasting models with the seasonal features. This should help in improving the accuracy of the model by providing it with information about the seasonality present in your data.

    Ensure that you adapt and incorporate this into your modeling workflow per the specific requirements of your forecasting task and with the latest functionalities provided by the StatsForecast library.

  7. User

    hold on, it says in the api mstl_decomposition (df:Union[pandas.core.frame.DataFrame,polars.datafram e.frame.DataFrame], model:statsforecast.models.MSTL, freq:str, h:int) so this also works with polars, how do we do this in polars freq = 1 season_length = 24 horizon = 2 * season_length valid = df.groupby('unique_id').tail(horizon) train = df.drop(valid.index) model = MSTL(season_length=24) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon) especially the train variable

  8. Assistant

    The mstl_decomposition function in StatsForecast can indeed work with both Pandas and Polars DataFrames. When working with Polars, you need to adjust your operations to use Polars' API, which has a slightly different syntax compared to Pandas.

    Here's how you would use mstl_decomposition in Polars:

    from statsforecast.feature_engineering import MSTL, mstl_decomposition
    import polars as pl
    
    # Assuming df is your initial Polars DataFrame with columns 'unique_id', 'ds' (for date), and 'y' (for target variable)
    # Make sure that the date column 'ds' is in a datetime format
    df = df.with_column(pl.col('ds').str.strptime(pl.Datetime))
    
    # Define the seasonal period and other parameters
    season_length = 52
    freq = 1 
    horizon = 2 * season_length
    
    # Split the data into training and validation sets based on the horizon
    # Use Polars expressions to replicate the Pandas tail and drop methods
    valid = df.groupby('unique_id').agg(pl.col('y').tail(horizon))
    train = df.join(valid, on=['unique_id', 'y'], how='anti')  # This is to drop rows that are in valid
    
    # Create the MSTL model with the appropriate season_length
    model = MSTL(season_length=season_length)
    
    # Generate features using the mstl_decomposition function with a Polars DataFrame
    transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)
    
    # X_df will now be a Polars DataFrame containing the decomposed features
    # You can continue with your forecasting models using the generated features
    

    Please bear in mind that the syntax for Polars is slightly different from that of Pandas:

    • Instead of using apply for transformations, Polars has a variety of built-in expression functions.
    • The join method can be used similarly to SQL joins, and here it is used to exclude rows in valid from df to create the train DataFrame.
    • The date column is parsed into datetime using Polars' str.strptime method.

    After obtaining the transformed_df and X_df with Polars DataFrames, you can proceed with your forecasting models as planned, using the decomposed features to improve model accuracy. Always check the documentation and API reference for the most current and appropriate methods when working with libraries like Polars and StatsForecast.

  9. User

    from statsforecast.feature_engineering import mstl_decomposition

    freq = 1 season_length = 52 horizon = 2 * season_length valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon)) train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') model = MSTL(season_length=season_length) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon) 6408 msg = f"expected other join table to be a DataFrame, got {type(other).name!r}" 6409 raise TypeError(msg) 6411 return ( -> 6412 self.lazy() 6413 .join( 6414 other=other.lazy(), 6415 left_on=left_on, 6416 right_on=right_on, 6417 on=on, 6418 how=how, 6419 suffix=suffix, 6420 validate=validate, 6421 join_nulls=join_nulls, 6422 ) 6423 .collect(_eager=True) 6424 )

    File ~/anaconda3/lib/python3.10/site-packages/polars/lazyframe/frame.py:1940, in LazyFrame.collect(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, no_optimization, streaming, background, _eager) 1937 if background: 1938 return InProcessQuery(ldf.collect_concurrently()) -> 1940 return wrap_df(ldf.collect())

    ComputeError: datatypes of join keys don't match - y: f64 on left does not match y: list[f64] on right

  10. Assistant

    The error that you're experiencing in Polars hints that during the join operation, the key columns have a type mismatch. To explain, while you're attempting to join on 'unique_id' and 'y', the 'valid' DataFrame has 'y' as a list due to the tail operation.

    In Polars, when you aggregate using tail (or similar functions), the resulting column is a list. To perform the anti-join you’re attempting (i.e., to exclude rows from y_cl4 that appear in valid), you would typically match on a "flat" column, not a list.

    Here's how you can fix the error:

    First, let's perform the tail operation correctly:

    # Perform the tail operation without aggregation to maintain flat vectors
    valid = y_cl4.sort('ds').groupby('unique_id').tail(horizon)
    

    Next, create a unique key on the valid DataFrame to perform the join:

    # Create a unique identifier for each row by combining 'unique_id' and 'ds'
    valid = valid.with_column((pl.col('unique_id') + pl.col('ds').cast(pl.Utf8)).alias('row_identifier'))
    train = y_cl4.with_column((pl.col('unique_id') + pl.col('ds').cast(pl.Utf8)).alias('row_identifier'))
    
    # Perform the anti join using the new unique key
    train = train.join(valid, on='row_identifier', how='anti').drop('row_identifier')
    

    With this approach, you create a unique identifier for each row based on the combined values of 'unique_id' and 'ds'. This ensures a proper anti join that excludes only the rows that appear in the valid DataFrame.

    Finally, you can continue with the MSTL decomposition:

    model = MSTL(season_length=season_length)
    transformed_df, X_df = mstl_decomposition(train.drop(['row_identifier']), model=model, freq=freq, h=horizon)
    

    Now, you should be able to use mstl_decomposition without errors. If after these modifications you still encounter issues, please let me know, and we'll troubleshoot further.

  11. User

    from statsforecast.models import MSTL from statsforecast.feature_engineering import mstl_decomposition

    freq = 1 season_length = 52 horizon = 2 * season_length valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon)) train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') model = MSTL(season_length=season_length) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon) --------------------------------------------------------------------------- ComputeError Traceback (most recent call last) Cell In[15], line 7 5 horizon = 2 * season_length 6 valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon)) ----> 7 train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') 8 model = MSTL(season_length=season_length) 9 transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)

    File ~/anaconda3/lib/python3.10/site-packages/polars/dataframe/frame.py:6412, in DataFrame.join(self, other, on, how, left_on, right_on, suffix, validate, join_nulls) 6408 msg = f"expected other join table to be a DataFrame, got {type(other).name!r}" 6409 raise TypeError(msg) 6411 return ( -> 6412 self.lazy() 6413 .join( 6414 other=other.lazy(), 6415 left_on=left_on, 6416 right_on=right_on, 6417 on=on, 6418 how=how, 6419 suffix=suffix, 6420 validate=validate, 6421 join_nulls=join_nulls, 6422 ) 6423 .collect(_eager=True) 6424 )

    File ~/anaconda3/lib/python3.10/site-packages/polars/lazyframe/frame.py:1940, in LazyFrame.collect(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, no_optimization, streaming, background, _eager) 1937 if background: 1938 return InProcessQuery(ldf.collect_concurrently()) -> 1940 return wrap_df(ldf.collect())

    ComputeError: datatypes of join keys don't match - y: f64 on left does not match y: list[f64] on right

  12. Assistant

    My apologies for the oversight in the previous responses. When you aggregate in Polars using functions like tail(), it indeed returns a list which is not suitable for join keys directly. Instead, we would like to keep a column as is without turning it into a list upon aggregation.

    Let's correct the logic to exclude the rows that are part of valid from y_cl4. We need to ensure we match rows with the same unique_id and the last horizon number of ds values per group.

    Here is how you can perform this operation in Polars, considering the unique_id and datetime ds:

    from statsforecast.models import MSTL
    from statsforecast.feature_engineering import mstl_decomposition
    
    # Assuming y_cl4 is your initial Polars DataFrame with columns 'unique_id', 'ds' and 'y'
    # where 'unique_id' represents each time series, 'ds' represents dates, and 'y' is the target variable.
    
    # Convert 'ds' to datetime if it is not already
    y_cl4 = y_cl4.with_column(pl.col('ds').str.strptime(pl.Datetime))
    
    # Define the seasonal period and other parameters
    season_length = 52
    freq = 1
    horizon = 2 * season_length
    
    # Sort the DataFrame by 'unique_id' and 'ds' and get the last 'horizon' rows for each 'unique_id'
    y_cl4_sorted = y_cl4.sort(['unique_id', 'ds'])
    valid_indices = y_cl4_sorted.groupby('unique_id').tail(horizon).select('ds').unique().sort('ds').row()
    
    # Exclude the rows that are part of `valid` from `y_cl4`
    train = y_cl4_sorted.filter(~pl.col('ds').is_in(valid_indices))
    
    # Now `train` contains the training data with the last `horizon` number of `ds` values per group excluded
    
    # Create the MSTL model with the appropriate season_length
    model = MSTL(season_length=season_length)
    
    # Generate features using the mstl_decomposition function
    transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)
    

    In this modified approach, after sorting y_cl4 by unique_id and ds, we get the unique dates of the last horizon number of rows for each unique_id. Then, we filter out rows from y_cl4 where ds matches these unique dates, which effectively creates a training set excluding the validation set.

    This should resolve the issue you're experiencing and allow you to continue with the MSTL decomposition and further model development.

  13. User

    why is there a sort for valid? y_cl4 is already sorted valid = df.groupby('unique_id').tail(horizon)

  14. Assistant

    You are correct; if y_cl4 is already sorted by 'unique_id' and 'ds', you don't need to sort it again to extract the validation set. However, it's crucial to handle the 'valid' DataFrame correctly because the issue arises from attempting to join using a list type in the 'y' column of 'valid'.

    Here’s how to properly create your training set by excluding the rows that are present in the validation set without causing the join to fail:

    from statsforecast.models import MSTL
    from statsforecast.feature_engineering import mstl_decomposition
    
    # Assuming y_cl4 is your initial Polars DataFrame with columns 'unique_id', and 'ds' (date), and 'y' (target variable)
    # Define the seasonal period and other parameters
    freq = 1
    season_length = 52
    horizon = 2 * season_length
    
    # Determine the row numbers to keep for validation. No need to sort as `y_cl4` is already sorted.
    row_nums = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon).alias('tail_rows'))
    
    # Flatten the list of row_nums using explode, and then join it back with the original DataFrame
    # to get a column indicating which rows are for validation.
    flat_valid_indicator = (
        row_nums.with_column(pl.arange(0, pl.count()).alias("row_num"))
        .explode("tail_rows")
        .with_column(pl.col("tail_rows").alias("y"))
        .select(["unique_id", "y", "row_num"])
    )
    
    # Join with the original DataFrame to mark the rows.
    y_cl4_marked = y_cl4.join(flat_valid_indicator, on=["unique_id", "y"], how="left")
    
    # Filter out the rows marked for validation.
    train = y_cl4_marked.filter(pl.col("row_num").is_null()).drop("row_num")
    
    # Create the MSTL model with the appropriate season_length
    model = MSTL(season_length=season_length)
    
    # Generate features using the mstl_decomposition function
    # Make sure to drop any extra columns you might have added for the join
    transformed_df, X_df = mstl_decomposition(train.drop("row_num"), model=model, freq=freq, h=horizon)
    

    In this approach, rather than using the join on a list, we:

    1. Create an indicator DataFrame flat_valid_indicator that contains individual rows for each 'y' we wish to use for validation, along with a 'row_num' column to uniquely identify each row.
    2. Join this indicator DataFrame with the original y_cl4 to mark the rows to keep for validation.
    3. Use this marked DataFrame to filter out the training set.
    4. Decompose using mstl_decomposition with the correct training set that excludes validation rows.
  15. User

    hold on, this is the original example freq = 1 season_length = 24 horizon = 2 * season_length valid = df.groupby('unique_id').tail(horizon) train = df.drop(valid.index) model = MSTL(season_length=24) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon), we only convert pandas to polars why are you changing everything for valid and train, just stick to the example and use polars instead

  16. Assistant

    Understood, let's translate the example from Pandas to Polars without changing the logic for creating valid and train. Note that Polars does not have an equivalent to Pandas's drop by index, so we'll need to implicitly do this operation differently.

    Here is how the example would be straightforwardly translated to Polars:

    from statsforecast.models import MSTL
    from statsforecast.feature_engineering import mstl_decomposition
    
    # Assuming y_cl4 is your initial Polars DataFrame with columns 'unique_id', 'ds' and 'y'
    # and that 'y_cl4' is already sorted by 'ds'.
    # Define the seasonal period and other parameters
    freq = 1
    season_length = 52
    horizon = 2 * season_length
    
    # Get the last 'horizon' number of observations for each 'unique_id' as your validation set.
    valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon).alias('valid_y'))
    
    # Create an exclusion flag in the original dataframe based on the 'valid' set.
    y_cl4_marked = y_cl4.join(
        valid,
        on='unique_id',
        how='left'
    )
    
    # Filter out those rows where 'y' is present in 'valid_y' to create the training set.
    train = y_cl4_marked.filter(~pl.col('y').is_in(valid['valid_y'])).select(y_cl4.columns)
    
    # Create the MSTL model with the appropriate season_length
    model = MSTL(season_length=season_length)
    
    # Generate features using the mstl_decomposition function
    transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)
    

    In this translation:

    • We use groupby and agg with tail to create the valid DataFrame, which would contain lists in the valid_y column.
    • We use join to flag rows in y_cl4 that correspond to the validation data.
    • We use filter to exclude validation rows from y_cl4 based on the flag created by the join, resulting in the train DataFrame.
    • We then proceed with mstl_decomposition.

    One important thing to note is that because Polars functions differently compared to Pandas regarding the relationship between grouped data and the original DataFrame, the final step with filter might not directly exclude the desired rows. If you find that this is the case, you will need to create a unique identifier for each row (combining 'unique_id' and 'ds') and use that for filtering, as I suggested in my previous messages.

  17. User

    hold on, this is what you gave me in the beginning from statsforecast.models import MSTL from statsforecast.feature_engineering import mstl_decomposition

    freq = 1 season_length = 52 horizon = 2 * season_length valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon)) train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') model = MSTL(season_length=season_length) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon) output for valid valid unique_id y str list[f64] "12522127_US01_… [484.0, 66.0, … 198.0] "12421246_US03_… [1.0] "90084048_US01_… [1.0] "12446749_US01_… [1008.0, 324.0, 126.0] "11004185_US01_… [48.0] "12421025_US01_… [1.0] "12497240_US01_… [75.0, 165.0, … 30.0] "12460984_US05_… [96.0] "12069355_US01_… [234.0, 156.0, … 52.0] "12522129_US01_… [450.0, 150.0, … 250.0] "12405540_US01_… [840.0, 1386.0, … 420.0] "12303754_US03_… [360.0, 360.0, … 180.0] … … "12505780_US01_… [96.0, 96.0, … 48.0] "12458826_US01_… [182.0, 196.0] "12522129_US01_… [25.0, 250.0, … 125.0] "12482932_US01_… [1.0] "12523899_US01_… [1200.0] "12511934_US01_… [375.0, 150.0, … 75.0] "12421371_US01_… [10.0] "12482459_US03_… [2.0] "12420784_US01_… [6.0, 3.0, 6.0] "12384547_US01_… [248.0] "12421374_US01_… [10.0] "12472039_US01_… [1.0]

  18. Assistant

    The output shows that valid is a Polars DataFrame with a list of the last horizon values of y for each unique_id. Because the join operation in Polars cannot operate directly on the list type, as described previously, my earlier response did not handle this correctly.

    To achieve the same result as the original Pandas example, you can use the anti join in Polars by directly excluding the validation set rows from your training set without encountering the join data type mismatch.

    Let's rework the code snippet, using an additional column for indexing the rows:

    from statsforecast.models import MSTL
    from statsforecast.feature_engineering import mstl_decomposition
    import polars as pl
    
    # Assuming y_cl4 is your initial Polars DataFrame with columns 'unique_id', 'ds' and 'y'
    # Define the seasonal period and other parameters
    freq = 1
    season_length = 52
    horizon = 2 * season_length
    
    # Assign row numbers, which are used to differentiate between rows with otherwise identical data
    y_cl4 = y_cl4.with_column(pl.arange(0, pl.count()).alias("row_num"))
    
    # Extract the rows to be used as a validation set, which are the last 'horizon' rows for each unique_id
    valid = y_cl4.groupby('unique_id').agg([
        pl.col('y').take(pl.col('y').arr.tail(horizon)).alias('valid_y'),
        pl.col('row_num').take(pl.col('row_num').arr.tail(horizon)).alias('valid_row_nums')
    ]).explode(['valid_y', 'valid_row_nums'])
    
    # Perform an anti-join to exclude the validation rows from the training dataset
    train = y_cl4.join(valid, on=['unique_id', 'row_num'], how='anti', suffix="_valid")\
                 .filter(pl.col("valid_row_nums").is_null())\
                 .drop("valid_row_nums")
    
    # Create the MSTL model with the appropriate season_length
    model = MSTL(season_length=season_length)
    
    # Generate features using the mstl_decomposition function
    transformed_df, X_df = mstl_decomposition(train.drop("row_num"), model=model, freq=freq, h=horizon)
    

    In the revised code:

    1. We obtain a row number for each row using pl.arange(). This acts as a unique identifier.
    2. We group by unique_id and use take with arr.tail() to get the last few rows' numbers (row_num) and their corresponding y values for validation.
    3. We explode both the valid_y and valid_row_nums columns, so they are no longer lists.
    4. We perform an anti-join with the row_num column, using a suffix to prevent column name clashes.
    5. We filter out any non-null valid_row_nums, since those are the validation rows.
    6. Finally, we clean up the DataFrame by dropping the valid_row_nums and auxiliary row_num before calling mstl_decomposition.

    This approach should correctly provide you with the training set you want to use for model fitting, excluding the validation rows.

  19. User

    okay hold on, I'm sure each unique_id is like unique for sure, dont have to create row number, again go back to this, stop making everything more complicated, from statsforecast.models import MSTL from statsforecast.feature_engineering import mstl_decomposition

    freq = 1 season_length = 52 horizon = 2 * season_length valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon)) train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') model = MSTL(season_length=season_length) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon) its saying 'y' having list[f64] instead of f64 unique_id y str list[f64] "12522127_US01_… [484.0, 66.0, … 198.0] "12421246_US03_… [1.0] "90084048_US01_… [1.0] "12446749_US01_… [1008.0, 324.0, 126.0] ----> 7 train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') 8 model = MSTL(season_length=season_length) 9 transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)

    File ~/anaconda3/lib/python3.10/site-packages/polars/dataframe/frame.py:6412, in DataFrame.join(self, other, on, how, left_on, right_on, suffix, validate, join_nulls) 6408 msg = f"expected other join table to be a DataFrame, got {type(other).name!r}" 6409 raise TypeError(msg) 6411 return ( -> 6412 self.lazy() 6413 .join( 6414 other=other.lazy(), 6415 left_on=left_on, 6416 right_on=right_on, 6417 on=on, 6418 how=how, 6419 suffix=suffix, 6420 validate=validate, 6421 join_nulls=join_nulls, 6422 ) 6423 .collect(_eager=True) 6424 )

    File ~/anaconda3/lib/python3.10/site-packages/polars/lazyframe/frame.py:1940, in LazyFrame.collect(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, no_optimization, streaming, background, _eager) 1937 if background: 1938 return InProcessQuery(ldf.collect_concurrently()) -> 1940 return wrap_df(ldf.collect())

    ComputeError: datatypes of join keys don't match - y: f64 on left does not match y: list[f64] on right just have to make sure 'y' in valid gonna be f64

  20. Assistant

    Understood, if you're certain that each unique_id is unique, we can exclude the rows belonging to the valid dataset without using auxiliary columns.

    Since we have a mismatch in data types preventing the join (due to valid['y'] being a list), we need to explode this list into individual rows before performing the anti-join.

    The following steps will ensure that valid['y'] is of data type f64 instead of list[f64]:

    from statsforecast.models import MSTL
    from statsforecast.feature_engineering import mstl_decomposition
    
    # Assuming y_cl4 is your initial Polars DataFrame with columns 'unique_id', 'ds' and 'y'
    # and that 'y_cl4' is already sorted by 'unique_id' and 'ds'.
    
    # Define the seasonal period and other parameters
    freq = 1
    season_length = 52
    horizon = 2 * season_length
    
    # Extract the rows to be used as a validation set, which are the last 'horizon' rows for each `unique_id`
    valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon).alias('y').list()).explode('y')
    
    # Perform an anti-join to exclude the validation rows from the training dataset
    train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti')
    
    # Create the MSTL model with the appropriate season_length
    model = MSTL(season_length=season_length)
    
    # Generate features using the mstl_decomposition function
    transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)
    

    In the code above:

    1. We group by unique_id and use tail to select the last horizon rows of y, which are then exploded into individual rows using .explode('y'). This ensures that valid will contain individual rows instead of lists, making its y columns of type f64.
    2. We then perform an anti-join on y_cl4 with valid which excludes rows from the y_cl4 DataFrame that are present in the valid DataFrame.
    3. Finally, we proceed with the MSTL decomposition on the resulting train DataFrame.

    This should yield the correct training dataset you can then use for decomposition and modeling without having to deal with data type mismatches in the key of the join operation.

  21. User

    when i make the valid code simialr to the example I got this errro from statsforecast.models import MSTL from statsforecast.feature_engineering import mstl_decomposition

    freq = '1w' season_length = 52 horizon = 2 * season_length valid = y_cl4.groupby('unique_id').tail(horizon) train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') model = MSTL(season_length=season_length) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon) ---> 10 transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)

    File ~/anaconda3/lib/python3.10/site-packages/statsforecast/feature_engineering.py:57, in mstl_decomposition(df, model, freq, h) 55 df = drop_index_if_pandas(df) 56 sf = StatsForecast(models=[model], freq=freq) ---> 57 sf.fit(df=df) 58 X_df = sf._make_future_df(h=h) 59 if isinstance(X_df, pd.DataFrame) and _id_as_idx():

    File ~/anaconda3/lib/python3.10/site-packages/statsforecast/core.py:650, in StatsForecast.fit(self, df, sort_df, prediction_intervals, id_col, time_col, target_col) 646 self.fitted = self.ga.fit( 647 models=self.models, fallback_model=self.fallback_model 648 ) 649 else: --> 650 self.fitted_ = self._fit_parallel() 651 return self

    File ~/anaconda3/lib/python3.10/site-packages/statsforecast/core.py:1114, in _StatsForecast._fit_parallel(self) 1113 def _fit_parallel(self): -> 1114 gas = self.ga.split(self.n_jobs) 1115 Pool, pool_kwargs = self._get_pool() 1116 with Pool(self.n_jobs, **pool_kwargs) as executor:

    File ~/anaconda3/lib/python3.10/site-packages/statsforecast/core.py:379, in GroupedArray.split(self, n_chunks) 376 def split(self, n_chunks): 377 n_chunks = min(n_chunks, self.n_groups) 378 return [ --> 379 self.take(idxs) for idxs in np.array_split(range(self.n_groups), n_chunks) 380 ]

    File ~/anaconda3/lib/python3.10/site-packages/numpy/lib/shape_base.py:770, in array_split(ary, indices_or_sections, axis) 768 Nsections = int(indices_or_sections) 769 if Nsections <= 0: --> 770 raise ValueError('number sections must be larger than 0.') from None 771 Neach_section, extras = divmod(Ntotal, Nsections) 772 section_sizes = ([0] + 773 extras * [Neach_section+1] + 774 (Nsections-extras) * [Neach_section])

    ValueError: number sections must be larger than 0.

  22. Assistant

    The error you're encountering, ValueError: number sections must be larger than 0, suggests that at some stage of the mstl_decomposition process, your dataset is being split into chunks, but the number of groups or chunks calculated ends up being zero or negative. This could happen if your train DataFrame ends up being empty after the join operation.

    Let's ensure the train DataFrame is not empty after the anti-join.

    from statsforecast.models import MSTL
    from statsforecast.feature_engineering import mstl_decomposition
    import polars as pl
    
    freq = '1W'
    season_length = 52
    horizon = 2 * season_length
    
    # Select the last 'horizon' number of points for each unique_id
    valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon).alias('tail_y')).explode('tail_y')
    
    # Perform an anti-join to create the train DataFrame
    # We specify the DataFrame valid and the keys for the join
    train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti')
    
    # Check that train is not an empty DataFrame
    if train.height == 0:
        raise ValueError("The training dataset is empty after the join operation.")
    
    model = MSTL(season_length=season_length)
    transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)
    

    Notes:

    1. An anti-join is being performed on columns unique_id and y. This should remove all rows in y_cl4 that have a matching y value under the same unique_id in the valid DataFrame.
    2. Be sure y_cl4 and valid DataFrames are using compatible data types for the join operation (unique_id and y columns).
    3. Before calling mstl_decomposition, we check if train is non-empty.

    If train ends up empty, you will need to debug the code to understand why the anti-join is removing all rows. It might be an indicator that all rows of y_cl4 are being matched in valid, which would signify a logic issue.

    If required, you can print or inspect the size of valid and train DataFrames to make sure the anti-join operation is performing as expected and train retains data for fitting the models.

  23. User

    okay go back tot he example freq = 1 season_length = 24 horizon = 2 * season_length valid = df.groupby('unique_id').tail(horizon) train = df.drop(valid.index) model = MSTL(season_length=24) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon), this is out code using polars instead from statsforecast.models import MSTL from statsforecast.feature_engineering import mstl_decomposition

    freq = '1w' season_length = 52 horizon = 2 * season_length valid = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon)) train = y_cl4.join(valid, on=['unique_id', 'y'], how='anti') model = MSTL(season_length=season_length) transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon), this is the expected output unique_id ds y trend seasonal 0 H1 1 605.0 501.350550 124.683643 1 H1 2 586.0 506.424549 87.115039 2 H1 3 586.0 511.453736 79.479564 , this is the output for valid unique_id y str list[f64] "12420725_US01_… [1.0] "12505783_US01_… [9.0, 12.0, … 48.0] "12513091_US01_… [2.0, 1.0] "12408050_US01_… [2.0, 2.0] "12486859_US01_… [105.0]

  24. Assistant

    Given the example you provided and the valid output you've shown, the issue seems to stem from the aggregation in Polars, which creates a list rather than individual rows for the values of y. As Polars does not have a direct equivalent to the Pandas drop function using index labels, we should use an alternative approach using Polars operations to achieve the same result.

    We can fix this by first exploding the list created by tail into individual rows before performing the join operation:

    from statsforecast.models import MSTL
    from statsforecast.feature_engineering import mstl_decomposition
    import polars as pl
    
    freq = '1W'  # weekly frequency
    season_length = 52
    horizon = 2 * season_length
    
    # Get the last `horizon` rows for each `unique_id`
    valid_idx = y_cl4.groupby('unique_id').agg(pl.col('y').tail(horizon).alias('tail_y'))
    valid_idx = valid_idx.with_column(pl.explode(pl.col("tail_y")))
    
    # An anti join in Polars to exclude the rows present in valid
    train = y_cl4.join(valid_idx, on=['unique_id', 'tail_y'], how='anti').drop(['tail_y'])
    
    model = MSTL(season_length=season_length)
    transformed_df, X_df = mstl_decomposition(train, model=model, freq=freq, h=horizon)
    

    In the revised code:

    • We adjust the valid_idx DataFrame by exploding tail_y to ensure that we promote the lists to individual rows, which are then used as a join key.
    • We perform an anti-join between y_cl4 and valid_idx DataFrames, making sure that rows with values in tail_y are removed from y_cl4. This valid_idx now has the same structure as y_cl4 so, the join will remove y_cl4 rows that match the y values in tail_y under the same unique_id.
    • I dropped the tail_y column after joining to clean up the train DataFrame and preserve only the original columns.
    • Finally, we can go ahead with the mstl_decomposition.

    Please ensure that after the join, the train dataframe is not empty, and contains the data excluding the rows that were selected for validation within valid_idx.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.

my ensemble code from statsforecast import StatsForecast… · The Chat Archive