Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· openaiHugging Face Suzhen/CodeChat

from statsforecast import StatsForecast from statsforecast.models import AutoARIMA,…

  1. User

    from statsforecast import StatsForecast from statsforecast.models import AutoARIMA, AutoETS, DynamicOptimizedTheta from statsforecast.utils import ConformalIntervals import numpy as np import polars as pl

    Polars option to display all rows

    pl.Config.set_tbl_rows(None)

    Initialize the models

    models = [ AutoARIMA(season_length=12), AutoETS(season_length=12), DynamicOptimizedTheta(season_length=12) ]

    Initialize the StatsForecast model

    sf = StatsForecast(models=models, freq='1w', n_jobs=-1)

    Perform cross-validation with a step size of 1 to mimic an expanding window

    crossvalidation_df = sf.cross_validation(df=y_cl4_filtered, h=2, step_size=1, n_windows=8, sort_df=True)

    Calculate the ensemble mean

    ensemble = crossvalidation_df[['AutoARIMA', 'AutoETS', 'DynamicOptimizedTheta']].mean(axis=1)

    Create a Series for the ensemble mean

    ensemble_series = pl.Series('Ensemble', ensemble)

    Add the ensemble mean as a new column to the DataFrame

    crossvalidation_df = crossvalidation_df.with_columns(ensemble_series)

    def wmape(y_true, y_pred): return np.abs(y_true - y_pred).sum() / np.abs(y_true).sum()

    Calculate the WMAPE for the ensemble model

    wmape_value = wmape(crossvalidation_df['y'], crossvalidation_df['Ensemble']) print('Average WMAPE for Ensemble: ', round(wmape_value, 4))

    Calculate the errors for the ensemble model

    errors = crossvalidation_df['y'] - crossvalidation_df['Ensemble']

    For an individual forecast

    individual_accuracy = 1 - (abs(crossvalidation_df['y'] - crossvalidation_df['Ensemble']) / crossvalidation_df['y']) individual_bias = (crossvalidation_df['Ensemble'] / crossvalidation_df['y']) - 1

    Add these calculations as new columns to DataFrame

    crossvalidation_df = crossvalidation_df.with_columns([ individual_accuracy.alias("individual_accuracy"), individual_bias.alias("individual_bias") ])

    Print the individual accuracy and bias for each week

    for row in crossvalidation_df.to_dicts(): id = row['unique_id'] date = row['ds'] accuracy = row['individual_accuracy'] bias = row['individual_bias'] print(f"{id}, {date}, Individual Accuracy: {accuracy:.4f}, Individual Bias: {bias:.4f}")

    For groups of forecasts

    group_accuracy = 1 - (errors.abs().sum() / crossvalidation_df['y'].sum()) group_bias = (crossvalidation_df['Ensemble'].sum() / crossvalidation_df['y'].sum()) - 1

    Print the average group accuracy and group bias over all folds for the ensemble model

    print('Average Group Accuracy: ', round(group_accuracy, 4)) print('Average Group Bias: ', round(group_bias, 4))

    Fit the models on the entire dataset

    sf.fit(y_cl4_fit_1)

    Instantiate the ConformalIntervals class

    prediction_intervals = ConformalIntervals()

    Generate 24 months forecasts

    forecasts_df = sf.forecast(h=52*2, prediction_intervals=prediction_intervals, level=[95], id_col='unique_id', sort_df=True)

    Apply the non-negative constraint to the forecasts of individual models

    forecasts_df = forecasts_df.with_columns([ pl.when(pl.col('AutoARIMA') < 0).then(0).otherwise(pl.col('AutoARIMA')).alias('AutoARIMA'), pl.when(pl.col('AutoETS') < 0).then(0).otherwise(pl.col('AutoETS')).alias('AutoETS'), pl.when(pl.col('DynamicOptimizedTheta') < 0).then(0).otherwise(pl.col('DynamicOptimizedTheta')).alias('DynamicOptimizedTheta'), ])

    Calculate the ensemble forecast

    ensemble_forecast = forecasts_df[['AutoARIMA', 'AutoETS', 'DynamicOptimizedTheta']].mean(axis=1)

    Calculate the lower and upper prediction intervals for the ensemble forecast

    ensemble_lo_95 = forecasts_df.select( [ pl.when(pl.col('AutoARIMA-lo-95') < 0).then(0).otherwise(pl.col('AutoARIMA-lo-95')).alias('AutoARIMA-lo-95'), pl.when(pl.col('AutoETS-lo-95') < 0).then(0).otherwise(pl.col('AutoETS-lo-95')).alias('AutoETS-lo-95'), pl.when(pl.col('DynamicOptimizedTheta-lo-95') < 0).then(0).otherwise(pl.col('DynamicOptimizedTheta-lo-95')).alias('DynamicOptimizedTheta-lo-95'), ] ).mean(axis=1) ensemble_hi_95 = forecasts_df[['AutoARIMA-hi-95', 'AutoETS-hi-95', 'DynamicOptimizedTheta-hi-95']].mean(axis=1)

    Create Series for the ensemble forecast and its prediction intervals

    ensemble_forecast_series = pl.Series('EnsembleForecast', ensemble_forecast) ensemble_lo_95_series = pl.Series('Ensemble-lo-95', ensemble_lo_95) ensemble_hi_95_series = pl.Series('Ensemble-hi-95', ensemble_hi_95)

    Add the ensemble forecast and its prediction intervals as new columns to the DataFrame

    forecasts_df = forecasts_df.with_columns([ensemble_forecast_series, ensemble_lo_95_series, ensemble_hi_95_series])

    Round the ensemble forecast and prediction intervals and convert to integer

    forecasts_df = forecasts_df.with_columns([ pl.col("EnsembleForecast").round().cast(pl.Int32), pl.col("Ensemble-lo-95").round().cast(pl.Int32), pl.col("Ensemble-hi-95").round().cast(pl.Int32) ])

    Split the unique_id concat into the original columns

    forecasts_df = ( forecasts_df .with_columns(pl.col('unique_id').str.split('_').alias('id_list'))
    .with_columns(pl.col('id_list').arr.get(0).alias('MaterialID'))
    .with_columns(pl.col('id_list').arr.get(1).alias('SalesOrg'))
    .with_columns(pl.col('id_list').arr.get(2).alias('DistrChan')) .with_columns(pl.col('id_list').arr.get(3).alias('CL4'))
    .drop('id_list')
    .drop('unique_id')
    )

    Rename ‘ds’ to ‘WeekDate’

    forecasts_df = forecasts_df.rename({'ds': 'WeekDate'})

    Reorder the columns

    forecasts_df = forecasts_df.select([ "MaterialID", "SalesOrg", "DistrChan", "CL4", "WeekDate", "EnsembleForecast", "Ensemble-lo-95", "Ensemble-hi-95", "AutoARIMA", "AutoARIMA-lo-95", "AutoARIMA-hi-95", "AutoETS", "AutoETS-lo-95", "AutoETS-hi-95", "DynamicOptimizedTheta", "DynamicOptimizedTheta-lo-95", "DynamicOptimizedTheta-hi-95" ])

    Create an empty list

    forecasts_list = []

    Append each row to the list

    for row in forecasts_df.to_dicts(): forecasts_list.append(row)

    Print the list

    for forecast in forecasts_list: print(forecast) --------------------------------------------------------------------------- SchemaError Traceback (most recent call last) Cell In[14], line 117 107 forecasts_df = forecasts_df.with_columns([ 108 pl.col("EnsembleForecast").round().cast(pl.Int32), 109 pl.col("Ensemble-lo-95").round().cast(pl.Int32), 110 pl.col("Ensemble-hi-95").round().cast(pl.Int32) 111 ]) 113 # Split the unique_id concat into the original columns 114 forecasts_df = ( 115 forecasts_df 116 .with_columns(pl.col('unique_id').str.split('_').alias('id_list'))
    --> 117 .with_columns(pl.col('id_list').arr.get(0).alias('MaterialID'))
    118 .with_columns(pl.col('id_list').arr.get(1).alias('SalesOrg'))
    119 .with_columns(pl.col('id_list').arr.get(2).alias('DistrChan')) 120 .with_columns(pl.col('id_list').arr.get(3).alias('CL4'))
    121 .drop('id_list')
    122 .drop('unique_id')
    123 ) 125 # Rename ‘ds’ to ‘WeekDate’ 126 forecasts_df = forecasts_df.rename({'ds': 'WeekDate'})

    File ~/anaconda3/lib/python3.10/site-packages/polars/dataframe/frame.py:8315, in DataFrame.with_columns(self, *exprs, **named_exprs) 8169 def with_columns( 8170 self, 8171 *exprs: IntoExpr | Iterable[IntoExpr], 8172 **named_exprs: IntoExpr, 8173 ) -> DataFrame: 8174 """ 8175 Add columns to this DataFrame. 8176 (...) 8313 └─────┴──────┴─────────────┘ 8314 """ -> 8315 return self.lazy().with_columns(*exprs, **named_exprs).collect(_eager=True)

    File ~/anaconda3/lib/python3.10/site-packages/polars/lazyframe/frame.py:1940, in LazyFrame.collect(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, no_optimization, streaming, background, _eager) 1937 if background: 1938 return InProcessQuery(ldf.collect_concurrently()) -> 1940 return wrap_df(ldf.collect())

    SchemaError: invalid series dtype: expected FixedSizeList, got list[str]

  2. Assistant

    It looks like there is a SchemaError occurring because you're trying to use .str.split('_') on a column named 'unique_id', which seems to be either not present in the dataframe forecasts_df at that point or not formatted as expected. The error message indicates that the split operation is resulting in a list of strings (Python list) which Polars recognizes as FixedSizeList, and it seems to have a problem with that.

    To debug this, let's step through the code where the error occurs:

    The .with_columns(pl.col('unique_id').str.split('_').alias('id_list')) is where you are splitting the 'unique_id' value and creating a new column 'id_list' containing lists of strings. The pl.col('id_list').arr.get(0) and similar statements are attempting to retrieve elements from those lists, but the error is indicating that the expected result type is not being met.

    Here's what might be going wrong:

    1. The column 'unique_id' might not exist in the forecasts_df DataFrame at the time you're trying to split it.
    2. The column 'unique_id' exists but is not a string type suitable for splitting.
    3. You are trying to fetch list elements using arr.get(i), but the split result is not what Polars expects.

    To resolve the issue, make sure the 'unique_id' column exists in the DataFrame and is of the correct string type before splitting. Also, ensure that the DataFrame you are working with is properly sorted and has the expected columns.

    Here's what you can do to fix the code:

    • Check if 'unique_id' column exists and make sure it is a string type that can be split.
    • If 'unique_id' does not exist at that point, track back in the code to find where it should have been created or why it is missing.
    • Make sure the split operation is giving the expected output.

    You may need to comment out the problematic parts of the code, and instead add debug statements to print out the schema or contents of forecasts_df just before the error occurs.

    Example of debugging code:

    print(forecasts_df.schema)
    print(forecasts_df.get_column_names())
    

    Ensure the 'unique_id' column is present and then continue with the splitting operation.

  3. User

    forecasts_df unique_id ds AutoARIMA AutoARIMA-lo-95 AutoARIMA-hi-95 AutoETS AutoETS-lo-95 AutoETS-hi-95 DynamicOptimizedTheta DynamicOptimizedTheta-lo-95 DynamicOptimizedTheta-hi-95 EnsembleForecast Ensemble-lo-95 Ensemble-hi-95 str datetime[μs] f32 f32 f32 f32 f32 f32 f32 f32 f32 i32 i32 i32 "11000263_US01_… 2023-11-13 00:00:00 17.284355 -5.480006 40.048717 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2023-11-20 00:00:00 17.284351 -5.480011 40.04871 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2023-11-27 00:00:00 17.284346 -5.480016 40.048706 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2023-12-04 00:00:00 17.28434 -5.480021 40.048702 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2023-12-11 00:00:00 17.284336 -5.480026 40.048698 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2023-12-18 00:00:00 17.28433 -5.480031 40.048691 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2023-12-25 00:00:00 17.284325 -5.480036 40.048687 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2024-01-01 00:00:00 17.284321 -5.480041 40.048683 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2024-01-08 00:00:00 17.284315 -5.480046 40.048676 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2024-01-15 00:00:00 17.284311 -5.480051 40.048672 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 "11000263_US01_… 2024-01-22 00:00:00 17.284306 -5.480056 40.048668 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212

  4. Assistant

    The schema output suggests that there is a column unique_id in forecasts_df, but for some reason, the split operation you are using is resulting in a FixedSizeList which is causing issues.

    Polars uses its own List/Array types, and when splitting a string column, the results can sometimes differ from what you might expect if you are assuming Python list behaviors.

    Let's try to fix the error using Polars properly:

    1. Instead of trying to split the 'unique_id' column into lists and then applying arr.get(i) on those lists (which seems to be causing the issue), we'll split the 'unique_id' and then immediately take the elements to create new columns.

    Here's the corrected code block:

    # Split the 'unique_id' column into separate columns by splitting on the underscore '_'
    forecasts_df = (
        forecasts_df
        .with_columns([
            pl.col('unique_id').str.split("_").get(0).alias('MaterialID'),
            pl.col('unique_id').str.split("_").get(1).alias('SalesOrg'),
            pl.col('unique_id').str.split("_").get(2).alias('DistrChan'),
            pl.col('unique_id').str.split("_").get(3).alias('CL4')
        ])
        .drop('unique_id')  # If you no longer need the 'unique_id' you can drop it
    )
    
    # Rename ‘ds’ to ‘WeekDate’
    forecasts_df = forecasts_df.rename({'ds': 'WeekDate'})
    
    # Reorder the columns as you prefer
    forecasts_df = forecasts_df.select([
        "MaterialID",
        "SalesOrg",
        "DistrChan", 
        "CL4",
        "WeekDate",
        "EnsembleForecast", 
        "Ensemble-lo-95", 
        "Ensemble-hi-95",
        "AutoARIMA", 
        "AutoARIMA-lo-95", 
        "AutoARIMA-hi-95", 
        "AutoETS", 
        "AutoETS-lo-95", 
        "AutoETS-hi-95", 
        "DynamicOptimizedTheta", 
        "DynamicOptimizedTheta-lo-95", 
        "DynamicOptimizedTheta-hi-95"
    ]) 
    
    # ... continue with the rest of your code ...
    

    This change avoids the creation of a Python list that Polars doesn't know how to handle with its arr.get() method, and instead assigns each split element directly as a new column.

    When working with Polars, it's important to make use of chainable methods rather than relying on list-like behavior that you might expect when working with regular Python lists; Polars operates on its own set of types and methods for efficiency and optimization reasons.

  5. User

    forecasts_df ds AutoARIMA AutoARIMA-lo-95 AutoARIMA-hi-95 AutoETS AutoETS-lo-95 AutoETS-hi-95 DynamicOptimizedTheta DynamicOptimizedTheta-lo-95 DynamicOptimizedTheta-hi-95 EnsembleForecast Ensemble-lo-95 Ensemble-hi-95 MaterialID SalesOrg DistrChan CL4 datetime[μs] f32 f32 f32 f32 f32 f32 f32 f32 f32 i32 i32 i32 list[str] list[str] list[str] list[str] 2023-11-13 00:00:00 17.284355 -5.480006 40.048717 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-11-20 00:00:00 17.284351 -5.480011 40.04871 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-11-27 00:00:00 17.284346 -5.480016 40.048706 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-12-04 00:00:00 17.28434 -5.480021 40.048702 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-12-11 00:00:00 17.284336 -5.480026 40.048698 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-12-18 00:00:00 17.28433 -5.480031 40.048691 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"]

  6. Assistant

    It seems that even after the split operation, the values in the columns MaterialID, SalesOrg, DistrChan, and CL4 are being stored as lists, not strings. This typically happens if str.split() operates on a column that has been grouped or if the .alias('MaterialID') and similar patterns follow a previous str.split() call and aren't directly chained.

    Let's try this approach to ensure that we extract the first element after splitting each time:

    # Split the 'unique_id' column and directly access the elements to create new columns
    forecasts_df = (
        forecasts_df
        .with_columns([
            pl.col('unique_id').str.split("_").arr.get(0).alias('MaterialID'),
            pl.col('unique_id').str.split("_").arr.get(1).alias('SalesOrg'),
            pl.col('unique_id').str.split("_").arr.get(2).alias('DistrChan'),
            pl.col('unique_id').str.split("_").arr.get(3).alias('CL4')
        ])
        .drop('unique_id')
    )
    
    # Rename ‘ds’ to ‘WeekDate’
    forecasts_df = forecasts_df.rename({'ds': 'WeekDate'})
    
    # Reorder the columns as you prefer
    forecasts_df = forecasts_df.select([
        "MaterialID",
        "SalesOrg",
        "DistrChan", 
        "CL4",
        "WeekDate",
        # ... (rest of your columns)
    ])
    

    Make sure the unique_id column exists when you perform the splitting. The arr.get(i) call will correctly retrieve the element at index i from each of the resulting lists after splitting the string.

    If you still encounter issues, it may be helpful to print out the forecasts_df DataFrame after the split and before the additional column extractions to ensure the split is resulting in the expected lists.

    Also, please note that if forecasts_df is being updated from an external source or due to previous operations that weren't successful (which might leave the DataFrame in an unexpected state), you would need to ensure that those operations are correct before this step.

  7. User

    error 8 pl.col('unique_id').str.split("_").arr.get(3).alias('CL4') 9 ]) 10 .drop('unique_id') 11 )

    File ~/anaconda3/lib/python3.10/site-packages/polars/dataframe/frame.py:8315, in DataFrame.with_columns(self, *exprs, **named_exprs) 8169 def with_columns( 8170 self, 8171 *exprs: IntoExpr | Iterable[IntoExpr], 8172 **named_exprs: IntoExpr, 8173 ) -> DataFrame: 8174 """ 8175 Add columns to this DataFrame. 8176 (...) 8313 └─────┴──────┴─────────────┘ 8314 """ -> 8315 return self.lazy().with_columns(*exprs, **named_exprs).collect(_eager=True)

    File ~/anaconda3/lib/python3.10/site-packages/polars/lazyframe/frame.py:1940, in LazyFrame.collect(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, no_optimization, streaming, background, _eager) 1937 if background: 1938 return InProcessQuery(ldf.collect_concurrently()) -> 1940 return wrap_df(ldf.collect())

    SchemaError: invalid series dtype: expected FixedSizeList, got list[str] , this one didnt get error forecasts_df = ( forecasts_df .with_columns([ pl.col(‘unique_id’).str.split("“).get(0).alias(‘MaterialID’), pl.col(‘unique_id’).str.split(”“).get(1).alias(‘SalesOrg’), pl.col(‘unique_id’).str.split(”“).get(2).alias(‘DistrChan’), pl.col(‘unique_id’).str.split(”").get(3).alias(‘CL4’) ]) .drop(‘unique_id’) # If you no longer need the ‘unique_id’ you can drop it ) but it shows materialID, salesorg, distrchan, cl4 all have the same value "11000263_US01…

  8. Assistant

    It looks like the code snippet you provided that did not cause an error uses the wrong quote characters (‘’). This might have happened due to copying and pasting code from a document that automatically converts straight quotes into curly quotes. Please ensure you are using the standard single quotes (') or double quotes (") in your code.

    Additionally, the ellipsis in the unique_id column values ("11000263_US01_…") suggests that the value might be truncated or not properly displayed. Ensure that the unique_id column contains the complete and correct unique identifiers composed of the parts you want to split into separate columns.

    Here's how you can proceed with the corrected code:

    # Correcting the quote characters
    forecasts_df = (
        forecasts_df
        .with_columns([
            pl.col('unique_id').str.split("_").arr.get(0).alias('MaterialID'),
            pl.col('unique_id').str.split("_").arr.get(1).alias('SalesOrg'),
            pl.col('unique_id').str.split("_").arr.get(2).alias('DistrChan'),
            pl.col('unique_id').str.split("_").arr.get(3).alias('CL4')
        ])
        .drop('unique_id')  # If you no longer need the 'unique_id' you can drop it
    )
    
    # Rename 'ds' to 'WeekDate'
    forecasts_df = forecasts_df.rename({'ds': 'WeekDate'})
    
    # ... rest of your code ...
    

    Make sure that the unique_id is complete and properly formatted, and that all operations are using the correct quote characters.

    If the unique_id values are still appearing as truncated in the output (which is indicated by the ellipsis "…"), you will need to inspect the unique_id values to make sure they are correct. You can print the first few unique_id values before the splitting operation to verify:

    print(forecasts_df[:, 'unique_id'].head())
    

    Once you are sure the unique_id column has the proper format, the split operation should work correctly, and you should be able to extract the desired components into the new columns. If you still face problems, please provide the exact error message, and I’ll be glad to assist you further.

  9. User

    I told the arr.get got error , this one working # Split the unique_id concat into the original columns forecasts_df2 = ( forecasts_df .with_columns([ pl.col('unique_id').str.split("").get(0).alias('MaterialID'), pl.col('unique_id').str.split("").get(1).alias('SalesOrg'), pl.col('unique_id').str.split("").get(2).alias('DistrChan'), pl.col('unique_id').str.split("").get(3).alias('CL4') ]) .drop('unique_id') ) forecasts_df2 but the resulr MaterialID SalesOrg DistrChan CL4 datetime[μs] f32 f32 f32 f32 f32 f32 f32 f32 f32 i32 i32 i32 list[str] list[str] list[str] list[str] 2023-11-13 00:00:00 17.284355 -5.480006 40.048717 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-11-20 00:00:00 17.284351 -5.480011 40.04871 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-11-27 00:00:00 17.284346 -5.480016 40.048706 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"]

  10. Assistant

    Thank you for the clarification. If using .get(i) is working for you without error, then it's the correct approach to use with polars, and my previous suggestions were based on the assumption that .get(i) was not working.

    Now, considering that .get(i) is working but the resulting columns are of type list[str] instead of simply str, we'll need to adjust the code to extract the string elements from these lists.

    Polars expects a str type when accessing elements of a list within a DataFrame column but if .get(i) is providing a list you would need to convert the lists to their first elements. You can use the .apply() method of Series to extract the first element of each list like this:

    # Splitting the 'unique_id' column and extracting the individual components
    def extract_first_element(lst):
        return lst[0] if lst else None  # Return the first element, or None if the list is empty
    
    forecasts_df = (
        forecasts_df
        .with_columns([
            pl.col('unique_id').str.split("_").arr.get(0).apply(extract_first_element).alias('MaterialID'),
            pl.col('unique_id').str.split("_").arr.get(1).apply(extract_first_element).alias('SalesOrg'),
            pl.col('unique_id').str.split("_").arr.get(2).apply(extract_first_element).alias('DistrChan'),
            pl.col('unique_id').str.split("_").arr.get(3).apply(extract_first_element).alias('CL4'),
        ])
        .drop('unique_id')  # If you no longer need the 'unique_id' you can drop it
    )
    
    # ... rest of your code ...
    

    With the .apply(extract_first_element) method chained after .arr.get(i), the lists should be converted to individual string elements as expected. Make sure to replace unique_id with the correct column name or identifier where the split should be applied.

    Keep in mind that Polars is under active development, and its API may be updated, so make sure to check the Polars documentation for the most current methods for performing operations like this.

  11. User

    Splitting the ‘unique_id’ column and extracting the individual components

    def extract_first_element(lst): return lst[0] if lst else None # Return the first element, or None if the list is empty

    forecasts_df3 = ( forecasts_df .with_columns([ pl.col('unique_id').str.split("").arr.get(0).apply(extract_first_element).alias('MaterialID'), pl.col('unique_id').str.split("").arr.get(1).apply(extract_first_element).alias('SalesOrg'), pl.col('unique_id').str.split("").arr.get(2).apply(extract_first_element).alias('DistrChan'), pl.col('unique_id').str.split("").arr.get(3).apply(extract_first_element).alias('CL4'), ]) .drop('unique_id') # If you no longer need the ‘unique_id’ you can drop itsame error ----> 7 .with_columns([ 8 pl.col('unique_id').str.split("").arr.get(0).apply(extract_first_element).alias('MaterialID'), 9 pl.col('unique_id').str.split("").arr.get(1).apply(extract_first_element).alias('SalesOrg'), 10 pl.col('unique_id').str.split("").arr.get(2).apply(extract_first_element).alias('DistrChan'), 11 pl.col('unique_id').str.split("").arr.get(3).apply(extract_first_element).alias('CL4'), 12 ]) 13 .drop('unique_id') # If you no longer need the ‘unique_id’ you can drop it 14 )

    File ~/anaconda3/lib/python3.10/site-packages/polars/dataframe/frame.py:8315, in DataFrame.with_columns(self, *exprs, **named_exprs) 8169 def with_columns( 8170 self, 8171 *exprs: IntoExpr | Iterable[IntoExpr], 8172 **named_exprs: IntoExpr, 8173 ) -> DataFrame: 8174 """ 8175 Add columns to this DataFrame. 8176 (...) 8313 └─────┴──────┴─────────────┘ 8314 """ -> 8315 return self.lazy().with_columns(*exprs, **named_exprs).collect(_eager=True)

    File ~/anaconda3/lib/python3.10/site-packages/polars/lazyframe/frame.py:1940, in LazyFrame.collect(self, type_coercion, predicate_pushdown, projection_pushdown, simplify_expression, slice_pushdown, comm_subplan_elim, comm_subexpr_elim, no_optimization, streaming, background, _eager) 1937 if background: 1938 return InProcessQuery(ldf.collect_concurrently()) -> 1940 return wrap_df(ldf.collect())

    SchemaError: invalid series dtype: expected FixedSizeList, got list[str] )

  12. Assistant

    It appears there's a misunderstanding with the use of Polars' methods, and I apologize for the confusion. The .arr.get(i) method is actually not the correct approach in this context, given how Polars manages list types and the fact that the unique_id is directly being split without immediately accessing results makes troubleshooting challenging.

    Let's try a different approach which is more appropriate for Polars—a method chain that explicitly splits and selects the components. Here's the correct workflow:

    # Let's use the 'apply' method but with a lambda that truly extracts the item
    forecasts_df = (
        forecasts_df.lazy()
        .with_column(pl.col('unique_id').str.split("_").apply(lambda lst: lst[0]).alias('MaterialID'))
        .with_column(pl.col('unique_id').str.split("_").apply(lambda lst: lst[1]).alias('SalesOrg'))
        .with_column(pl.col('unique_id').str.split("_").apply(lambda lst: lst[2]).alias('DistrChan'))
        .with_column(pl.col('unique_id').str.split("_").apply(lambda lst: lst[3]).alias('CL4'))
        .drop('unique_id')
        .collect()
    )
    
    # Rename 'ds' to 'WeekDate'
    forecasts_df = forecasts_df.rename({'ds': 'WeekDate'})
    
    # Verify the result
    print(forecasts_df.schema)
    

    In this case, we're using the apply() method on the result of split() to directly pick out the element we want for the column in question. This should give us the correct string types, not lists.

    You will notice now we're using the lazy API (.lazy()) and at the end, we collect the result with .collect(). This could also be necessary if your dataset is large or if it's advantageous to benefit from Polars' lazy evaluation and optimizations.

    Also, make sure that all entries in the unique_id column have at least 4 underscore-separated parts, as we're directly accessing up to the fourth element with lst[3]. If any unique_id doesn't match this pattern, it'll throw an IndexError, and you may want to handle that scenario in your lambda function.

  13. User

    go back to this one, # Split the unique_id concat into the original columns forecasts_df6 = ( forecasts_df .with_columns([ pl.col('unique_id').str.split('').get(0).alias('MaterialID'), pl.col('unique_id').str.split('').get(1).alias('SalesOrg'), pl.col('unique_id').str.split('').get(2).alias('DistrChan'), pl.col('unique_id').str.split('').get(3).alias('CL4') ]) .drop('unique_id') ) it still working butthe split but not splitting out right since ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-11-20 00:00:00 17.284351 -5.480011 40.04871 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-11-27 00:00:00 17.284346 -5.480016 40.048706 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] 2023-12-04 00:00:00 17.28434 -5.480021 40.048702 17.82925 10.164697 25.493803 13.231607 0.037003 26.426212 16 3 31 ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] ["11000263", "US01", … "6067493"] go back to this 12528190_US01_8_6922332, there are 3 _ _ _, do we need to include in the split for it to work right?

  14. Assistant

    I see, it looks like the splitting operation is not isolating individual elements correctly into the columns MaterialID, SalesOrg, DistrChan, CL4 due to the way it's performed and assigned. Let's correct this.

    If your unique_id is of the form 12528190_US01_8_6922332, and you want to split it into four parts, here's the right way to do it with Polars:

    # Split the unique_id concat into the original columns
    forecasts_df6 = (
        forecasts_df
        .with_columns([
            pl.col('unique_id').str.split('_').arr.get(0).alias('MaterialID'),
            pl.col('unique_id').str.split('_').arr.get(1).alias('SalesOrg'),
            pl.col('unique_id').str.split('_').arr.get(2).alias('DistrChan'),
            pl.col('unique_id').str.split('_').arr.get(3).alias('CL4')
        ])
        .drop('unique_id')
    )
    

    This code uses str.split('_').arr.get(i) correctly (based on your original code that was working), to split the unique_id into its respective parts and assign them to the new columns.

    Here's what each line is doing:

    • str.split('_') - Splits the string at each underscore.
    • arr.get(i) - Gets the ith element from the array resulting from the split.

    Make sure that you're not seeing truncation (…) because of the display configuration of the DataFrame. It's often the case that longer strings are condensed to fit the display.

    If you still see something like ["11000263", "US01", … "6067493"] in every new column, it suggests that the items aren't actually being split and fetched properly. In that case, you may want to confirm how the unique_id values are displayed in your dataframe before applying the split, e.g., check with print(forecasts_df['unique_id'].to_series().to_list()) to see the contents of the unique_id column.

    If everything seems fine after checking the unique_id contents, try running the split operation again and ensure you're having the updated polars library. If problems persist, consider revisiting the behavior of str.split() and arr.get(i) in the context of your Polars dataframe or seek support from the Polars community.

  15. User

    but it give me error whenever arr.get. is used

  16. Assistant

    Given that arr.get is causing issues, let's try a different approach that should work correctly with the Polars library. We will use the apply method to manually process each item after splitting.

    The following code will perform the split operation while explicitly extracting single string values from the list for each piece of the unique_id. Since apply can operate on each value of the Series, we'll write a custom function to handle the split and extraction:

    def split_unique_id(unique_id):
        parts = unique_id.split('_')
        return parts if len(parts) >= 4 else (parts + [None] * (4 - len(parts)))
    
    # Apply the split function and extract the elements
    forecasts_df6 = (
        forecasts_df
        .with_columns([
            pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[0]).alias('MaterialID'),
            pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[1]).alias('SalesOrg'),
            pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[2]).alias('DistrChan'),
            pl.col('unique_id').apply(lambda uid: split_unique_id(uid)[3]).alias('CL4'),
        ])
        .drop('unique_id')
    )
    

    In the above code:

    • We define a custom function split_unique_id that splits the unique_id by the underscore character and ensures the result has at least four parts by padding with None if necessary.
    • We then use apply to map this function to the unique_id column of the DataFrame. By applying the split_unique_id function to each item and indexing the result ([0], [1], [2], [3]), we extract the parts for MaterialID, SalesOrg, DistrChan, and CL4 columns respectively.
    • We drop the original unique_id column since it’s no longer needed after splitting.

    This should work provided that your initial DataFrame forecasts_df includes a column named 'unique_id'. Make sure all the unique_ids have at least three underscores to properly split into four parts. If any unique_id doesn't have enough parts, the function split_unique_id will pad it with None, but you can adjust that behavior as needed.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.