Skip to content
TheChatArchiveThe Chat Archive
o1-preview-2024-09-12· OpenAIAi2 WildChat-4.8M (AI2)

import pandas as pd import numpy as np import yfinance as yf from sklearn.preprocessing…

  1. User

    import pandas as pd import numpy as np import yfinance as yf from sklearn.preprocessing import MinMaxScaler from sklearn.model_selection import train_test_split from sklearn.ensemble import RandomForestRegressor, GradientBoostingRegressor from sklearn.metrics import mean_squared_error, r2_score from tensorflow.keras.models import Sequential from tensorflow.keras.layers import LSTM, Dense, Dropout from tensorflow.keras.optimizers import Adam from tensorflow.keras.callbacks import EarlyStopping import tensorflow as tf from concurrent.futures import ThreadPoolExecutor, as_completed import time from tqdm.notebook import tqdm import warnings import pandas_ta as ta # Ensure consistent usage

    warnings.filterwarnings('ignore') tf.random.set_seed(42) np.random.seed(42)

    Function to get list of Indian stocks

    def get_indian_stocks(): nse_url = "https://archives.nseindia.com/content/equities/EQUITY_L.csv" try: nse_df = pd.read_csv(nse_url) return [f"{symbol}.NS" for symbol in nse_df['SYMBOL'].tolist()] except Exception as e: print(f"Error fetching Indian stocks list: {str(e)}") return []

    Function to fetch stock data with retries and multiple periods

    def get_stock_data(symbol, periods=["5y", "max"], retries=3): """ Fetches historical stock data for a given symbol using available periods.

    Parameters:
    - symbol (str): Stock symbol.
    - periods (list): List of periods to attempt fetching.
    - retries (int): Number of retry attempts per period.
    
    Returns:
    - pd.DataFrame or None: Historical stock data or None if fetching fails.
    """
    for period in periods:
        for attempt in range(retries):
            try:
                stock = yf.Ticker(symbol)
                df = stock.history(period=period)
                if df.empty:
                    raise ValueError("Empty DataFrame returned")
                if len(df) < 400:  # Ensure we have enough data
                    print(f"Insufficient data for {symbol} with period '{period}'. Trying next period...")
                    break  # Move to the next period
                return df
            except Exception as e:
                if attempt == retries - 1:
                    print(f"Error fetching data for {symbol} with period '{period}': {str(e)}")
                time.sleep(1)  # Wait before retrying
    return None
    

    Function to create features

    def create_features(df): # Calculate Percentage Returns df['Returns'] = df['Close'].pct_change()

    # Simple Moving Averages
    df['SMA_50'] = df['Close'].rolling(window=50).mean()
    df['SMA_200'] = df['Close'].rolling(window=200).mean()
    
    # Volatility
    df['Volatility'] = df['Returns'].rolling(window=20).std() * np.sqrt(252)
    
    # **Corrected Technical Indicators Using pandas_ta**
    # RSI
    df['RSI'] = ta.rsi(df['Close'], length=14)  # Default length is 14
    
    # MACD
    macd = ta.macd(df['Close'], fast=12, slow=26, signal=9)
    df['MACD'] = macd['MACD_12_26_9']
    df['MACD_Signal'] = macd['MACDs_12_26_9']
    
    # Bollinger Bands
    bb = ta.bbands(df['Close'], length=20, std=2)
    df['BB_High'] = bb['BBU_20_2.0']
    df['BB_Low'] = bb['BBL_20_2.0']
    
    # On-Balance Volume
    df['OBV'] = ta.obv(df['Close'], df['Volume'])
    
    # Drop Rows with NaN Values
    df = df.dropna()
    
    return df
    

    Function to prepare data for LSTM

    def prepare_data_for_lstm(data, look_back=60): scaler = MinMaxScaler(feature_range=(0, 1)) scaled_data = scaler.fit_transform(data)

    X, y = [], []
    for i in range(look_back, len(scaled_data)):
        X.append(scaled_data[i-look_back:i])
        y.append(scaled_data[i, 3])  # 'Close' is the 4th column (0-based indexing)
    
    return np.array(X), np.array(y), scaler
    

    Function to create and train LSTM model

    def create_lstm_model(input_shape): model = Sequential([ LSTM(100, return_sequences=True, input_shape=input_shape), Dropout(0.2), LSTM(100, return_sequences=False), Dropout(0.2), Dense(50, activation='relu'), Dense(1) ]) model.compile(optimizer=Adam(learning_rate=0.001), loss='mse') return model

    Function to train model and make prediction

    def analyze_stock(symbol): df = get_stock_data(symbol) if df is None or len(df) < 400: return None

    df = create_features(df)
    
    features = ['Open', 'High', 'Low', 'Close', 'Volume', 'Returns', 'SMA_50', 'SMA_200', 
                'Volatility', 'RSI', 'MACD', 'MACD_Signal', 'BB_High', 'BB_Low', 'OBV']
    X = df[features]
    y = df['Close']
    
    # Prepare data for LSTM
    X_lstm, y_lstm, scaler = prepare_data_for_lstm(df[features].values)
    X_train_lstm, X_test_lstm, y_train_lstm, y_test_lstm = train_test_split(
        X_lstm, y_lstm, test_size=0.2, random_state=42
    )
    
    # Train LSTM model
    lstm_model = create_lstm_model((X_train_lstm.shape[1], X_train_lstm.shape[2]))
    early_stopping = EarlyStopping(monitor='val_loss', patience=10, restore_best_weights=True)
    lstm_model.fit(
        X_train_lstm, y_train_lstm, epochs=100, batch_size=32, validation_split=0.2, 
        callbacks=[early_stopping], verbose=0
    )
    
    # Train Random Forest and Gradient Boosting models
    X_train, X_test, y_train, y_test = train_test_split(
        X, y, test_size=0.2, random_state=42
    )
    
    rf_model = RandomForestRegressor(n_estimators=200, random_state=42, n_jobs=-1)
    rf_model.fit(X_train, y_train)
    
    gb_model = GradientBoostingRegressor(n_estimators=200, random_state=42)
    gb_model.fit(X_train, y_train)
    
    # Make predictions
    lstm_pred = lstm_model.predict(X_test_lstm)
    # Inverse transform only the 'Close' price
    lstm_pred_full = np.zeros((len(lstm_pred), X.shape[1]))
    lstm_pred_full[:, 3] = lstm_pred  # 'Close' is the 4th column
    lstm_pred_inv = scaler.inverse_transform(lstm_pred_full)[:, 3]
    
    rf_pred = rf_model.predict(X_test)
    gb_pred = gb_model.predict(X_test)
    
    # Ensemble prediction
    ensemble_pred = (lstm_pred_inv + rf_pred + gb_pred) / 3
    
    # Calculate metrics
    mse = mean_squared_error(y_test, ensemble_pred)
    r2 = r2_score(y_test, ensemble_pred)
    
    # Make future prediction
    last_60_days = df[features].values[-60:]
    last_60_days_scaled = scaler.transform(last_60_days)
    last_60_days_reshaped = last_60_days_scaled.reshape(1, 60, X.shape[1])
    lstm_future = lstm_model.predict(last_60_days_reshaped)
    lstm_future_full = np.zeros((1, X.shape[1]))
    lstm_future_full[0, 3] = lstm_future[0, 0]  # 'Close' is the 4th column
    lstm_future_inv = scaler.inverse_transform(lstm_future_full)[0, 3]
    
    rf_future = rf_model.predict(df[features].iloc[-1:].values)
    gb_future = gb_model.predict(df[features].iloc[-1:].values)
    
    ensemble_future = (lstm_future_inv + rf_future[0] + gb_future[0]) / 3
    
    current_price = df['Close'].iloc[-1]
    potential_return = (ensemble_future - current_price) / current_price
    
    avg_volume = df['Volume'].mean()
    volatility = df['Volatility'].iloc[-1]
    
    return {
        'symbol': symbol,
        'current_price': current_price,
        'predicted_price': ensemble_future,
        'potential_return': potential_return,
        'mse': mse,
        'r2': r2,
        'avg_volume': avg_volume,
        'volatility': volatility
    }
    

    Function to calculate confidence score

    def calculate_confidence(stock_data): r2_weight = 0.4 volume_weight = 0.3 volatility_weight = 0.3

    r2_score_val = max(min(stock_data['r2'], 1), 0)  # Clip R2 score between 0 and 1
    volume_score = min(stock_data['avg_volume'] / 1e6, 1)  # Normalize volume, cap at 1
    volatility_score = 1 - min(stock_data['volatility'], 1)  # Lower volatility is better, cap at 1
    
    confidence = (r2_weight * r2_score_val +
                  volume_weight * volume_score +
                  volatility_weight * volatility_score)
    
    return confidence * 100  # Convert to percentage
    

    Function to scan all stocks

    def scan_stocks(stocks): results = []

    with ThreadPoolExecutor(max_workers=5) as executor:
        future_to_stock = {executor.submit(analyze_stock, symbol): symbol for symbol in stocks}
        for future in tqdm(as_completed(future_to_stock), total=len(stocks), desc="Analyzing Stocks"):
            result = future.result()
            if result:
                confidence = calculate_confidence(result)
                result['confidence'] = confidence
                results.append(result)
    
    return sorted(results, key=lambda x: x['confidence'], reverse=True)
    

    Main function

    def main(): print("Fetching list of Indian stocks...") stocks = get_indian_stocks() print(f"Found {len(stocks)} stocks. Starting analysis...")

    if not stocks:
        print("No stocks to analyze. Exiting.")
        return
    
    # Analyze top 100 stocks for demonstration; adjust as needed
    stocks_to_analyze = stocks[:100]
    results = scan_stocks(stocks_to_analyze)
    
    if not results:
        print("No stocks were successfully analyzed. Please check the error logs for details.")
        return
    
    print("\nTop 10 suggested stocks for investment:")
    for i, stock in enumerate(results[:10], 1):
        print(f"{i}. {stock['symbol']}:")
        print(f"   Current Price: ₹{stock['current_price']:.2f}")
        print(f"   Predicted Price: ₹{stock['predicted_price']:.2f}")
        print(f"   Potential Return: {stock['potential_return']:.2%}")
        print(f"   Confidence: {stock['confidence']:.2f}%")
        print(f"   R2 Score: {stock['r2']:.4f}")
        print(f"   MSE: {stock['mse']:.4f}")
        print(f"   Average Daily Volume: {stock['avg_volume']:,.0f}")
        print(f"   Volatility: {stock['volatility']:.2%}")
        print()
    
    print(f"Total stocks analyzed: {len(results)}")
    print("Note: This analysis is based on historical data and should not be the sole basis for investment decisions.")
    print("Always conduct thorough research and consult with a qualified financial advisor before investing.")
    

    if name == "main": main()

    This is my code. First, let's limit our stocks from the top stocks to smallcap shares - currently it is very slow as it tries to process all 2022 shares.

    Second, can you try improving logic on this code?

    Third, it is extremely slow - Fetching list of Indian stocks... Found 2022 stocks. Starting analysis... Analyzing Stocks:   0%  0/100 [03:17<?, ?it/s] WARNING:tensorflow:5 out of the last 9 calls to <function TensorFlowTrainer.make_train_function..one_step_on_iterator at 0x7c77f32aa0e0> triggered tf.function retracing. Tracing is expensive and the excessive number of tracings could be due to (1) creating @tf.function repeatedly in a loop, (2) passing tensors with different shapes, (3) passing Python objects instead of tensors. For (1), please define your @tf.function outside of the loop. For (2), @tf.function has reduce_retracing=True option that can avoid unnecessary retracing. For (3), please refer to https://www.tensorflow.org/guide/function#controlling_retracing and https://www.tensorflow.org/api_docs/python/tf/function for more details. 7/7 ━━━━━━━━━━━━━━━━━━━━ 3s 380ms/step

    This has been the output in over 6m of processing so far.

  2. Assistant

    It seems you're facing several challenges with your current code: fetching and processing a large number of stocks (2022 stocks), slow performance due to resource-intensive computations, and some technical warnings from TensorFlow.

    Let's address your concerns one by one and provide solutions:

    1. Limiting Analysis to Small-Cap Stocks

    To analyze only small-cap stocks instead of all 2022 stocks, we need to obtain a list of small-cap stocks. Here's how you can modify your code to fetch and analyze only small-cap stocks:

    a. Fetching Small-Cap Stocks from NSE

    Unfortunately, NSEIndia doesn't provide a straightforward CSV file for small-cap stocks like it does for the complete list. However, we can utilize the Nifty Smallcap 100 index constituents. NSE provides a CSV file for this index, but accessing it requires setting appropriate headers due to their website's policy.

    Here's how you can fetch the list of small-cap stocks:

    import pandas as pd
    import requests
    
    def get_smallcap_stocks():
        url = "https://www.nseindia.com/api/index-component?index=NIFTY%20SMALLCAP%20100"
        headers = {
            "User-Agent": "Mozilla/5.0",
            "Accept-Language": "en-US,en;q=0.9"
        }
        try:
            session = requests.Session()
            session.get("https://www.nseindia.com", headers=headers)  # Establish session
            response = session.get(url, headers=headers)
            data = response.json()
            symbols = [item['symbol'] + ".NS" for item in data['data']]
            return symbols
        except Exception as e:
            print(f"Error fetching small-cap stocks: {str(e)}")
            return []
    

    Notes:

    • Accessing NSE Data: NSE now requires setting headers to mimic a browser request. We use the requests library to set a User-Agent and Accept-Language header.
    • Fetching Nifty Smallcap 100 Constituents: We access the API endpoint https://www.nseindia.com/api/index-component with the index parameter set to NIFTY SMALLCAP 100.

    b. Modifying Your main Function

    Now, replace your call to get_indian_stocks() with get_smallcap_stocks():

    def main():
        print("Fetching list of Small-Cap stocks...")
        stocks = get_smallcap_stocks()
        print(f"Found {len(stocks)} small-cap stocks. Starting analysis...")
        
        if not stocks:
            print("No stocks to analyze. Exiting.")
            return
        
        # Analyze small-cap stocks
        stocks_to_analyze = stocks  # Analyze all small-cap stocks
        results = scan_stocks(stocks_to_analyze)
        
        # ... rest of your code remains the same ...
    

    2. Improving the Logic and Performance of the Code

    a. Identifying the Bottleneck

    The primary reason your code is slow is due to the per-stock training of computationally expensive models, especially the LSTM neural network. Training neural networks, even small ones, can be time-consuming, particularly when done sequentially for many stocks.

    b. Reducing Training Time

    To improve performance:

    • Reduce the Number of Epochs: Decrease the number of epochs in LSTM training from 100 to a lower number, such as 5 or 10.
    • Reduce the Number of Trees: Decrease n_estimators in both RandomForestRegressor and GradientBoostingRegressor from 200 to 50.
    • Adjust EarlyStopping Patience: Reduce the patience parameter in EarlyStopping to prevent unnecessary epochs.

    Here's how you can modify the relevant parts of your analyze_stock function:

    def analyze_stock(symbol):
        # ... Your existing code ...
    
        # Train LSTM model
        lstm_model = create_lstm_model((X_train_lstm.shape[1], X_train_lstm.shape[2]))
        early_stopping = EarlyStopping(monitor='val_loss', patience=2, restore_best_weights=True)
        lstm_model.fit(
            X_train_lstm, y_train_lstm, epochs=10, batch_size=16, validation_split=0.2, 
            callbacks=[early_stopping], verbose=0
        )
        
        # Train Random Forest and Gradient Boosting models
        rf_model = RandomForestRegressor(n_estimators=50, random_state=42, n_jobs=-1)
        rf_model.fit(X_train, y_train)
        
        gb_model = GradientBoostingRegressor(n_estimators=50, random_state=42)
        gb_model.fit(X_train, y_train)
    
        # ... Rest of your code ...
    

    c. Clearing TensorFlow Sessions

    The warning you're seeing about TensorFlow retracing functions can be caused by not properly clearing the Keras/TensorFlow session between model trainings. Add K.clear_session() at the end of your analyze_stock function:

    from tensorflow.keras import backend as K
    
    def analyze_stock(symbol):
        # ... Your existing code ...
    
        # At the very end of the function
        K.clear_session()
        return {
            # ... Your return dictionary ...
        }
    

    This helps in preventing potential memory leaks and reduces retracing warnings.

    d. Simplifying the Model (Optional)

    If performance is still an issue, consider:

    • Removing the LSTM Model: LSTMs are resource-intensive. You can focus on the Random Forest and Gradient Boosting models which are faster to train.
    • Using Pre-trained Models: If feasible, train your models once on aggregated data and use them for prediction across stocks.

    e. Avoid Unnecessary Retracing

    The warning also mentions avoiding passing Python objects instead of tensors and reducing retracing by ensuring consistent input shapes. Since your data might vary in length for each stock, this can trigger retracing. Clearing the session as shown above should help mitigate this issue.

    3. Overall Code Optimization

    By limiting the stocks to small-cap companies and reducing the computational load per stock, your code should perform significantly faster.

    However, processing many stocks with complex models will still take time. To further speed up your code:

    • Parallel Processing: You're already using ThreadPoolExecutor, but you can increase max_workers, keeping in mind your system's CPU core count. Be cautious as too many threads can lead to contention.
    • Data Sampling: If you have very long time series data per stock, consider downsampling or using a rolling window approach to reduce data size.
    • Resource Monitoring: Ensure your system has sufficient memory and CPU resources to handle the computations.

    Updated main Function Example

    Here's how your updated main function might look:

    def main():
        print("Fetching list of Small-Cap stocks...")
        stocks = get_smallcap_stocks()
        print(f"Found {len(stocks)} small-cap stocks. Starting analysis...")
        
        if not stocks:
            print("No stocks to analyze. Exiting.")
            return
        
        # Analyze small-cap stocks
        stocks_to_analyze = stocks
        results = scan_stocks(stocks_to_analyze)
        
        if not results:
            print("No stocks were successfully analyzed. Please check the error logs for details.")
            return
        
        print("\nTop 10 suggested small-cap stocks for investment:")
        for i, stock in enumerate(results[:10], 1):
            print(f"{i}. {stock['symbol']}:")
            print(f"   Current Price: ₹{stock['current_price']:.2f}")
            print(f"   Predicted Price: ₹{stock['predicted_price']:.2f}")
            print(f"   Potential Return: {stock['potential_return']:.2%}")
            print(f"   Confidence: {stock['confidence']:.2f}%")
            print(f"   R2 Score: {stock['r2']:.4f}")
            print(f"   MSE: {stock['mse']:.4f}")
            print(f"   Average Daily Volume: {stock['avg_volume']:,.0f}")
            print(f"   Volatility: {stock['volatility']:.2%}")
            print()
        
        print(f"Total small-cap stocks analyzed: {len(results)}")
        print("Note: This analysis is based on historical data and should not be the sole basis for investment decisions.")
        print("Always conduct thorough research and consult with a qualified financial advisor before investing.")
    

    Final Remarks

    • Ensure Dependencies Are Up to Date: Make sure all your libraries (pandas-ta, tensorflow, etc.) are updated to the latest versions.
    • Monitoring Progress: Use tqdm to display progress bars, which you've already included. This helps in keeping track of the analysis progress.
    • Testing with Fewer Stocks First: Before running the full analysis, test your code with a small subset (e.g., 5 stocks) to ensure everything works correctly.

    By implementing these changes, you should see a significant improvement in performance, and your code will focus specifically on small-cap stocks as per your requirement.


    Keep in mind: Stock market predictions involve a high degree of uncertainty. The models and methods used here are based on historical data and may not accurately predict future performance. Always use such tools as part of a broader investment strategy and consider consulting financial professionals.