Skip to content
TheChatArchiveThe Chat Archive
gpt-4-0125-preview· OpenAIAi2 WildChat-4.8M (AI2)

You are a Python expert who can provide clear, concise, high-quality code. " import…

  1. User

    You are a Python expert who can provide clear, concise, high-quality code. " import matplotlib.pyplot as plt import numpy as np import pandas as pd import seaborn as sns import cv2 import random import tensorflow as tf import tkinter as tk from tkinter import filedialog from PIL import ImageTk, Image from ipywidgets import interact, interactive, fixed, interact_manual, HBox, VBox import ipywidgets as widgets from IPython.display import display, clear_output from tensorflow.keras.preprocessing import image from tensorflow.keras.optimizers import Adam, SGD, RMSprop, AdamW, Adadelta, Adagrad, Adamax, Adafactor, Nadam, Ftrl from tensorflow.keras.preprocessing.image import ImageDataGenerator from tqdm import tqdm import os from sklearn.utils import shuffle from sklearn.preprocessing import LabelEncoder from sklearn.model_selection import train_test_split from tensorflow.keras.models import Sequential, Model, load_model from tensorflow.keras.layers import ( GlobalAveragePooling2D, Dropout, Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization, Activation, concatenate, Conv2DTranspose, Input, Reshape, UpSampling2D, ) from tensorflow.keras.applications import ( EfficientNetV2B0, EfficientNetV2B1, EfficientNetV2B2, EfficientNetV2B3, EfficientNetV2L, EfficientNetV2M, EfficientNetV2S, ) from tensorflow.keras.applications import Xception from tensorflow.keras.applications import VGG16, VGG19 from tensorflow.keras.applications import ResNet50, ResNet101, ResNet152, ResNetRS50, ResNetRS101 from tensorflow.keras.applications import InceptionResNetV2, ConvNeXtXLarge, ConvNeXtBase, DenseNet121, MobileNetV2, NASNetLarge, NASNetMobile from tensorflow.keras.utils import to_categorical from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, TensorBoard, ModelCheckpoint from sklearn.metrics import classification_report, confusion_matrix import ipywidgets as widgets import io from PIL import Image from IPython.display import display, clear_output from ipywidgets import Output, Button from warnings import filterwarnings from google.colab import drive drive.mount("/content/gdrive") def load_data(data_folders): X_data = [] # Combined data y_class_labels = [] # Combined classification labels y_seg_labels = [] # Combined segmentation labels for folderPath in data_folders: for label in labels: label_folder_path = os.path.join(folderPath, label) for filename in tqdm(os.listdir(label_folder_path)): if filename.endswith(".jpg"): img = cv2.imread(os.path.join(label_folder_path, filename)) img = cv2.resize(img, (image_size, image_size)) X_data.append(img) y_class_labels.append(label) seg_filename = filename.split(".")[0] + ".png" seg_img = cv2.imread(os.path.join(label_folder_path, seg_filename), 0) seg_img = cv2.resize(seg_img, (image_size, image_size)) seg_img = np.where(seg_img > 0, 1, 0) # Convert segmentation mask to binary y_seg_labels.append(seg_img) X_data = np.array(X_data) y_class_labels = np.array(y_class_labels) y_seg_labels = np.array(y_seg_labels) X_data, y_class_labels, y_seg_labels = shuffle(X_data, y_class_labels, y_seg_labels, random_state=101) return X_data, y_class_labels, y_seg_labels

    def split_data(X_data, y_class_labels, y_seg_labels, train_ratio, val_ratio, test_ratio): num_samples = len(X_data) num_train_samples = int(train_ratio * num_samples) num_val_samples = int(val_ratio * num_samples) num_test_samples = int(test_ratio * num_samples) X_train = X_data[:num_train_samples] y_train_class = y_class_labels[:num_train_samples] y_train_seg = y_seg_labels[:num_train_samples] X_val = X_data[num_train_samples: num_train_samples + num_val_samples] y_val_class = y_class_labels[num_train_samples: num_train_samples + num_val_samples] y_val_seg = y_seg_labels[num_train_samples: num_train_samples + num_val_samples] X_test = X_data[num_train_samples + num_val_samples: num_train_samples + num_val_samples + num_test_samples] y_test_class = y_class_labels[num_train_samples + num_val_samples: num_train_samples + num_val_samples + num_test_samples] y_test_seg = y_seg_labels[num_train_samples + num_val_samples: num_train_samples + num_val_samples + num_test_samples] # Convert class labels to categorical label_encoder = LabelEncoder() y_train_class_encoded = label_encoder.fit_transform(y_train_class) y_train_class_categorical = to_categorical(y_train_class_encoded) y_val_class_encoded = label_encoder.transform(y_val_class) y_val_class_categorical = to_categorical(y_val_class_encoded) y_test_class_encoded = label_encoder.transform(y_test_class) y_test_class_categorical = to_categorical(y_test_class_encoded) return ( X_train, y_train_class_categorical, y_train_seg, X_val, y_val_class_categorical, y_val_seg, X_test, y_test_class_categorical, y_test_seg, )

    def build_model(input_shape, num_classes): num_filter = 32 # 16/32 best, 8: best classification but no segment # Encoder (Done) inputs = Input(input_shape) conv1 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(inputs) bn1 = BatchNormalization()(conv1) relu1 = Activation("relu")(bn1) conv2 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(relu1) bn2 = BatchNormalization()(conv2) relu2 = Activation("relu")(bn2) down1 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu2) conv3 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(down1) bn3 = BatchNormalization()(conv3) relu3 = Activation("relu")(bn3) conv4 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(relu3) bn4 = BatchNormalization()(conv4) relu4 = Activation("relu")(bn4) down2 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu4) conv5 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(down2) bn5 = BatchNormalization()(conv5) relu5 = Activation("relu")(bn5) conv6 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(relu5) bn6 = BatchNormalization()(conv6) relu6 = Activation("relu")(bn6) down3 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu6) conv7 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(down3) bn7 = BatchNormalization()(conv7) relu7 = Activation("relu")(bn7) conv8 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(relu7) bn8 = BatchNormalization()(conv8) relu8 = Activation("relu")(bn8) # Middle down4 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu8) conv9 = Conv2D(num_filter * 16, 3, activation="linear", padding="same", strides=1)(down4) bn9 = BatchNormalization()(conv9) relu9 = Activation("relu")(bn9) conv10 = Conv2D(num_filter * 16, 3, activation="linear", padding="same", strides=1)(relu9) bn10 = BatchNormalization()(conv10) relu10 = Activation("relu")(bn10) up1 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu10) # Decoder (Done) concat1 = concatenate([up1, relu8], axis=-1) # , axis=3 conv11 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(concat1) bn11 = BatchNormalization()(conv11) relu11 = Activation("relu")(bn11) conv12 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(relu11) bn12 = BatchNormalization()(conv12) relu12 = Activation("relu")(bn12) up2 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu12) concat2 = concatenate([up2, relu6], axis=-1) # , axis=3 conv13 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(concat2) bn13 = BatchNormalization()(conv13) relu13 = Activation("relu")(bn13) conv14 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(relu13) bn14 = BatchNormalization()(conv14) relu14 = Activation("relu")(bn14) up3 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu14) concat3 = concatenate([up3, relu4], axis=-1) # , axis=3 conv15 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(concat3) bn15 = BatchNormalization()(conv15) relu15 = Activation("relu")(bn15) conv16 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(relu15) bn16 = BatchNormalization()(conv16) relu16 = Activation("relu")(bn16) up4 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu16) concat4 = concatenate([up4, relu2], axis=-1) # , axis=3 conv17 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(concat4) bn17 = BatchNormalization()(conv17) relu17 = Activation("relu")(bn17) conv18 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(relu17) bn18 = BatchNormalization()(conv18) relu18 = Activation("relu")(bn18) # Segmentation branch segmentation_output = Conv2D(1, 1, activation="sigmoid", name="segmentation_output")(relu18) # original # Classification branch (Not done) gap1 = GlobalAveragePooling2D()(relu8) gap2 = GlobalAveragePooling2D()(relu10) gap3 = GlobalAveragePooling2D()(relu12) conv20 = Conv2D(16, 3, activation="linear", padding="same", strides=1)(segmentation_output) bn20 = BatchNormalization()(conv20) relu20 = Activation("relu")(bn20) down5 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu20) conv21 = Conv2D(32, 3, activation="linear", padding="same", strides=1)(down5) bn21 = BatchNormalization()(conv21) relu21 = Activation("relu")(bn21) down6 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu21) conv22 = Conv2D(64, 3, activation="linear", padding="same", strides=1)(down6) bn22 = BatchNormalization()(conv22) relu22 = Activation("relu")(bn22) down7 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu22) flatten1 = Flatten()(down7) concat5 = concatenate([gap1, gap2, gap3, flatten1], axis=-1) # FC layers fc1 = Dense(1024, activation="relu")(concat5) dropout1 = Dropout(0.5)(fc1) fc2 = Dense(1024, activation="relu")(dropout1) dropout2 = Dropout(0.5)(fc2) classification_output = Dense(num_classes, activation="softmax", name="classification_output")(dropout2) # Define the model model = Model(inputs=inputs, outputs=[classification_output, segmentation_output]) return model

    def segmentation_loss(y_true, y_pred): y_true = tf.cast(y_true, tf.float32) y_pred = tf.cast(y_pred, tf.float32) bce_loss = tf.keras.losses.binary_crossentropy(y_true, y_pred) smooth = 1e-5 intersection = tf.reduce_sum(y_true * y_pred) union = tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) dice_loss = 1.0 - 2.0 * (intersection + smooth) / (union + smooth) segmentation_loss = bce_loss + 1 * dice_loss return segmentation_loss

    def train_model(model, X_train, y_train_class, y_train_seg, X_val, y_val_class, y_val_seg, batch_size, epochs): checkpoint = ModelCheckpoint( "multitask_best_weights.h5", monitor="val_classification_output_accuracy", save_best_only=True, mode="max", verbose=1,) reduce_lr = ReduceLROnPlateau( monitor="val_classification_output_accuracy", factor=0.3, patience=2, min_delta=0.001, mode="auto", verbose=1,) tensorboard = TensorBoard(log_dir="logs") model.compile( optimizer=Adam(lr=0.001), loss={"classification_output": "categorical_crossentropy", "segmentation_output": segmentation_loss}, metrics={"classification_output": "accuracy", "segmentation_output": "accuracy"}, loss_weights={"classification_output": 1, "segmentation_output": 1},) history = model.fit( X_train, {"classification_output": y_train_class, "segmentation_output": y_train_seg}, validation_data=(X_val, {"classification_output": y_val_class, "segmentation_output": y_val_seg}), epochs=epochs, verbose=1, batch_size=batch_size, callbacks=[checkpoint, reduce_lr, tensorboard],) return history

    def evaluate_model(model, X_test, y_test_class, y_test_seg): with tf.keras.utils.custom_object_scope({"segmentation_loss": segmentation_loss}): # Load the best model weights best_model = load_model("multitask_best_weights.h5") # Evaluate the model on test data test_loss, test_class_loss, test_seg_loss, test_class_acc, test_seg_acc = best_model.evaluate( X_test, {"classification_output": y_test_class, "segmentation_output": y_test_seg}) print("Test Classification Loss:", test_class_loss) print("Test Segmentation Loss:", test_seg_loss) print("Test Classification Accuracy:", test_class_acc) print("Test Segmentation Accuracy:", test_seg_acc) # Evaluate the model on validation data val_loss, val_class_loss, val_seg_loss, val_class_acc, val_seg_acc = best_model.evaluate( X_val, {'classification_output': y_val_class, 'segmentation_output': y_val_seg}) print("Validation Classification Loss:", val_class_loss) print("Validation Segmentation Loss:", val_seg_loss) print("Validation Classification Accuracy:", val_class_acc) print("Validation Segmentation Accuracy:", val_seg_acc) # Evaluate the model on training data train_loss, train_class_loss, train_seg_loss, train_class_acc, train_seg_acc = best_model.evaluate(X_train, {'classification_output': y_train_class, 'segmentation_output': y_train_seg}) print("Train Classification Loss:", train_class_loss) print("Train Segmentation Loss:", train_seg_loss) print("Train Classification Accuracy:", train_class_acc) print("Train Segmentation Accuracy:", train_seg_acc) # Return test classification accuracy return test_class_acc

    def plot_performance(history): # Plot classification accuracy classification_train_accuracy = history.history["classification_output_accuracy"] classification_val_accuracy = history.history["val_classification_output_accuracy"] plt.figure(figsize=(7, 3)) plt.plot(classification_train_accuracy, label="Training Accuracy") plt.plot(classification_val_accuracy, label="Validation Accuracy") plt.title("Classification Accuracy") plt.xlabel("Epochs") plt.ylabel("Accuracy") plt.legend() plt.show() # Plot classification loss classification_train_loss = history.history["classification_output_loss"] classification_val_loss = history.history["val_classification_output_loss"] plt.figure(figsize=(7, 3)) plt.plot(classification_train_loss, "b", label="Training Loss") plt.plot(classification_val_loss, "r", label="Validation Loss") plt.title("Classification Loss") plt.xlabel("Epochs") plt.ylabel("Loss") plt.legend() plt.show() # Plot segmentation accuracy segmentation_train_accuracy = history.history["segmentation_output_accuracy"] segmentation_val_accuracy = history.history["val_segmentation_output_accuracy"] plt.figure(figsize=(7, 3)) plt.plot(segmentation_train_accuracy, label="Training Accuracy") plt.plot(segmentation_val_accuracy, label="Validation Accuracy") plt.title("Segmentation Accuracy") plt.xlabel("Epochs") plt.ylabel("Accuracy") plt.legend() plt.show() # Plot segmentation loss segmentation_train_loss = history.history["segmentation_output_loss"] segmentation_val_loss = history.history["val_segmentation_output_loss"] plt.figure(figsize=(7, 3)) plt.plot(segmentation_train_loss, "b", label="Training Loss") plt.plot(segmentation_val_loss, "r", label="Validation Loss") plt.title("Segmentation Loss") plt.xlabel("Epochs") plt.ylabel("Loss") plt.legend() plt.show()

    def test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5): random_indices = np.random.choice(len(X_test), num_images, replace=False) selected_images = X_test[random_indices] selected_labels = y_test_class[random_indices] selected_masks = y_test_seg[random_indices] predictions = model.predict(selected_images) predicted_classes = np.argmax(predictions[0], axis=1) predicted_masks = predictions[1] for i in range(num_images): plt.figure(figsize=(10, 5)) plt.subplot(2, 3, 1) plt.imshow(selected_images[i]) plt.title(f"Input Image\nGround Truth Label: {labels[np.argmax(selected_labels[i])]}\nPredicted Label: {labels[predicted_classes[i]]}") plt.subplot(2, 3, 2) plt.imshow(selected_masks[i].squeeze(), cmap="gray") plt.title("Ground Truth Segmentation Mask") plt.subplot(2, 3, 3) plt.imshow(predicted_masks[i].squeeze(), cmap="gray") plt.title("Predicted Segmentation Mask") plt.show()

    preprocess and predict on the selected image

    out = Output() def predict_image(image_path): # Load and preprocess the image img = cv2.imread(image_path) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # Convert BGR to RGB img_resized = cv2.resize(img, (image_size, image_size)) img_expanded = np.expand_dims(img_resized, axis=0) # Predict using the model class_output, seg_output = model.predict(img_expanded) # Get the predicted class predicted_class_index = np.argmax(class_output) predicted_class = labels[predicted_class_index] # Clear the previous output with out: clear_output(wait=True) # Visualize the input and segmentation output fig, axs = plt.subplots(1, 3, figsize=(15, 5)) axs[0].imshow(img) axs[0].set_title("Input Image") axs[0].axis("off")

        axs[1].imshow(seg_output[0, :, :, 0], cmap='gray')
        axs[1].set_title("Segmentation Output")
        axs[1].axis("off")
        
        class_probabilities = class_output[0]
        x_pos = np.arange(len(labels))
        axs[2].bar(x_pos, class_probabilities, align='center', alpha=0.7, color='blue')
        axs[2].set_xticks(x_pos)
        axs[2].set_xticklabels(labels, rotation=45, ha="right")
        axs[2].set_ylabel('Probability')
        axs[2].set_ylim([0, 1])
        axs[2].set_title(f'Classification Output\nPredicted Class: {predicted_class}')  # Increased title font size
        
        for i, v in enumerate(class_probabilities):
            axs[2].text(i, v + 0.02, f"{v:.2f}", color='black', ha='center')
        
        plt.tight_layout()
        plt.show()
    

    Define a callback function to handle the uploaded file

    def on_upload(change): # Get the uploaded file uploaded_filename = next(iter(uploader.value)) uploaded_file = uploader.value[uploaded_filename] # Save the uploaded file with open(uploaded_filename, 'wb') as f: f.write(uploaded_file['content']) # Predict and visualize the image predict_image(uploaded_filename)

    Define a function to handle the button click event and clear the output

    clear_button = Button(description='Clear') def clear_output_button_clicked(button): out.clear_output() clear_button.on_click(clear_output_button_clicked)

    Set image size

    image_size = 224

    Define labels

    labels = ["bridge", "excess", "good", "insuff", "no"]

    Set data folders

    data_folders = [ "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/train", "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/val", "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/test",]

    Load data

    X_data, y_class_labels, y_seg_labels = load_data(data_folders)

    Set image size

    image_size = 224

    Split data

    train_ratio = 0.2 val_ratio = 0.2 test_ratio = 0.6 X_train, y_train_class, y_train_seg, X_val, y_val_class, y_val_seg, X_test, y_test_class, y_test_seg = split_data( X_data, y_class_labels, y_seg_labels, train_ratio, val_ratio, test_ratio)

    Print the counts

    print("Number of train images:", len(X_train)) print("Number of train binary masks:", len(y_train_seg)) print("Number of validation images:", len(X_val)) print("Number of validation binary masks:", len(y_val_seg)) print("Number of test images:", len(X_test)) print("Number of test binary masks:", len(y_test_seg)) input_shape = (image_size, image_size, 3) num_classes = len(labels) saved_model_path = "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/model/pcbmtl_bdip.h5" custom_objects = {"segmentation_loss": segmentation_loss}

    Load model

    model = load_model(saved_model_path, custom_objects=custom_objects)

    Test model

    test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5)

    file uploader widget

    uploader = widgets.FileUpload(accept='image/*', multiple=False) uploader.observe(on_upload, names='value') buttons_layout = HBox([uploader, clear_button]) display(buttons_layout) display(out)

    " The above is the Python code with Keras to do inference using the multi-task learning model, this model will take the image as input and provide one classification output and one segmentation output, this code let user to select one image and performance inference. Help me to make the classification plot become dark theme, more pro and high-tech. Show code.

  2. Assistant

    To modify the visualization for classification outputs to adopt a dark theme, making it appear more "high-tech", you can leverage Matplotlib's support for style sheets and its comprehensive customization options for plotting. In the predict_image function, you will adjust the plotting section related to classification to apply a dark theme. Here's how you can do it:

    1. Apply a dark theme using plt.style.use('dark_background').
    2. Customize the bar plot to make it fit better with the new theme, such as adjusting colors, bar edge color, etc.
    3. Adjust text color for better visibility on a dark background.

    I'll modify the relevant part of your predict_image function to demonstrate these changes:

    def predict_image(image_path):
        # Load and preprocess the image
        img = cv2.imread(image_path)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB
        img_resized = cv2.resize(img, (image_size, image_size))
        img_expanded = np.expand_dims(img_resized, axis=0)
        # Predict using the model
        class_output, seg_output = model.predict(img_expanded)
        # Get the predicted class
        predicted_class_index = np.argmax(class_output)
        predicted_class = labels[predicted_class_index]
        # Clear the previous output
        with out:
            clear_output(wait=True)
            # Visualization adjustments for dark theme
            plt.style.use('dark_background')
            # Visualize the input and segmentation output
            with plt.style.context('dark_background'):  # Ensuring dark theme is applied locally
                fig, axs = plt.subplots(1, 3, figsize=(15, 5))
                axs[0].imshow(img)
                axs[0].set_title("Input Image")
                axs[0].axis("off")
    
                axs[1].imshow(seg_output[0, :, :, 0], cmap='gray')
                axs[1].set_title("Segmentation Output")
                axs[1].axis("off")
                
                # Enhanced plot for classification with a dark theme
                class_probabilities = class_output[0]
                x_pos = np.arange(len(labels))
                bars = axs[2].bar(x_pos, class_probabilities, align='center', alpha=0.7, color='cyan', edgecolor='white')
                axs[2].set_xticks(x_pos)
                axs[2].set_xticklabels(labels, rotation=45, ha="right", color='white')
                axs[2].set_ylabel('Probability', color='white')
                axs[2].set_ylim([0, 1])
                axs[2].spines['bottom'].set_color('white')
                axs[2].spines['left'].set_color('white')
                axs[2].tick_params(axis='x', colors='white')
                axs[2].tick_params(axis='y', colors='white')
                axs[2].set_title(f'Classification Output\nPredicted Class: {predicted_class}', color='white')  # Adjusted title color
                
                for i, v in enumerate(class_probabilities):
                    axs[2].text(i, v + 0.02, f"{v:.2f}", color='white', ha='center')  # Adjust text color
                
                plt.tight_layout()
                plt.show()
    

    This modification applies a dark theme to your plots for a "high-tech" look. The key changes are:

    • Use plt.style.use('dark_background') to switch the plotting style to a dark theme.
    • Customize the plot for classification, making text, edges, and bars contrast well against the dark background.

    Remember, these styles are customizable, and you can experiment with colors and styles to achieve the appearance that best fits your application's aesthetic.

  3. User

    Too dark, I cannot see the segmentation image as the segmentation mask is also dark.

  4. Assistant

    Understanding the concern with visibility, especially for dark segmentation masks on a dark theme, you can adjust the color palette specifically for the segmentation output to enhance visibility while keeping the high-tech, dark theme for the classification plot. Here's an adjustment to the predict_image function to address this:

    def predict_image(image_path):
        # Load and preprocess the image
        img = cv2.imread(image_path)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB
        img_resized = cv2.resize(img, (image_size, image_size))
        img_expanded = np.expand_dims(img_resized, axis=0)
        # Predict using the model
        class_output, seg_output = model.predict(img_expanded)
        # Get the predicted class
        predicted_class_index = np.argmax(class_output)
        predicted_class = labels[predicted_class_index]
        # Clear the previous output
        with out:
            clear_output(wait=True)
            # Apply dark theme for classification plot only
            plt.style.use('dark_background')
            with plt.style.context(('dark_background')):
                fig, axs = plt.subplots(1, 3, figsize=(15, 5))
                axs[0].imshow(img)
                axs[0].set_title("Input Image")
                axs[0].axis("off")
    
                # Adjust segmentation output visualization for better visibility
                axs[1].imshow(seg_output[0, :, :, 0], cmap='plasma')  # Using a brighter colormap
                axs[1].set_title("Segmentation Output")
                axs[1].axis("off")
                
                # Plotting classification with dark theme adjustments
                class_probabilities = class_output[0]
                x_pos = np.arange(len(labels))
                bars = axs[2].bar(x_pos, class_probabilities, align='center', color='cyan', edgecolor='white')
                axs[2].set_xticks(x_pos)
                axs[2].set_xticklabels(labels, rotation=45, ha="right", color='white')
                axs[2].set_ylabel('Probability', color='white')
                axs[2].set_ylim([0, 1])
                axs[2].spines['bottom'].set_color('white')
                axs[2].spines['left'].set_color('white')
                axs[2].tick_params(axis='x', colors='white')
                axs[2].tick_params(axis='y', colors='white')
                axs[2].set_title(f'Classification Output\nPredicted Class: {predicted_class}', color='white') 
                
                for i, v in enumerate(class_probabilities):
                    axs[2].text(i, v + 0.02, f"{v:.2f}", color='white', ha='center')
                
                plt.tight_layout()
                plt.show()
    

    Key Adjustments:

    • For the segmentation output, we switch to using a brighter colormap (such as plasma) that stands out on a dark background. This helps maintain visibility while keeping the overall aesthetic dark and high-tech.
    • We retained the dark theme for the classification output bar plot to maintain the professional, tech-savvy look.

    This approach ensures that both segmentation and classification outputs are visible and distinct while achieving the desired visual theme. Feel free to experiment with other colormaps (cmap parameter) for the segmentation output to find the one that best balances between visibility and aesthetics. Colormaps like viridis, inferno, magma, and cividis are also suitable alternatives that perform well on dark backgrounds.

  5. User

    No! Don't change the segmentation mask, just make the background less dark.

  6. Assistant

    If adjusting the segmentation mask's visualization is not desired and the goal is to lighten the background of the classification plot to make it less dark while still maintaining a stylish, professional appearance, you can customize the plot further. Instead of using the dark_background style which affects all plot elements, we'll manually configure the plot's background, grid lines, and text to achieve a suitable balance. The idea is to retain the dark theme charisma but with a slightly lighter tone for better visibility and contrast, especially for elements like the segmentation mask.

    Here's how you can adjust the plotting code in the predict_image function to customize the background and plot elements:

    def predict_image(image_path):
        # Load and preprocess the image
        img = cv2.imread(image_path)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB
        img_resized = cv2.resize(img, (image_size, image_size))
        img_expanded = np.expand_dims(img_resized, axis=0)
        # Predict using the model
        class_output, seg_output = model.predict(img_expanded)
        # Get the predicted class
        predicted_class_index = np.argmax(class_output)
        predicted_class = labels[predicted_class_index]
        # Clear the previous output
        with out:
            clear_output(wait=True)
            # Configure specific style options for lighter background
            fig, axs = plt.subplots(1, 3, figsize=(15, 5))
            
            # Lighten background
            bg_color = '#303030'  # Lighter than black for contrast
            fig.patch.set_facecolor(bg_color)
            for ax in axs:
                ax.set_facecolor(bg_color)
                # Set spines color to lighter tone
                ax.spines['bottom'].set_color('grey')
                ax.spines['top'].set_color('grey')
                ax.spines['right'].set_color('grey')
                ax.spines['left'].set_color('grey')
                ax.tick_params(axis='x', colors='white')
                ax.tick_params(axis='y', colors='white')
            
            axs[0].imshow(img)
            axs[0].set_title("Input Image", color='white')
            axs[0].axis("off")
    
            axs[1].imshow(seg_output[0, :, :, 0], cmap='gray')
            axs[1].set_title("Segmentation Output", color='white')
            axs[1].axis("off")
            
            # Adjusted plot for classification
            class_probabilities = class_output[0]
            x_pos = np.arange(len(labels))
            bars = axs[2].bar(x_pos, class_probabilities, align='center', alpha=0.7, color='cyan')
            axs[2].set_xticks(x_pos)
            axs[2].set_xticklabels(labels, rotation=45, ha="right", color='white')
            axs[2].set_ylabel('Probability', color='white')
            axs[2].set_ylim([0, 1])
            axs[2].set_title(f'Classification Output\nPredicted Class: {predicted_class}', color='white')
            
            for i, v in enumerate(class_probabilities):
                axs[2].text(i, v + 0.02, f"{v:.2f}", color='white', ha='center')
                  
            plt.tight_layout()
            plt.show()
    

    This approach manually sets the plot's background to a medium-dark gray (#303030) which is lighter than black but retains the "dark theme" essence. Each aspect of the plot, including tick labels and spines, is customized to ensure readability and style consistency. This should help the segmentation mask stand out more while maintaining the overall aesthetic.

    Feel free to adjust the bg_color value to find the right balance for your visualizations.

  7. User

    Good, make the title Bold.

  8. Assistant

    To make the titles bold in your plot, you can simply adjust the set_title method calls by adding the fontweight='bold' parameter within the predict_image function. This change will apply to both the input image title, the segmentation output title, and the classification output title, making them stand out more prominently.

    Here's how you can update the predict_image function to incorporate bold titles:

    def predict_image(image_path):
        # Load and preprocess the image
        img = cv2.imread(image_path)
        img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB)  # Convert BGR to RGB
        img_resized = cv2.resize(img, (image_size, image_size))
        img_expanded = np.expand_dims(img_resized, axis=0)
        # Predict using the model
        class_output, seg_output = model.predict(img_expanded)
        # Get the predicted class
        predicted_class_index = np.argmax(class_output)
        predicted_class = labels[predicted_class_index]
        # Clear the previous output
        with out:
            clear_output(wait=True)
            # Configure specific style options for lighter background
            fig, axs = plt.subplots(1, 3, figsize=(15, 5))
            
            bg_color = '#303030'  # Lighter than black for contrast, as per the previous specification
            fig.patch.set_facecolor(bg_color)
            for ax in axs:
                ax.set_facecolor(bg_color)
                # Set spines color to lighter tone
                ax.spines['bottom'].set_color('grey')
                ax.spines['top'].set_color('grey')
                ax.spines['right'].set_color('grey')
                ax.spines['left'].set_color('grey')
                ax.tick_params(axis='x', colors='white')
                ax.tick_params(axis='y', colors='white')
            
            axs[0].imshow(img)
            axs[0].set_title("Input Image", color='white', fontweight='bold')
            axs[0].axis("off")
    
            axs[1].imshow(seg_output[0, :, :, 0], cmap='gray')
            axs[1].set_title("Segmentation Output", color='white', fontweight='bold')
            axs[1].axis("off")
            
            class_probabilities = class_output[0]
            x_pos = np.arange(len(labels))
            bars = axs[2].bar(x_pos, class_probabilities, align='center', alpha=0.7, color='cyan')
            axs[2].set_xticks(x_pos)
            axs[2].set_xticklabels(labels, rotation=45, ha="right", color='white')
            axs[2].set_ylabel('Probability', color='white', fontweight='bold')
            axs[2].set_ylim([0, 1])
            axs[2].set_title(f'Classification Output\nPredicted Class: {predicted_class}', color='white', fontweight='bold')
            
            for i, v in enumerate(class_probabilities):
                axs[2].text(i, v + 0.02, f"{v:.2f}", color='white', ha='center')
            
            plt.tight_layout()
            plt.show()
    

    By adding fontweight='bold' to the set_title and set_ylabel calls, the titles and the y-axis label of the classification output plot are now bold, making them more pronounced and visually striking. The plt.tight_layout() call ensures that all elements are nicely fitted within the plot area, considering the new bold titles.

  9. User

    Good, also make my other plot become this background color, recall my original code: " import matplotlib.pyplot as plt import numpy as np import pandas as pd import seaborn as sns import cv2 import random import tensorflow as tf import tkinter as tk from tkinter import filedialog from PIL import ImageTk, Image from ipywidgets import interact, interactive, fixed, interact_manual, HBox, VBox import ipywidgets as widgets from IPython.display import display, clear_output from tensorflow.keras.preprocessing import image from tensorflow.keras.optimizers import Adam, SGD, RMSprop, AdamW, Adadelta, Adagrad, Adamax, Adafactor, Nadam, Ftrl from tensorflow.keras.preprocessing.image import ImageDataGenerator from tqdm import tqdm import os from sklearn.utils import shuffle from sklearn.preprocessing import LabelEncoder from sklearn.model_selection import train_test_split from tensorflow.keras.models import Sequential, Model, load_model from tensorflow.keras.layers import ( GlobalAveragePooling2D, Dropout, Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization, Activation, concatenate, Conv2DTranspose, Input, Reshape, UpSampling2D, ) from tensorflow.keras.applications import ( EfficientNetV2B0, EfficientNetV2B1, EfficientNetV2B2, EfficientNetV2B3, EfficientNetV2L, EfficientNetV2M, EfficientNetV2S, ) from tensorflow.keras.applications import Xception from tensorflow.keras.applications import VGG16, VGG19 from tensorflow.keras.applications import ResNet50, ResNet101, ResNet152, ResNetRS50, ResNetRS101 from tensorflow.keras.applications import InceptionResNetV2, ConvNeXtXLarge, ConvNeXtBase, DenseNet121, MobileNetV2, NASNetLarge, NASNetMobile from tensorflow.keras.utils import to_categorical from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, TensorBoard, ModelCheckpoint from sklearn.metrics import classification_report, confusion_matrix import ipywidgets as widgets import io from PIL import Image from IPython.display import display, clear_output from ipywidgets import Output, Button from warnings import filterwarnings from google.colab import drive drive.mount("/content/gdrive") def load_data(data_folders): X_data = [] # Combined data y_class_labels = [] # Combined classification labels y_seg_labels = [] # Combined segmentation labels for folderPath in data_folders: for label in labels: label_folder_path = os.path.join(folderPath, label) for filename in tqdm(os.listdir(label_folder_path)): if filename.endswith(".jpg"): img = cv2.imread(os.path.join(label_folder_path, filename)) img = cv2.resize(img, (image_size, image_size)) X_data.append(img) y_class_labels.append(label) seg_filename = filename.split(".")[0] + ".png" seg_img = cv2.imread(os.path.join(label_folder_path, seg_filename), 0) seg_img = cv2.resize(seg_img, (image_size, image_size)) seg_img = np.where(seg_img > 0, 1, 0) # Convert segmentation mask to binary y_seg_labels.append(seg_img) X_data = np.array(X_data) y_class_labels = np.array(y_class_labels) y_seg_labels = np.array(y_seg_labels) X_data, y_class_labels, y_seg_labels = shuffle(X_data, y_class_labels, y_seg_labels, random_state=101) return X_data, y_class_labels, y_seg_labels

    def split_data(X_data, y_class_labels, y_seg_labels, train_ratio, val_ratio, test_ratio): num_samples = len(X_data) num_train_samples = int(train_ratio * num_samples) num_val_samples = int(val_ratio * num_samples) num_test_samples = int(test_ratio * num_samples) X_train = X_data[:num_train_samples] y_train_class = y_class_labels[:num_train_samples] y_train_seg = y_seg_labels[:num_train_samples] X_val = X_data[num_train_samples: num_train_samples + num_val_samples] y_val_class = y_class_labels[num_train_samples: num_train_samples + num_val_samples] y_val_seg = y_seg_labels[num_train_samples: num_train_samples + num_val_samples] X_test = X_data[num_train_samples + num_val_samples: num_train_samples + num_val_samples + num_test_samples] y_test_class = y_class_labels[num_train_samples + num_val_samples: num_train_samples + num_val_samples + num_test_samples] y_test_seg = y_seg_labels[num_train_samples + num_val_samples: num_train_samples + num_val_samples + num_test_samples] # Convert class labels to categorical label_encoder = LabelEncoder() y_train_class_encoded = label_encoder.fit_transform(y_train_class) y_train_class_categorical = to_categorical(y_train_class_encoded) y_val_class_encoded = label_encoder.transform(y_val_class) y_val_class_categorical = to_categorical(y_val_class_encoded) y_test_class_encoded = label_encoder.transform(y_test_class) y_test_class_categorical = to_categorical(y_test_class_encoded) return ( X_train, y_train_class_categorical, y_train_seg, X_val, y_val_class_categorical, y_val_seg, X_test, y_test_class_categorical, y_test_seg, )

    def build_model(input_shape, num_classes): num_filter = 32 # 16/32 best, 8: best classification but no segment # Encoder (Done) inputs = Input(input_shape) conv1 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(inputs) bn1 = BatchNormalization()(conv1) relu1 = Activation("relu")(bn1) conv2 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(relu1) bn2 = BatchNormalization()(conv2) relu2 = Activation("relu")(bn2) down1 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu2) conv3 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(down1) bn3 = BatchNormalization()(conv3) relu3 = Activation("relu")(bn3) conv4 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(relu3) bn4 = BatchNormalization()(conv4) relu4 = Activation("relu")(bn4) down2 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu4) conv5 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(down2) bn5 = BatchNormalization()(conv5) relu5 = Activation("relu")(bn5) conv6 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(relu5) bn6 = BatchNormalization()(conv6) relu6 = Activation("relu")(bn6) down3 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu6) conv7 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(down3) bn7 = BatchNormalization()(conv7) relu7 = Activation("relu")(bn7) conv8 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(relu7) bn8 = BatchNormalization()(conv8) relu8 = Activation("relu")(bn8) # Middle down4 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu8) conv9 = Conv2D(num_filter * 16, 3, activation="linear", padding="same", strides=1)(down4) bn9 = BatchNormalization()(conv9) relu9 = Activation("relu")(bn9) conv10 = Conv2D(num_filter * 16, 3, activation="linear", padding="same", strides=1)(relu9) bn10 = BatchNormalization()(conv10) relu10 = Activation("relu")(bn10) up1 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu10) # Decoder (Done) concat1 = concatenate([up1, relu8], axis=-1) # , axis=3 conv11 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(concat1) bn11 = BatchNormalization()(conv11) relu11 = Activation("relu")(bn11) conv12 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(relu11) bn12 = BatchNormalization()(conv12) relu12 = Activation("relu")(bn12) up2 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu12) concat2 = concatenate([up2, relu6], axis=-1) # , axis=3 conv13 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(concat2) bn13 = BatchNormalization()(conv13) relu13 = Activation("relu")(bn13) conv14 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(relu13) bn14 = BatchNormalization()(conv14) relu14 = Activation("relu")(bn14) up3 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu14) concat3 = concatenate([up3, relu4], axis=-1) # , axis=3 conv15 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(concat3) bn15 = BatchNormalization()(conv15) relu15 = Activation("relu")(bn15) conv16 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(relu15) bn16 = BatchNormalization()(conv16) relu16 = Activation("relu")(bn16) up4 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu16) concat4 = concatenate([up4, relu2], axis=-1) # , axis=3 conv17 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(concat4) bn17 = BatchNormalization()(conv17) relu17 = Activation("relu")(bn17) conv18 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(relu17) bn18 = BatchNormalization()(conv18) relu18 = Activation("relu")(bn18) # Segmentation branch segmentation_output = Conv2D(1, 1, activation="sigmoid", name="segmentation_output")(relu18) # original # Classification branch (Not done) gap1 = GlobalAveragePooling2D()(relu8) gap2 = GlobalAveragePooling2D()(relu10) gap3 = GlobalAveragePooling2D()(relu12) conv20 = Conv2D(16, 3, activation="linear", padding="same", strides=1)(segmentation_output) bn20 = BatchNormalization()(conv20) relu20 = Activation("relu")(bn20) down5 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu20) conv21 = Conv2D(32, 3, activation="linear", padding="same", strides=1)(down5) bn21 = BatchNormalization()(conv21) relu21 = Activation("relu")(bn21) down6 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu21) conv22 = Conv2D(64, 3, activation="linear", padding="same", strides=1)(down6) bn22 = BatchNormalization()(conv22) relu22 = Activation("relu")(bn22) down7 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu22) flatten1 = Flatten()(down7) concat5 = concatenate([gap1, gap2, gap3, flatten1], axis=-1) # FC layers fc1 = Dense(1024, activation="relu")(concat5) dropout1 = Dropout(0.5)(fc1) fc2 = Dense(1024, activation="relu")(dropout1) dropout2 = Dropout(0.5)(fc2) classification_output = Dense(num_classes, activation="softmax", name="classification_output")(dropout2) # Define the model model = Model(inputs=inputs, outputs=[classification_output, segmentation_output]) return model

    def segmentation_loss(y_true, y_pred): y_true = tf.cast(y_true, tf.float32) y_pred = tf.cast(y_pred, tf.float32) bce_loss = tf.keras.losses.binary_crossentropy(y_true, y_pred) smooth = 1e-5 intersection = tf.reduce_sum(y_true * y_pred) union = tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) dice_loss = 1.0 - 2.0 * (intersection + smooth) / (union + smooth) segmentation_loss = bce_loss + 1 * dice_loss return segmentation_loss

    def train_model(model, X_train, y_train_class, y_train_seg, X_val, y_val_class, y_val_seg, batch_size, epochs): checkpoint = ModelCheckpoint( "multitask_best_weights.h5", monitor="val_classification_output_accuracy", save_best_only=True, mode="max", verbose=1,) reduce_lr = ReduceLROnPlateau( monitor="val_classification_output_accuracy", factor=0.3, patience=2, min_delta=0.001, mode="auto", verbose=1,) tensorboard = TensorBoard(log_dir="logs") model.compile( optimizer=Adam(lr=0.001), loss={"classification_output": "categorical_crossentropy", "segmentation_output": segmentation_loss}, metrics={"classification_output": "accuracy", "segmentation_output": "accuracy"}, loss_weights={"classification_output": 1, "segmentation_output": 1},) history = model.fit( X_train, {"classification_output": y_train_class, "segmentation_output": y_train_seg}, validation_data=(X_val, {"classification_output": y_val_class, "segmentation_output": y_val_seg}), epochs=epochs, verbose=1, batch_size=batch_size, callbacks=[checkpoint, reduce_lr, tensorboard],) return history

    def evaluate_model(model, X_test, y_test_class, y_test_seg): with tf.keras.utils.custom_object_scope({"segmentation_loss": segmentation_loss}): # Load the best model weights best_model = load_model("multitask_best_weights.h5") # Evaluate the model on test data test_loss, test_class_loss, test_seg_loss, test_class_acc, test_seg_acc = best_model.evaluate( X_test, {"classification_output": y_test_class, "segmentation_output": y_test_seg}) print("Test Classification Loss:", test_class_loss) print("Test Segmentation Loss:", test_seg_loss) print("Test Classification Accuracy:", test_class_acc) print("Test Segmentation Accuracy:", test_seg_acc) # Evaluate the model on validation data val_loss, val_class_loss, val_seg_loss, val_class_acc, val_seg_acc = best_model.evaluate( X_val, {'classification_output': y_val_class, 'segmentation_output': y_val_seg}) print("Validation Classification Loss:", val_class_loss) print("Validation Segmentation Loss:", val_seg_loss) print("Validation Classification Accuracy:", val_class_acc) print("Validation Segmentation Accuracy:", val_seg_acc) # Evaluate the model on training data train_loss, train_class_loss, train_seg_loss, train_class_acc, train_seg_acc = best_model.evaluate(X_train, {'classification_output': y_train_class, 'segmentation_output': y_train_seg}) print("Train Classification Loss:", train_class_loss) print("Train Segmentation Loss:", train_seg_loss) print("Train Classification Accuracy:", train_class_acc) print("Train Segmentation Accuracy:", train_seg_acc) # Return test classification accuracy return test_class_acc

    def plot_performance(history): # Plot classification accuracy classification_train_accuracy = history.history["classification_output_accuracy"] classification_val_accuracy = history.history["val_classification_output_accuracy"] plt.figure(figsize=(7, 3)) plt.plot(classification_train_accuracy, label="Training Accuracy") plt.plot(classification_val_accuracy, label="Validation Accuracy") plt.title("Classification Accuracy") plt.xlabel("Epochs") plt.ylabel("Accuracy") plt.legend() plt.show() # Plot classification loss classification_train_loss = history.history["classification_output_loss"] classification_val_loss = history.history["val_classification_output_loss"] plt.figure(figsize=(7, 3)) plt.plot(classification_train_loss, "b", label="Training Loss") plt.plot(classification_val_loss, "r", label="Validation Loss") plt.title("Classification Loss") plt.xlabel("Epochs") plt.ylabel("Loss") plt.legend() plt.show() # Plot segmentation accuracy segmentation_train_accuracy = history.history["segmentation_output_accuracy"] segmentation_val_accuracy = history.history["val_segmentation_output_accuracy"] plt.figure(figsize=(7, 3)) plt.plot(segmentation_train_accuracy, label="Training Accuracy") plt.plot(segmentation_val_accuracy, label="Validation Accuracy") plt.title("Segmentation Accuracy") plt.xlabel("Epochs") plt.ylabel("Accuracy") plt.legend() plt.show() # Plot segmentation loss segmentation_train_loss = history.history["segmentation_output_loss"] segmentation_val_loss = history.history["val_segmentation_output_loss"] plt.figure(figsize=(7, 3)) plt.plot(segmentation_train_loss, "b", label="Training Loss") plt.plot(segmentation_val_loss, "r", label="Validation Loss") plt.title("Segmentation Loss") plt.xlabel("Epochs") plt.ylabel("Loss") plt.legend() plt.show()

    def test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5): random_indices = np.random.choice(len(X_test), num_images, replace=False) selected_images = X_test[random_indices] selected_labels = y_test_class[random_indices] selected_masks = y_test_seg[random_indices] predictions = model.predict(selected_images) predicted_classes = np.argmax(predictions[0], axis=1) predicted_masks = predictions[1] for i in range(num_images): plt.figure(figsize=(10, 5)) plt.subplot(2, 3, 1) plt.imshow(selected_images[i]) plt.title(f"Input Image\nGround Truth Label: {labels[np.argmax(selected_labels[i])]}\nPredicted Label: {labels[predicted_classes[i]]}") plt.subplot(2, 3, 2) plt.imshow(selected_masks[i].squeeze(), cmap="gray") plt.title("Ground Truth Segmentation Mask") plt.subplot(2, 3, 3) plt.imshow(predicted_masks[i].squeeze(), cmap="gray") plt.title("Predicted Segmentation Mask") plt.show()

    preprocess and predict on the selected image

    out = Output() def predict_image(image_path): # Load and preprocess the image img = cv2.imread(image_path) img = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # Convert BGR to RGB img_resized = cv2.resize(img, (image_size, image_size)) img_expanded = np.expand_dims(img_resized, axis=0) # Predict using the model class_output, seg_output = model.predict(img_expanded) # Get the predicted class predicted_class_index = np.argmax(class_output) predicted_class = labels[predicted_class_index] # Clear the previous output with out: clear_output(wait=True) # Visualize the input and segmentation output fig, axs = plt.subplots(1, 3, figsize=(15, 5)) axs[0].imshow(img) axs[0].set_title("Input Image") axs[0].axis("off")

        axs[1].imshow(seg_output[0, :, :, 0], cmap='gray')
        axs[1].set_title("Segmentation Output")
        axs[1].axis("off")
        
        class_probabilities = class_output[0]
        x_pos = np.arange(len(labels))
        axs[2].bar(x_pos, class_probabilities, align='center', alpha=0.7, color='blue')
        axs[2].set_xticks(x_pos)
        axs[2].set_xticklabels(labels, rotation=45, ha="right")
        axs[2].set_ylabel('Probability')
        axs[2].set_ylim([0, 1])
        axs[2].set_title(f'Classification Output\nPredicted Class: {predicted_class}')  # Increased title font size
        
        for i, v in enumerate(class_probabilities):
            axs[2].text(i, v + 0.02, f"{v:.2f}", color='black', ha='center')
        
        plt.tight_layout()
        plt.show()
    

    Define a callback function to handle the uploaded file

    def on_upload(change): # Get the uploaded file uploaded_filename = next(iter(uploader.value)) uploaded_file = uploader.value[uploaded_filename] # Save the uploaded file with open(uploaded_filename, 'wb') as f: f.write(uploaded_file['content']) # Predict and visualize the image predict_image(uploaded_filename)

    Define a function to handle the button click event and clear the output

    clear_button = Button(description='Clear') def clear_output_button_clicked(button): out.clear_output() clear_button.on_click(clear_output_button_clicked)

    Set image size

    image_size = 224

    Define labels

    labels = ["bridge", "excess", "good", "insuff", "no"]

    Set data folders

    data_folders = [ "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/train", "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/val", "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/test",]

    Load data

    X_data, y_class_labels, y_seg_labels = load_data(data_folders)

    Set image size

    image_size = 224

    Split data

    train_ratio = 0.2 val_ratio = 0.2 test_ratio = 0.6 X_train, y_train_class, y_train_seg, X_val, y_val_class, y_val_seg, X_test, y_test_class, y_test_seg = split_data( X_data, y_class_labels, y_seg_labels, train_ratio, val_ratio, test_ratio)

    Print the counts

    print("Number of train images:", len(X_train)) print("Number of train binary masks:", len(y_train_seg)) print("Number of validation images:", len(X_val)) print("Number of validation binary masks:", len(y_val_seg)) print("Number of test images:", len(X_test)) print("Number of test binary masks:", len(y_test_seg)) input_shape = (image_size, image_size, 3) num_classes = len(labels) saved_model_path = "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/model/pcbmtl_bdip.h5" custom_objects = {"segmentation_loss": segmentation_loss}

    Load model

    model = load_model(saved_model_path, custom_objects=custom_objects)

    Test model

    test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5)

    file uploader widget

    uploader = widgets.FileUpload(accept='image/*', multiple=False) uploader.observe(on_upload, names='value') buttons_layout = HBox([uploader, clear_button]) display(buttons_layout) display(out)

    "

  10. Assistant

    To apply the requested background color (#303030) and make titles bold across all the plots in your provided code, you need to make adjustments in every plotting function. In your original code, the key areas to adjust are within the plotting sections of the plot_performance function and the test_model function. Here's an example of how to apply these adjustments to the plot_performance function to ensure consistency in the appearance of all plots:

    def plot_performance(history):
        bg_color = '#303030' # Background color for plots
        text_color = 'white' # Text color for better visibility on darker backgrounds
        
        plt.rcParams['figure.facecolor'] = bg_color
        plt.rcParams['axes.facecolor'] = bg_color
        plt.rcParams['axes.edgecolor'] = 'grey'
        plt.rcParams['axes.labelcolor'] = text_color
        plt.rcParams['text.color'] = text_color
        plt.rcParams['xtick.color'] = text_color
        plt.rcParams['ytick.color'] = text_color
    
        # Plot classification accuracy
        classification_train_accuracy = history.history["classification_output_accuracy"]
        classification_val_accuracy = history.history["val_classification_output_accuracy"]
        plt.figure(figsize=(7, 3))
        plt.plot(classification_train_accuracy, label="Training Accuracy")
        plt.plot(classification_val_accuracy, label="Validation Accuracy")
        plt.title("Classification Accuracy", fontweight='bold')
        plt.xlabel("Epochs")
        plt.ylabel("Accuracy")
        plt.legend()
        plt.show()
    
        # Plot classification loss
        classification_train_loss = history.history["classification_output_loss"]
        classification_val_loss = history.history["val_classification_output_loss"]
        plt.figure(figsize=(7, 3))
        plt.plot(classification_train_loss, "b", label="Training Loss")
        plt.plot(classification_val_loss, "r", label="Validation Loss")
        plt.title("Classification Loss", fontweight='bold')
        plt.xlabel("Epochs")
        plt.ylabel("Loss")
        plt.legend()
        plt.show()
    
        # Plot segmentation accuracy
        segmentation_train_accuracy = history.history["segmentation_output_accuracy"]
        segmentation_val_accuracy = history.history["val_segmentation_output_accuracy"]
        plt.figure(figsize=(7, 3))
        plt.plot(segmentation_train_accuracy, label="Training Accuracy")
        plt.plot(segmentation_val_accuracy, label="Validation Accuracy")
        plt.title("Segmentation Accuracy", fontweight='bold')
        plt.xlabel("Epochs")
        plt.ylabel("Accuracy")
        plt.legend()
        plt.show()
    
        # Plot segmentation loss
        segmentation_train_loss = history.history["segmentation_output_loss"]
        segmentation_val_loss = history.history["val_segmentation_output_loss"]
        plt.figure(figsize=(7, 3))
        plt.plot(segmentation_train_loss, "b", label="Training Loss")
        plt.plot(segmentation_val_loss, "r", label="Validation Loss")
        plt.title("Segmentation Loss", fontweight='bold')
        plt.xlabel("Epochs")
        plt.ylabel("Loss")
        plt.legend()
        plt.show()
    
        # Reset to defaults for other plots that may not want these settings
        plt.rcParams['figure.facecolor'] = 'white'
        plt.rcParams['axes.facecolor'] = 'white'
        plt.rcParams['axes.edgecolor'] = 'black'
        plt.rcParams['axes.labelcolor'] = 'black'
        plt.rcParams['text.color'] = 'black'
        plt.rcParams['xtick.color'] = 'black'
        plt.rcParams['ytick.color'] = 'black'
    

    Notice how plt.rcParams is used to set the color configurations that will be applied to all plots. This approach ensures that the background color, text color, and title weight are consistently applied across your plots for a uniform appearance.

    For the test_model function, you would apply similar changes within each section where plotting occurs. Remember to adjust the plt.rcParams as shown above before each visualize section where you plot images and after to reset to defaults if needed elsewhere.

    These adjustments give your plots a consistent look and feel with the desired background color and bold titles, enhancing readability and visual appeal.

  11. User

    Also for test_model

  12. Assistant

    Certainly! To apply the same styling with the background color #303030, white text, and bold titles to the plots generated in the test_model function, you can use a similar approach by setting plt.rcParams or directly applying styling within the function.

    Given the specific way you might want to control each plot (especially when creating subplots or when dealing with images), it might be more straightforward to apply the styling directly within each plotting command for test_model. Here's how you can adjust your test_model function to match the requested styling:

    def test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5):
        random_indices = np.random.choice(len(X_test), num_images, replace=False)
        selected_images = X_test[random_indices]
        selected_labels = y_test_class[random_indices]
        selected_masks = y_test_seg[random_indices]
        predictions = model.predict(selected_images)
        predicted_classes = np.argmax(predictions[0], axis=1)
        predicted_masks = predictions[1]
        
        bg_color = '#303030'  # Lighter than black, as per the previous specification
        text_color = 'white'
        
        for i in range(num_images):
            plt.figure(figsize=(10, 5))
            
            plt.subplot(2, 3, 1)
            plt.imshow(selected_images[i])
            plt.title(f"Input Image\nGround Truth Label: {labels[np.argmax(selected_labels[i])]}\nPredicted Label: {labels[predicted_classes[i]]}", color=text_color, fontweight='bold')
            plt.axis("off")
            plt.gcf().set_facecolor(bg_color)
            
            plt.subplot(2, 3, 2)
            plt.imshow(selected_masks[i].squeeze(), cmap="gray")
            plt.title("Ground Truth Segmentation Mask", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.subplot(2, 3, 3)
            plt.imshow(predicted_masks[i].squeeze(), cmap="gray")
            plt.title("Predicted Segmentation Mask", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.tight_layout()
            plt.show()
    
    

    Key adjustments:

    • Before each subplot (plt.subplot(...)), it configures text color and font weight directly in the plotting commands like plt.title(...).
    • Sets the facecolor of the figure to the specified background color right after calling plt.figure(...).

    These tweaks ensure that your plots from the test_model function will have a consistent aesthetic with the rest of your visualizations: dark-themed background, white text, and bold titles. This approach does not modify the global plt.rcParams for the current script execution context, hence affecting only the plots within this specific function call, offering you finer control over the styling of each plot.

  13. User

    can you make the resolution of the plot higher?

  14. Assistant

    To enhance the resolution of plots, especially when dealing with images and intricate visualizations, you can set the dpi (dots per inch) parameter when creating a figure with plt.figure(). A higher dpi value results in higher resolution images. This can be particularly useful when you want to scrutinize details within images or when preparing plots for high-quality reports and presentations.

    Here's how you can adjust the test_model function to increase the plot resolution by specifying a higher dpi value:

    def test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5):
        random_indices = np.random.choice(len(X_test), num_images, replace=False)
        selected_images = X_test[random_indices]
        selected_labels = y_test_class[random_indices]
        selected_masks = y_test_seg[random_indices]
        predictions = model.predict(selected_images)
        predicted_classes = np.argmax(predictions[0], axis=1)
        predicted_masks = predictions[1]
        
        bg_color = '#303030'  # Background color
        text_color = 'white'  # Text color
        dpi_setting = 150  # Higher dpi for better resolution
        
        for i in range(num_images):
            plt.figure(figsize=(10, 5), dpi=dpi_setting)
            
            plt.subplot(2, 3, 1)
            plt.imshow(selected_images[i])
            plt.title(f"Input Image\nGround Truth Label: {labels[np.argmax(selected_labels[i])]}\nPredicted Label: {labels[predicted_classes[i]]}", color=text_color, fontweight='bold')
            plt.axis("off")
            plt.gcf().set_facecolor(bg_color)
            
            plt.subplot(2, 3, 2)
            plt.imshow(selected_masks[i].squeeze(), cmap="gray")
            plt.title("Ground Truth Segmentation Mask", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.subplot(2, 3, 3)
            plt.imshow(predicted_masks[i].squeeze(), cmap="gray")
            plt.title("Predicted Segmentation Mask", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.tight_layout()
            plt.show()
    

    In this adjusted version, by setting dpi=dpi_setting when calling plt.figure(), the resolution of each plot is increased according to the dpi_setting value. You may adjust this value to suit your needs; a higher dpi results in higher resolution, but also larger file sizes for saved images and potentially longer rendering times.

    The value of 150 dpi is chosen as a compromise between quality and performance, but for publication-quality images, you might consider increasing this to 300 dpi or higher. Remember, when displaying high-dpi images on screen (e.g., in Jupyter Notebooks), the perceived quality improvement may vary depending on your display's characteristics. However, this increase will be more apparent when saving figures to files, especially for printing purposes or inclusion in high-quality documents.

  15. User

    The plot is become bigger

  16. Assistant

    When you increase the dpi (dots per inch) parameter, you enhance the resolution of the figure without intending to scale up its size. However, the physical or displayed size of the figure might seem larger, particularly in environments like Jupyter notebooks or other screen-based displays, because the number of pixels increases while the physical size (in inches) set by figsize remains the same.

    If the plot appears too large due to the increased dpi, but you want to maintain the high resolution, you can adjust the figsize parameter accordingly to reduce the displayed size of the plot. The figsize parameter controls the size of the figure in inches, and coupled with a higher dpi, you can achieve a higher resolution image without unnecessarily increasing its displayed or physical size.

    Here's how you can adjust the plot size while keeping a higher dpi for better resolution:

    def test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5):
        random_indices = np.random.choice(len(X_test), num_images, replace=False)
        selected_images = X_test[random_indices]
        selected_labels = y_test_class[random_indices]
        selected_masks = y_test_seg[random_indices]
        predictions = model.predict(selected_images)
        predicted_classes = np.argmax(predictions[0], axis=1)
        predicted_masks = predictions[1]
        
        bg_color = '#303030'  # Background color
        text_color = 'white'  # Text color
        dpi_setting = 150  # Higher dpi for better resolution
        figure_size = (10, 5)  # Adjust figure size here if it seems too large on your display
    
        for i in range(num_images):
            # Adjust figsize according to your display or requirements
            plt.figure(figsize=figure_size, dpi=dpi_setting)
            
            plt.subplot(2, 3, 1)
            plt.imshow(selected_images[i])
            plt.title(f"Input Image\nGround Truth Label: {labels[np.argmax(selected_labels[i])]}\nPredicted Label: {labels[predicted_classes[i]]}", color=text_color, fontweight='bold')
            plt.axis("off")
            plt.gcf().set_facecolor(bg_color)
            
            plt.subplot(2, 3, 2)
            plt.imshow(selected_masks[i].squeeze(), cmap="gray")
            plt.title("Ground Truth Segmentation Mask", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.subplot(2, 3, 3)
            plt.imshow(predicted_masks[i].squeeze(), cmap="gray")
            plt.title("Predicted Segmentation Mask", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.tight_layout()
            plt.show()
    

    By carefully choosing figsize and dpi, you can manage both the visual quality of your figures and their size on the screen or page. This adjustment helps in maintaining clarity and detail for both real-time presentation and for prints or documentation purposes, without overwhelming the viewer with unnecessarily large images. If you're still finding the images too large, consider reducing the figure_size dimensions further.

  17. User

    What are the value in figure_size = (10, 5)

  18. Assistant

    The figure_size = (10, 5) in the code represents the dimensions of the figure in inches. This parameter sets the width and height of the figure that will be created. Specifically:

    • 10 is the width of the figure in inches.
    • 5 is the height of the figure in inches.

    These values determine the size of the figure "canvas" onto which your plots are drawn. When you're working in environments like Jupyter notebooks, matplotlib visualizations are translated to a display size on your screen based on these inch measurements in combination with the dpi (dots per inch) setting.

    Adjusting the figsize:

    • Reducing the figsize values will produce a smaller figure, handy if the plot seems too large for your screen or if you aim for a more compact visualization.
    • Increasing the figsize values results in a larger figure, which might be useful when you need to highlight details or for presentations where larger visuals are preferred.

    The dpi setting further influences the resolution of the figure. A higher dpi means more pixels will be used to draw the figure, leading to clearer and more detailed images, which is particularly beneficial when the plot contains fine details or when the figure is meant for high-quality print publication.

    Together, figsize and dpi allow you to control both the physical size and the resolution (quality) of your plots in Matplotlib. If you want a high-resolution plot without physically enlarging the figure on your display or in print, you would increase dpi while maintaining or slightly adjusting the figsize to your preference.

  19. User

    " import matplotlib.pyplot as plt import numpy as np import pandas as pd import seaborn as sns import cv2 import random import tensorflow as tf import tkinter as tk from tkinter import filedialog from PIL import ImageTk, Image from ipywidgets import interact, interactive, fixed, interact_manual import ipywidgets as widgets from ipywidgets import Output, Button from warnings import filterwarnings from IPython.display import display, clear_output from tensorflow.keras.preprocessing import image from tensorflow.keras.optimizers import Adam from tensorflow.keras.preprocessing.image import ImageDataGenerator from tqdm import tqdm import os from sklearn.utils import shuffle from sklearn.preprocessing import LabelEncoder from sklearn.model_selection import train_test_split from tensorflow.keras.models import Sequential, Model, load_model from tensorflow.keras.layers import ( GlobalAveragePooling2D, Dropout, Dense, Conv2D, MaxPooling2D, Flatten, Dropout, BatchNormalization, Activation, concatenate, Conv2DTranspose, Input, Reshape, UpSampling2D, ) from tensorflow.keras.utils import to_categorical from tensorflow.keras.callbacks import EarlyStopping, ReduceLROnPlateau, TensorBoard, ModelCheckpoint from sklearn.metrics import classification_report, confusion_matrix from google.colab import drive drive.mount("/content/gdrive") def load_data(data_folders): X_data = []
    y_class_labels = []
    y_seg_labels = [] for folderPath in data_folders: for label in labels: label_folder_path = os.path.join(folderPath, label) for filename in tqdm(os.listdir(label_folder_path)): if filename.endswith(".jpg"): img = cv2.imread(os.path.join(label_folder_path, filename)) img = cv2.resize(img, (image_size, image_size)) X_data.append(img) y_class_labels.append(label) seg_filename = filename.split(".")[0] + ".png" seg_img = cv2.imread(os.path.join(label_folder_path, seg_filename), 0) seg_img = cv2.resize(seg_img, (image_size, image_size)) seg_img = np.where(seg_img > 0, 1, 0) y_seg_labels.append(seg_img) X_data = np.array(X_data) y_class_labels = np.array(y_class_labels) y_seg_labels = np.array(y_seg_labels) X_data, y_class_labels, y_seg_labels = shuffle(X_data, y_class_labels, y_seg_labels, random_state=101) return X_data, y_class_labels, y_seg_labels

    def split_data(X_data, y_class_labels, y_seg_labels, class_data_counts): X_train = [] y_train_class = [] y_train_seg = [] X_val = [] y_val_class = [] y_val_seg = [] X_test = [] y_test_class = [] y_test_seg = [] for label, count in class_data_counts.items(): label_indices = np.where(y_class_labels == label)[0] class_X_data = X_data[label_indices] class_y_class_labels = y_class_labels[label_indices] class_y_seg_labels = y_seg_labels[label_indices] train_count = count[0] val_count = count[1] test_count = count[2] class_X_train = class_X_data[:train_count] class_y_train_class = class_y_class_labels[:train_count] class_y_train_seg = class_y_seg_labels[:train_count] class_X_val = class_X_data[train_count: train_count + val_count] class_y_val_class = class_y_class_labels[train_count: train_count + val_count] class_y_val_seg = class_y_seg_labels[train_count: train_count + val_count] class_X_test = class_X_data[train_count + val_count: train_count + val_count + test_count] class_y_test_class = class_y_class_labels[train_count + val_count: train_count + val_count + test_count] class_y_test_seg = class_y_seg_labels[train_count + val_count: train_count + val_count + test_count] X_train.extend(class_X_train) y_train_class.extend(class_y_train_class) y_train_seg.extend(class_y_train_seg) X_val.extend(class_X_val) y_val_class.extend(class_y_val_class) y_val_seg.extend(class_y_val_seg) X_test.extend(class_X_test) y_test_class.extend(class_y_test_class) y_test_seg.extend(class_y_test_seg) label_encoder = LabelEncoder() y_train_class_encoded = label_encoder.fit_transform(y_train_class) y_train_class_categorical = to_categorical(y_train_class_encoded) y_val_class_encoded = label_encoder.transform(y_val_class) y_val_class_categorical = to_categorical(y_val_class_encoded) y_test_class_encoded = label_encoder.transform(y_test_class) y_test_class_categorical = to_categorical(y_test_class_encoded)

    return (
        np.array(X_train),
        np.array(y_train_class_categorical),
        np.array(y_train_seg),
        np.array(X_val),
        np.array(y_val_class_categorical),
        np.array(y_val_seg),
        np.array(X_test),
        np.array(y_test_class_categorical),
        np.array(y_test_seg),
    )
    

    def count_labels(y_class_categorical, label_encoder): y_class_labels = np.argmax(y_class_categorical, axis=1) y_class_names = label_encoder.inverse_transform(y_class_labels) unique, counts = np.unique(y_class_names, return_counts=True) return dict(zip(unique, counts))

    def build_model(input_shape, num_classes): num_filter = 16
    inputs = Input(input_shape) conv1 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(inputs) bn1 = BatchNormalization()(conv1) relu1 = Activation("relu")(bn1) conv2 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(relu1) bn2 = BatchNormalization()(conv2) relu2 = Activation("relu")(bn2) down1 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu2) conv3 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(down1) bn3 = BatchNormalization()(conv3) relu3 = Activation("relu")(bn3) conv4 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(relu3) bn4 = BatchNormalization()(conv4) relu4 = Activation("relu")(bn4) down2 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu4) conv5 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(down2) bn5 = BatchNormalization()(conv5) relu5 = Activation("relu")(bn5) conv6 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(relu5) bn6 = BatchNormalization()(conv6) relu6 = Activation("relu")(bn6) down3 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu6) conv7 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(down3) bn7 = BatchNormalization()(conv7) relu7 = Activation("relu")(bn7) conv8 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(relu7) bn8 = BatchNormalization()(conv8) relu8 = Activation("relu")(bn8) down4 = MaxPooling2D(pool_size=(2, 2), strides=2)(relu8) conv9 = Conv2D(num_filter * 16, 3, activation="linear", padding="same", strides=1)(down4) bn9 = BatchNormalization()(conv9) relu9 = Activation("relu")(bn9) conv10 = Conv2D(num_filter * 16, 3, activation="linear", padding="same", strides=1)(relu9) bn10 = BatchNormalization()(conv10) relu10 = Activation("relu")(bn10) up1 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu10) concat1 = concatenate([up1, relu8], axis=-1)
    conv11 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(concat1) bn11 = BatchNormalization()(conv11) relu11 = Activation("relu")(bn11) conv12 = Conv2D(num_filter * 8, 3, activation="linear", padding="same", strides=1)(relu11) bn12 = BatchNormalization()(conv12) relu12 = Activation("relu")(bn12) up2 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu12) concat2 = concatenate([up2, relu6], axis=-1)
    conv13 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(concat2) bn13 = BatchNormalization()(conv13) relu13 = Activation("relu")(bn13) conv14 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(relu13) bn14 = BatchNormalization()(conv14) relu14 = Activation("relu")(bn14) up3 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu14) concat3 = concatenate([up3, relu4], axis=-1)
    conv15 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(concat3) bn15 = BatchNormalization()(conv15) relu15 = Activation("relu")(bn15) conv16 = Conv2D(num_filter * 2, 3, activation="linear", padding="same", strides=1)(relu15) bn16 = BatchNormalization()(conv16) relu16 = Activation("relu")(bn16) up4 = UpSampling2D(size=(2, 2), interpolation="bilinear")(relu16) concat4 = concatenate([up4, relu2], axis=-1)
    conv17 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(concat4) bn17 = BatchNormalization()(conv17) relu17 = Activation("relu")(bn17) conv18 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(relu17) bn18 = BatchNormalization()(conv18) relu18 = Activation("relu")(bn18) segmentation_output = Conv2D(1, 1, activation="sigmoid", name="segmentation_output")(relu18) gap1 = GlobalAveragePooling2D()(relu8) gap2 = GlobalAveragePooling2D()(relu10) gap3 = GlobalAveragePooling2D()(relu12) conv20 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(segmentation_output) bn20 = BatchNormalization()(conv20) relu20 = Activation("relu")(bn20) down5 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu20) conv21 = Conv2D(num_filter * 4, 3, activation="linear", padding="same", strides=1)(down5) bn21 = BatchNormalization()(conv21) relu21 = Activation("relu")(bn21) down6 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu21) conv22 = Conv2D(num_filter * 1, 3, activation="linear", padding="same", strides=1)(down6) bn22 = BatchNormalization()(conv22) relu22 = Activation("relu")(bn22) down7 = MaxPooling2D(pool_size=(4, 4), strides=4)(relu22) flatten1 = Flatten()(down7) concat5 = concatenate([gap1, gap2, gap3, flatten1], axis=-1) fc1 = Dense(1024, activation="relu")(concat5) dropout1 = Dropout(0.5)(fc1) fc2 = Dense(1024, activation="relu")(dropout1) dropout2 = Dropout(0.5)(fc2) classification_output = Dense(num_classes, activation="softmax", name="classification_output")(dropout2) model = Model(inputs=inputs, outputs=[classification_output, segmentation_output]) return model

    def segmentation_loss(y_true, y_pred): y_true = tf.cast(y_true, tf.float32) y_pred = tf.cast(y_pred, tf.float32) bce_loss = tf.keras.losses.binary_crossentropy(y_true, y_pred) smooth = 1e-5 intersection = tf.reduce_sum(y_true * y_pred) union = tf.reduce_sum(y_true) + tf.reduce_sum(y_pred) dice_loss = 1.0 - 2.0 * (intersection + smooth) / (union + smooth) segmentation_loss = bce_loss + 1 * dice_loss return segmentation_loss

    Categorical_Focal_loss = tf.keras.losses.CategoricalFocalCrossentropy( alpha=0.25, gamma=2.0, from_logits=False, label_smoothing=0.0, axis=-1,)

    def train_model(model, X_train, y_train_class, y_train_seg, X_val, y_val_class, y_val_seg, batch_size, epochs): checkpoint = ModelCheckpoint( "multitask_best_weights.h5", monitor="val_classification_output_accuracy", save_best_only=True, mode="max", verbose=1,) reduce_lr = ReduceLROnPlateau( monitor="val_classification_output_accuracy", factor=0.3, patience=2, min_delta=0.001, mode="auto", verbose=1,) tensorboard = TensorBoard(log_dir="logs") model.compile( optimizer=Adam(learning_rate=0.001), loss={"classification_output": Categorical_Focal_loss, "segmentation_output": segmentation_loss}, metrics={"classification_output": "accuracy", "segmentation_output": "accuracy"}, loss_weights={"classification_output": 1, "segmentation_output": 1},) history = model.fit( X_train, {"classification_output": y_train_class, "segmentation_output": y_train_seg}, validation_data=(X_val, {"classification_output": y_val_class, "segmentation_output": y_val_seg}), epochs=epochs, verbose=1, batch_size=batch_size, callbacks=[checkpoint, reduce_lr, tensorboard],) return history

    def evaluate_model(model, X_test, y_test_class, y_test_seg): with tf.keras.utils.custom_object_scope({"segmentation_loss": segmentation_loss}): best_model = load_model("multitask_best_weights.h5") test_loss, test_class_loss, test_seg_loss, test_class_acc, test_seg_acc = best_model.evaluate( X_test, {"classification_output": y_test_class, "segmentation_output": y_test_seg}) print("Test Classification Loss:", test_class_loss) print("Test Segmentation Loss:", test_seg_loss) print("Test Classification Accuracy:", test_class_acc) print("Test Segmentation Accuracy:", test_seg_acc) val_loss, val_class_loss, val_seg_loss, val_class_acc, val_seg_acc = best_model.evaluate( X_val, {'classification_output': y_val_class, 'segmentation_output': y_val_seg}) print("Validation Classification Loss:", val_class_loss) print("Validation Segmentation Loss:", val_seg_loss) print("Validation Classification Accuracy:", val_class_acc) print("Validation Segmentation Accuracy:", val_seg_acc) train_loss, train_class_loss, train_seg_loss, train_class_acc, train_seg_acc = best_model.evaluate(X_train, {'classification_output': y_train_class, 'segmentation_output': y_train_seg}) print("Train Classification Loss:", train_class_loss) print("Train Segmentation Loss:", train_seg_loss) print("Train Classification Accuracy:", train_class_acc) print("Train Segmentation Accuracy:", train_seg_acc) return test_class_acc

    def plot_performance(history): bg_color = '#303030'
    text_color = 'white' plt.rcParams['figure.facecolor'] = bg_color plt.rcParams['axes.facecolor'] = bg_color plt.rcParams['axes.edgecolor'] = 'grey' plt.rcParams['axes.labelcolor'] = text_color plt.rcParams['text.color'] = text_color plt.rcParams['xtick.color'] = text_color plt.rcParams['ytick.color'] = text_color

    # Plot classification accuracy
    classification_train_accuracy = history.history["classification_output_accuracy"]
    classification_val_accuracy = history.history["val_classification_output_accuracy"]
    plt.figure(figsize=(7, 3))
    plt.plot(classification_train_accuracy, label="Training Accuracy")
    plt.plot(classification_val_accuracy, label="Validation Accuracy")
    plt.title("Classification Accuracy", fontweight='bold')
    plt.xlabel("Epochs")
    plt.ylabel("Accuracy")
    plt.legend()
    plt.show()
    
    # Plot classification loss
    classification_train_loss = history.history["classification_output_loss"]
    classification_val_loss = history.history["val_classification_output_loss"]
    plt.figure(figsize=(7, 3))
    plt.plot(classification_train_loss, "b", label="Training Loss")
    plt.plot(classification_val_loss, "r", label="Validation Loss")
    plt.title("Classification Loss", fontweight='bold')
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.legend()
    plt.show()
    
    # Plot segmentation accuracy
    segmentation_train_accuracy = history.history["segmentation_output_accuracy"]
    segmentation_val_accuracy = history.history["val_segmentation_output_accuracy"]
    plt.figure(figsize=(7, 3))
    plt.plot(segmentation_train_accuracy, label="Training Accuracy")
    plt.plot(segmentation_val_accuracy, label="Validation Accuracy")
    plt.title("Segmentation Accuracy", fontweight='bold')
    plt.xlabel("Epochs")
    plt.ylabel("Accuracy")
    plt.legend()
    plt.show()
    
    # Plot segmentation loss
    segmentation_train_loss = history.history["segmentation_output_loss"]
    segmentation_val_loss = history.history["val_segmentation_output_loss"]
    plt.figure(figsize=(7, 3))
    plt.plot(segmentation_train_loss, "b", label="Training Loss")
    plt.plot(segmentation_val_loss, "r", label="Validation Loss")
    plt.title("Segmentation Loss", fontweight='bold')
    plt.xlabel("Epochs")
    plt.ylabel("Loss")
    plt.legend()
    plt.show()
    
    # Reset to defaults for other plots that may not want these settings
    plt.rcParams['figure.facecolor'] = 'white'
    plt.rcParams['axes.facecolor'] = 'white'
    plt.rcParams['axes.edgecolor'] = 'black'
    plt.rcParams['axes.labelcolor'] = 'black'
    plt.rcParams['text.color'] = 'black'
    plt.rcParams['xtick.color'] = 'black'
    plt.rcParams['ytick.color'] = 'black'
    

    def test_model(model, X_test, y_test_class, y_test_seg, labels, num_images=5): random_indices = np.random.choice(len(X_test), num_images, replace=False) selected_images = X_test[random_indices] selected_labels = y_test_class[random_indices] selected_masks = y_test_seg[random_indices] predictions = model.predict(selected_images) predicted_classes = np.argmax(predictions[0], axis=1) predicted_masks = predictions[1] for i in range(num_images): plt.figure(figsize=(10, 5)) plt.subplot(2, 3, 1) plt.imshow(selected_images[i]) plt.title(f"Input Image\nGround Truth Label: {labels[np.argmax(selected_labels[i])]}\nPredicted Label: {labels[predicted_classes[i]]}") plt.subplot(2, 3, 2) plt.imshow(selected_masks[i].squeeze(), cmap="gray") plt.title("Ground Truth Segmentation Mask") plt.subplot(2, 3, 3) plt.imshow(predicted_masks[i].squeeze(), cmap="gray") plt.title("Predicted Segmentation Mask") plt.show() out = Output() def predict_image(image_path): img = cv2.imread(image_path) img = cv2.resize(img, (image_size, image_size)) img = np.expand_dims(img, axis=0) class_output, seg_output = model.predict(img) predicted_class_index = np.argmax(class_output) predicted_class = labels[predicted_class_index] with out: clear_output(wait=True) plt.figure(figsize=(10, 5)) plt.subplot(1, 3, 1) plt.imshow(cv2.cvtColor(cv2.imread(image_path), cv2.COLOR_BGR2RGB)) plt.title("Input Image") plt.axis("off") plt.subplot(1, 3, 2) plt.imshow(seg_output[0, :, :, 0], cmap='gray') plt.title("Segmentation Output") plt.axis("off") plt.subplot(1, 3, 3) class_probabilities = class_output[0] y_pos = np.arange(len(labels)) plt.barh(y_pos, class_probabilities, align='center') plt.yticks(y_pos, labels) plt.xlabel('Probability') plt.title('Classification Output') plt.gca().invert_yaxis() plt.tight_layout() print("Predicted Class:", predicted_class) print("Class Probabilities:") for label, prob in zip(labels, class_probabilities): print(f"{label}: {prob:.4f}") plt.show()

    def on_upload(change): uploaded_filename = next(iter(uploader.value)) uploaded_file = uploader.value[uploaded_filename] with open(uploaded_filename, 'wb') as f: f.write(uploaded_file['content']) predict_image(uploaded_filename)

    clear_button = Button(description='Clear') def clear_output_button_clicked(button): out.clear_output() clear_button.on_click(clear_output_button_clicked)

    image_size = 224 labels = ["bridge", "excess", "good", "insuff", "no"] data_folders = [ "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/train", "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/val", "/content/gdrive/MyDrive/Deep learning/FYP_2/4 Dataset Ratio 60 20 20/jit012/jit0/b_dip/test",] X_data, y_class_labels, y_seg_labels = load_data(data_folders) class_data_counts = { "bridge": [80, 80, 80], "excess": [80, 80, 80], "good": [80, 80, 80], "insuff": [80, 80, 80], "no": [80, 80, 80]}

    X_train, y_train_class, y_train_seg, X_val, y_val_class, y_val_seg, X_test, y_test_class, y_test_seg = split_data( X_data, y_class_labels, y_seg_labels, class_data_counts) label_encoder = LabelEncoder() label_encoder.fit(y_class_labels) train_counts = count_labels(y_train_class, label_encoder) val_counts = count_labels(y_val_class, label_encoder) test_counts = count_labels(y_test_class, label_encoder) print("Train counts: ", train_counts," Total in train set:", sum(train_counts.values())) print("Validation counts:", val_counts, " Total in validation set:", sum(val_counts.values())) print("Test counts: ", test_counts," Total in test set:", sum(test_counts.values())) input_shape = (image_size, image_size, 3) num_classes = len(labels) model = build_model(input_shape, num_classes) model.summary() model = build_model(input_shape, num_classes) batch_size = 16 epochs = 2 history = train_model(model, X_train, y_train_class, y_train_seg, X_val, y_val_class, y_val_seg, batch_size, epochs) test_class_acc = evaluate_model(model, X_test, y_test_class, y_test_seg) plot_performance(history) " The above is the Python code with Keras to train the multi-task learning model, this model will take the image as input and provide one classification output and one segmentation output. I want to plot some images (e.g. 10) with their segmentation masks and labels randomly before training. Use '#303030' as background color.

  20. Assistant

    To plot some images with their segmentation masks and labels randomly before training, and using '#303030' as the background color, you can create a function plot_sample_images that randomly selects images, displays them alongside their segmentation masks, and prints their labels. This function can be called before starting the training process. I'll integrate this function into your existing code structure.

    Here's how you could implement and call this function:

    import matplotlib.pyplot as plt
    import numpy as np
    
    def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=10):
        bg_color = '#303030'  # Background color
        text_color = 'white'  # Text color
        indices = np.random.choice(range(len(X_data)), num_images, replace=False)
        
        plt.rcParams['figure.facecolor'] = bg_color
        plt.rcParams['axes.facecolor'] = bg_color
        plt.rcParams['axes.edgecolor'] = 'grey'
        plt.rcParams['axes.labelcolor'] = text_color
        plt.rcParams['text.color'] = text_color
        plt.rcParams['xtick.color'] = text_color
        plt.rcParams['ytick.color'] = text_color
        
        for i, index in enumerate(indices):
            img = X_data[index]
            seg = y_seg_labels[index] * 255  # Assuming the segmentation masks are normalized
            label = y_class_labels[index]
            
            plt.figure(figsize=(10, 5))
            
            plt.subplot(1, 3, 1)
            plt.imshow(img)
            plt.title(f"Input Image - {labels[label]}", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.subplot(1, 3, 2)
            plt.imshow(seg, cmap='gray')
            plt.title("Segmentation Mask", color=text_color, fontweight='bold')
            plt.axis("off")
            
            plt.tight_layout()
            plt.show()
            
        # Reset to defaults to avoid affecting other plots
        plt.rcParams['figure.facecolor'] = 'white'
        plt.rcParams['axes.facecolor'] = 'white'
        plt.rcParams['axes.edgecolor'] = 'black'
        plt.rcParams['axes.labelcolor'] = 'black'
        plt.rcParams['text.color'] = 'black'
        plt.rcParams['xtick.color'] = 'black'
        plt.rcParams['ytick.color'] = 'black'
    
    # Example usage (call this before starting the training process):
    plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=10)
    

    Notes:

    1. The function plot_sample_images selects a specified number (defaulting to 10) of random images from your dataset using np.random.choice.
    2. It then iterates over these selected indices, retrieves the corresponding image data, segmentation mask data, and labels, and plots them using matplotlib. The segmentation mask is assumed to be normalized (0 to 1 range), so it's multiplied by 255 for visualization purposes. If your segmentation masks are already in the appropriate format (e.g., 0 to 255 range), you might need to adjust this line accordingly.
    3. I've incorporated the desired dark background color and text color settings through plt.rcParams and reset these settings to their defaults after plotting to avoid affecting other plots in your notebook or script.
    4. Make sure the labels array correctly maps the class indices in y_class_labels to their respective class names for proper annotation in the plot titles.

    By calling this function before your training process, you can visually inspect a sample of your images and their corresponding segmentation masks, helping to validate data preparation steps or for exploratory data analysis purposes.

  21. User

    TypeError Traceback (most recent call last) in <cell line: 1>() ----> 1 plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=10)

    in plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images) 103 plt.subplot(1, 3, 1) 104 plt.imshow(img) --> 105 plt.title(f"Input Image - {labels[label]}", color=text_color, fontweight='bold') 106 plt.axis("off") 107 plt.subplot(1, 3, 2)

    TypeError: list indices must be integers or slices, not numpy.str_

  22. Assistant

    The error you encountered,

    TypeError: list indices must be integers or slices, not numpy.str_
    

    suggests that the label variable is a string (numpy.str_ type) when the code expects an integer index to access an item in the labels list. This discrepancy likely stems from how the y_class_labels array is formatted. If y_class_labels is an array of string labels, then you cannot use these strings directly as indices to access elements in the labels list.

    To resolve this issue, there are a couple of approaches you could take depending on how your labels and data are structured:

    Approach 1: If labels is a List of Class Names

    If labels directly corresponds to the class names (and assuming y_class_labels are the actual textual class labels), then you wouldn't use labels[label] but rather just label directly in the plotting function. Update the line causing the error from:

    plt.title(f"Input Image - {labels[label]}", color=text_color, fontweight='bold')
    

    To:

    plt.title(f"Input Image - {label}", color=text_color, fontweight='bold')
    

    Approach 2: If y_class_labels Needs Mapping Through LabelEncoder

    If your y_class_labels needs to be mapped to indices first through a LabelEncoder to then access names in labels, you would ensure label is an integer index. Since the code provided seems to be structured as though y_class_labels are already the class labels as strings directly, this approach might not be necessary unless your y_class_labels are actually encoded values instead of string labels.

    However, if y_class_labels are indeed encoded and labels is an array where each index corresponds to a class name, you'll want to first ensure y_class_labels is properly encoded to numerical values, and label should be an integer. This discrepancy might also arise if y_class_labels are not encoded as numerical indices but directly used as such.

    Debugging Tip

    You can print out or inspect the type and value of label before the error occurs to understand better what's being passed and why it may not be compatible with indexing labels:

    print(f"label: {label}, type: {type(label)}")
    

    This step will help verify what label contains and why it may not be an integer as expected.

    Ensure that labels is a list where indices correspond to class names and that label is an integer index into this list. If there's still confusion about your data's structure, you might want to revisit how y_class_labels is generated or post more details about how labels and y_class_labels are defined and used in your code.

  23. User

    Too big

  24. Assistant

    If the provided explanation seemed too detailed and you're looking for a focused solution to the error based on the fact that label is expected to be an integer index but is being treated as a string (numpy.str_), here is a concise approach to adjust the plot_sample_images function:

    import numpy as np
    import matplotlib.pyplot as plt
    
    def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=10):
        indices = np.random.choice(range(len(X_data)), num_images, replace=False)
        
        for i, index in enumerate(indices):
            img = X_data[index]
            seg = y_seg_labels[index]  # Assuming your seg masks are 0 or 1, no need to multiply by 255
            label_index = y_class_labels[index]  # Assuming this is already the descriptive label
    
            fig, axs = plt.subplots(1, 3, figsize=(15, 5), facecolor='#303030')
            axs[0].imshow(img)
            axs[0].set_title(f"Input Image - {label_index}", color='white', fontweight='bold')
            axs[0].axis('off')
    
            axs[1].imshow(seg.squeeze(), cmap='gray')
            axs[1].set_title("Segmentation Mask", color='white', fontweight='bold')
            axs[1].axis('off')
    
            plt.tight_layout()
            plt.show()
    

    Key Adjustments:

    1. If y_class_labels directly provides the labels in a text/string format (e.g., "good", "bad"), ensure it's used directly for display purposes without indexing labels list/array.

    2. From your error details and usage, it appears that the original issue was around using the y_class_labels directly as indices, which won't work if they're strings. The adjustment assumes y_class_labels[index] gives you the label as a string directly applicable for titles.

    3. The plot_sample_images function is simplified to avoid label indexing errors and encourages direct use of string labels for titles in plots.

    This approach should integrate well, excluding any complexity about label encoding/decoding or mistreatment of string labels as indices. If labels already come in a descriptive string form but need correlation with labels for index-based access, ensure conversion processes (encoding or decoding) align with your dataset's structure and intended usage.

  25. User

    No, the problem is solved, I mean the plots are too big, I want to show them at the same time, recall the code: def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=10): bg_color = '#303030'
    text_color = 'white'
    indices = np.random.choice(range(len(X_data)), num_images, replace=False) plt.rcParams['figure.facecolor'] = bg_color plt.rcParams['axes.facecolor'] = bg_color plt.rcParams['axes.edgecolor'] = 'grey' plt.rcParams['axes.labelcolor'] = text_color plt.rcParams['text.color'] = text_color plt.rcParams['xtick.color'] = text_color plt.rcParams['ytick.color'] = text_color for i, index in enumerate(indices): img = X_data[index] seg = y_seg_labels[index] * 255
    label = y_class_labels[index] plt.figure(figsize=(10, 5), dpi=200) plt.subplot(1, 3, 1) plt.imshow(img) plt.title(f"Input Image - {label}", color=text_color, fontweight='bold') plt.axis("off") plt.subplot(1, 3, 2) plt.imshow(seg, cmap='gray') plt.title("Segmentation Mask", color=text_color, fontweight='bold') plt.axis("off") plt.tight_layout() plt.show()
    plt.rcParams['figure.facecolor'] = 'white' plt.rcParams['axes.facecolor'] = 'white' plt.rcParams['axes.edgecolor'] = 'black' plt.rcParams['axes.labelcolor'] = 'black' plt.rcParams['text.color'] = 'black' plt.rcParams['xtick.color'] = 'black' plt.rcParams['ytick.color'] = 'black'

  26. Assistant

    If you want the plots to be smaller or to arrange them more compactly so you could see multiple images at once, consider arranging them in a grid layout within a single figure. This approach reduces the overall space required and allows for a more consolidated view of your sample images and their corresponding segmentation masks.

    You can achieve this by creating subplots in a grid with multiple rows and adjusting figsize and dpi to make individual plots smaller and the overall figure more manageable on screen.

    Here is a revised version of the plot_sample_images function that places multiple images in a grid layout:

    import matplotlib.pyplot as plt
    import numpy as np
    
    def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=10):
        bg_color = '#303030'  
        text_color = 'white'  
        indices = np.random.choice(range(len(X_data)), num_images, replace=False)
        
        n_cols = 3  # Number of columns per row; adjust based on how many types of images you want side by side
        n_rows = (num_images * 2) // n_cols  # Calculate rows needed, times 2 because of image + segmentation mask
        
        fig, axs = plt.subplots(n_rows, n_cols, figsize=(15, n_rows * 2.5))
        fig.patch.set_facecolor(bg_color)
        
        for i, ax in enumerate(axs.flat):
            index = indices[i // 2]  # Integer division to use the same image for both original and mask
            if i % 2 == 0:  # Even index, show original image
                img = X_data[index]
                ax.imshow(img)
                ax.set_title(f"Input Image - {y_class_labels[index]}", color=text_color, fontweight='bold')
            else:  # Odd index, show segmentation mask
                seg = y_seg_labels[index]
                ax.imshow(seg, cmap='gray')
                ax.set_title("Segmentation Mask", color=text_color, fontweight='bold')
            
            ax.axis("off")
    
        plt.tight_layout()
        plt.show()
        
        # Reset to defaults
        plt.rcParams['figure.facecolor'] = 'white'
        plt.rcParams['axes.facecolor'] = 'white'
        plt.rcParams['axes.edgecolor'] = 'black'
        plt.rcParams['axes.labelcolor'] = 'black'
        plt.rcParams['text.color'] = 'black'
        plt.rcParams['xtick.color'] = 'black'
        plt.rcParams['ytick.color'] = 'black'
    

    Key Adjustments

    • Grid Layout: This version organizes the images and masks into a grid. The number of columns n_cols is set to 3 as before, but now you effectively get rows that contain both an image and its corresponding mask side by side.
    • Shared Figures: Instead of creating a new figure for each image/mask pair, this approach places all visualizations within a single figure to conserve space and make comparison easier.
    • figsize and dpi: Adjust figsize (width, height in inches) and dpi (dots per inch) according to your preference or screen resolution to make the image display size appropriate for your needs. Find a balance where individual subplots are visible but don't take up too much space.
    • Indexing for Images and Masks: Uses i // 2 to ensure the same index is used for both an image and its subsequent mask within the loop.

    This solution lets you visualize multiple samples simultaneously, making it easier to compare images and their segmentation masks in a concise layout.

  27. User

    I want to plot 12 images

  28. Assistant

    To plot 12 images alongside their segmentation masks in a structured and compact layout, considering that we plot each image with its corresponding segmentation mask side by side, we'll use a grid layout of rows and columns. Given that we have 12 images, the total number of subplots needed will be 24 (12 images + 12 masks).

    A good arrangement could be 4 rows and 6 columns where each image and its mask are placed next to each other. Here's the adjusted function:

    def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=12):
        bg_color = '#303030'  # Background color for the figure
        text_color = 'white'  # Text color for titles
        
        # Ensuring we don't request more images than available
        num_images = min(num_images, len(X_data))
        indices = np.random.choice(range(len(X_data)), num_images, replace=False)
        
        n_cols = 6  # 3 pairs of image and its mask side by side
        n_rows = num_images // 2  # Total number of rows needed
        
        # Creating a large figure to accommodate all subplots
        fig, axs = plt.subplots(n_rows, n_cols, figsize=(15, 2.5 * n_rows))
        fig.patch.set_facecolor(bg_color)
    
        for i in range(0, num_images * 2, 2):
            row_idx = i // n_cols
            col_idx_image = (i % n_cols)  # Column for the image
            col_idx_mask = (i % n_cols) + 1  # Column for the mask (next to the image)
    
            # Handling case where axs is a 2D array and when it's not
            ax_img = axs[row_idx, col_idx_image] if num_images > 1 else axs[col_idx_image]
            ax_mask = axs[row_idx, col_idx_mask] if num_images > 1 else axs[col_idx_mask]
    
            selected_index = indices[i // 2]  # Get the index for image and mask
            
            # Plotting the image
            ax_img.imshow(X_data[selected_index])
            ax_img.set_title(f"Image - {y_class_labels[selected_index]}", color=text_color, fontweight='bold')
            ax_img.axis("off")
            
            # Plotting the segmentation mask
            seg = y_seg_labels[selected_index] * 255  # Assuming masks need to be scaled
            ax_mask.imshow(seg, cmap='gray')
            ax_mask.set_title("Mask", color=text_color, fontweight='bold')
            ax_mask.axis("off")
    
        plt.tight_layout()
        plt.show()
    
        # Reset matplotlib's styling to defaults
        plt.rcParams['figure.facecolor'] = 'white'
        plt.rcParams['axes.facecolor'] = 'white'
        plt.rcParams['axes.edgecolor'] = 'black'
        plt.rcParams['axes.labelcolor'] = 'black'
        plt.rcParams['text.color'] = 'black'
        plt.rcParams['xtick.color'] = 'black'
        plt.rcParams['ytick.color'] = 'black'
    

    Key Points:

    • The above function generates a grid structure that accommodates 12 images and their corresponding segmentation masks.
    • Each pair (an image and its mask) occupies two columns on the same row, leading to a comprehensive view where each image directly precedes its segmentation mask.
    • The backgrounds and titles are styled according to your specifications, using a dark theme (#303030 background with white text).
    • The layout dynamically adjusts with n_rows and n_cols to optimally display the requested number of images (in this case, 12). However, the logic ensures each image and its corresponding mask are placed side by side for easy comparison.

    Ensure your dataset (X_data, y_class_labels, y_seg_labels) is correctly loaded, and num_images is set to 12 as you requested, for this function to work as intended.

  29. User

    adjust dpi

  30. Assistant

    Adjusting the dpi parameter enhances the overall resolution of the plots, making them clearer, especially when you are displaying detailed images like segmentation masks. To adjust the dpi for higher resolution within the plot_sample_images function, you simply include a dpi= argument when you create your figure with plt.subplots.

    I'll add a dpi=100 to the function, but feel free to increase this value if you need even higher resolution. Keep in mind that higher dpi values produce higher-resolution images, which can lead to larger file sizes if you save the plots and potentially slower rendering times on screen.

    Here's the function with dpi set to 100:

    def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=12):
        bg_color = '#303030'  # Background color for the figure
        text_color = 'white'  # Text color for titles
        dpi_setting = 100  # Adjust DPI for better resolution
    
        # Ensuring we don't request more images than available
        num_images = min(num_images, len(X_data))
        indices = np.random.choice(range(len(X_data)), num_images, replace=False)
        
        n_cols = 6  # 3 pairs of image and its mask side by side
        n_rows = num_images // 2  # Total number of rows needed
        
        # Creating a large figure to accommodate all subplots
        # Adding the dpi parameter here
        fig, axs = plt.subplots(n_rows, n_cols, figsize=(15, 2.5 * n_rows), dpi=dpi_setting)
        fig.patch.set_facecolor(bg_color)
    
        for i in range(0, num_images * 2, 2):
            row_idx = i // n_cols
            col_idx_image = (i % n_cols)  # Column for the image
            col_idx_mask = (i % n_cols) + 1  # Column for the mask (next to the image)
    
            # Handling case where axs is a 2D array and when it's not
            ax_img = axs[row_idx, col_idx_image] if num_images > 1 else axs[col_idx_image]
            ax_mask = axs[row_idx, col_idx_mask] if num_images > 1 else axs[col_idx_mask]
    
            selected_index = indices[i // 2]  # Get the index for image and mask
            
            # Plotting the image
            ax_img.imshow(X_data[selected_index])
            ax_img.set_title(f"Image - {y_class_labels[selected_index]}", color=text_color, fontweight='bold')
            ax_img.axis("off")
            
            # Plotting the segmentation mask
            seg = y_seg_labels[selected_index]  # Assuming masks are already in the correct scale (0-255)
            ax_mask.imshow(seg, cmap='gray')
            ax_mask.set_title("Mask", color=text_color, fontweight='bold')
            ax_mask.axis("off")
    
        plt.tight_layout()
        plt.show()
    
        # Reset matplotlib's styling to defaults
        plt.rcParams['figure.facecolor'] = 'white'
        plt.rcParams['axes.facecolor'] = 'white'
        plt.rcParams['axes.edgecolor'] = 'black'
        plt.rcParams['axes.labelcolor'] = 'black'
        plt.rcParams['text.color'] = 'black'
        plt.rcParams['xtick.color'] = 'black'
        plt.rcParams['ytick.color'] = 'black'
    

    Notes:

    • dpi_setting = 100 sets the dots per inch for the figure. Increase this number for higher resolution. A setting of 200 or 300 dpi is common for high-resolution images needed in publications or presentations.
    • The figsize=(15, 2.5 * n_rows) alongside dpi=dpi_setting defines the size and resolution of the figure. Adjust these values to fit your display or desired output resolution better.
  31. User

    Now my code is : def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=12): bg_color = '#303030' # Background color for the figure text_color = 'white' # Text color for titles

    # Ensuring we don't request more images than available
    num_images = min(num_images, len(X_data))
    indices = np.random.choice(range(len(X_data)), num_images, replace=False)
    
    n_cols = 6  # 3 pairs of image and its mask side by side
    n_rows = num_images // 2  # Total number of rows needed
    
    # Creating a large figure to accommodate all subplots
    fig, axs = plt.subplots(n_rows, n_cols, figsize=(15, 2.5 * n_rows))
    fig.patch.set_facecolor(bg_color)
    
    for i in range(0, num_images * 2, 2):
        row_idx = i // n_cols
        col_idx_image = (i % n_cols)  # Column for the image
        col_idx_mask = (i % n_cols) + 1  # Column for the mask (next to the image)
    
        # Handling case where axs is a 2D array and when it's not
        ax_img = axs[row_idx, col_idx_image] if num_images > 1 else axs[col_idx_image]
        ax_mask = axs[row_idx, col_idx_mask] if num_images > 1 else axs[col_idx_mask]
    
        selected_index = indices[i // 2]  # Get the index for image and mask
        
        # Plotting the image
        ax_img.imshow(X_data[selected_index])
        ax_img.set_title(f"Image - {y_class_labels[selected_index]}", color=text_color, fontweight='bold')
        ax_img.axis("off")
        
        # Plotting the segmentation mask
        seg = y_seg_labels[selected_index] * 255  # Assuming masks need to be scaled
        ax_mask.imshow(seg, cmap='gray')
        ax_mask.set_title("Mask", color=text_color, fontweight='bold')
        ax_mask.axis("off")
    
    plt.tight_layout()
    plt.show()
    

    But there are some white plot with nothing in the last few images no matter how I adjust the num_images in plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=12)

  32. Assistant

    The issue with white plots appearing in your grid likely arises from a mismatch between the total number of subplots created (n_rows * n_cols) and the actual number of images and segmentation masks plotted. This mismatch can occur when specifying a grid that doesn't exactly fit the number of images and segmentation masks you're plotting.

    In your current setup, where you plot both the image and its corresponding segmentation mask side by side within the same row, the total number of subplots (n_rows * n_cols) should exactly match num_images * 2. However, if num_images is not perfectly divisible by n_cols / 2, you'll end up with extra, unused subplot slots.

    To resolve this issue, you need to ensure that any extra subplot slots are appropriately handled either by not creating them in the first place or by turning off their visibility if they're not used. Let's adjust the calculation of n_rows to better fit the number of images being plotted and then turn off any unused subplots:

    import matplotlib.pyplot as plt
    import numpy as np
    
    def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=12):
        bg_color = '#303030'  # Background color for the figure
        text_color = 'white'  # Text color for titles
        
        # Ensuring we don't request more images than available
        num_images = min(num_images, len(X_data))
        indices = np.random.choice(range(len(X_data)), num_images, replace=False)
        
        n_cols = 6  # 3 pairs of image and its mask, making 6 columns needed per row
        # Ensure we have an even number of total plots for images and their masks, adjusting rows as needed
        total_plots_needed = num_images * 2  # Total plots needed
        n_rows = (total_plots_needed + n_cols - 1) // n_cols  # Calculate rows needed, ensure no extra row
        
        fig, axs = plt.subplots(n_rows, n_cols, figsize=(15, 2.5 * n_rows), facecolor=bg_color)
        fig.patch.set_facecolor(bg_color)  # Set figure background color
      
        axs = axs.ravel()  # Flatten the array to make iteration easier
        for i in range(0, total_plots_needed, 2):
            selected_index = indices[i // 2]  # Get the index for image and mask
            
            # Plotting the image
            axs[i].imshow(X_data[selected_index])
            axs[i].set_title(f"Image - {y_class_labels[selected_index]}", color=text_color, fontweight='bold')
            axs[i].axis("off")
            
            # Plotting the segmentation mask
            seg = y_seg_labels[selected_index]  # Assuming masks are scaled 0 to 1
            axs[i+1].imshow(seg, cmap='gray')
            axs[i+1].set_title("Mask", color=text_color, fontweight='bold')
            axs[i+1].axis("off")
    
        # Turn off any unused subplots that may remain due to fixed column count
        for i in range(total_plots_needed, n_rows * n_cols):
            axs[i].axis('off')
    
        plt.tight_layout()
        plt.show()
    

    Key Adjustments:

    • Ax Flattening: The axs array is flattened using ravel() to simplify the iteration process for plotting images and masks. This step allows treating axs as a 1D array, easing access to individual subplots.
    • Unused Subplots: After plotting all images and masks, any remaining subplots in the grid (which would have appeared as empty white plots) are turned off by setting their axis 'off'.
    • Segmentation Mask Plotting: The seg calculation was adjusted back to assume the masks are already appropriately scaled since a prior assumption of needing * 255 might not apply to all cases. Ensure this step matches your data normalization.

    This solution dynamically adjusts to the number of images you wish to plot and ensures no extra, unused subplot slots are visible.

  33. User

    can you make it select at least 1 image from each class?

  34. Assistant

    To ensure that your plotting function selects at least one image from each class, you should modify the selection process to explicitly include images from each class before randomly selecting the remaining images to fill up to num_images. This approach guarantees that each class is represented in the plots. Given that num_images could be less than or equal to the total number of classes, the function will first accommodate one image per class up to the num_images limit and would only then randomly select additional images if num_images exceeds the number of classes.

    Here's how to adjust the plot_sample_images function accordingly:

    import numpy as np
    import matplotlib.pyplot as plt
    
    def plot_sample_images(X_data, y_class_labels, y_seg_labels, labels, num_images=12):
        bg_color = '#303030'  
        text_color = 'white'  
        unique_classes = np.unique(y_class_labels)
        num_classes = len(unique_classes)
        
        # Adjust num_images if it's less than the number of classes to ensure at least one image per class
        num_images = max(num_images, num_classes)
        
        selected_indices = []
        for cls in unique_classes:
            class_indices = np.where(y_class_labels == cls)[0]
            selected_indices.append(np.random.choice(class_indices, 1)[0])
        
        # After ensuring one image per class, randomly select additional images if needed
        additional_selections = num_images - num_classes
        if additional_selections > 0:
            available_indices = [idx for idx in range(len(X_data)) if idx not in selected_indices]
            selected_indices.extend(np.random.choice(available_indices, additional_selections, replace=False))
        
        n_cols = 6
        total_plots_needed = len(selected_indices) * 2  # One image and one mask per selection
        n_rows = (total_plots_needed + n_cols - 1) // n_cols
        
        fig, axs = plt.subplots(n_rows, n_cols, figsize=(15, 2.5 * n_rows), facecolor=bg_color)
        fig.patch.set_facecolor(bg_color)
    
        axs = axs.ravel()
        for i, idx in enumerate(selected_indices*2):
            if i % 2 == 0:
                img_idx = selected_indices[i // 2]
                axs[i].imshow(X_data[img_idx])
                axs[i].set_title(f"Image - {y_class_labels[img_idx]}", color=text_color, fontweight='bold')
                axs[i].axis("off")
            else:
                mask_idx = selected_indices[(i-1) // 2]
                seg = y_seg_labels[mask_idx]  # Assuming masks are scaled 0 to 1
                axs[i].imshow(seg, cmap='gray')
                axs[i].set_title("Mask", color=text_color, fontweight='bold')
                axs[i].axis("off")
                
        # Turn off any unused subplots that may remain due to fixed column count
        for j in range(total_plots_needed, n_rows * n_cols):
            axs[j].axis('off')
    
        plt.tight_layout()
        plt.show()
    

    Key Adjustments:

    • The function now first iterates over each unique class in y_class_labels to select an image. It ensures diversity in the plotted selection by guaranteeing at least one image from each class when possible.
    • After ensuring one image per class, it then fills the remaining slots by randomly choosing from the images that were not yet selected until reaching the desired num_images.
    • The loop for plotting handles both images and their masks by checking the index (i % 2 == 0 for images and i % 2 != 0 for masks).

    Remember to adjust n_cols, figsize, and any other parameter to better fit the format and size preferences for your specific use case.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.