Skip to content
TheChatArchiveThe Chat Archive
gpt-4o-2024-08-06· OpenAIAi2 WildChat-4.8M (AI2)

from future import absoluteimport, division, printfunction import os import…

  1. User

    from future import absolute_import, division, print_function

    import os

    import matplotlib.pyplot as plt

    os.environ["MKL_NUM_THREADS"] = "1" os.environ["NUMEXPR_NUM_THREADS"] = "1" os.environ["OMP_NUM_THREADS"] = "1" os.environ["KMP_DUPLICATE_LIB_OK"] = "TRUE" import random import numpy as np from PIL import Image # using pillow-simd for increased speed import random import numpy as np from PIL import Image # using pillow-simd for increased speed from skimage.filters import gaussian, sobel from scipy.interpolate import griddata import cv2 from utils import transfer_color import torch import torch.utils.data as data from torchvision import transforms

    def pil_loader(path): # print(len(path)) with open(path, 'rb') as f: with Image.open(f) as img: return img.convert('RGB')

    def read_pfm(file): with open(file, 'rb') as fh: fh.readline() width, height = str(fh.readline().rstrip())[2:-1].split() fh.readline() disp = np.fromfile(fh, '<f') return np.flipud(disp.reshape(int(height), int(width)))

    class BaseDataset: def init(self, data_path, feed_height=1024, feed_width=768, max_disparity=20, is_train=True, disable_synthetic_augmentation=True, disable_sharpening=True, monodepth_model='midas', disable_background=False, has_gt=True, disable_normalisation=False, keep_aspect_ratio=True):

        itxt_path = '/home/notebook/data/group/Ruby/stereo-from-mono-master/img.txt'
        dtxt_path = '/home/notebook/data/group/Ruby/stereo-from-mono-master/depth.txt'
        all_img = []
        all_depth = []
        with open( itxt_path , 'r' ) as fi:
            for li in fi.readlines():
                li = li.strip( '\n' )
                all_img.append(li)
    
            # print('all_image_len',len(all_img[0]))
            # print('all_image_len',all_img[0])
        with open( dtxt_path , 'r' ) as fd:
            for ld in fd.readlines():
                ld = ld.strip( '\n' )
                all_depth.append(ld)
    
            # print('all_depth',len(all_depth[0]))
                  
        self.data_path = data_path
        # self.depth_path=self.data_path+"/"
        # self.img_path=self.data_path+"/"
        self.depth_path= all_depth
        self.img_path= all_img
        # self.filenames = filenames
        self.feed_height = feed_height
        self.feed_width = feed_width
        self.is_train = is_train
        self.has_gt = has_gt
        self.disable_normalisation = disable_normalisation
        self.keep_aspect_ratio = keep_aspect_ratio
    
        self.loader = pil_loader
        self.read_pfm = read_pfm
        self.to_tensor = transforms.ToTensor()
        # self.filenames=[file for file in os.listdir(self.img_path)  if(".png"  in file)]
        # self.filenames=[file for file in os.listdir(self.depth_path)  if(".png"  in file)]
        self.max_disparity = max_disparity
        self.disable_synthetic_augmentation = disable_synthetic_augmentation
        self.disable_sharpening = disable_sharpening
        self.monodepth_model = monodepth_model
        self.disable_background = disable_background
    
        # do image generation for a wider image so we can crop off missing pixels
        self.process_width = self.feed_width + self.max_disparity
    
        self.xs, self.ys = np.meshgrid(np.arange(self.process_width), np.arange(self.feed_height))
    
        # We need to specify augmentations differently in newer versions of torchvision.
        # We first try the newer tuple version; if this fails we fall back to scalars
        try:
            self.stereo_brightness = (0.8, 1.2)
            self.stereo_contrast = (0.8, 1.2)
            self.stereo_saturation = (0.8, 1.2)
            self.stereo_hue = (-0.01, 0.01)
            transforms.ColorJitter.get_params(
                self.stereo_brightness, self.stereo_contrast, self.stereo_saturation,
                self.stereo_hue)
        except TypeError:
            self.stereo_brightness = 0.2
            self.stereo_contrast = 0.2
            self.stereo_saturation = 0.2
            self.stereo_hue = 0.01
        try:
            self.brightness = (0.8, 1.2)
            self.contrast = (0.8, 1.2)
            self.saturation = (0.8, 1.2)
            self.hue = (-0.1, 0.1)
            transforms.ColorJitter.get_params(
                self.brightness, self.contrast, self.saturation, self.hue)
        except TypeError:
            self.brightness = 0.2
            self.contrast = 0.2
            self.saturation = 0.2
            self.hue = 0.1
    
        self.silly_svsm = False
    
    def __len__(self):
        # return len(self.filenames)
        return len(self.img_path)
    
    
    def load_images(self, idx, do_flip=False):
        """ Load an image to use as left and a random background image to fill in occlusion holes"""
    
        # frame = self.filenames[idx]
        # image = self.loader(os.path.join(self.img_path, frame))
        # print('img_path',self.img_path)
        ###这边是单张数据读取
        print('idx',idx)
        print('img',self.img_path[idx])
        image = self.loader(self.img_path[idx])
    
        if do_flip:
            image = image.transpose(Image.FLIP_LEFT_RIGHT)
        # bg_path=r"/home/notebook/data/group/Ruby/stereo-from-mono-master/flower/img"
        # frame = random.choice(os.listdir(bg_path))
        frame = random.choice(self.img_path)
        # background = self.loader(os.path.join(bg_path, frame))
        background = self.loader(frame)
        return image, background
    
    def load_disparity(self, idx, do_flip=False):
        # frame = self.filenames[idx]
        # print(self.depth_path+frame.replace(".png","_depth.png"))
        # depth=self.loader(self.depth_path+frame.replace(".png","_depth.png")).convert("L")
        depth=self.loader(self.depth_path[idx]).convert("L")
        disparity = np.squeeze(depth)
        # disparity=25-disparity
    
        if do_flip:
            disparity = disparity[:, ::-1]
        return disparity
    
    def preprocess(self, inputs):
    
        # do color augmentation
        if self.is_train and random.random() > 0.5:
            color_aug = transforms.ColorJitter.get_params(
                self.brightness, self.contrast, self.saturation, self.hue)
    
            for key in ['image', 'stereo_image']:
                inputs[key] = color_aug(inputs[key])
    
        # convert to tensors and standardise using ImageNet
        for key in ['image', 'stereo_image']:
            if self.disable_normalisation:
                inputs[key] = self.to_tensor(inputs[key])
            else:
                inputs[key] = (self.to_tensor(inputs[key]) - 0.45) / 0.225
        if self.has_gt:
            inputs['disparity'] = torch.from_numpy(inputs['disparity']).float()
    
        if inputs.get('mono_disparity') is not None:
            inputs['mono_disparity'] = torch.from_numpy(inputs['mono_disparity']).float()
    def process_disparity(self, disparity, max_disparity_range=(40, 196)):
        """ Depth predictions have arbitrary scale - need to convert to a pixel disparity"""
    
        disparity = disparity.copy()
    
        # make disparities positive
        min_disp = disparity.min()
        if min_disp < 0:
            disparity += np.abs(min_disp)
    
        # if random.random() < 0.01:
        #     # make max warped disparity bigger than network max -> will be clipped to max disparity,
        #     # but will mean network is robust to disparities which are too big
        #     max_disparity_range = (self.max_disparity * 1.05, self.max_disparity * 1.15)
    
        disparity /= disparity.max()  # now 0-1
    
        scaling_factor = (max_disparity_range[0] + random.random() *
                          (max_disparity_range[1] - max_disparity_range[0]))
        disparity *= scaling_factor
    
        if not self.disable_sharpening:
            # now find disparity gradients and set to nearest - stop flying pixels
            edges = sobel(disparity) > 3
            disparity[edges] = 0
            mask = disparity > 0
    
            try:
                disparity = griddata(np.stack([self.ys[mask].ravel(), self.xs[mask].ravel()], 1),
                                     disparity[mask].ravel(), np.stack([self.ys.ravel(),
                                                                        self.xs.ravel()], 1),
                                     method='nearest').reshape(self.feed_height, self.process_width)
            except (ValueError, IndexError) as e:
                pass  # just return disparity
    
        return disparity
    
    def prepare_sizes(self, inputs):
    
        height, width, _ = np.array(inputs['left_image']).shape
    
        if self.keep_aspect_ratio:
            if self.feed_height <= height and self.process_width <= width:
                # can simply crop the image
                target_height = height
                target_width = width
    
            else:
                # check the constraint
                current_ratio = height / width
                target_ratio = self.feed_height / self.process_width
    
                if current_ratio < target_ratio:
                    # height is the constraint
                    target_height = self.feed_height
                    target_width = int(self.feed_height / height * width)
                elif current_ratio > target_ratio:
                    # width is the constraint
                    target_height = int(self.process_width / width * height)
                    target_width = self.process_width
                else:
                    # ratio is the same - just resize
                    target_height = self.feed_height
                    target_width = self.process_width
    
        else:
            target_height = self.feed_height
            target_width = self.process_width
    
        inputs = self.resize_all(inputs, target_height, target_width)
    
        # now do cropping
        if target_height == self.feed_height and target_width == self.process_width:
            # we are already at the correct size - no cropping
            pass
        else:
            self.crop_all(inputs)
    
        return inputs
    
    def crop_all(self, inputs):
    
        # get crop parameters
        height, width, _ = np.array(inputs['left_image']).shape
        top = int(random.random() * (height - self.feed_height))
        left = int(random.random() * (width - self.process_width))
        right, bottom = left + self.process_width, top + self.feed_height
    
        for key in ['left_image', 'background']:
            inputs[key] = inputs[key].crop((left, top, right, bottom))
        inputs['loaded_disparity'] = inputs['loaded_disparity'][top:bottom, left:right]
    
        return inputs
    
    @staticmethod
    def resize_all(inputs, height, width):
    
        # images
        img_resizer = transforms.Resize(size=(height, width))
        for key in ['left_image', 'background']:
            inputs[key] = img_resizer(inputs[key])
        # disparity - needs rescaling
        disp = inputs['loaded_disparity']
        disp=np.array(disp).astype(np.float32)
        disp *= width / disp.shape[1]
    
        disp = cv2.resize(disp.astype(float), (width, height))  # ensure disp is float32 for cv2
        inputs['loaded_disparity'] = disp
    
        return inputs
    
    def get_occlusion_mask(self, shifted):
    
        mask_up = shifted > 0
        mask_down = shifted > 0
    
        shifted_up = np.ceil(shifted)
        shifted_down = np.floor(shifted)
    
        for col in range(self.process_width - 2):
            loc = shifted[:, col:col + 1]  # keepdims
            loc_up = np.ceil(loc)
            loc_down = np.floor(loc)
    
            _mask_down = ((shifted_down[:, col + 2:] != loc_down) * (
            (shifted_up[:, col + 2:] != loc_down))).min(-1)
            _mask_up = ((shifted_down[:, col + 2:] != loc_up) * (
            (shifted_up[:, col + 2:] != loc_up))).min(-1)
    
            mask_up[:, col] = mask_up[:, col] * _mask_up
            mask_down[:, col] = mask_down[:, col] * _mask_down
    
        mask = mask_up + mask_down
        return mask
    
    def project_image(self, image, disp_map, background_image):
    
        image = np.array(image)
        background_image = np.array(background_image)
    
        # set up for projection
        warped_image = np.zeros_like(image).astype(float)
        warped_image = np.stack([warped_image] * 2, 0)
        pix_locations = self.xs - disp_map
    
        # find where occlusions are, and remove from disparity map
        mask = self.get_occlusion_mask(pix_locations)
        masked_pix_locations = pix_locations * mask - self.process_width * (1 - mask)
    
        # do projection - linear interpolate up to 1 pixel away
        weights = np.ones((2, self.feed_height, self.process_width)) * 10000
    
        for col in range(self.process_width - 1, -1, -1):
            loc = masked_pix_locations[:, col]
            loc_up = np.ceil(loc).astype(int)
            loc_down = np.floor(loc).astype(int)
            weight_up = loc_up - loc
            weight_down = 1 - weight_up
    
            mask = loc_up >= 0
            mask[mask] = \
                weights[0, np.arange(self.feed_height)[mask], loc_up[mask]] > weight_up[mask]
            weights[0, np.arange(self.feed_height)[mask], loc_up[mask]] = \
                weight_up[mask]
            warped_image[0, np.arange(self.feed_height)[mask], loc_up[mask]] = \
                image[:, col][mask] / 255.
    
            mask = loc_down >= 0
            mask[mask] = \
                weights[1, np.arange(self.feed_height)[mask], loc_down[mask]] > weight_down[mask]
            weights[1, np.arange(self.feed_height)[mask], loc_down[mask]] = weight_down[mask]
            warped_image[1, np.arange(self.feed_height)[mask], loc_down[mask]] = \
                image[:, col][mask] / 255.
    
        weights /= weights.sum(0, keepdims=True) + 1e-7  # normalise
        weights = np.expand_dims(weights, -1)
        warped_image = warped_image[0] * weights[1] + warped_image[1] * weights[0]
        warped_image *= 255.
    
        # now fill occluded regions with random background
        if not self.disable_background:
            warped_image[warped_image.max(-1) == 0] = background_image[warped_image.max(-1) == 0]
    
        warped_image = warped_image.astype(np.uint8)
    
        return warped_image
    
    def augment_synthetic_image(self, image):
    
        if self.disable_synthetic_augmentation:
    
            return Image.fromarray(image.astype(np.uint8))
    
        # add some noise to stereo image
        noise = np.random.randn(self.feed_height, self.process_width, 3) / 50
        image = np.clip(image / 255 + noise, 0, 1) * 255
    
        # add blurring
        if random.random() > 0.5:
            image = gaussian(image,
                             sigma=random.random())
        image = np.clip(image, 0, 255)
    
        # color augmentation
        # stereo_aug = transforms.ColorJitter.get_params(
        #     self.stereo_brightness, self.stereo_contrast, self.stereo_saturation,
        #     self.stereo_hue)
        # print(image.shape)
        # image = stereo_aug(Image.fromarray(image.astype(np.uint8)))
        image=Image.fromarray(image.astype(np.uint8))
        return image
    
    def __getitem__(self, idx):
        inputs = {}
        do_flip = False
        if self.is_train and random.random() > 0.5:
            do_flip = True
        # load from disk
        left_image, background_image = self.load_images(idx, do_flip=do_flip)
        loaded_disparity = self.load_disparity(idx, do_flip=do_flip)
    
        inputs['left_image'] = left_image
        inputs['background'] = background_image
        inputs['loaded_disparity'] = loaded_disparity
    
        # resize and/or crop
        inputs = self.prepare_sizes(inputs)
    
        # match color in background image
        inputs['background'] = transfer_color(np.array(inputs['background']),
                                              np.array(inputs['left_image']))
    
        # convert scaleless disparity to pixel disparity
        inputs['disparity'] = \
            self.process_disparity(inputs['loaded_disparity'],
                                   max_disparity_range=(10, self.max_disparity))
    
        # now generate synthetic stereo image###
        projection_disparity = inputs['disparity']
        right_image = self.project_image(inputs['left_image'],
                                         projection_disparity, inputs['background'])
    
        # augmentation ###
        right_image = self.augment_synthetic_image(right_image)
    
        # only keep required keys and prepare for network
        inputs = {'image': inputs['left_image'],
                  'stereo_image': right_image,
                  'disparity': projection_disparity.astype(float),
                  'mono_disparity': inputs['loaded_disparity'].astype(float),
                  }
    
        # finally crop to feed width
        for key in ['image', 'stereo_image']:
            inputs[key] = inputs[key].crop((0, 0, self.feed_width, self.feed_height))
        for key in ['disparity', 'mono_disparity']:
            inputs[key] = inputs[key][:, :self.feed_width]
    
        # self.preprocess(inputs)
        return inputs
    

    if name=="main":

    dataset=BaseDataset(r"/home/notebook/data/group/Ruby/stereo-from-mono-master/")
    # 输入数据path
    # save_path=r"F:\code\stereo-from-mono-master\langan_refine_data/"
    save_path=r"/home/notebook/data/group/Ruby/stereo-from-mono-master/refine_results/"
    # 输出数据path
    count=5000 # 原值是5000
    for data in dataset:
        print(count)
        left=np.array(data["image"])
        right=np.array(data["stereo_image"])
        disparity=np.array(data["disparity"])
        mono_disparity=np.array(data["mono_disparity"])
        left=cv2.cvtColor(left,cv2.COLOR_RGB2BGR)
        right=cv2.cvtColor(right,cv2.COLOR_RGB2BGR)
        print(save_path+str(count)+"_left.jpg")
        cv2.imwrite(save_path+str(count)+"_left.jpg",left)
        cv2.imwrite(save_path+str(count)+"_right.jpg",right)
        cv2.imwrite(save_path+str(count)+"_disp.png",disparity*10)
        count+=1
    
        # plt.subplot(1,4,1)
        # plt.imshow(left)
        # plt.subplot(1,4,2)
        # plt.imshow(right)
        # plt.subplot(1,4,3)
        # plt.imshow(disparity,cmap="gray")
        #
        # plt.subplot(1,4,4)
        # plt.imshow(mono_disparity,cmap="gray")
        # plt.show()
    
  2. Assistant

    Your code defines a Python script for data preprocessing and generation of synthetic stereo images. Here's a brief breakdown of what's happening:

    1. Imports and Setup:

      • Various libraries are imported, including PyTorch, OpenCV, Pillow, NumPy, Skimage, Scipy, and Matplotlib.
      • OpenMP and other threading-related environment settings are configured to control parallelism.
    2. Utility Functions:

      • pil_loader(path): Loads an image using Pillow and returns it in RGB format.
      • read_pfm(file): Reads a PFM file, typically storing disparity data.
    3. BaseDataset Class:

      • This class defines a dataset loader designed to load images and their corresponding disparity maps, perform various transformations, and generate synthetic stereo pairs for training models.
    4. Initialization:

      • The class initializes with paths to data, settings pertinent to training, and augmentation toggles. It reads lists of image and disparity file paths from text files.
    5. Main Methods:

      • load_images and load_disparity: Load images and disparities for a given index, possibly performing horizontal flip for augmentation.
      • process_disparity: Processes disparity maps, scaling them to fall within a given range.
      • prepare_sizes and crop_all: Resize and crop images to the desired dimensions while maintaining aspect ratio when specified.
      • get_occlusion_mask, project_image, and augment_synthetic_image: Generate occlusion masks, warp images using disparity maps to create synthetic right images, and apply augmentations respectively.
      • __len__ and __getitem__: Define the length of the dataset and how data is accessed, respectively.
    6. Main Script:

      • The script iterates through the dataset, generating and saving left/right image pairs and disparity maps.

    If you have specific questions or need further explanations or improvements, feel free to ask! The current setup should allow you to preprocess datasets for training stereo vision models effectively.

  3. User

    以上述代码为基础,将这个代码改成能在GPU上面运行的

  4. Assistant

    Converting your data processing and augmentation pipeline to use the GPU primarily involves leveraging PyTorch's GPU capabilities. Here's a step-by-step guide on how you can modify the script to run parts of it on the GPU:

    1. Data Loading:

      • Ensure that tensors that need to be processed on the GPU are moved appropriately.
      • Utilize PyTorch transforms that can operate on GPU rather than CPU.
    2. Torch Device:

      • Define a torch device, typically cuda if a GPU is available. Use this device to move data and perform computations.
    3. Adjustments:

      • When converting or handling NumPy arrays, ensure that they are moved to GPU appropriately and handled as tensors.

    Here is a revised version that includes these modifications:

    import torch
    from torchvision import transforms
    
    # Define the device
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    
    # Modify the BaseDataset class to use the device
    class BaseDataset:
        def __init__(self, data_path, feed_height=1024, feed_width=768, max_disparity=20, is_train=True, disable_synthetic_augmentation=True, disable_sharpening=True, monodepth_model='midas', disable_background=False, has_gt=True, disable_normalisation=False, keep_aspect_ratio=True):
            # [Initialization code stays as it is]
    
            # Update transform initialization
            self.to_tensor = transforms.ToTensor()
        
        # Other methods remain unchanged
        
        def preprocess(self, inputs):
            # Ensure image inputs are tensors and transferred to the proper device
            for key in ['image', 'stereo_image']:
                if self.disable_normalisation:
                    inputs[key] = self.to_tensor(inputs[key]).to(device)
                else:
                    inputs[key] = ((self.to_tensor(inputs[key]) - 0.45) / 0.225).to(device)
            
            # if has_gt is defined, move disparity to device
            if self.has_gt:
                inputs['disparity'] = torch.from_numpy(inputs['disparity']).float().to(device)
            
            if inputs.get('mono_disparity') is not None:
                inputs['mono_disparity'] = torch.from_numpy(inputs['mono_disparity']).float().to(device)
                
        # The __getitem__ method will remain the same apart from tensor conversions
    
        def __getitem__(self, idx):
            inputs = {}
            do_flip = False
            if self.is_train and random.random() > 0.5:
                do_flip = True
            # load from disk
            left_image, background_image = self.load_images(idx, do_flip=do_flip)
            loaded_disparity = self.load_disparity(idx, do_flip=do_flip)
    
            inputs['left_image'] = left_image
            inputs['background'] = background_image
            inputs['loaded_disparity'] = loaded_disparity
    
            # resize and/or crop
            inputs = self.prepare_sizes(inputs)
    
            # match color in background image
            inputs['background'] = transfer_color(np.array(inputs['background']),
                                                  np.array(inputs['left_image']))
    
            # convert scaleless disparity to pixel disparity
            inputs['disparity'] = \
                self.process_disparity(inputs['loaded_disparity'],
                                       max_disparity_range=(10, self.max_disparity))
    
            # now generate synthetic stereo image
            projection_disparity = inputs['disparity']
            right_image = self.project_image(inputs['left_image'],
                                             projection_disparity, inputs['background'])
    
            # augmentation 
            right_image = self.augment_synthetic_image(right_image)
    
            # only keep required keys and prepare for network
            inputs = {'image': inputs['left_image'],
                      'stereo_image': right_image,
                      'disparity': projection_disparity.astype(float),
                      'mono_disparity': inputs['loaded_disparity'].astype(float),
                      }
    
            # finally crop to feed width
            for key in ['image', 'stereo_image']:
                inputs[key] = inputs[key].crop((0, 0, self.feed_width, self.feed_height))
            for key in ['disparity', 'mono_disparity']:
                inputs[key] = inputs[key][:, :self.feed_width]
    
            # Ensure tensors are moved to the GPU device
            self.preprocess(inputs)
    
            return inputs
    
    if __name__ == "__main__":
        dataset = BaseDataset(r"/home/notebook/data/group/Ruby/stereo-from-mono-master/")
        save_path = r"/home/notebook/data/group/Ruby/stereo-from-mono-master/refine_results/"
        count = 5000 # Original value is 5000
        for data in dataset:
            print(count)
            left = np.array(data["image"].cpu())  # Move tensor back to CPU for saving
            right = np.array(data["stereo_image"].cpu())
            disparity = np.array(data["disparity"].cpu())
            mono_disparity = np.array(data["mono_disparity"].cpu())
            
            left = cv2.cvtColor(left, cv2.COLOR_RGB2BGR)
            right = cv2.cvtColor(right, cv2.COLOR_RGB2BGR)
            cv2.imwrite(save_path + str(count) + "_left.jpg", left)
            cv2.imwrite(save_path + str(count) + "_right.jpg", right)
            cv2.imwrite(save_path + str(count) + "_disp.png", disparity*10)
            count += 1
    

    Key Changes:

    • Device Management: Defined device at the start to check for a GPU and used .to(device) to transfer data to the GPU.
    • Tensor Handling: Moved any tensor operations like normalization or processing within preprocess to make sure they are done on the GPU.

    Keep in mind, file I/O operations and data movements are typically done on the CPU. The actual data processing can be moved to GPU. If further optimizations are needed, especially for large-scale operations, consider employing dataloaders with appropriate num_workers and pin_memory settings.

Responses were produced in different, uncontrolled contexts (system prompts, settings, prior turns). This is not a benchmark.