Skip to content
TheChatArchiveThe Chat Archive
gpt-4-1106-preview· openaiHugging Face Suzhen/CodeChat

import numpy as np import os import time import torch from torch.utils.data import…

  1. User

    import numpy as np import os import time import torch from torch.utils.data import Dataset, DataLoader import random

    DEBUG = 1

    class SpatialSlicesDataset(Dataset): def init(self, spatial_slices): self.spatial_slices = spatial_slices self.num, self.x, self.y, _ = self.spatial_slices.shape

    def __getitem__(self, idx):
        spatial_slice = self.spatial_slices[idx]
        coords = torch.stack([
            (torch.arange(self.x).float() / (self.x - 1) * 2 - 1).unsqueeze(1).repeat(1, self.y),
            (torch.arange(self.y).float() / (self.y - 1) * 2 - 1).unsqueeze(0).repeat(self.x, 1)
            # (torch.arange(self.x).float() / (self.x - 1) * 2 - 1).expand(-1, self.y),
            # (torch.arange(self.y).float() / (self.y - 1) * 2 - 1).expand(self.x, -1)
        ], dim=-1).to(torch.float32)
        targets = spatial_slice.to(torch.float32)
        return coords, targets
    
    
    def __len__(self):
        return self.num
    
    
    def get_batch(self, batch_size, sample_num):
        n_indices = torch.randint(0, self.num-1, (batch_size,))
        timesteps = (n_indices.float() / 25).int()
    
        selected_slices = self.spatial_slices[n_indices]
    
        x_indices = torch.randint(0, self.x, (batch_size, sample_num))
        y_indices = torch.randint(0, self.y, (batch_size, sample_num))
    
        # - make sure both corrds and targets are float type
        # targets = torch.stack([selected_slices[i][x_indices[i], y_indices[i]].float() for i in range(batch_size)])
        targets = selected_slices[torch.arange(batch_size)[:, None], x_indices, y_indices].float()
    
        coords = torch.stack([
            (x_indices.float() / (self.x - 1) * 2 - 1),
            (y_indices.float() / (self.y - 1) * 2 - 1)
        ], dim=-1)
    
        return coords, targets, timesteps
    
    
    def get_slice(self, index):
        return self.__getitem__(index)
    

    class SpatialFieldsDataset(Dataset): def init(self, data_dir): self.data_dir = data_dir self.field_idx = []

        for filename in os.listdir(data_dir):
            # if filename.endswith(".pt"):  
            self.field_idx.append(filename[:4])
    
        #%TODO
        self.t = 1001
        self.x = 512
        self.y = 512
    
    
    def __len__(self):
        return len(self.field_idx)
    
    
    def __getitem__(self, idx):
        #%TODO n_idx is not continuous integer, n_idx+=160 if we pick 50 fields in all
        n_idx, t_idx = divmod(idx, self.t)
        n_idx = n_idx*160
        spatial_slice = torch.load(os.path.join(self.data_dir, f'/{n_idx}/{n_idx}_{t_idx}.pt'))
        
        coords = torch.stack([
            (torch.arange(self.x).float() / (self.x - 1) * 2 - 1).expand(-1, self.y),
            (torch.arange(self.y).float() / (self.y - 1) * 2 - 1).expand(self.x, -1)
        ], dim=-1)
        
        # - reshaping the spatial slice to match the coordinates shape
        targets = spatial_slice
        
        return coords, targets                
    
    def get_batch(self, batch_size, sample_num):
        coords = torch.empty((batch_size, sample_num, 2))
        targets = torch.empty((batch_size, sample_num, 2))
    
        n_indices = random.choices(self.field_idx, k=batch_size)
        t_indices = torch.randint(0, self.t, (batch_size,))
    
        # - load spatial slices in batch
        spatial_slices = [torch.load(os.path.join(self.data_dir, f'{n_idx}/{n_idx}_{t_idx}.pt')) for n_idx, t_idx in zip(n_indices, t_indices)]
        spatial_slices_tensor = torch.stack(spatial_slices)
    
        # - randomly select x and y coordinates within the field for each sample in the batch
        x_indices = torch.randint(0, self.x, (batch_size, sample_num))
        y_indices = torch.randint(0, self.y, (batch_size, sample_num))
    
        # - use advanced indexing to select targets
        targets = spatial_slices_tensor[torch.arange(batch_size)[:, None], x_indices, y_indices]
    
        coords = torch.stack([
            (x_indices.float() / (self.x - 1) * 2 - 1).unsqueeze(-1), 
            (y_indices.float() / (self.y - 1) * 2 - 1).unsqueeze(-1)
        ], dim=-1)
        
        return coords, targets
    

    # - test SpatialSlicesDataset

    dataset = torch.load(os.path.join('input/7999_train.pt'))

    data_loader = SpatialSlicesDataset(dataset)

    batch = data_loader.get_batch(8, 500)

    if DEBUG:

    print(batch[0].shape)

    print(batch[1].shape)

    print(f'coords range: {batch[0].min()},{batch[0].max()}')

    print(f'target range: {batch[1].min()},{batch[1].max()}')

    print(f'timestep range: {batch[2].min()},{batch[2].max()}')

    # - test SpatialSlicesDataset

    data_loader = SpatialFieldsDataset('/media/wangx84/Elements/fuild_flow_eth')

    batch = data_loader.get_batch(8, 100)

    if DEBUG:

    print(batch[0].shape)

    print(batch[1].shape)

    print(f'coords range: {batch[0].min()},{batch[0].max()}')

    print(f'target range: {batch[1].min()},{batch[1].max()}')

    为什么在spatial_slices = [torch.load(os.path.join(self.data_dir, f'{n_idx}/{n_idx}_{t_idx}.pt')) for n_idx, t_idx in zip(n_indices, t_indices)]这一行,n_idx=spat?

  2. Assistant

    在这行代码:

    spatial_slices = [torch.load(os.path.join(self.data_dir, f'{n_idx}/{n_idx}_{t_idx}.pt')) for n_idx, t_idx in zip(n_indices, t_indices)]
    

    n_idx 的赋值来源于之前的列表推导式,它取自 n_indices,这个列表是通过 random.choices 方法从 self.field_idx 中随机选取的一些字段索引。

    n_idx=spat 这部分看起来是注释或者编辑问题,应该不是代码的一部分。如果 n_idx 被赋予 spat,那么它可能是在其他部分的代码中进行了赋值,或者 spat 应该代表某种含义(比如可能是一个错误的变量名或者是一个占位符)。

    实际上,代码执行时,n_idx 会是从 n_indices 中选取的一个值,每次循环它都会被赋予一个新值。循环迭代时填充路径字符串时,会根据 n_idx 和 t_idx 的组合加载对应的 .pt 文件。

    根据你的注释 %TODO 和 #%TODO,这是一个正在开发或者未完成的部分,所以最好检查一下代码并确保所有需要的变量都被正确定义了。 如果遇到错误,确保调试时遵照代码的上下文来理解变量的含义和用途。

  3. User

    n_indices = random.choices(self.field_idx, k=batch_size) 为什么n_indices会有一个spat?

  4. Assistant

    The line

    n_indices = random.choices(self.field_idx, k=batch_size)
    

    is using Python's random.choices() method to select batch_size random elements from the list self.field_idx. If self.field_idx contains a string "spat", then it is possible for n_indices to include "spat" as one of the elements because random.choices() selects from self.field_idx with replacement.

    If you are seeing "spat" in the n_indices list and you are wondering why it is there, you should check how self.field_idx is being populated. The list comprehension in the __init__ method of SpatialFieldsDataset looks like:

    for filename in os.listdir(data_dir):
        self.field_idx.append(filename[:4])
    

    According to this code, self.field_idx will contain the first four characters of every file name in data_dir. If any of the file names start with the string "spat", that substring will be added to self.field_idx and subsequently n_indices can include "spat" after the call to random.choices().

    Here's a possible scenario:

    1. There is a file in data_dir named something like "spat_0001.pt".
    2. The list comprehension takes the first four characters ("spat") and adds them to self.field_idx.
    3. When random.choices() is called, it might select "spat" from self.field_idx as part of n_indices.

    To prevent unintended strings in self.field_idx, you can add more checks in the list comprehension such as confirming that the filenames match a certain pattern or contain valid numeric indices. Consider using regular expressions or other string methods to validate or parse relevant information from the filenames.

    If "spat" is not an expected or desired value, you may need to modify the way self.field_idx is built to exclude non-numeric identifiers or to ensure it matches the expected format of your dataset files.