import numpy as np import os import time import torch from torch.utils.data import…
- User
import numpy as np import os import time import torch from torch.utils.data import Dataset, DataLoader import random
DEBUG = 1
class SpatialSlicesDataset(Dataset): def init(self, spatial_slices): self.spatial_slices = spatial_slices self.num, self.x, self.y, _ = self.spatial_slices.shape
def __getitem__(self, idx): spatial_slice = self.spatial_slices[idx] coords = torch.stack([ (torch.arange(self.x).float() / (self.x - 1) * 2 - 1).unsqueeze(1).repeat(1, self.y), (torch.arange(self.y).float() / (self.y - 1) * 2 - 1).unsqueeze(0).repeat(self.x, 1) # (torch.arange(self.x).float() / (self.x - 1) * 2 - 1).expand(-1, self.y), # (torch.arange(self.y).float() / (self.y - 1) * 2 - 1).expand(self.x, -1) ], dim=-1).to(torch.float32) targets = spatial_slice.to(torch.float32) return coords, targets def __len__(self): return self.num def get_batch(self, batch_size, sample_num): n_indices = torch.randint(0, self.num-1, (batch_size,)) timesteps = (n_indices.float() / 25).int() selected_slices = self.spatial_slices[n_indices] x_indices = torch.randint(0, self.x, (batch_size, sample_num)) y_indices = torch.randint(0, self.y, (batch_size, sample_num)) # - make sure both corrds and targets are float type # targets = torch.stack([selected_slices[i][x_indices[i], y_indices[i]].float() for i in range(batch_size)]) targets = selected_slices[torch.arange(batch_size)[:, None], x_indices, y_indices].float() coords = torch.stack([ (x_indices.float() / (self.x - 1) * 2 - 1), (y_indices.float() / (self.y - 1) * 2 - 1) ], dim=-1) return coords, targets, timesteps def get_slice(self, index): return self.__getitem__(index)class SpatialFieldsDataset(Dataset): def init(self, data_dir): self.data_dir = data_dir self.field_idx = []
for filename in os.listdir(data_dir): # if filename.endswith(".pt"): self.field_idx.append(filename[:4]) #%TODO self.t = 1001 self.x = 512 self.y = 512 def __len__(self): return len(self.field_idx) def __getitem__(self, idx): #%TODO n_idx is not continuous integer, n_idx+=160 if we pick 50 fields in all n_idx, t_idx = divmod(idx, self.t) n_idx = n_idx*160 spatial_slice = torch.load(os.path.join(self.data_dir, f'/{n_idx}/{n_idx}_{t_idx}.pt')) coords = torch.stack([ (torch.arange(self.x).float() / (self.x - 1) * 2 - 1).expand(-1, self.y), (torch.arange(self.y).float() / (self.y - 1) * 2 - 1).expand(self.x, -1) ], dim=-1) # - reshaping the spatial slice to match the coordinates shape targets = spatial_slice return coords, targets def get_batch(self, batch_size, sample_num): coords = torch.empty((batch_size, sample_num, 2)) targets = torch.empty((batch_size, sample_num, 2)) n_indices = random.choices(self.field_idx, k=batch_size) t_indices = torch.randint(0, self.t, (batch_size,)) # - load spatial slices in batch spatial_slices = [torch.load(os.path.join(self.data_dir, f'{n_idx}/{n_idx}_{t_idx}.pt')) for n_idx, t_idx in zip(n_indices, t_indices)] spatial_slices_tensor = torch.stack(spatial_slices) # - randomly select x and y coordinates within the field for each sample in the batch x_indices = torch.randint(0, self.x, (batch_size, sample_num)) y_indices = torch.randint(0, self.y, (batch_size, sample_num)) # - use advanced indexing to select targets targets = spatial_slices_tensor[torch.arange(batch_size)[:, None], x_indices, y_indices] coords = torch.stack([ (x_indices.float() / (self.x - 1) * 2 - 1).unsqueeze(-1), (y_indices.float() / (self.y - 1) * 2 - 1).unsqueeze(-1) ], dim=-1) return coords, targets# - test SpatialSlicesDataset
dataset = torch.load(os.path.join('input/7999_train.pt'))
data_loader = SpatialSlicesDataset(dataset)
batch = data_loader.get_batch(8, 500)
if DEBUG:
print(batch[0].shape)
print(batch[1].shape)
print(f'coords range: {batch[0].min()},{batch[0].max()}')
print(f'target range: {batch[1].min()},{batch[1].max()}')
print(f'timestep range: {batch[2].min()},{batch[2].max()}')
# - test SpatialSlicesDataset
data_loader = SpatialFieldsDataset('/media/wangx84/Elements/fuild_flow_eth')
batch = data_loader.get_batch(8, 100)
if DEBUG:
print(batch[0].shape)
print(batch[1].shape)
print(f'coords range: {batch[0].min()},{batch[0].max()}')
print(f'target range: {batch[1].min()},{batch[1].max()}')
为什么在spatial_slices = [torch.load(os.path.join(self.data_dir, f'{n_idx}/{n_idx}_{t_idx}.pt')) for n_idx, t_idx in zip(n_indices, t_indices)]这一行,n_idx=spat?
- Assistant
在这行代码:
spatial_slices = [torch.load(os.path.join(self.data_dir, f'{n_idx}/{n_idx}_{t_idx}.pt')) for n_idx, t_idx in zip(n_indices, t_indices)]n_idx的赋值来源于之前的列表推导式,它取自n_indices,这个列表是通过random.choices方法从self.field_idx中随机选取的一些字段索引。n_idx=spat这部分看起来是注释或者编辑问题,应该不是代码的一部分。如果n_idx被赋予spat,那么它可能是在其他部分的代码中进行了赋值,或者spat应该代表某种含义(比如可能是一个错误的变量名或者是一个占位符)。实际上,代码执行时,
n_idx会是从n_indices中选取的一个值,每次循环它都会被赋予一个新值。循环迭代时填充路径字符串时,会根据n_idx和t_idx的组合加载对应的.pt文件。根据你的注释
%TODO和#%TODO,这是一个正在开发或者未完成的部分,所以最好检查一下代码并确保所有需要的变量都被正确定义了。 如果遇到错误,确保调试时遵照代码的上下文来理解变量的含义和用途。 - User
n_indices = random.choices(self.field_idx, k=batch_size) 为什么n_indices会有一个spat?
- Assistant
The line
n_indices = random.choices(self.field_idx, k=batch_size)is using Python's
random.choices()method to selectbatch_sizerandom elements from the listself.field_idx. Ifself.field_idxcontains a string"spat", then it is possible forn_indicesto include"spat"as one of the elements becauserandom.choices()selects fromself.field_idxwith replacement.If you are seeing
"spat"in then_indiceslist and you are wondering why it is there, you should check howself.field_idxis being populated. The list comprehension in the__init__method ofSpatialFieldsDatasetlooks like:for filename in os.listdir(data_dir): self.field_idx.append(filename[:4])According to this code,
self.field_idxwill contain the first four characters of every file name indata_dir. If any of the file names start with the string"spat", that substring will be added toself.field_idxand subsequentlyn_indicescan include"spat"after the call torandom.choices().Here's a possible scenario:
- There is a file in
data_dirnamed something like"spat_0001.pt". - The list comprehension takes the first four characters (
"spat") and adds them toself.field_idx. - When
random.choices()is called, it might select"spat"fromself.field_idxas part ofn_indices.
To prevent unintended strings in
self.field_idx, you can add more checks in the list comprehension such as confirming that the filenames match a certain pattern or contain valid numeric indices. Consider using regular expressions or other string methods to validate or parse relevant information from the filenames.If "spat" is not an expected or desired value, you may need to modify the way
self.field_idxis built to exclude non-numeric identifiers or to ensure it matches the expected format of your dataset files. - There is a file in