Skip to content
TheChatArchiveThe Chat Archive
gpt-3.5-turbo-0613· OpenAIAi2 WildChat-4.8M (AI2)

int32 CreateIoStoreContainerFiles(const TCHAR* CmdLine) {…

  1. User

    int32 CreateIoStoreContainerFiles(const TCHAR* CmdLine) { IOSTORE_CPU_SCOPE(CreateIoStoreContainerFiles);

    UE_LOG(LogIoStore, Display, TEXT("==================== IoStore Utils ===================="));
    
    FIoStoreArguments Arguments;
    
    LoadKeyChain(FCommandLine::Get(), Arguments.KeyChain);
    
    if (FParse::Param(FCommandLine::Get(), TEXT("sign")))
    {
    	Arguments.bSign = true;
    }
    
    UE_LOG(LogIoStore, Display, TEXT("Container signing - %s"), Arguments.bSign ? TEXT("ENABLED") : TEXT("DISABLED"));
    
    Arguments.bCreateDirectoryIndex = !FParse::Param(FCommandLine::Get(), TEXT("NoDirectoryIndex"));
    UE_LOG(LogIoStore, Display, TEXT("Directory index - %s"), Arguments.bCreateDirectoryIndex  ? TEXT("ENABLED") : TEXT("DISABLED"));
    
    FString PatchReferenceCryptoKeysFilename;
    FKeyChain PatchKeyChain;
    if (FParse::Value(FCommandLine::Get(), TEXT("PatchCryptoKeys="), PatchReferenceCryptoKeysFilename))
    {
    	LoadKeyChainFromFile(PatchReferenceCryptoKeysFilename, Arguments.PatchKeyChain);
    }
    
    FString GameOrderFilePath;
    if (FParse::Value(FCommandLine::Get(), TEXT("GameOrder="), GameOrderFilePath))
    {
    	if (!ParsePakOrderFile(*GameOrderFilePath, Arguments.GameOrderMap))
    	{
    		return -1;
    	}
    }
    
    FString CookerOrderFilePath;
    if (FParse::Value(FCommandLine::Get(), TEXT("CookerOrder="), CookerOrderFilePath))
    {
    	if (!ParsePakOrderFile(*CookerOrderFilePath, Arguments.CookerOrderMap))
    	{
    		return -1;
    	}
    }
    
    FIoStoreWriterSettings GeneralIoWriterSettings { DefaultCompressionMethod, DefaultCompressionBlockSize, false };
    GeneralIoWriterSettings.bEnableCsvOutput = FParse::Param(CmdLine, TEXT("-csvoutput"));
    
    TArray<FName> CompressionFormats;
    FString DesiredCompressionFormats;
    if (FParse::Value(CmdLine, TEXT("-compressionformats="), DesiredCompressionFormats) ||
    	FParse::Value(CmdLine, TEXT("-compressionformat="), DesiredCompressionFormats))
    {
    	TArray<FString> Formats;
    	DesiredCompressionFormats.ParseIntoArray(Formats, TEXT(","));
    	for (FString& Format : Formats)
    	{
    		// look until we have a valid format
    		FName FormatName = *Format;
    
    		if (FCompression::IsFormatValid(FormatName))
    		{
    			GeneralIoWriterSettings.CompressionMethod = FormatName;
    			break;
    		}
    	}
    
    	if (GeneralIoWriterSettings.CompressionMethod == NAME_None)
    	{
    		UE_LOG(LogIoStore, Warning, TEXT("Failed to find desired compression format(s) '%s'. Using falling back to '%s'"),
    			*DesiredCompressionFormats, *DefaultCompressionMethod.ToString());
    	}
    	else
    	{
    		UE_LOG(LogIoStore, Display, TEXT("Using compression format '%s'"), *GeneralIoWriterSettings.CompressionMethod.ToString());
    	}
    }
    
    ParseSizeArgument(CmdLine, TEXT("-alignformemorymapping="), GeneralIoWriterSettings.MemoryMappingAlignment, DefaultMemoryMappingAlignment);
    ParseSizeArgument(CmdLine, TEXT("-compressionblocksize="), GeneralIoWriterSettings.CompressionBlockSize, DefaultCompressionBlockSize);
    bool bUseDefaultCompressionBlockAlignment = true;
    if (ParseSizeArgument(CmdLine, TEXT("-blocksize="), GeneralIoWriterSettings.CompressionBlockAlignment))
    {
    	bUseDefaultCompressionBlockAlignment = false;
    }
    
    uint64 PatchPaddingAlignment = 0;
    if (ParseSizeArgument(CmdLine, TEXT("-patchpaddingalign"), PatchPaddingAlignment))
    {
    	bUseDefaultCompressionBlockAlignment = false;
    	if (!GeneralIoWriterSettings.CompressionBlockAlignment || PatchPaddingAlignment < GeneralIoWriterSettings.CompressionBlockAlignment)
    	{
    		GeneralIoWriterSettings.CompressionBlockAlignment = PatchPaddingAlignment;
    	}
    }
    
    if (bUseDefaultCompressionBlockAlignment)
    {
    	GeneralIoWriterSettings.CompressionBlockAlignment = DefaultCompressionBlockAlignment;
    }
    
    UE_LOG(LogIoStore, Display, TEXT("Using memory mapping alignment '%llu'"), GeneralIoWriterSettings.MemoryMappingAlignment);
    UE_LOG(LogIoStore, Display, TEXT("Using compression block size '%llu'"), GeneralIoWriterSettings.CompressionBlockSize);
    UE_LOG(LogIoStore, Display, TEXT("Using compression block alignment '%llu'"), GeneralIoWriterSettings.CompressionBlockAlignment);
    
    FParse::Value(CmdLine, TEXT("-MetaOutputDirectory="), Arguments.MetaOutputDir);
    FParse::Value(CmdLine, TEXT("-MetaInputDirectory="), Arguments.MetaInputDir);
    
    FString CommandListFile;
    if (FParse::Value(FCommandLine::Get(), TEXT("Commands="), CommandListFile))
    {
    	UE_LOG(LogIoStore, Display, TEXT("Using command list file: '%s'"), *CommandListFile);
    	TArray<FString> Commands;
    	if (!FFileHelper::LoadFileToStringArray(Commands, *CommandListFile))
    	{
    		UE_LOG(LogIoStore, Error, TEXT("Failed to read command list file '%s'."), *CommandListFile);
    		return -1;
    	}
    
    	Arguments.Containers.Reserve(Commands.Num());
    	for (const FString& Command : Commands)
    	{
    		FContainerSourceSpec& ContainerSpec = Arguments.Containers.AddDefaulted_GetRef();
    
    		if (!FParse::Value(*Command, TEXT("Output="), ContainerSpec.OutputPath))
    		{
    			UE_LOG(LogIoStore, Error, TEXT("Output argument missing from command '%s'"), *Command);
    			return -1;
    		}
    		ContainerSpec.OutputPath = FPaths::ChangeExtension(ContainerSpec.OutputPath, TEXT(""));
    
    		FString ContainerName;
    		if (FParse::Value(*Command, TEXT("ContainerName="), ContainerName))
    		{
    			ContainerSpec.Name = FName(ContainerName);
    		}
    
    		FString PatchSourceWildcard;
    		if (FParse::Value(*Command, TEXT("PatchSource="), PatchSourceWildcard))
    		{
    			IFileManager::Get().FindFiles(ContainerSpec.PatchSourceContainerFiles, *PatchSourceWildcard, true, false);
    			FString PatchSourceContainersDirectory = FPaths::GetPath(*PatchSourceWildcard);
    			for (FString& PatchSourceContainerFile : ContainerSpec.PatchSourceContainerFiles)
    			{
    				PatchSourceContainerFile = PatchSourceContainersDirectory / PatchSourceContainerFile;
    				FPaths::NormalizeFilename(PatchSourceContainerFile);
    			}
    		}
    
    		ContainerSpec.bGenerateDiffPatch = FParse::Param(*Command, TEXT("GenerateDiffPatch"));
    
    		FParse::Value(*Command, TEXT("PatchTarget="), ContainerSpec.PatchTargetFile);
    
    		FString ResponseFilePath;
    		if (FParse::Value(*Command, TEXT("ResponseFile="), ResponseFilePath))
    		{
    			if (!ParsePakResponseFile(*ResponseFilePath, ContainerSpec.SourceFiles))
    			{
    				UE_LOG(LogIoStore, Error, TEXT("Failed to parse Pak response file '%s'"), *ResponseFilePath);
    				return -1;
    			}
    
    			FString EncryptionKeyOverrideGuidString;
    			if (FParse::Value(*Command, TEXT("EncryptionKeyOverrideGuid="), EncryptionKeyOverrideGuidString))
    			{
    				FGuid::Parse(EncryptionKeyOverrideGuidString, ContainerSpec.EncryptionKeyOverrideGuid);
    			}
    		}
    	}
    }
    
    if (FParse::Value(FCommandLine::Get(), TEXT("BasedOnReleaseVersionPath="), Arguments.BasedOnReleaseVersionPath))
    {
    	UE_LOG(LogIoStore, Display, TEXT("Based on release version path: '%s'"), *Arguments.BasedOnReleaseVersionPath);
    }
    
    if (FParse::Value(FCommandLine::Get(), TEXT("DLCFile="), Arguments.DLCPluginPath))
    {
    	Arguments.DLCName = FPaths::GetBaseFilename(*Arguments.DLCPluginPath);
    	Arguments.bRemapPluginContentToGame = FParse::Param(FCommandLine::Get(), TEXT("RemapPluginContentToGame"));
    
    	UE_LOG(LogIoStore, Display, TEXT("DLC: '%s'"), *Arguments.DLCPluginPath);
    	UE_LOG(LogIoStore, Display, TEXT("Remapping plugin content to game: '%s'"), Arguments.bRemapPluginContentToGame ? TEXT("True") : TEXT("False"));
    
    	if (Arguments.BasedOnReleaseVersionPath.IsEmpty())
    	{
    		UE_LOG(LogIoStore, Error, TEXT("Based on release version path is needed for DLC"));
    		return -1;
    	}
    
    	FString DevelopmentAssetRegistryPath = FPaths::Combine(Arguments.BasedOnReleaseVersionPath, TEXT("Metadata/DevelopmentAssetRegistry.bin"));
    
    	bool bAssetRegistryLoaded = false;
    	FArrayReader SerializedAssetData;
    	if (FFileHelper::LoadFileToArray(SerializedAssetData, *DevelopmentAssetRegistryPath))
    	{
    		FAssetRegistrySerializationOptions Options;
    		if (Arguments.ReleaseAssetRegistry.Serialize(SerializedAssetData, Options))
    		{
    			UE_LOG(LogIoStore, Display, TEXT("Loaded asset registry '%s'"), *DevelopmentAssetRegistryPath);
    			bAssetRegistryLoaded = true;
    
    			TArray<FName> PackageNames;
    			Arguments.ReleaseAssetRegistry.GetPackageNames(PackageNames);
    			Arguments.ReleasedPackages.PackageNames.Reserve(PackageNames.Num());
    			Arguments.ReleasedPackages.PackageIdToName.Reserve(PackageNames.Num());
    
    			for (FName PackageName : PackageNames)
    			{
    				Arguments.ReleasedPackages.PackageNames.Add(PackageName);
    				Arguments.ReleasedPackages.PackageIdToName.Add(FPackageId::FromName(PackageName), PackageName);
    			}
    		}
    	}
    
    	if (!bAssetRegistryLoaded)
    	{
    		UE_LOG(LogIoStore, Warning, TEXT("Failed to load Asset registry '%s'. Needed to verify DLC package names"), *DevelopmentAssetRegistryPath);
    	}
    }
    
    if (FParse::Value(FCommandLine::Get(), TEXT("CreateGlobalContainer="), Arguments.GlobalContainerPath))
    {
    	Arguments.GlobalContainerPath = FPaths::ChangeExtension(Arguments.GlobalContainerPath, TEXT(""));
    }
    
    if (Arguments.ShouldCreateContainers())
    {
    	FString TargetPlatform;
    	if (FParse::Value(FCommandLine::Get(), TEXT("TargetPlatform="), TargetPlatform))
    	{
    		UE_LOG(LogIoStore, Display, TEXT("Using target platform '%s'"), *TargetPlatform);
    		ITargetPlatformManagerModule& TPM = GetTargetPlatformManagerRef();
    		Arguments.TargetPlatform = TPM.FindTargetPlatform(TargetPlatform);
    		if (!Arguments.TargetPlatform)
    		{
    			UE_LOG(LogIoStore, Error, TEXT("Invalid TargetPlatform: '%s'"), *TargetPlatform);
    			return 1;
    		}
    	}
    	else
    	{
    		UE_LOG(LogIoStore, Error, TEXT("TargetPlatform must be specified"));
    		return 1;
    	}
    
    	// Now that we have the target platform we need to error out if LegacyBulkDataOffsets is enabled for it
    	{
    		FConfigFile PlatformEngineIni;
    		FConfigCacheIni::LoadLocalIniFile(PlatformEngineIni, TEXT("Engine"), true, *Arguments.TargetPlatform->IniPlatformName());
    
    		bool bLegacyBulkDataOffsets = false;
    		PlatformEngineIni.GetBool(TEXT("Core.System"), TEXT("LegacyBulkDataOffsets"), bLegacyBulkDataOffsets);
    
    		if (bLegacyBulkDataOffsets)
    		{
    			UE_LOG(LogIoStore, Error, TEXT("'LegacyBulkDataOffsets' is enabled for the target platform '%s', this needs to be disabled and the data recooked in order for the IoStore to work"), *Arguments.TargetPlatform->IniPlatformName());
    			return 1;
    		}
    	}
    
    	if (!FParse::Value(FCommandLine::Get(), TEXT("CookedDirectory="), Arguments.CookedDir))
    	{
    		UE_LOG(LogIoStore, Error, TEXT("CookedDirectory must be specified"));
    		return 1;
    	}
    
    	for (const FContainerSourceSpec& Container : Arguments.Containers)
    	{
    		if (Container.Name.IsNone())
    		{
    			UE_LOG(LogIoStore, Error, TEXT("ContainerName argument missing for container '%s'"), *Container.OutputPath);
    			return -1;
    		}
    	}
    
    	// Enable file region metadata if required by the target platform.
    	bool bFileRegions = Arguments.TargetPlatform->SupportsFeature(ETargetPlatformFeatures::CookFileRegionMetadata);
    	GeneralIoWriterSettings.bEnableFileRegions = bFileRegions;
    
    	UE_LOG(LogIoStore, Display, TEXT("Searching for cooked assets in folder '%s'"), *Arguments.CookedDir);
    	FCookedFileVisitor CookedFileVistor(Arguments.CookedFileStatMap, nullptr, bFileRegions);
    	IFileManager::Get().IterateDirectoryStatRecursively(*Arguments.CookedDir, CookedFileVistor);
    	UE_LOG(LogIoStore, Display, TEXT("Found '%d' files"), Arguments.CookedFileStatMap.Num());
    
    	int32 ReturnValue = CreateTarget(Arguments, GeneralIoWriterSettings);
    	if (ReturnValue != 0)
    	{
    		return ReturnValue;
    	}
    }
    else if (FParse::Param(FCommandLine::Get(), TEXT("CreateContentPatch")))
    {
    	for (const FContainerSourceSpec& Container : Arguments.Containers)
    	{
    		if (Container.PatchTargetFile.IsEmpty())
    		{
    			UE_LOG(LogIoStore, Error, TEXT("PatchTarget argument missing for container '%s'"), *Container.OutputPath);
    			return -1;
    		}
    	}
    
    	int32 ReturnValue = CreateContentPatch(Arguments, GeneralIoWriterSettings);
    	if (ReturnValue != 0)
    	{
    		return ReturnValue;
    	}
    }
    else
    {
    	FString ContainerPathOrWildcard;
    	if (FParse::Value(FCommandLine::Get(), TEXT("List="), ContainerPathOrWildcard))
    	{
    		FString CsvPath;
    		if (!FParse::Value(FCommandLine::Get(), TEXT("csv="), CsvPath))
    		{
    			UE_LOG(LogIoStore, Error, TEXT("Incorrect arguments. Expected: -list=<ContainerFile> -csv=<path>"));
    		}
    
    		return ListContainer(Arguments, ContainerPathOrWildcard, CsvPath);
    	}
    	else if (FParse::Value(FCommandLine::Get(), TEXT("Describe="), ContainerPathOrWildcard))
    	{
    		FString PackageFilter;
    		FParse::Value(FCommandLine::Get(), TEXT("PackageFilter="), PackageFilter);
    		FString OutPath;
    		FParse::Value(FCommandLine::Get(), TEXT("DumpToFile="), OutPath);
    		bool bIncludeExportHashes = FParse::Param(FCommandLine::Get(), TEXT("IncludeExportHashes"));
    		return Describe(ContainerPathOrWildcard, Arguments.KeyChain, PackageFilter, OutPath, bIncludeExportHashes);
    	}
    	else
    	{
    		UE_LOG(LogIoStore, Display, TEXT("Usage:"));
    		UE_LOG(LogIoStore, Display, TEXT(" -List=</path/to/[container.utoc|*.utoc]> -CSV=<list.csv> [-CryptoKeys=</path/to/crypto.json>]"));
    		UE_LOG(LogIoStore, Display, TEXT(" -Describe=</path/to/global.utoc> [-PackageFilter=<PackageName>] [-DumpToFile=<describe.txt>] [-CryptoKeys=</path/to/crypto.json>]"));
    		return -1;
    	}
    }
    
    return 0;
    

    } 过程中怎么做iostore的分包?例如定义不同目录放到不同的.ucas中

  2. Assistant

    在函数CreateIoStoreContainerFiles中,根据命令行参数解析,通过调用CreateTarget和CreateContentPatch函数来创建iostore的分包。

    在CreateTarget函数中,遍历传入的Arguments结构体中的Containers数组,每个FContainerSourceSpec结构体定义了一个分包的配置信息。根据配置信息,遍历CookedFileStatMap中的文件列表,将文件按照容器的规则进行分配,并写入到相应的.ucas文件中。

    在CreateContentPatch函数中,实现了生成补丁包的逻辑。对于每个分包,根据定义的PatchSourceContainerFiles和PatchTargetFile,计算出需要补丁的文件和目标文件,并写入到相应的.ucas文件中。

    具体的分包逻辑可以参考这两个函数的实现。

  3. User

    解释一下 Ue4 的 streaming,例如 texture streaming

  4. Assistant

    在UE4中,Streaming是指一种动态加载资源的机制,它可以根据需要在运行时异步加载和丢弃资源,以提供更好的性能和优化资源使用。

    Texture Streaming 是 UE4 中的一种流媒体技术,用于管理纹理资源的加载和释放。它允许在运行时异步加载和卸载纹理资源,以适应场景中的视距变化和设备性能。当相机接近一个纹理时,Texture Streaming 会将其从磁盘加载到 GPU 内存,当相机离开时,它会优先释放相机视野外的纹理,以回收资源并减少内存占用。

    Texture Streaming 基于纹理 mipmaps(MIP 纹理),它将纹理分割为多个不同分辨率的层级,每个层级通过降低分辨率来表示纹理的细节。当相机远离一个纹理时,Texture Streaming 可以丢弃更高分辨率级别的纹理,只保留低分辨率级别,从而减少纹理的内存占用。而当相机接近一个纹理时,Texture Streaming 可以逐渐加载更高分辨率级别的纹理,以提供更高的细节质量。

    Texture Streaming 的好处是它在运行时自动管理纹理资源的加载和释放,以适应不同的设备性能和场景需求。通过动态调整纹理的分辨率级别,Texture Streaming 可以提高游戏的性能和内存使用效率,并且可以根据设备的性能和内存情况来提供更