import torch import torch.nn as nn from timm.layers import trunc_normal_ from onescience.modules.mlp.MLP import StandardMLP from onescience.modules.transformer.factformer_block import Factformer_block from onescience.modules.embedding import timestep_embedding, unified_pos_embedding class Model(nn.Module): """ FactFormer 模型。 基于 Factorized Attention 的 Transformer 架构,用于处理结构化网格数据。 """ def __init__(self, args, device): super(Model, self).__init__() self.args = args self.__name__ = "Factformer" # 1. 几何类型检查 if args.geotype == "unstructured": raise ValueError( "Factformer does not support unstructured geometry, please try to integrate GeoFNO layer" ) # 2. Embedding & Preprocessing input_dim = args.fun_dim if args.unified_pos: # only for structured mesh self.pos = unified_pos_embedding(args.shapelist, args.ref, device=device) input_dim += args.ref ** len(args.shapelist) else: input_dim += args.space_dim self.preprocess = StandardMLP( input_dim=input_dim, output_dim=args.n_hidden, hidden_dims=[args.n_hidden * 2], activation=args.act, use_bias=True, res=False, ) if args.time_input: self.time_fc = nn.Sequential( nn.Linear(args.n_hidden, args.n_hidden), nn.SiLU(), nn.Linear(args.n_hidden, args.n_hidden), ) # 3. Transformer Blocks self.blocks = nn.ModuleList([ Factformer_block( num_heads=args.n_heads, hidden_dim=args.n_hidden, dropout=args.dropout, act=args.act, mlp_ratio=args.mlp_ratio, out_dim=args.out_dim, last_layer=(_ == args.n_layers - 1), geotype=args.geotype, shapelist=args.shapelist, ) for _ in range(args.n_layers) ]) self.placeholder = nn.Parameter( (1 / (args.n_hidden)) * torch.rand(args.n_hidden, dtype=torch.float) ) self.initialize_weights() def initialize_weights(self): self.apply(self._init_weights) def _init_weights(self, m): if isinstance(m, nn.Linear): trunc_normal_(m.weight, std=0.02) if isinstance(m, nn.Linear) and m.bias is not None: nn.init.constant_(m.bias, 0) elif isinstance(m, (nn.LayerNorm, nn.BatchNorm1d)): nn.init.constant_(m.bias, 0) nn.init.constant_(m.weight, 1.0) def structured_geo(self, x, fx, T=None): if self.args.unified_pos: x = self.pos.repeat(x.shape[0], 1, 1) if fx is not None: fx = torch.cat((x, fx), -1) fx = self.preprocess(fx) else: fx = self.preprocess(x) fx = fx + self.placeholder[None, None, :] if T is not None: Time_emb = timestep_embedding(T, self.args.n_hidden) # (B, C) Time_emb = self.time_fc(Time_emb) if Time_emb.ndim == 2: Time_emb = Time_emb.unsqueeze(1) # (B, 1, C) fx = fx + Time_emb for block in self.blocks: fx = block(fx) return fx def unstructured_geo(self, x, fx, T=None): # 保持对非结构化网格的异常抛出 raise ValueError( "Factformer does not support unstructured geometry, please try to integrate GeoFNO layer" ) def forward(self, x, fx, T=None, geo=None): if self.args.geotype == "unstructured": return self.unstructured_geo(x, fx, T) else: return self.structured_geo(x, fx, T)