我们从Python开源项目中，提取了以下50个代码示例，用于说明如何使用torch.nn.ModuleList()。
def __init__(self, mode, anchors=9, classes=80, depth=4, base_activation=F.relu, output_activation=F.sigmoid): super(SubNet, self).__init__() self.anchors = anchors self.classes = classes self.depth = depth self.base_activation = base_activation self.output_activation = output_activation self.subnet_base = nn.ModuleList([conv3x3(256, 256, padding=1) for _ in range(depth)]) if mode == 'boxes': self.subnet_output = conv3x3(256, 4 * self.anchors, padding=1) elif mode == 'classes': # add an extra dim for confidence self.subnet_output = conv3x3(256, (1 + self.classes) * self.anchors, padding=1) self._output_layer_init(self.subnet_output.bias.data)
def __init__(self, phase, base, extras, head, num_classes): super(SSD, self).__init__() self.phase = phase self.num_classes = num_classes # TODO: implement __call__ in PriorBox self.priorbox = PriorBox(v2) self.priors = Variable(self.priorbox.forward(), volatile=True) self.size = 300 # SSD network self.vgg = nn.ModuleList(base) # Layer learns to scale the l2 normalized features from conv4_3 self.L2Norm = L2Norm(512, 20) self.extras = nn.ModuleList(extras) self.loc = nn.ModuleList(head[0]) self.conf = nn.ModuleList(head[1]) if phase == 'test': self.softmax = nn.Softmax() self.detect = Detect(num_classes, 0, 200, 0.01, 0.45)
def __init__(self, dictionary, encoder_embed_dim=512, embed_dim=512, out_embed_dim=512, num_layers=1, dropout_in=0.1, dropout_out=0.1, attention=True): super().__init__() self.dictionary = dictionary self.dropout_in = dropout_in self.dropout_out = dropout_out num_embeddings = len(dictionary) padding_idx = dictionary.pad() self.embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx) self.layers = nn.ModuleList([ LSTMCell(encoder_embed_dim + embed_dim if layer == 0 else embed_dim, embed_dim) for layer in range(num_layers) ]) self.attention = AttentionLayer(encoder_embed_dim, embed_dim) if embed_dim != out_embed_dim: self.additional_fc = Linear(embed_dim, out_embed_dim) self.fc_out = Linear(out_embed_dim, num_embeddings, dropout=dropout_out)
def __init__(self, input_size, hidden_size, num_layers, dropout_rate=0, dropout_output=False, rnn_type=nn.LSTM, concat_layers=False, padding=False): super(StackedBRNN, self).__init__() self.padding = padding self.dropout_output = dropout_output self.dropout_rate = dropout_rate self.num_layers = num_layers self.concat_layers = concat_layers self.rnns = nn.ModuleList() for i in range(num_layers): input_size = input_size if i == 0 else 2 * hidden_size #self.rnns.append(rnn_type(input_size, hidden_size, # num_layers=1, # bidirectional=True)) self.rnns.append(MF.SRUCell(input_size, hidden_size, dropout=dropout_rate, rnn_dropout=dropout_rate, use_tanh=1, bidirectional=True))
def __init__(self, input_size, hidden_size, num_layers=2, dropout=0, rnn_dropout=0, bidirectional=False, use_tanh=1, use_relu=0): super(SRU, self).__init__() self.n_in = input_size self.n_out = hidden_size self.depth = num_layers self.dropout = dropout self.rnn_dropout = rnn_dropout self.rnn_lst = nn.ModuleList() self.bidirectional = bidirectional self.out_size = hidden_size*2 if bidirectional else hidden_size for i in range(num_layers): l = SRUCell( n_in = self.n_in if i==0 else self.out_size, n_out = self.n_out, dropout = dropout if i+1 != num_layers else 0, rnn_dropout = rnn_dropout, bidirectional = bidirectional, use_tanh = use_tanh, use_relu = use_relu, ) self.rnn_lst.append(l)
def __init__(self, args): super(CNN_Text,self).__init__() self.args = args V = args.embed_num D = args.embed_dim C = args.class_num Ci = 1 Co = args.kernel_num Ks = args.kernel_sizes self.embed = nn.Embedding(V, D) #self.convs1 = [nn.Conv2d(Ci, Co, (K, D)) for K in Ks] self.convs1 = nn.ModuleList([nn.Conv2d(Ci, Co, (K, D)) for K in Ks]) ''' self.conv13 = nn.Conv2d(Ci, Co, (3, D)) self.conv14 = nn.Conv2d(Ci, Co, (4, D)) self.conv15 = nn.Conv2d(Ci, Co, (5, D)) ''' self.dropout = nn.Dropout(args.dropout) self.fc1 = nn.Linear(len(Ks)*Co, C)
def __init__( self, n_src_vocab, n_max_seq, n_layers=6, n_head=8, d_k=64, d_v=64, d_word_vec=512, d_model=512, d_inner_hid=1024, dropout=0.1): super(Encoder, self).__init__() n_position = n_max_seq + 1 self.n_max_seq = n_max_seq self.d_model = d_model self.position_enc = nn.Embedding(n_position, d_word_vec, padding_idx=Constants.PAD) self.position_enc.weight.data = position_encoding_init(n_position, d_word_vec) self.src_word_emb = nn.Embedding(n_src_vocab, d_word_vec, padding_idx=Constants.PAD) self.layer_stack = nn.ModuleList([ EncoderLayer(d_model, d_inner_hid, n_head, d_k, d_v, dropout=dropout) for _ in range(n_layers)])
def __init__( self, n_tgt_vocab, n_max_seq, n_layers=6, n_head=8, d_k=64, d_v=64, d_word_vec=512, d_model=512, d_inner_hid=1024, dropout=0.1): super(Decoder, self).__init__() n_position = n_max_seq + 1 self.n_max_seq = n_max_seq self.d_model = d_model self.position_enc = nn.Embedding( n_position, d_word_vec, padding_idx=Constants.PAD) self.position_enc.weight.data = position_encoding_init(n_position, d_word_vec) self.tgt_word_emb = nn.Embedding( n_tgt_vocab, d_word_vec, padding_idx=Constants.PAD) self.dropout = nn.Dropout(dropout) self.layer_stack = nn.ModuleList([ DecoderLayer(d_model, d_inner_hid, n_head, d_k, d_v, dropout=dropout) for _ in range(n_layers)])
def setUttEncoder(module): # set utterance encoder to the module if SharedModel.args.utt_enc_noise == True: module.uttEncNoise = Variable(torch.FloatTensor(), volatile=True) if SharedModel.args.no_cuda == False: module.uttEncNoise = module.uttEncNoise.cuda() if SharedModel.args.utt_enc_type >= 2: module.uttEncoder = nn.ModuleList() for i in [int(x) for x in SharedModel.args.conv_filters.split('_')]: module.uttEncoder.append( nn.Conv1d(2*SharedModel.args.hid_dim * (2 if SharedModel.args.attn == 2 else 1), SharedModel.args.conv_out_dim, i, 1, int(math.ceil((i-1)/2))) ) if SharedModel.args.utt_enc_bn == True: uttEncOutSize = 2 * SharedModel.args.hid_dim if SharedModel.args.utt_enc_type >= 2: uttEncOutSize = 3 * SharedModel.args.conv_out_dim elif SharedModel.args.attn == 2: uttEncOutSize = 4 * SharedModel.args.hid_dim module.uttBn = nn.BatchNorm1d(uttEncOutSize)
def __init__(self, input_dim, conv_bank_dim, conv_dim1, conv_dim2, gru_dim, num_filters, is_masked): super(CBHG, self).__init__() self.num_filters = num_filters bank_out_dim = num_filters * conv_bank_dim self.conv_bank = nn.ModuleList() for i in range(num_filters): self.conv_bank.append(nn.Conv1d(input_dim, conv_bank_dim, i + 1, stride=1, padding=int(np.ceil(i / 2)))) # define batch normalization layer, we use BN1D since the sequence length is not fixed self.bn_list = nn.ModuleList() self.bn_list.append(nn.BatchNorm1d(bank_out_dim)) self.bn_list.append(nn.BatchNorm1d(conv_dim1)) self.bn_list.append(nn.BatchNorm1d(conv_dim2)) self.conv1 = nn.Conv1d(bank_out_dim, conv_dim1, 3, stride=1, padding=1) self.conv2 = nn.Conv1d(conv_dim1, conv_dim2, 3, stride=1, padding=1) if input_dim != conv_dim2: self.residual_proj = nn.Linear(input_dim, conv_dim2) self.highway = Highway(conv_dim2, 4) self.BGRU = nn.GRU(input_size=conv_dim2, hidden_size=gru_dim, num_layers=1, batch_first=True, bidirectional=True)
def __init__(self, in_planes, out_planes, pool_method, stride): super(Block, self).__init__() self.branches = nn.ModuleList([ nn.Sequential( _make_conv(in_planes, out_planes, kernel_size=1, padding=0), _make_conv(out_planes, out_planes, stride=stride) ), nn.Sequential( _make_conv(in_planes, out_planes, kernel_size=1, padding=0), _make_conv(out_planes, out_planes), _make_conv(out_planes, out_planes, stride=stride)) ]) if pool_method == 'Avg': assert stride == 1 self.branches.append( _make_conv(in_planes, out_planes, kernel_size=1, padding=0)) self.branches.append(nn.Sequential( nn.AvgPool2d(kernel_size=3, stride=1, padding=1), _make_conv(in_planes, out_planes, kernel_size=1, padding=0))) else: self.branches.append( nn.MaxPool2d(kernel_size=3, stride=stride, padding=1))
def __init__(self, base, extras, head, num_classes): super(SSD, self).__init__() self.num_classes = num_classes # TODO: implement __call__ in PriorBox self.priorbox = PriorBox(v2) self.priors = Variable(self.priorbox.forward(), volatile=True) self.num_priors = self.priors.size(0) self.size = 300 # SSD network self.vgg = nn.ModuleList(base) # Layer learns to scale the l2 normalized features from conv4_3 self.L2Norm = L2Norm(512, 20) self.extras = nn.ModuleList(extras) self.loc = nn.ModuleList(head[0]) self.conf = nn.ModuleList(head[1]) self.softmax = nn.Softmax().cuda() # self.detect = Detect(num_classes, 0, 200, 0.001, 0.45)
def __init__(self, dictionary, embed_dim=512, max_positions=1024, convolutions=((512, 3),) * 20, dropout=0.1): super().__init__() self.dictionary = dictionary self.dropout = dropout self.num_attention_layers = None num_embeddings = len(dictionary) padding_idx = dictionary.pad() self.embed_tokens = Embedding(num_embeddings, embed_dim, padding_idx) self.embed_positions = Embedding(max_positions, embed_dim, padding_idx) in_channels = convolutions[0][0] self.fc1 = Linear(embed_dim, in_channels, dropout=dropout) self.projections = nn.ModuleList() self.convolutions = nn.ModuleList() for (out_channels, kernel_size) in convolutions: pad = (kernel_size - 1) / 2 self.projections.append(Linear(in_channels, out_channels) if in_channels != out_channels else None) self.convolutions.append( ConvTBC(in_channels, out_channels * 2, kernel_size, padding=pad, dropout=dropout)) in_channels = out_channels self.fc2 = Linear(in_channels, embed_dim)
def __init__(self, opt, visual_embedding=True, question_embedding=True): super(MutanFusion, self).__init__(opt) self.visual_embedding = visual_embedding self.question_embedding = question_embedding # Modules if self.visual_embedding: self.linear_v = nn.Linear(self.opt['dim_v'], self.opt['dim_hv']) else: print('Warning fusion.py: no visual embedding before fusion') if self.question_embedding: self.linear_q = nn.Linear(self.opt['dim_q'], self.opt['dim_hq']) else: print('Warning fusion.py: no question embedding before fusion') self.list_linear_hv = nn.ModuleList([ nn.Linear(self.opt['dim_hv'], self.opt['dim_mm']) for i in range(self.opt['R'])]) self.list_linear_hq = nn.ModuleList([ nn.Linear(self.opt['dim_hq'], self.opt['dim_mm']) for i in range(self.opt['R'])])
def __init__(self, opt={}, vocab_words=[], vocab_answers=[]): # TODO: deep copy ? opt['attention']['dim_v'] = opt['attention']['dim_h'] opt['attention']['dim_q'] = opt['attention']['dim_h'] opt['attention']['dim_mm'] = opt['attention']['dim_h'] super(MLBAtt, self).__init__(opt, vocab_words, vocab_answers) # Modules for classification self.list_linear_v_fusion = nn.ModuleList([ nn.Linear(self.opt['dim_v'], self.opt['fusion']['dim_h']) for i in range(self.opt['attention']['nb_glimpses'])]) self.linear_q_fusion = nn.Linear(self.opt['dim_q'], self.opt['fusion']['dim_h'] * self.opt['attention']['nb_glimpses']) self.linear_classif = nn.Linear(self.opt['fusion']['dim_h'] * self.opt['attention']['nb_glimpses'], self.num_classes)
def __init__(self, opt={}, vocab_words=[], vocab_answers=[]): # TODO: deep copy ? opt['attention']['dim_v'] = opt['attention']['dim_hv'] opt['attention']['dim_q'] = opt['attention']['dim_hq'] super(MutanAtt, self).__init__(opt, vocab_words, vocab_answers) # Modules for classification self.fusion_att = fusion.MutanFusion2d(self.opt['attention'], visual_embedding=False, question_embedding=False) self.list_linear_v_fusion = nn.ModuleList([ nn.Linear(self.opt['dim_v'], int(self.opt['fusion']['dim_hv'] / opt['attention']['nb_glimpses'])) for i in range(self.opt['attention']['nb_glimpses'])]) self.linear_q_fusion = nn.Linear(self.opt['dim_q'], self.opt['fusion']['dim_hq']) self.linear_classif = nn.Linear(self.opt['fusion']['dim_mm'], self.num_classes) self.fusion_classif = fusion.MutanFusion(self.opt['fusion'], visual_embedding=False, question_embedding=False)
def __init__(self, args): super(TextCNN, self).__init__() self.args = args V = args.vocab_size D = args.embed_dim C = args.num_classes Cin = 1 Cout = args.kernel_num Ks = args.kernel_sizes self.embeding = nn.Embedding(V, D) self.convs = nn.ModuleList([nn.Conv2d(Cin, Cout, (K, D)) for K in Ks]) self.dropout = nn.Dropout(args.dropout) self.fc = nn.Linear(len(Ks)*Cout, C)
def __init__(self, input_size, layer_type, layer_sizes=(64, 64), *args, **kwargs): super(MultiLayerLSTM, self).__init__() rnn = layer_type layers = [] prev_size = input_size for size in layer_sizes[:-1]: layer = rnn(input_size=prev_size, hidden_size=size, *args, **kwargs) layers.append(layer) prev_size = size if 'dropout' in kwargs: del kwargs['dropout'] layer = rnn(input_size=prev_size, hidden_size=size, dropout=0.0, *args, **kwargs) layers.append(layer) self.layers = layers self.layer_sizes = layer_sizes self.input_size = input_size self.params = nn.ModuleList(layers)
def __init__(self, input_size, hidden_size, num_layers=2, dropout=0, rnn_dropout=0, bidirectional=False, use_tanh=1, use_relu=0, use_kernel=True): super(SRU, self).__init__() self.n_in = input_size self.n_out = hidden_size self.depth = num_layers self.dropout = dropout self.rnn_dropout = rnn_dropout self.rnn_lst = nn.ModuleList() self.bidirectional = bidirectional self.use_kernel = use_kernel self.out_size = hidden_size*2 if bidirectional else hidden_size for i in range(num_layers): l = SRUCell( n_in = self.n_in if i==0 else self.out_size, n_out = self.n_out, dropout = dropout if i+1 != num_layers else 0, rnn_dropout = rnn_dropout, bidirectional = bidirectional, use_tanh = use_tanh, use_relu = use_relu, use_kernel = use_kernel, ) self.rnn_lst.append(l)
def __init__(self, opt ): super(MultiModelAll2, self).__init__() self.model_name = 'MultiModelAll2' self.opt=opt self.models = [] for _name,_path in zip(opt.model_names, opt.model_paths): tmp_config = Config().parse(opt.state_dict(),print_=False) # tmp_config.static=True tmp_config.embedding_path=None _model = getattr(models,_name)(tmp_config) if _path is not None: _model.load(_path) self.models.append(_model) self.models = nn.ModuleList(self.models) self.model_num = len(self.models) self.weights = nn.Parameter(t.ones(opt.num_classes,self.model_num)) assert self.opt.loss=='bceloss' self.eval()
def __init__(self, opt ): super(MultiModelAll4zhihu, self).__init__() self.model_name = 'MultiModelAll4zhihu' self.opt=opt self.models = [] self.word_embedding=nn.Embedding(411720,256) self.char_embedding=nn.Embedding(11973,256) model_opts = t.load(opt.model_path+'.json') for _name,_path,model_opt_ in zip(opt.model_names, opt.model_paths,model_opts): tmp_config = Config().parse(model_opt_,print_=False) tmp_config.embedding_path=None _model = getattr(models,_name)(tmp_config) _model.encoder=(self.char_embedding if _model.opt.type_=='char' else self.word_embedding) self.models.append(_model) self.models = nn.ModuleList(self.models) self.model_num = len(self.models) self.weights = nn.Parameter(t.ones(opt.num_classes,self.model_num)) self.load(opt.model_path)
def __init__(self, vocab_size, hidden_size=512, embedding_size=None, num_layers=6, num_heads=8, inner_linear=1024, mask_symbol=PAD, dropout=0): super(TransformerAttentionEncoder, self).__init__() embedding_size = embedding_size or hidden_size self.hidden_size = hidden_size self.batch_first = True self.mask_symbol = mask_symbol self.embedder = nn.Embedding( vocab_size, embedding_size, padding_idx=PAD) self.scale_embedding = hidden_size ** 0.5 self.dropout = nn.Dropout(dropout, inplace=True) self.blocks = nn.ModuleList([EncoderBlock(hidden_size, num_heads, inner_linear, dropout) for _ in range(num_layers) ])
def __init__(self, vocab_size, hidden_size=512, embedding_size=None, num_layers=6, num_heads=8, dropout=0, inner_linear=1024, mask_symbol=PAD, tie_embedding=True): super(TransformerAttentionDecoder, self).__init__() embedding_size = embedding_size or hidden_size self.batch_first = True self.mask_symbol = mask_symbol self.embedder = nn.Embedding( vocab_size, embedding_size, padding_idx=PAD) self.scale_embedding = hidden_size ** 0.5 self.dropout = nn.Dropout(dropout, inplace=True) self.blocks = nn.ModuleList([DecoderBlock(hidden_size, num_heads, inner_linear, dropout) for _ in range(num_layers) ]) self.classifier = nn.Linear(hidden_size, vocab_size) if tie_embedding: self.embedder.weight = self.classifier.weight
def __init__(self, input_size, hidden_size, kernel_size=3, num_layers=4, bias=True, dropout=0, causal=True): super(StackedConv, self).__init__() self.convs = nn.ModuleList() size = input_size for l in range(num_layers): self.convs.append(GatedConv1d(size, hidden_size, 1, bias=bias, causal=False)) self.convs.append(nn.BatchNorm1d(hidden_size)) self.convs.append(MaskedConv1d(hidden_size, hidden_size, kernel_size, bias=bias, groups=hidden_size, causal=causal)) self.convs.append(nn.BatchNorm1d(hidden_size)) size = hidden_size
def _init_pool(self, dsz, **kwargs): filtsz = kwargs['filtsz'] cmotsz = kwargs['cmotsz'] convs = [] for i, fsz in enumerate(filtsz): pad = fsz//2 conv = nn.Sequential( nn.Conv1d(dsz, cmotsz, fsz, padding=pad), pytorch_activation("relu") ) convs.append(conv) # Add the module so its managed correctly self.convs = nn.ModuleList(convs) # Width of concat of parallel convs self.conv_drop = nn.Dropout(self.pdrop) return cmotsz * len(filtsz)
def __init__(self, conv_channels, input_nch=3, output_nch=2, use_bn=True): super(UNet, self).__init__() self.n_stages = len(conv_channels) # define convolution blocks down_convs = [] up_convs = [] self.max_pooling = nn.MaxPool2d(kernel_size=3, stride=2, padding=1) in_nch = input_nch for i, out_nch in enumerate(conv_channels): down_convs.append(UNetConvBlock(in_nch, out_nch, use_bn=use_bn)) up_conv_in_ch = 2 * out_nch if i < self.n_stages - 1 else out_nch # first up conv with equal channels up_conv_out_ch = out_nch if i == 0 else in_nch # last up conv with channels equal to labels up_convs.insert(0, UNetConvBlock(up_conv_in_ch, up_conv_out_ch, use_bn=use_bn)) in_nch = out_nch self.down_convs = nn.ModuleList(down_convs) self.up_convs = nn.ModuleList(up_convs) # define output convolution self.out_conv = nn.Conv2d(conv_channels[0], output_nch, 1)
def __init__(self, conv_channels, input_nch, output_nch, groups=1): super(TriangleNet, self).__init__() self.input_nch = input_nch self.output_nch = output_nch self.pyramid_height = len(conv_channels) blocks = [list() for _ in range(self.pyramid_height)] for i in range(self.pyramid_height): for j in range(i, self.pyramid_height): if i == 0 and j == 0: blocks[i].append(BasicResBlock(input_nch, conv_channels[j], groups=groups)) else: blocks[i].append(BasicResBlock(conv_channels[j-1], conv_channels[j], groups=groups)) for i in range(self.pyramid_height): blocks[i] = nn.ModuleList(blocks[i]) self.blocks = nn.ModuleList(blocks) self.down_sample = nn.MaxPool2d(3, 2, 1) self.up_samples = nn.ModuleList([nn.Upsample(scale_factor=2**i, mode='bilinear') for i in range(1, self.pyramid_height)]) self.channel_out_convs = nn.ModuleList([nn.Conv2d(conv_channels[-1], output_nch, 1) for _ in range(self.pyramid_height)]) self.out_conv = nn.Conv2d(self.pyramid_height * conv_channels[-1], output_nch, 1)
def __init__(self, conv_channels, input_nch, output_nch, groups): super(PSPTriangleNet, self).__init__() self.input_nch = input_nch self.output_nch = output_nch self.pyramid_height = len(conv_channels) blocks = [] for i in range(self.pyramid_height-1): if i == 0: blocks.append(BasicResBlock(input_nch, conv_channels[i], groups=groups)) else: blocks.append(BasicResBlock(conv_channels[i-1], conv_channels[i], groups=groups)) ms_blocks = [] for i in range(self.pyramid_height): ms_blocks.append(BasicResBlock(conv_channels[-2], conv_channels[-1]//self.pyramid_height)) self.blocks = nn.ModuleList(blocks) self.ms_blocks = nn.ModuleList(ms_blocks) self.down_samples = nn.ModuleList([nn.MaxPool2d(2**i+1, 2**i, 2**(i-1)) for i in range(1, self.pyramid_height)]) self.up_samples = nn.ModuleList([nn.Upsample(scale_factor=2**i, mode='bilinear') for i in range(1, self.pyramid_height)]) self.channel_out_convs = nn.ModuleList([nn.Conv2d(conv_channels[-1]//self.pyramid_height, output_nch, 1) for _ in range(self.pyramid_height)]) self.out_conv = nn.Conv2d(conv_channels[-1], output_nch, 1)
def __init__(self, n_in, n_out, batchnorm=False, preactivation=True, gate_style='add_split', kernel_size=7): super(SMASHLayer, self).__init__() self.n_out = n_out self.n_in = n_in self.batchnorm = batchnorm self.preactivation = preactivation self.gate_style = gate_style ''' may want to make n_in and n_out more dynamic here''' self.op = nn.ModuleList([SMASHseq(n_in=n_in if not i%2 else n_out, n_out=n_out, dilation=1, batchnorm=self.batchnorm, preactivation=self.preactivation, kernel_size=kernel_size) for i in range(4)]) # Op represents the op definition, gate whether to use tanh-sig mult gates, # dilation the individual dilation factors, and NL the particular # activation to use at each ungated conv. # Groups is currently unactivated, we'd need to make sure we slice differently # if using variable group.
def __init__(self, cell, num_layers, in_dim, hid_dim, dropout=0.0, **kwargs): """ cell: str or custom cell class """ super(BaseStackedRNN, self).__init__() self.in_dim = in_dim self.hid_dim = hid_dim self.has_dropout = False if dropout: self.has_dropout = True self.dropout = nn.Dropout(dropout) self.num_layers = num_layers self.layers = nn.ModuleList() if isinstance(cell, str): cell = getattr(nn, cell) for i in range(num_layers): self.layers.append(cell(in_dim, hid_dim, **kwargs)) in_dim = hid_dim
def __init__(self, embed_dim, hidden_dim=64): super(HierarchialNetwork1D, self).__init__() self.layers = nn.ModuleList() first_block = nn.Sequential( nn.Conv1d(in_channels=embed_dim, out_channels=hidden_dim, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.BatchNorm1d(hidden_dim) ) self.layers.append(first_block) for layer_index in range(4): conv_block = nn.Sequential( nn.Conv1d(in_channels=hidden_dim, out_channels=hidden_dim, kernel_size=3, padding=1), nn.ReLU(inplace=True), nn.BatchNorm1d(hidden_dim) ) self.layers.append(conv_block)
def __init__(self, feature_size=64): super(OmniglotEncoder, self).__init__() self.layers = nn.ModuleList() first_block = nn.Sequential( nn.Conv2d(in_channels=3, out_channels=feature_size, kernel_size=3, stride=1, padding=1), nn.BatchNorm2d(feature_size), nn.LeakyReLU(inplace=True), nn.MaxPool2d(kernel_size=2) ) self.layers.append(first_block) for layer_index in range(3): block = nn.Sequential( nn.Conv2d(in_channels=64, out_channels=feature_size, kernel_size=3, stride=1, padding=1), nn.BatchNorm2d(feature_size), nn.LeakyReLU(inplace=True), nn.MaxPool2d(kernel_size=2) ) self.layers.append(block) self.fc = nn.Linear(feature_size, feature_size)
def __set_update(self, update_def, args): self.u_definition = update_def.lower() self.u_function = { 'duvenaud': self.u_duvenaud, 'ggnn': self.u_ggnn, 'intnet': self.u_intnet, 'mpnn': self.u_mpnn }.get(self.u_definition, None) if self.u_function is None: print('WARNING!: Update Function has not been set correctly\n\tIncorrect definition ' + update_def) init_parameters = { 'duvenaud': self.init_duvenaud, 'ggnn': self.init_ggnn, 'intnet': self.init_intnet, 'mpnn': self.init_mpnn }.get(self.u_definition, lambda x: (nn.ParameterList([]), nn.ModuleList([]), {})) self.learn_args, self.learn_modules, self.args = init_parameters(args) # Get the name of the used update function
def init_duvenaud(self, params): learn_args = [] learn_modules = [] args = {} # Filter degree 0 (the message will be 0 and therefore there is no update args['deg'] = [i for i in params['deg'] if i!=0] args['in'] = params['in'] args['out'] = params['out'] # Define a parameter matrix H for each degree. learn_args.append(torch.nn.Parameter(torch.randn(len(args['deg']), args['in'], args['out']))) return nn.ParameterList(learn_args), nn.ModuleList(learn_modules), args # GG-NN, Li et al.
def __init__(self, in_n, out_message, out_update, l_target, type='regression'): super(MpnnIntNet, self).__init__() n_layers = len(out_update) # Define message 1 & 2 self.m = nn.ModuleList([MessageFunction('intnet', args={'in': 2*in_n[0] + in_n[1], 'out': out_message[i]}) if i == 0 else MessageFunction('intnet', args={'in': 2*out_update[i-1] + in_n[1], 'out': out_message[i]}) for i in range(n_layers)]) # Define Update 1 & 2 self.u = nn.ModuleList([UpdateFunction('intnet', args={'in': in_n[0]+out_message[i], 'out': out_update[i]}) if i == 0 else UpdateFunction('intnet', args={'in': out_update[i-1]+out_message[i], 'out': out_update[i]}) for i in range(n_layers)]) # Define Readout self.r = ReadoutFunction('intnet', args={'in': out_update[-1], 'target': l_target}) self.type = type
def __init__(self, in_n, hidden_state_size, message_size, n_layers, l_target, type='regression'): super(MPNN, self).__init__() # Define message self.m = nn.ModuleList( [MessageFunction('mpnn', args={'edge_feat': in_n[1], 'in': hidden_state_size, 'out': message_size})]) # Define Update self.u = nn.ModuleList([UpdateFunction('mpnn', args={'in_m': message_size, 'out': hidden_state_size})]) # Define Readout self.r = ReadoutFunction('mpnn', args={'in': hidden_state_size, 'target': l_target}) self.type = type self.args = {} self.args['out'] = hidden_state_size self.n_layers = n_layers
def __init__(self, d, in_n, out_update, hidden_state_readout, l_target, type='regression'): super(MpnnDuvenaud, self).__init__() n_layers = len(out_update) # Define message 1 & 2 self.m = nn.ModuleList([MessageFunction('duvenaud') for _ in range(n_layers)]) # Define Update 1 & 2 self.u = nn.ModuleList([UpdateFunction('duvenaud', args={'deg': d, 'in': self.m[i].get_out_size(in_n[0], in_n[1]), 'out': out_update[0]}) if i == 0 else UpdateFunction('duvenaud', args={'deg': d, 'in': self.m[i].get_out_size(out_update[i-1], in_n[1]), 'out': out_update[i]}) for i in range(n_layers)]) # Define Readout self.r = ReadoutFunction('duvenaud', args={'layers': len(self.m) + 1, 'in': [in_n[0] if i == 0 else out_update[i-1] for i in range(n_layers+1)], 'out': hidden_state_readout, 'target': l_target}) self.type = type
def __set_readout(self, readout_def, args): self.r_definition = readout_def.lower() self.r_function = { 'duvenaud': self.r_duvenaud, 'ggnn': self.r_ggnn, 'intnet': self.r_intnet, 'mpnn': self.r_mpnn }.get(self.r_definition, None) if self.r_function is None: print('WARNING!: Readout Function has not been set correctly\n\tIncorrect definition ' + readout_def) quit() init_parameters = { 'duvenaud': self.init_duvenaud, 'ggnn': self.init_ggnn, 'intnet': self.init_intnet, 'mpnn': self.init_mpnn }.get(self.r_definition, lambda x: (nn.ParameterList([]), nn.ModuleList([]), {})) self.learn_args, self.learn_modules, self.args = init_parameters(args) # Get the name of the used readout function
def init_ggnn(self, params): learn_args = [] learn_modules = [] args = {} # i learn_modules.append(NNet(n_in=2*params['in'], n_out=params['target'])) # j learn_modules.append(NNet(n_in=params['in'], n_out=params['target'])) args['out'] = params['target'] return nn.ParameterList(learn_args), nn.ModuleList(learn_modules), args # Battaglia et al. (2016), Interaction Networks
def __init__(self, params): super(AutoEncoder, self).__init__() self.img_sz = params.img_sz self.img_fm = params.img_fm self.instance_norm = params.instance_norm self.init_fm = params.init_fm self.max_fm = params.max_fm self.n_layers = params.n_layers self.n_skip = params.n_skip self.deconv_method = params.deconv_method self.dropout = params.dec_dropout self.attr = params.attr self.n_attr = params.n_attr enc_layers, dec_layers = build_layers(self.img_sz, self.img_fm, self.init_fm, self.max_fm, self.n_layers, self.n_attr, self.n_skip, self.deconv_method, self.instance_norm, 0, self.dropout) self.enc_layers = nn.ModuleList(enc_layers) self.dec_layers = nn.ModuleList(dec_layers)
def __init__(self, input_dims, code_dims, layers=[2, 2], beta=1.0, hidden=400, activacation="lrelu", decoder="Bernoulli"): super(BetaVAE, self).__init__(input_dims, code_dims, hidden=400, activacation="lrelu", decoder="Bernoulli") self.beta = beta self.encode_layers = nn.ModuleList([self.fc1]) for i in range(layers[0]-2): l = nn.Linear(hidden, hidden) self.encode_layers.append(l) self.decode_layers = nn.ModuleList([self.fc3]) for i in range(layers[0]-2): l = nn.Linear(hidden, hidden) self.decode_layers.append(l)
def __init__(self, num_layers, hidden_size, attn_type, copy_attn, dropout, embeddings): super(TransformerDecoder, self).__init__() # Basic attributes. self.decoder_type = 'transformer' self.num_layers = num_layers self.embeddings = embeddings # Build TransformerDecoder. self.transformer_layers = nn.ModuleList( [TransformerDecoderLayer(hidden_size, dropout) for _ in range(num_layers)]) # TransformerDecoder has its own attention mechanism. # Set up a separated copy attention layer, if needed. self._copy = False if copy_attn: self.copy_attn = onmt.modules.GlobalAttention( hidden_size, attn_type=attn_type) self._copy = True self.layer_norm = onmt.modules.BottleLayerNorm(hidden_size)
def __init__(self, size, num_layers, f): super(Highway, self).__init__() self.num_layers = num_layers self.nonlinear = nn.ModuleList([nn.Linear(size, size) for _ in range(num_layers)]) self.linear = nn.ModuleList([nn.Linear(size, size) for _ in range(num_layers)]) self.gate = nn.ModuleList([nn.Linear(size, size) for _ in range(num_layers)]) self.f = f
def __init__(self, enc_vocab_size, max_word_len, n_enc, d_model, d_ff, n_head, dropout): super().__init__() self.n_position = max_word_len + 1 self.enc_vocab_size = enc_vocab_size self.d_model = d_model self.enc_ebd = nn.Embedding(enc_vocab_size, d_model, padding_idx=PAD) self.pos_ebd = nn.Embedding(self.n_position, d_model, padding_idx=PAD) self.encodes = nn.ModuleList([ EncoderLayer(d_model, d_ff, n_head, dropout) for _ in range(n_enc)]) self._init_weight()
