我们从Python开源项目中，提取了以下14个代码示例，用于说明如何使用torch.optim.Adamax()。
def test_adamax(self): self._test_rosenbrock( lambda params: optim.Adamax(params, lr=1e-1), wrap_old_fn(old_optim.adamax, learningRate=1e-1) ) self._test_rosenbrock( lambda params: optim.Adamax(params, lr=1e-1, weight_decay=1e-2), wrap_old_fn(old_optim.adamax, learningRate=1e-1, weightDecay=1e-2) ) self._test_rosenbrock( lambda params: optim.Adamax(params, lr=1e-1, betas=(0.95, 0.998)), wrap_old_fn(old_optim.adamax, learningRate=1e-1, beta1=0.95, beta2=0.998) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad([weight, bias], lr=1e-1) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad( self._build_params_dict(weight, bias, lr=1e-2), lr=1e-1) )
def init_optimizer(self, state_dict=None): """Initialize an optimizer for the free parameters of the network. Args: state_dict: network parameters """ if self.args.fix_embeddings: for p in self.network.embedding.parameters(): p.requires_grad = False parameters = [p for p in self.network.parameters() if p.requires_grad] if self.args.optimizer == 'sgd': self.optimizer = optim.SGD(parameters, self.args.learning_rate, momentum=self.args.momentum, weight_decay=self.args.weight_decay) elif self.args.optimizer == 'adamax': self.optimizer = optim.Adamax(parameters, weight_decay=self.args.weight_decay) else: raise RuntimeError('Unsupported optimizer: %s' % self.args.optimizer) # -------------------------------------------------------------------------- # Learning # --------------------------------------------------------------------------
def __init__(self, opt, word_dict, feature_dict, state_dict=None): # Book-keeping. self.opt = opt self.word_dict = word_dict self.feature_dict = feature_dict self.updates = 0 self.train_loss = AverageMeter() # Building network. self.network = RnnDocReader(opt) if state_dict: new_state = set(self.network.state_dict().keys()) for k in list(state_dict['network'].keys()): if not k in new_state: del state_dict['network'][k] self.network.load_state_dict(state_dict['network']) # Building optimizer. parameters = [p for p in self.network.parameters() if p.requires_grad] if opt['optimizer'] == 'sgd': self.optimizer = optim.SGD(parameters, opt['learning_rate'], momentum=opt['momentum'], weight_decay=opt['weight_decay']) elif opt['optimizer'] == 'adamax': self.optimizer = optim.Adamax(parameters, weight_decay=opt['weight_decay']) else: raise RuntimeError('Unsupported optimizer: %s' % opt['optimizer'])
def __init__(self, opt, embedding=None, state_dict=None): # Book-keeping. self.opt = opt self.updates = state_dict['updates'] if state_dict else 0 self.train_loss = AverageMeter() # Building network. self.network = RnnDocReader(opt, embedding=embedding) if state_dict: new_state = set(self.network.state_dict().keys()) for k in list(state_dict['network'].keys()): if k not in new_state: del state_dict['network'][k] self.network.load_state_dict(state_dict['network']) # Building optimizer. parameters = [p for p in self.network.parameters() if p.requires_grad] if opt['optimizer'] == 'sgd': self.optimizer = optim.SGD(parameters, opt['learning_rate'], momentum=opt['momentum'], weight_decay=opt['weight_decay']) elif opt['optimizer'] == 'adamax': self.optimizer = optim.Adamax(parameters, opt['learning_rate'], weight_decay=opt['weight_decay']) else: raise RuntimeError('Unsupported optimizer: %s' % opt['optimizer']) if state_dict: self.optimizer.load_state_dict(state_dict['optimizer']) num_params = sum(p.data.numel() for p in parameters if p.data.data_ptr() != self.network.embedding.weight.data.data_ptr()) print ("{} parameters".format(num_params))
def adamax(w, lr=0.002, betas=(0.9, 0.999), eps=1e-08, w_decay=0): return nn.Adamax(params=w, lr=lr, betas=betas, eps=eps, weight_decay=w_decay)
def __init__(self, opt, embedding=None, state_dict=None): # Book-keeping. self.opt = opt self.updates = state_dict['updates'] if state_dict else 0 self.train_loss = AverageMeter() # Building network. self.network = RnnDocReader(opt, embedding=embedding) if state_dict: new_state = set(self.network.state_dict().keys()) for k in list(state_dict['network'].keys()): if k not in new_state: del state_dict['network'][k] self.network.load_state_dict(state_dict['network']) # Building optimizer. parameters = [p for p in self.network.parameters() if p.requires_grad] if opt['optimizer'] == 'sgd': self.optimizer = optim.SGD(parameters, opt['learning_rate'], momentum=opt['momentum'], weight_decay=opt['weight_decay']) elif opt['optimizer'] == 'adamax': self.optimizer = optim.Adamax(parameters, weight_decay=opt['weight_decay']) else: raise RuntimeError('Unsupported optimizer: %s' % opt['optimizer']) if state_dict: self.optimizer.load_state_dict(state_dict['optimizer'])
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params
def get_optimizer(model, s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam optim_params['betas'] = (optim_params.get('beta1', 0.5), optim_params.get('beta2', 0.999)) optim_params.pop('beta1', None) optim_params.pop('beta2', None) elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn(model.parameters(), **optim_params)
