我们从Python开源项目中，提取了以下33个代码示例，用于说明如何使用torch.optim.Adagrad()。
def test_adagrad(self): self._test_rosenbrock( lambda params: optim.Adagrad(params, lr=1e-1), wrap_old_fn(old_optim.adagrad, learningRate=1e-1) ) self._test_rosenbrock( lambda params: optim.Adagrad(params, lr=1e-1, lr_decay=1e-3), wrap_old_fn(old_optim.adagrad, learningRate=1e-1, learningRateDecay=1e-3) ) self._test_rosenbrock( lambda params: optim.Adagrad(params, lr=1e-1, weight_decay=1e-2), wrap_old_fn(old_optim.adagrad, learningRate=1e-1, weightDecay=1e-2) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad([weight, bias], lr=1e-1) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad( self._build_params_dict(weight, bias, lr=1e-2), lr=1e-1) )
def test_adamax(self): self._test_rosenbrock( lambda params: optim.Adamax(params, lr=1e-1), wrap_old_fn(old_optim.adamax, learningRate=1e-1) ) self._test_rosenbrock( lambda params: optim.Adamax(params, lr=1e-1, weight_decay=1e-2), wrap_old_fn(old_optim.adamax, learningRate=1e-1, weightDecay=1e-2) ) self._test_rosenbrock( lambda params: optim.Adamax(params, lr=1e-1, betas=(0.95, 0.998)), wrap_old_fn(old_optim.adamax, learningRate=1e-1, beta1=0.95, beta2=0.998) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad([weight, bias], lr=1e-1) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad( self._build_params_dict(weight, bias, lr=1e-2), lr=1e-1) )
def test_rmsprop(self): self._test_rosenbrock( lambda params: optim.RMSprop(params, lr=1e-2), wrap_old_fn(old_optim.rmsprop, learningRate=1e-2) ) self._test_rosenbrock( lambda params: optim.RMSprop(params, lr=1e-2, weight_decay=1e-2), wrap_old_fn(old_optim.rmsprop, learningRate=1e-2, weightDecay=1e-2) ) self._test_rosenbrock( lambda params: optim.RMSprop(params, lr=1e-2, alpha=0.95), wrap_old_fn(old_optim.rmsprop, learningRate=1e-2, alpha=0.95) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad([weight, bias], lr=1e-2) ) self._test_basic_cases( lambda weight, bias: optim.Adagrad( self._build_params_dict(weight, bias, lr=1e-3), lr=1e-2) )
def get_opt(name): opts = { 'SGD': optim.SGD, 'Adam': optim.Adam, 'Adagrad': optim.Adagrad, 'RMSprop': optim.RMSprop, } return opts[name]
def _makeOptimizer(self): if self.method == 'sgd': self.optimizer = optim.SGD(self.params, lr=self.lr) elif self.method == 'adagrad': self.optimizer = optim.Adagrad(self.params, lr=self.lr) elif self.method == 'adadelta': self.optimizer = optim.Adadelta(self.params, lr=self.lr) elif self.method == 'adam': self.optimizer = optim.Adam(self.params, lr=self.lr) else: raise RuntimeError("Invalid optim method: " + self.method)
def choose_optimizer(args, model): if args.optim =='adam': return optim.Adam(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.wd) elif args.optim=='adagrad': # optimizer = optim.Adagrad(filter(lambda p: p.requires_grad, model.parameters()), lr=args.lr, weight_decay=args.wd) return optim.Adagrad([ {'params': model.parameters(), 'lr': args.lr} ], lr=args.lr, weight_decay=args.wd)
def set_parameters(self, params): self.params = list(params) # careful: params may be a generator if self.method == 'sgd': self.optimizer = optim.SGD(self.params, lr=self.lr) elif self.method == 'adagrad': self.optimizer = optim.Adagrad(self.params, lr=self.lr) elif self.method == 'adadelta': self.optimizer = optim.Adadelta(self.params, lr=self.lr) elif self.method == 'adam': self.optimizer = optim.Adam(self.params, lr=self.lr) else: raise RuntimeError("Invalid optim method: " + self.method)
def test_adagrad_sparse(self): self._test_rosenbrock_sparse( lambda params: optim.Adagrad(params, lr=1e-1) )
def adagrad(w, lr=0.01, lr_decay=0, w_decay=0): return nn.Adagrad(params=w, lr=lr, lr_decay=lr_decay, weight_decay=w_decay)
def _makeOptimizer(self): if self.method == 'sgd': self.optimizer = optim.SGD(self.params, lr=self.lr) elif self.method == 'adagrad': self.optimizer = optim.Adagrad(self.params, lr=self.lr) elif self.method == 'adadelta': self.optimizer = optim.Adadelta(self.params, lr=self.lr) elif self.method == 'adam': self.optimizer = optim.Adam(self.params, lr=self.lr, betas=(0.5, 0.999)) else: raise RuntimeError("Invalid optim method: " + self.method)
def create_optimizer(model, new_lr): # setup optimizer if args.optimizer == 'sgd': optimizer = optim.SGD(model.parameters(), lr=new_lr, momentum=0.9, dampening=0.9, weight_decay=args.wd) elif args.optimizer == 'adam': optimizer = optim.Adam(model.parameters(), lr=new_lr, weight_decay=args.wd, betas=(args.beta1, 0.999)) elif args.optimizer == 'adagrad': optimizer = optim.Adagrad(model.parameters(), lr=new_lr, lr_decay=args.lr_decay, weight_decay=args.wd) return optimizer
def create_optimizer(model, new_lr): # setup optimizer if args.optimizer == 'sgd': optimizer = optim.SGD(model.parameters(), lr=new_lr, momentum=0.9, dampening=0.9, weight_decay=args.wd) elif args.optimizer == 'adam': optimizer = optim.Adam(model.parameters(), lr=new_lr, weight_decay=args.wd) elif args.optimizer == 'adagrad': optimizer = optim.Adagrad(model.parameters(), lr=new_lr, lr_decay=args.lr_decay, weight_decay=args.wd) return optimizer
def _makeOptimizer(self): if self.method == 'sgd': self.optimizer = optim.SGD(self.params, lr=self.lr) elif self.method == 'adagrad': self.optimizer = optim.Adagrad(self.params, lr=self.lr) elif self.method == 'adadelta': self.optimizer = optim.Adadelta(self.params, lr=self.lr) elif self.method == 'adam': self.optimizer = optim.Adam(self.params, lr=self.lr) elif self.method == 'rmsprop': self.optimizer = optim.RMSprop(self.params, lr=self.lr) else: raise RuntimeError("Invalid optim method: " + self.method)
def set_parameters(self, params): self.params = [p for p in params if p.requires_grad] if self.method == 'sgd': self.optimizer = optim.SGD(self.params, lr=self.lr) elif self.method == 'adagrad': self.optimizer = optim.Adagrad(self.params, lr=self.lr) for group in self.optimizer.param_groups: for p in group['params']: self.optimizer.state[p]['sum'] = self.optimizer\ .state[p]['sum'].fill_(self.adagrad_accum) elif self.method == 'adadelta': self.optimizer = optim.Adadelta(self.params, lr=self.lr) elif self.method == 'adam': self.optimizer = optim.Adam(self.params, lr=self.lr, betas=self.betas, eps=1e-9) else: raise RuntimeError("Invalid optim method: " + self.method) # We use the default parameters for Adam that are suggested by # the original paper https://arxiv.org/pdf/1412.6980.pdf # These values are also used by other established implementations, # e.g. https://www.tensorflow.org/api_docs/python/tf/train/AdamOptimizer # https://keras.io/optimizers/ # Recently there are slightly different values used in the paper # "Attention is all you need" # https://arxiv.org/pdf/1706.03762.pdf, particularly the value beta2=0.98 # was used there however, beta2=0.999 is still arguably the more # established value, so we use that here as well
def get_optimizer(s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn, optim_params
def get_optimizer(model, s): """ Parse optimizer parameters. Input should be of the form: - "sgd,lr=0.01" - "adagrad,lr=0.1,lr_decay=0.05" """ if "," in s: method = s[:s.find(',')] optim_params = {} for x in s[s.find(',') + 1:].split(','): split = x.split('=') assert len(split) == 2 assert re.match("^[+-]?(\d+(\.\d*)?|\.\d+)$", split[1]) is not None optim_params[split[0]] = float(split[1]) else: method = s optim_params = {} if method == 'adadelta': optim_fn = optim.Adadelta elif method == 'adagrad': optim_fn = optim.Adagrad elif method == 'adam': optim_fn = optim.Adam optim_params['betas'] = (optim_params.get('beta1', 0.5), optim_params.get('beta2', 0.999)) optim_params.pop('beta1', None) optim_params.pop('beta2', None) elif method == 'adamax': optim_fn = optim.Adamax elif method == 'asgd': optim_fn = optim.ASGD elif method == 'rmsprop': optim_fn = optim.RMSprop elif method == 'rprop': optim_fn = optim.Rprop elif method == 'sgd': optim_fn = optim.SGD assert 'lr' in optim_params else: raise Exception('Unknown optimization method: "%s"' % method) # check that we give good parameters to the optimizer expected_args = inspect.getargspec(optim_fn.__init__)[0] assert expected_args[:2] == ['self', 'params'] if not all(k in expected_args[2:] for k in optim_params.keys()): raise Exception('Unexpected parameters: expected "%s", got "%s"' % ( str(expected_args[2:]), str(optim_params.keys()))) return optim_fn(model.parameters(), **optim_params)
