class torch.optim.Adam(params, lr=0.001, betas=(0.9, 0.999), eps=1e-08, weight_decay=0)

 import math

 from .optimizer import Optimizer

 class Adam(Optimizer):

     def __init__(self, params, lr=1e-3, betas=(0.9, 0.999), eps=1e-8，weight_decay=0):

         defaults = dict(lr=lr, betas=betas, eps=eps,weight_decay=weight_decay)

         super(Adam, self).__init__(params, defaults)

     def step(self, closure=None):

         loss = None

         if closure is not None:

             loss = closure()

         for group in self.param_groups:

             for p in group['params']:

                 if p.grad is None:

                     continue

                 grad = p.grad.data

                 state = self.state[p]

                 # State initialization

                 if len(state) == 0:

                     state['step'] = 0

                     # Exponential moving average of gradient values

                     state['exp_avg'] = grad.new().resize_as_(grad).zero_()

                     # Exponential moving average of squared gradient values

                     state['exp_avg_sq'] = grad.new().resize_as_(grad).zero_()

                 exp_avg, exp_avg_sq = state['exp_avg'], state['exp_avg_sq']

                 beta1, beta2 = group['betas']

                 state['step'] += 1

                 if group['weight_decay'] != 0:

                     grad = grad.add(group['weight_decay'], p.data)

                 # Decay the first and second moment running average coefficient

                 exp_avg.mul_(beta1).add_(1 - beta1, grad)

                 exp_avg_sq.mul_(beta2).addcmul_(1 - beta2, grad, grad)

                 denom = exp_avg_sq.sqrt().add_(group['eps'])

                 bias_correction1 = 1 - beta1 ** state['step']

                 bias_correction2 = 1 - beta2 ** state['step']

                 step_size = group['lr'] * math.sqrt(bias_correction2) / bias_correction1

                 p.data.addcdiv_(-step_size, exp_avg, denom)

         return loss

 import torch

 # N is batch size; D_in is input dimension;

 # H is hidden dimension; D_out is output dimension.

 N, D_in, H, D_out = 64, 1000, 100, 10

 # Create random Tensors to hold inputs and outputs

 x = torch.randn(N, D_in)

 y = torch.randn(N, D_out)

 # Use the nn package to define our model and loss function.

 model = torch.nn.Sequential(

     torch.nn.Linear(D_in, H),

     torch.nn.ReLU(),

     torch.nn.Linear(H, D_out),

 )

 loss_fn = torch.nn.MSELoss(reduction='sum')

 # Use the optim package to define an Optimizer that will update the weights of

 # the model for us. Here we will use Adam; the optim package contains many other

 # optimization algoriths. The first argument to the Adam constructor tells the

 # optimizer which Tensors it should update.

 learning_rate = 1e-4

 optimizer = torch.optim.Adam(model.parameters(), lr=learning_rate)

 for t in range(500):

     # Forward pass: compute predicted y by passing x to the model.

     y_pred = model(x)

     # Compute and print loss.

     loss = loss_fn(y_pred, y)

     print(t, loss.item())

     # Before the backward pass, use the optimizer object to zero all of the

     # gradients for the variables it will update (which are the learnable

     # weights of the model). This is because by default, gradients are

     # accumulated in buffers( i.e, not overwritten) whenever .backward()

     # is called. Checkout docs of torch.autograd.backward for more details.

     optimizer.zero_grad()

     # Backward pass: compute gradient of the loss with respect to model

     # parameters

     loss.backward()

     # Calling the step function on an Optimizer makes an update to its

     # parameters

     optimizer.step()