a65614aa1f06b3679234254958c12a3ae31061529cdd2800ec252c


import matplotlib.pyplot as plt
y_true = 2
x = 1
w1 = 0.5
w2 = 0.5

def ReLU(x):
ย  ย  return max(0, x)


def predict(x, w1, w2):
ย  ย  return w2 * ReLU(w1 * x)


def grad_w2(x, y_true, w1, w2):
ย  ย  y = predict(x, w1, w2)
ย  ย  return 2 * (y - y_true) * ReLU(w1 * x)

def grad_w1(x, y_true, w1, w2):
ย  ย  y = predict(x, w1, w2)
ย  ย  if w1 * x > 0:
ย  ย  ย  ย  return 2 * (y - y_true) * w2 * x
ย  ย  else:
ย  ย  ย  ย  return 0


def adam(m, v, t, w, grad):
ย  ย  beta1 = 0.9
ย  ย  beta2 = 0.999
ย  ย  epsilon = 1e-8
ย  ย  eta = 0.001
ย  ย 
ย  ย  m = beta1 * m + (1 - beta1) * grad
ย  ย  v = beta2 * v + (1 - beta2) * (grad ** 2)
ย  ย  m_hat = m / (1 - beta1 ** t)
ย  ย  v_hat = v / (1 - beta2 ** t)
ย  ย  w = w - eta * m_hat / (v_hat ** 0.5 + epsilon)
ย  ย 
ย  ย  return (w, m, v)

def train(x, y_true, w1, w2, epochs):
ย  ย  m1, v1 = 0, 0
ย  ย  m2, v2 = 0, 0
ย  ย  x_data = []
ย  ย  y_data = []
ย  ย  for i in range(epochs):
ย  ย  ย  ย  t = i + 1
ย  ย  ย  ย  g1 = grad_w1(x, y_true, w1, w2)
ย  ย  ย  ย  g2 = grad_w2(x, y_true, w1, w2)
ย  ย  ย  ย  w1, m1, v1 = adam(m1, v1, t, w1, g1)
ย  ย  ย  ย  w2, m2, v2 = adam(m2, v2, t, w2, g2)
ย  ย  ย  ย  x_data.append(t)
ย  ย  ย  ย  loss = (predict(x, w1, w2) - y_true) ** 2
ย  ย  ย  ย  y_data.append(loss)
ย  ย  plt.plot(x_data, y_data, linestyle='-', color='b', linewidth=1)
ย  ย  return w1, w2

w1, w2 = train(x, y_true, w1, w2, 10000)
plt.title('Adam Optimizer')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.show()
ย  ย 


ํ•™๋ถ€ 2ํ•™๋…„์ธ๋ฐ ์ง„์งœ ์—ด์‹ฌํžˆ ๋งŒ๋“ค์—ˆ์Œ ...์•„๋‹ด ์ดํ•ดํ•˜๋Š”๋ฐ 2์‹œ๊ฐ„ ๋„˜๊ฒŒ๊ฑธ๋ ธ์Œ,,