저는 지금 오목 인공지능을 개발하고 있습니다. 지도학습과 강화학습을 합쳐서 해보려고 있는데, 문제는 지도학습에서 발생했습니다.

처음에는 Adam으로 학습했는데 0.35까지는 빠르게 상승하다 그 이후부터는 거의 일정한 값을 나타냈습니다.
두 번째로는 SGD를 lr=1e-5으로 학습했는데 학습률이 너무나도 저조했습니다.



적어도 128만 개의 데이터를 사용했는데도 불구하고 말입니다.

소스코드는 다음과 같습니다.

모델은 policy net입니다.


model = Sequential()
model.add(Conv2D(64, (7, 7), input_shape=input_shape, padding='same', activation='relu', data_format='channels_first'))
model.add(BatchNormalizationV2())
model.add(Conv2D(64, (7, 7), input_shape=input_shape, padding='same', activation='relu', data_format='channels_first'))
model.add(BatchNormalizationV2())
model.add(Dropout(0.2))
model.add(Conv2D(128, (5, 5), input_shape=input_shape, padding='same', activation='relu', data_format='channels_first'))
model.add(BatchNormalizationV2())
model.add(Conv2D(128, (5, 5), input_shape=input_shape, padding='same', activation='relu', data_format='channels_first'))
model.add(BatchNormalizationV2())
model.add(Dropout(0.2))
model.add(Conv2D(64, (3, 3), input_shape=input_shape, padding='same', activation='relu', data_format='channels_first'))
model.add(BatchNormalizationV2())
model.add(Conv2D(64, (3, 3), input_shape=input_shape, padding='same', activation='relu', data_format='channels_first'))
model.add(BatchNormalizationV2())
model.add(Dropout(0.2))

if is_policy_net:
model.add(Conv2D(filters=1, kernel_size=1, padding='same', data_format='channels_first', activation='softmax'))
model.add(Flatten())
return model
else:
model.add(Conv2D(64, (3, 3), input_shape=input_shape, padding='same', activation='relu', data_format='channels_first'))
model.add(Conv2D(filters=1, kernel_size=1, padding='same', data_format='channels_first', activation='relu'))
model.add(Flatten())
model.add(Dense(256, activation='relu'))
model.add(Dense(1, activation='tanh'))
return model



def train(self, experience, lr=0.00001, clipnorm=1.0, batch_size:int=256, epochs:int = 1):
opt = SGD(learning_rate=lr, clipnorm=clipnorm)
#opt = Adam()
self.model.compile(loss='categorical_crossentropy', optimizer=opt, metrics=['accuracy'])

n = experience.states.shape[0]

#current_time = datetime.datetime.now().strftime("%Y%m%d-%H%M%S")
#tensorboard_callback = TensorBoard(log_dir=f"./logs_{current_time}")
checkpoint_callback = ModelCheckpoint(
filepath=f"{self.encoder.name()}_alphago_sl_checkpoint_{{epoch:02d}}",
save_best_only=False,
period=10 # Save model every 10 epochs
)
generator = self.data_generator(experience.states, experience.actions, experience.rewards, batch_size)
self.model.fit(
generator,
steps_per_epoch=n // batch_size,
epochs=epochs,
#callbacks=[tensorboard_callback, checkpoint_callback]
)
def data_generator(self, states, actions, rewards, batch_size):
n = states.shape[0]
num_moves = len(actions)
indices = np.arange(n)
while True:
np.random.shuffle(indices)
for start_idx in range(0, n, batch_size):
end_idx = min(start_idx + batch_size, n)
batch_indices = indices[start_idx:end_idx]
batch_states = states[batch_indices]
batch_actions = actions[batch_indices]
batch_rewards = rewards[batch_indices]
y = np.zeros((batch_indices.shape[0], num_moves))
for i, action in enumerate(batch_actions):
reward = batch_rewards[i]
y[i][action] = reward
yield batch_states, y