Neat + Pygame Snake. Обучение ИИ стоит на месте... Не могу найти проблему
Всем добрый день. Господа, возникла проблема. Около 3ех месяцев, с перерывами, не могу решить задачку и найти в ней ошибку...Уже отчаялся и решил обратиться к Вам, сколько раз Вы мне помогали, стоило лишь задать правильный вопрос поиску, но не в этот раз. Сейчас решил обратится к Вам лично.
Суть проблемы: Есть скрипт змейки, довольно не замысловатый, к нему прикручена библиотека Neat. Скрипт, по одному, в цикле for i, прогоняет 20 змеек и я надеюсь на эволюцию. Мотивирую их тем что приближаясь к еде +1 к награде, выходя за рамки (смерть) -10 награды, если съел еду +5. Но с каждой новой генерацией обучение стоит на месте...
Все так же змейка не реагирует на:
- Выход из стены - смерть и ее надо бояться
- Взятие еды не мотивирует след. поколения. стремится к ней
- Приближение к еде, не мотивирует след. поколения стремится двигаться к ней.
Читал, что возможно я не правильно мотивирую змею. НО.. я менял значения награды, убирал мотивацию приближения к еде и я так же убирал и всякую мотивацию вовсе, кроме съедания еды - все без толку. Склоняюсь к тому, что в списке генерации не происходит модель сложения опыта и последующей эволюции, но не могу найти где же я допустил этот промах.
Прошу Вас помочь найти ошибку, которая не дает змейкам обучаться.
import random, pygame, sys, neat
from pygame.locals import *
FPS = 25
WINDOWWIDTH = 640
WINDOWHEIGHT = 480
CELLSIZE = 20
assert WINDOWWIDTH % CELLSIZE == 0, "Window width must be a multiple of cell size."
assert WINDOWHEIGHT % CELLSIZE == 0, "Window height must be a multiple of cell size."
CELLWIDTH = int(WINDOWWIDTH / CELLSIZE)
CELLHEIGHT = int(WINDOWHEIGHT / CELLSIZE)
# R G B
WHITE = (255, 255, 255)
BLACK = ( 0, 0, 0)
RED = (255, 0, 0)
GREEN = ( 0, 255, 0)
DARKGREEN = ( 0, 155, 0)
DARKGRAY = ( 40, 40, 40)
BGCOLOR = BLACK
txt_on = 1
UP = 'up'
DOWN = 'down'
LEFT = 'left'
RIGHT = 'right'
HEAD = 0 # syntactic sugar: index of the worm's head
info_list = [0,0,0,0,0,0]
list_apple = []
score_outzero = ['0']
location = {}
was_reward_x = 0
was_reward_y = 0
generation = 0
def main_proc(genomes, config):
global FPSCLOCK, DISPLAYSURF, BASICFONT, generation
pygame.init()
FPSCLOCK = pygame.time.Clock()
DISPLAYSURF = pygame.display.set_mode((WINDOWWIDTH, WINDOWHEIGHT))
BASICFONT = pygame.font.Font('freesansbold.ttf', 18)
pygame.display.set_caption('Wormy')
nets = []
ge = [] # Змейки
for i, g in genomes:
net = neat.nn.FeedForwardNetwork.create(g, config)
nets.append(net)
g.fitness = 0
#print('NETS:', len(nets))
ge.append(g)
print(g)
generation += 1
for ka in range(len(ge)): # пока в списке змеек есть живые
k = 19 - ka # переменная на убывание для того чтобы вырезая из списка змейку не нарваться на index error в 190 строке
wormCoords = [{'x': 15, 'y': 15},
{'x': 15 - 1, 'y': 15},
{'x': 15 - 2, 'y': 15}]
direction = RIGHT
# Start the apple in a random place.
apple = getRandomLocation(k, len(wormCoords) - 3)
savex = 0
savey = 0
rewx = []
rewy = []
print('NETS:', len(nets))
for l in range(601): # чтобы бесконечно не крутились у них 601 попытка
def minus_stata(xa):
pass
genomes[k][1].fitness -= xa
genomes.pop(k)
nets.pop(k)
for event in pygame.event.get(): # event handling loop
if event.type == QUIT:
terminate()
elif event.type == KEYDOWN:
if (event.key == K_LEFT or event.key == K_a) and direction != RIGHT:
direction = LEFT
elif (event.key == K_RIGHT or event.key == K_d) and direction != LEFT:
direction = RIGHT
elif (event.key == K_UP or event.key == K_w) and direction != DOWN:
direction = UP
elif (event.key == K_DOWN or event.key == K_s) and direction != UP:
direction = DOWN
elif event.key == K_ESCAPE:
terminate()
#print(info_list, k, len(nets))
# check if the worm has hit itself or the edge
#print(output)
count_de = ['0'] # переменная смерти
def death_check():
if wormCoords[HEAD]['x'] == -1 or wormCoords[HEAD]['x'] == CELLWIDTH or wormCoords[HEAD]['y'] == -1 or \
wormCoords[HEAD]['y'] == CELLHEIGHT:
count_de.clear()
count_de.append('1')
for wormBody in wormCoords[1:]:
if wormBody['x'] == wormCoords[HEAD]['x'] and wormBody['y'] == wormCoords[HEAD]['y']:
count_de.clear()
count_de.append('1')
#print(wormCoords[HEAD]['x'], wormCoords[HEAD]['y'], CELLWIDTH, CELLHEIGHT)
# check if worm has eaten an apply
if wormCoords[HEAD]['x'] == apple['x'] and wormCoords[HEAD]['y'] == apple['y']:
# don't remove worm's tail segment
print('Я ВЗЯЛ ЯБЛОКО ИЧКИ', len(wormCoords) - 2)
apple = getRandomLocation(k, len(wormCoords) - 2) # set a new apple somewhere
genomes[k][1].fitness += 600
rewx.clear()
rewy.clear()
else:
del wormCoords[-1] # remove worm's tail segment
w_app_x1 = int(apple['x']) - int(wormCoords[HEAD]['x'])
w_app_y1 = int(apple['y']) - int(wormCoords[HEAD]['y'])
if w_app_x1 == 0:
w_app_x1 = w_app_x1 * -1
#print(genomes[k])
genomes[k][1].fitness += 10
if w_app_y1 == 0:
w_app_y1 = w_app_y1 * -1
genomes[k][1].fitness += 10
#time.sleep(1)
def check_pos_to_reward(): # Функция выдачи награды
global was_reward_x, was_reward_y
if savex < w_app_x1:
pass
#print('отдаляюсь по Х', savex)
genomes[k][1].fitness -= 10
elif savex == w_app_x1:
#print('Без изменений Х', savex)
pass
else:
if was_reward_x == 0 and savex not in rewx:
#print('Приближаюсь Х', savex)
genomes[k][1].fitness += 40
was_reward_x += 1
was_reward_y *= 0
rewx.append(savex)
if savey < w_app_y1:
pass
#print('отдаляюсь по y', savey)
genomes[k][1].fitness -= 10
elif savey == w_app_y1:
pass
#print('Без изменений Y', savey)
else:
if was_reward_y == 0 and savey not in rewy:
#print('Приближаюсь Y', savey)
genomes[k][1].fitness += 40
rewy.append(savey)
was_reward_y += 1
was_reward_x *= 0
# print(w_app_x1, w_app_y1 )
w_app_x = int(apple['x'])
w_app_y = int(apple['y'])
#print(w_app_x1, w_app_y1, w_app_x, w_app_y)
def get_info(): # Тут добавляются значения с карты (где змейка, еда, сколько поинтоа до еды и тд и тп)
info_list.clear()
info_list1 = [wormCoords[HEAD]['x'], wormCoords[HEAD]['y'], w_app_x1, w_app_y1, w_app_x, w_app_y]
for i in range(len(info_list1)):
info_list.append(info_list1[i])
get_info()
#print(info_list, nets)
mjh = 19 - k
#print(len(nets), k)
output = nets[k].activate(info_list)
#genomes[k].fitness += 1
death_check()
if l == 600:
minus_stata(20)
break
if int(count_de[0]) == 1:
minus_stata(30)
break
if max(output) == output[0] and direction != DOWN and direction != UP:
if txt_on == 1:
pass
#genomes[k].fitness -= 10
#print('Выбираю нопку ВВЕРХ', output, action_button)
direction = UP
if l == 0:
pass
elif max(output) == output[1] and direction != UP and direction != DOWN:
if txt_on == 1:
#genomes[k].fitness -= 10
pass
#print('Выбираю нопку ВНИЗ', output, action_button)
direction = DOWN
if l == 0:
pass
check_pos_to_reward()
#get_info()
elif max(output) == output[2] and direction != LEFT and direction != RIGHT:
if txt_on == 1:
pass
#genomes[k].fitness -= 10
#print('Выбираю нопку ВПРАВО', output, action_button)
direction = RIGHT
if l == 0:
pass
check_pos_to_reward()
#get_info()
# move the worm by adding a segment in the direction it is moving
if direction == UP:
newHead = {'x': wormCoords[HEAD]['x'], 'y': wormCoords[HEAD]['y'] - 1}
elif direction == DOWN:
newHead = {'x': wormCoords[HEAD]['x'], 'y': wormCoords[HEAD]['y'] + 1}
elif direction == LEFT:
newHead = {'x': wormCoords[HEAD]['x'] - 1, 'y': wormCoords[HEAD]['y']}
elif direction == RIGHT:
newHead = {'x': wormCoords[HEAD]['x'] + 1, 'y': wormCoords[HEAD]['y']}
wormCoords.insert(0, newHead)
check_pos_to_reward()
savex = w_app_x1
savey = w_app_y1
DISPLAYSURF.fill(BGCOLOR)
drawGrid()
drawWorm(wormCoords)
drawApple(apple)
drawScore(len(wormCoords) - 3, generation, k)
pygame.display.update()
FPSCLOCK.tick(FPS)
print(len(genomes))
#minus_stata()
def drawPressKeyMsg():
pressKeySurf = BASICFONT.render('Press a key to play.', True, DARKGRAY)
pressKeyRect = pressKeySurf.get_rect()
pressKeyRect.topleft = (WINDOWWIDTH - 200, WINDOWHEIGHT - 30)
DISPLAYSURF.blit(pressKeySurf, pressKeyRect)
def checkForKeyPress():
if len(pygame.event.get(QUIT)) > 0:
terminate()
keyUpEvents = pygame.event.get(KEYUP)
if len(keyUpEvents) == 0:
return None
if keyUpEvents[0].key == K_ESCAPE:
terminate()
return keyUpEvents[0].key
def showStartScreen():
titleFont = pygame.font.Font('freesansbold.ttf', 100)
titleSurf1 = titleFont.render('Wormy!', True, WHITE, DARKGREEN)
titleSurf2 = titleFont.render('Wormy!', True, GREEN)
degrees1 = 0
degrees2 = 0
while True:
DISPLAYSURF.fill(BGCOLOR)
rotatedSurf1 = pygame.transform.rotate(titleSurf1, degrees1)
rotatedRect1 = rotatedSurf1.get_rect()
rotatedRect1.center = (WINDOWWIDTH / 2, WINDOWHEIGHT / 2)
DISPLAYSURF.blit(rotatedSurf1, rotatedRect1)
rotatedSurf2 = pygame.transform.rotate(titleSurf2, degrees2)
rotatedRect2 = rotatedSurf2.get_rect()
rotatedRect2.center = (WINDOWWIDTH / 2, WINDOWHEIGHT / 2)
DISPLAYSURF.blit(rotatedSurf2, rotatedRect2)
drawPressKeyMsg()
if checkForKeyPress():
pygame.event.get() # clear event queue
return
pygame.display.update()
FPSCLOCK.tick(FPS)
degrees1 += 3 # rotate by 3 degrees each frame
degrees2 += 7 # rotate by 7 degrees each frame
def terminate():
pygame.quit()
sys.exit()
score_count = 0
xrandom = []
yrandom = []
start_game = 0
def getRandomLocation(i, b,): # Случайное появление еды, раз в генерацию
global score_count, start_game
#m = {'x': random.randint(0, CELLWIDTH - 1), 'y': random.randint(0, CELLHEIGHT - 1)}
#return {'x': random.randint(0, CELLWIDTH - 1), 'y': random.randint(0, CELLHEIGHT - 1)}
#print('MAX', score_count, 'СЧЕТ: ',b)
#if True:
#if start_game == 0 or score_count < b:
print(i)
if i == 19:
print('СМЕНА КООРДИНАТ ЕДЫ')
xrandom.insert(b, random.randint(0, CELLWIDTH - 1))
yrandom.insert(b, random.randint(0, CELLHEIGHT - 1))
if score_count < b:
score_count += 1
xrandom.insert(b, random.randint(0, CELLWIDTH - 1))
yrandom.insert(b, random.randint(0, CELLHEIGHT - 1))
m = {'x': xrandom[b], 'y': yrandom[b]}
location.clear()
location.update(m)
#print(i,location)
return location
def showGameOverScreen():
gameOverFont = pygame.font.Font('freesansbold.ttf', 150)
gameSurf = gameOverFont.render('Game', True, WHITE)
overSurf = gameOverFont.render('Over', True, WHITE)
gameRect = gameSurf.get_rect()
overRect = overSurf.get_rect()
gameRect.midtop = (WINDOWWIDTH / 2, 10)
overRect.midtop = (WINDOWWIDTH / 2, gameRect.height + 10 + 25)
DISPLAYSURF.blit(gameSurf, gameRect)
DISPLAYSURF.blit(overSurf, overRect)
drawPressKeyMsg()
pygame.display.update()
pygame.time.wait(500)
#checkForKeyPress() # clear out any key presses in the event queue
while True:
if checkForKeyPress():
pygame.event.get() # clear event queue
return
def drawScore(score, generation, k):
scoreSurf = BASICFONT.render('Score: %s' % (score), True, WHITE)
scoreRect = scoreSurf.get_rect()
scoreRect.topleft = (WINDOWWIDTH - 120, 10)
DISPLAYSURF.blit(scoreSurf, scoreRect)
gener = BASICFONT.render('WORMS LEFT: %s' % k, True, WHITE)
gener1 = gener.get_rect()
gener1.topright = (WINDOWWIDTH - 220, 10)
DISPLAYSURF.blit(gener, gener1)
gener3 = BASICFONT.render('GENERATION: %s' % (generation), True, WHITE)
gener4 = gener3.get_rect()
gener4.topright = (WINDOWWIDTH - 420, 10)
DISPLAYSURF.blit(gener3, gener4)
def drawWorm(wormCoords):
for coord in wormCoords:
x = coord['x'] * CELLSIZE
y = coord['y'] * CELLSIZE
wormSegmentRect = pygame.Rect(x, y, CELLSIZE, CELLSIZE)
pygame.draw.rect(DISPLAYSURF, DARKGREEN, wormSegmentRect)
wormInnerSegmentRect = pygame.Rect(x + 4, y + 4, CELLSIZE - 8, CELLSIZE - 8)
pygame.draw.rect(DISPLAYSURF, GREEN, wormInnerSegmentRect)
def drawApple(coord):
x = coord['x'] * CELLSIZE
y = coord['y'] * CELLSIZE
appleRect = pygame.Rect(x, y, CELLSIZE, CELLSIZE)
pygame.draw.rect(DISPLAYSURF, WHITE, appleRect)
def drawGrid():
for x in range(0, WINDOWWIDTH, CELLSIZE): # draw vertical lines
pygame.draw.line(DISPLAYSURF, DARKGRAY, (x, 0), (x, WINDOWHEIGHT))
for y in range(0, WINDOWHEIGHT, CELLSIZE): # draw horizontal lines
pygame.draw.line(DISPLAYSURF, DARKGRAY, (0, y), (WINDOWWIDTH, y))
if __name__ == '__main__':
config_path = "./config-feedforward.txt"
config = neat.config.Config(neat.DefaultGenome, neat.DefaultReproduction, neat.DefaultSpeciesSet,
neat.DefaultStagnation, config_path)
# init NEAT
p = neat.Population(config)
p.add_reporter(neat.StdOutReporter(True))
# run NEAT
p.run(main_proc, 100)
#main()
Конфиг Фаил NEAT ./config-feedforward.txt
[NEAT]
fitness_criterion = max
fitness_threshold = 10000
pop_size = 20
reset_on_extinction = False
[DefaultGenome]
# node activation options
activation_default = tanh
activation_mutate_rate = 0.01
activation_options = tanh
# node aggregation options
aggregation_default = sum
aggregation_mutate_rate = 0.01
aggregation_options = sum
# node bias options
bias_init_mean = 0.0
bias_init_stdev = 1.0
bias_max_value = 30.0
bias_min_value = -30.0
bias_mutate_power = 0.5
bias_mutate_rate = 0.7
bias_replace_rate = 0.1
# genome compatibility options
compatibility_disjoint_coefficient = 1.0
compatibility_weight_coefficient = 0.5
# connection add/remove rates
conn_add_prob = 0.5
conn_delete_prob = 0.5
# connection enable options
enabled_default = True
enabled_mutate_rate = 0.01
feed_forward = True
initial_connection = full
# node add/remove rates
node_add_prob = 0.2
node_delete_prob = 0.2
# network parameters
num_hidden = 0
num_inputs = 6
num_outputs = 4
# node response options
response_init_mean = 1.0
response_init_stdev = 0.0
response_max_value = 30.0
response_min_value = -30.0
response_mutate_power = 0.0
response_mutate_rate = 0.0
response_replace_rate = 0.0
# connection weight options
weight_init_mean = 0.0
weight_init_stdev = 1.0
weight_max_value = 30
weight_min_value = -30
weight_mutate_power = 0.5
weight_mutate_rate = 0.8
weight_replace_rate = 0.1
[DefaultSpeciesSet]
compatibility_threshold = 3.0
[DefaultStagnation]
species_fitness_func = max
max_stagnation = 20
species_elitism = 2
[DefaultReproduction]
elitism = 3
survival_threshold = 0.2