Модель не обучается

Подскажите пожалуйста, почему моя модель не обучается?

import sys
import numpy as np
import pandas as pd
from keras.models import Sequential
from keras.layers import LSTM
from keras.layers import Dense
from keras.layers import RepeatVector
from keras.layers import TimeDistributed
import tensorflow as tf
import csv
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras import layers
from tensorflow.keras.models import load_model
import os, os.path
import re
from keras.layers import TimeDistributed
from sklearn.preprocessing import MinMaxScaler
import sys
import numpy as np
import pandas as pd
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense
from keras.layers import LSTM, Dense, RepeatVector, TimeDistributed
import tensorflow as tf
from sklearn.preprocessing import MinMaxScaler

TRAIN_FILE = 'datasets/typeRdpSSH.log'
SYSCALL_FILE = 'syscalls.txt'
TEST_FILE = 'datasets/typeRdpSSH.log' 
ROWS = 100
array_size = 100
sequence_length = 10

def extract_syscalls_for_process(log_file, syscall_file, array_size=100):
    """Извлекает системные вызовы для каждого процесса из файла логов.

    Args:
        log_file: Имя файла с логами.
        syscall_file: Имя файла с перечнем системных вызовов.
        array_size: Размер массива для хранения системных вызовов.

    Returns:
        Список массивов, где каждый массив - это последовательность системных вызовов
        для одного процесса, размером `array_size`.
    """

    syscalls = {}
    with open(syscall_file, "r") as f:
        for i, line in enumerate(f):
            syscalls[line.strip()] = i + 1

    process_sequences = []
    current_process_id = None
    syscall_array = [0] * array_size
    current_index = 0

    with open(log_file, "r") as f:
        for line in f:
            match = re.search(r'process name=(\w+) process id=(\d+) syscall=(\w+)', line)
            if match:
                process_id = int(match.group(2))

                if process_id != current_process_id and current_process_id is not None:
                    # Сохраняем массив для предыдущего процесса
                    process_sequences.append(syscall_array.copy())
                    # Создаем новый массив для текущего процесса
                    syscall_array = [0] * array_size
                    current_index = 0

                current_process_id = process_id
                syscall = match.group(3)
                if syscall in syscalls:
                    syscall_array[current_index] = syscalls[syscall]
                    current_index = (current_index + 1) % array_size

    if current_process_id:
        # Сохраняем массив для последнего процесса
        process_sequences.append(syscall_array.copy())

    return process_sequences

def create_sequences(process_sequences, sequence_length):
    """Создает последовательности из массивов системных вызовов.

    Args:
        process_sequences: Список массивов системных вызовов.
        sequence_length: Длина каждой последовательности.

    Returns:
        Список последовательностей.
    """
    sequences = []
    for i in range(len(process_sequences) - sequence_length + 1):
        sequences.append(process_sequences[i:i + sequence_length])
    return sequences

def loadTrain():
    process_sequences = extract_syscalls_for_process(TRAIN_FILE, SYSCALL_FILE, array_size)
    sequences = create_sequences(process_sequences, sequence_length)

    # Проверка наличия нулевых значений в sequences
    if np.any(np.isnan(sequences)):
        print("Sequences contain NaN values. Replacing with 0.")
        sequences = np.nan_to_num(sequences)

    num_sequences = len(sequences)
    sequences = np.array(sequences).reshape(-1, num_sequences, ROWS)
    sequences = sequences.reshape(sequences.shape[0], num_sequences, ROWS)

    # Масштабирование данных
    scaler = MinMaxScaler()
    sequences = scaler.fit_transform(sequences.reshape(-1, ROWS)).reshape(sequences.shape)

    # Модель автоэнкодера
    model = Sequential()
    model.add(LSTM(ROWS, activation='relu', input_shape=(num_sequences, ROWS), return_sequences=True))
    model.add(LSTM(100, activation='relu', return_sequences=False))
    model.add(RepeatVector(num_sequences))
    model.add(LSTM(100, activation='relu', return_sequences=True))
    model.add(LSTM(ROWS, activation='relu', return_sequences=True))
    model.add(TimeDistributed(Dense(ROWS)))
    model.compile(optimizer='adam', loss='mse')
    model.summary()

    # Обучение модели
    model.fit(sequences, sequences, epochs=5, batch_size=1, shuffle=False)

    # Сохранение модели и весов
    print("Save model: models/AE.keras")
    model.save("models/AE.keras")
    print("Save weights: models/AE.weights.h5")
    model.save_weights("models/AE.weights.h5")

##### ЗАГРУЗИТЬ И ОПРЕДЕЛИТЬ НАБОР ДАННЫХ #####
def loadDetect():
    process_sequences = extract_syscalls_for_process(TEST_FILE, SYSCALL_FILE, array_size)
    sequences = create_sequences(process_sequences, sequence_length)

    # Проверка наличия нулевых значений в sequences
    if np.any(np.isnan(sequences)):
        print("Sequences contain NaN values. Replacing with 0.")
        sequences = np.nan_to_num(sequences)

    num_sequences = len(sequences)
    sequences = np.array(sequences).reshape(-1, num_sequences, ROWS)
    sequences = sequences.reshape(sequences.shape[0], num_sequences, ROWS)

    # Масштабирование данных
    scaler = MinMaxScaler()
    sequences = scaler.fit_transform(sequences.reshape(-1, ROWS)).reshape(sequences.shape)

    # Загрузка модели из файла
    print("Load model: models/AE.keras")
    detectModel = tf.keras.models.load_model("models/AE.keras")
    print("Load weights: models/AE.weights.h5")
    detectWeights = detectModel.load_weights("models/AE.weights.h5")

    # Демонстрация реконструкции
    yhat = detectModel.predict(sequences)

    mse = np.mean(np.square(sequences - yhat), axis=(1, 2))
    threshold = np.mean(mse) + np.std(mse)  # Пороговое значение для определения аномалий
    print(mse)
    print(threshold)

    anomaly_indices = np.where(mse > threshold)[0]

    if len(anomaly_indices) == 0:
        print("Anomaly no detection!")
    else:
        print("Anomaly detection!")
        original_sequences = scaler.inverse_transform(sequences.reshape(-1, ROWS)).reshape(sequences.shape)
        # Вывод аномальных последовательностей
        for i in anomaly_indices:
          print(f"Anomalous sequence {i}:")
          print(original_sequences[i])  # Вывод аномальной последовательности в исходном масштабе

if __name__ == "__main__":
    if len(sys.argv) > 0:
        param_name = sys.argv[1]
        if (param_name == "--train" or param_name == "-t"):
            loadTrain()
        elif (param_name == "--detect" or param_name == "-d"):
            loadDetect()
        else:
            print ("Error. Unknown parameter: '{}'".format (param_name) )
            sys.exit (1)


        
        
        
          


Файл syscalls

getpid
openat
rt_sigprocmask
lseek
chown
connect
bind
pipe
execve
rt_sigaction
close
exit_group
fstat
clone
poll
...

Файл typeRdpSSH


15:56:19.441261781: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441269911: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441282541: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441291880: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441294532: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441299407: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441301539: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441310335: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441314380: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441337657: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441353807: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=epoll_wait)
15:56:19.441361616: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=select)
15:56:19.441365916: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=select)
15:56:19.441369527: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441372835: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441378048: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441391880: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441420458: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441423533: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441427599: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441429969: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441434742: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=epoll_wait)
15:56:19.441450307: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441457140: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441467447: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441475299: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441478101: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441482394: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441484434: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=recvmsg)
15:56:19.441490616: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441494560: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441514107: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=xautolock process id=4091 syscall=poll)
15:56:19.441528618: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=epoll_wait)
15:56:19.441535500: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=select)
15:56:19.441539807: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=select)
15:56:19.441543258: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441546292: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441550486: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441562936: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441652741: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441661055: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=recvmsg)
15:56:19.441670367: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441673129: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=setitimer)
15:56:19.441679475: Warning (user=ubuntu container_id=8a9d84239520 container_name=uxrdp process name=Xorg process id=3994 syscall=epoll_wait)
...

Ответы (0 шт):