Помогите с вводом из таблицы, и выводом из нее PyQt5 Python

Я не проверял вашу логику, а только сделал рабочим ввод и вывод данных из таблицы. Над приложение еще надо хорошенько поработать, так как любой чих и приложение вылетает.

QTableWidgetItem *QTableWidget::item(int row, int column) const

Возвращает элемент/item (QTableWidgetItem) для данной строки и столбца, если он был установлен; в противном случае возвращает nullptr.

QString QTableWidgetItem::text() const

Возвращает текст элемента/item`s.

Меняем

            H[i][j] = int(self.ui.tableWidget.item(i, j))
            

на

            item = self.ui.tableWidget.item(i, j).text()
            item = item if item else '0'             
            H[i][j] = int(item)   
        

void QTableWidget::setItem(int row, int column, *QTableWidgetItem item)

Устанавливает элемент для данной строки и столбца как item. Таблица становится владельцем item.

Меняем

        self.ui.tableWidget_3.setItem(i, 0, X[i])
        

на

        self.ui.tableWidget_3.setItem(i, 0, QtWidgets.QTableWidgetItem(str(X[i])))

import sys
from PyQt5 import QtWidgets, QtCore, QtGui


#from Kui import Ui_MainWindow
class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(1124, 509)
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        MainWindow.setFont(font)
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.groupBox_2 = QtWidgets.QGroupBox(self.centralwidget)
        self.groupBox_2.setGeometry(QtCore.QRect(50, 120, 801, 351))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.groupBox_2.setFont(font)
        self.groupBox_2.setLayoutDirection(QtCore.Qt.LeftToRight)
        self.groupBox_2.setObjectName("groupBox_2")
        self.tableWidget = QtWidgets.QTableWidget(self.groupBox_2)
        self.tableWidget.setGeometry(QtCore.QRect(10, 20, 630, 320))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.tableWidget.setFont(font)
        self.tableWidget.setStyleSheet("QTableWidget {\n"
"background-color: white \n"
"}\n"
"")
        self.tableWidget.setObjectName("tableWidget")
        self.tableWidget.setColumnCount(1)
        self.tableWidget.setRowCount(1)
        item = QtWidgets.QTableWidgetItem()
        self.tableWidget.setVerticalHeaderItem(0, item)
        item = QtWidgets.QTableWidgetItem()
        self.tableWidget.setHorizontalHeaderItem(0, item)
        item = QtWidgets.QTableWidgetItem()
        self.tableWidget.setItem(0, 0, item)
        self.tableWidget_2 = QtWidgets.QTableWidget(self.groupBox_2)
        self.tableWidget_2.setGeometry(QtCore.QRect(640, 20, 150, 320))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.tableWidget_2.setFont(font)
        self.tableWidget_2.setObjectName("tableWidget_2")
        self.tableWidget_2.setColumnCount(1)
        self.tableWidget_2.setRowCount(1)
        item = QtWidgets.QTableWidgetItem()
        self.tableWidget_2.setVerticalHeaderItem(0, item)
        item = QtWidgets.QTableWidgetItem()
        self.tableWidget_2.setHorizontalHeaderItem(0, item)
        self.groupBox_3 = QtWidgets.QGroupBox(self.centralwidget)
        self.groupBox_3.setGeometry(QtCore.QRect(870, 120, 251, 351))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.groupBox_3.setFont(font)
        self.groupBox_3.setLayoutDirection(QtCore.Qt.LeftToRight)
        self.groupBox_3.setObjectName("groupBox_3")
        self.tableWidget_3 = QtWidgets.QTableWidget(self.groupBox_3)
        self.tableWidget_3.setGeometry(QtCore.QRect(10, 20, 230, 320))
        self.tableWidget_3.setStyleSheet("background-color: white ")
        self.tableWidget_3.setObjectName("tableWidget_3")
        self.tableWidget_3.setColumnCount(1)
        self.tableWidget_3.setRowCount(1)
        item = QtWidgets.QTableWidgetItem()
        self.tableWidget_3.setVerticalHeaderItem(0, item)
        item = QtWidgets.QTableWidgetItem()
        self.tableWidget_3.setHorizontalHeaderItem(0, item)
        self.pushButton_2 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_2.setGeometry(QtCore.QRect(930, 80, 161, 31))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.pushButton_2.setFont(font)
        self.pushButton_2.setStyleSheet("QPushButton:hover {\n"
"background-color: white\n"
"\n"
"}")
        self.pushButton_2.setObjectName("pushButton_2")
        self.pushButton_3 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_3.setGeometry(QtCore.QRect(550, 90, 140, 30))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.pushButton_3.setFont(font)
        self.pushButton_3.setStyleSheet("QPushButton:hover {\n"
"background-color: white\n"
"\n"
"}")
        self.pushButton_3.setObjectName("pushButton_3")
        self.groupBox = QtWidgets.QGroupBox(self.centralwidget)
        self.groupBox.setGeometry(QtCore.QRect(50, 50, 241, 61))
        font = QtGui.QFont()
        font.setPointSize(10)
        self.groupBox.setFont(font)
        self.groupBox.setLayoutDirection(QtCore.Qt.RightToLeft)
        self.groupBox.setStyleSheet("QPushButton:hover {\n"
"background-color: white\n"
"\n"
"}")
        self.groupBox.setObjectName("groupBox")
        self.pushButton = QtWidgets.QPushButton(self.groupBox)
        self.pushButton.setGeometry(QtCore.QRect(90, 20, 140, 30))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.pushButton.setFont(font)
        self.pushButton.setStyleSheet("")
        self.pushButton.setObjectName("pushButton")
        self.lineEdit = QtWidgets.QLineEdit(self.groupBox)
        self.lineEdit.setGeometry(QtCore.QRect(12, 19, 60, 30))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.lineEdit.setFont(font)
        self.lineEdit.setObjectName("lineEdit")
        self.label = QtWidgets.QLabel(self.centralwidget)
        self.label.setGeometry(QtCore.QRect(350, 10, 531, 41))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(16)
        font.setItalic(False)
        self.label.setFont(font)
        self.label.setLayoutDirection(QtCore.Qt.LeftToRight)
        self.label.setAlignment(QtCore.Qt.AlignCenter)
        self.label.setObjectName("label")
        self.pushButton_4 = QtWidgets.QPushButton(self.centralwidget)
        self.pushButton_4.setGeometry(QtCore.QRect(780, 100, 71, 21))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.pushButton_4.setFont(font)
        self.pushButton_4.setStyleSheet("QPushButton:hover {\n"
"background-color: white\n"
"\n"
"}")
        self.pushButton_4.setObjectName("pushButton_4")
        self.lineEdit_2 = QtWidgets.QLineEdit(self.centralwidget)
        self.lineEdit_2.setGeometry(QtCore.QRect(370, 80, 101, 30))
        font = QtGui.QFont()
        font.setFamily("Times New Roman")
        font.setPointSize(10)
        self.lineEdit_2.setFont(font)
        self.lineEdit_2.setObjectName("lineEdit_2")
        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 1124, 21))
        self.menubar.setObjectName("menubar")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "MainWindow"))
        self.groupBox_2.setTitle(_translate("MainWindow", "Шаг 2: Ввод переменных "))
        item = self.tableWidget.verticalHeaderItem(0)
        item.setText(_translate("MainWindow", "1"))
        item = self.tableWidget.horizontalHeaderItem(0)
        item.setText(_translate("MainWindow", "1"))
        __sortingEnabled = self.tableWidget.isSortingEnabled()
        self.tableWidget.setSortingEnabled(False)
        self.tableWidget.setSortingEnabled(__sortingEnabled)
        item = self.tableWidget_2.verticalHeaderItem(0)
        item.setText(_translate("MainWindow", "1"))
        item = self.tableWidget_2.horizontalHeaderItem(0)
        item.setText(_translate("MainWindow", "b"))
        self.groupBox_3.setTitle(_translate("MainWindow", " Решение данной системы:"))
        item = self.tableWidget_3.verticalHeaderItem(0)
        item.setText(_translate("MainWindow", "1"))
        item = self.tableWidget_3.horizontalHeaderItem(0)
        item.setText(_translate("MainWindow", "x"))
        self.pushButton_2.setText(_translate("MainWindow", "Решение"))
        self.pushButton_3.setText(_translate("MainWindow", "Очистка таблиц"))
        self.groupBox.setTitle(_translate("MainWindow", "Шаг 1:  Выбор кол-ва переменных "))
        self.pushButton.setText(_translate("MainWindow", "Построить таблицу "))
        self.label.setText(_translate("MainWindow", "Решение системы линейных уравнений методом Гаусса  "))
        self.pushButton_4.setText(_translate("MainWindow", "Справка"))


class MyWin(QtWidgets.QMainWindow):
    def __init__(self, parent=None):
        QtWidgets.QWidget.__init__(self, parent)
        self.ui = Ui_MainWindow()
        self.ui.setupUi(self)

        self.ui.pushButton.clicked.connect(self.bp)
        self.ui.pushButton_2.clicked.connect(self.bp_2)
        self.ui.pushButton_3.clicked.connect(self.bp_3)

    def bp(self):
        self.text = int(self.ui.lineEdit.text())
        self.ui.tableWidget.setRowCount(self.text)
        self.ui.tableWidget.setColumnCount(self.text)
        self.ui.tableWidget_2.setColumnCount(1)
        self.ui.tableWidget_2.setRowCount(self.text)
        self.ui.tableWidget_3.setColumnCount(1)
        self.ui.tableWidget_3.setRowCount(self.text)
        print(f'self.text = {self.text}') #
# ???        return self.text

    def bp_2(self):
        # Перемена местами двух строк системы
        def SwapRows(A, B, row1, row2):
            A[row1], A[row2] = A[row2], A[row1]
            B[row1], B[row2] = B[row2], B[row1]

        # Деление строки системы на числа
        def DivideRow(A, B, row, divider):
            A[row] = [a / divider for a in A[row]]
            B[row] /= divider

        # Сложение строки системы с другой строкой, умноженной на число
        def CombineRows(A, B, row, source_row, weight):
            A[row] = [(a + k * weight) for a, k in zip(A[row], A[source_row])]
            B[row] += B[source_row] * weight

        # Решение системы методом Гаусса (приведением к треугольному виду)
        D = int(self.ui.lineEdit.text())
        H = [[0] * D for i in range(D)]
        for i in range(D):
            for j in range(D):
#                 H[i][j] = int(self.ui.tableWidget.item(i, j))
                item = self.ui.tableWidget.item(i, j).text()             # +++
                item = item if item else '0'                             # +++
                H[i][j] = int(item)                                      # +++

            j = 0
        i = 0
        M =[]
        while i != D:
#            M.insert(i, int(self.ui.tableWidget_2.item(i, 0)))
            M.insert(i, int(self.ui.tableWidget_2.item(i, 0).text()))    # +++
            
            i = i + 1
        column = 0
        while (column < len(M)):
        # Ищем максимальный по модулю элемент в {0}-м столбце: .format(column + 1)
            current_row = None
            for r in range(column, len(H)):
                if current_row is None or abs(H[r][column]) > abs(H[current_row][column]):
                    current_row = r
            if current_row is None:
                # Решений нет
                self.ui.tableWidget_3.setItem(0, 0, 'Решений нет')
            if current_row != column:
                # Переставляем строку с найденным элементом повыше
                SwapRows(H, M, current_row, column)
            # Нормализуем строку с найденным элементом
            DivideRow(H, M, column, H[column][column])
            # Обрабатываем нижележащие строки
            for r in range(column + 1, len(H)):
                CombineRows(H, M, r, column, -H[r][column])
            column += 1
        # Матрица приведена к треугольному виду, считаем решение
        X = [0 for b in M]
        for i in range(len(M) - 1, -1, -1):
            X[i] = M[i] - sum(x * a for x, a in zip(X[(i + 1):], H[i][(i + 1):]))

#            self.ui.tableWidget_3.setItem(i, 0, X[i])
            self.ui.tableWidget_3.setItem(i, 0, QtWidgets.QTableWidgetItem(str(X[i]))) # +++
            
    def bp_3(self):
        self.ui.tableWidget.clearContents()
        self.ui.tableWidget_2.clearContents()


if __name__ == "__main__":
    app = QtWidgets.QApplication(sys.argv)
    myapp = MyWin()
    myapp.show()
    sys.exit(app.exec_())