Выключить виджеты QCheckBox

enabled : bool

Это свойство указывает, включен ли виджет.

Не изменяйте код, сгенерированный Qt Designer. Создайте другой класс, который наследуется от соответствующего виджета, и используйте созданный класс для его заполнения.

import sys
from PyQt5 import QtWidgets, QtCore, QtGui
from PyQt5.Qt import *


class Ui_MainWindow(object):
    def setupUi(self, MainWindow):
        MainWindow.setObjectName("MainWindow")
        MainWindow.resize(421, 407)
        MainWindow.setStyleSheet("QStatusBar {\n"
"    background-color: rgb(207, 207, 207)\n"
"}")
        self.centralwidget = QtWidgets.QWidget(MainWindow)
        self.centralwidget.setObjectName("centralwidget")
        self.label_2 = QtWidgets.QLabel(self.centralwidget)
        self.label_2.setGeometry(QtCore.QRect(10, 170, 241, 17))
        self.label_2.setStyleSheet("font: 9pt \"Ubuntu\";")
        self.label_2.setObjectName("label_2")
        self.browse = QtWidgets.QPushButton(self.centralwidget)
        self.browse.setGeometry(QtCore.QRect(240, 190, 89, 25))
        self.browse.setObjectName("browse")
        self.logo = QtWidgets.QLabel(self.centralwidget)
        self.logo.setGeometry(QtCore.QRect(90, -10, 251, 81))
        self.logo.setText("")
        self.logo.setPixmap(QtGui.QPixmap("./logo.png"))
        self.logo.setObjectName("logo")
        self.filename = QtWidgets.QLineEdit(self.centralwidget)
        self.filename.setGeometry(QtCore.QRect(10, 190, 221, 25))
        self.filename.setObjectName("filename")
        self.line = QtWidgets.QFrame(self.centralwidget)
        self.line.setGeometry(QtCore.QRect(-20, 230, 481, 31))
        self.line.setFrameShape(QtWidgets.QFrame.HLine)
        self.line.setFrameShadow(QtWidgets.QFrame.Sunken)
        self.line.setObjectName("line")
        self.btncryptmsg = QtWidgets.QPushButton(self.centralwidget)
        self.btncryptmsg.setGeometry(QtCore.QRect(240, 110, 171, 21))
        self.btncryptmsg.setStyleSheet("font: 10pt \"Ubuntu\";")
        self.btncryptmsg.setObjectName("btncryptmsg")
        self.line_2 = QtWidgets.QFrame(self.centralwidget)
        self.line_2.setGeometry(QtCore.QRect(-30, 60, 471, 20))
        self.line_2.setFrameShape(QtWidgets.QFrame.HLine)
        self.line_2.setFrameShadow(QtWidgets.QFrame.Sunken)
        self.line_2.setObjectName("line_2")
        self.label_4 = QtWidgets.QLabel(self.centralwidget)
        self.label_4.setGeometry(QtCore.QRect(10, 90, 161, 17))
        self.label_4.setStyleSheet("font: 9pt \"Ubuntu\";")
        self.label_4.setObjectName("label_4")
        self.textEdit = QtWidgets.QTextEdit(self.centralwidget)
        self.textEdit.setGeometry(QtCore.QRect(10, 110, 221, 51))
        self.textEdit.setObjectName("textEdit")
        self.btnencryptmsg = QtWidgets.QPushButton(self.centralwidget)
        self.btnencryptmsg.setGeometry(QtCore.QRect(240, 140, 171, 21))
        self.btnencryptmsg.setStyleSheet("font: 10pt \"Ubuntu\";")
        self.btnencryptmsg.setObjectName("btnencryptmsg")
        self.textEdit_2 = QtWidgets.QTextEdit(self.centralwidget)
        self.textEdit_2.setGeometry(QtCore.QRect(10, 250, 401, 71))
        self.textEdit_2.setStyleSheet("QFrame {\n"
"    background-color: rgb(207, 207, 207)\n"
"}")
        self.textEdit_2.setObjectName("textEdit_2")
        self.progressBar = QtWidgets.QProgressBar(self.centralwidget)
        self.progressBar.setGeometry(QtCore.QRect(10, 330, 401, 16))
        self.progressBar.setLayoutDirection(QtCore.Qt.LeftToRight)
        self.progressBar.setStyleSheet("QProgressBar{\n"
"    background-color: rgb(207, 207, 207)\n"
"}\n"
"\n"
"QProgressBar::chunk{\n"
"    background-color: rgb(85, 87, 83)\n"
"}")
        self.progressBar.setProperty("value", 0)
        self.progressBar.setAlignment(QtCore.Qt.AlignCenter)
        self.progressBar.setObjectName("progressBar")
        self.checkBoxText = QtWidgets.QCheckBox(self.centralwidget)
        self.checkBoxText.setGeometry(QtCore.QRect(10, 220, 92, 23))
        self.checkBoxText.setObjectName("checkBoxText")
        self.checkBoxIMG = QtWidgets.QCheckBox(self.centralwidget)
        self.checkBoxIMG.setGeometry(QtCore.QRect(80, 220, 121, 23))
        self.checkBoxIMG.setObjectName("checkBoxIMG")
#        self.checkBoxIMG.stateChanged.connect(lambda:(self.checkBoxText.setChecked(False), self.checkBoxVideo.setChecked(False)))
        self.checkBoxVideo = QtWidgets.QCheckBox(self.centralwidget)
        self.checkBoxVideo.setGeometry(QtCore.QRect(210, 220, 92, 23))
        self.checkBoxVideo.setObjectName("checkBoxVideo")
        MainWindow.setCentralWidget(self.centralwidget)
        self.menubar = QtWidgets.QMenuBar(MainWindow)
        self.menubar.setGeometry(QtCore.QRect(0, 0, 421, 22))
        self.menubar.setObjectName("menubar")
        self.menuHELP = QtWidgets.QMenu(self.menubar)
        self.menuHELP.setObjectName("menuHELP")
        MainWindow.setMenuBar(self.menubar)
        self.statusbar = QtWidgets.QStatusBar(MainWindow)
        self.statusbar.setObjectName("statusbar")
        MainWindow.setStatusBar(self.statusbar)
        self.menubar.addAction(self.menuHELP.menuAction())

        self.retranslateUi(MainWindow)
        QtCore.QMetaObject.connectSlotsByName(MainWindow)

    def retranslateUi(self, MainWindow):
        _translate = QtCore.QCoreApplication.translate
        MainWindow.setWindowTitle(_translate("MainWindow", "Cryptone тестовая версия v1.0."))
        self.label_2.setText(_translate("MainWindow", "Укажите путь к  файлу:"))
        self.browse.setText(_translate("MainWindow", "Обзор..."))
        self.btncryptmsg.setText(_translate("MainWindow", "зашифровать"))
        self.label_4.setText(_translate("MainWindow", "Введите сообщение:"))
        self.btnencryptmsg.setText(_translate("MainWindow", "расшифровать"))
        self.checkBoxText.setText(_translate("MainWindow", "Текст"))
        self.checkBoxIMG.setText(_translate("MainWindow", "Изображение"))
        self.checkBoxVideo.setText(_translate("MainWindow", "Видео"))
        self.menuHELP.setTitle(_translate("MainWindow", "FAQ по работе с порграммой"))
        

class MainWindow(QMainWindow, Ui_MainWindow):
    def __init__(self):
        super().__init__()
        self.setupUi(self)

        self.checkBoxIMG.stateChanged.connect(
            lambda state: (                                        # +++ state
#                self.checkBoxText.setChecked(False), 
#                self.checkBoxVideo.setChecked(False)
                self.checkBoxText.setEnabled(not state),           # +++
                self.checkBoxVideo.setEnabled(not state)           # +++
            )
        )

                

if __name__ == "__main__":
    app = QApplication(sys.argv)
    w = MainWindow()
    w.show()
    sys.exit(app.exec_())