Start QProcess from within QDialog that is used as a progress monitor

Question

I have a main pyqt program that needs to run external program with arguments. I would like to use a QDialog as a sort of a status monitor that would capture the external program's stdout while it is executing and display them in a textbox inside the QDialog. I have the following status monitor code:

class ProgressInfo(QtGui.QDialog):
    def __init__(self, cmd, args, parent=None):
        #super(self).__init__(parent)
        QDialog.__init__(self)
        self.ui = Ui_Dialog()
        self.ui.setupUi(self)
        self.cmd = cmd
        self.args = args
        self.keepGoing = True
        layout = QFormLayout()
        layout.setContentsMargins(10, 10, 10, 10)
        self.output = QtGui.QTextEdit()
        layout.addRow(self.output)
        layout.addRow(self.ui.buttonBox)
        self.setLayout(layout)

        self.ext_process = QtCore.QProcess(self)
        #self.ext_process.waitForFinished(-1)
        #self.ext_process.waitForStarted()
        #self.ext_process.readyRead.connect(self.dataReady)
        self.ext_process.started.connect(self.open)
        self.ext_process.readyReadStandardOutput.connect(self.dataReady)
        self.ext_process.finished.connect(self.onProcessFinished)
        self.ext_process.start(self.cmd, self.args)

    def dataReady(self):
        cursor = self.output.textCursor()
        cursor.movePosition(cursor.End)
        cursor.insertText(str(self.ext_process.readAll()))
        self.output.ensureCursorVisible()

    def onProcessFinished(self):
        cursor = self.output.textCursor()
        cursor.movePosition(cursor.End)
        #cursor.insertText(str(self.ext_process.readAll()))
        cursor.insertText(str(self.ext_process.readAllStandardOutput()))
        self.output.ensureCursorVisible()

Then I would instantiate this with the following command:

prog='C:/Program Files (x86)/My Program/Execute.exe'
margs=['D:/Data/Input1.txt', 'D:/Data/Input2.txt']
status = ProgressInfo(prog, margs, self)

So far this hasn't worked yet.

Problem 1: the external program will run only after I uncommented out the waitForFinished(-1) line.

Problem 2. the QDialog box only open in a flash, then disappears.

Problem 3. obviously, no standout from the running program is displayed.

Lastly, the code I put together draws on many people's ideas and lessons, but I look at it, it seems that it can only print out all of the standout after the program finished, but I was hoping it will display line by line as the program is writing them out at runtime.

My tool chains: Python 64-bit version 2.7.5 and I am developing on Windows 7 box

('Qt version:', '4.8.5')
('SIP version:', '4.14.7')
('PyQt version:', '4.10.2')

Thanks for any help.

Answer 1

Here is an example how you can do it (I use QWidget but you can also use QDialog or whatever). I don't use a separate thread because the UI doesn't need to be interactive. If you want to add buttons etc. then you should consider going for the good old QThread running a QObject model provided by Qt.

#!/usr/bin/python
from PyQt4.QtGui import * 
from PyQt4.QtCore import * 
import sys


class MyQProcess(QWidget):     
  def __init__(self):    
   super(QWidget, self).__init__()

   # Add the UI components (here we use a QTextEdit to display the stdout from the process)
   layout = QVBoxLayout()
   self.edit = QTextEdit()
   self.edit.setWindowTitle("QTextEdit Standard Output Redirection")
   layout.addWidget(self.edit)
   self.setLayout(layout)

   # Add the process and start it
   self.process = QProcess()
   self.setupProcess()   

   # Show the widget
   self.show()

  def setupProcess(self):
    # Set the channels
    self.process.setProcessChannelMode(QProcess.MergedChannels)
    # Connect the signal readyReadStandardOutput to the slot of the widget
    self.process.readyReadStandardOutput.connect(self.readStdOutput)
    # Run the process with a given command
    self.process.start("df -h")

  def __del__(self):
    # If QApplication is closed attempt to kill the process
    self.process.terminate()
    # Wait for Xms and then elevate the situation to terminate
    if not self.process.waitForFinished(10000):
      self.process.kill()

  @pyqtSlot()
  def readStdOutput(self):
    # Every time the process has something to output we attach it to the QTextEdit
    self.edit.append(QString(self.process.readAllStandardOutput()))


def main():  
    app = QApplication(sys.argv)
    w   = MyQProcess()

    return app.exec_()

if __name__ == '__main__':
    main()

Notice that the command I'm using (df -h) runs once (it's a Linux command which displays the disk usage on your hard drives) and then it's over. You can replace it also with your Execute.exe which can run indefinitely. I have tested it with htop (a terminal-based advanced task manager), which once started doesn't stop unless the user wants it to or the system stops (crash, shutdown etc.).

Note that you have to ensure that the external process is stopped in a clean manner. This can be done inside __del__ (destructor) or another function invoked at the end of the life of a given widget. What I've done is basically send a SIGTERM (terminate) to the external process and once a given amount of time has passed but the process is still running I elevate the situation to SIGKILL (kill).

The code needs more work obviously but it should be enough to give you an idea how things work.

Here is the same version of the code above but with an extra thread. Note that I am redirecting the output from the external process to a slot in my worker. You don't have to do that unless you want to maybe work on that output. So you can skip this and connect your process signal to the slot in your widget that receives it and outputs its content. The processing of the output will be done again inside the separate thread so you can go the distance instead of freezing your UI (which will happen if you follow the

from PyQt4.QtGui import * 
from PyQt4.QtCore import * 
import sys

class Worker(QObject):
  sendOutput = pyqtSignal(QString)  

  def __init__(self):
    super(Worker, self).__init__()
    self.process = QProcess()
    self.setupProcess()

  def __del__(self):
    self.process.terminate()
    if not self.process.waitForFinished(10000):
      self.process.kill()

  def setupProcess(self):
    self.process.setProcessChannelMode(QProcess.MergedChannels)
    self.process.readyReadStandardOutput.connect(self.readStdOutput)
    self.process.start("htop")

  @pyqtSlot()
  def readStdOutput(self):
    output = QString(self.process.readAllStandardOutput())
    # Do some extra processing of the output here if required
    # ...
    self.sendOutput.emit(output)



class MyQProcess(QWidget):     
  def __init__(self):    
   super(QWidget, self).__init__()
   layout = QVBoxLayout()
   self.edit = QTextEdit()
   self.thread = QThread()

   self.setupConnections()

   self.edit.setWindowTitle("QTextEdit Standard Output Redirection")
   layout.addWidget(self.edit)
   self.setLayout(layout)
   self.show()

  def setupConnections(self):
    self.worker = Worker()
    self.thread.finished.connect(self.worker.deleteLater)
    self.worker.sendOutput.connect(self.showOutput)

    self.worker.moveToThread(self.thread)
    self.thread.start()

  def __del__(self):
    if self.thread.isRunning():
      self.thread.quit()
      # Do some extra checking if thread has finished or not here if you want to

  #Define Slot Here 
  @pyqtSlot(QString)
  def showOutput(self, output):
    #self.edit.clear()
    self.edit.append(output)


def main():  
    app = QApplication(sys.argv)
    w   = MyQProcess()

    return app.exec_()

if __name__ == '__main__':
    main()

Further clarification: As I've told @BrendanAbel in the comment section of his answer the issue with using slots with QThread is that the slots have the same thread affinity (=the thread they belong to) as the QThread instance itself, which is the same thread where the QThread was created from. The only - I repeat the only - thing that runs in a separate thread when it comes to a QThread is its event loop represented by QThread.run(). If you look on the Internet you will find out that this way of doing things is discouraged (unless you really, really know that you have to subclass QThread) because since the early versions of Qt 4 run() was abstract and you had to subclass QThread in order to use a QThread. Later the abstract run() got a concrete implementation hence the need of subclassing QThread was removed. About thread-safety and signals what @BrendanAbel wrote is only partially true. It comes down to the connection type (default is AutoConnection). If you manually specify the connection type you may actually render the signals thread-unsafe. Read more about this in the Qt documentation.

Answer 2

You can't wait on a process and update the GUI in the same thread. The GUI only updates on each iteration of the event loop. If the event loop is stuck waiting on a process, it can't update the GUI.

The solution is to monitor the process in a separate thread, freeing up the main thread to continue updating the GUI. Most GUI elements aren't thread-safe, so you can't write the output to the QTextEdit directly from the monitoring thread. But Signals are thread-safe, so you can send output from the monitoring thread back to the main thread using a signal, which the QDialog in the main thread can handle and print the output to the QTextEdit

import subprocess
import sys

from PyQt4.QtCore import QObject, QThread, pyqtSignal
from PyQt4.QtGui import QDialog, QTextEdit, QVBoxLayout, QPushButton, QApplication


class MyDialog(QDialog):

    def __init__(self):
        super(MyDialog, self).__init__()
        self.ui_lay = QVBoxLayout()
        self.setLayout(self.ui_lay)
        self.ui_txt = QTextEdit(self)
        self.ui_lay.addWidget(self.ui_txt)
        self.ui_btn = QPushButton('Ping', self)
        self.ui_lay.addWidget(self.ui_btn)

        self.thread = MyThread(self)
        self.thread.line_printed.connect(self.handle_line)

        self.ui_btn.clicked.connect(self.run_thread)

    def run_thread(self):
        self.thread.start_command('ping google.com')

    def handle_line(self, line):
        cursor = self.ui_txt.textCursor()
        cursor.movePosition(cursor.End)
        cursor.insertText(line)
        self.ui_txt.ensureCursorVisible()


class MyThread(QThread):

    line_printed = pyqtSignal(str)

    def __init__(self, parent):
        super(MyThread, self).__init__(parent)
        self.cmd = None

    def start_command(self, cmd):
        self.cmd = cmd
        self.start()

    def run(self):
        if self.cmd:
            popen = subprocess.Popen(self.cmd, stdout=subprocess.PIPE, shell=True)
            lines_iterator = iter(popen.stdout.readline, b"")
            for line in lines_iterator:
                self.line_printed.emit(line)


if __name__ == '__main__':
    app = QApplication(sys.argv)
    dlg = MyDialog()
    dlg.show()
    app.exec_()