subprocess source code learning - posix - 1

Just start from the initialization function Popen. Code like this:

    def __init__(self, args, bufsize=0, executable=None,
                 stdin=None, stdout=None, stderr=None,
                 preexec_fn=None, close_fds=False, shell=False,
                 cwd=None, env=None, universal_newlines=False,
                 startupinfo=None, creationflags=0):
        """Create new Popen instance."""
        _cleanup() # refer to Note 1

        self._child_created = False
        if not isinstance(bufsize, (int, long)):
            raise TypeError("bufsize must be an integer")

        # preexec_fn and close_fds are not supported by Windows
        if mswindows:
            if preexec_fn is not None:
                raise ValueError("preexec_fn is not supported on Windows "
                                 "platforms")
            if close_fds and (stdin is not None or stdout is not None or
                              stderr is not None):
                raise ValueError("close_fds is not supported on Windows "
                                 "platforms if you redirect stdin/stdout/stderr")
        
        # startupinfo and creationflags are not suppported by posix systems
        else:
            # POSIX
            if startupinfo is not None:
                raise ValueError("startupinfo is only supported on Windows "
                                 "platforms")
            if creationflags != 0:
                raise ValueError("creationflags is only supported on Windows "
                                 "platforms")

        self.stdin = None
        self.stdout = None
        self.stderr = None
        self.pid = None
        self.returncode = None
        self.universal_newlines = universal_newlines

        # Input and output objects. The general principle is like
        # this:
        #
        # Parent                   Child
        # ------                   -----
        # p2cwrite   ---stdin--->  p2cread
        # c2pread    <--stdout---  c2pwrite
        # errread    <--stderr---  errwrite
        #
        # On POSIX, the child objects are file descriptors.  On
        # Windows, these are Windows file handles.  The parent objects
        # are file descriptors on both platforms.  The parent objects
        # are None when not using PIPEs. The child objects are None
        # when not redirecting.

        (p2cread, p2cwrite,
         c2pread, c2pwrite,
         errread, errwrite) = self._get_handles(stdin, stdout, stderr) # refer to Note 2

        self._execute_child(args, executable, preexec_fn, close_fds,
                            cwd, env, universal_newlines,
                            startupinfo, creationflags, shell,
                            p2cread, p2cwrite,
                            c2pread, c2pwrite,
                            errread, errwrite) # launch the sub process. Refer to Note 3

        if mswindows:
            if p2cwrite is not None:
                p2cwrite = msvcrt.open_osfhandle(p2cwrite.Detach(), 0)
            if c2pread is not None:
                c2pread = msvcrt.open_osfhandle(c2pread.Detach(), 0)
            if errread is not None:
                errread = msvcrt.open_osfhandle(errread.Detach(), 0)

        # open fds of p2cwrite, c2pread, errread
        if p2cwrite is not None:
            self.stdin = os.fdopen(p2cwrite, 'wb', bufsize)
        if c2pread is not None:
            if universal_newlines:
                # if universal_newlines is indicated
                # tips about universal_newlines support:
                # In addition to the standard fopen() values mode may be 'U' or 'rU'. 
                # Python is usually built with universal newline support; 
                # supplying 'U' opens the file as a text file, but lines may be terminated by any of the following: 
                # the Unix end-of-line convention '\n', the Macintosh convention '\r', or the Windows convention '\r\n'. 
                # All of these external representations are seen as '\n' by the Python program. 
                # If Python is built without universal newline support a mode with 'U' is the same as normal text mode. 
                # Note that file objects so opened also have an attribute called newlines which has a value of None (if no newlines have yet been seen), '\n', '\r', '\r\n', or a tuple containing all the newline types seen.
                self.stdout = os.fdopen(c2pread, 'rU', bufsize) 
            else:
                self.stdout = os.fdopen(c2pread, 'rb', bufsize)
        if errread is not None:
            if universal_newlines:
                self.stderr = os.fdopen(errread, 'rU', bufsize)
            else:
                self.stderr = os.fdopen(errread, 'rb', bufsize)

Note 1:

Code of _cleanup() is like this:

def _cleanup():
    '''
    This function is used to remove the finished child process
    '''
    for inst in _active[:]: 
        if inst._internal_poll(_deadstate=sys.maxint) >= 0: # refer to Note 1-1
            # return_value >= 0, means that the child process is finished.
            try:
                _active.remove(inst) # _active is initialized to [] when module of subprocess is first present (importted).
            except ValueError:
                # This can happen if two threads create a new Popen instance.
                # It's harmless that it was already removed, so ignore.
                pass

Note 1-1:

Code of self._internal_poll() is like this, which is defined within class Popen.

For finished sub processes, this function will return an integer GE 0, while None for the unfinished sub processes.

        def _internal_poll(self, _deadstate=None):
            """Check if child process has terminated.  Returns returncode
            attribute."""
            if self.returncode is None: 
                # means the child is not finished yet, then check it for another time.
                try:
                    pid, sts = os.waitpid(self.pid, os.WNOHANG) # this will return immediately, if sub process finished, the return value is (pid, status)
                                                                # else, the return value is (0, 0)                        
                        if pid == self.pid: 
                        # if child process finished, deal with return code
                        self._handle_exitstatus(sts) # refer to Note 1-1-1
                    else: 
                        # else, do nothing, self.returncode would be None.
                        pass
                except os.error:
                    if _deadstate is not None: 
                        # if error occurs during watipid(), return the max int
                        self.returncode = _deadstate

            # return the return code directly, if the process already finished
            return self.returncode

Note 1-1-1:

Code of self._handle_exitstatus() is like the following code clips. And this function is implemented within class Popen.

If the child is terminated by a signal, then set the return code as the signal number;

if the child exits with system call exit(), then return the parameter of exit(). For example, if the exit statement is "exit(3)", then, 3 will be assigned to self.returncode.

        def _handle_exitstatus(self, sts):                      
            if os.WIFSIGNALED(sts):
                # Return True if the process exited due to a signal, otherwise return False. Availability: Unix.
                self.returncode = -os.WTERMSIG(sts) # Return the signal which caused the process to exit. Availability: Unix.
            elif os.WIFEXITED(sts):
                # Return True if the process exited using the exit(2) system call, otherwise return False. Availability: Unix.
                self.returncode = os.WEXITSTATUS(sts) # If WIFEXITED(status) is true, return the integer parameter to the exit(2) system call. Otherwise, the return value is meaningless. Availability: Unix.
            else:
                # Should never happen
                raise RuntimeError("Unknown child exit status!")

Note 2:

self._get_handles() is preparing the pipes for the sub process. 

According to the official comments:

        # Parent                   Child
        # ------                   -----
        # p2cwrite   ---stdin--->  p2cread
        # c2pread    <--stdout---  c2pwrite
        # errread    <--stderr---  errwrite

we call tell that,

p2cread is used for child to get the input from parent;

p2cwrite is used for parent to send input to child;

c2pread is used by parent to get output from chuild;

c2pwrite is used by child to send output to parent;

errread is used by parent to get error info from child;

errwirte is used by child to send error info to parent.

Code is like this:

        def _get_handles(self, stdin, stdout, stderr):
            """Construct and return tupel with IO objects:
            p2cread, p2cwrite, c2pread, c2pwrite, errread, errwrite
            """
            p2cread, p2cwrite = None, None
            c2pread, c2pwrite = None, None
            errread, errwrite = None, None

            # stdin, stdout can be:
            # None;
            # PIPE;
            # a file descirptor (a positive number);
            # a file-object; 

            # With None, no redirection will occur; the child’s file handles will be inherited from the parent
            # with PIPE, a new pipe will be craeted;

            if stdin is None:
                pass
            elif stdin == PIPE:
                p2cread, p2cwrite = os.pipe()
            elif isinstance(stdin, int): 
                p2cread = stdin 
            else:
                p2cread = stdin.fileno() # get the fd of the file-object 

            if stdout is None:
                pass
            elif stdout == PIPE:
                c2pread, c2pwrite = os.pipe()
            elif isinstance(stdout, int):
                c2pwrite = stdout
            else:
                c2pwrite = stdout.fileno()

            if stderr is None:
                pass
            elif stderr == PIPE:
                errread, errwrite = os.pipe()
            elif stderr == STDOUT:
                # stedrr has another more option
                errwrite = c2pwrite
            elif isinstance(stderr, int):
                errwrite = stderr
            else:
                # Assuming file-like object
                errwrite = stderr.fileno()

            return (p2cread, p2cwrite,
                    c2pread, c2pwrite,
                    errread, errwrite)

Note 3:

The child process is being launched by this, of which the code is:

        def _execute_child(self, args, executable, preexec_fn, close_fds,
                           cwd, env, universal_newlines,
                           startupinfo, creationflags, shell,
                           p2cread, p2cwrite,
                           c2pread, c2pwrite,
                           errread, errwrite):
            """Execute program (POSIX version)"""

            # convert the args to list
            if isinstance(args, types.StringTypes): # module of types defines names for all type symbols known in the standard interpreter.
                                                    # types.StringTypes = (<type 'int'>, <type 'unicode'>)
                                                    # if args are passed in as string, code will arrive at here
                args = [args]
            else:
                # if args are passed in as sequence, code will come here
                args = list(args)

            # handle shell
            if shell:
                args = ["/bin/sh", "-c"] + args

            # handle executable
            # consider this scenario, exectable is not indicated, means, it's None as default, and
            # the args are passed in as a string, for example, 'ls -l'
            # then what's will happen?
            # this section of code is consider 'ls -l' as the executable, and absoultly, will get error
            if executable is None:
                executable = args[0]

            # For transferring possible exec failure from child to parent
            # The first char specifies the exception type: 0 means
            # OSError, 1 means some other error.
            # this pipe is used to get errors when try to luanch the executable.
            # and after the executable is launched successfully, errread and errwrite will be used get the errors            
            errpipe_read, errpipe_write = os.pipe()
            try:
                try:
                    self._set_cloexec_flag(errpipe_write) # ?? don't konw what's this used for.

                    gc_was_enabled = gc.isenabled() # ?? don't konw why
                    # Disable gc to avoid bug where gc -> file_dealloc ->
                    # write to stderr -> hang.  http://bugs.python.org/issue1336
                    # disable gobage collection before fork a clild process and then enable it.
                    gc.disable()
                    try:
                        self.pid = os.fork() # fork a sub process
                    except:
                        if gc_was_enabled:
                            gc.enable()
                        raise
                    self._child_created = True
                    if self.pid == 0:
                        # Child
                        try:
                            # Close parent's pipe ends
                            if p2cwrite is not None:
                                os.close(p2cwrite)
                            if c2pread is not None:
                                os.close(c2pread)
                            if errread is not None:
                                os.close(errread)
                            os.close(errpipe_read)

                            # Dup fds for child
                            # This is preparing for the call of self._close_fds() if close_fds is indicated
                            if p2cread is not None:
                                os.dup2(p2cread, 0)
                            if c2pwrite is not None:
                                os.dup2(c2pwrite, 1)
                            if errwrite is not None:
                                os.dup2(errwrite, 2)

                            # Close pipe fds.  Make sure we don't close the same
                            # fd more than once, or standard fds.
                            if p2cread is not None and p2cread not in (0,):
                                os.close(p2cread)
                            if c2pwrite is not None and c2pwrite not in (p2cread, 1):
                                os.close(c2pwrite)
                            if errwrite is not None and errwrite not in (p2cread, c2pwrite, 2):
                                os.close(errwrite)

                            # Close all other fds, if asked for
                            # If close_fds is true, all file descriptors except 0, 1 and 2 will be closed before the child process is executed. (Unix only).
                            if close_fds:
                                self._close_fds(but=errpipe_write) # Refer to Note 3-1
                            
                            # change cwd if cwd is indicated
                            if cwd is not None:
                                os.chdir(cwd)

                            # run preexec_fn if it's indicated
                            if preexec_fn:
                                preexec_fn()
                                
                            # launch the executable
                            # On Unix, the new executable is loaded into the current process, and will have the same process id as the caller. 
                            # So, the executable will have the same pid with the child process
                            if env is None:
                                os.execvp(executable, args)
                            else:
                                os.execvpe(executable, args, env)

                        except:
                            exc_type, exc_value, tb = sys.exc_info() # get exception info
                            exc_lines = traceback.format_exception(exc_type,
                                                                   exc_value,
                                                                   tb) # format exception info
                            exc_value.child_traceback = ''.join(exc_lines) # ?? Why do we need attribute of child_traceback here??
                            os.write(errpipe_write, pickle.dumps(exc_value)) # pickle Exception object to string and wirte the pipe.
                                                                             # note, the errpipe_write is being used here, instead of errwite
                        # This exitcode won't be reported to applications, so it
                        # really doesn't matter what we return.
                        os._exit(255) # note the difference here between os.exit() and os._exit()

                    # Parent
                    if gc_was_enabled:
                        gc.enable()
                finally:
                    # be sure the FD is closed no matter what, and it's closed by both parent and child
                    # Parent is closing this, becuase it's no use to parent;
                    # Child is closing this, becuase it has wirotten error info to it
                    os.close(errpipe_write)

                if p2cread is not None and p2cwrite is not None:
                    os.close(p2cread) # close p2cread, which are intend for child, and leave p2cwrite to input something to child
                if c2pwrite is not None and c2pread is not None:
                    os.close(c2pwrite) # close c2pwrite, which is intend for child, and leave c2pread to get output of child
                if errwrite is not None and errread is not None:
                    os.close(errwrite) # close errwrite, which is intend for child, and leave errread to get the errors of child

                # Wait for exec to fail or succeed; possibly raising exception
                # Exception limited to 1M
                # check the status of launching exectable.
                data = _eintr_retry_call(os.read, errpipe_read, 1048576) # Refer to Note 3-2
            finally:
                # be sure the FD is closed no matter what
                # errpipe_read is only closed in parent process, becuase it's no use now
                # and the child has already closed it's own errpipe_read. 
                os.close(errpipe_read)

            # data != "" means, there is somthing wrong when launching the executable 
            # if the executable is not launched successfully, should:
            # wait sub process finishing,
            # load the exception object,
            # close the pipes,
            # and then raise error
            if data != "":
                _eintr_retry_call(os.waitpid, self.pid, 0)
                child_exception = pickle.loads(data)
                for fd in (p2cwrite, c2pread, errread):
                    if fd is not None:
                        os.close(fd)
                raise child_exception

Note 3-1:

Code of self._close_fds() is like this. And as you see, this function will close all the fds except 0, 1, 2 and the one indicated by "but"

        def _close_fds(self, but):
            os.closerange(3, but) # close all fds fo [3, but), but will not be closed.
            os.closerange(but + 1, MAXFD)

Note 3-2:

Code of _eintr_retry_call() is like this. Not very sure about this function, but looks like its behavior is like this:

If a system call (if the func is a system call) is interrupted, then try again, till it finishes or other error occurs.

def _eintr_retry_call(func, *args):
    while True:
        try:
            return func(*args)
        except OSError, e:
            if e.errno == errno.EINTR:
                # errno.EINTR means interupted system call why this??
                continue
            raise

Other functions except the initialization function are very easy:

self.poll()

    def poll(self):
        return self._internal_poll() # Refer to Note 1-1

self.wait()

        def wait(self):
            """Wait for child process to terminate.  Returns returncode
            attribute."""
            if self.returncode is None:
                pid, sts = _eintr_retry_call(os.waitpid, self.pid, 0) # please note, the parent should be blocked here. Refer to Note 3-2
                self._handle_exitstatus(sts) # Refer to Note 1-1-1
            return self.returncode

self.send_signal, self.terminate and self.kill():

        def send_signal(self, sig):
            """Send a signal to the process
            """
            os.kill(self.pid, sig) # Send signal sig to the process pid. 
                                   # Constants for the specific signals available on the host platform are defined in the signal module. 
                                   # Availability: Unix.
                                   
        def terminate(self):
            """Terminate the process with SIGTERM
            """
            self.send_signal(signal.SIGTERM)

        def kill(self):
            """Kill the process with SIGKILL
            """
            self.send_signal(signal.SIGKILL)

To Be Continued.