#Copyright (c) 2012, 2019, Oracle and/or its affiliates. All rights reserved.
#
#This program is free software; you can redistribute it and/or modify it under the terms of the GNU
#General Public License as published by the Free Software Foundation; version 2 of the License.
#
#This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without
#even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
#General Public License for more details.
#
#You should have received a copy of the GNU General Public License along with this program; if not,
#write to the Free Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA

"""Tools for performing tasks on (possibly remote) Cluster hosts."""

import time
import posixpath
import ntpath
import abc
import os
import socket
import subprocess
import tempfile
import shutil
import logging
import platform
import json
import shlex

import util
from util import bcolors

import request_handler

_logger = logging.getLogger(__name__)
# Lock for executing commands fast on localhost.
# CMD_LOCK = threading.Lock()
class ExecException(Exception):
    """Exception type thrown when process-spawning fails on the local host. """

    def __init__(self, cmd, exitstatus, out):
        self.cmd = cmd
        self.exitstatus = exitstatus
        self.out = out.read()
    def __str__(self):
        return bcolors.FAIL + 'Command ' + self.cmd + ' exited with ' + self.exitstatus + ':\n' + \
            self.out + bcolors.ENDC.format

class HostInfo(object):
    """Class which provides host information from a Linux-style /proc file system."""
    def __init__(self, ch, uname, machine, osver, osflavor):
        self.ch = ch
        self.pm = posixpath
        self.uname = uname
        self.machine = machine
        self.envcmd = ['env']
        self.osver = osver
        self.osflavor = osflavor

    @property
    def _host_info_path(self):
        return self.path_module.join(request_handler.CONFIGDIR, 'host_info', 'binaries', self.uname,
                                     self.machine, 'host_info')

    def _run_host_info(self):
        host_res = json.loads(self.ch._exec_pkg_cmdv([self._host_info_path]))
        host_res['uname'] = self.uname
        return {'host': {'name' : self.ch.host}, 'hostRes': host_res}

    @property
    def ram(self):
        """Caching?"""
        with self.ch.open('/proc/meminfo') as meminfo:
            return int(meminfo.readline().split()[1]) // 1024

    @property
    def cores(self):
        """x"""
        with self.ch.open('/proc/cpuinfo') as cpuinfo:
            return len([ln for ln in cpuinfo.readlines() if 'processor' in ln])

    @property
    def installdir(self):
        """x"""
        return None # Don't know how to get this for remote hosts

    @property
    def homedir(self):
        """x"""
        return self.ch.env['HOME']

    @property
    def path_module(self):
        """Returns the python path module to use when manipulating path names on this host."""
        return self.pm

    @property
    def disk_free(self):
        """Returns the free space for homedir on host."""
        hd = self.ch.env['HOME']
        dtest = self.ch.exec_blocking(['df', '-h', hd])

        #There is a chance there will be an error output from opening the console.
        # grab df output, skip to last line, go one back and grab Avail (4th).
        lc = dtest.count('\n')
        #There has to be 2 lines of df output + errors from console (if any).
        if lc > 1:
            ds = str(((dtest).split('\n')[lc-1]).split()[3:4]).strip('[]').strip("''")
            if len(ds) >= 2:
                return ds
            wrn_msg = 'CHS--> Length of parsed df output was less than 2 characters wide.'
            _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC)
            return 'unknown'
        else:
            wrn_msg = 'CHS--> df output had less than 2 lines.'
            _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC)
            return 'unknown'

    @property
    def docker_info(self):
        """Returns NOT INSTALLED, NOT RUNNING or RUNNING for Docker."""
        # We will ask for this info if necessary. IF docker is requested and it's NOT INSTALLED, we
        # will install and run docker. IF docker NOT RUNNING we will start it. IF docker is RUNNING,
        # front-end will do nothing. All *nix platforms share same code.
        d_test = None
        try:
            d_test = self.ch.exec_blocking(['docker', '-v'])
            _logger.debug('CHS--> docker -v finished with dtest = %s', d_test)
        except:
            _logger.debug('CHS--> docker -v failed with dtest = %s', d_test)
            d_test = ""
        #We ignore ver. info for now and send new request for result of command.
        if d_test != "":
            #Now check if docker is running.
            try:
                d_test = self.ch.exec_blocking(['ps', '-e', '-o', 'comm', '|', 'grep', '[d]ocker'])
                _logger.debug('CHS--> ps finished with dtest = %s', d_test)
            except:
                wrn_msg = 'CHS--> ps failed with dtest = %s'
                _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC, d_test)
                d_test = ""
            if d_test != "":
                return 'RUNNING'
            return 'NOT RUNNING'
        return 'NOT INSTALLED'

    @property
    def intip(self):
        """Returns the internal IP address of the host. Should work for all but Mac and Win."""
        _logger.debug('CHS--> Running ifconfig')
        d_test = None
        if isinstance(self.ch, LocalClusterHost):
            # localhost providing addressed via external IP can still be part of Cluster
            # requiring InternalIP
            d_test = subprocess.Popen([
                r'ifconfig -a | grep -A 1 "RUNNING" | grep -v "LOOPBACK" | grep -v "127\.0\.0\.1"' +
                r' | grep -oP "(?<=inet\s)\d+(\.\d+){3}"'], stdout=subprocess.PIPE, shell=True)
            dt = d_test.communicate()[0]
            return str(dt).split("\n")
        else:
            d_test = self.ch.exec_blocking(
                ['/usr/sbin/ifconfig', '-a', '|', 'grep', '-A', '1', '"RUNNING"', '|', 'grep', \
                 '-v', '"LOOPBACK"', '|', 'grep', '-v', r'"127\.0\.0\.1"', \
                 '|', 'grep', '-oP', r'"(?<=inet\s)\d+(\.\d+){3}"'])

            if d_test is None:
                _logger.warning(bcolors.WARNING + 'CHS--> ifconfig failed.' + bcolors.ENDC)
                return ''
            lc = d_test.count('\n')
            # Oracle cloud machines return single internal IP address! There is 1+ ip
            # addresses, return that and let FE sort things out.
            if lc >= 1:
                return d_test
            wrn_msg = 'CHS--> ifconfig output had less than 2 lines.'
            _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC)
            return ''

class  SolarisHostInfo(HostInfo):
    """Specialization for Solaris which uses prtconf and psrinfo to get host information."""

    @property
    def ram(self):
        return int(self.ch.exec_blocking(['/usr/sbin/prtconf']).split()[7])

    @property
    def cores(self):
        return len(
            self.ch.exec_blocking(['/usr/sbin/prtconf']).split('\n')[0:-1])

    @property
    def intip(self):
        return "unknown"

class MacHostInfo(HostInfo):
    """Specialization for MacOS which uses sysctl to get host information."""

    @property
    def ram(self):
        sysinfo = self.ch.exec_blocking(['/usr/sbin/sysctl', 'hw.'])
        return [int(filter(str.isdigit, ln.split()[1])) for ln \
            in sysinfo.split('\n') if 'hw.memsize:' in ln][0] // 1024 // 1024

    @property
    def cores(self):
        sysinfo = self.ch.exec_blocking(['/usr/sbin/sysctl', 'hw.'])
        return [int(filter(str.isdigit, ln.split()[1])) for ln in \
            sysinfo.split('\n') if 'hw.ncpu:' in ln][0]

    @property
    def intip(self):
        d_test = None
        dt = []

        if isinstance(self.ch, LocalClusterHost):
            _logger.debug('CHS--> ifconfig local Mac')
            d_test = subprocess.Popen(
                ["ifconfig -a | grep -A 4 'RUNNING' | grep 'inet' | grep -v" +
                 r" 'LOOPBACK' | grep -v '127\.0\.0\.1' | grep -v 'inet6' | " +
                 "awk '''{ print $2 }'''"], stdout=subprocess.PIPE, shell=True)
            dt = d_test.communicate()[0]
            return str(dt).split("\n")
        else:
            d_test = self.ch.exec_blocking(
                ['ifconfig', '-a', '|', 'grep', '-A', '4', '"RUNNING"', '|', 'grep', '"inet"',\
                 '|', 'grep', '-v', '"LOOPBACK"', '|', 'grep', '-v', r'"127\.0\.0\.1"', '|',\
                 'grep', '-v', '"inet6"'])
            if d_test is None:
                _logger.warning(bcolors.WARNING + 'CHS--> ifconfig failed.' + bcolors.ENDC)
                return ''
            lc = d_test.count('\n')
            dt1 = []
            if lc >= 1:
                dt1 = d_test.split('\n')
                for x in xrange(len(dt1)):
                    if 'inet ' in dt1[x]:
                        _logger.debug('CHS--> Appending intIP:%s',
                                      dt1[x].lstrip().split('inet ')[1].split(' ')[0])
                        dt.append(dt1[x].lstrip().split('inet ')[1].split(' ')[0])
                return dt
            wrn_msg = 'CHS--> ifconfig output had less than 2 lines.'
            _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC)
            return ''
class CygwinHostInfo(HostInfo):
    """Specialization for Cygwin which uses systeminfo and wmic to retrieve host information, but
     retains posixpath as the path module."""

    @property
    def _host_info_path(self):
        return self.path_module.join('install', 'host_info', 'Windows', 'host_info.exe')

    @property
    def ram(self):
        cmdv = ['cmd.exe', '/c', 'wmic', 'ComputerSystem', 'get', 'TotalPhysicalMemory', '/value']
        rd = self.ch.exec_blocking(cmdv).split("\r") # TotalPhysicalMemory=34190766080
        ram = ""
        for t in rd:
            if t.strip() != "":
                if t.find("TotalPhysicalMemory") != -1:
                    ram = t.split("TotalPhysicalMemory=")[1]
                    break
        return int(ram) // 1024 // 1024 # Return MB same as systeminfo would.

    @property
    def cores(self):
        """Returns number of logical CPU's available on box."""
        wmic = self.ch.exec_blocking(['C:/Windows/System32/Wbem/wmic', 'CPU', 'GET', '/VALUE'])
        return sum(
            [int(ln.split('=')[1]) for ln in wmic.split('\n') if 'NumberOfLogicalProcessors' in ln])

    @property
    def docker_info(self):
        """Returns state (NOT INSTALLED, NOT RUNNING and RUNNING) of docker."""
        cmdv = ['C:/Windows/System32/sc.exe', 'queryex', 'com.docker.service']
        # This one is expected to fail mostly with 1060:
        # The specified service does not exist as an installed service.
        d_test = self.ch.exec_cmdv(cmdv, {'noRaise': 1060, 'waitForCompletion': True}, None)
        if len(d_test) > 10:
            if d_test.find("STATE") != -1:
                gds = d_test.split("STATE              :")[1].split("WIN32_EXIT_CODE")[0].strip()
                if gds.find("4  RUNNING") != -1:
                    return "RUNNING"
                return "NOT RUNNING"
            else:
                return "NOT INSTALLED"
        else:
            # Actually this is error in sc but the result is the same.
            return 'NOT INSTALLED(error)'

    @property
    def intip(self):
        mt = self.ch.exec_blocking(['C:/Windows/System32/ipconfig', '/all'])
        if len(mt) > 6:
            tmp = [ln for ln in mt.split("\r\n") if "IPv4" in ln]
            return [ln.split(":")[1].replace("(Preferred)", "").strip() for ln in tmp]
        return "unknown"

class WindowsHostInfo(CygwinHostInfo):
    """Specialization of CygwinHostInfo for native Windows which uses ntpath as the path module."""

    def __init__(self, ch, uname, machine, osver, osflavor):
        super(type(self), self).__init__(ch, uname, machine, osver, osflavor)
        self.pm = ntpath
        self.envcmd = ['cmd.exe', '/c', 'set']

    @property
    def homedir(self):
        en = self.ch.env
        if en.has_key('USERPROFILE'):
            return en['USERPROFILE']
        return en['HOMEDRIVE']+en['HOMEPATH']

    @property
    def path_module(self):
        return ntpath

    @property
    def disk_free(self):
        en = self.ch.env
        if en.has_key('USERPROFILE'):
            hd = en['USERPROFILE']
        else:
            hd = en['HOMEDRIVE']+en['HOMEPATH']

        mt = (self.ch.exec_blocking(['C:/Windows/System32/Wbem/wmic', 'logicaldisk', hd[:2],
                                     'GET', 'freespace'])).split('\n')[1]
        s = ""
        for i in range(len(mt)-1):
            if ord(mt[i]) >= 48:
                s += str(mt[i])
        if len(s) < 2:
            return 'unknown'
        try:
            return str(int(s) // 1024 // 1024 // 1024) + 'G'
        except (TypeError, ValueError): # hope that's all
            return 'unknown'

    @property
    def intip(self):
        """ Returns (Array of) internal IP address(es)."""
        # This should be it:
        mt = self.ch.exec_blocking(['C:/Windows/System32/ipconfig', '/all'])
        if len(mt) > 6:
            tmp = [ln for ln in mt.split("\r\n") if "IPv4" in ln]
            return [
                ln.split(":")[1].replace("(Preferred)", "").strip() for ln in tmp]
        return "unknown"

# Map from uname string to HostInfo type
HOST_INFO_MAP = {'SunOS' : SolarisHostInfo,
                 'Darwin' : MacHostInfo,
                 'Windows' : WindowsHostInfo,
                 'CYGWIN' : CygwinHostInfo}

class ABClusterHost(object):
    """Base class providing common interface."""
    __meta_class__ = abc.ABCMeta

    def __enter__(self):
        return self

    def __exit__(self, exc_type, exc_val, exc_tb):
        self.drop()
        return False

    def __init__(self):
        self._hostInfo = None
        self._env = None

    @abc.abstractmethod
    def _get_system_tuple(self):
        """Return a string identifying the hosts operating system."""
        pass

    @abc.abstractmethod
    def _exec_pkg_cmdv(self, cmdv):
        """Execute a package binary on this ClusterHost."""
        pass

    @abc.abstractmethod
    def open(self, filename, mode='r'):
        """Abstract"""
        pass

    @abc.abstractmethod
    def get(self, host, filename, localname):
        """Abstract"""
        pass

    @abc.abstractmethod
    def put(self, host, localname, filename):
        """Abstract"""
        pass

    @property
    def hostInfo(self):
        """Create an appropriate HostInfo object for localhost. Uses Pythons platform.system() to
         determine the type of HostInfo object to return."""

        if self._hostInfo is None:
            (system, machine, osver, osflavor) = self._get_system_tuple()
            if HOST_INFO_MAP.has_key(system):
                self._hostInfo = HOST_INFO_MAP[system](self, system, machine, osver, osflavor)
            else:
                dbg_msg = 'CHS--> Using default HostInfo for host %s (%s, %s)'
                _logger.debug(dbg_msg, self.host, system, machine)
                return HostInfo(self, system, machine, osver, osflavor)

        return self._hostInfo

    @property
    def path_module(self):
        """Path module to use when manipulating file paths."""
        return self.hostInfo.path_module

    @property
    def env(self):
        """environment"""
        if not self._env:
            ctx = {'str': self.exec_blocking(self.hostInfo.envcmd), 'properties': {}}
            util.parse_properties(ctx)
            self._env = ctx['properties']

        return self._env

    @property
    def ram(self):
        """Returns total amount of RAM in MB"""
        return self.hostInfo.ram

    @property
    def cores(self):
        """Returns how many logical processors host has."""
        return self.hostInfo.cores

    @property
    def uname(self):
        """UNAME for host."""
        return self.hostInfo.uname

    @property
    def installdir(self):
        """Where to find MySQL executables"""
        return self.hostInfo.installdir

    @property
    def homedir(self):
        """HOME for logged in user"""
        return self.hostInfo.homedir

    @property
    def disk_free(self):
        """Free disk space on HOME partition or 'unknown'."""
        return self.hostInfo.disk_free

    @property
    def docker_info(self):
        """Return Docker service status on host; 'RUNNING', 'NOT RUNNING' or 'NOT INSTALLED'"""
        return self.hostInfo.docker_info

    @property
    def intip(self):
        """Depending on how many IP addresses various NET commands filtered out, we return single
         or array of IP addresses deemed as "Internal"""
        return self.hostInfo.intip

    @abc.abstractmethod
    def drop(self, paths=[]):
        """Close open connections and remove files.
        paths - list of files to remove from host before closing connection
        """
        map(self.rm_r, paths)

    @abc.abstractmethod
    def file_exists(self, path):
        """Test for the existence of a file. If the file actually exists, its stat object is
         returned, otherwise None.
        path - file to check the existence of
        """
        pass

    @abc.abstractmethod
    def list_dir(self, path):
        """List the files in a directory.
        path - directory to list
        """
        pass

    @abc.abstractmethod
    def mkdir_p(self, path):
        """Provides mkdir -p type functionality. I.e, missing parent directories are also created.
        If the directory we are trying to create already exists, we silently do nothing. If path or
         any of its parents is not a directory an exception is raised.
        path - directory to create on remote host """
        pass

    @abc.abstractmethod
    def rm_r(self, path):
        """Provides rm -r type functionality. All files and directories are removed recursively.
        path - file or directory to remove."""
        pass

    def auto_complete(self, basedir, locations, executable):
        """Find the absolute path of an executable given a prefix directory, a set of possible
         directories, and the basename of the executable.
        basedir - basedir of a cluster installation
        locations - list of directories to try
        executable - basename of executable to auto-complete"""
        for l in locations:
            choice = posixpath.join(basedir, l, executable)
            _logger.debug('Testing if %s exists.', choice)
            if self.file_exists(choice):
                _logger.debug('%s exists.', choice)
                return choice
            else:
                _logger.debug('%s doesn\'t exist.', choice)
        raise Exception('Cannot locate '+executable+' in '+
                        posixpath.join(basedir, str(locations)) + ' on host ' + self.host)

    @abc.abstractmethod
    def _exec_cmdv(self, cmdv, procCtrl, std_in_file):
        pass

    def _execfast(self, cmdv):
        pass

    def execfast(self, cmdv):
        """Forwards to virtual."""
        return self._execfast(cmdv)

    def exec_cmdv(self, cmdv, procCtrl={'waitForCompletion': True}, std_in_file=None):
        """Forwards to virtual."""
        return self._exec_cmdv(cmdv, procCtrl, std_in_file)

    def exec_blocking(self, cmdv):
        """Convenience method."""
        assert(isinstance(cmdv, list))
        return self.exec_cmdv(cmdv, {'waitForCompletion': True})

    def exec_cluster_daemon(self, cluster_daemon, waitsec):
        """Convenience method."""
        assert(isinstance(cluster_daemon, list))
        return self.exec_cmdv(cluster_daemon, {'daemonWait': waitsec})

class LocalClusterHost(ABClusterHost):
    """Implement the ABClusterHost interface for access to the localhost without using SSH over
     Paramiko. Note that this implies that there will be no authentication."""
    def __init__(self, host):
        super(type(self), self).__init__()
        self.host = host
        self.osver = None
        self.osflavor = None
        self.hlpstr = None
        self.system = None

    def _get_system_tuple(self):
        if self.system is not None:
            return (self.system, self.hlpstr, self.osver, self.osflavor)

        osver = None
        osflavor = None
        hlp_str = None
        system = platform.system()
        if system == 'Windows':
            try:
                subprocess.check_call(['uname'])
            except WindowsError:
                _logger.debug('CHS--> No uname available, assuming native Windows')
                # systeminfo slows things down significantly.
                m = platform.platform()
                #system = m.split("-")[0]
                osver = m.split("-")[1]
                osflavor = "Microsoft " + system + " " + osver
                self.osver = osver
                self.osflavor = osflavor
                self.hlpstr = platform.uname()[-2]
                self.system = system
                return (system, platform.uname()[-2], osver, osflavor)
            else:
                self.osver = osver
                self.osflavor = osflavor
                self.hlpstr = 'Unknown'
                self.system = 'CYGWIN'
                return ('CYGWIN', 'Unknown', osver, osflavor)
        #System is either Linux, SunOS or Darwin
        if system.startswith('Darwin'):
            osver = str(self.exec_blocking(['uname', '-r']))
            osflavor = "MacOSX"

        if system.startswith("SunOS"): #or system.startswith("Solaris"):
            osver = str(self.exec_blocking(['uname', '-v']))
            osflavor = "Solaris"

        if "Linux" in system:
            system = "Linux"
            # Assumption is all Linux flavors will have /etc/os-release file.
            if self.file_exists('/etc/os-release'):
                hlp_str = self.exec_blocking(['test', '-f', '/etc/os-release'])
            else:
                hlp_str = "0"
            if hlp_str != "0":
                hlp_str = self.exec_blocking(['cat', '/etc/os-release'])
                matched_lines = [line for line in hlp_str.split('\n') if "ID=" in line]
                hlp = (str(matched_lines[0]).split("ID=", 1)[1]).strip('"')
                osflavor = hlp
                matched_lines = [line for line in hlp_str.split('\n') if "VERSION_ID=" in line]
                hlp = (str(matched_lines[0]).split("VERSION_ID=", 1)[1]).strip('"')
                osver = hlp
            else:
                #Bail out, no file
                wrn_msg = 'CHS--> %s, %s, %s does not have /etc/os-release file!'
                _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC, system, osflavor, osver)

        self.osver = osver
        self.osflavor = osflavor
        self.hlpstr = platform.uname()[-1]
        self.system = system
        return (system, platform.uname()[-1], osver, osflavor)

    def _exec_pkg_cmdv(self, cmdv):
        """Locally this just forwards to exec_cmdv."""
        return self.exec_cmdv(cmdv)

    @property
    def env(self):
        return os.environ

    @property
    def cores(self):
        return self.hostInfo.cores

    @property
    def installdir(self):
        system = platform.system()
        if system == 'Windows':
            # On Windows, there is no path auto-complete and the startup script (ndb_setup.py)
            # returns BASEDIR so we need to add \bin.
            pat = request_handler.BASEDIR
            if not "\bin" in pat or not "/bin" in pat:
                pat = os.path.join(pat, 'bin')
            return pat
        else:
            return request_handler.BASEDIR

    def open(self, filename, mode='r'):
        """Open a file on ABClusterHost."""
        return open(filename, mode)

    def get(self, filename, localname):
        """Get a file on ABClusterHost and save to ~/.mcc."""
        #File names are complete with path.
        return shutil.copy(filename, localname)

    def put(self, localname, filename):
        """Copy file."""
        #File names are complete with path.
        return shutil.copy(localname, filename)

    def drop(self, paths=[]):
        """Close open connections and remove files.
        paths - list of files to remove from host before closing connection
        """
        map(self.rm_r, paths)

    def file_exists(self, path):
        """Test for the existence of a file on the local host. If the file
        actually exists, its stat result object is returned, otherwise None.
        path - file to check the existence of
        """
        if os.path.exists(path):
            return os.stat(path)
        return None

    def list_dir(self, path):
        """List the files in a directory on the local host. Forwards to os.listdir().
        path - directory to list
        """
        return os.listdir(path)

    def mkdir_p(self, path):
        """Provides mkdir -p type functionality on localhost. Does nothing if
        the directory already exists, otherwise forwards to os.makedirs.
        path - directory to create on remote host
        """
        if os.path.exists(path) and os.path.isdir(path):
            return
        os.makedirs(path)

    def rm_r(self, path):
        """Provides rm -r type functionality on localhost. Forwards to os.rmdirs.
        path - file or directory to remove
        """
        shutil.rmtree(path)

    def _execfast(self, cmdv):
        """Quick and dirty exec for localhost. Not sure locking is even needed."""
        # global CMD_LOCK
        # with CMD_LOCK:
        proc = subprocess.Popen(cmdv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT,
                                shell=True)
        out = proc.communicate()[0]
        rc = proc.returncode
        if rc == 0:
            return out
        return 'errcode:'+str(rc)+'\n' + out

    def _exec_cmdv(self, cmdv, procCtrl, stdin_file):
        """Execute an OS command on the local host, using subprocess module.
        cmdv - complete command vector (argv) of the OS command
        procCtrl - procCtrl object from message which controls how the process
        is started (blocking vs non-blocking and output reporting)
        """
        # Add nohup when running locally to prevent server from waiting on the children
        if util.get_val(procCtrl, 'nohup'):
            cmdv[:0] = ['nohup']
        stdin = None
        if stdin_file != None:
            stdin = self.open(stdin_file)

        #We need Popen to have all possible FE commands to work. Using single mechanism to run all
        # commands adds to readability. I am hoping to reduce number of exec_cmd functions too :-/
        # Several possible cases:
        # 1) !waitForCompletion and !daemonWait: Just fire and forget.
        # 2) waitForCompletion and !daemonWait: Wait for result.
        # 3) waitForCompletion and daemonWait: Wait for result but poll after daemonWait seconds.
        # 4) !waitForCompletion and daemonWait: Wait for result only daemonWait seconds.
        # Old code was partial 2) and 3)+4).

        #New code:
        # Wait for daemonWait AND MAX 120 seconds for command to return but check if it finished
        # every second (after waiting daemonWait).

        #Rationale: We have commands that can not return immediately even on localhost. For such
        # commands we set daemonWait member of procControl to appropriate amount of seconds (say 2)
        # forcing this exec routine to always wait that long for response. After that, we allow
        # maximum of 120s for command to complete before we kill it. If command does not have
        # procControl.daemonWait this means we expect entire process to finish as soon as command
        # returns. For such commands, we wait max 120s.
        proc_DW_tot = 120
        proc_DW = 0
        if procCtrl.has_key('daemonWait'):
            proc_DW = util.get_val(procCtrl, 'daemonWait')
        tst = ""
        if isinstance(cmdv, list):
            tst = ' '.join(cmdv)
        else:
            tst = str(cmdv)
        _logger.debug("CHS--> daemonWait is %s", proc_DW)
        _logger.debug("CHS--> cmdv %s", tst)
        # hasKill = procCtrl.has_key('kill'). Commands with Kill member have different processing.
        kill_it = True
        try:
            if isinstance(cmdv, list):
                proc = subprocess.Popen(cmdv, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
            else:
                proc = subprocess.Popen(shlex.split(cmdv), stdout=subprocess.PIPE,
                                        stderr=subprocess.STDOUT)
            # There are commands we have to wait for to start with, such as NET START/STOP etc.
            _logger.debug("CHS--> proc %s executed", tst)
            if proc_DW > 0:
                _logger.debug("CHS--> proc %s executed, sleeping", tst)
                time.sleep(proc_DW)
            #We have waited set amount of time, now see if it returns in 2min and if not, kill it.
            for t in xrange(proc_DW_tot):
                time.sleep(0.5)
                if proc.poll() is not None:
                    kill_it = False
                    _logger.debug("CHS--> %ss, breaking for %s", t/2, tst)
                    if proc_DW > 0:
                        dbg_msg = "CHS--> Waited %ssec (on top of %ssec) for %s to complete."
                        _logger.debug(dbg_msg, t/2, proc_DW, tst)
                    else:
                        _logger.debug("CHS--> Waited %s seconds for %s to complete.", t/2, tst)
                    break

            if kill_it:
                _logger.warning(bcolors.WARNING + "CHS--> killing %s" + bcolors.ENDC, tst)
                proc.kill()
                retcode = proc.returncode
                if retcode is None:
                    retcode = 1
                rd = proc.stdout.read()
                wrn_msg = "CHS--> killing, raising %s rc %s, msg %s"
                _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC, tst, retcode, rd)
                raise ExecException(tst, retcode, proc.stdout)
            else:
                retcode = proc.returncode
                rd = proc.stdout.read()
                if retcode is None: #REALLY BAD except if it's meant to be so, such as with KILL -9
                    retcode = retcode or -1
                    rd = rd or "No output from command."
                    wrn_msg = "CHS--> killing, raising %s rc %s, msg %s"
                    _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC, tst, retcode, rd)
                    raise ExecException(tst, proc.returncode, proc.stdout)
                elif retcode == 0  or (retcode != 0 and \
                    retcode == util.get_val(procCtrl, 'noRaise')):
                    _logger.debug("CHS--> Conditional succ. noRise:%s.", retcode or -1)
                    # When successful, cut it to command and 100 characters of response. To remove
                    # just empty lines and not lines with spaces:
                    #"".join([s for s in t.strip().splitlines(True) if s.strip("\r\n")])
                    rd = rd or "No output from command." # rd can be None, for example for KILL -9
                    _logger.debug("CHS--> Succ. %s(%s), %s...", tst, retcode, \
                        "".join([t for t in rd.strip().splitlines(True) if t.strip()])[0:100])
                    return rd
                elif retcode <= 125:
                    wrn_msg = "CHS--> %s failed, exit-code=%d error = %s"
                    _logger.warning(bcolors.WARNING + wrn_msg + bcolors.ENDC, tst, retcode, rd)
                    return 'errcode:'+str(retcode)+'\n' + rd
                elif retcode == 127:
                    wrn_msg = bcolors.WARNING + "CHS--> %s, program not found: %s" + bcolors.ENDC
                    _logger.warning(wrn_msg, tst, rd)
                    return 'errcode:'+str(retcode)+'\n' + rd
                else:
                    # Things get hairy and unportable - different shells return
                    # different values for coredumps, signals, etc.
                    rd = rd or "No output from command." # rd can be None
                    _logger.warning(bcolors.WARNING + "CHS--> %s, program reported OS-dependant "
                                    "exit-code=%d error = %s" + bcolors.ENDC, tst, retcode, rd)
                    return 'errcode:'+str(retcode)+'\n' + rd
        finally:
            if stdin is not None:
                stdin.close()

    def execute_command(self, cmdv, inFile=None):
        """Execute an OS command blocking on the local host, using subprocess module. Returns dict
        containing output from process.
        cmdv - complete command vector (argv) of the OS command."""
        out_file = tempfile.TemporaryFile()
        err_file = tempfile.TemporaryFile()
        # This will fail if command needs shell-expansion. Also, no timeout.
        result = {
            'exitstatus': subprocess.call(args=cmdv, stdin=inFile, stdout=out_file, stderr=err_file)
            }
        out_file.seek(0)
        err_file.seek(0)
        result['out'] = out_file.read()
        result['err'] = err_file.read()
        _logger.debug('CHS--> execute_command %s on localhost, result %s', ' '.join(cmdv), result)
        return result

def produce_ABClusterHost(hostname='localhost', key_based=None, user=None, pwd=None, key_file=None):
    """Factory method which returns RemoteClusterHost or LocalClusterHost depending on the value of
     hostname.."""
    _logger.debug('CHS--> Produce ABC, hostname passed %s', hostname)
    shn = socket.gethostbyname_ex(socket.gethostname())
    #Say ('host.dom.comp.com', [], ['10.17.16.24'])
    _logger.debug('CHS--> Produce ABC socket host name is %s.', shn)
    if hostname == 'localhost' or hostname == '127.0.0.1' or hostname == shn[0]:
        _logger.debug('CHS--> Host is local.')
        return LocalClusterHost(hostname)

    for x in range(len(shn[2])):
        if hostname == shn[2][x]:
            return LocalClusterHost(hostname)

    # Sanitize input:
    if not (user and user.strip()):
        user = None
    if not (pwd and pwd.strip()):
        pwd = None
    else:
        if pwd.startswith('**'):
            pwd = None
    if not (key_file and key_file.strip()):
        key_file = None
    import remote_clusterhost
    _logger.debug('CHS--> Producing remote ABC.')
    return remote_clusterhost.RemoteClusterHost(hostname, key_based, user, pwd, key_file)