diff --git a/.travis.yml b/.travis.yml index 39a6a06..4152079 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,24 @@ language: python +sudo: required +services: + - docker python: - - "2.6" - - "2.7" - - "3.4" +- '2.7' +- '3.4' +- '3.5' +- '3.6' install: - - pip install tox +- pip install tox tox-travis codecov script: - - tox +- tox +after_success: +- codecov +deploy: + provider: pypi + user: lukas-bednar + password: + secure: AlKh1Zv2+PjPa5W2RhHT0OtECmmNzDpSr+sWCDBFUnuZFiCLqXiD+aMxLCIbq0cndODhOd3MMXu3lKxNtmO5/7zUrSkHG0VRsqBj7yNabzJgp+rLX1FmKaw/20IuiNVhV31uh3MU1sHPBDNoU9S67SCiKk/L/k6t2wFTbac6gyiJ5D8HD4bukyhbGRf4GIFlibHLCq6GvSTCqEdSSnCVF/jSai4r5C7MicDpc3Osxud2EohnbcRT9aujKPxYzoTmworF2ZgIZqOUHoBo2oC8GdJlNOYdx/pBFYipMFxuWAHwZGiywxnob0ORgd54UxvTcvbXCIBPv4fjgrJRsJzoB2bH6QtG48RyCQIFrNOSDr/bPNP3XnvzG3yeAZ/Mxy50i9O4YhqgYum/VUBzvdU6SqFaDc/3FXt8dCz9AlPW1f2NMaElHREUJCXH17BjP1ziccUse7AhunHqMMLtIbuLAT8vZe0tJIAwgB/MSn9fFm4ME2oEeVRW4s8LVDIgc2LLKJ4AxwaZXfL6G8BgRV26EHgbPLwDMKDDKtaUL8KcY6TtTUx3pwDFrL+hOkhfE0Q302t6GI0UqG1t0pRweVfgpDHni4ReaHjDNXQqhKmMwrTRIQMVXPKoD3oHZMAyRtJW85e55aKVIO7iEEvhJCstU7tVmkCsJZcfFU//H+i9mVQ= + on: + branch: master + tags: true + skip_upload_docs: true diff --git a/README.md b/README.md deleted file mode 100644 index 1431093..0000000 --- a/README.md +++ /dev/null @@ -1,133 +0,0 @@ -# python-rrmngmnt -Remote Resources MaNaGeMeNT -[![Build Status](https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt.svg?branch=master)](https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt) - -## Intro -This tool helps you manage remote machines and services running on that. -It is targeted to Linux based machines. All is done via SSH connection, -that means SSH server must be running there already. -```python -from rrmngmnt import Host, RootUser - -h = Host("10.11.12.13") -h.users.append(RootUser('123456')) -exec = h.executor() -print exec.run_cmd(['echo', 'Hello World']) -``` - -## Features -List of provided interfaces to manage resources on machine, and examples. - -### Filesystem -Basic file operations, you can find there subset of python 'os' module related -to files. -```python -print h.fs.exists("/path/to/file") -h.fs.chown("/path/to/file", "root", "root") -h.fs.chmod("/path/to/file", "644") -h.fs.unlink("/path/to/file") -``` - -### Network -It allows to manage network configuration. -```python -print h.network.hostname -h.network.hostname = "my.machine.org" -print h.network.all_interfaces() -print h.network.list_bridges() -``` - -### Package Management -It encapsulates various package managements. It is able to determine -which package management to use. You can still specify package management -explicitly. - -Implemented managements: - - * APT - * YUM - * DNF - * RPM - -```python -# install htop package using implicit management -h.package_management.install('htop') -# remove htop package using rpm explicitly -h.package_management('rpm').remove('htop') -``` - -### System Services -You can toggle system services, it encapsulates various service managements. -It is able to determine which service management to use in most cases. - -Implemented managements: - - * Systemd - * SysVinit - * InitCtl - -```python -if h.service('httpd').status(): - h.service('httpd').stop() -if h.service('httpd').is_enabled(): - h.service('httpd').disable() -``` - -### Operating System Info -Host provide `os` attribute which allows obtain basic operating system info. -Note that `os.release_info` depends on systemd init system. - -```python -print h.os.distribution -# Distribution(distname='Fedora', version='23', id='Twenty Three') - -print h.os.release_info -# {'HOME_URL': 'https://fedoraproject.org/', -# 'ID': 'fedora', -# 'NAME': 'Fedora', -# 'PRETTY_NAME': 'Fedora 23 (Workstation Edition)', -# 'VARIANT': 'Workstation Edition', -# 'VARIANT_ID': 'workstation', -# 'VERSION': '23 (Workstation Edition)', -# 'VERSION_ID': '23', -# ... -# } - -print h.os.release_str -# Fedora release 23 (Twenty Three) -``` - -### Storage Management -It is in PROGRESS state. Planed are NFS & LVM services. - -## Requires -* paramiko -* netaddr - -### Power Management -Give you possibility to control host power state, you can restart, poweron, -poweroff host and get host power status. - -Implemented managements: - - * SSH - * IPMI - -```python -ipmi_user = User(pm_user, pm_password) -ipmi_params = { - 'pm_if_type': 'lan', - 'pm_address': 'test-mgmt.testdomain', - 'user': ipmi_user -} -h.add_power_manager( - power_manager.IPMI_TYPE, **ipmi_params -) -# restart host via ipmitool -h.power_manager.restart() -``` - -## Install -```python -python setup.py devop -``` diff --git a/README.rst b/README.rst new file mode 100644 index 0000000..de3ecbb --- /dev/null +++ b/README.rst @@ -0,0 +1,225 @@ +|Build Status| +|Code Coverage| +|Code Health| + +python-rrmngmnt +=============== + +Remote Resources MaNaGeMeNT + +Intro +----- + +This tool helps you manage remote machines and services running on that. +It is targeted to Linux based machines. All is done via SSH connection, +that means SSH server must be running there already. + +.. code:: python + + from rrmngmnt import Host, RootUser + + h = Host("10.11.12.13") + h.users.append(RootUser('123456')) + exec = h.executor() + print exec.run_cmd(['echo', 'Hello World']) + +Features +-------- + +List of provided interfaces to manage resources on machine, and +examples. + +Filesystem +~~~~~~~~~~ + +Basic file operations, you can find there subset of python 'os' module +related to files. + +.. code:: python + + print h.fs.exists("/path/to/file") + h.fs.chown("/path/to/file", "root", "root") + h.fs.chmod("/path/to/file", "644") + h.fs.unlink("/path/to/file") + +In additional there are methods to fetch / put file from / to remote system +to / from local system. + +.. code:: python + + h.fs.get("/path/to/remote/file", "/path/to/local/file/or/target/dir") + h.fs.put("/path/to/local/file", "/path/to/remote/file/or/target/dir") + +There is one special method which allows transfer file between hosts. + +.. code:: python + + h1.fs.transfer( + "/path/to/file/on/h1", + h2, "/path/to/file/on/h2/or/target/dir", + ) + +You can also mount devices. + +.. code:: python + + with h.fs.mount_point( + '//example.com/share', opts='ro,guest', + fstype='cifs', target='/mnt/netdisk' + ) as mp: + h.fs.listdir(mp.target) # list mounted directory + mp.remount('rw,sync,guest') # remount with different options + h.fs.touch('%s/new_file' % mp.target) # touch file + +Firewall +~~~~~~~~ + +Allows to manage firewall configurarion. Check which firewall service is +running on host (firewalld/iptables) and make configure this service. + +.. code:: python + + h.firewall.is_active('iptables') + h.firewall.chain('OUTPUT').list_rules() + h.firewall.chain('OUTPUT').add_rule('1.1.1.1', 'DROP') + + +Network +~~~~~~~ + +It allows to manage network configuration. + +.. code:: python + + print h.network.hostname + h.network.hostname = "my.machine.org" + print h.network.all_interfaces() + print h.network.list_bridges() + +Package Management +~~~~~~~~~~~~~~~~~~ + +It encapsulates various package managements. It is able to determine +which package management to use. You can still specify package management +explicitly. + + +Implemented managements: + +- APT +- YUM +- DNF +- RPM + +.. code:: python + + # install htop package using implicit management + h.package_management.install('htop') + # remove htop package using rpm explicitly + h.package_management('rpm').remove('htop') + +System Services +~~~~~~~~~~~~~~~ + +You can toggle system services, it encapsulates various service managements. +It is able to determine which service management to use in most cases. + + +Implemented managements: + +- Systemd +- SysVinit +- InitCtl + +.. code:: python + + if h.service('httpd').status(): + h.service('httpd').stop() + if h.service('httpd').is_enabled(): + h.service('httpd').disable() + +Operating System Info +~~~~~~~~~~~~~~~~~~~~~ + +Host provide ``os`` attribute which allows obtain basic operating +system info. +Note that ``os.release_info`` depends on systemd init system. + +.. code:: python + + print h.os.distribution + # Distribution(distname='Fedora', version='23', id='Twenty Three') + + print h.os.release_info + # {'HOME_URL': 'https://fedoraproject.org/', + # 'ID': 'fedora', + # 'NAME': 'Fedora', + # 'PRETTY_NAME': 'Fedora 23 (Workstation Edition)', + # 'VARIANT': 'Workstation Edition', + # 'VARIANT_ID': 'workstation', + # 'VERSION': '23 (Workstation Edition)', + # 'VERSION_ID': '23', + # ... + # } + + print h.os.release_str + # Fedora release 23 (Twenty Three) + +Storage Management +~~~~~~~~~~~~~~~~~~ + +It is in PROGRESS state. Planed are NFS & LVM services. + +Power Management +~~~~~~~~~~~~~~~~ + +Give you possibility to control host power state, you can restart, +poweron, poweroff host and get host power status. + + +Implemented managements: + +- SSH +- IPMI + +.. code:: python + + ipmi_user = User(pm_user, pm_password) + ipmi_params = { + 'pm_if_type': 'lan', + 'pm_address': 'test-mgmt.testdomain', + 'user': ipmi_user + } + h.add_power_manager( + power_manager.IPMI_TYPE, **ipmi_params + ) + # restart host via ipmitool + h.power_manager.restart() + +Requires +-------- + +- paramiko +- netaddr +- six + +Install +------- + +.. code:: sh + + python setup.py devop + +Test +---- + +.. code:: sh + + tox + +.. |Build Status| image:: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt.svg?branch=master + :target: https://travis-ci.org/rhevm-qe-automation/python-rrmngmnt +.. |Code Coverage| image:: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt/branch/master/graph/badge.svg + :target: https://codecov.io/gh/rhevm-qe-automation/python-rrmngmnt +.. |Code Health| image:: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master/landscape.svg?style=flat + :target: https://landscape.io/github/rhevm-qe-automation/python-rrmngmnt/master diff --git a/rrmngmnt/__init__.py b/rrmngmnt/__init__.py index a952651..1cfbccc 100644 --- a/rrmngmnt/__init__.py +++ b/rrmngmnt/__init__.py @@ -10,11 +10,11 @@ __all__ = [ - Host, - User, - RootUser, - Domain, - InternalDomain, - ADUser, - Database, + 'Host', + 'User', + 'RootUser', + 'Domain', + 'InternalDomain', + 'ADUser', + 'Database', ] diff --git a/rrmngmnt/common.py b/rrmngmnt/common.py index 4845f6f..884506b 100644 --- a/rrmngmnt/common.py +++ b/rrmngmnt/common.py @@ -1,3 +1,4 @@ +import six import socket @@ -5,10 +6,11 @@ def fqdn2ip(fqdn): """ translate fqdn to IP - :param fqdn: host name - :type fqdn: string - :return: IP - :rtype: string + Args: + fqdn (str): host name + + Returns: + str: IP address """ try: return socket.gethostbyname(fqdn) @@ -19,3 +21,73 @@ def fqdn2ip(fqdn): ex.strerror = message ex.args = tuple(args) raise + + +def normalize_string(data): + """ + get normalized string + + Args: + data (object): data to process + Returns: + object: normalized string + """ + if isinstance(data, six.binary_type): + data = data.decode('utf-8', errors='replace') + return data + + +class CommandReader(object): + """ + This class is for gradual reading of commands output lines as they come in. + Each instance of CommandReader is tied to one command and one executor. + The executor calls the command only once the method read_lines is called. + After the execution of command finishes, CommandReader object may be + queried for return code, stdout and stderr of the command. + + Example usage: + my_host = Host("1.2.3.4") + my_host.users.append(RootUser("1234")) + my_executor = my_host.executor() + cr = CommandReader(my_executor, ['ansible-playbook', 'long_task.yml'] + for line in cr.read_lines(): + print(line) + """ + + def __init__(self, executor, cmd, cmd_input=None): + """ + Args: + executor (rrmngmnt.Executor): instance of rrmngmnt.Executor class + or one of its subclasses that executes provided command + cmd (list): Command to be executed + cmd_input(str): Input for the command + """ + self.executor = executor + self.cmd = cmd + self.cmd_input = cmd_input + self.rc = None + self.out = '' + self.err = '' + + def read_lines(self): + """ + Generator that yields lines of command output as they come to + underlying file handler. + + Yields: + str: Line of command's output stripped of newline character + """ + with self.executor.session() as ss: + command = ss.command(self.cmd) + with command.execute() as (in_, out, err): + if self.cmd_input: + in_.write(self.cmd_input) + in_.close() + while True: + line = out.readline() + self.out += line + if not line: + break + yield line.strip('\n') + self.rc = command.rc + self.err = err.read() diff --git a/rrmngmnt/db.py b/rrmngmnt/db.py index 57501cd..5534353 100644 --- a/rrmngmnt/db.py +++ b/rrmngmnt/db.py @@ -5,12 +5,10 @@ class Database(Service): def __init__(self, host, name, user): """ - :param host: Remote resouce to DB machine - :type host: instance of Host - :param name: database name - :type name: str - :param user: user/role - :type user: instance of User + Args: + host (Host): Remote resouce to DB machine + name (str): database name + user (User): user/role """ super(Database, self).__init__(host) self.name = name @@ -18,14 +16,14 @@ def __init__(self, host, name, user): def psql(self, sql, *args): """ - Execute psql command on host + Execute sql command on host - :param sql: sql command - :type sql: string - :param args: positional format arguments for command - :type args: list of arguments - :return: list of lines with records - :rtype: list(list(string, string, ...)) + Args: + sql (str): sql command + args (list): positional format arguments for command + + Returns: + list: list of lines with records. """ separator = '__RECORD_SEPARATOR__' sql = sql % tuple(args) @@ -48,7 +46,35 @@ def psql(self, sql, *args): ] # NOTE: I am considering to use Psycopg2 to access DB directly. # I need to think whether it is better or not. - # We need to realize that connection can be forbiden from outside ... + # We need to realize that connection can be forbidden from outside ... + + def psql_cmd(self, command): + """ + Execute psql special command on host (e.g. \\dt, \\dv, ...) + + Args: + command (str): special psql command + Returns: + str: output of the command + """ + cmd = [ + 'export', 'PGPASSWORD=%s;' % self.user.password, + 'psql', '-d', self.name, '-U', self.user.name, '-h', 'localhost', + '-c', command + ] + executor = self.host.executor() + with executor.session() as ss: + rc, out, err = ss.run_cmd(cmd) + if rc: + raise Exception( + "Failed to exec command: %s" % err + ) + if not out and err: + out = err + return out def restart(self): + """ + Restart postgresql service + """ self.host.service('postgresql').restart() diff --git a/rrmngmnt/errors.py b/rrmngmnt/errors.py index 8391052..8854487 100644 --- a/rrmngmnt/errors.py +++ b/rrmngmnt/errors.py @@ -10,14 +10,11 @@ class CommandExecutionFailure(GeneralResourceError): """ def __init__(self, executor, cmd, rc, err): """ - :param executor: executor used for execution - :type executor: instance of RemoteExecutor - :param cmd: executed command - :type cmd: list - :param rc: return code - :type rc: int - :param err: standard error output if provided - :type err: string + Args: + executor (RemoteExecutor): executor used for execution + cmd (list): executed command + rc (int): return code + err (str): standard error output if provided """ super(CommandExecutionFailure, self).__init__(executor, cmd, rc, err) @@ -51,12 +48,10 @@ class UnsupportedOperation(GeneralResourceError): """ def __init__(self, host, operation, reason): """ - :param host: relevant host - :type host: instance of Host - :param operation: name of unsupported operation - :type operation: str - :param reason: message - :type message: str + Args: + host (Host): relevant host + operation (str): name of unsupported operation + reason (str): message """ super(UnsupportedOperation, self).__init__(host, operation, reason) @@ -76,3 +71,48 @@ def __str__(self): return "Operation '{0}' is not supported for {1}: {2}".format( self.operation, self.host, self.reason ) + + +class FileSystemError(GeneralResourceError): + pass + + +class MountError(FileSystemError): + def __init__(self, mp): + self.mp = mp + + +class FailCreateTemp(FileSystemError): + pass + + +class MountCommandError(MountError): + def __init__(self, mp, stdout, stderr): + super(MountCommandError, self).__init__(mp) + self.stdout = stdout + self.stderr = stderr + + def __str__(self): + return ( + """ + stdout:{out} + stderr:{err} + {mp} + """.format( + out=self.stdout, + err=self.stderr, + mp=self.mp, + ) + ) + + +class FailToMount(MountCommandError): + pass + + +class FailToUmount(MountCommandError): + pass + + +class FailToRemount(MountCommandError): + pass diff --git a/rrmngmnt/executor.py b/rrmngmnt/executor.py index a9f2251..3bfe09d 100644 --- a/rrmngmnt/executor.py +++ b/rrmngmnt/executor.py @@ -81,8 +81,8 @@ def rc(self): def __init__(self, user): """ - :param user: user - :type user: instance of user + Args: + user (User): user """ super(Executor, self).__init__() self.user = user @@ -92,10 +92,14 @@ def session(self): def run_cmd(self, cmd, input_=None): """ - :param cmd: command - :type cmd: list - :param input_: input data - :type input_: str + Args: + cmd (list): command + input_(str): input data """ with self.session() as session: return session.run_cmd(cmd, input_) + + +class ExecutorFactory(object): + def build(self, host, user): + raise NotImplementedError() diff --git a/rrmngmnt/filesystem.py b/rrmngmnt/filesystem.py index 278887e..64a928b 100644 --- a/rrmngmnt/filesystem.py +++ b/rrmngmnt/filesystem.py @@ -1,6 +1,11 @@ import os -from rrmngmnt.service import Service + +import six +import warnings + from rrmngmnt import errors +from rrmngmnt.service import Service +from rrmngmnt.resource import Resource class FileSystem(Service): @@ -10,11 +15,12 @@ class FileSystem(Service): """ def _exec_command(self, cmd): host_executor = self.host.executor() - rc, _, err = host_executor.run_cmd(cmd) + rc, out, err = host_executor.run_cmd(cmd) if rc: raise errors.CommandExecutionFailure( cmd=cmd, executor=host_executor, rc=rc, err=err ) + return out def _exec_file_test(self, op, path): return self.host.executor().run_cmd( @@ -30,6 +36,9 @@ def isfile(self, path): def isdir(self, path): return self._exec_file_test('d', path) + def isexec(self, path): + return self._exec_file_test('x', path) + def remove(self, path): return self.host.executor().run_cmd( ['rm', '-f', path] @@ -48,71 +57,143 @@ def listdir(self, path): ['ls', '-A1', path] )[1].split() - def touch(self, file_name, path): + def touch(self, *args): + """ + Creates files on host + + __author__ = "vkondula" + + Args: + args (list): Paths of files to create + + Returns: + bool: True when file creation succeeds, False otherwise + """ + if len(args) == 2 and self.isdir(args[1]): + warnings.warn( + "This usecase is deprecated and will be removed. " + "Use list of fullpaths instead" + ) + return self._deprecated_touch(args[0], args[1]) + return self.host.run_command(['touch'] + list(args))[0] == 0 + + def _deprecated_touch(self, file_name, path): """ Creates a file on host __author__ = "ratamir" - :param file_name: The file to create - :type file_name: str - :param path: The path under which the file will be created - :type path: str - :returns: True when file creation succeeds, False otherwise - False otherwise - :rtype: bool + + Args: + file_name (str): The file to create + path (str): The path under which the file will be created + + Returns: + bool: True when file creation succeeds, False otherwise """ full_path = os.path.join(path, file_name) return self.host.run_command(['touch', full_path])[0] == 0 + def flush_file(self, file_path): + """ + Flushes the file. + + Args: + file_path (str): The path of file to flush. + + Returns: + bool: True if truncated, False otherwise + """ + cmd = ["truncate", "-s", "0", file_path] + return self.host.run_command(cmd)[0] == 0 + def read_file(self, path): """ Reads a content of a file in a given path - :param path: The path from where to take a content from - :type path: str - :return: Content of a file - :rtype: str + Args: + path (str): The path from where to take a content from + + Returns: + str: Content of a file """ cmd = ["cat", path] rc, out, _ = self.host.run_command(cmd) return out if not rc else "" + def move(self, source_path, destination_path): + """ + Moves a file or directory from source to destination. + + Args: + source_path (str): The source path to move from. + destination_path (str): The destination path to move to. + + Returns: + bool: True if there were no errors, False otherwise. + """ + cmd = ["mv", source_path, destination_path] + return self.host.run_command(cmd)[0] == 0 + + def create_file(self, content, path): + """ + Create file with given content on filesystem. + + Args: + content (str): content of the file. + path (str): destination path of the file. + """ + executor = self.host.executor() + with executor.session() as session: + with session.open_file(path, 'wb') as fh: + fh.write(six.b(content)) + def create_script(self, content, path): """ Create script on filesystem, and make it executable. - :param content: content of the script - :type content: str - :param path: path to script to create - :type path: str + Args: + content (str): content of the script + path (str): path to script to create """ executor = self.host.executor() with executor.session() as session: with session.open_file(path, 'wb') as fh: - fh.write(content) + fh.write(six.b(content)) self.chmod(path=path, mode="+x") - def mkdir(self, path): + def mkdir(self, path, parents=False, mode=None): """ Create directory on host - :param path: directory path - :type path: str - :raises: CommandExecutionFailure, if mkdir failed + Args: + path (str): directory path + parents (bool): True - no error if existing, make parent + directories as needed, False - error when parent + doesn't exist (default False) + mode (str): permission mode(600 for example or u+x) + + Raises: + CommandExecutionFailure: If mkdir failed """ - self._exec_command(['mkdir', path]) + cmd = ['mkdir'] + if parents: + cmd.append('-p') + if mode: + cmd.extend(['-m', mode]) + cmd.append(path) + self._exec_command(cmd) def chown(self, path, username, groupname): """ Change owner of file or directory - :param path: file or directory path - :type path: str - :param username: change user owner to username - :type username: str - :param groupname: change group owner to groupname - :type groupname: str - :raises: CommandExecutionFailure, if chown failed + Args: + path (str): file or directory path + username (str): change user owner to username + groupname (str): change group owner to groupname + + Raises: + CommandExecutionFailure: If chown failed """ self._exec_command(['chown', '%s:%s' % (username, groupname), path]) @@ -120,26 +201,86 @@ def chmod(self, path, mode): """ Change permission of directory or file - :param path: file or directory path - :type path: str - :param mode: permission mode(600 for example or u+x) - :type mode: str - :raises: CommandExecutionFailure, if chmod failed + Args: + path (str): file or directory path + mode (str): permission mode(600 for example or u+x) + + Raises: + CommandExecutionFailure: If chmod failed """ self._exec_command(['chmod', mode, path]) + def get(self, path_src, path_dst): + """ + Fetch file from Host and store on local system + + Args: + path_src (str): path to file on remote system + path_dst (str): path to file on local system or directory + + Returns: + str: Path to destination file + """ + if os.path.isdir(path_dst): + path_dst = os.path.join(path_dst, os.path.basename(path_src)) + with self.host.executor().session() as ss: + with ss.open_file(path_src, 'rb') as rh: + with open(path_dst, 'wb') as wh: + wh.write(rh.read()) + return path_dst + + def put(self, path_src, path_dst): + """ + Upload file from local system to Host + + Args: + path_src (str): path to file on local system + path_dst (str): path to file on remote system or directory + + Returns: + str: path to destination file + """ + if self.isdir(path_dst): + path_dst = os.path.join(path_dst, os.path.basename(path_src)) + with self.host.executor().session() as ss: + with open(path_src, 'rb') as rh: + with ss.open_file(path_dst, 'wb') as wh: + wh.write(rh.read()) + return path_dst + + def transfer(self, path_src, target_host, path_dst): + """ + Transfer file from one remote system (self) to other + remote system (target_host). + + Args: + path_src (str): path to file on local system + target_host (Host): target system + path_dst (str): path to file on remote system or directory + + Returns: + str: path to destination file + """ + if target_host.fs.isdir(path_dst): + path_dst = os.path.join(path_dst, os.path.basename(path_src)) + with self.host.executor().session() as h1s: + with target_host.executor().session() as h2s: + with h1s.open_file(path_src, 'rb') as rh: + with h2s.open_file(path_dst, 'wb') as wh: + wh.write(rh.read()) + return path_dst + def wget(self, url, output_file, progress_handler=None): """ Download file on the host from given url - :param url: url to file - :type url: str - :param output_file: full path to output file - :type output_file: str - :param progress_handler: progress handler function - :type progress_handler: func - :return: absolute path to file - :rtype: str + Args: + url (str): url to file + output_file (str): full path to output file + progress_handler (func): progress handler function + + Returns: + str: absolute path to file """ rc = None host_executor = self.host.executor() @@ -161,3 +302,140 @@ def wget(self, url, output_file, progress_handler=None): "Failed to download file from url {0}".format(url) ) return output_file + + def mktemp(self, template=None, tmpdir=None, directory=False): + """ + Make temporary file + + Args: + template (str): template for path, 'X's are replaced + tmpdir (str): where to create file, if not specified + use $TMPDIR if set, else /tmp + directory (bool): create directory instead of a file + + Returns: + str: absolute path to file or None if failed + """ + cmd = ['mktemp'] + if tmpdir: + cmd.extend(['-p', tmpdir]) + if directory: + cmd.append('-d') + if template: + cmd.append(template) + rc, out, _ = self.host.run_command(cmd) + if rc: + raise errors.FailCreateTemp(cmd) + return out.replace('\n', '') + + def mount_point( + self, source, target=None, fs_type=None, opts=None + ): + return MountPoint( + self, + source=source, + target=target, + fs_type=fs_type, + opts=opts, + ) + + +class MountPoint(Resource): + """ + Class for mounting devices. + """ + def __init__(self, fs, source, target=None, fs_type=None, opts=None): + """ + Mounts source to target mount point + + __author__ = "vkondula" + + Args: + fs (FileSystem): FileSystem object instance + source (str): Full path to source + target (str): Path to target directory, if omitted, a temporary + folder is created instead + fs_type (str): File system type + opts (str): Mount options separated by a comma such as: + 'sync,rw,guest' + """ + super(MountPoint, self).__init__() + self.fs = fs + self.source = source + self.opts = opts + self.fs_type = fs_type + self.target = target + self._tmp = not bool(target) + self._mounted = False + + def __enter__(self): + self.mount() + return self + + def __exit__(self, type_, value, tb): + try: + self.umount() + except errors.MountError as e: + self.logger.error(e) + if not type_: + raise + + def __str__(self): + return ( + """ + Mounting point: + source: {source} + target: {target} + file system: {fs} + options: {opts} + """.format( + source=self.source, + target=self.target or "*tmp*", + fs=self.fs_type or "DEFAULT", + opts=self.opts or "DEFAULT", + ) + ) + + def mount(self): + if self._tmp: + self.target = self.fs.mktemp(directory=True) + cmd = ['mount', '-v'] + if self.fs_type: + cmd.extend(['-t', self.fs_type]) + if self.opts: + cmd.extend(['-o', self.opts]) + cmd.extend([self.source, self.target]) + rc, out, err = self.fs.host.run_command(cmd) + if rc: + raise errors.FailToMount(self, out, err) + self._mounted = True + + def umount(self, force=True): + cmd = ['umount', '-v'] + if force: + cmd.append('-f') + cmd.append(self.target) + rc, out, err = self.fs.host.run_command(cmd) + if rc: + raise errors.FailToUmount(self, out, err) + if self._tmp and not self.fs.listdir(self.target): + self.fs.rmdir(self.target) + self._mounted = False + + def remount(self, opts): + """ + Remount disk. 'remount' option is implicit + + Args: + opts (str): Mount options separated by a comma such as: + 'sync,rw,guest' + """ + if not self._mounted: + raise errors.FailToRemount(self, '', 'not mounted!') + cmd = ['mount', '-v'] + cmd.extend(['-o', 'remount,%s' % opts]) + cmd.append(self.target) + rc, out, err = self.fs.host.run_command(cmd) + if rc: + raise errors.FailToRemount(self, out, err) + self.opts = opts diff --git a/rrmngmnt/firewall.py b/rrmngmnt/firewall.py new file mode 100644 index 0000000..70589a6 --- /dev/null +++ b/rrmngmnt/firewall.py @@ -0,0 +1,211 @@ +from rrmngmnt.service import Service + +IPTABLES = 'iptables' + + +class Firewall(Service): + """ + Class for firewall services + """ + def __init__(self, host): + """ + Args: + host (host): Host object to run commands on + """ + super(Firewall, self).__init__(host) + self.host = host + + def is_active(self, firewall_service): + """ + Check if the relevant firewall service is active on the host + + Args: + firewall_service (str): Service name + + Returns: + bool: True if the service is active on host, False if not + + """ + return self.host.service(firewall_service).status() + + def chain(self, chain_name): + """ + Return Chain class to run commands on specefic firewall chain + Args: + chain_name (str): Name of chain to make changes + + Returns: + chain: Chain class object + + """ + return Chain(self.host, chain_name) + + +class Chain(Service): + """ + Class for Firewall specific chain commands + """ + def __init__(self, host, chain_name): + """ + Args: + host (host): Host object to run commands on + chain_name (str): Name of the firewall chain + """ + super(Chain, self).__init__(host) + self.host = host + self.firewall_service = IPTABLES + self.chain_name = chain_name.upper() + if self.chain_name == 'OUTPUT': + self.address_type = '--destination' + elif self.chain_name == 'INPUT': + self.address_type = '--source' + else: + raise NotImplementedError("only INPUT/OUTPUT chains are supported") + + def edit_chain( + self, action, chain_name, address_type, dest, target, protocol='all', + ports=None, rule_num=None + ): + """ + Changes firewall configuration + + Args: + action (str): action to perform + chain_name (str): affected chain name + address_type (str): '--destination' for outgoing rules, + '--source' for incoming + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + rule_num (str): the number given after the chain name indicates the + position where the rule will be inserted + + Returns: + bool: True if configuration change succeeded, False otherwise + + Raises: + NotImplementedError: In case the users specifies more than 15 ports + to block + + Example: + edit_chain( + action='--append',chain='OUTPUT', + rule_num='1', + address_type='--destination', + dest={'address': nfs_server}, + target='DROP' + ) + """ + cmd = [ + self.firewall_service, action, chain_name + ] + + if rule_num: + cmd.extend([rule_num]) + + dest = ",".join(dest['address']) + cmd.extend( + [ + address_type, dest, '--jump', target.upper(), + '--protocol', protocol + ] + ) + + if ports: + # Iptables multiport module accepts up to 15 ports + if len(ports) > 15: + raise NotImplementedError("Up to 15 ports can be specified") + ports = ",".join(ports) + + if protocol.lower() == 'all': + # Adjust the protocol type, '--dports' option requires specific + # type + cmd[-1] = 'tcp' + + cmd.extend(['--match', 'multiport', '--dports', ports]) + + return not self.host.executor().run_cmd(cmd)[0] + + def list_rules(self): + """ + List all existing rules in a specific Chain + + Returns: + list: List of existing rules + """ + cmd = [self.firewall_service, '--list-rules', self.chain_name] + rules = self.host.executor().run_cmd(cmd)[1] + return rules.splitlines() + + def add_rule(self, dest, target, protocol='all', ports=None): + """ + Add new firewall rule to a specific chain + + Args: + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): Target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + + Returns: + bool: False if adding new rule failed, True if it succeeded + """ + return self.edit_chain( + '--append', self.chain_name, self.address_type, dest, target, + protocol, ports + ) + + def insert_rule(self, dest, target, protocol='all', ports=None, + rule_num=None): + """ + Insert new firewall rule to a specific chain + + Args: + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): Target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + rule_num (str): the number given after the chain name indicates + the position where the rule will be inserted. If the rule_num is + not given , the new rule is inserted in the line 1. + + Returns: + bool: False if inserting new rule failed, True if it succeeded + """ + return self.edit_chain( + '--insert', self.chain_name, self.address_type, dest, target, + protocol, ports, rule_num + ) + + def delete_rule(self, dest, target, protocol='all', ports=None): + """ + Delete existing firewall rule from a specific chain + + Args: + dest (dict): 'address' key and value containing destination host or + list of destination hosts + target (str): Target rule to apply + protocol (str): affected network protocol, Default is 'all' + ports (list): list of ports to configure + + Returns: + bool: False if deleting rule failed, True if it succeeded + """ + return self.edit_chain( + '--delete', self.chain_name, self.address_type, dest, target, + protocol, ports + ) + + def clean_rules(self): + """ + Delete all rules in a specific chain + + Returns: + bool: True if succeeded, False otherwise + """ + cmd = [self.firewall_service, '--flush', self.chain_name] + return not self.host.executor().run_cmd(cmd)[0] diff --git a/rrmngmnt/host.py b/rrmngmnt/host.py index 64d6e77..e0ec769 100644 --- a/rrmngmnt/host.py +++ b/rrmngmnt/host.py @@ -3,23 +3,26 @@ It should hold methods / properties which returns you Instance of specific Service hosted on that Host. """ -import os import copy +import os import socket -import netaddr +import threading import warnings -from rrmngmnt import ssh +import netaddr from rrmngmnt import errors from rrmngmnt import power_manager +from rrmngmnt import ssh from rrmngmnt.common import fqdn2ip -from rrmngmnt.network import Network -from rrmngmnt.storage import NFSService, LVMService -from rrmngmnt.service import Systemd, SysVinit, InitCtl -from rrmngmnt.resource import Resource from rrmngmnt.filesystem import FileSystem -from rrmngmnt.package_manager import PackageManagerProxy +from rrmngmnt.firewall import Firewall +from rrmngmnt.network import Network from rrmngmnt.operatingsystem import OperatingSystem +from rrmngmnt.package_manager import PackageManagerProxy +from rrmngmnt.playbook_runner import PlaybookRunner +from rrmngmnt.resource import Resource +from rrmngmnt.service import Systemd, SysVinit, InitCtl +from rrmngmnt.storage import NFSService, LVMService class Host(Resource): @@ -30,12 +33,14 @@ class Host(Resource): # The purpose of inventory variable is keeping all instances of # interesting resources in single place. inventory = list() + lock = threading.Lock() default_service_providers = [ Systemd, SysVinit, InitCtl, ] + executor_factory = ssh.RemoteExecutorFactory() class LoggerAdapter(Resource.LoggerAdapter): """ @@ -53,13 +58,12 @@ def process(self, msg, kwargs): def __init__(self, ip, service_provider=None): """ - :param ip: IP address of machine or resolvable FQDN - :type ip: string - :param service_provider: system service handler - :type service_provider: class which implement SystemService interface + Args: + ip (str): IP address of machine or resolvable FQDN + service_provider (Service): system service handler """ super(Host, self).__init__() - if not netaddr.valid_ipv4(ip): + if not netaddr.valid_ipv4(ip) and not netaddr.valid_ipv6(ip): ip = fqdn2ip(ip) self.ip = ip self.users = list() @@ -77,10 +81,12 @@ def __str__(self): def get(cls, ip): """ Get host from inventory - :param ip: IP address of machine or resolvable FQDN - :type ip: str - :return: host - :rtype: Host + + Args: + ip (str): IP address of machine or resolvable FQDN + + Returns: + Host: host instance """ host = [h for h in cls.inventory if h.ip == ip or h.fqdn == ip] if not host: @@ -91,14 +97,15 @@ def add(self): """ Add host to inventory """ - try: - host = self.get(self.ip) - except ValueError: - pass - else: - self.inventory.remove(host) - self.logger.debug("Adding host with ip '%s' to inventory", self.ip) - self.inventory.append(self) + with self.lock: + try: + host = self.get(self.ip) + except ValueError: + pass + else: + self.inventory.remove(host) + self.logger.debug("Adding host with ip '%s' to inventory", self.ip) + self.inventory.append(self) @property def fqdn(self): @@ -108,10 +115,10 @@ def add_power_manager(self, pm_type, **init_params): """ Add power power manager to host - :param pm_type: power manager type(power_manager.SSH_TYPE for example) - :type pm_type: str - :param init_params: power manager init parameters - :type init_params: dict + Args: + pm_type (str): power manager type + (power_manager.SSH_TYPE for example) + init_params (dict): power manager init parameters """ self._power_managers[pm_type] = getattr( power_manager, power_manager.MANAGERS[pm_type] @@ -121,11 +128,15 @@ def get_power_manager(self, pm_type=None): """ Get host power manager - :param pm_type: power manager type(power_manager.SSH_TYPE for example) - :type pm_type: str - :return: instance of PowerManager - :rtype: PowerManager - :raises: Exception + Args: + pm_type (str): power manager type(power_manager.SSH_TYPE for + example) + + Returns: + PowerManager: instance of powermanager + + Raises: + Exception: If power manager not supported """ if self._power_managers: if pm_type: @@ -136,7 +147,7 @@ def get_power_manager(self, pm_type=None): (pm_type, self) ) else: - return self._power_managers.values()[0] + return list(self._power_managers.values())[0] raise Exception("No PM is associated with the host %s" % self) def get_user(self, name): @@ -151,8 +162,8 @@ def add_user(self, user): """ Adds user to users collection, and tries remove duplicities. - :param user: user to add - :type user: instance of rrmngmnt.User + Args: + user (User): user to add """ for u in self.users[:]: if user.get_full_name() == u.get_full_name(): @@ -162,11 +173,10 @@ def add_user(self, user): def _set_executor_user(self, user): """ This method explicitly set user which is used to execute commands - on host. - And adds user into users collection. + on host. And adds user into users collection. - :param user: specific user - :type user: instance of rrmngmnt.User + Args: + user (User): specific user """ self._executor_user = user self.add_user(user) @@ -175,8 +185,8 @@ def _get_executor_user(self): """ The user which is supposed to be used for command execution. - :return: user - :rtype: instance of rrmngmnt.User + Returns: + user: instance of User """ if self._executor_user: return copy.copy(self._executor_user) @@ -204,13 +214,22 @@ def executor(self, user=None, pkey=False): """ Gives you executor to allowing command execution - :param user: the executed commands will be executed under this user. - when it is None, the default executor user is used, - see set_executor_user method for more info. + Args: + user (User): the executed commands will be executed under this + user. when it is None, the default executor user is used, + see set_executor_user method for more info. """ if user is None: user = self.executor_user - return ssh.RemoteExecutor(user, self.ip, use_pkey=pkey) + if pkey: + warnings.warn( + "Parameter 'pkey' is deprecated and will be removed in future." + "Please use ssh.RemoteExecutorFactory to set this parameter." + ) + ef = copy.copy(ssh.RemoteExecutorFactory) + ef.use_pkey = pkey + return ef(self.ip, user) + return self.executor_factory.build(self, user) def run_command( self, command, input_=None, tcp_timeout=None, io_timeout=None, @@ -219,16 +238,14 @@ def run_command( """ Run command on host - :param command: command - :type command: list - :param input_: input data - :type input_: str - :param tcp_timeout: tcp timeout - :type tcp_timeout: float - :param io_timeout: timeout for data operation (read/write) - :type io_timeout: float - :return: tuple of (rc, out, err) - :rtype: tuple + Args: + command (list): command + input_ (str): input data + tcp_timeout (float): tcp timeout + `io_timeout (float): timeout for data operation (read/write) + + Returns: + tuple: tuple of (rc, out, err) """ self.logger.info("Executing command %s", ' '.join(command)) rc, out, err = self.executor(user=user, pkey=pkey).run_cmd( @@ -241,22 +258,30 @@ def run_command( ) return rc, out, err - def copy_to(self, resource, src, dst): + def copy_to(self, resource, src, dst, mode=None, ownership=None): """ Copy to host from another resource - :param resource: resource to copy from - :type resource: instance of Host - :param src: path to source - :type src: str - :param dst: path to destination - :type dst: str + Args: + src (str): Path to source + dst (str): Path to destination + resource (instance of Host): Resource to copy from + mode (str): File permissions + ownership (tuple): File ownership(ex. ('root', 'root')) """ + warnings.warn( + "This method is deprecated and will be removed. " + "Use Host.fs.transfer instead." + ) with resource.executor().session() as resource_session: with self.executor().session() as host_session: with resource_session.open_file(src, 'rb') as resource_file: with host_session.open_file(dst, 'wb') as host_file: host_file.write(resource_file.read()) + if mode: + self.fs.chmod(path=dst, mode=mode) + if ownership: + self.fs.chown(dst, *ownership) def _create_service(self, name, timeout): for provider in self.default_service_providers: @@ -283,12 +308,13 @@ def service(self, name, timeout=None): """ Create service provider for desired service - :param name: service name - :type name: string - :param timeout: expected time to complete operations - :type timeout: int - :return: service provider for desired service - :rtype: instance of SystemService + :Args: + name (string): Service name + timeout (int): Expected time to complete operations + + Returns: + instance of SystemService: Service provider for desired service + """ if self._service_provider is None: # we need to pick up service provider, @@ -310,10 +336,13 @@ def get_ssh_public_key(self, user=None): """ Get SSH public key - :param user: what user to get ssh keys for, default is root - :type user: instance of rrmngmnt.User - :return: SSH public key - :rtype: str + Args: + user (instance of rrmngmnt.User): What user to get ssh keys for, + default is root + + Returns: + str: Ssh public key + """ if user is None: user = copy.copy(self.root_user) @@ -340,12 +369,13 @@ def remove_remote_host_ssh_key(self, remote_host, user=None): """ Remove remote host keys (ip, fqdn) from KNOWN_HOSTS file - :param remote_host: Remote host resource object - :type remote_host: Host - :param user: what user to remove ssh keys for, default is root - :type user: instance of rrmngmnt.User - :return: True/False - :rtype: bool + Args: + remote_host (Host): Remote host resource object + user (instance of rrmngmnt.User): What user to remove ssh keys for, + default is root + + Returns: + bool: True/false """ if user is None: user = copy.copy(self.root_user) @@ -365,10 +395,12 @@ def remove_remote_key_from_authorized_keys(self, user=None): """ Remove remote ssh key from AUTHORIZED_KEYS file - :param user: what user to remove from authorized_keys, default is root - :type user: instance of rrmngmnt.User - :return: True/False - :rtype: bool + Args: + user (instance of rrmngmnt.User): What user to remove from + authorized_keys, default is root + + Returns: + bool: True/false """ if user is None: user = copy.copy(self.root_user) @@ -386,12 +418,15 @@ def get_os_info(self): """ Get OS info (Distro, version and code name) - :return: Results {dist: , ver: , name:} - example: - {'dist': 'Red Hat Enterprise Linux Server', + Returns: + dict: Results {dist: , ver: , name:} + + Examples: + { + 'dist': 'Red Hat Enterprise Linux Server', 'name': 'Maipo', - 'ver': '7.1'} - :rtype: dict + 'ver': '7.1' + } """ warnings.warn( "This method is deprecated and will be removed. " @@ -426,6 +461,10 @@ def lvm(self): def fs(self): return FileSystem(self) + @property + def playbook(self): + return PlaybookRunner(self) + @property def ssh_public_key(self): return self.get_ssh_public_key() @@ -440,15 +479,14 @@ def create_script( """ Create script on resource - :param content: content of the script - :type content: str - :param name_of_script: name of script to create - :type name_of_script: str - :param destination_path: directory on host to copy script - :type destination_path: str - :returns: Script absolute path, if creation success, - otherwise empty string - :rtype: str + Args: + content (str): Content of the script + name_of_script (str): Name of script to create + destination_path (str): Directory on host to copy script + + Returns: + str: Script absolute path, if creation success, otherwise empty + string """ warnings.warn( "This method is deprecated and will be removed. " @@ -465,13 +503,18 @@ def is_connective(self, tcp_timeout=20.0): """ Check if host is connective via ssh - :param tcp_timeout: time to wait for response - :type tcp_timeout: float - :return: True if host is connective, False otherwise - :rtype: bool + Args: + tcp_timeout (float): Time to wait for response + + Returns: + bool: True if host is connective, false otherwise """ warnings.warn( "This method is deprecated and will be removed. " "Use Host.executor().is_connective() instead." ) return self.executor().is_connective(tcp_timeout=tcp_timeout) + + @property + def firewall(self): + return Firewall(self) diff --git a/rrmngmnt/network.py b/rrmngmnt/network.py index 0bb373a..f82c622 100644 --- a/rrmngmnt/network.py +++ b/rrmngmnt/network.py @@ -5,6 +5,7 @@ import shlex import six import subprocess +from rrmngmnt.errors import CommandExecutionFailure from rrmngmnt.service import Service @@ -22,6 +23,10 @@ def __init__(self, host): self._s = None self._c = 0 + @property + def executor(self): + return self._e + def runCmd(self, cmd): return self._s.run_cmd(cmd) @@ -61,10 +66,11 @@ def __init__(self, session): def get_hostname(self): """ Get hostname - :return: hostname - :rtype: string + + Returns: + str: Hostname """ - rc, out, _ = self._m.runCmd(['hostname']) + rc, out, _ = self._m.runCmd(['hostname', '-f']) if rc: return None return out.strip() @@ -73,8 +79,9 @@ def get_hostname(self): def set_hostname(self, name): """ Set hostname persistently - :param name: hostname to be set - :type name: string + + Args: + name (str): Hostname to be set """ net_config = '/etc/sysconfig/network' cmd = [ @@ -99,8 +106,9 @@ class HostnameCtlHandler(HostnameHandler): def get_hostname(self): """ Get hostname - :return: hostname - :rtype: string + + Returns: + str: Hostname """ cmd = [ 'hostnamectl', 'status', '|', @@ -117,13 +125,15 @@ def get_hostname(self): def set_hostname(self, name): """ Set hostname persistently - :param name: hostname to be set - :type name: string + + Args: + name (str): Hostname to be set """ cmd = ['hostnamectl', 'set-hostname', name] rc, _, err = self._m.runCmd(cmd) if rc: - raise Exception("Unable to set hostname: %s" % err) + raise CommandExecutionFailure( + self._m.executor, cmd, rc, "Unable to set hostname: %s" % err) class Network(Service): @@ -137,9 +147,8 @@ def _cmd(self, cmd): rc, out, err = self._m.runCmd(cmd) if rc: - cmd_out = " ".join(cmd) - raise Exception( - "Fail to run command %s: %s ; %s" % (cmd_out, out, err)) + raise CommandExecutionFailure( + self._m.executor, cmd, rc, "OUT: %s\nERR: %s" % (out, err)) return out @keep_session @@ -177,11 +186,11 @@ def all_interfaces(self): """ Lists interfaces - :return: list of interfaces - :rtype: list of strings + Returns: + list of strings: List of interfaces """ out = self._cmd( - "ls -la /sys/class/net | grep 'dummy_\|pci' | grep -o '[" + "ls -la /sys/class/net | grep 'dummy_\\|pci' | grep -o '[" "^/]*$'".split() ) out = out.strip().splitlines() @@ -193,8 +202,8 @@ def find_default_gw(self): """ Find host default gateway - :return: default gateway - :rtype: string + Returns: + str: Default gateway """ out = self._cmd(["ip", "route"]).splitlines() for i in out: @@ -204,13 +213,32 @@ def find_default_gw(self): return default_gw[0] return None + @keep_session + def find_default_gwv6(self): + """ + Find host default ipv6 gateway + + Returns: + str: Default gateway + """ + out = self._cmd(["ip", "-6", "route"]).splitlines() + for i in out: + if re.search("default", i): + default_gw = re.findall( + r'(?<=\s)[0-9a-fA-F:]{3,}(?=\s)', i + ) + if netaddr.valid_ipv6(default_gw[0]): + return default_gw[0] + return None + @keep_session def find_ips(self): """ Find host IPs - :return: list of ips and list of cird ips - :rtype: tuple(list of strings, list of strings) + Returns: + tuple(list of strings, list of strings): List of ips and list of + cird ips """ ips = [] ip_and_netmask = [] @@ -229,12 +257,13 @@ def find_ip_by_default_gw(self, default_gw, ips_and_mask): """ Find IP by default gateway - :param default_gw: default gw of the host - :type default_gw: string - :param ips_and_mask: list of host ips with mask x.x.x.x/xx - :type ips_and_mask: list of strings - :return: ip - :rtype: string + Args: + ips_and_mask (list of strings): List of host ips with + mask x.x.x.x/xx + default_gw (str): Default gw of the host + + Returns: + str: Ip """ dgw = netaddr.IPAddress(default_gw) for ip_mask in ips_and_mask: @@ -249,10 +278,11 @@ def find_int_by_ip(self, ip): """ Find host interface or bridge by IP - :param ip: ip of the interface to find - :type ip: string - :return: interface - :rtype: string + Args: + ip (str): Ip of the interface to find + + Returns: + str: Interface """ out = self._cmd(["ip", "addr", "show", "to", ip]) return out.split(":")[1].strip() @@ -260,12 +290,13 @@ def find_int_by_ip(self, ip): @keep_session def find_ip_by_int(self, interface): """ - Find host interface by interface or Bridge name + Find host ipv4 by interface or Bridge name + + Args: + interface (str): Interface to get ip from - :param interface: interface to get ip from - :type interface: string - :return: IP or None - :rtype: string or None + Returns: + str or None: Ip or none """ out = self._cmd(["ip", "addr", "show", interface]) match_ip = re.search(r'[0-9]+(?:\.[0-9]+){3}', out) @@ -275,15 +306,40 @@ def find_ip_by_int(self, interface): return interface_ip return None + @keep_session + def find_ipv6_by_int(self, interface): + """ + Find host global ipv6 by interface or Bridge name + + Args: + interface (str): Interface to get ipv6 from + + Returns: + str or None: Ip or none + """ + out = self._cmd(["ip", "-6", "addr", "show", interface]) + for line in out.splitlines(): + if re.search("global", line): + match_ip = re.search( + r'(?<=\s)[0-9a-fA-F:]{3,}(?=/[0-9]{1,3}\s)', + line, + ) + if match_ip: + interface_ip = match_ip.group() + if netaddr.valid_ipv6(interface_ip): + return interface_ip + return None + @keep_session def find_int_by_bridge(self, bridge): """ Find host interface by Bridge name - :param bridge: bridge to get ip from - :type bridge: string - :return: interface - :rtype: string + Args: + bridge (str): Bridge to get ip from + + Returns: + str: Interface """ bridge = self.get_bridge(bridge) try: @@ -299,10 +355,11 @@ def find_mac_by_int(self, interfaces): """ Find interfaces MAC by interface name - :param interfaces: list of interfaces - :type interfaces: list of strings - :return: list of macs - :rtype: list of strings + Args: + interfaces (list of strings): List of interfaces + + Returns: + list of strings: List of macs """ mac_list = list() for interface in interfaces: @@ -316,11 +373,11 @@ def find_mac_by_int(self, interfaces): @keep_session def find_mgmt_interface(self): """ - Find host mgmt interface (interface with IP that lead - to default gateway) + Find host mgmt interface (interface with IP that lead to default + gateway) - :return: interface - :rtype: string + Returns: + str: Interface """ host_ip = self.find_ips() host_dg = self.find_default_gw() @@ -333,15 +390,15 @@ def list_bridges(self): """ List of bridges on host - :return: list of bridges - :rtype: list of dict(name, id, stp, interfaces) + Returns: + list of dict(name, id, stp, interfaces): List of bridges """ bridges = [] cmd = [ 'brctl', 'show', '|', 'sed', '-e', '/^bridge name/ d', # remove header # deal with multiple interfaces - '-e', "'s/^\s\s*\(\S\S*\)$/CONT:\\1/I'" + '-e', "'s/^\\s\\s*\\(\\S\\S*\\)$/CONT:\\1/I'" ] out = self._cmd(cmd).strip() if not out: @@ -368,8 +425,8 @@ def get_bridge(self, name): """ Find bridge by name - :return: bridge - :rtype: dict(name, id, stp, interfaces) + Returns: + dict(name, id, stp, interfaces): Bridge """ bridges = [ bridge for bridge in self.list_bridges() @@ -384,12 +441,12 @@ def add_bridge(self, bridge, network): """ Add bridge and add network to the bridge on host - :param bridge: Bridge name - :type bridge: str - :param network: Network name - :type network: str - :return: True/False - :rtype: bool + Args: + bridge (str): Bridge name + network (str): Network name + + Returns: + bool: True/false """ cmd_add_br = ["brctl", "addbr", bridge] cmd_add_if = ["brctl", "addif", bridge, network] @@ -402,10 +459,11 @@ def delete_bridge(self, bridge): """ Add bridge and add network to the bridge on host - :param bridge: Bridge name - :type bridge: str - :return: True/False - :rtype: bool + Args: + bridge (str): Bridge name + + Returns: + bool: True/false """ cmd_br_down = ["ip", "link", "set", "down", bridge] cmd_del_br = ["brctl", "delbr", bridge] @@ -418,8 +476,8 @@ def get_info(self): """ Get network info for host, return info for main IP. - :return: network info - :rtype: dict + Returns: + dict: Network info """ net_info = {} gateway = self.find_default_gw() @@ -429,6 +487,10 @@ def get_info(self): ip = self.find_ip_by_default_gw(gateway, ips_and_mask) net_info["ip"] = ip if ip is not None: + mask = [ + mask.split("/")[-1] for mask in ips_and_mask if ip in mask + ] + net_info["prefix"] = mask[0] if mask else "N/A" interface = self.find_int_by_ip(ip) # strip @NONE for PPC try: @@ -452,12 +514,10 @@ def create_ifcfg_file(self, nic, params, ifcfg_path=IFCFG_PATH): """ Create ifcfg file - :param nic: NIC name - :type nic: str - :param params: Ifcfg file content - :type params: dict - :param ifcfg_path: Ifcfg files path - :type ifcfg_path: str + Args: + nic (str): Nic name + ifcfg_path (str): Ifcfg files path + params (dict): Ifcfg file content """ dst = os.path.join(ifcfg_path, "ifcfg-%s" % nic) self.logger.info("Creating %s on %s", dst, self.host.fqdn) @@ -471,12 +531,12 @@ def delete_ifcfg_file(self, nic, ifcfg_path=IFCFG_PATH): """ Delete ifcfg file - :param nic: NIC name - :type nic: str - :param ifcfg_path: Ifcfg files path - :type ifcfg_path: str - :return: True/False - :rtype: bool + Args: + nic (str): Nic name + ifcfg_path (str): Ifcfg files path + + Returns: + bool: True/false """ dst = os.path.join(ifcfg_path, "ifcfg-%s" % nic) logger.info("Delete %s ", dst) @@ -485,25 +545,23 @@ def delete_ifcfg_file(self, nic, ifcfg_path=IFCFG_PATH): return False return True - def send_icmp(self, dst, count="5", size="1500", extra_args=None): + def send_icmp(self, dst, count="5", size=None, extra_args=None): """ Send ICMP to destination IP/FQDN - :param dst: IP/FQDN to send ICMP to - :type dst: str - :param count: Number of ICMP packets to send - :type count: str - :param size: Size of the ICMP packet - :type size: str - :param extra_args: Extra args for ping command - :type extra_args: str - :return: True/False - :rtype: bool - """ - cmd = ["ping", dst, "-c", count, "-s", size] - if size != "1500": - cmd.extend(["-M", "do"]) - if extra_args is not None: + Args: + count (str): Number of icmp packets to send + extra_args (str): Extra args for ping command + dst (str): Ip/fqdn to send icmp to + size (str): Size of the icmp packet + + Returns: + bool: True/false + """ + cmd = ["ping", dst, "-c", count] + if size: + cmd.extend(["-s", size, "-M", "do"]) + if extra_args: for ar in extra_args.split(): cmd.extend(ar.split()) try: @@ -517,12 +575,12 @@ def set_mtu(self, nics, mtu="1500"): """ Set MTU on NICs - :param nics: List on NICs - :type nics: list - :param mtu: MTU size - :type mtu: str - :return: True or raise Exception - :rtype: bool or Exception + Args: + nics (list): List on nics + mtu (str): Mtu size + + Returns: + bool or Exception: True or raise exception """ base_cmd = "ip link set mtu %s %s" for nic in nics: @@ -534,10 +592,11 @@ def delete_interface(self, interface): """ Delete interface from host - :param interface: Interface name - :type interface: str - :return: True/False - :rtype: bool + Args: + interface (str): Interface name + + Returns: + bool: True/false """ cmd = "ip link del %s" % interface try: @@ -552,10 +611,11 @@ def get_mac_by_ip(self, ip): """ Get mac address by ip address - :param ip: ip address - :type ip: str - :return: mac address - :rtype: str + Args: + ip (str): Ip address + + Returns: + str: Mac address """ interface = self.find_int_by_ip(ip=ip) return self.find_mac_by_int([interface])[0] @@ -564,37 +624,63 @@ def if_up(self, nic): """ Set nic up - :param nic: NIC name - :type nic: str - :return: True if setting NIC up succeeded, False otherwise - :rtype: bool + Args: + nic (str): Nic name + + Returns: + bool: True if setting nic up succeeded, false otherwise """ - cmd = "ip link set up %s" % nic + cmd = "ip link set {nic} up".format(nic=nic) rc, _, _ = self.host.run_command(shlex.split(cmd)) return not bool(rc) - def if_down(self, nic): + def if_down(self, nic, tcp_timeout=20, io_timeout=20): """ Set nic down - :param nic: NIC name - :type nic: str - :return: True if setting NIC down succeeded, False otherwise - :rtype: bool + Args: + nic (str): Nic name + tcp_timeout (float): TCP timeout + io_timeout (float): Timeout for data operation (read/write) + + Returns: + bool: True if setting nic down succeeded, false otherwise """ - cmd = "ip link set down %s" % nic - rc, _, _ = self.host.run_command(shlex.split(cmd)) + cmd = "ip link set {nic} down".format(nic=nic) + rc, _, _ = self.host.run_command( + command=shlex.split(cmd), + tcp_timeout=tcp_timeout, + io_timeout=io_timeout + ) return not bool(rc) + def add_ip(self, nic, ip, mask): + """ + Add IP address to interface + + Args: + nic (str): Interface name + ip (str): IP address to add + mask (str): IP netmask + + Returns: + bool: True if add IP was success, False otherwise + """ + cmd = "ip address add {ip}/{mask} dev {nic}".format( + ip=ip, mask=mask, nic=nic + ) + return not bool(self.host.run_command(command=shlex.split(cmd))[0]) + def is_connective(self, ping_timeout=20.0): """ Check if host network is connective via ping command - :param ping_timeout: time to wait for response - :type ping_timeout: float - :return: True if address is connective via ping command, - False otherwise - :rtype: bool + Args: + ping_timeout (float): Time to wait for response + + Returns: + bool: True if address is connective via ping command, false + otherwise """ host_address = self.host.ip # Leave it for future support of IPV6 @@ -619,3 +705,33 @@ def is_connective(self, ping_timeout=20.0): ) return False return True + + def get_interface_speed(self, interface): + """ + Get network interface speed + + Args: + interface (str): Interface name + + Returns: + str: Interface speed, or empty string if error has occurred + """ + ethtool_cmd = "ethtool -i {iface}".format(iface=interface) + self._cmd(shlex.split(ethtool_cmd)) + speed_cmd = "cat /sys/class/net/{iface}/speed".format(iface=interface) + out = self._cmd(shlex.split(speed_cmd)) + return out.strip() + + def get_interface_status(self, interface): + """ + Get interface status + + Args: + interface (str): Interface name + + Returns: + str: Interface status (up/down) + """ + cmd = "cat /sys/class/net/{iface}/operstate".format(iface=interface) + out = self._cmd(shlex.split(cmd)) + return out.strip() diff --git a/rrmngmnt/operatingsystem.py b/rrmngmnt/operatingsystem.py index 517e4b2..dac2561 100644 --- a/rrmngmnt/operatingsystem.py +++ b/rrmngmnt/operatingsystem.py @@ -14,15 +14,22 @@ def __init__(self, host): self._release_info = None self._dist = None - def get_release_str(self): - cmd = ['cat', '/etc/system-release'] - executor = self.host.executor() - rc, out, err = executor.run_cmd(cmd) + def _exec_command(self, cmd, err_msg=None): + host_executor = self.host.executor() + rc, out, err = host_executor.run_cmd(cmd) + if err_msg: + err = "{err_msg}: {err}".format(err_msg=err_msg, err=err) if rc: raise errors.CommandExecutionFailure( - executor, cmd, rc, - "Failed to obtain release string: {0}".format(err) + executor=host_executor, cmd=cmd, rc=rc, err=err ) + return out + + def get_release_str(self): + cmd = ['cat', '/etc/system-release'] + out = self._exec_command( + cmd=cmd, err_msg="Failed to obtain release string" + ) return out.strip() @property @@ -38,7 +45,8 @@ def get_release_info(self): It might raise exception in case the systemd is not deployed on system. - :raises: UnsupportedOperation + Raises: + UnsupportedOperation """ os_release_file = '/etc/os-release' cmd = ['cat', os_release_file] @@ -78,27 +86,25 @@ def get_distribution(self): """ Get OS info (Distro, version and code name) - :return: Results tuple(distname, version, id} - example: - Distribution( - distname='Red Hat Enterprise Linux Server', - id='Maipo', - version'='7.1' - ) - :rtype: namedtuple Distribution + Returns: + namedtuple Distribution: Results tuple(distname, version, id} + + Examples: + distribution( + distname='red hat enterprise linux server', + id='maipo', + version'='7.1' + ) + """ values = ["distname", "version", "id"] cmd = [ "python", "-c", "import platform;print(','.join(platform.linux_distribution()))" ] - executor = self.host.executor() - rc, out, err = executor.run_cmd(cmd) - if rc: - raise errors.CommandExecutionFailure( - executor, cmd, rc, - "Failed to obtain release info: {0}".format(err) - ) + out = self._exec_command( + cmd=cmd, err_msg="Failed to obtain release info" + ) Distribution = namedtuple('Distribution', values) return Distribution(*[i.strip() for i in out.split(",")]) @@ -107,3 +113,101 @@ def distribution(self): if not self._dist: self._dist = self.get_distribution() return self._dist + + def stat(self, path): + """ + Get file or directory stats + + Returns: + collections.namedtuple: File stats + """ + type_map = { + 'st_mode': ('0x%f', lambda x: int(x, 16)), + 'st_ino': ('%i', int), + 'st_dev': ('%d', int), + 'st_nlink': ('%h', int), + 'st_uid': ('%u', int), + 'st_gid': ('%g', int), + 'st_size': ('%s', int), + 'st_atime': ('%X', int), + 'st_mtime': ('%Y', int), + 'st_ctime': ('%W', int), + 'st_blocks': ('%b', int), + 'st_blksize': ('%o', int), + 'st_rdev': ('%t', int), + } + posix_stat_result = namedtuple( + "posix_stat_result", type_map.keys() + ) + + cmd = [ + "stat", + "-c", + ",".join(["%s=%s" % (k, v[0]) for k, v in type_map.items()]), + path + ] + out = self._exec_command(cmd=cmd) + out = out.strip().split(',') + + data = {} + + for pair in out: + key, value = pair.split('=') + data[key] = type_map[key][1](value) + + return posix_stat_result(**data) + + def get_file_permissions(self, path): + """ + Get file permissions + + Returns: + str: File permission in octal form(example 0644) + """ + cmd = ["stat", "-c", "%a", path] + return self._exec_command(cmd=cmd).strip() + + def get_file_owner(self, path): + """ + Get file user and group owner name + + Returns: + list: File user and group owner names(example ['root', 'root']) + """ + cmd = ["stat", "-c", "%U %G", path] + return self._exec_command(cmd=cmd).split() + + def user_exists(self, user_name): + """ + Check if user exist on system + + Args: + user_name (str): User name + + Returns: + bool: True, if user exist, otherwise false + """ + try: + cmd = ["id", "-u", user_name] + self._exec_command(cmd=cmd) + except errors.CommandExecutionFailure: + return False + return True + + def group_exists(self, group_name): + """ + Check if group exist on system + + Args: + group_name (str): Group name + + Returns: + bool: True, if group exist, otherwise false + + """ + try: + cmd = ["id", "-g", group_name] + self._exec_command(cmd=cmd) + except errors.CommandExecutionFailure: + return False + return True diff --git a/rrmngmnt/package_manager.py b/rrmngmnt/package_manager.py index 6d29f01..acdb61f 100644 --- a/rrmngmnt/package_manager.py +++ b/rrmngmnt/package_manager.py @@ -30,10 +30,11 @@ def _run_command_on_host(self, cmd): """ Run given command on host - :param cmd: command to run - :type cmd: list - :return: True, if command success, otherwise False - :rtype: bool + Args: + cmd (list): Command to run + + Returns: + bool: True, if command success, otherwise false """ self.logger.info( "Execute command '%s' on host %s", " ".join(cmd), self.host @@ -51,11 +52,14 @@ def exist(self, package): """ Check if package exist on host - :param package: name of package - :type package: str - :return: True, if package exist, otherwise False - :rtype: bool - :raise: NotImplementedError + Args: + package (str): Name of package + + Returns: + bool: True, if package exist, otherwise false + + Raises: + NotImplementedError """ if not self.exist_command_d: raise NotImplementedError("There is no 'exist' command defined.") @@ -68,11 +72,13 @@ def exist(self, package): def list_(self): """ - List installled packages on host + List installed packages on host - :return: installed packages - :rtype: list - :raise: NotImplementedError, CommandExecutionFailure + Returns: + list: Installed packages + + Raises: + NotImplementedError, CommandExecutionFailure """ if not self.list_command_d: raise NotImplementedError( @@ -97,11 +103,14 @@ def install(self, package): """ Install package on host - :param package: name of package - :type package: str - :return: True, if package installation success, otherwise False - :rtype: bool - :raise: NotImplementedError + Args: + package (str): Name of package + + Returns: + bool: True, if package installation success, otherwise false + + Raises: + NotImplementedError """ if not self.install_command_d: raise NotImplementedError("There is no 'install' command defined.") @@ -122,13 +131,16 @@ def remove(self, package, pattern=False): Remove package from host, or packages which match pattern if pattern is set to True - :param package: name of package or extended regular expression pattern - take a look at -E option in man grep - :type package: str - :param pattern: If True package name is pattern - :return: True, if package(s) removal success, otherwise False - :rtype: bool - :raise: NotImplementedError + Args: + pattern (bool): If true package name is pattern + package (str): Name of package or extended regular expression + pattern take a look at -e option in man grep + + Returns: + bool: True, if package(s) removal success, otherwise false + + Raises: + NotImplementedError """ if not self.remove_command_d: raise NotImplementedError("There is no 'remove' command defined.") @@ -174,11 +186,15 @@ def update(self, packages=None): if no packages are specified __author__ = "omachace" - :param packages: Packages to be updated, if empty, update system - :type packages: list - :return: True when updates succeed, False otherwise - :rtype: bool - :raise: NotImplementedError + + Args: + packages (list): Packages to be updated, if empty, update system + + Returns: + bool: True when updates succeed, false otherwise + + Raises: + NotImplementedError """ if not self.update_command_d: raise NotImplementedError("There is no 'update' command defined.") diff --git a/rrmngmnt/playbook_runner.py b/rrmngmnt/playbook_runner.py new file mode 100644 index 0000000..cd3a19c --- /dev/null +++ b/rrmngmnt/playbook_runner.py @@ -0,0 +1,192 @@ +import contextlib +import json +import os.path +import uuid + +from rrmngmnt.common import CommandReader +from rrmngmnt.resource import Resource +from rrmngmnt.service import Service + + +class PlaybookRunner(Service): + """ + Class for working with and especially executing Ansible playbooks on hosts. + On your Host instance, it might be accessed (similar to other services) by + playbook property. + + Example: + rc,out, err = my_host.playbook.run('long_task.yml') + + In such case, the default logger of this class (called PlaybookRunner) will + be used to log playbook's output. It will propagate events to handlers of + ancestor loggers. However, PlaybookRunner might also be directly + instantiated with instance of logging.Logger passed to logger parameter. + + Example: + my_runner = PlaybookRunner(my_host, logging.getLogger('playbook')) + rc, out, err = my_runner.run('long_task.yml') + + In that case, custom provided logger will be used instead. In both cases, + each log record will be prefixed with UUID generated specifically for that + one playbook execution. + """ + class LoggerAdapter(Resource.LoggerAdapter): + + def process(self, msg, kwargs): + return "[%s] %s" % (self.extra['self'].short_run_uuid, msg), kwargs + + tmp_dir = "/tmp" + binary = "ansible-playbook" + extra_vars_file = "extra_vars.json" + default_inventory_name = "inventory" + default_inventory_content = "localhost ansible_connection=local" + ssh_common_args_param = "--ssh-common-args" + check_mode_param = "--check" + + def __init__(self, host, logger=None): + """ + Args: + host (rrmngmnt.Host): Underlying host for this service + logger (logging.Logger): Alternate logger for Ansible output + """ + super(PlaybookRunner, self).__init__(host) + if logger: + self.set_logger(logger) + self.run_uuid = uuid.uuid4() + self.short_run_uuid = str(self.run_uuid).split('-')[0] + self.tmp_exec_dir = None + self.cmd = [self.binary] + self.rc = None + self.out = None + self.err = None + + @contextlib.contextmanager + def _exec_dir(self): + """ + Context manager that makes sure that for each execution of playbook, + temporary directory (whose name is the same as run's UUID) is created + on the host and removed afterwards. + """ + exec_dir_path = os.path.join(self.tmp_dir, self.short_run_uuid) + self.host.fs.rmdir(exec_dir_path) + self.host.fs.mkdir(exec_dir_path) + self.tmp_exec_dir = exec_dir_path + try: + yield + finally: + self.tmp_exec_dir = None + self.host.fs.rmdir(exec_dir_path) + + def _upload_file(self, file_): + file_path_on_host = os.path.join( + self.tmp_exec_dir, os.path.basename(file_) + ) + self.host.fs.put(path_src=file_, path_dst=file_path_on_host) + return file_path_on_host + + def _dump_vars_to_json_file(self, vars_): + file_path_on_host = os.path.join( + self.tmp_exec_dir, self.extra_vars_file + ) + self.host.fs.create_file( + content=json.dumps(vars_), path=file_path_on_host + ) + return file_path_on_host + + def _generate_default_inventory(self): + file_path_on_host = os.path.join( + self.tmp_exec_dir, self.default_inventory_name + ) + self.host.fs.create_file( + content=self.default_inventory_content, + path=file_path_on_host + ) + return file_path_on_host + + def run( + self, playbook, extra_vars=None, vars_files=None, inventory=None, + verbose_level=1, run_in_check_mode=False, ssh_common_args=None, + ): + """ + Run Ansible playbook on host + + Args: + playbook (str): Path to playbook you want to execute (on your + machine) + extra_vars (dict): Dictionary of extra variables that are to be + passed to playbook execution. They will be dumped to JSON file + and included using -e@ parameter + vars_files (list): List of additional variable files to be included + using -e@ parameter. If one variable is specified both in + extra_vars and in one of the vars_files, the one in vars_files + takes precedence. + inventory (str): Path to an inventory file (on your machine) to be + used for playbook execution. If none is provided, default + inventory including only localhost will be generated and used + verbose_level (int): How much should playbook be verbose. Possible + values are 1 through 5 with 1 being the most quiet and 5 being + the most verbose + run_in_check_mode (bool): If True, playbook will not actually be + executed, but instead run with --check parameter + ssh_common_args (list): List of options that will extend (not + replace) the list of default options that Ansible uses when + calling ssh/sftp/scp. Example: ["-o StrictHostKeyChecking=no", + "-o UserKnownHostsFile=/dev/null"] + + Returns: + tuple: tuple of (rc, out, err) + """ + self.logger.info( + "Running playbook {} on {}".format( + os.path.basename(playbook), + self.host.fqdn + ) + ) + + with self._exec_dir(): + + if extra_vars: + self.cmd.append( + "-e@{}".format(self._dump_vars_to_json_file(extra_vars)) + ) + + if vars_files: + for f in vars_files: + self.cmd.append("-e@{}".format(self._upload_file(f))) + + self.cmd.append("-i") + if inventory: + self.cmd.append(self._upload_file(inventory)) + else: + self.cmd.append(self._generate_default_inventory()) + + self.cmd.append("-{}".format("v" * verbose_level)) + + if run_in_check_mode: + self.cmd.append(self.check_mode_param) + + if ssh_common_args: + self.cmd.append( + "{}={}".format( + self.ssh_common_args_param, " ".join(ssh_common_args) + ) + ) + + self.cmd.append(self._upload_file(playbook)) + + self.logger.debug("Executing: {}".format(" ".join(self.cmd))) + + playbook_reader = CommandReader(self.host.executor(), self.cmd) + for line in playbook_reader.read_lines(): + self.logger.debug(line) + self.rc, self.out, self.err = ( + playbook_reader.rc, + playbook_reader.out, + playbook_reader.err + ) + + self.logger.debug( + "Ansible playbook finished with RC: {}".format(self.rc) + ) + + return self.rc, self.out, self.err diff --git a/rrmngmnt/power_manager.py b/rrmngmnt/power_manager.py index 76b031c..bfb1463 100644 --- a/rrmngmnt/power_manager.py +++ b/rrmngmnt/power_manager.py @@ -62,7 +62,9 @@ def _exec_pm_command(self, command, *args): try: t_command = list(command) t_command += args - self.host.executor().run_cmd(t_command) + self.host.executor().run_cmd( + t_command, tcp_timeout=20, io_timeout=20 + ) except socket.timeout as e: self.logger.debug("Socket timeout: %s", e) except Exception as e: @@ -97,12 +99,10 @@ def __init__(self, h, pm_if_type, pm_address, user): """ Initialize IPMIPowerManagement instance - :param pm_if_type: ipmi interface type(lan, lanplus) - :type pm_if_type: str - :param pm_address: power management address - :type pm_address: str - :param user: instance of User with pm username and password - :type user: User + Args: + pm_if_type (str): Ipmi interface type(lan, lanplus) + pm_address (str): Power management address + user (User): Instance of user with pm username and password """ super(IPMIPowerManager, self).__init__(h) self.pm_if_type = pm_if_type diff --git a/rrmngmnt/resource.py b/rrmngmnt/resource.py index add7855..9764a72 100644 --- a/rrmngmnt/resource.py +++ b/rrmngmnt/resource.py @@ -16,8 +16,14 @@ def warn(self, *args, **kwargs): def __init__(self): super(Resource, self).__init__() logger = logging.getLogger(self.__class__.__name__) - self._logger_adapter = self.LoggerAdapter(logger, {'self': self}) + self.set_logger(logger) @property def logger(self): return self._logger_adapter + + def set_logger(self, logger): + if isinstance(logger, logging.Logger): + self._logger_adapter = self.LoggerAdapter(logger, {'self': self}) + elif isinstance(logger, logging.LoggerAdapter): + self._logger_adapter = logger diff --git a/rrmngmnt/service.py b/rrmngmnt/service.py index 9222678..066f4ce 100644 --- a/rrmngmnt/service.py +++ b/rrmngmnt/service.py @@ -1,4 +1,5 @@ from rrmngmnt.resource import Resource +import re class Service(Resource): @@ -82,7 +83,8 @@ def unmask(self): def _can_handle(self): """ - :raises: CanNotHandle + Raises: + CanNotHandle """ executor = self.host.executor() rc, _, _ = executor.run_cmd( @@ -169,6 +171,11 @@ class Systemd(SystemService): def _can_handle(self): super(Systemd, self)._can_handle() + + orig_name = self.name + if "@" in self.name: + self.name = re.match(r'^.*@', self.name).group(0) + cmd = ( 'systemctl', 'list-unit-files', '|', 'grep', '-o', '^[^.][^.]*.service', '|', @@ -180,8 +187,9 @@ def _can_handle(self): out = out.strip().splitlines() if rc or self.name not in out: raise self.CanNotHandle( - "%s is not listed in %s" % (self.name, out) + "%s is not listed in %s" % (orig_name, out) ) + self.name = orig_name def _execute(self, action): cmd = [ @@ -191,6 +199,12 @@ def _execute(self, action): ] executor = self.host.executor() rc, _, _ = executor.run_cmd(cmd, io_timeout=self.timeout) + + if rc: + cmd = ['journalctl', '-u', self.name + ".service"] + _, out, _ = executor.run_cmd(cmd, io_timeout=self.timeout) + self.logger.warning(out) + return rc == 0 def is_enabled(self): diff --git a/rrmngmnt/ssh.py b/rrmngmnt/ssh.py index 1743111..32811dd 100644 --- a/rrmngmnt/ssh.py +++ b/rrmngmnt/ssh.py @@ -4,7 +4,9 @@ import paramiko import contextlib import subprocess -from rrmngmnt.executor import Executor +from rrmngmnt.common import normalize_string +from rrmngmnt.executor import Executor, ExecutorFactory + AUTHORIZED_KEYS = os.path.join("%s", ".ssh/authorized_keys") KNOWN_HOSTS = os.path.join("%s", ".ssh/known_hosts") @@ -12,6 +14,7 @@ ID_RSA_PRV = os.path.join("%s", ".ssh/id_rsa") CONNECTIVITY_TIMEOUT = 600 CONNECTIVITY_SAMPLE_TIME = 20 +TCP_CONNECTION_TIMEOUT = 20 class RemoteExecutor(Executor): @@ -19,12 +22,12 @@ class RemoteExecutor(Executor): Any resource which provides SSH service. This class is meant to replace our current utilities.machine.LinuxMachine - classs. This allows you to lower access to communicate with ssh. + class. This allows you to lower access to communicate with ssh. Like a live interaction, getting rid of True/False results, and mixing stdout with stderr. You can still use use 'run_cmd' method if you don't care. - But I would recommed you to work like this: + But I would recommend you to work like this: """ TCP_TIMEOUT = 10.0 @@ -49,14 +52,14 @@ class Session(Executor.Session): """ Represents active ssh connection """ - def __init__(self, executor, timeout=None, use_pkey=False): + def __init__(self, executor, timeout=None): super(RemoteExecutor.Session, self).__init__(executor) if timeout is None: timeout = RemoteExecutor.TCP_TIMEOUT self._timeout = timeout self._ssh = paramiko.SSHClient() self._ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - if use_pkey: + if self._executor.use_pkey: self.pkey = paramiko.RSAKey.from_private_key_file( ID_RSA_PRV % os.path.expanduser('~') ) @@ -85,7 +88,8 @@ def open(self): username=self._executor.user.name, password=self._executor.user.password, timeout=self._timeout, - pkey=self.pkey + pkey=self.pkey, + port=self._executor.port, ) except (socket.gaierror, socket.herror) as ex: args = list(ex.args) @@ -196,44 +200,43 @@ def run(self, input_, timeout=None, get_pty=False): if input_: in_.write(input_) in_.close() - self.out = out.read() - self.err = err.read() + self.out = normalize_string(out.read()) + self.err = normalize_string(err.read()) return self.rc, self.out, self.err - def __init__(self, user, address, use_pkey=False): + def __init__(self, user, address, use_pkey=False, port=22): """ - :param user: user - :type user: instance of User - :param address: ip / hostname - :type address: str - :param use_pkey: use ssh private key in the connection - :type use_pkey: bool + Args: + use_pkey (bool): Use ssh private key in the connection + user (instance of User): User + address (str): Ip / hostname + port (int): Port to connect """ super(RemoteExecutor, self).__init__(user) self.address = address self.use_pkey = use_pkey + self.port = port def session(self, timeout=None): """ - :param timeout: tcp timeout - :type timeout: float - :return: the session - :rtype: instance of RemoteExecutor.Session + Args: + timeout (float): Tcp timeout + + Returns: + instance of RemoteExecutor.Session: The session """ - return RemoteExecutor.Session(self, timeout, self.use_pkey) + return RemoteExecutor.Session(self, timeout) def run_cmd(self, cmd, input_=None, tcp_timeout=None, io_timeout=None): """ - :param cmd: command - :type cmd: list - :param input_: input data - :type input_: str - :param tcp_timeout: tcp timeout - :type tcp_timeout: float - :param io_timeout: timeout for data operation (read/write) - :type io_timeout: float - :return: rc, out, err - :rtype: tuple (int, str, str) + Args: + tcp_timeout (float): Tcp timeout + cmd (list): Command + input_ (str): Input data + io_timeout (float): Timeout for data operation (read/write) + + Returns: + tuple (int, str, str): Rc, out, err """ with self.session(tcp_timeout) as session: return session.run_cmd(cmd, input_, io_timeout) @@ -242,10 +245,11 @@ def is_connective(self, tcp_timeout=20.0): """ Check if address is connective via ssh - :param tcp_timeout: time to wait for response - :type tcp_timeout: float - :return: True if address is connective, False otherwise - :rtype: bool + Args: + tcp_timeout (float): Time to wait for response + + Returns: + bool: True if address is connective, false otherwise """ try: self.logger.info( @@ -263,24 +267,25 @@ def is_connective(self, tcp_timeout=20.0): def wait_for_connectivity_state( self, positive, timeout=CONNECTIVITY_TIMEOUT, - sample_time=CONNECTIVITY_SAMPLE_TIME + sample_time=CONNECTIVITY_SAMPLE_TIME, + tcp_connection_timeout=TCP_CONNECTION_TIMEOUT ): """ Wait until address will be connective or not via ssh - :param positive: wait for the positive or negative connective state - :type positive: bool - :param timeout: wait timeout - :type timeout: int - :param sample_time: sample the ssh each sample_time seconds - :type sample_time: int - :return: True, if positive and ssh is connective or - negative and ssh does not connective, otherwise False - :rtype: bool + Args: + positive (bool): Wait for the positive or negative connective state + timeout (int): Wait timeout + sample_time (int): Sample the ssh each sample_time seconds + tcp_connection_timeout (int): TCP connection timeout + + Returns: + bool: True, if positive and ssh is connective or negative and ssh + does not connective, otherwise false """ reachable = "unreachable" if positive else "reachable" timeout_counter = 0 - while self.is_connective() != positive: + while self.is_connective(tcp_timeout=tcp_connection_timeout) != positive: if timeout_counter > timeout: self.logger.error( "Address %s is still %s via ssh, after %s seconds", @@ -290,3 +295,13 @@ def wait_for_connectivity_state( time.sleep(sample_time) timeout_counter += sample_time return True + + +class RemoteExecutorFactory(ExecutorFactory): + def __init__(self, use_pkey=False, port=22): + self.use_pkey = use_pkey + self.port = port + + def build(self, host, user): + return RemoteExecutor( + user, host.ip, use_pkey=self.use_pkey, port=self.port) diff --git a/rrmngmnt/storage.py b/rrmngmnt/storage.py index 179a010..f76cf5d 100644 --- a/rrmngmnt/storage.py +++ b/rrmngmnt/storage.py @@ -17,16 +17,15 @@ def mount(self, source, target=None, opts=None): Mounts source to target mount point __author__ = "ratamir" - :param source: Full path to source - :type source: str - :param target: Path to target directory, if omitted, a temporary - folder is created instead - :type target: str - :param opts: List of mount options such as: - ['-t', 'nfs', '-o', 'vers=3'] - :type opts: list - :return: Path to mount point if succeeded, None otherwise - :rtype: str + + Args: + source (str): Full path to source + target (str): Path to target directory, if omitted, a temporary + folder is created instead + opts (list): List of mount options such as + + Returns: + str: Path to mount point if succeeded, none otherwise """ target = '/tmp/mnt_point' if target is None else target cmd = ['mkdir', '-p', target] @@ -59,17 +58,17 @@ def umount(self, mount_point, force=True, remove_mount_point=True): optionally removes 'mount_point' __author__ = "ratamir" - :param mount_point: Path to directory that should be unmounted - :type mount_point: str - :param force: True if the mount point should be forcefully removed - (such as in the case of an unreachable NFS server) - :type force: bool - :param remove_mount_point: True if mount point should be deleted - after 'umount' operation completes, False otherwise - :type remove_mount_point: bool - :return: True if umount operation and mount point removal - succeeded, False otherwise - :rtype: bool + + Args: + mount_point (str): Path to directory that should be unmounted + force (bool): True if the mount point should be forcefully removed + (such as in the case of an unreachable nfs server) + remove_mount_point (bool): True if mount point should be deleted + after 'umount' operation completes, false otherwise + + Returns: + bool: True if umount operation and mount point removal succeeded, + false otherwise """ cmd = ['umount', mount_point, '-v'] if force: @@ -99,21 +98,22 @@ def lvchange(self, vg_name, lv_name, activate=True): (by setting it's 'active' attribute) __author__ = "ratamir" - :param vg_name: The name of the Volume group under which the LV resides - :type vg_name: str - :param lv_name: The name of the logical volume which will be - activated or deactivated - :type lv_name: str - :param activate: True when the logical volume should be activated, - False when it should be deactivated - :type activate: bool - :returns: True if setting the logical volume 'active' flag - succeeded, False otherwise - :rtype: bool + + Args: + activate (bool): True when the logical volume should be activated, + false when it should be deactivated + vg_name (str): The name of the volume group under which the lv + resides + lv_name (str): The name of the logical volume which will be + activated or deactivated + + Returns: + bool: True if setting the logical volume 'active' flag succeeded, + false otherwise """ active = 'y' if activate else 'n' return self.host.run_command( - shlex.split(LV_CHANGE_CMD % active, vg_name, lv_name) + shlex.split(LV_CHANGE_CMD % (active, vg_name, lv_name)) )[0] == 0 def pvscan(self): @@ -121,7 +121,8 @@ def pvscan(self): Execute 'pvscan' in order to get the current list of physical volumes __author__ = "ratamir" - :returns: True if the pvscan command succeded, False otherwise - :rtype: bool + + Returns: + bool: True if the pvscan command succeded, false otherwise """ return self.host.run_command(['pvscan'])[0] == 0 diff --git a/rrmngmnt/user.py b/rrmngmnt/user.py index 2fd0b94..ff590c8 100644 --- a/rrmngmnt/user.py +++ b/rrmngmnt/user.py @@ -4,12 +4,11 @@ class User(Resource): def __init__(self, name, password): """ - :param name: user name - :type name: str - :param password: password - :type password: str + Args: + password (str): Password + name (str): User name """ - super(Resource, self).__init__() + super(User, self).__init__() self.name = name self.password = password @@ -31,12 +30,10 @@ def __init__(self, password): class Domain(Resource): def __init__(self, name, provider=None, server=None): """ - :param name: name of domain - :type name: str - :param provider: name of provider / type of domain - :type provider: str - :param server: server address - :type server: str + Args: + server (str): Server address + name (str): Name of domain + provider (str): Name of provider / type of domain """ super(Domain, self).__init__() self.name = name @@ -54,12 +51,10 @@ def __init__(self): class ADUser(User): def __init__(self, name, password, domain): """ - :param name: user name - :type name: str - :param password: password - :type password: str - :param domain: user domain - :type domain: instance of Domain + Args: + domain (instance of Domain): User domain + password (str): Password + name (str): User name """ super(ADUser, self).__init__(name, password) self.domain = domain diff --git a/setup.cfg b/setup.cfg index 94edc43..7b9940a 100644 --- a/setup.cfg +++ b/setup.cfg @@ -3,7 +3,7 @@ name = python-rrmngmnt author = Lukas Bednar author-email = lukyn17@gmail.com summary = Tool to manage remote systems and services -description-file = README.md +description-file = README.rst home-page = https://github.com/rhevm-qe-automation/python-rrmngmnt license = GPLv2 classifier = @@ -13,9 +13,12 @@ classifier = License :: OSI Approved :: GNU General Public License v2 (GPLv2) Operating System :: POSIX Programming Language :: Python - Programming Language :: Python :: 2.6 + Programming Language :: Python :: 2 Programming Language :: Python :: 2.7 + Programming Language :: Python :: 3 Programming Language :: Python :: 3.4 + Programming Language :: Python :: 3.5 + Programming Language :: Python :: 3.6 keywords = remote resource @@ -32,3 +35,5 @@ build_requires = python python-setuptools python-pbr +[sdist] +formats = zip diff --git a/test-requirements.txt b/test-requirements.txt index 5b6c82e..9252814 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -1,2 +1,5 @@ pytest>=2.8.7 pytest-cov +pytest-docker +docker-compose +attrs==19.1.0 diff --git a/tests/common.py b/tests/common.py index 362da02..77f72e6 100644 --- a/tests/common.py +++ b/tests/common.py @@ -1,6 +1,6 @@ import contextlib from subprocess import list2cmdline -from rrmngmnt.executor import Executor +from rrmngmnt.executor import Executor, ExecutorFactory import six @@ -21,6 +21,23 @@ def close(self): six.StringIO.close(self) +class ByteFakeFile(six.BytesIO): + def __init__(self, buf=six.b('')): + six.BytesIO.__init__(self, six.b(buf)) + self.data = None + + def __exit__(self, *args): + self.close() + + def __enter__(self): + return self + + def close(self): + self.seek(0) + self.data = self.read().decode("utf-8", errors="replace") + six.BytesIO.close(self) + + class FakeExecutor(Executor): cmd_to_data = None files_content = {} @@ -64,7 +81,10 @@ def open_file(self, name, mode): raise else: data = '' - data = FakeFile(data) + if len(mode) == 2 and mode[1] == 'b': + data = ByteFakeFile(data) + else: + data = FakeFile(data) if mode[0] == 'w': data.seek(0) self._executor.files_content[name] = data @@ -86,8 +106,8 @@ def run(self, input_, timeout=None): @contextlib.contextmanager def execute(self, bufsize=-1, timeout=None): rc, out, err = self._ss.get_data(self.cmd) - yield six.StringIO(), six.StringIO(out), six.StringIO(err) self._rc = rc + yield six.StringIO(), six.StringIO(out), six.StringIO(err) def __init__(self, user, address): super(FakeExecutor, self).__init__(user) @@ -101,13 +121,13 @@ def run_cmd(self, cmd, input_=None, tcp_timeout=None, io_timeout=None): return session.run_cmd(cmd, input_, io_timeout) -if __name__ == "__main__": - from rrmngmnt import RootUser - u = RootUser('password') - e = FakeExecutor(u) - e.cmd_to_data = {'echo ahoj': (0, 'ahoj', '')} - print(e.run_cmd(['echo', 'ahoj'])) - with e.session() as ss: - with ss.open_file('/tmp/a', 'w') as fh: - fh.write("ahoj") - print(e.files_content['/tmp/a'], e.files_content['/tmp/a'].data) +class FakeExecutorFactory(ExecutorFactory): + def __init__(self, cmd_to_data, files_content): + self.cmd_to_data = cmd_to_data.copy() + self.files_content = files_content + + def build(self, host, user): + fe = FakeExecutor(user, host.ip) + fe.cmd_to_data = self.cmd_to_data.copy() + fe.files_content = self.files_content + return fe diff --git a/tests/docker-compose.yml b/tests/docker-compose.yml new file mode 100644 index 0000000..3c5e189 --- /dev/null +++ b/tests/docker-compose.yml @@ -0,0 +1,7 @@ +--- +version: '3' +services: + ubuntu: + ports: + - "22221:22" + image: "chrismeyers/ubuntu12.04" diff --git a/tests/test_common.py b/tests/test_common.py index 315a0b7..63f09c4 100644 --- a/tests/test_common.py +++ b/tests/test_common.py @@ -1,11 +1,15 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- +import types + import pytest import netaddr -from rrmngmnt import common +from rrmngmnt import common, Host, User +from .common import FakeExecutorFactory +import six def test_fqdn2ip_positive(): - ip = common.fqdn2ip('github.org') + ip = common.fqdn2ip('github.com') assert netaddr.valid_ipv4(ip) @@ -13,3 +17,118 @@ def test_fqdn2ip_negative(): with pytest.raises(Exception) as ex_info: common.fqdn2ip('github.or') assert 'github.or' in str(ex_info.value) + + +class TestCommandReader(object): + + data = { + 'cat shopping_list.txt': (0, 'bananas\nmilk\nhuge blender', ''), + 'cat milk_shake_recipe.txt': ( + 1, '', 'cat: milk_shake_recipe.txt: No such file or directory' + ), + } + files = {} + + @classmethod + @pytest.fixture(scope='class') + def fake_host(cls): + fh = Host('1.1.1.1') + fh.add_user(User('root', '11111')) + fh.executor_factory = FakeExecutorFactory(cls.data, cls.files) + return fh + + def test_return_type(self, fake_host): + """ Test that CommandReader returns generator type """ + cmd = 'cat shopping_list.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + ret = cmd_reader.read_lines() + + assert isinstance(ret, types.GeneratorType) + + expected_output = self.data[cmd][1].split('\n') + for i in range(len(expected_output)): + assert next(ret) == expected_output[i] + + with pytest.raises(StopIteration): + next(ret) + + def test_iterate_over_output(self, fake_host): + """ + Test that we can iterate over CommandReader's output using for loop + """ + cmd = 'cat shopping_list.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + expected_output = self.data[cmd][1].split('\n') + cmd_reader_output = [] + + for line in cmd_reader.read_lines(): + cmd_reader_output.append(line) + + assert cmd_reader_output == expected_output + + def test_return_code(self, fake_host): + """ Test that rc of command is captured by CommandReader """ + cmd = 'cat shopping_list.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + + assert cmd_reader.rc is None + + for line in cmd_reader.read_lines(): + pass + + assert not cmd_reader.rc + + def test_stderr(self, fake_host): + """ Test that error output is captured by CommandReader """ + cmd = 'cat milk_shake_recipe.txt' + cmd_reader = common.CommandReader(fake_host.executor(), cmd.split()) + + assert not cmd_reader.err + + for line in cmd_reader.read_lines(): + pass + + assert cmd_reader.rc + assert cmd_reader.err + + +def test_normalize_string_bytes_input(): + """ + Test 'normalize_string' function with 'bytes' input + + In python3 we want to convert bytes() to str(), + to eliminate TypeError exception when mixing between bytes & str + at the calling function + In python2 there is no meaning if the output will by bytes() or str() + python will not raise a TypeError exception when mixing between them + """ + if six.PY3: + assert type(common.normalize_string(data=bytes())) == str + + +def test_normalize_string_str_input(): + """ + Test 'normalize_string' function with 'str' input + + Keep the str type in python3 + Convert the str type to unicode in python2 + """ + try: + expected_type = unicode # Python2 + except NameError: + expected_type = str # Python3 + + assert type(common.normalize_string(data=str())) == expected_type + + +def test_normalize_string_unicode_input(): + """ + Test 'normalize_string' function with 'unicode' input (PY2 only) + + In python2 unicode input should not be converted + 'unicode' is not supported at python3 + """ + if six.PY2: + assert type( + common.normalize_string(data=unicode()) # noqa: F821 + ) == unicode # noqa: F821 diff --git a/tests/test_db.py b/tests/test_db.py index c05260a..970cf9a 100644 --- a/tests/test_db.py +++ b/tests/test_db.py @@ -1,25 +1,20 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- import pytest from rrmngmnt import Host, User from rrmngmnt.db import Database -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files_content = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestDb(object): @@ -39,6 +34,25 @@ class TestDb(object): '-R __RECORD_SEPARATOR__ -t -A -c "SELECT * FROM table ERROR"': ( 1, "", "Syntax Error" ), + 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' + '-c \\\\dt': ( + 0, + ( + "List of relations\n" + " Schema | Name | Type | Owner\n" + "--------+----------------------+-------+---------\n" + " public | test_table | table | postgres\n" + ), + "" + ), + 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' + '-c \\\\dv': ( + 0, "", "Did not find any relations." + ), + 'export PGPASSWORD=db_pass; psql -d db_name -U db_user -h localhost ' + '-c \\\\gg': ( + 1, "", "invalid command \\gg" + ), } files = {} @@ -51,10 +65,10 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_db(self, ip='1.1.1.1'): + h = Host(ip) + h.add_user(User('root', '34546')) return Database( - Host(ip), - self.db_name, - User(self.db_user, self.db_pass), + h, self.db_name, User(self.db_user, self.db_pass), ) def test_restart(self): @@ -71,3 +85,16 @@ def test_negative(self): with pytest.raises(Exception) as ex_info: db.psql("SELECT * FROM table ERROR") assert "Syntax Error" in str(ex_info.value) + + def test_psql_cmd(self): + db = self.get_db() + res = db.psql_cmd('\\\\dt') + assert 'List of relations' in res + res = db.psql_cmd('\\\\dv') + assert res == 'Did not find any relations.' + + def test_negative_cmd(self): + db = self.get_db() + with pytest.raises(Exception) as ex_info: + db.psql_cmd('\\\\gg') + assert 'invalid command' in str(ex_info.value) diff --git a/tests/test_docker.py b/tests/test_docker.py new file mode 100644 index 0000000..23bfe83 --- /dev/null +++ b/tests/test_docker.py @@ -0,0 +1,25 @@ +import pytest +from rrmngmnt import Host, User +from rrmngmnt.ssh import RemoteExecutorFactory + + +@pytest.fixture(scope='session') +def provisioned_hosts(docker_ip, docker_services): + hosts = {} + for h in ('ubuntu',): + host = Host(docker_ip) + host.add_user(User("root", "docker.io")) + host.executor_factory = RemoteExecutorFactory( + port=docker_services.port_for(h, 22)) + executor = host.executor() + docker_services.wait_until_responsive( + timeout=30.0, pause=1, + check=lambda: executor.is_connective, + ) + hosts[h] = host + return hosts + + +def test_echo(provisioned_hosts): + ubuntu_host = provisioned_hosts['ubuntu'] + ubuntu_host.executor().run_cmd(['echo', 'hello']) diff --git a/tests/test_filesystem.py b/tests/test_filesystem.py index b5bdb25..441c88a 100644 --- a/tests/test_filesystem.py +++ b/tests/test_filesystem.py @@ -1,25 +1,20 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- import pytest from rrmngmnt import Host, User from rrmngmnt import errors -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=User('fakeuser', 'password'), pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files_content = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestFilesystem(object): @@ -30,6 +25,11 @@ class TestFilesystem(object): '[ -f /tmp/nofile ]': (1, '', ''), '[ -d /tmp/dir ]': (0, '', ''), '[ -d /tmp/nodir ]': (1, '', ''), + '[ -d /path/to/file1 ]': (1, '', ''), + '[ -d /path/to ]': (0, '', ''), + '[ -d somefile ]': (1, '', ''), + '[ -x /tmp/executable ]': (0, '', ''), + '[ -x /tmp/nonexecutable ]': (1, '', ''), 'rm -f /path/to/remove': (0, '', ''), 'rm -f /dir/to/remove': ( 1, '', 'rm: cannot remove ‘.tox/’: Is a directory', @@ -38,16 +38,28 @@ class TestFilesystem(object): 'cat %s' % "/tmp/file": (0, 'data', ''), 'chmod +x /tmp/hello.sh': (0, '', ''), 'mkdir /dir/to/remove': (0, '', ''), + 'mkdir -p -m 600 /dir/to/remove2/remove': (0, '', ''), 'chown root:root /dir/to/remove': (0, '', ''), 'chmod 600 /dir/to/remove': (0, '', ''), 'chmod 600 /tmp/nofile': ( 1, '', 'chmod: cannot access ‘/tmp/nofile’: No such file or directory', ), - 'touch /path/to/file': (0, '', ''), + 'touch /path/to/file /path/to/file1': (0, '', ''), + 'touch /path/to/file2': (0, '', ''), 'touch /path/to/nopermission': (1, '', ''), 'ls -A1 /path/to/empty': (0, '\n', ''), 'ls -A1 /path/to/two': (0, 'first\nsecond\n', ''), + 'mktemp -d': (0, '/path/to/tmpdir', ''), + 'mount -v -t xfs -o bind,ro /path/to /path/to/tmpdir': (0, '', ''), + 'ls -A1 /path/to/tmpdir': (0, 'first\nsecond\n', ''), + 'mount -v -o remount,bind,rw /path/to/tmpdir': (0, '', ''), + 'umount -v -f /path/to/tmpdir': (0, '', ''), + 'mount -v /not/device /path/to/tmpdir': ( + 32, '', '/not/device is not a block device\n' + ), + 'truncate -s 0 /tmp/file_to_flush': (0, '', ''), + 'mv /tmp/source /tmp/destination': (0, '', ''), } files = {} @@ -56,7 +68,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def test_exists_positive(self): assert self.get_host().fs.exists('/tmp/exits') @@ -76,6 +90,12 @@ def test_isdir_positive(self): def test_isdir_negative(self): assert not self.get_host().fs.isdir('/tmp/nodir') + def test_isexec_positive(self): + assert self.get_host().fs.isexec('/tmp/executable') + + def test_isexec_negative(self): + assert not self.get_host().fs.isexec('/tmp/nonexecutable') + def test_remove_positive(self): assert self.get_host().fs.remove('/path/to/remove') @@ -92,7 +112,19 @@ def test_rmdir_negative(self): def test_read_file(self): assert self.get_host().fs.read_file("/tmp/file") == "data" - def test_create_sctript(self): + def test_move(self): + assert self.get_host().fs.move("/tmp/source", "/tmp/destination") + + def test_flush_file(self): + assert self.get_host().fs.flush_file("/tmp/file_to_flush") + + def test_create_file(self): + data = "hello world" + path = "/tmp/hello.txt" + self.get_host().fs.create_file(data, path) + assert self.files[path].data == data + + def test_create_script(self): data = "echo hello" path = '/tmp/hello.sh' self.get_host().fs.create_script(data, path) @@ -101,6 +133,11 @@ def test_create_sctript(self): def test_mkdir_positive(self): self.get_host().fs.mkdir('/dir/to/remove') + def test_mkdir_pm_positive(self): + self.get_host().fs.mkdir( + '/dir/to/remove2/remove', parents=True, mode='600' + ) + def test_chown_positive(self): self.get_host().fs.chown('/dir/to/remove', 'root', 'root') @@ -113,10 +150,13 @@ def test_chmod_negative(self): assert "No such file or directory" in str(ex_info.value) def test_touch_positive(self): - assert self.get_host().fs.touch('/path/to/file', '') + assert self.get_host().fs.touch('/path/to/file', '/path/to/file1') def test_touch_negative(self): - assert not self.get_host().fs.touch('/path/to/nopermission', '') + assert not self.get_host().fs.touch('/path/to/nopermission') + + def test_backwards_comp_touch(self): + assert self.get_host().fs.touch('file2', '/path/to') def test_touch_wrong_params(self): with pytest.raises(Exception) as ex_info: @@ -130,3 +170,70 @@ def test_listdir_two(self): assert self.get_host().fs.listdir('/path/to/two') == [ 'first', 'second', ] + + def test_mount_point(self): + with self.get_host().fs.mount_point( + '/path/to', opts='bind,ro', fs_type='xfs' + ) as mp: + assert not mp.remount('bind,rw') + + def test_fail_mount(self): + with pytest.raises(errors.MountError) as ex_info: + with self.get_host().fs.mount_point('/not/device'): + pass + assert "is not a block device" in str(ex_info.value) + + +class TestFSGetPutFile(object): + data = { + "[ -d /path/to/put_dir ]": (0, "", ""), + } + files = { + "/path/to/get_file": "data of get_file", + } + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + h = Host(ip) + h.add_user(User('root', '11111')) + return h + + def test_get(self, tmpdir): + self.get_host().fs.get("/path/to/get_file", str(tmpdir)) + assert tmpdir.join("get_file").read() == "data of get_file" + + def test_put(self, tmpdir): + p = tmpdir.join("put_file") + p.write("data of put_file") + self.get_host().fs.put(str(p), "/path/to/put_dir") + assert self.files[ + '/path/to/put_dir/put_file'].data == "data of put_file" + + +class TestTransfer(object): + data = { + "[ -d /path/to/dest_dir ]": (0, "", ""), + } + files = { + "/path/to/file_to_transfer": "data to transfer", + } + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + h = Host(ip) + h.add_user(User('root', '11111')) + return h + + def test_transfer(self): + self.get_host().fs.transfer( + "/path/to/file_to_transfer", self.get_host("1.1.1.2"), + "/path/to/dest_dir", + ) + assert self.files[ + '/path/to/dest_dir/file_to_transfer'].data == "data to transfer" diff --git a/tests/test_firewall.py b/tests/test_firewall.py new file mode 100644 index 0000000..98c655f --- /dev/null +++ b/tests/test_firewall.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +import pytest +from rrmngmnt import Host +from rrmngmnt.user import RootUser +from .common import FakeExecutorFactory + +host_executor_factory = Host.executor_factory + + +def teardown_module(): + Host.executor_factory = host_executor_factory + + +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) + + +def get_host(ip='1.1.1.1'): + h = Host(ip) + h.users.append(RootUser('123456')) + return h + + +class TestFirewall(object): + + data = { + 'which systemctl': (0, '/usr/bin/systemctl', ''), + 'systemctl list-unit-files | grep -o ^[^.][^.]*.service ' + '| cut -d. -f1 | sort | uniq': ( + 0, + '\n'.join( + [ + 'iptables', + 'noniptables', + ] + ), + '' + ), + 'systemctl status iptables.service': (0, '', ''), + 'systemctl status noniptables.service': (1, '', ''), + } + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data) + + def test_running_service_positive(self): + assert get_host().firewall.is_active('iptables') + + +class TestChain(object): + + data = { + 'iptables --append OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --append INPUT --source 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --insert OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --insert INPUT --source 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --delete OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --delete INPUT --source 2.2.2.2 --jump DROP ' + '--protocol all': (0, '', ''), + 'iptables --append OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol tcp --match multiport --dports ' + '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15': (0, '', ''), + 'iptables --append OUTPUT --destination 2.2.2.2 --jump DROP ' + '--protocol tcp --match multiport --dports ' + '1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, 16': ( + 4, '', 'iptables v1.4.21: too many ports specified' + ), + 'iptables --flush OUTPUT': (0, '', '') + } + + destination_host = {'address': ['2.2.2.2']} + ports = [ + '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', + '14', '15' + ] + too_many_ports = [ + '1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11', '12', '13', + '14', '15', '16' + ] + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data) + + def test_wrong_chain_name(self): + with pytest.raises(NotImplementedError): + get_host().firewall.chain('CHAIN') + + def test_add_outgoing_rule(self): + assert get_host().firewall.chain('OUTPUT').add_rule( + self.destination_host, 'DROP' + ) + + def test_add_incoming_rule(self): + assert get_host().firewall.chain('INPUT').add_rule( + self.destination_host, 'DROP' + ) + + def test_insert_outgoing_rule(self): + assert get_host().firewall.chain('OUTPUT').insert_rule( + self.destination_host, 'DROP' + ) + + def test_insert_incoming_rule(self): + assert get_host().firewall.chain('INPUT').insert_rule( + self.destination_host, 'DROP' + ) + + def test_delete_outgoing_rule(self): + assert get_host().firewall.chain('OUTPUT').delete_rule( + self.destination_host, 'DROP' + ) + + def test_delete_incoming_rule(self): + assert get_host().firewall.chain('OUTPUT').delete_rule( + self.destination_host, 'DROP' + ) + + def test_add_outgoing_rule_with_ports(self): + assert get_host().firewall.chain('OUTPUT').add_rule( + self.destination_host, 'DROP', ports=self.ports + ) + + def test_add_outgoing_rule_with_too_many_ports(self): + with pytest.raises(NotImplementedError): + get_host().firewall.chain('OUTPUT').add_rule( + self.destination_host, 'DROP', ports=self.too_many_ports + ) + + def test_clean_firewall_rules(self): + assert get_host().firewall.chain('OUTPUT').clean_rules() diff --git a/tests/test_host.py b/tests/test_host.py index 9e0116f..2face3e 100644 --- a/tests/test_host.py +++ b/tests/test_host.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import Host, User, RootUser import pytest @@ -30,3 +30,16 @@ def test_executor_user(self): h.executor_user = user e = h.executor() e.user.name == 'lukas' + + +class TestHostFqdnIp(object): + + def test_host_ip(self): + h = Host('127.0.0.1') + assert h.ip == '127.0.0.1' + assert 'localhost' in h.fqdn + + def test_host_fqdn(self): + h = Host('localhost') + assert h.ip == '127.0.0.1' + assert 'localhost' in h.fqdn diff --git a/tests/test_network.py b/tests/test_network.py index 9fb2c2e..a184b58 100644 --- a/tests/test_network.py +++ b/tests/test_network.py @@ -1,22 +1,17 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import Host, RootUser -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) def get_host(ip='1.1.1.1'): @@ -48,6 +43,19 @@ class TestNetwork(object): ), '', ), + 'ip -6 route': ( + 0, + '\n'.join( + [ + 'unreachable ::/96 dev lo metric 1024 error -101', + 'unreachable 2002:a9fe::/32 lo metric 1024 error -101', + 'unreachable 2002:ac10::/28 lo metric 1024 error -101', + 'fe80:52:0::3fe dev eth0 proto static metric 100 ', + 'default via fe80::0:3fe dev eth0 proto static metric 100', + ] + ), + '', + ), 'ip addr': ( 0, ''.join( @@ -115,6 +123,36 @@ class TestNetwork(object): ), '' ), + 'ip addr show eth0': ( + 0, + '\n'.join( + [ + '2: eth0: mtu ...', + 'link/ether 00:1a:4a:01:3f:1c brd ff:ff:ff:ff:ff:ff', + 'inet 10.11.12.84/22 brd 10.11.12.255 scope global', + 'valid_lft 20343sec preferred_lft 20343sec', + 'inet6 2620:52:0::fe01:3f1c/64 scope global dynamic', + 'valid_lft 2591620sec preferred_lft 604420sec', + 'inet6 fe80::4aff:fe01:3f1c/64 scope link ', + 'valid_lft forever preferred_lft forever', + ] + ), + '' + ), + 'ip -6 addr show eth0': ( + 0, + '\n'.join( + [ + '2: eth0: mtu ...', + 'link/ether 00:1a:4a:01:3f:1c brd ff:ff:ff:ff:ff:ff', + 'inet6 2620:52:0::fe01:3f1c/64 scope global dynamic', + 'valid_lft 2591620sec preferred_lft 604420sec', + 'inet6 fe80::4aff:fe01:3f1c/64 scope link ', + 'valid_lft forever preferred_lft forever', + ] + ), + '' + ), 'brctl show | sed -e "/^bridge name/ d" ' '-e \'s/^\\s\\s*\\(\\S\\S*\\)$/CONT:\\1/I\'': ( 0, @@ -128,7 +166,7 @@ class TestNetwork(object): ), 'brctl addbr br1': (0, '', ''), 'brctl addif br1 net1': (0, '', ''), - 'ls -la /sys/class/net | grep \'dummy_\|pci\' | grep -o \'[^/]*$\'': ( + 'ls -la /sys/class/net | grep \'dummy_\\|pci\' | grep -o \'[^/]*$\'': ( 0, '\n'.join( [ @@ -145,8 +183,14 @@ class TestNetwork(object): "Permanent address: 44:1e:a1:73:3c:98", '' ), - 'ip link set up interface': True, - 'ip link set down interface': True, + 'ip link set interface up': True, + 'ip link set interface down': True, + "ethtool -i eth0": (0, "driver: e1000", ""), + "cat /sys/class/net/eth0/speed": (0, "1000", ""), + "cat /sys/class/net/eth0/operstate": (0, "up", ""), + "ping 1.2.3.4 -c 5 -s 10 -M do": (0, "something", ""), + 'ip address add 1.2.3.4/24 dev eth0': (0, "", ""), + 'ip address add 1.2.3.4/255.255.255.0 dev eth0': (0, "", ""), } files = { } @@ -162,6 +206,7 @@ def test_get_info(self): 'ip': '10.11.12.83', 'gateway': '10.11.12.254', 'interface': 'enp5s0f0', + 'prefix': '24' } assert info == expected_info @@ -201,12 +246,39 @@ def test_get_mac_address_by_ip(self): expected = "44:1e:a1:73:3c:98" assert get_host().network.get_mac_by_ip("10.11.12.83") == expected + def test_find_ip_by_int(self): + assert get_host().network.find_ip_by_int("eth0") == "10.11.12.84" + + def test_find_ipv6_by_int(self): + expected = "2620:52:0::fe01:3f1c" + assert get_host().network.find_ipv6_by_int("eth0") == expected + + def test_find_default_gwv6(self): + assert get_host().network.find_default_gwv6() == "fe80::0:3fe" + def if_up(self): assert get_host().network.if_up("interface") def if_down(self): assert get_host().network.if_down("interface") + def test_get_interface_speed(self): + assert get_host().network.get_interface_speed("eth0") == "1000" + + def test_get_interface_status(self): + assert get_host().network.get_interface_status("eth0") == "up" + + def test_send_icmp(self): + assert get_host().network.send_icmp('1.2.3.4', size="10") + + def test_add_ip_with_bitmask(self): + assert get_host().network.add_ip(nic="eth0", ip="1.2.3.4", mask="24") + + def test_add_ip_with_subnet_mask(self): + assert get_host().network.add_ip( + nic="eth0", ip="1.2.3.4", mask="255.255.255.0" + ) + class TestHostNameCtl(object): @@ -235,7 +307,7 @@ class TestHostNameEtc(object): data = { 'which hostnamectl': (1, '', ''), - 'hostname': (0, 'local', ''), + 'hostname -f': (0, 'local', ''), 'hostname something ; sed -i -e /^HOSTNAME/d /etc/sysconfig/network ' '&& echo HOSTNAME=something >> /etc/sysconfig/network': (0, '', ''), } diff --git a/tests/test_os.py b/tests/test_os.py index 9622e74..2c53713 100644 --- a/tests/test_os.py +++ b/tests/test_os.py @@ -1,25 +1,20 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- import pytest from rrmngmnt import Host, User from rrmngmnt import errors -from .common import FakeExecutor +from .common import FakeExecutorFactory -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data, files): - def executor(self, user=User("fake", "pass"), pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - e.files_content = files - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestOperatingSystem(object): @@ -61,7 +56,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def test_get_release_str(self): result = self.get_host().os.release_str @@ -106,7 +103,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11114')) + return h def test_get_release_str(self): with pytest.raises(errors.CommandExecutionFailure) as ex_info: @@ -138,7 +137,9 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '22222')) + return h def test_get_release_info(self): with pytest.raises(errors.UnsupportedOperation) as ex_info: @@ -169,9 +170,166 @@ def setup_class(cls): fake_cmd_data(cls.data, cls.files) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '3333')) + return h def test_get_release_info(self): info = self.get_host().os.release_info assert 'VERSION_ID' not in info assert len(info) == 4 + + +type_map = { + 'st_mode': ('0x%f', lambda x: int(x, 16)), + 'st_ino': ('%i', int), + 'st_dev': ('%d', int), + 'st_nlink': ('%h', int), + 'st_uid': ('%u', int), + 'st_gid': ('%g', int), + 'st_size': ('%s', int), + 'st_atime': ('%X', int), + 'st_mtime': ('%Y', int), + 'st_ctime': ('%W', int), + 'st_blocks': ('%b', int), + 'st_blksize': ('%o', int), + 'st_rdev': ('%t', int), +} + + +class TestFileStats(object): + data = { + 'stat -c %s /tmp/test' % + ','.join(["%s=%s" % (k, v[0]) for k, v in type_map.items()]): ( + 0, + ( + 'st_ctime=0,' + 'st_rdev=0,' + 'st_blocks=1480,' + 'st_nlink=1,' + 'st_gid=0,' + 'st_dev=2051,' + 'st_ino=11804680,' + 'st_mode=0x81a4,' + 'st_mtime=1463487739,' + 'st_blksize=4096,' + 'st_size=751764,' + 'st_uid=0,' + 'st_atime=1463487196' + ), + '' + ), + 'stat -c "%U %G" /tmp/test': ( + 0, + 'root root', + '' + ), + 'stat -c %a /tmp/test': ( + 0, + '644\n', + '' + ), + 'id -u root': ( + 0, + '', + '' + ), + 'id -g root': ( + 0, + '', + '' + ) + } + files = {} + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + h = Host(ip) + h.add_user(User('root', '11331')) + return h + + def test_get_file_stats(self): + file_stats = self.get_host().os.stat('/tmp/test') + assert ( + file_stats.st_mode == 33188 and + file_stats.st_uid == 0 and + file_stats.st_gid == 0 + ) + + def test_get_file_owner(self): + file_user, file_group = self.get_host().os.get_file_owner('/tmp/test') + assert file_user == 'root' and file_group == 'root' + + def test_get_file_permissions(self): + assert self.get_host().os.get_file_permissions('/tmp/test') == '644' + + def test_user_exists(self): + assert self.get_host().os.user_exists('root') + + def test_group_exists(self): + assert self.get_host().os.group_exists('root') + + +class TestFileStatsNegative(object): + data = { + 'stat -c %s /tmp/negative_test' % + ','.join(["%s=%s" % (k, v[0]) for k, v in type_map.items()]): ( + 1, + '', + 'cannot stat ‘/tmp/negative_test’: No such file or directory' + ), + 'stat -c "%U %G" /tmp/negative_test': ( + 1, + '', + 'cannot stat ‘/tmp/negative_test’: No such file or directory' + ), + 'stat -c %a /tmp/negative_test': ( + 1, + '', + 'cannot stat ‘/tmp/negative_test’: No such file or directory' + ), + 'id -u test': ( + 1, + '', + '' + ), + 'id -g test': ( + 1, + '', + '' + ) + } + files = {} + + @classmethod + def setup_class(cls): + fake_cmd_data(cls.data, cls.files) + + def get_host(self, ip='1.1.1.1'): + h = Host(ip) + h.add_user(User('root', '155511')) + return h + + def test_get_file_stats(self): + with pytest.raises(errors.CommandExecutionFailure) as ex_info: + self.get_host().os.stat('/tmp/negative_test') + assert "No such file" in str(ex_info.value) + + def test_get_file_owner(self): + with pytest.raises(errors.CommandExecutionFailure) as ex_info: + self.get_host().os.get_file_owner('/tmp/negative_test') + assert "No such file" in str(ex_info.value) + + def test_get_file_permissions(self): + with pytest.raises(errors.CommandExecutionFailure) as ex_info: + self.get_host().os.get_file_permissions('/tmp/negative_test') + assert "No such file" in str(ex_info.value) + + def test_user_exists(self): + assert not self.get_host().os.user_exists('test') + + def test_group_exists(self): + assert not self.get_host().os.group_exists('test') diff --git a/tests/test_package_manager.py b/tests/test_package_manager.py index c7ff2c1..d968e04 100644 --- a/tests/test_package_manager.py +++ b/tests/test_package_manager.py @@ -1,11 +1,11 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import Host, User -from .common import FakeExecutor +from .common import FakeExecutorFactory import rrmngmnt.package_manager as pm from rrmngmnt.package_manager import PackageManagerProxy as PMProxy from subprocess import list2cmdline -host_executor = Host.executor +host_executor_factory = Host.executor_factory def extend_cmd(cmd, *args): @@ -22,15 +22,11 @@ def join_cmds(*args): def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data): - def executor(self, user=User('fakeuser', 'password'), pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class BasePackageManager(object): @@ -130,7 +126,9 @@ def set_base_data(cls): }) def get_host(self, ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '11111')) + return h def get_pm(self): return self.get_host().package_manager diff --git a/tests/test_playbook_runner.py b/tests/test_playbook_runner.py new file mode 100644 index 0000000..66d7f90 --- /dev/null +++ b/tests/test_playbook_runner.py @@ -0,0 +1,286 @@ +import os.path + +import pytest + +from rrmngmnt import Host, User +from rrmngmnt.playbook_runner import PlaybookRunner +from .common import FakeExecutorFactory + + +class PlaybookRunnerBase(object): + + # The fake run UUID will be used instead of unique ID that's auto-generated + # for each playbook execution + fake_run_uuid = '123' + + playbook_name = 'test.yml' + playbook_content = '' + vars_file_name = 'my_vars.yml' + vars_file_content = '' + inventory_name = 'my_inventory' + inventory_content = '' + ssh_no_strict_host_key_checking = "-o StrictHostKeyChecking=no" + + tmp_dir = os.path.join(PlaybookRunner.tmp_dir, fake_run_uuid) + + success = (0, '', '') + failure = (1, '', '') + + data = { + # Filesystem-related operations + 'rm -rf {}'.format(tmp_dir): success, + 'mkdir {}'.format(tmp_dir): success, + '[ -d {tmp_dir}/{playbook} ]'.format( + tmp_dir=tmp_dir, playbook=playbook_name + ): failure, + '[ -d {tmp_dir}/{vars_file} ]'.format( + tmp_dir=tmp_dir, vars_file=vars_file_name + ): failure, + '[ -d {tmp_dir}/{inventory} ]'.format( + tmp_dir=tmp_dir, inventory=inventory_name + ): failure, + # Actual execution of ansible-playbook + # Basic scenario + '{bin} -i {tmp_dir}/{inventory} -v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # Extra vars have been provided + '{bin} -e@{tmp_dir}/{extra_vars} -i {tmp_dir}/{inventory} ' + '-v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + extra_vars=PlaybookRunner.extra_vars_file, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # File with additional variables has been provided + '{bin} -e@{tmp_dir}/{vars_file} -i {tmp_dir}/{inventory} ' + '-v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + vars_file=vars_file_name, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # Custom inventory has been provided + '{bin} -i {tmp_dir}/{inventory} -v {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=inventory_name, + playbook=playbook_name + ): success, + # Verbosity has been increased to max + '{bin} -i {tmp_dir}/{inventory} -vvvvv {tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + playbook=playbook_name + ): success, + # Running in check mode + '{bin} -i {tmp_dir}/{inventory} -v {check_mode_param} ' + '{tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + check_mode_param=PlaybookRunner.check_mode_param, + playbook=playbook_name + ): success, + # Running with extended SSH common args + '{bin} -i {tmp_dir}/{inventory} ' + '-v "{ssh_common_args_param}={ssh_common_args}" ' + '{tmp_dir}/{playbook}'.format( + bin=PlaybookRunner.binary, + tmp_dir=tmp_dir, + inventory=PlaybookRunner.default_inventory_name, + ssh_common_args_param=PlaybookRunner.ssh_common_args_param, + ssh_common_args=ssh_no_strict_host_key_checking, + playbook=playbook_name + ): success, + } + + @classmethod + @pytest.fixture(scope='class') + def fake_host(cls): + fh = Host('1.1.1.1') + fh.add_user(User('root', '11111')) + fh.executor_factory = FakeExecutorFactory(cls.data, cls.files) + return fh + + @pytest.fixture() + def fake_playbook(self, tmpdir): + fp = tmpdir.join(self.playbook_name) + fp.write(self.playbook_content) + return str(fp) + + @pytest.fixture() + def playbook_runner(self, fake_host): + playbook_runner = PlaybookRunner(fake_host) + playbook_runner.short_run_uuid = self.fake_run_uuid + return playbook_runner + + def check_files_on_host(self, files=None): + """ + Check that all files provided to files parameter (and only those) have + been "copied" to our imaginary host. In reality, they should not be + present on host once the playbook's execution is done. However here + we'll use the fact that our fake host does not really implements file + removal. Because of this, in the end of the test case, we can check + that files that should have been copied to the host (by using + FileSystem service) have actually been sent there. + + Args: + files (list): List of files that should have been copied to the + host. Don't include test playbook into this list since its + presence is implicitly expected. You can also provide only one + file as a string. + + Returns: + bool: True if files expected on host and those present match, + False otherwise + """ + if files is None: + files = [] + if isinstance(files, str): + files = [files] + expected_files = [os.path.join(self.tmp_dir, self.playbook_name)] + expected_files.extend(files) + return sorted(expected_files) == sorted(list(self.files.keys())) + + +class TestBasic(PlaybookRunnerBase): + + files = {} + + def test_basic_scenario(self, playbook_runner, fake_playbook): + """ User has provided only playbook """ + rc, _, _ = playbook_runner.run(playbook=fake_playbook) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) + + +class TestExtraVars(PlaybookRunnerBase): + + files = {} + + def test_extra_vars(self, playbook_runner, fake_playbook): + """ User has provided extra vars as a dictionary """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + extra_vars={ + "greetings": "hello", + } + ) + assert not rc + assert self.check_files_on_host( + [ + os.path.join( + self.tmp_dir, PlaybookRunner.default_inventory_name + ), + os.path.join(self.tmp_dir, PlaybookRunner.extra_vars_file) + ] + ) + + +class TestVarsFile(PlaybookRunnerBase): + + files = {} + + @pytest.fixture() + def fake_vars_file(self, tmpdir): + fvf = tmpdir.join(self.vars_file_name) + fvf.write(self.vars_file_content) + return str(fvf) + + def test_vars_file(self, playbook_runner, fake_playbook, fake_vars_file): + """ User has provided YAML file with custom variables """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + vars_files=[fake_vars_file] + ) + assert not rc + assert self.check_files_on_host( + [ + os.path.join( + self.tmp_dir, PlaybookRunner.default_inventory_name + ), + os.path.join(self.tmp_dir, self.vars_file_name) + ] + ) + + +class TestInventory(PlaybookRunnerBase): + + files = {} + + @pytest.fixture() + def fake_inventory(self, tmpdir): + fi = tmpdir.join(self.inventory_name) + fi.write(self.inventory_content) + return str(fi) + + def test_inventory(self, playbook_runner, fake_playbook, fake_inventory): + """ User has provided custom inventory instead of the default one """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + inventory=fake_inventory + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, self.inventory_name) + ) + + +class TestVerbosity(PlaybookRunnerBase): + + files = {} + + def test_max_verbosity(self, playbook_runner, fake_playbook): + """ User has increased verbosity to maximum level """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + verbose_level=5 + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) + + +class TestCheckMode(PlaybookRunnerBase): + + files = {} + + def test_check_mode(self, playbook_runner, fake_playbook): + """ User is running the playbook with --check param """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + run_in_check_mode=True + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) + + +class TestSSHCommonArgs(PlaybookRunnerBase): + + files = {} + + def test_no_strict_host_key_checking(self, playbook_runner, fake_playbook): + """ + User has provided custom SSH argument that extend default Ansible SSH + arguments + """ + rc, _, _ = playbook_runner.run( + playbook=fake_playbook, + ssh_common_args=[self.ssh_no_strict_host_key_checking] + ) + assert not rc + assert self.check_files_on_host( + os.path.join(self.tmp_dir, PlaybookRunner.default_inventory_name) + ) diff --git a/tests/test_power_manager.py b/tests/test_power_manager.py index cd83cc2..7b0fcf3 100644 --- a/tests/test_power_manager.py +++ b/tests/test_power_manager.py @@ -1,7 +1,7 @@ import pytest from rrmngmnt import Host, User, power_manager -from .common import FakeExecutor +from .common import FakeExecutorFactory PM_TYPE = 'lanplus' PM_ADDRESS = 'test-mgmt.test' @@ -16,19 +16,15 @@ pm_password=PM_PASSWORD ) -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) class TestPowerManager(object): @@ -50,7 +46,9 @@ def setup_class(cls): @staticmethod def get_host(ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', '123456')) + return h class TestSSHPowerManager(TestPowerManager): diff --git a/tests/test_service.py b/tests/test_service.py index e8f4176..246bda7 100644 --- a/tests/test_service.py +++ b/tests/test_service.py @@ -1,27 +1,25 @@ -# -*- coding: utf8 -*- -from rrmngmnt import Host +# -*- coding: utf-8 -*- +from rrmngmnt import Host, User from rrmngmnt.service import SysVinit, Systemd, InitCtl -from .common import FakeExecutor +from .common import FakeExecutorFactory import pytest -host_executor = Host.executor +host_executor_factory = Host.executor_factory def teardown_module(): - Host.executor = host_executor + Host.executor_factory = host_executor_factory -def fake_cmd_data(cmd_to_data): - def executor(self, user=None, pkey=False): - e = FakeExecutor(user, self.ip) - e.cmd_to_data = cmd_to_data.copy() - return e - Host.executor = executor +def fake_cmd_data(cmd_to_data, files=None): + Host.executor_factory = FakeExecutorFactory(cmd_to_data, files) def get_host(ip='1.1.1.1'): - return Host(ip) + h = Host(ip) + h.add_user(User('root', 'fakepasswd')) + return h class TestSystemService(object): @@ -105,13 +103,17 @@ class TestSystemd(TestSystemService): ), 'systemctl is-enabled s-enabled.service': (0, '', ''), 'systemctl is-enabled s-disabled.service': (1, '', ''), + 'journalctl -u s-disabled.service': (0, '-- No entries --', ''), 'systemctl enable s-disabled.service': (0, '', ''), 'systemctl disable s-enabled.service': (0, '', ''), 'systemctl status s-running.service': (0, '', ''), 'systemctl status s-stopped.service': (1, '', ''), + 'journalctl -u s-stopped.service': (0, '-- No entries --', ''), 'systemctl start s-stopped.service': (0, '', ''), 'systemctl start s-running.service': (1, '', ''), + 'journalctl -u s-running.service': (0, '-- No entries --', ''), 'systemctl stop s-stopped.service': (1, '', ''), + 'journalctl -u s-stopped.service': (0, '-- No entries --', ''), 'systemctl stop s-running.service': (0, '', ''), 'systemctl restart s-running.service': (0, '', ''), 'systemctl reload s-running.service': (0, '', ''), diff --git a/tests/test_user.py b/tests/test_user.py index baeb552..00cd647 100644 --- a/tests/test_user.py +++ b/tests/test_user.py @@ -1,4 +1,4 @@ -# -*- coding: utf8 -*- +# -*- coding: utf-8 -*- from rrmngmnt import User, InternalDomain, ADUser, Domain diff --git a/tox.ini b/tox.ini index 7db2330..d54d9ca 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,10 @@ [tox] -envlist = py26,py27,py34,pep8 +envlist = py27,py34,py35,py36,pep8 +[tox:travis] +2.7 = py27, pep8 +3.4 = py34, pep8 +3.5 = py35, pep8 +3.6 = py36, pep8 [testenv] deps = -r{toxinidir}/test-requirements.txt -r{toxinidir}/requirements.txt