diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..c3a3cd2 --- /dev/null +++ b/.flake8 @@ -0,0 +1,5 @@ +[flake8] +max-line-length = 79 +max-complexity = 18 +select = B,C,E,F,W,T4,B9 +ignore = E203, W503 diff --git a/.gitignore b/.gitignore index ca1df98..8ce4558 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,9 @@ tags _last_py2_res *.pyc .cache +_trial_temp* +*.snap +landscape-client_amd64*.txt +landscape-client_arm64*.txt +landscape-client_ppc64el*.txt +landscape-client_s390x*.txt diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml new file mode 100644 index 0000000..df96450 --- /dev/null +++ b/.pre-commit-config.yaml @@ -0,0 +1,43 @@ +# See https://pre-commit.com for more information +# See https://pre-commit.com/hooks.html for more hooks +repos: +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v4.4.0 + hooks: + - id: trailing-whitespace + - id: end-of-file-fixer + - id: check-yaml + - id: check-added-large-files + - id: debug-statements +- repo: https://github.com/pre-commit/pre-commit-hooks + rev: v2.3.0 + hooks: + - id: flake8 + args: + - "--max-line-length=79" + - "--select=B,C,E,F,W,T4,B9" + - "--ignore=E203,W503" +- repo: https://github.com/psf/black + rev: 22.12.0 + hooks: + - id: black + args: + - --line-length=79 + - --include='\.pyi?$' +- repo: https://github.com/asottile/reorder_python_imports + rev: v2.3.0 + hooks: + - id: reorder-python-imports + args: [--py3-plus] +- repo: https://github.com/asottile/add-trailing-comma + rev: v2.0.1 + hooks: + - id: add-trailing-comma + args: [--py36-plus] +exclude: > + (?x)( + \.git + | \.csv$ + | \.__pycache__ + | \.log$ + ) diff --git a/LICENSE b/LICENSE index 5b6e7c6..dcfa4c2 100644 --- a/LICENSE +++ b/LICENSE @@ -55,7 +55,7 @@ patent must be licensed for everyone's free use or not licensed at all. The precise terms and conditions for copying, distribution and modification follow. - + GNU GENERAL PUBLIC LICENSE TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION @@ -110,7 +110,7 @@ above, provided that you also meet all of these conditions: License. (Exception: if the Program itself is interactive but does not normally print such an announcement, your work based on the Program is not required to print an announcement.) - + These requirements apply to the modified work as a whole. If identifiable sections of that work are not derived from the Program, and can be reasonably considered independent and separate works in @@ -168,7 +168,7 @@ access to copy from a designated place, then offering equivalent access to copy the source code from the same place counts as distribution of the source code, even though third parties are not compelled to copy the source along with the object code. - + 4. You may not copy, modify, sublicense, or distribute the Program except as expressly provided under this License. Any attempt otherwise to copy, modify, sublicense or distribute the Program is @@ -225,7 +225,7 @@ impose that choice. This section is intended to make thoroughly clear what is believed to be a consequence of the rest of this License. - + 8. If the distribution and/or use of the Program is restricted in certain countries either by patents or by copyrighted interfaces, the original copyright holder who places the Program under this License @@ -278,7 +278,7 @@ PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES. END OF TERMS AND CONDITIONS - + How to Apply These Terms to Your New Programs If you develop a new program, and you want it to be of the greatest diff --git a/Makefile b/Makefile index fe183ec..09ade0e 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,15 @@ PYDOCTOR ?= pydoctor TXT2MAN ?= txt2man PYTHON2 ?= python2 PYTHON3 ?= python3 +SNAPCRAFT = SNAPCRAFT_BUILD_INFO=1 snapcraft TRIAL ?= -m twisted.trial TRIAL_ARGS ?= +# PEP8 rules ignored: +# W503 https://www.flake8rules.com/rules/W503.html +# E203 Whitespace before ':' (enforced by Black) +PEP8_IGNORED = W503,E203 + .PHONY: help help: ## Print help about available targets @grep -h -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' @@ -15,11 +21,15 @@ depends: depends3 ## py2 is deprecated .PHONY: depends2 depends2: - sudo apt-get -y install python-twisted-core python-distutils-extra python-mock python-configobj python-netifaces python-pycurl + sudo apt-get -y install python-twisted-core python-distutils-extra python-mock python-configobj python-netifaces python-pycurl python-pip + pip install pre-commit + pre-commit install .PHONY: depends3 depends3: - sudo apt-get -y install python3-twisted python3-distutils-extra python3-mock python3-configobj python3-netifaces python3-pycurl + sudo apt-get -y install python3-twisted python3-distutils-extra python3-mock python3-configobj python3-netifaces python3-pycurl python3-pip + pip3 install pre-commit + pre-commit install all: build @@ -56,13 +66,15 @@ coverage: .PHONY: lint lint: - $(PYTHON3) -m flake8 --ignore E24,E121,E123,E125,E126,E221,E226,E266,E704,E265,W504 \ - `find landscape -name \*.py` + $(PYTHON3) -m flake8 --ignore $(PEP8_IGNORED) `find landscape -name \*.py` .PHONY: pyflakes pyflakes: -pyflakes `find landscape -name \*.py` +pre-commit: + -pre-commit run -a + clean: -find landscape -name __pycache__ -exec rm -rf {} \; -find landscape -name \*.pyc -exec rm -f {} \; @@ -115,6 +127,35 @@ tags: etags: -etags --languages=python -R . +snap-install: + sudo snap install --devmode landscape-client_0.1_amd64.snap +.PHONY: snap-install + +snap-remote-build: + snapcraft remote-build +.PHONY: snap-remote-build + +snap-remove: + sudo snap remove --purge landscape-client +.PHONY: snap-remove + +snap-shell: snap-install + sudo snap run --shell landscape-client.landscape-client +.PHONY: snap-shell + +snap-debug: + $(SNAPCRAFT) -v --debug +.PHONY: snap-debug + +snap-clean: snap-remove + $(SNAPCRAFT) clean + -rm landscape-client_0.1_amd64.snap +.PHONY: snap-clean + +snap: + $(SNAPCRAFT) +.PHONY: snap + include Makefile.packaging .DEFAULT_GOAL := help diff --git a/README b/README index 06be0e1..8751ef3 100644 --- a/README +++ b/README @@ -5,7 +5,7 @@ Add our beta PPA to get the latest updates to the landscape-client package -#### Add repo to an Ubuntu series +#### Add repo to an Ubuntu series ``` sudo add-apt-repository ppa:landscape/self-hosted-beta ``` @@ -50,7 +50,7 @@ monitor_only = true ## Running -Now you can complete the configuration of your client and register with the +Now you can complete the configuration of your client and register with the Landscape service. There are two ways to do this: 1. `sudo landscape-config` and answer interactive prompts to finalize your configuration @@ -86,3 +86,104 @@ Before opening a PR, make sure to run the full testsuite and lint make check3 make lint ``` + +### Building the Landscape Client snap + +First, you need to ensure that you have the appropriate tools installed: +``` +$ sudo snap install snapcraft --classic +$ lxd init --auto +``` + +There are various make targets defined to assist in the lifecycle of +building and testing the snap. To simply build the snap with the minimum +of debug information displayed: +``` +$ make snap +``` + +If you would prefer to see more information displayed showing the progress +of the build, and would like to get dropped into a debug shell within the +snap container in the event of an error: +``` +$ make snap-debug +``` + +To install the resulting snap: +``` +$ make snap-install +``` + +To remove a previously installed snap: +``` +$ make snap-remove +``` + +To clean the intermediate files as well as the snap itself from the local +build environment: +``` +$ make snap-clean +``` + +To enter into a shell environment within the snap container: +``` +$ make snap-shell +``` + +If you wish to upload the snap to the store, you will first need to get +credentials with the store that allow you to log in locally and publish +binaries. This can be done at: + +https://snapcraft.io/docs/creating-your-developer-account + +After obtaining and confirming your store credentials, you then need to +log in using the snapcraft tool: +``` +$ snapcraft login +``` + +Since snapcraft version 7.x and higher, the credentials are stored in the +gnome keyring on your local workstation. If you are building in an +environment without a GUI (e.g. in a multipass or lxc container), you +will need to install the gnome keyring tools: +``` +$ sudo apt install gnome-keyring +``` + +You will then need to initialze the default keyring as follows: +``` +$ dbus-run-session -- bash +$ gnome-keyring-daemon --unlock +``` +The gnome-keyring-daemon will prompt you (without a prompt) to type in +the initial unlock password (typically the same password for the account +you are using - if you are using the default multipass or lxc "ubuntu" +login, use sudo passwd ubuntu to set it to a known value before doing +the above step). + +Type the login password and hit followed by +D to end +the input. + +At this point, you should be able to log into snapcraft: +``` +$ snapcraft login +``` +You will be prompted for your UbuntuOne email address, password and, +if set up this way, your second factor for authentication. If you +are successful, you should be able to query your login credentials +with: +``` +$ snapcraft whoami +``` +A common mistake that first-time users of this process might make +is that after running the gnome-keyring-daemon command, they will +exit the dbus session shell. Do NOT do that. Do all subsequent +work in that bash shell that dbus set up for you because it will +have access to your gnome-keyring. + +If you need to leave the environment and get back in, keep in mind +that you do not have to be logged into the store UNLESS you are +uploading the snap or otherwise manipulating things in your store +account. You will need to repeat the dbus-run-session and +gnome-keyring-daemon steps BEFORE logging in if you do need to be +logged into the store. diff --git a/debian/README.source b/debian/README.source index d21d7e4..20371fc 100644 --- a/debian/README.source +++ b/debian/README.source @@ -12,4 +12,4 @@ appreciated if you also updated the UPSTREAM_VERSION variable in landscape/__init__.py to include the entire new client version number. This helps us keep track of exact version of clients in use. There's no need to update the DEBIAN_REVISION variable, as it gets automatically set at build -time. +time. diff --git a/debian/changelog b/debian/changelog index 4ba984f..4b2ab6d 100644 --- a/debian/changelog +++ b/debian/changelog @@ -934,11 +934,11 @@ landscape-client (1.0.21.1-0ubuntu2) intrepid; urgency=low landscape-client (1.0.21.1-0ubuntu1) intrepid; urgency=low - * New upstream version: + * New upstream version: * Add ok-no-register option to landscape-config script to not fail if dbus isn't started or landscape-client isn't running. * lower timeout related to package management in landscape. - * debian/control: Depend on cron. + * debian/control: Depend on cron. * debian/landscape-client.postinst: use ok-no-register option so that the postinst script doesn't fail when run from the installer. (LP: #274573). @@ -986,7 +986,7 @@ landscape-client (1.0.18-0ubuntu4) intrepid; urgency=low [ Mathias Gug ] * debian/landscape-common.postinst, debian/landscape-common.prerm: don't call update-motd init script as it's no longer available in the - update-motd package. Call directly /usr/sbin/update-motd instead. + update-motd package. Call directly /usr/sbin/update-motd instead. (LP: #271854) -- Mathias Gug Thu, 18 Sep 2008 16:47:08 -0400 @@ -1006,15 +1006,15 @@ landscape-client (1.0.18-0ubuntu2) intrepid; urgency=low command. A landscape account is not required to use this package. - landscape-client: has all the binaries required to run the landscape-client. Requires a landscape account. - - debian/control: + - debian/control: + move some dependencies to landscape-client so that - landscape-common doesn't install unecessary packages in the + landscape-common doesn't install unecessary packages in the default -server install. + move python-gobject to a Pre-Depends so that landscape-config can register the system during the postinst (LP: #268838). * debian/control: - depend on python-smartpm instead of smartpm-core. - * debian/landscape-client.postrm: delete /etc/landscape/client.conf when + * debian/landscape-client.postrm: delete /etc/landscape/client.conf when the package is purged. * debian/landscape-client.postinst: remove sysinfo_in_motd debconf question as it wasn't used. @@ -1030,7 +1030,7 @@ landscape-client (1.0.18-0ubuntu2) intrepid; urgency=low landscape-client (1.0.18-0ubuntu1) intrepid; urgency=low - * New upstream release + * New upstream release -- Rick Clark Mon, 08 Sep 2008 16:35:57 -0500 @@ -1149,7 +1149,7 @@ landscape-client (1.0.11-hardy1-landscape1) hardy; urgency=low landscape-client (1.0.10-hardy1-landscape1) hardy; urgency=low - * Change the utilisation of the csv module to be compatible with + * Change the utilisation of the csv module to be compatible with python 2.4 as used in Dapper. -- Andreas Hasenack Thu, 12 Jun 2008 17:58:05 +0000 @@ -1398,7 +1398,7 @@ landscape-client (0.10.3-1ubuntu1) feisty; urgency=low fixed (#114829). -- Landscape Team Mon, 15 May 2007 16:31:00 -0700 - + landscape-client (0.10.2-1ubuntu1) feisty; urgency=low * New upstream release. @@ -1412,7 +1412,7 @@ landscape-client (0.10.1-1ubuntu1) feisty; urgency=low * Minor fix in package management plugin timings. -- Landscape Team Thu, 10 May 2007 10:00:00 -0700 - + landscape-client (0.10.0-1ubuntu1) feisty; urgency=low * New upstream release. @@ -1422,9 +1422,9 @@ landscape-client (0.10.0-1ubuntu1) feisty; urgency=low * The client uses the new operations system. Support for the now unused action info system is gone. * A minor bpickle bug was fixed. - + -- Jamshed Kakar Mon, 7 May 2007 20:16:00 -0700 - + landscape-client (0.9.6-1ubuntu1) feisty; urgency=low * New upstream release. @@ -1507,7 +1507,7 @@ landscape-client (0.8.1-1) unstable; urgency=low * New upstream release. * Account registration log message no longer exposes account password. - + -- Jamshed Kakar Thu, 18 Jan 2007 23:07:00 -0800 landscape-client (0.8.0-1) unstable; urgency=low @@ -1518,7 +1518,7 @@ landscape-client (0.8.0-1) unstable; urgency=low * Client includes an SSL certificate to verify the server with. * Package depends on python2.4-pycurl, which is necessary for the new SSL support. - + -- Jamshed Kakar Thu, 18 Jan 2007 10:48:00 -0800 landscape-client (0.7.0-1) unstable; urgency=low @@ -1529,7 +1529,7 @@ landscape-client (0.7.0-1) unstable; urgency=low added to control log verbosity. * Package depends on perl-modules, which is necessary for bare-bones server installs. - + -- Jamshed Kakar Tue, 2 Jan 2007 12:54:00 -0800 landscape-client (0.6.1-1) unstable; urgency=low diff --git a/debian/copyright b/debian/copyright index 479d234..8b528c8 100644 --- a/debian/copyright +++ b/debian/copyright @@ -33,4 +33,3 @@ License: On Debian systems, the complete text of the GNU General Public License can be found in `/usr/share/common-licenses/GPL-2'. - diff --git a/debian/landscape-client.postrm b/debian/landscape-client.postrm index 10f79b8..aa34ed2 100644 --- a/debian/landscape-client.postrm +++ b/debian/landscape-client.postrm @@ -29,7 +29,7 @@ case "$1" in rm -f "${LOG_DIR}/package-reporter.log"* rm -f "${LOG_DIR}/package-changer.log"* - rm -f "${GPG_DIR}/landscape-server"*.asc + rm -f "${GPG_DIR}/landscape-server-mirror"*.asc rm -rf "${DATA_DIR}/client" rm -rf "${DATA_DIR}/.gnupg" diff --git a/debian/source.lintian-overrides b/debian/source.lintian-overrides index 720d1be..137ce40 100644 --- a/debian/source.lintian-overrides +++ b/debian/source.lintian-overrides @@ -1,4 +1,4 @@ # we use dh_python or dh_python2 depending on the ubuntu release # the package is being built on, this is detected dynamically # in the rules file -landscape-client source: dh_python-is-obsolete \ No newline at end of file +landscape-client source: dh_python-is-obsolete diff --git a/dev/landscape-client-vm b/dev/landscape-client-vm index 4d13b6f..a2a1e57 100755 --- a/dev/landscape-client-vm +++ b/dev/landscape-client-vm @@ -37,7 +37,7 @@ landscape-devel account on the Landscape staging server (or you can specify another account with the --account parameter). The built VM will be stored under ./build/intrepid along with some other -files. To launch the VM, cd to ./build/intrepid and issue: +files. To launch the VM, cd to ./build/intrepid and issue: $ ./run Once it's booted you can log into it with: $ ./ssh @@ -136,7 +136,7 @@ cat > script < /etc/default/landscape-client EOF chmod 755 script diff --git a/dev/upload-to-ppa b/dev/upload-to-ppa index 2cc01b8..f65b42d 100755 --- a/dev/upload-to-ppa +++ b/dev/upload-to-ppa @@ -112,7 +112,7 @@ for release in $releases; do else message="Built for $codename, no source changes" fi - cp debian/changelog ../ + cp debian/changelog ../ dch --force-distribution -b -v $version -D $codename -m $message dpkg-buildpackage -S $source_opt -k$key dput $ppa ../${package}_${version}_source.changes diff --git a/display_py2_testresults b/display_py2_testresults index 021681d..e683c1d 100755 --- a/display_py2_testresults +++ b/display_py2_testresults @@ -1,5 +1,5 @@ #!/usr/bin/python -with open('_last_py2_res', 'r') as py2res: +with open("_last_py2_res", "r") as py2res: lines = py2res.readlines() lastline = lines[-1] @@ -7,7 +7,9 @@ lastline = lines[-1] time, total, total, err, fail, skip = lastline.split() if "".join((err, fail, skip)) != "000": - print("Python 2: \033[91mFAILED\033[0m (skips={}, failures={}, " - "errors={}, total={})".format(skip, fail, err, total)) + print( + "Python 2: \033[91mFAILED\033[0m (skips={}, failures={}, " + "errors={}, total={})".format(skip, fail, err, total), + ) else: - print("Python 2: \033[92mOK\033[0m (total={})".format(total)) + print(f"Python 2: \033[92mOK\033[0m (total={total})") diff --git a/example.conf b/example.conf index 4b7fd94..16b710e 100644 --- a/example.conf +++ b/example.conf @@ -77,7 +77,6 @@ ignore_sigusr1 = False # # ActiveProcessInfo - lists active processes # ComputerInfo - various information -# HardwareInventory - information provided by the "lshw" command # LoadAverage - load information # MemoryInfo - memory information # MountInfo - information about mount points (space available, used) @@ -86,9 +85,17 @@ ignore_sigusr1 = False # PackageMonitor - packages installed, available, versions # UserMonitor - users, groups # RebootRequired - whether a reboot is required or not +# AptPreferences - system APT preferences configuration # NetworkActivity - network information (TX, RX) # NetworkDevice - a list of connected network devices # UpdateManager - controls when distribution upgrades are prompted +# CPUUsage - CPU usage information +# SwiftUsage - Swift cluster usage +# CephUsage - Ceph usage information +# ComputerTags - changes in computer tags +# UbuntuProInfo - Ubuntu Pro registration information +# LivePatch - Livepath status information +# UbuntuProRebootRequired - informs if the system needs to be rebooted # # The special value "ALL" is an alias for the full list of plugins. monitor_plugins = ALL @@ -130,6 +137,9 @@ apt_update_interval = 21600 # The number of seconds between package monitor runs. package_monitor_interval = 1800 +# The number of seconds between snap monitor runs. +snap_monitor_interval = 1800 + # The URL of the http proxy to use, if any. # This value is optional. # @@ -182,3 +192,9 @@ script_users = ALL # The default is 512kB # 2MB is allowed in this example #script_output_limit=2048 + +# Whether files in /etc/apt/sources.list.d are removed when a repository +# profile is added to this machine. +# +# The default is True +#manage_sources_list_d = True diff --git a/landscape-client.conf b/landscape-client.conf index cf96100..af4faaf 100644 --- a/landscape-client.conf +++ b/landscape-client.conf @@ -9,4 +9,3 @@ log_dir = /tmp/landscape/ log_level = debug pid_file = /tmp/landscape/landscape-client.pid ping_url = http://localhost:8081/ping - diff --git a/landscape/__init__.py b/landscape/__init__.py index 23f8f79..27b3f46 100644 --- a/landscape/__init__.py +++ b/landscape/__init__.py @@ -1,6 +1,6 @@ DEBIAN_REVISION = "" UPSTREAM_VERSION = "23.02" -VERSION = "%s%s" % (UPSTREAM_VERSION, DEBIAN_REVISION) +VERSION = f"{UPSTREAM_VERSION}{DEBIAN_REVISION}" # The minimum server API version that all Landscape servers are known to speak # and support. It can serve as fallback in case higher versions are not there. diff --git a/landscape/client/accumulate.py b/landscape/client/accumulate.py index 2d17c06..ce14c88 100644 --- a/landscape/client/accumulate.py +++ b/landscape/client/accumulate.py @@ -72,24 +72,31 @@ representative data at each step boundary. """ -class Accumulator(object): - +class Accumulator: def __init__(self, persist, step_size): self._persist = persist self._step_size = step_size def __call__(self, new_timestamp, new_free_space, key): previous_timestamp, accumulated_value = self._persist.get(key, (0, 0)) - accumulated_value, step_data = \ - accumulate(previous_timestamp, accumulated_value, - new_timestamp, new_free_space, self._step_size) + accumulated_value, step_data = accumulate( + previous_timestamp, + accumulated_value, + new_timestamp, + new_free_space, + self._step_size, + ) self._persist.set(key, (new_timestamp, accumulated_value)) return step_data -def accumulate(previous_timestamp, accumulated_value, - new_timestamp, new_value, - step_size): +def accumulate( + previous_timestamp, + accumulated_value, + new_timestamp, + new_value, + step_size, +): previous_step = previous_timestamp // step_size new_step = new_timestamp // step_size step_boundary = new_step * step_size diff --git a/landscape/client/amp.py b/landscape/client/amp.py index df4ef92..5b5d448 100644 --- a/landscape/client/amp.py +++ b/landscape/client/amp.py @@ -10,14 +10,15 @@ This module implements a few conveniences built around L{landscape.lib.amp} to let the various services connect to each other in an easy and idiomatic way, and have them respond to standard requests like "ping" or "exit". """ -import os import logging +import os -from landscape.lib.amp import ( - MethodCallClientFactory, MethodCallServerFactory, RemoteObject) +from landscape.lib.amp import MethodCallClientFactory +from landscape.lib.amp import MethodCallServerFactory +from landscape.lib.amp import RemoteObject -class ComponentPublisher(object): +class ComponentPublisher: """Publish a Landscape client component using a UNIX socket. Other Landscape client processes can then connect to the socket and invoke @@ -72,7 +73,7 @@ def remote(method): return method -class ComponentConnector(object): +class ComponentConnector: """Utility superclass for creating connections with a Landscape component. @cvar component: The class of the component to connect to, it is expected @@ -90,6 +91,7 @@ class ComponentConnector(object): @see: L{MethodCallClientFactory}. """ + factory = MethodCallClientFactory component = None # Must be defined by sub-classes remote = RemoteObject @@ -123,14 +125,14 @@ class ComponentConnector(object): factory.factor = factor def fire_reconnect(ignored): - self._reactor.fire("%s-reconnect" % self.component.name) + self._reactor.fire(f"{self.component.name}-reconnect") def connected(remote): factory.notifyOnConnect(fire_reconnect) return remote def log_error(failure): - logging.error("Error while connecting to %s", self.component.name) + logging.error(f"Error while connecting to {self.component.name}") return failure socket_path = _get_socket_path(self.component, self._config) diff --git a/landscape/client/broker/amp.py b/landscape/client/broker/amp.py index 94a1fa2..2278b08 100644 --- a/landscape/client/broker/amp.py +++ b/landscape/client/broker/amp.py @@ -1,16 +1,19 @@ -from twisted.internet.defer import maybeDeferred, execute, succeed +from twisted.internet.defer import execute +from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import succeed from twisted.python.compat import iteritems -from landscape.lib.amp import RemoteObject, MethodCallArgument -from landscape.client.amp import ComponentConnector, get_remote_methods -from landscape.client.broker.server import BrokerServer +from landscape.client.amp import ComponentConnector +from landscape.client.amp import get_remote_methods from landscape.client.broker.client import BrokerClient -from landscape.client.monitor.monitor import Monitor +from landscape.client.broker.server import BrokerServer from landscape.client.manager.manager import Manager +from landscape.client.monitor.monitor import Monitor +from landscape.lib.amp import MethodCallArgument +from landscape.lib.amp import RemoteObject class RemoteBroker(RemoteObject): - def call_if_accepted(self, type, callable, *args): """Call C{callable} if C{type} is an accepted message type.""" deferred_types = self.get_accepted_message_types() @@ -18,6 +21,7 @@ class RemoteBroker(RemoteObject): def got_accepted_types(result): if type in result: return callable(*args) + deferred_types.addCallback(got_accepted_types) return deferred_types @@ -30,11 +34,10 @@ class RemoteBroker(RemoteObject): callable will be fired. """ result = self.listen_events(list(handlers.keys())) - return result.addCallback( - lambda args: handlers[args[0]](**args[1])) + return result.addCallback(lambda args: handlers[args[0]](**args[1])) -class FakeRemoteBroker(object): +class FakeRemoteBroker: """Looks like L{RemoteBroker}, but actually talks to local objects.""" def __init__(self, exchanger, message_store, broker_server): @@ -48,16 +51,19 @@ class FakeRemoteBroker(object): that they're encodable with AMP. """ original = getattr(self.broker_server, name, None) - if (name in get_remote_methods(self.broker_server) and - original is not None and - callable(original) - ): + if ( + name in get_remote_methods(self.broker_server) + and original is not None + and callable(original) + ): + def method(*args, **kwargs): for arg in args: assert MethodCallArgument.check(arg) for k, v in iteritems(kwargs): assert MethodCallArgument.check(v) return execute(original, *args, **kwargs) + return method else: raise AttributeError(name) @@ -76,8 +82,7 @@ class FakeRemoteBroker(object): callable will be fired. """ result = self.broker_server.listen_events(handlers.keys()) - return result.addCallback( - lambda args: handlers[args[0]](**args[1])) + return result.addCallback(lambda args: handlers[args[0]](**args[1])) def register(self): return succeed(None) @@ -114,8 +119,8 @@ def get_component_registry(): RemoteBrokerConnector, RemoteClientConnector, RemoteMonitorConnector, - RemoteManagerConnector + RemoteManagerConnector, ] - return dict( - (connector.component.name, connector) - for connector in all_connectors) + return { + connector.component.name: connector for connector in all_connectors + } diff --git a/landscape/client/broker/client.py b/landscape/client/broker/client.py index 042c18a..edbc2c2 100644 --- a/landscape/client/broker/client.py +++ b/landscape/client/broker/client.py @@ -1,19 +1,23 @@ -from logging import info, exception, error, debug -import sys import random +import sys +from logging import debug +from logging import error +from logging import exception +from logging import info -from twisted.internet.defer import maybeDeferred, succeed +from twisted.internet.defer import maybeDeferred +from twisted.internet.defer import succeed +from landscape.client.amp import remote from landscape.lib.format import format_object from landscape.lib.twisted_util import gather_results -from landscape.client.amp import remote class HandlerNotFoundError(Exception): """A handler for the given message type was not found.""" -class BrokerClientPlugin(object): +class BrokerClientPlugin: """A convenience for writing L{BrokerClient} plugins. This provides a register method which will set up a bunch of @@ -32,6 +36,7 @@ class BrokerClientPlugin(object): sent. See L{landscape.broker.server.BrokerServer.send_message} for more details. """ + run_interval = 5 run_immediately = False scope = None # Global scope @@ -58,8 +63,10 @@ class BrokerClientPlugin(object): if acceptance: return callable(*args, **kwargs) - self.client.reactor.call_on(("message-type-acceptance-changed", type), - acceptance_changed) + self.client.reactor.call_on( + ("message-type-acceptance-changed", type), + acceptance_changed, + ) def _resynchronize(self, scopes=None): """ @@ -106,18 +113,27 @@ class BrokerClientPlugin(object): if self.run_immediately: self._run_with_error_log() if self.run_interval is not None: - delay = (random.random() * self.run_interval * - self.client.config.stagger_launch) - debug("delaying start of %s for %d seconds", - format_object(self), delay) + delay = ( + random.random() + * self.run_interval + * self.client.config.stagger_launch + ) + debug( + "delaying start of %s for %d seconds", + format_object(self), + delay, + ) self._loop = self.client.reactor.call_later( - delay, self._start_loop) + delay, + self._start_loop, + ) def _start_loop(self): """Launch the client loop.""" self._loop = self.client.reactor.call_every( self.run_interval, - self._run_with_error_log) + self._run_with_error_log, + ) def _run_with_error_log(self): """Wrap self.run in a Deferred with a logging error handler.""" @@ -134,7 +150,7 @@ class BrokerClientPlugin(object): return failure -class BrokerClient(object): +class BrokerClient: """Basic plugin registry for clients that have to deal with the broker. This knows about the needs of a client when dealing with the Landscape @@ -148,10 +164,11 @@ class BrokerClient(object): @param reactor: A L{LandscapeReactor}. """ + name = "client" def __init__(self, reactor, config): - super(BrokerClient, self).__init__() + super().__init__() self.reactor = reactor self.broker = None self.config = config @@ -179,7 +196,7 @@ class BrokerClient(object): """ info("Registering plugin %s.", format_object(plugin)) self._plugins.append(plugin) - if hasattr(plugin, 'plugin_name'): + if hasattr(plugin, "plugin_name"): self._plugin_names[plugin.plugin_name] = plugin plugin.register(self) @@ -210,15 +227,16 @@ class BrokerClient(object): @return: The return value of the handler, if found. @raises: HandlerNotFoundError if the handler was not found """ - type = message["type"] - handler = self._registered_messages.get(type) + typ = message["type"] + handler = self._registered_messages.get(typ) if handler is None: - raise HandlerNotFoundError(type) + raise HandlerNotFoundError(typ) try: return handler(message) except Exception: - exception("Error running message handler for type %r: %r" - % (type, handler)) + exception( + f"Error running message handler for type {typ!r}: {handler!r}", + ) @remote def message(self, message): @@ -259,8 +277,9 @@ class BrokerClient(object): results = self.reactor.fire((event_type, message_type), acceptance) else: results = self.reactor.fire(event_type, *args, **kwargs) - return gather_results([ - maybeDeferred(lambda x: x, result) for result in results]) + return gather_results( + [maybeDeferred(lambda x: x, result) for result in results], + ) def handle_reconnect(self): """Called when the connection with the broker is established again. @@ -273,8 +292,8 @@ class BrokerClient(object): - Re-register ourselves as client, so the broker knows we exist and will talk to us firing events and dispatching messages. """ - for type in self._registered_messages: - self.broker.register_client_accepted_message_type(type) + for typ in self._registered_messages: + self.broker.register_client_accepted_message_type(typ) self.broker.register_client(self.name) @remote diff --git a/landscape/client/broker/config.py b/landscape/client/broker/config.py index 71e65e1..6b222fb 100644 --- a/landscape/client/broker/config.py +++ b/landscape/client/broker/config.py @@ -1,5 +1,4 @@ """Configuration class for the broker.""" - import os from landscape.client.deployment import Configuration @@ -12,7 +11,7 @@ class BrokerConfiguration(Configuration): """ def __init__(self): - super(BrokerConfiguration, self).__init__() + super().__init__() self._original_http_proxy = os.environ.get("http_proxy") self._original_https_proxy = os.environ.get("https_proxy") @@ -33,35 +32,67 @@ class BrokerConfiguration(Configuration): - C{http_proxy} - C{https_proxy} """ - parser = super(BrokerConfiguration, self).make_parser() + parser = super().make_parser() - parser.add_option("-a", "--account-name", metavar="NAME", - help="The account this computer belongs to.") - parser.add_option("-p", "--registration-key", metavar="KEY", - help="The account-wide key used for " - "registering clients.") - parser.add_option("-t", "--computer-title", metavar="TITLE", - help="The title of this computer") - parser.add_option("--exchange-interval", default=15 * 60, type="int", - metavar="INTERVAL", - help="The number of seconds between server " - "exchanges.") - parser.add_option("--urgent-exchange-interval", default=1 * 60, - type="int", metavar="INTERVAL", - help="The number of seconds between urgent server " - "exchanges.") - parser.add_option("--ping-interval", default=30, type="int", - metavar="INTERVAL", - help="The number of seconds between pings.") - parser.add_option("--http-proxy", metavar="URL", - help="The URL of the HTTP proxy, if one is needed.") - parser.add_option("--https-proxy", metavar="URL", - help="The URL of the HTTPS proxy, if one is needed.") - parser.add_option("--access-group", default="", - help="Suggested access group for this computer.") - parser.add_option("--tags", - help="Comma separated list of tag names to be sent " - "to the server.") + parser.add_option( + "-a", + "--account-name", + metavar="NAME", + help="The account this computer belongs to.", + ) + parser.add_option( + "-p", + "--registration-key", + metavar="KEY", + help="The account-wide key used for " "registering clients.", + ) + parser.add_option( + "-t", + "--computer-title", + metavar="TITLE", + help="The title of this computer", + ) + parser.add_option( + "--exchange-interval", + default=15 * 60, + type="int", + metavar="INTERVAL", + help="The number of seconds between server " "exchanges.", + ) + parser.add_option( + "--urgent-exchange-interval", + default=1 * 60, + type="int", + metavar="INTERVAL", + help="The number of seconds between urgent server " "exchanges.", + ) + parser.add_option( + "--ping-interval", + default=30, + type="int", + metavar="INTERVAL", + help="The number of seconds between pings.", + ) + parser.add_option( + "--http-proxy", + metavar="URL", + help="The URL of the HTTP proxy, if one is needed.", + ) + parser.add_option( + "--https-proxy", + metavar="URL", + help="The URL of the HTTPS proxy, if one is needed.", + ) + parser.add_option( + "--access-group", + default="", + help="Suggested access group for this computer.", + ) + parser.add_option( + "--tags", + help="Comma separated list of tag names to be sent " + "to the server.", + ) return parser @@ -78,7 +109,7 @@ class BrokerConfiguration(Configuration): C{http_proxy} and C{https_proxy} environment variables based on that config data. """ - super(BrokerConfiguration, self).load(args) + super().load(args) if self.http_proxy: os.environ["http_proxy"] = self.http_proxy elif self._original_http_proxy: diff --git a/landscape/client/broker/exchange.py b/landscape/client/broker/exchange.py index 924cb9a..31f32ff 100644 --- a/landscape/client/broker/exchange.py +++ b/landscape/client/broker/exchange.py @@ -342,23 +342,28 @@ Diagram:: 14. Schedule exchange """ -import time import logging -from landscape.lib.hashlib import md5 +import time -from twisted.internet.defer import Deferred, succeed -from landscape.lib.compat import _PY3 +from twisted.internet.defer import Deferred +from twisted.internet.defer import succeed +from landscape import CLIENT_API +from landscape import DEFAULT_SERVER_API +from landscape import SERVER_API from landscape.lib.backoff import ExponentialBackoff -from landscape.lib.fetch import HTTPCodeError, PyCurlError +from landscape.lib.compat import _PY3 +from landscape.lib.fetch import HTTPCodeError +from landscape.lib.fetch import PyCurlError from landscape.lib.format import format_delta -from landscape.lib.message import got_next_expected, RESYNC -from landscape.lib.versioning import is_version_higher, sort_versions - -from landscape import DEFAULT_SERVER_API, SERVER_API, CLIENT_API +from landscape.lib.hashlib import md5 +from landscape.lib.message import got_next_expected +from landscape.lib.message import RESYNC +from landscape.lib.versioning import is_version_higher +from landscape.lib.versioning import sort_versions -class MessageExchange(object): +class MessageExchange: """Schedule and handle message exchanges with the server. The L{MessageExchange} is the place where messages are sent to go out @@ -376,8 +381,16 @@ class MessageExchange(object): # The highest server API that we are capable of speaking _api = SERVER_API - def __init__(self, reactor, store, transport, registration_info, - exchange_store, config, max_messages=100): + def __init__( + self, + reactor, + store, + transport, + registration_info, + exchange_store, + config, + max_messages=100, + ): """ @param reactor: The L{LandscapeReactor} used to fire events in response to messages received by the server. @@ -421,15 +434,16 @@ class MessageExchange(object): A message is considered obsolete if the secure ID changed since it was received. """ - if 'operation-id' not in message: + if "operation-id" not in message: return False - operation_id = message['operation-id'] + operation_id = message["operation-id"] context = self._exchange_store.get_message_context(operation_id) if context is None: logging.warning( - "No message context for message with operation-id: %s" - % operation_id) + "No message context for message with " + f"operation-id: {operation_id}", + ) return False # Compare the current secure ID with the one that was in effect when @@ -450,7 +464,7 @@ class MessageExchange(object): max_bytes = self._max_log_text_bytes if len(value) > max_bytes: value = value[:max_bytes] - value += '...MESSAGE TRUNCATED DUE TO SIZE' + value += "...MESSAGE TRUNCATED DUE TO SIZE" message[field] = value def send(self, message, urgent=False): @@ -463,14 +477,15 @@ class MessageExchange(object): """ if self._message_is_obsolete(message): logging.info( - "Response message with operation-id %s was discarded " - "because the client's secure ID has changed in the meantime" - % message.get('operation-id')) + "Response message with operation-id " + f"{message.get('operation-id')} was discarded " + "because the client's secure ID has changed in the meantime", + ) return None # These fields sometimes have really long output we need to trim - self.truncate_message_field('err', message) - self.truncate_message_field('result-text', message) + self.truncate_message_field("err", message) + self.truncate_message_field("result-text", message) if "timestamp" not in message: message["timestamp"] = int(self._reactor.time()) @@ -535,12 +550,16 @@ class MessageExchange(object): def _handle_set_intervals(self, message): if "exchange" in message: self._config.exchange_interval = message["exchange"] - logging.info("Exchange interval set to %d seconds." % - self._config.exchange_interval) + logging.info( + "Exchange interval set " + f"to {self._config.exchange_interval:d} seconds.", + ) if "urgent-exchange" in message: self._config.urgent_exchange_interval = message["urgent-exchange"] - logging.info("Urgent exchange interval set to %d seconds." % - self._config.urgent_exchange_interval) + logging.info( + "Urgent exchange interval set " + f"to {self._config.urgent_exchange_interval:d} seconds.", + ) self._config.write() def exchange(self): @@ -565,19 +584,24 @@ class MessageExchange(object): start_time = time.time() if self._urgent_exchange: - logging.info("Starting urgent message exchange with %s." - % self._transport.get_url()) + logging.info( + "Starting urgent message exchange " + f"with {self._transport.get_url()}.", + ) else: - logging.info("Starting message exchange with %s." - % self._transport.get_url()) + logging.info( + f"Starting message exchange with {self._transport.get_url()}.", + ) deferred = Deferred() def exchange_completed(): self.schedule_exchange(force=True) self._reactor.fire("exchange-done") - logging.info("Message exchange completed in %s.", - format_delta(time.time() - start_time)) + logging.info( + "Message exchange completed in %s.", + format_delta(time.time() - start_time), + ) deferred.callback(None) def handle_result(result): @@ -627,7 +651,7 @@ class MessageExchange(object): # The error returned is an SSL error, most likely the server # is using a self-signed certificate. Let's fire a special # event so that the GUI can display a nice message. - logging.error("Message exchange failed: %s" % error.message) + logging.error(f"Message exchange failed: {error.message}") ssl_error = True self._reactor.fire("exchange-failed", ssl_error=ssl_error) @@ -636,16 +660,19 @@ class MessageExchange(object): logging.info("Message exchange failed.") exchange_completed() - self._reactor.call_in_thread(handle_result, handle_failure, - self._transport.exchange, payload, - self._registration_info.secure_id, - self._get_exchange_token(), - payload.get("server-api")) + self._reactor.call_in_thread( + handle_result, + handle_failure, + self._transport.exchange, + payload, + self._registration_info.secure_id, + self._get_exchange_token(), + payload.get("server-api"), + ) return deferred def is_urgent(self): - """Return a bool showing whether there is an urgent exchange scheduled. - """ + """Return bool showing whether there is an urgent exchange scheduled""" return self._urgent_exchange def schedule_exchange(self, urgent=False, force=False): @@ -666,9 +693,12 @@ class MessageExchange(object): # The 'not self._exchanging' check below is currently untested. # It's a bit tricky to test as it is preventing rehooking 'exchange' # while there's a background thread doing the exchange itself. - if (not self._exchanging and - (force or self._exchange_id is None or - urgent and not self._urgent_exchange)): + if not self._exchanging and ( + force + or self._exchange_id is None + or urgent + and not self._urgent_exchange + ): if urgent: self._urgent_exchange = True if self._exchange_id: @@ -680,18 +710,24 @@ class MessageExchange(object): interval = self._config.exchange_interval backoff_delay = self._backoff_counter.get_random_delay() if backoff_delay: - logging.warning("Server is busy. Backing off client for {} " - "seconds".format(backoff_delay)) + logging.warning( + "Server is busy. Backing off client for {} " + "seconds".format(backoff_delay), + ) interval += backoff_delay if self._notification_id is not None: self._reactor.cancel_call(self._notification_id) notification_interval = interval - 10 self._notification_id = self._reactor.call_later( - notification_interval, self._notify_impending_exchange) + notification_interval, + self._notify_impending_exchange, + ) self._exchange_id = self._reactor.call_later( - interval, self.exchange) + interval, + self.exchange, + ) def _get_exchange_token(self): """Get the token given us by the server at the last exchange. @@ -743,13 +779,15 @@ class MessageExchange(object): del messages[i:] else: server_api = store.get_server_api() - payload = {"server-api": server_api, - "client-api": CLIENT_API, - "sequence": store.get_sequence(), - "accepted-types": accepted_types_digest, - "messages": messages, - "total-messages": total_messages, - "next-expected-sequence": store.get_server_sequence()} + payload = { + "server-api": server_api, + "client-api": CLIENT_API, + "sequence": store.get_sequence(), + "accepted-types": accepted_types_digest, + "messages": messages, + "total-messages": total_messages, + "next-expected-sequence": store.get_server_sequence(), + } accepted_client_types = self.get_client_accepted_message_types() accepted_client_types_hash = self._hash_types(accepted_client_types) if accepted_client_types_hash != self._client_accepted_types_hash: @@ -776,7 +814,8 @@ class MessageExchange(object): """ message_store = self._message_store self._client_accepted_types_hash = result.get( - "client-accepted-types-hash") + "client-accepted-types-hash", + ) next_expected = result.get("next-expected-sequence") old_sequence = message_store.get_sequence() if next_expected is None: @@ -793,8 +832,10 @@ class MessageExchange(object): # let's fire an event to tell all the plugins that they # ought to generate new messages so the server gets some # up-to-date data. - logging.info("Server asked for ancient data: resynchronizing all " - "state with the server.") + logging.info( + "Server asked for ancient data: resynchronizing all " + "state with the server.", + ) self.send({"type": "resynchronize"}) self._reactor.fire("resynchronize-clients") @@ -808,8 +849,9 @@ class MessageExchange(object): if new_uuid and isinstance(new_uuid, bytes): new_uuid = new_uuid.decode("ascii") if new_uuid != old_uuid: - logging.info("Server UUID changed (old=%s, new=%s)." - % (old_uuid, new_uuid)) + logging.info( + f"Server UUID changed (old={old_uuid}, new={new_uuid}).", + ) self._reactor.fire("server-uuid-changed", old_uuid, new_uuid) message_store.set_server_uuid(new_uuid) @@ -874,12 +916,14 @@ class MessageExchange(object): Any message handlers registered with L{register_message} will be called. """ - if 'operation-id' in message: + if "operation-id" in message: # This is a message that requires a response. Store the secure ID # so we can check for obsolete results later. self._exchange_store.add_message_context( - message['operation-id'], self._registration_info.secure_id, - message['type']) + message["operation-id"], + self._registration_info.secure_id, + message["type"], + ) self._reactor.fire("message", message) # This has plan interference! but whatever. @@ -903,9 +947,9 @@ def get_accepted_types_diff(old_types, new_types): stable_types = old_types & new_types removed_types = old_types - new_types diff = [] - diff.extend(["+%s" % type for type in added_types]) - diff.extend(["%s" % type for type in stable_types]) - diff.extend(["-%s" % type for type in removed_types]) + diff.extend([f"+{typ}" for typ in added_types]) + diff.extend([f"{typ}" for typ in stable_types]) + diff.extend([f"-{typ}" for typ in removed_types]) return " ".join(diff) diff --git a/landscape/client/broker/exchangestore.py b/landscape/client/broker/exchangestore.py index aaa9192..4fb855a 100644 --- a/landscape/client/broker/exchangestore.py +++ b/landscape/client/broker/exchangestore.py @@ -9,7 +9,7 @@ except ImportError: from landscape.lib.store import with_cursor -class MessageContext(object): +class MessageContext: """Stores a context for incoming messages that require a response. The context consists of @@ -39,10 +39,11 @@ class MessageContext(object): def remove(self, cursor): cursor.execute( "DELETE FROM message_context WHERE operation_id=?", - (self.operation_id,)) + (self.operation_id,), + ) -class ExchangeStore(object): +class ExchangeStore: """Message meta data required by the L{MessageExchange}. The implementation uses a SQLite database as backend, with a single table @@ -51,6 +52,7 @@ class ExchangeStore(object): @param filename: The name of the file that contains the sqlite database. """ + _db = None def __init__(self, filename): @@ -61,13 +63,20 @@ class ExchangeStore(object): @with_cursor def add_message_context( - self, cursor, operation_id, secure_id, message_type): + self, + cursor, + operation_id, + secure_id, + message_type, + ): """Add a L{MessageContext} with the given data.""" params = (operation_id, secure_id, message_type, time.time()) cursor.execute( "INSERT INTO message_context " " (operation_id, secure_id, message_type, timestamp) " - " VALUES (?,?,?,?)", params) + " VALUES (?,?,?,?)", + params, + ) return MessageContext(self._db, *params) @with_cursor @@ -75,7 +84,9 @@ class ExchangeStore(object): """The L{MessageContext} for the given C{operation_id} or C{None}.""" cursor.execute( "SELECT operation_id, secure_id, message_type, timestamp " - "FROM message_context WHERE operation_id=?", (operation_id,)) + "FROM message_context WHERE operation_id=?", + (operation_id,), + ) row = cursor.fetchone() if row: return MessageContext(self._db, *row) @@ -101,10 +112,12 @@ def ensure_exchange_schema(db): "CREATE TABLE message_context" " (id INTEGER PRIMARY KEY, timestamp TIMESTAMP, " " secure_id TEXT NOT NULL, operation_id INTEGER NOT NULL, " - " message_type text NOT NULL)") + " message_type text NOT NULL)", + ) cursor.execute( "CREATE UNIQUE INDEX msgctx_operationid_idx ON " - "message_context(operation_id)") + "message_context(operation_id)", + ) except (sqlite3.OperationalError, sqlite3.DatabaseError): cursor.close() db.rollback() diff --git a/landscape/client/broker/ping.py b/landscape/client/broker/ping.py index 5c8062e..153be3c 100644 --- a/landscape/client/broker/ping.py +++ b/landscape/client/broker/ping.py @@ -48,7 +48,7 @@ from landscape.lib.fetch import fetch from landscape.lib.log import log_failure -class PingClient(object): +class PingClient: """An HTTP client which knows how to talk to the ping server.""" def __init__(self, reactor, get_page=None): @@ -74,10 +74,16 @@ class PingClient(object): def errback(type, value, tb): page_deferred.errback(Failure(value, type, tb)) - self._reactor.call_in_thread(page_deferred.callback, errback, - self.get_page, url, - post=True, data=data, - headers=headers) + + self._reactor.call_in_thread( + page_deferred.callback, + errback, + self.get_page, + url, + post=True, + data=data, + headers=headers, + ) page_deferred.addCallback(self._got_result) return page_deferred return defer.succeed(False) @@ -92,7 +98,7 @@ class PingClient(object): return True -class Pinger(object): +class Pinger: """ A plugin which pings the Landscape server with HTTP requests to see if a full exchange should be initiated. @@ -107,8 +113,14 @@ class Pinger(object): scheduled ping. """ - def __init__(self, reactor, identity, exchanger, config, - ping_client_factory=PingClient): + def __init__( + self, + reactor, + identity, + exchanger, + config, + ping_client_factory=PingClient, + ): self._config = config self._identity = identity self._reactor = reactor @@ -132,33 +144,42 @@ class Pinger(object): def ping(self): """Perform a ping; if there are messages, fire an exchange.""" deferred = self._ping_client.ping( - self._config.ping_url, self._identity.insecure_id) + self._config.ping_url, + self._identity.insecure_id, + ) deferred.addCallback(self._got_result) deferred.addErrback(self._got_error) deferred.addBoth(lambda _: self._schedule()) def _got_result(self, exchange): if exchange: - info("Ping indicates message available. " - "Scheduling an urgent exchange.") + info( + "Ping indicates message available. " + "Scheduling an urgent exchange.", + ) self._exchanger.schedule_exchange(urgent=True) def _got_error(self, failure): - log_failure(failure, - "Error contacting ping server at %s" % - (self._ping_client.url,)) + log_failure( + failure, + f"Error contacting ping server at {self._ping_client.url}", + ) def _schedule(self): """Schedule a new ping using the current ping interval.""" - self._call_id = self._reactor.call_later(self._config.ping_interval, - self.ping) + self._call_id = self._reactor.call_later( + self._config.ping_interval, + self.ping, + ) def _handle_set_intervals(self, message): if message["type"] == "set-intervals" and "ping" in message: self._config.ping_interval = message["ping"] self._config.write() - info("Ping interval set to %d seconds." % - self._config.ping_interval) + info( + f"Ping interval set to {self._config.ping_interval:d} " + "seconds.", + ) if self._call_id is not None: self._reactor.cancel_call(self._call_id) self._schedule() @@ -170,8 +191,7 @@ class Pinger(object): self._call_id = None -class FakePinger(object): - +class FakePinger: def __init__(self, *args, **kwargs): pass diff --git a/landscape/client/broker/registration.py b/landscape/client/broker/registration.py index 864ce99..c21186e 100644 --- a/landscape/client/broker/registration.py +++ b/landscape/client/broker/registration.py @@ -16,10 +16,11 @@ from twisted.internet.defer import Deferred from landscape.client.broker.exchange import maybe_bytes from landscape.client.monitor.ubuntuproinfo import get_ubuntu_pro_info from landscape.lib.juju import get_juju_info -from landscape.lib.tag import is_valid_tag_list from landscape.lib.network import get_fqdn -from landscape.lib.vm_info import get_vm_info, get_container_info +from landscape.lib.tag import is_valid_tag_list from landscape.lib.versioning import is_version_higher +from landscape.lib.vm_info import get_container_info +from landscape.lib.vm_info import get_vm_info class RegistrationError(Exception): @@ -31,7 +32,6 @@ class RegistrationError(Exception): def persist_property(name): - def get(self): value = self._persist.get(name) try: @@ -46,14 +46,13 @@ def persist_property(name): def config_property(name): - def get(self): return getattr(self._config, name) return property(get) -class Identity(object): +class Identity: """Maintains details about the identity of this Landscape client. @ivar secure_id: A server-provided ID for secure message exchange. @@ -83,7 +82,7 @@ class Identity(object): self._persist = persist.root_at("registration") -class RegistrationHandler(object): +class RegistrationHandler: """ An object from which registration can be requested of the server, and which will handle forced ID changes from the server. @@ -91,8 +90,16 @@ class RegistrationHandler(object): L{register} should be used to perform initial registration. """ - def __init__(self, config, identity, reactor, exchange, pinger, - message_store, fetch_async=None): + def __init__( + self, + config, + identity, + reactor, + exchange, + pinger, + message_store, + fetch_async=None, + ): self._config = config self._identity = identity self._reactor = reactor @@ -104,8 +111,10 @@ class RegistrationHandler(object): self._reactor.call_on("exchange-done", self._handle_exchange_done) self._exchange.register_message("set-id", self._handle_set_id) self._exchange.register_message("unknown-id", self._handle_unknown_id) - self._exchange.register_message("registration", - self._handle_registration) + self._exchange.register_message( + "registration", + self._handle_registration, + ) self._should_register = None self._fetch_async = fetch_async self._juju_data = None @@ -116,9 +125,11 @@ class RegistrationHandler(object): if id.secure_id: return False - return bool(id.computer_title and - id.account_name and - self._message_store.accepts("register")) + return bool( + id.computer_title + and id.account_name + and self._message_store.accepts("register"), + ) def register(self): """ @@ -186,14 +197,16 @@ class RegistrationHandler(object): tags = None logging.error("Invalid tags provided for registration.") - message = {"type": "register", - "hostname": get_fqdn(), - "account_name": account_name, - "computer_title": identity.computer_title, - "registration_password": identity.registration_key, - "tags": tags, - "container-info": get_container_info(), - "vm-info": get_vm_info()} + message = { + "type": "register", + "hostname": get_fqdn(), + "account_name": account_name, + "computer_title": identity.computer_title, + "registration_password": identity.registration_key, + "tags": tags, + "container-info": get_container_info(), + "vm-info": get_vm_info(), + } if self._clone_secure_id: # We use the secure id here because the registration is encrypted @@ -216,19 +229,20 @@ class RegistrationHandler(object): message["juju-info"] = { "environment-uuid": self._juju_data["environment-uuid"], "api-addresses": self._juju_data["api-addresses"], - "machine-id": self._juju_data["machine-id"]} + "machine-id": self._juju_data["machine-id"], + } # The computer is a normal computer, possibly a container. with_word = "with" if bool(registration_key) else "without" - with_tags = "and tags %s " % tags if tags else "" - with_group = "in access group '%s' " % group if group else "" + with_tags = f"and tags {tags} " if tags else "" + with_group = f"in access group '{group}' " if group else "" message["ubuntu_pro_info"] = json.dumps(get_ubuntu_pro_info()) logging.info( - u"Queueing message to register with account %r %s%s" - "%s a password." % ( - account_name, with_group, with_tags, with_word)) + f"Queueing message to register with account {account_name!r} " + f"{with_group}{with_tags}{with_word} a password.", + ) self._exchange.send(message) def _handle_set_id(self, message): @@ -239,15 +253,18 @@ class RegistrationHandler(object): Fire C{"registration-done"} and C{"resynchronize-clients"}. """ - id = self._identity - if id.secure_id: - logging.info("Overwriting secure_id with '%s'" % id.secure_id) + cid = self._identity + if cid.secure_id: + logging.info(f"Overwriting secure_id with '{cid.secure_id}'") - id.secure_id = message.get("id") - id.insecure_id = message.get("insecure-id") - logging.info("Using new secure-id ending with %s for account %s.", - id.secure_id[-10:], id.account_name) - logging.debug("Using new secure-id: %s", id.secure_id) + cid.secure_id = message.get("id") + cid.insecure_id = message.get("insecure-id") + logging.info( + "Using new secure-id ending with %s for account %s.", + cid.secure_id[-10:], + cid.account_name, + ) + logging.debug("Using new secure-id: %s", cid.secure_id) self._reactor.fire("registration-done") self._reactor.fire("resynchronize-clients") @@ -257,19 +274,21 @@ class RegistrationHandler(object): self._reactor.fire("registration-failed", reason=message_info) def _handle_unknown_id(self, message): - id = self._identity + cid = self._identity clone = message.get("clone-of") if clone is None: - logging.info("Client has unknown secure-id for account %s." - % id.account_name) + logging.info( + "Client has unknown secure-id for account " + f"{cid.account_name}.", + ) else: # Save the secure id as the clone, and clear it so it's renewed - logging.info("Client is clone of computer %s" % clone) - self._clone_secure_id = id.secure_id - id.secure_id = None - id.insecure_id = None + logging.info(f"Client is clone of computer {clone}") + self._clone_secure_id = cid.secure_id + cid.secure_id = None + cid.insecure_id = None -class RegistrationResponse(object): +class RegistrationResponse: """A helper for dealing with the response of a single registration request. @ivar deferred: The L{Deferred} that will be fired as per diff --git a/landscape/client/broker/server.py b/landscape/client/broker/server.py index 1f8d708..766c6a0 100644 --- a/landscape/client/broker/server.py +++ b/landscape/client/broker/server.py @@ -42,15 +42,14 @@ Diagram:: : exchange """ - import logging from twisted.internet.defer import Deferred -from landscape.lib.compat import _PY3 -from landscape.lib.twisted_util import gather_results from landscape.client.amp import remote from landscape.client.manager.manager import FAILED +from landscape.lib.compat import _PY3 +from landscape.lib.twisted_util import gather_results def event(method): @@ -71,7 +70,7 @@ def event(method): return broadcast_event -class BrokerServer(object): +class BrokerServer: """ A broker server capable of handling messages from plugins connected using the L{BrokerProtocol}. @@ -82,11 +81,20 @@ class BrokerServer(object): @param registration: The {RegistrationHandler}. @param message_store: The broker's L{MessageStore}. """ + name = "broker" - def __init__(self, config, reactor, exchange, registration, - message_store, pinger): + def __init__( + self, + config, + reactor, + exchange, + registration, + message_store, + pinger, + ): from landscape.client.broker.amp import get_component_registry + self.connectors_registry = get_component_registry() self._config = config self._reactor = reactor @@ -99,8 +107,10 @@ class BrokerServer(object): reactor.call_on("message", self.broadcast_message) reactor.call_on("impending-exchange", self.impending_exchange) - reactor.call_on("message-type-acceptance-changed", - self.message_type_acceptance_changed) + reactor.call_on( + "message-type-acceptance-changed", + self.message_type_acceptance_changed, + ) reactor.call_on("server-uuid-changed", self.server_uuid_changed) reactor.call_on("package-data-changed", self.package_data_changed) reactor.call_on("resynchronize-clients", self.resynchronize) @@ -201,7 +211,9 @@ class BrokerServer(object): message = {k.decode("ascii"): v for k, v in message.items()} message["type"] = message["type"].decode("ascii") if isinstance(session_id, bool) and message["type"] in ( - "operation-result", "change-packages-result"): + "operation-result", + "change-packages-result", + ): # XXX This means we're performing a Landscape-driven upgrade and # we're being invoked by a package-changer or release-upgrader # process that is running code which doesn't know about the @@ -218,7 +230,8 @@ class BrokerServer(object): if session_id is None: raise RuntimeError( - "Session ID must be set before attempting to send a message") + "Session ID must be set before attempting to send a message", + ) if self._message_store.is_valid_session_id(session_id): return self._exchanger.send(message, urgent=urgent) @@ -320,7 +333,6 @@ class BrokerServer(object): calls = [] def get_handler(event_type): - def handler(**kwargs): for call in calls: self._reactor.cancel_call(call) @@ -367,26 +379,30 @@ class BrokerServer(object): indicating as such. """ opid = message.get("operation-id") - if (True not in results and - opid is not None and - message["type"] != "resynchronize" - ): + if ( + True not in results + and opid is not None + and message["type"] != "resynchronize" + ): mtype = message["type"] - logging.error("Nobody handled the %s message." % (mtype,)) + logging.error(f"Nobody handled the {mtype} message.") result_text = """\ -Landscape client failed to handle this request (%s) because the +Landscape client failed to handle this request ({}) because the plugin which should handle it isn't available. This could mean that the plugin has been intentionally disabled, or that the client isn't running properly, or you may be running an older version of the client that doesn't support this feature. -""" % (mtype,) +""".format( + mtype, + ) response = { "type": "operation-result", "status": FAILED, "result-text": result_text, - "operation-id": opid} + "operation-id": opid, + } self._exchanger.send(response, urgent=True) @remote diff --git a/landscape/client/broker/service.py b/landscape/client/broker/service.py index 7e36149..9c86267 100644 --- a/landscape/client/broker/service.py +++ b/landscape/client/broker/service.py @@ -1,17 +1,19 @@ """Deployment code for the monitor.""" - import os -from landscape.client.service import LandscapeService, run_landscape_service from landscape.client.amp import ComponentPublisher -from landscape.client.broker.registration import RegistrationHandler, Identity from landscape.client.broker.config import BrokerConfiguration -from landscape.client.broker.transport import HTTPTransport from landscape.client.broker.exchange import MessageExchange from landscape.client.broker.exchangestore import ExchangeStore from landscape.client.broker.ping import Pinger -from landscape.client.broker.store import get_default_message_store +from landscape.client.broker.registration import Identity +from landscape.client.broker.registration import RegistrationHandler from landscape.client.broker.server import BrokerServer +from landscape.client.broker.store import get_default_message_store +from landscape.client.broker.transport import HTTPTransport +from landscape.client.service import LandscapeService +from landscape.client.service import run_landscape_service +from landscape.client.watchdog import bootstrap_list class BrokerService(LandscapeService): @@ -44,48 +46,82 @@ class BrokerService(LandscapeService): service_name = BrokerServer.name def __init__(self, config): + self._config = config self.persist_filename = os.path.join( - config.data_path, "%s.bpickle" % (self.service_name,)) - super(BrokerService, self).__init__(config) + config.data_path, + f"{self.service_name}.bpickle", + ) + super().__init__(config) self.transport = self.transport_factory( - self.reactor, config.url, config.ssl_public_key) + self.reactor, + config.url, + config.ssl_public_key, + ) self.message_store = get_default_message_store( - self.persist, config.message_store_path) + self.persist, + config.message_store_path, + ) self.identity = Identity(self.config, self.persist) exchange_store = ExchangeStore(self.config.exchange_store_path) self.exchanger = MessageExchange( - self.reactor, self.message_store, self.transport, self.identity, - exchange_store, config) + self.reactor, + self.message_store, + self.transport, + self.identity, + exchange_store, + config, + ) self.pinger = self.pinger_factory( - self.reactor, self.identity, self.exchanger, config) + self.reactor, + self.identity, + self.exchanger, + config, + ) self.registration = RegistrationHandler( - config, self.identity, self.reactor, self.exchanger, self.pinger, - self.message_store) - self.broker = BrokerServer(self.config, self.reactor, self.exchanger, - self.registration, self.message_store, - self.pinger) - self.publisher = ComponentPublisher(self.broker, self.reactor, - self.config) - - def startService(self): + config, + self.identity, + self.reactor, + self.exchanger, + self.pinger, + self.message_store, + ) + self.broker = BrokerServer( + self.config, + self.reactor, + self.exchanger, + self.registration, + self.message_store, + self.pinger, + ) + self.publisher = ComponentPublisher( + self.broker, + self.reactor, + self.config, + ) + + def startService(self): # noqa: N802 """Start the broker. Create a L{BrokerServer} listening on C{broker_socket_path} for clients connecting with the L{BrokerServerConnector}, and start the L{MessageExchange} and L{Pinger} services. """ - super(BrokerService, self).startService() + super().startService() + bootstrap_list.bootstrap( + data_path=self._config.data_path, + log_dir=self._config.log_dir, + ) self.publisher.start() self.exchanger.start() self.pinger.start() - def stopService(self): + def stopService(self): # noqa: N802 """Stop the broker.""" deferred = self.publisher.stop() self.exchanger.stop() self.pinger.stop() - super(BrokerService, self).stopService() + super().stopService() return deferred diff --git a/landscape/client/broker/store.py b/landscape/client/broker/store.py index ebab5cf..7557d45 100644 --- a/landscape/client/broker/store.py +++ b/landscape/client/broker/store.py @@ -91,25 +91,28 @@ See L{MessageStore} for details about how messages are stored on the file system and L{landscape.lib.message.got_next_expected} to check how the strategy for updating the pending offset and the sequence is implemented. """ - import itertools import logging import os +import shutil +import traceback import uuid from twisted.python.compat import iteritems from landscape import DEFAULT_SERVER_API from landscape.lib import bpickle -from landscape.lib.fs import create_binary_file, read_binary_file -from landscape.lib.versioning import sort_versions, is_version_higher +from landscape.lib.fs import create_binary_file +from landscape.lib.fs import read_binary_file +from landscape.lib.versioning import is_version_higher +from landscape.lib.versioning import sort_versions HELD = "h" BROKEN = "b" -class MessageStore(object): +class MessageStore: """A message store which stores its messages in a file system hierarchy. Beside the "sequence" and the "pending offset" values described in the @@ -134,9 +137,12 @@ class MessageStore(object): # in case the server supports it. _api = DEFAULT_SERVER_API - def __init__(self, persist, directory, directory_size=1000): + def __init__(self, persist, directory, directory_size=1000, max_dirs=4, + max_size_mb=400): self._directory = directory self._directory_size = directory_size + self._max_dirs = max_dirs # Maximum number of directories in store + self._max_size_mb = max_size_mb # Maximum size of message store self._schemas = {} self._original_persist = persist self._persist = persist.root_at("message-store") @@ -273,7 +279,7 @@ class MessageStore(object): logging.exception(e) self._add_flags(filename, BROKEN) else: - if u"type" not in message: + if "type" not in message: # Special case to decode keys for messages which were # serialized by py27 prior to py3 upgrade, and having # implicit byte message keys. Message may still get @@ -281,8 +287,9 @@ class MessageStore(object): # broker. (lp: #1718689) message = { (k if isinstance(k, str) else k.decode("ascii")): v - for k, v in message.items()} - message[u"type"] = message[u"type"].decode("ascii") + for k, v in message.items() + } + message["type"] = message["type"].decode("ascii") unknown_type = message["type"] not in accepted_types unknown_api = not is_version_higher(server_api, message["api"]) @@ -292,10 +299,53 @@ class MessageStore(object): messages.append(message) return messages + def get_messages_total_size(self): + """Get total size of messages directory""" + sizes = [] + for dirname in os.listdir(self._directory): + dirpath = os.path.join(self._directory, dirname) + dirsize = sum(file.stat().st_size for file in os.scandir(dirpath)) + sizes.append(dirsize) + return sum(sizes) + + def delete_messages_over_limit(self): + """ + Delete messages dirs if there's any over the max, which happens if + messages are queued up but not able to be sent + """ + + cur_dirs = os.listdir(self._directory) + cur_dirs.sort(key=int) # Since you could have 0, .., 9, 10 + num_dirs = len(cur_dirs) + + num_dirs_to_delete = max(0, num_dirs - self._max_dirs) # No negatives + dirs_to_delete = cur_dirs[:num_dirs_to_delete] # Chop off beginning + + for dirname in dirs_to_delete: + dirpath = os.path.join(self._directory, dirname) + try: + logging.debug(f"Trimming message store: {dirpath}") + shutil.rmtree(dirpath) + except Exception: # We want to continue like normal if any error + logging.warning(traceback.format_exc()) + logging.warning("Unable to delete message directory!") + logging.warning(dirpath) + + # Something is wrong if after deleting a bunch of files, we are still + # using too much space. Rather then look around for big files, we just + # start over. + num_bytes = self.get_messages_total_size() + num_mb = num_bytes / 1e6 + if num_mb > self._max_size_mb: + logging.warning("Messages too large! Clearing all messages!") + self.delete_all_messages() + def delete_old_messages(self): """Delete messages which are unlikely to be needed in the future.""" - for fn in itertools.islice(self._walk_messages(exclude=HELD + BROKEN), - self.get_pending_offset()): + for fn in itertools.islice( + self._walk_messages(exclude=HELD + BROKEN), + self.get_pending_offset(), + ): os.unlink(fn) containing_dir = os.path.split(fn)[0] if not os.listdir(containing_dir): @@ -326,9 +376,9 @@ class MessageStore(object): pending_offset = self.get_pending_offset() for filename in self._walk_messages(exclude=BROKEN): flags = self._get_flags(filename) - if ((HELD in flags or i >= pending_offset) and - os.stat(filename).st_ino == message_id - ): + if (HELD in flags or i >= pending_offset) and os.stat( + filename, + ).st_ino == message_id: return True if BROKEN not in flags and HELD not in flags: i += 1 @@ -347,7 +397,8 @@ class MessageStore(object): if not self._persist.has("first-failure-time"): self._persist.set("first-failure-time", timestamp) continued_failure_time = timestamp - self._persist.get( - "first-failure-time") + "first-failure-time", + ) if self._persist.get("blackhole-messages"): # Already added the resync message return @@ -357,7 +408,8 @@ class MessageStore(object): self._persist.set("blackhole-messages", True) logging.warning( "Unable to succesfully communicate with Landscape server " - "for more than a week. Waiting for resync.") + "for more than a week. Waiting for resync.", + ) def add(self, message): """Queue a message for delivery. @@ -373,6 +425,8 @@ class MessageStore(object): logging.debug("Dropped message, awaiting resync.") return + self.delete_messages_over_limit() + server_api = self.get_server_api() if "api" not in message: @@ -432,7 +486,8 @@ class MessageStore(object): """Walk the files which are definitely pending.""" pending_offset = self.get_pending_offset() for i, filename in enumerate( - self._walk_messages(exclude=HELD + BROKEN)): + self._walk_messages(exclude=HELD + BROKEN), + ): if i >= pending_offset: yield filename @@ -443,12 +498,15 @@ class MessageStore(object): for message_dir in message_dirs: for filename in self._get_sorted_filenames(message_dir): flags = set(self._get_flags(filename)) - if (not exclude or not exclude & flags): + if not exclude or not exclude & flags: yield self._message_dir(message_dir, filename) def _get_sorted_filenames(self, dir=""): - message_files = [x for x in os.listdir(self._message_dir(dir)) - if not x.endswith(".tmp")] + message_files = [ + x + for x in os.listdir(self._message_dir(dir)) + if not x.endswith(".tmp") + ] message_files.sort(key=lambda x: int(x.split("_")[0])) return message_files @@ -549,6 +607,7 @@ def get_default_message_store(*args, **kwargs): Get a L{MessageStore} object with all Landscape message schemas added. """ from landscape.message_schemas.server_bound import message_schemas + store = MessageStore(*args, **kwargs) for schema in message_schemas: store.add_schema(schema) diff --git a/landscape/client/broker/tests/helpers.py b/landscape/client/broker/tests/helpers.py index 367a03f..2ce52b0 100644 --- a/landscape/client/broker/tests/helpers.py +++ b/landscape/client/broker/tests/helpers.py @@ -7,23 +7,24 @@ connected to remote test L{BrokerClient}. """ import os -from landscape.lib.persist import Persist -from landscape.lib.testing import FakeReactor -from landscape.client.watchdog import bootstrap_list from landscape.client.amp import ComponentPublisher -from landscape.client.broker.transport import FakeTransport +from landscape.client.broker.amp import RemoteBrokerConnector +from landscape.client.broker.client import BrokerClient +from landscape.client.broker.config import BrokerConfiguration from landscape.client.broker.exchange import MessageExchange from landscape.client.broker.exchangestore import ExchangeStore -from landscape.client.broker.store import get_default_message_store -from landscape.client.broker.registration import Identity, RegistrationHandler from landscape.client.broker.ping import Pinger -from landscape.client.broker.config import BrokerConfiguration +from landscape.client.broker.registration import Identity +from landscape.client.broker.registration import RegistrationHandler from landscape.client.broker.server import BrokerServer -from landscape.client.broker.amp import RemoteBrokerConnector -from landscape.client.broker.client import BrokerClient +from landscape.client.broker.store import get_default_message_store +from landscape.client.broker.transport import FakeTransport +from landscape.client.watchdog import bootstrap_list +from landscape.lib.persist import Persist +from landscape.lib.testing import FakeReactor -class BrokerConfigurationHelper(object): +class BrokerConfigurationHelper: """Setup a L{BrokerConfiguration} instance with some test config values. The following attributes will be set on your test case: @@ -37,8 +38,10 @@ class BrokerConfigurationHelper(object): def set_up(self, test_case): data_path = test_case.makeDir() log_dir = test_case.makeDir() - test_case.config_filename = os.path.join(test_case.makeDir(), - "client.conf") + test_case.config_filename = os.path.join( + test_case.makeDir(), + "client.conf", + ) with open(test_case.config_filename, "w") as fh: fh.write( @@ -47,8 +50,9 @@ class BrokerConfigurationHelper(object): "computer_title = Some Computer\n" "account_name = some_account\n" "ping_url = http://localhost:91910\n" - "data_path = %s\n" - "log_dir = %s\n" % (data_path, log_dir)) + f"data_path = {data_path}\n" + f"log_dir = {log_dir}\n", + ) bootstrap_list.bootstrap(data_path=data_path, log_dir=log_dir) @@ -87,20 +91,31 @@ class ExchangeHelper(BrokerConfigurationHelper): """ def set_up(self, test_case): - super(ExchangeHelper, self).set_up(test_case) + super().set_up(test_case) test_case.persist_filename = test_case.makePersistFile() test_case.persist = Persist(filename=test_case.persist_filename) test_case.mstore = get_default_message_store( - test_case.persist, test_case.config.message_store_path) + test_case.persist, + test_case.config.message_store_path, + ) test_case.identity = Identity(test_case.config, test_case.persist) - test_case.transport = FakeTransport(None, test_case.config.url, - test_case.config.ssl_public_key) + test_case.transport = FakeTransport( + None, + test_case.config.url, + test_case.config.ssl_public_key, + ) test_case.reactor = FakeReactor() test_case.exchange_store = ExchangeStore( - test_case.config.exchange_store_path) + test_case.config.exchange_store_path, + ) test_case.exchanger = MessageExchange( - test_case.reactor, test_case.mstore, test_case.transport, - test_case.identity, test_case.exchange_store, test_case.config) + test_case.reactor, + test_case.mstore, + test_case.transport, + test_case.identity, + test_case.exchange_store, + test_case.config, + ) class RegistrationHelper(ExchangeHelper): @@ -116,16 +131,27 @@ class RegistrationHelper(ExchangeHelper): """ def set_up(self, test_case): - super(RegistrationHelper, self).set_up(test_case) - test_case.pinger = Pinger(test_case.reactor, test_case.identity, - test_case.exchanger, test_case.config) + super().set_up(test_case) + test_case.pinger = Pinger( + test_case.reactor, + test_case.identity, + test_case.exchanger, + test_case.config, + ) test_case.config.cloud = getattr(test_case, "cloud", False) if hasattr(test_case, "juju_contents"): test_case.makeFile( - test_case.juju_contents, path=test_case.config.juju_filename) + test_case.juju_contents, + path=test_case.config.juju_filename, + ) test_case.handler = RegistrationHandler( - test_case.config, test_case.identity, test_case.reactor, - test_case.exchanger, test_case.pinger, test_case.mstore) + test_case.config, + test_case.identity, + test_case.reactor, + test_case.exchanger, + test_case.pinger, + test_case.mstore, + ) class BrokerServerHelper(RegistrationHelper): @@ -139,10 +165,15 @@ class BrokerServerHelper(RegistrationHelper): """ def set_up(self, test_case): - super(BrokerServerHelper, self).set_up(test_case) - test_case.broker = BrokerServer(test_case.config, test_case.reactor, - test_case.exchanger, test_case.handler, - test_case.mstore, test_case.pinger) + super().set_up(test_case) + test_case.broker = BrokerServer( + test_case.config, + test_case.reactor, + test_case.exchanger, + test_case.handler, + test_case.mstore, + test_case.pinger, + ) class RemoteBrokerHelper(BrokerServerHelper): @@ -168,13 +199,17 @@ class RemoteBrokerHelper(BrokerServerHelper): """ def set_up(self, test_case): - super(RemoteBrokerHelper, self).set_up(test_case) - - self._publisher = ComponentPublisher(test_case.broker, - test_case.reactor, - test_case.config) - self._connector = RemoteBrokerConnector(test_case.reactor, - test_case.config) + super().set_up(test_case) + + self._publisher = ComponentPublisher( + test_case.broker, + test_case.reactor, + test_case.config, + ) + self._connector = RemoteBrokerConnector( + test_case.reactor, + test_case.config, + ) self._publisher.start() deferred = self._connector.connect() @@ -183,7 +218,7 @@ class RemoteBrokerHelper(BrokerServerHelper): def tear_down(self, test_case): self._connector.disconnect() self._publisher.stop() - super(RemoteBrokerHelper, self).tear_down(test_case) + super().tear_down(test_case) class BrokerClientHelper(RemoteBrokerHelper): @@ -204,7 +239,7 @@ class BrokerClientHelper(RemoteBrokerHelper): """ def set_up(self, test_case): - super(BrokerClientHelper, self).set_up(test_case) + super().set_up(test_case) # The client needs its own reactor to avoid infinite loops # when the broker broadcasts and event test_case.client_reactor = FakeReactor() @@ -227,10 +262,12 @@ class RemoteClientHelper(BrokerClientHelper): """ def set_up(self, test_case): - super(RemoteClientHelper, self).set_up(test_case) - self._client_publisher = ComponentPublisher(test_case.client, - test_case.reactor, - test_case.config) + super().set_up(test_case) + self._client_publisher = ComponentPublisher( + test_case.client, + test_case.reactor, + test_case.config, + ) self._client_publisher.start() test_case.remote.register_client("client") test_case.remote_client = test_case.broker.get_client("client") @@ -239,4 +276,4 @@ class RemoteClientHelper(BrokerClientHelper): def tear_down(self, test_case): self._client_connector.disconnect() self._client_publisher.stop() - super(RemoteClientHelper, self).tear_down(test_case) + super().tear_down(test_case) diff --git a/landscape/client/broker/tests/test_amp.py b/landscape/client/broker/tests/test_amp.py index a017dc1..bc992b4 100644 --- a/landscape/client/broker/tests/test_amp.py +++ b/landscape/client/broker/tests/test_amp.py @@ -1,10 +1,10 @@ -import mock +from unittest import mock +from landscape.client.broker.tests.helpers import RemoteBrokerHelper +from landscape.client.broker.tests.helpers import RemoteClientHelper +from landscape.client.tests.helpers import DEFAULT_ACCEPTED_TYPES +from landscape.client.tests.helpers import LandscapeTest from landscape.lib.amp import MethodCallError -from landscape.client.tests.helpers import ( - LandscapeTest, DEFAULT_ACCEPTED_TYPES) -from landscape.client.broker.tests.helpers import ( - RemoteBrokerHelper, RemoteClientHelper) class RemoteBrokerTest(LandscapeTest): @@ -41,13 +41,13 @@ class RemoteBrokerTest(LandscapeTest): session_id = self.successResultOf(self.remote.get_session_id()) message_id = self.successResultOf( - self.remote.send_message(message, session_id)) + self.remote.send_message(message, session_id), + ) self.assertTrue(isinstance(message_id, int)) self.assertTrue(self.mstore.is_pending(message_id)) self.assertFalse(self.exchanger.is_urgent()) - self.assertMessages(self.mstore.get_pending_messages(), - [message]) + self.assertMessages(self.mstore.get_pending_messages(), [message]) def test_send_message_with_urgent(self): """ @@ -56,8 +56,9 @@ class RemoteBrokerTest(LandscapeTest): message = {"type": "test"} self.mstore.set_accepted_types(["test"]) session_id = self.successResultOf(self.remote.get_session_id()) - message_id = self.successResultOf(self.remote.send_message( - message, session_id, urgent=True)) + message_id = self.successResultOf( + self.remote.send_message(message, session_id, urgent=True), + ) self.assertTrue(isinstance(message_id, int)) self.assertTrue(self.exchanger.is_urgent()) @@ -95,8 +96,9 @@ class RemoteBrokerTest(LandscapeTest): a L{Deferred}. """ # This should make the registration succeed - self.transport.responses.append([{"type": "set-id", "id": "abc", - "insecure-id": "def"}]) + self.transport.responses.append( + [{"type": "set-id", "id": "abc", "insecure-id": "def"}], + ) result = self.remote.register() return self.assertSuccess(result, None) @@ -130,7 +132,8 @@ class RemoteBrokerTest(LandscapeTest): self.assertEqual(response, None) self.assertEqual( self.exchanger.get_client_accepted_message_types(), - sorted(["type"] + DEFAULT_ACCEPTED_TYPES)) + sorted(["type"] + DEFAULT_ACCEPTED_TYPES), + ) result = self.remote.register_client_accepted_message_type("type") return result.addCallback(assert_response) @@ -159,8 +162,11 @@ class RemoteBrokerTest(LandscapeTest): The L{RemoteBroker.call_if_accepted} method doesn't do anything if the given message type is not accepted. """ - function = (lambda: 1 / 0) - result = self.remote.call_if_accepted("test", function) + + def division_by_zero(): + raise ZeroDivisionError() + + result = self.remote.call_if_accepted("test", division_by_zero) return self.assertSuccess(result, None) def test_listen_events(self): @@ -181,8 +187,9 @@ class RemoteBrokerTest(LandscapeTest): """ callback1 = mock.Mock() callback2 = mock.Mock(return_value=123) - deferred = self.remote.call_on_event({"event1": callback1, - "event2": callback2}) + deferred = self.remote.call_on_event( + {"event1": callback1, "event2": callback2}, + ) self.reactor.call_later(0.05, self.reactor.fire, "event2") self.reactor.advance(0.05) self.remote._factory.fake_connection.flush() @@ -228,8 +235,10 @@ class RemoteClientTest(LandscapeTest): a L{Deferred}. """ handler = mock.Mock() - with mock.patch.object(self.client.broker, - "register_client_accepted_message_type") as m: + with mock.patch.object( + self.client.broker, + "register_client_accepted_message_type", + ) as m: # We need to register a test message handler to let the dispatch # message method call succeed self.client.register_message("test", handler) diff --git a/landscape/client/broker/tests/test_client.py b/landscape/client/broker/tests/test_client.py index 3295b02..219cc0c 100644 --- a/landscape/client/broker/tests/test_client.py +++ b/landscape/client/broker/tests/test_client.py @@ -1,14 +1,14 @@ -import mock +from unittest import mock from twisted.internet import reactor from twisted.internet.defer import Deferred -from landscape.lib.twisted_util import gather_results -from landscape.client.tests.helpers import ( - LandscapeTest, DEFAULT_ACCEPTED_TYPES) +from landscape.client.broker.client import BrokerClientPlugin +from landscape.client.broker.client import HandlerNotFoundError from landscape.client.broker.tests.helpers import BrokerClientHelper -from landscape.client.broker.client import ( - BrokerClientPlugin, HandlerNotFoundError) +from landscape.client.tests.helpers import DEFAULT_ACCEPTED_TYPES +from landscape.client.tests.helpers import LandscapeTest +from landscape.lib.twisted_util import gather_results class BrokerClientTest(LandscapeTest): @@ -45,7 +45,8 @@ class BrokerClientTest(LandscapeTest): getting a session id. """ test_session_id = self.successResultOf( - self.client.broker.get_session_id(scope="test")) + self.client.broker.get_session_id(scope="test"), + ) plugin = BrokerClientPlugin() plugin.scope = "test" self.client.add(plugin) @@ -130,8 +131,10 @@ class BrokerClientTest(LandscapeTest): If a plugin has a run method, the reactor will call it every run_interval, but will stop and log if it raises unhandled exceptions. """ + class RunFailure(Exception): pass + # log helper should not complain on the error we're testing self.log_helper.ignore_errors("BrokerClientPlugin.*") plugin = BrokerClientPlugin() @@ -146,7 +149,8 @@ class BrokerClientTest(LandscapeTest): # message entry that would be present on a live client. self.assertIn( "ERROR: BrokerClientPlugin raised an uncaught exception", - self.logfile.getvalue()) + self.logfile.getvalue(), + ) def test_run_interval_blocked_during_resynch(self): """ @@ -220,7 +224,8 @@ class BrokerClientTest(LandscapeTest): def got_result(result): self.assertEqual( self.exchanger.get_client_accepted_message_types(), - sorted(["bar", "foo"] + DEFAULT_ACCEPTED_TYPES)) + sorted(["bar", "foo"] + DEFAULT_ACCEPTED_TYPES), + ) return gather_results([result1, result2]).addCallback(got_result) @@ -251,8 +256,10 @@ class BrokerClientTest(LandscapeTest): def dispatch_message(result): self.assertIs(self.client.dispatch_message(message), None) - self.assertTrue("Error running message handler for type 'foo'" in - self.logfile.getvalue()) + self.assertTrue( + "Error running message handler for type 'foo'" + in self.logfile.getvalue(), + ) handle_message.assert_called_once_with(message) result = self.client.register_message("foo", handle_message) @@ -263,8 +270,11 @@ class BrokerClientTest(LandscapeTest): L{BrokerClient.dispatch_message} raises an error if no handler was found for the given message. """ - error = self.assertRaises(HandlerNotFoundError, - self.client.dispatch_message, {"type": "x"}) + error = self.assertRaises( + HandlerNotFoundError, + self.client.dispatch_message, + {"type": "x"}, + ) self.assertEqual(str(error), "x") def test_message(self): @@ -326,8 +336,9 @@ class BrokerClientTest(LandscapeTest): self.client.add(plugin1) self.client.add(plugin2) self.client.exchange() - self.assertTrue("Error during plugin exchange" in - self.logfile.getvalue()) + self.assertTrue( + "Error during plugin exchange" in self.logfile.getvalue(), + ) self.assertTrue("ZeroDivisionError" in self.logfile.getvalue()) plugin1.exchange.assert_called_once_with() plugin2.exchange.assert_called_once_with() @@ -342,8 +353,10 @@ class BrokerClientTest(LandscapeTest): plugin.exchange = mock.Mock() self.client.add(plugin) self.client_reactor.fire("impending-exchange") - self.assertTrue("Got notification of impending exchange. " - "Notifying all plugins." in self.logfile.getvalue()) + self.assertTrue( + "Got notification of impending exchange. " + "Notifying all plugins." in self.logfile.getvalue(), + ) plugin.exchange.assert_called_once_with() def test_fire_event(self): @@ -415,7 +428,9 @@ class BrokerClientTest(LandscapeTest): calls = [mock.call("bar"), mock.call("foo")] broker.register_client_accepted_message_type.assert_has_calls( - calls, any_order=True) + calls, + any_order=True, + ) broker.register_client.assert_called_once_with("client") return gather_results([result1, result2]).addCallback(got_result) diff --git a/landscape/client/broker/tests/test_config.py b/landscape/client/broker/tests/test_config.py index 0b60fc4..b5236ae 100644 --- a/landscape/client/broker/tests/test_config.py +++ b/landscape/client/broker/tests/test_config.py @@ -1,8 +1,8 @@ import os from landscape.client.broker.config import BrokerConfiguration -from landscape.lib.testing import EnvironSaverHelper from landscape.client.tests.helpers import LandscapeTest +from landscape.lib.testing import EnvironSaverHelper class ConfigurationTests(LandscapeTest): @@ -20,9 +20,16 @@ class ConfigurationTests(LandscapeTest): del os.environ["https_proxy"] configuration = BrokerConfiguration() - configuration.load(["--http-proxy", "foo", - "--https-proxy", "bar", - "--url", "whatever"]) + configuration.load( + [ + "--http-proxy", + "foo", + "--https-proxy", + "bar", + "--url", + "whatever", + ], + ) self.assertEqual(os.environ["http_proxy"], "foo") self.assertEqual(os.environ["https_proxy"], "bar") @@ -53,9 +60,9 @@ class ConfigurationTests(LandscapeTest): os.environ["https_proxy"] = "originals" configuration = BrokerConfiguration() - configuration.load(["--http-proxy", "x", - "--https-proxy", "y", - "--url", "whatever"]) + configuration.load( + ["--http-proxy", "x", "--https-proxy", "y", "--url", "whatever"], + ) self.assertEqual(os.environ["http_proxy"], "x") self.assertEqual(os.environ["https_proxy"], "y") @@ -74,10 +81,12 @@ class ConfigurationTests(LandscapeTest): The 'urgent_exchange_interval, 'exchange_interval' and 'ping_interval' values specified in the configuration file are converted to integers. """ - filename = self.makeFile("[client]\n" - "urgent_exchange_interval = 12\n" - "exchange_interval = 34\n" - "ping_interval = 6\n") + filename = self.makeFile( + "[client]\n" + "urgent_exchange_interval = 12\n" + "exchange_interval = 34\n" + "ping_interval = 6\n", + ) configuration = BrokerConfiguration() configuration.load(["--config", filename, "--url", "whatever"]) @@ -91,8 +100,9 @@ class ConfigurationTests(LandscapeTest): The 'tags' value specified in the configuration file is not converted to a list (it must be a string). See bug #1228301. """ - filename = self.makeFile("[client]\n" - "tags = check,linode,profile-test") + filename = self.makeFile( + "[client]\ntags = check,linode,profile-test", + ) configuration = BrokerConfiguration() configuration.load(["--config", filename, "--url", "whatever"]) @@ -104,8 +114,7 @@ class ConfigurationTests(LandscapeTest): The 'access_group' value specified in the configuration file is passed through. """ - filename = self.makeFile("[client]\n" - "access_group = webserver") + filename = self.makeFile("[client]\naccess_group = webserver") configuration = BrokerConfiguration() configuration.load(["--config", filename, "--url", "whatever"]) @@ -122,5 +131,7 @@ class ConfigurationTests(LandscapeTest): configuration = BrokerConfiguration() configuration.load(["--config", filename]) - self.assertEqual(configuration.url, - "https://landscape.canonical.com/message-system") + self.assertEqual( + configuration.url, + "https://landscape.canonical.com/message-system", + ) diff --git a/landscape/client/broker/tests/test_exchange.py b/landscape/client/broker/tests/test_exchange.py index 3736bd4..bc7fb5f 100644 --- a/landscape/client/broker/tests/test_exchange.py +++ b/landscape/client/broker/tests/test_exchange.py @@ -1,22 +1,23 @@ -import mock +from unittest import mock from landscape import CLIENT_API -from landscape.lib.persist import Persist -from landscape.lib.fetch import HTTPCodeError, PyCurlError -from landscape.lib.hashlib import md5 -from landscape.lib.schema import Int -from landscape.message_schemas.message import Message from landscape.client.broker.config import BrokerConfiguration -from landscape.client.broker.exchange import ( - get_accepted_types_diff, MessageExchange) -from landscape.client.broker.transport import FakeTransport -from landscape.client.broker.store import MessageStore +from landscape.client.broker.exchange import get_accepted_types_diff +from landscape.client.broker.exchange import MessageExchange from landscape.client.broker.ping import Pinger from landscape.client.broker.registration import RegistrationHandler -from landscape.client.tests.helpers import ( - LandscapeTest, DEFAULT_ACCEPTED_TYPES) -from landscape.client.broker.tests.helpers import ExchangeHelper from landscape.client.broker.server import BrokerServer +from landscape.client.broker.store import MessageStore +from landscape.client.broker.tests.helpers import ExchangeHelper +from landscape.client.broker.transport import FakeTransport +from landscape.client.tests.helpers import DEFAULT_ACCEPTED_TYPES +from landscape.client.tests.helpers import LandscapeTest +from landscape.lib.fetch import HTTPCodeError +from landscape.lib.fetch import PyCurlError +from landscape.lib.hashlib import md5 +from landscape.lib.persist import Persist +from landscape.lib.schema import Int +from landscape.message_schemas.message import Message class MessageExchangeTest(LandscapeTest): @@ -24,11 +25,11 @@ class MessageExchangeTest(LandscapeTest): helpers = [ExchangeHelper] def setUp(self): - super(MessageExchangeTest, self).setUp() + super().setUp() self.mstore.add_schema(Message("empty", {})) self.mstore.add_schema(Message("data", {"data": Int()})) self.mstore.add_schema(Message("holdme", {})) - self.identity.secure_id = 'needs-to-be-set-for-tests-to-pass' + self.identity.secure_id = "needs-to-be-set-for-tests-to-pass" def wait_for_exchange(self, urgent=False, factor=1, delta=0): if urgent: @@ -52,9 +53,14 @@ class MessageExchangeTest(LandscapeTest): session IDs are expired, so any new messages being sent with those IDs will be discarded. """ - broker = BrokerServer(self.config, self.reactor, - self.exchanger, None, - self.mstore, None) + broker = BrokerServer( + self.config, + self.reactor, + self.exchanger, + None, + self.mstore, + None, + ) disk_session_id = self.mstore.get_session_id(scope="disk") package_session_id = self.mstore.get_session_id(scope="package") @@ -72,9 +78,14 @@ class MessageExchangeTest(LandscapeTest): When a resynchronisation event occurs with a scope existing session IDs for that scope are expired, all other session IDs are unaffected. """ - broker = BrokerServer(self.config, self.reactor, - self.exchanger, None, - self.mstore, None) + broker = BrokerServer( + self.config, + self.reactor, + self.exchanger, + None, + self.mstore, + None, + ) disk_session_id = self.mstore.get_session_id(scope="disk") package_session_id = self.mstore.get_session_id(scope="package") @@ -105,9 +116,10 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.exchange() self.assertEqual(len(self.transport.payloads), 1) messages = self.transport.payloads[0]["messages"] - self.assertEqual(messages, [{"type": "empty", - "timestamp": 0, - "api": b"3.2"}]) + self.assertEqual( + messages, + [{"type": "empty", "timestamp": 0, "api": b"3.2"}], + ) def test_send_urgent(self): """ @@ -118,8 +130,10 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.send({"type": "empty"}, urgent=True) self.wait_for_exchange(urgent=True) self.assertEqual(len(self.transport.payloads), 1) - self.assertMessages(self.transport.payloads[0]["messages"], - [{"type": "empty"}]) + self.assertMessages( + self.transport.payloads[0]["messages"], + [{"type": "empty"}], + ) def test_send_urgent_wont_reschedule(self): """ @@ -132,8 +146,10 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.send({"type": "empty"}, urgent=True) self.wait_for_exchange(urgent=True, factor=0.5) self.assertEqual(len(self.transport.payloads), 1) - self.assertMessages(self.transport.payloads[0]["messages"], - [{"type": "empty"}, {"type": "empty"}]) + self.assertMessages( + self.transport.payloads[0]["messages"], + [{"type": "empty"}, {"type": "empty"}], + ) def test_send_returns_message_id(self): """ @@ -151,14 +167,15 @@ class MessageExchangeTest(LandscapeTest): """ self.mstore.set_accepted_types(["package-reporter-result"]) self.exchanger._max_log_text_bytes = 5 - self.exchanger.send({"type": "package-reporter-result", "err": "E"*10, - "code": 0}) + self.exchanger.send( + {"type": "package-reporter-result", "err": "E" * 10, "code": 0}, + ) self.exchanger.exchange() self.assertEqual(len(self.transport.payloads), 1) messages = self.transport.payloads[0]["messages"] - self.assertIn('TRUNCATED', messages[0]['err']) - self.assertIn('EEEEE', messages[0]['err']) - self.assertNotIn('EEEEEE', messages[0]['err']) + self.assertIn("TRUNCATED", messages[0]["err"]) + self.assertIn("EEEEE", messages[0]["err"]) + self.assertNotIn("EEEEEE", messages[0]["err"]) def test_send_big_message_trimmed_result(self): """ @@ -166,14 +183,21 @@ class MessageExchangeTest(LandscapeTest): """ self.mstore.set_accepted_types(["operation-result"]) self.exchanger._max_log_text_bytes = 5 - self.exchanger.send({"type": "operation-result", "result-text": "E"*10, - "code": 0, "status": 0, "operation-id": 0}) + self.exchanger.send( + { + "type": "operation-result", + "result-text": "E" * 10, + "code": 0, + "status": 0, + "operation-id": 0, + }, + ) self.exchanger.exchange() self.assertEqual(len(self.transport.payloads), 1) messages = self.transport.payloads[0]["messages"] - self.assertIn('TRUNCATED', messages[0]['result-text']) - self.assertIn('EEEEE', messages[0]['result-text']) - self.assertNotIn('EEEEEE', messages[0]['result-text']) + self.assertIn("TRUNCATED", messages[0]["result-text"]) + self.assertIn("EEEEE", messages[0]["result-text"]) + self.assertNotIn("EEEEEE", messages[0]["result-text"]) def test_send_small_message_not_trimmed(self): """ @@ -181,13 +205,14 @@ class MessageExchangeTest(LandscapeTest): """ self.mstore.set_accepted_types(["package-reporter-result"]) self.exchanger._max_log_text_bytes = 4 - self.exchanger.send({"type": "package-reporter-result", "err": "E"*4, - "code": 0}) + self.exchanger.send( + {"type": "package-reporter-result", "err": "E" * 4, "code": 0}, + ) self.exchanger.exchange() self.assertEqual(len(self.transport.payloads), 1) messages = self.transport.payloads[0]["messages"] - self.assertNotIn('TRUNCATED', messages[0]['err']) - self.assertIn('EEEE', messages[0]['err']) + self.assertNotIn("TRUNCATED", messages[0]["err"]) + self.assertIn("EEEE", messages[0]["err"]) def test_wb_include_accepted_types(self): """ @@ -204,7 +229,8 @@ class MessageExchangeTest(LandscapeTest): types. """ self.exchanger.handle_message( - {"type": "accepted-types", "types": ["foo"]}) + {"type": "accepted-types", "types": ["foo"]}, + ) self.assertEqual(self.mstore.get_accepted_types(), ["foo"]) def test_message_type_acceptance_changed_event(self): @@ -212,13 +238,18 @@ class MessageExchangeTest(LandscapeTest): def callback(type, accepted): stash.append((type, accepted)) + self.reactor.call_on("message-type-acceptance-changed", callback) self.exchanger.handle_message( - {"type": "accepted-types", "types": ["a", "b"]}) + {"type": "accepted-types", "types": ["a", "b"]}, + ) self.exchanger.handle_message( - {"type": "accepted-types", "types": ["b", "c"]}) - self.assertCountEqual(stash, [("a", True), ("b", True), - ("a", False), ("c", True)]) + {"type": "accepted-types", "types": ["b", "c"]}, + ) + self.assertCountEqual( + stash, + [("a", True), ("b", True), ("a", False), ("c", True)], + ) def test_wb_accepted_types_roundtrip(self): """ @@ -226,11 +257,11 @@ class MessageExchangeTest(LandscapeTest): should affect its future payloads. """ self.exchanger.handle_message( - {"type": "accepted-types", "types": ["ack", "bar"]}) + {"type": "accepted-types", "types": ["ack", "bar"]}, + ) payload = self.exchanger._make_payload() self.assertIn("accepted-types", payload) - self.assertEqual(payload["accepted-types"], - md5(b"ack;bar").digest()) + self.assertEqual(payload["accepted-types"], md5(b"ack;bar").digest()) def test_accepted_types_causes_urgent_if_held_messages_exist(self): """ @@ -240,11 +271,14 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.send({"type": "holdme"}) self.assertEqual(self.mstore.get_pending_messages(), []) self.exchanger.handle_message( - {"type": "accepted-types", "types": ["holdme"]}) + {"type": "accepted-types", "types": ["holdme"]}, + ) self.wait_for_exchange(urgent=True) self.assertEqual(len(self.transport.payloads), 1) - self.assertMessages(self.transport.payloads[0]["messages"], - [{"type": "holdme"}]) + self.assertMessages( + self.transport.payloads[0]["messages"], + [{"type": "holdme"}], + ) def test_accepted_types_no_urgent_without_held(self): """ @@ -253,8 +287,10 @@ class MessageExchangeTest(LandscapeTest): """ self.exchanger.send({"type": "holdme"}) self.assertEqual(self.transport.payloads, []) - self.reactor.fire("message", - {"type": "accepted-types", "types": ["irrelevant"]}) + self.reactor.fire( + "message", + {"type": "accepted-types", "types": ["irrelevant"]}, + ) self.assertEqual(len(self.transport.payloads), 0) def test_sequence_is_committed_immediately(self): @@ -294,8 +330,7 @@ class MessageExchangeTest(LandscapeTest): def handler(message): Persist(filename=self.persist_filename) store = MessageStore(self.persist, self.config.message_store_path) - self.assertEqual(store.get_server_sequence(), - self.message_counter) + self.assertEqual(store.get_server_sequence(), self.message_counter) self.message_counter += 1 handled.append(True) @@ -324,8 +359,10 @@ class MessageExchangeTest(LandscapeTest): self.wait_for_exchange(urgent=True) self.assertEqual(len(self.transport.payloads), 2) - self.assertMessages(self.transport.payloads[1]["messages"], - [{"type": "empty"}]) + self.assertMessages( + self.transport.payloads[1]["messages"], + [{"type": "empty"}], + ) def test_server_expects_older_messages(self): """ @@ -361,22 +398,29 @@ class MessageExchangeTest(LandscapeTest): self.assertEqual(exchanged, [True]) payload = self.transport.payloads[-1] - self.assertMessages(payload["messages"], - [{"type": "data", "data": 1}, - {"type": "data", "data": 2}, - {"type": "data", "data": 3}]) + self.assertMessages( + payload["messages"], + [ + {"type": "data", "data": 1}, + {"type": "data", "data": 2}, + {"type": "data", "data": 3}, + ], + ) self.assertEqual(payload["sequence"], 1) self.assertEqual(payload["next-expected-sequence"], 0) - @mock.patch("landscape.client.broker.store.MessageStore" - ".delete_old_messages") - def test_pending_offset_when_next_expected_too_high(self, - mock_rm_all_messages): - ''' + @mock.patch( + "landscape.client.broker.store.MessageStore.delete_old_messages", + ) + def test_pending_offset_when_next_expected_too_high( + self, + mock_rm_all_messages, + ): + """ When next expected sequence received from server is too high, then the pending offset should reset to zero. This will cause the client to resend the pending messages. - ''' + """ self.mstore.set_accepted_types(["data"]) self.mstore.add({"type": "data", "data": 0}) @@ -399,12 +443,12 @@ class MessageExchangeTest(LandscapeTest): self.assertTrue(mock_rm_all_messages.called) def test_payloads_when_next_expected_too_high(self): - ''' + """ When next expected sequence received from server is too high, then the current messages should get sent again since we don't have confirmation that the server received it. Also previous messages should not get repeated. - ''' + """ self.mstore.set_accepted_types(["data"]) @@ -426,17 +470,16 @@ class MessageExchangeTest(LandscapeTest): self.assertTrue(last_messages) # Confirm earlier messages are not resent - self.assertNotIn(message0["data"], - [m["data"] for m in last_messages]) + self.assertNotIn(message0["data"], [m["data"] for m in last_messages]) # Confirm contents of payload self.assertEqual([message1, message2], last_messages) def test_resync_when_next_expected_too_high(self): - ''' + """ When next expected sequence received from the server is too high, then a resynchronize should happen - ''' + """ self.mstore.set_accepted_types(["empty", "resynchronize"]) self.mstore.add({"type": "empty"}) @@ -447,17 +490,24 @@ class MessageExchangeTest(LandscapeTest): self.reactor.call_on("resynchronize-clients", lambda scope=None: None) self.exchanger.exchange() - self.assertMessage(self.mstore.get_pending_messages()[-1], - {"type": "resynchronize"}) + self.assertMessage( + self.mstore.get_pending_messages()[-1], + {"type": "resynchronize"}, + ) def test_start_with_urgent_exchange(self): """ Immediately after registration, an urgent exchange should be scheduled. """ transport = FakeTransport() - exchanger = MessageExchange(self.reactor, self.mstore, transport, - self.identity, self.exchange_store, - self.config) + exchanger = MessageExchange( + self.reactor, + self.mstore, + transport, + self.identity, + self.exchange_store, + self.config, + ) exchanger.start() self.wait_for_exchange(urgent=True) self.assertEqual(len(transport.payloads), 1) @@ -499,13 +549,17 @@ class MessageExchangeTest(LandscapeTest): self.mstore.record_success = mock_record_success exchanger = MessageExchange( - self.reactor, self.mstore, self.transport, - self.identity, self.exchange_store, self.config) + self.reactor, + self.mstore, + self.transport, + self.identity, + self.exchange_store, + self.config, + ) exchanger.exchange() mock_record_success.assert_called_with(mock.ANY) - self.assertTrue( - type(mock_record_success.call_args[0][0]) is int) + self.assertTrue(type(mock_record_success.call_args[0][0]) is int) def test_ancient_causes_resynchronize(self): """ @@ -530,15 +584,20 @@ class MessageExchangeTest(LandscapeTest): # should come AFTER the "resynchronize" message that is generated # by the exchange code itself. self.mstore.add({"type": "data", "data": 999}) + self.reactor.call_on("resynchronize-clients", resynchronize) # This exchange call will notice the server is asking for an old # message and fire the event: self.exchanger.exchange() - self.assertMessages(self.mstore.get_pending_messages(), - [{"type": "empty"}, - {"type": "resynchronize"}, - {"type": "data", "data": 999}]) + self.assertMessages( + self.mstore.get_pending_messages(), + [ + {"type": "empty"}, + {"type": "resynchronize"}, + {"type": "data", "data": 999}, + ], + ) def test_resynchronize_msg_causes_resynchronize_response_then_event(self): """ @@ -551,15 +610,20 @@ class MessageExchangeTest(LandscapeTest): def resynchronized(scopes=None): self.mstore.add({"type": "empty"}) + self.reactor.call_on("resynchronize-clients", resynchronized) - self.transport.responses.append([{"type": "resynchronize", - "operation-id": 123}]) + self.transport.responses.append( + [{"type": "resynchronize", "operation-id": 123}], + ) self.exchanger.exchange() - self.assertMessages(self.mstore.get_pending_messages(), - [{"type": "resynchronize", - "operation-id": 123}, - {"type": "empty"}]) + self.assertMessages( + self.mstore.get_pending_messages(), + [ + {"type": "resynchronize", "operation-id": 123}, + {"type": "empty"}, + ], + ) def test_scopes_are_copied_from_incoming_resynchronize_messages(self): """ @@ -574,9 +638,15 @@ class MessageExchangeTest(LandscapeTest): self.reactor.call_on("resynchronize-clients", resynchronized) - self.transport.responses.append([{"type": "resynchronize", - "operation-id": 123, - "scopes": ["disk", "users"]}]) + self.transport.responses.append( + [ + { + "type": "resynchronize", + "operation-id": 123, + "scopes": ["disk", "users"], + }, + ], + ) self.exchanger.exchange() self.assertEqual(["disk", "users"], fired_scopes) @@ -622,8 +692,10 @@ class MessageExchangeTest(LandscapeTest): def test_old_sequence_id_does_not_cause_resynchronize(self): resynchronized = [] - self.reactor.call_on("resynchronize", - lambda: resynchronized.append(True)) + self.reactor.call_on( + "resynchronize", + lambda: resynchronized.append(True), + ) self.mstore.set_accepted_types(["empty"]) self.mstore.add({"type": "empty"}) @@ -663,9 +735,10 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.exchange() payload = self.transport.payloads[-1] - self.assertMessages(payload["messages"], - [{"type": "a", "api": b"1.0"}, - {"type": "b", "api": b"1.0"}]) + self.assertMessages( + payload["messages"], + [{"type": "a", "api": b"1.0"}, {"type": "b", "api": b"1.0"}], + ) self.assertEqual(payload.get("client-api"), CLIENT_API) self.assertEqual(payload.get("server-api"), b"1.0") self.assertEqual(self.transport.message_api, b"1.0") @@ -673,9 +746,10 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.exchange() payload = self.transport.payloads[-1] - self.assertMessages(payload["messages"], - [{"type": "c", "api": b"1.1"}, - {"type": "d", "api": b"1.1"}]) + self.assertMessages( + payload["messages"], + [{"type": "c", "api": b"1.1"}, {"type": "d", "api": b"1.1"}], + ) self.assertEqual(payload.get("client-api"), CLIENT_API) self.assertEqual(payload.get("server-api"), b"1.1") self.assertEqual(self.transport.message_api, b"1.1") @@ -732,9 +806,15 @@ class MessageExchangeTest(LandscapeTest): the total-messages is equivalent to the total number of messages pending. """ - exchanger = MessageExchange(self.reactor, self.mstore, self.transport, - self.identity, self.exchange_store, - self.config, max_messages=1) + exchanger = MessageExchange( + self.reactor, + self.mstore, + self.transport, + self.identity, + self.exchange_store, + self.config, + max_messages=1, + ) self.mstore.set_accepted_types(["empty"]) self.mstore.add({"type": "empty"}) self.mstore.add({"type": "empty"}) @@ -763,9 +843,14 @@ class MessageExchangeTest(LandscapeTest): # fixture has an urgent exchange interval of 10 seconds, which makes # testing this awkward. self.config.urgent_exchange_interval = 20 - exchanger = MessageExchange(self.reactor, self.mstore, self.transport, - self.identity, self.exchange_store, - self.config) + exchanger = MessageExchange( + self.reactor, + self.mstore, + self.transport, + self.identity, + self.exchange_store, + self.config, + ) exchanger.schedule_exchange(urgent=True) events = [] self.reactor.call_on("impending-exchange", lambda: events.append(True)) @@ -783,9 +868,14 @@ class MessageExchangeTest(LandscapeTest): """ self.config.exchange_interval = 60 * 60 self.config.urgent_exchange_interval = 20 - exchanger = MessageExchange(self.reactor, self.mstore, self.transport, - self.identity, self.exchange_store, - self.config) + exchanger = MessageExchange( + self.reactor, + self.mstore, + self.transport, + self.identity, + self.exchange_store, + self.config, + ) events = [] self.reactor.call_on("impending-exchange", lambda: events.append(True)) # This call will: @@ -806,11 +896,12 @@ class MessageExchangeTest(LandscapeTest): # schedule a regular exchange. # Let's make sure that that *original* impending-exchange event has # been cancelled: - TIME_UNTIL_EXCHANGE = 60 * 60 - TIME_UNTIL_NOTIFY = 10 - TIME_ADVANCED = 20 # time that we've already advanced - self.reactor.advance(TIME_UNTIL_EXCHANGE - - (TIME_UNTIL_NOTIFY + TIME_ADVANCED)) + time_until_exchange = 60 * 60 + time_until_notify = 10 + time_advanced = 20 + self.reactor.advance( + time_until_exchange - (time_until_notify + time_advanced), + ) self.assertEqual(events, [True]) # Ok, so no new events means that the original call was # cancelled. great. @@ -877,8 +968,13 @@ class MessageExchangeTest(LandscapeTest): the L{MessageExchange} are changed, the configuration values as well, and the configuration is written to disk to be persisted. """ - server_message = [{"type": "set-intervals", - "urgent-exchange": 1234, "exchange": 5678}] + server_message = [ + { + "type": "set-intervals", + "urgent-exchange": 1234, + "exchange": 5678, + }, + ] self.transport.responses.append(server_message) self.exchanger.exchange() @@ -993,6 +1089,7 @@ class MessageExchangeTest(LandscapeTest): def server_uuid_changed(old_uuid, new_uuid): called.append((old_uuid, new_uuid)) + self.reactor.call_on("server-uuid-changed", server_uuid_changed) # Set it for the first time, and it should emit the event @@ -1028,6 +1125,7 @@ class MessageExchangeTest(LandscapeTest): def server_uuid_changed(old_uuid, new_uuid): called.append((old_uuid, new_uuid)) + self.reactor.call_on("server-uuid-changed", server_uuid_changed) self.mstore.set_server_uuid("the-uuid") @@ -1039,8 +1137,10 @@ class MessageExchangeTest(LandscapeTest): self.transport.extra["server-uuid"] = "the-uuid" self.exchanger.exchange() - self.assertIn("INFO: Server UUID changed (old=None, new=the-uuid).", - self.logfile.getvalue()) + self.assertIn( + "INFO: Server UUID changed (old=None, new=the-uuid).", + self.logfile.getvalue(), + ) # An exchange with the same UUID shouldn't be logged. self.logfile.truncate(0) @@ -1063,9 +1163,11 @@ class MessageExchangeTest(LandscapeTest): [message] = messages self.assertIsNot( None, - self.exchange_store.get_message_context(message['operation-id'])) + self.exchange_store.get_message_context(message["operation-id"]), + ) message_context = self.exchange_store.get_message_context( - message['operation-id']) + message["operation-id"], + ) self.assertEqual(message_context.operation_id, 123456) self.assertEqual(message_context.message_type, "type-R") @@ -1099,12 +1201,13 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.exchange() # Change the secure ID so that the response message gets discarded. - self.identity.secure_id = 'brand-new' + self.identity.secure_id = "brand-new" ids_before = self.exchange_store.all_operation_ids() self.mstore.set_accepted_types(["resynchronize"]) message_id = self.exchanger.send( - {"type": "resynchronize", "operation-id": 234567}) + {"type": "resynchronize", "operation-id": 234567}, + ) self.exchanger.exchange() self.assertEqual(2, len(self.transport.payloads)) messages = self.transport.payloads[1]["messages"] @@ -1112,13 +1215,14 @@ class MessageExchangeTest(LandscapeTest): self.assertIs(None, message_id) expected_log_entry = ( "Response message with operation-id 234567 was discarded because " - "the client's secure ID has changed in the meantime") + "the client's secure ID has changed in the meantime" + ) self.assertIn(expected_log_entry, self.logfile.getvalue()) # The MessageContext was removed after utilisation. ids_after = self.exchange_store.all_operation_ids() self.assertEqual(len(ids_after), len(ids_before) - 1) - self.assertNotIn('234567', ids_after) + self.assertNotIn("234567", ids_after) def test_error_exchanging_causes_failed_exchange(self): """ @@ -1135,13 +1239,14 @@ class MessageExchangeTest(LandscapeTest): self.exchanger.exchange() self.assertEqual([None], events) - def test_SSL_error_exchanging_causes_failed_exchange(self): + def test_SSL_error_exchanging_causes_failed_exchange(self): # noqa: N802 """ If an SSL error occurs when exchanging, the 'exchange-failed' event should be fired with the optional "ssl_error" flag set to True. """ - self.log_helper.ignore_errors("Message exchange failed: Failed to " - "communicate.") + self.log_helper.ignore_errors( + "Message exchange failed: Failed to communicate.", + ) events = [] def failed_exchange(ssl_error): @@ -1149,8 +1254,9 @@ class MessageExchangeTest(LandscapeTest): events.append(None) self.reactor.call_on("exchange-failed", failed_exchange) - self.transport.responses.append(PyCurlError(60, - "Failed to communicate.")) + self.transport.responses.append( + PyCurlError(60, "Failed to communicate."), + ) self.exchanger.exchange() self.assertEqual([None], events) @@ -1167,8 +1273,9 @@ class MessageExchangeTest(LandscapeTest): events.append(None) self.reactor.call_on("exchange-failed", failed_exchange) - self.transport.responses.append(PyCurlError(10, # Not 60 - "Failed to communicate.")) + self.transport.responses.append( + PyCurlError(10, "Failed to communicate."), # Not 60 + ) self.exchanger.exchange() self.assertEqual([None], events) @@ -1278,14 +1385,23 @@ class AcceptedTypesMessageExchangeTest(LandscapeTest): helpers = [ExchangeHelper] def setUp(self): - super(AcceptedTypesMessageExchangeTest, self).setUp() - self.pinger = Pinger(self.reactor, self.identity, self.exchanger, - self.config) + super().setUp() + self.pinger = Pinger( + self.reactor, + self.identity, + self.exchanger, + self.config, + ) # The __init__ method of RegistrationHandler registers a few default # message types that we want to catch as well self.handler = RegistrationHandler( - self.config, self.identity, self.reactor, self.exchanger, - self.pinger, self.mstore) + self.config, + self.identity, + self.reactor, + self.exchanger, + self.pinger, + self.mstore, + ) def test_register_accepted_message_type(self): self.exchanger.register_client_accepted_message_type("type-B") @@ -1293,9 +1409,10 @@ class AcceptedTypesMessageExchangeTest(LandscapeTest): self.exchanger.register_client_accepted_message_type("type-C") self.exchanger.register_client_accepted_message_type("type-A") types = self.exchanger.get_client_accepted_message_types() - self.assertEqual(types, - sorted(["type-A", "type-B", "type-C"] + - DEFAULT_ACCEPTED_TYPES)) + self.assertEqual( + types, + sorted(["type-A", "type-B", "type-C"] + DEFAULT_ACCEPTED_TYPES), + ) def test_exchange_sends_message_type_when_no_hash(self): self.exchanger.register_client_accepted_message_type("type-A") @@ -1303,15 +1420,17 @@ class AcceptedTypesMessageExchangeTest(LandscapeTest): self.exchanger.exchange() self.assertEqual( self.transport.payloads[0]["client-accepted-types"], - sorted(["type-A", "type-B"] + DEFAULT_ACCEPTED_TYPES)) + sorted(["type-A", "type-B"] + DEFAULT_ACCEPTED_TYPES), + ) def test_exchange_does_not_send_message_types_when_hash_matches(self): self.exchanger.register_client_accepted_message_type("type-A") self.exchanger.register_client_accepted_message_type("type-B") types = sorted(["type-A", "type-B"] + DEFAULT_ACCEPTED_TYPES) accepted_types_digest = md5(";".join(types).encode("ascii")).digest() - self.transport.extra["client-accepted-types-hash"] = \ - accepted_types_digest + self.transport.extra[ + "client-accepted-types-hash" + ] = accepted_types_digest self.exchanger.exchange() self.exchanger.exchange() self.assertNotIn("client-accepted-types", self.transport.payloads[1]) @@ -1327,7 +1446,8 @@ class AcceptedTypesMessageExchangeTest(LandscapeTest): self.exchanger.exchange() self.assertEqual( self.transport.payloads[1]["client-accepted-types"], - sorted(["type-A", "type-B"] + DEFAULT_ACCEPTED_TYPES)) + sorted(["type-A", "type-B"] + DEFAULT_ACCEPTED_TYPES), + ) def test_exchange_sends_new_accepted_types_hash(self): """ @@ -1342,7 +1462,8 @@ class AcceptedTypesMessageExchangeTest(LandscapeTest): self.exchanger.exchange() self.assertEqual( self.transport.payloads[1]["client-accepted-types"], - sorted(["type-A", "type-B"] + DEFAULT_ACCEPTED_TYPES)) + sorted(["type-A", "type-B"] + DEFAULT_ACCEPTED_TYPES), + ) def test_exchange_sends_new_types_when_server_screws_up(self): """ @@ -1359,7 +1480,8 @@ class AcceptedTypesMessageExchangeTest(LandscapeTest): self.exchanger.exchange() self.assertEqual( self.transport.payloads[2]["client-accepted-types"], - sorted(["type-A"] + DEFAULT_ACCEPTED_TYPES)) + sorted(["type-A"] + DEFAULT_ACCEPTED_TYPES), + ) def test_register_message_adds_accepted_type(self): """ @@ -1373,7 +1495,6 @@ class AcceptedTypesMessageExchangeTest(LandscapeTest): class GetAcceptedTypesDiffTest(LandscapeTest): - def test_diff_empty(self): self.assertEqual(get_accepted_types_diff([], []), "") @@ -1387,6 +1508,7 @@ class GetAcceptedTypesDiffTest(LandscapeTest): self.assertEqual(get_accepted_types_diff(["ooga"], ["ooga"]), "ooga") def test_diff_complex(self): - self.assertEqual(get_accepted_types_diff(["foo", "bar"], - ["foo", "ooga"]), - "+ooga foo -bar") + self.assertEqual( + get_accepted_types_diff(["foo", "bar"], ["foo", "ooga"]), + "+ooga foo -bar", + ) diff --git a/landscape/client/broker/tests/test_exchangestore.py b/landscape/client/broker/tests/test_exchangestore.py index 45354db..76a7d9f 100644 --- a/landscape/client/broker/tests/test_exchangestore.py +++ b/landscape/client/broker/tests/test_exchangestore.py @@ -14,7 +14,7 @@ class ExchangeStoreTest(LandscapeTest): """Unit tests for the C{ExchangeStore}.""" def setUp(self): - super(ExchangeStoreTest, self).setUp() + super().setUp() self.filename = self.makeFile() self.store1 = ExchangeStore(self.filename) @@ -23,19 +23,21 @@ class ExchangeStoreTest(LandscapeTest): def test_add_message_context(self): """Adding a message context works correctly.""" now = time.time() - self.store1.add_message_context(123, 'abc', 'change-packages') + self.store1.add_message_context(123, "abc", "change-packages") db = sqlite3.connect(self.store2._filename) cursor = db.cursor() cursor.execute( "SELECT operation_id, secure_id, message_type, timestamp " - "FROM message_context WHERE operation_id=?", (123,)) + "FROM message_context WHERE operation_id=?", + (123,), + ) results = cursor.fetchall() self.assertEqual(1, len(results)) [row] = results self.assertEqual(123, row[0]) - self.assertEqual('abc', row[1]) - self.assertEqual('change-packages', row[2]) + self.assertEqual("abc", row[1]) + self.assertEqual("change-packages", row[2]) self.assertTrue(row[3] > now) def test_add_message_context_with_duplicate_operation_id(self): @@ -43,18 +45,22 @@ class ExchangeStoreTest(LandscapeTest): self.store1.add_message_context(123, "abc", "change-packages") self.assertRaises( (sqlite3.IntegrityError, sqlite3.OperationalError), - self.store1.add_message_context, 123, "def", "change-packages") + self.store1.add_message_context, + 123, + "def", + "change-packages", + ) def test_get_message_context(self): """ Accessing a C{MessageContext} with an existing C{operation-id} works. """ now = time.time() - self.store1.add_message_context(234, 'bcd', 'change-packages') + self.store1.add_message_context(234, "bcd", "change-packages") context = self.store2.get_message_context(234) self.assertEqual(234, context.operation_id) - self.assertEqual('bcd', context.secure_id) - self.assertEqual('change-packages', context.message_type) + self.assertEqual("bcd", context.secure_id) + self.assertEqual("change-packages", context.message_type) self.assertTrue(context.timestamp > now) def test_get_message_context_with_nonexistent_operation_id(self): @@ -65,7 +71,10 @@ class ExchangeStoreTest(LandscapeTest): def test_message_context_remove(self): """C{MessageContext}s are deleted correctly.""" context = self.store1.add_message_context( - 345, 'opq', 'change-packages') + 345, + "opq", + "change-packages", + ) context.remove() self.assertIs(None, self.store1.get_message_context(345)) @@ -78,7 +87,7 @@ class ExchangeStoreTest(LandscapeTest): def test_all_operation_ids(self): """C{all_operation_ids} works correctly.""" - self.store1.add_message_context(456, 'cde', 'change-packages') + self.store1.add_message_context(456, "cde", "change-packages") self.assertEqual([456], self.store2.all_operation_ids()) - self.store2.add_message_context(567, 'def', 'change-packages') + self.store2.add_message_context(567, "def", "change-packages") self.assertEqual([456, 567], self.store1.all_operation_ids()) diff --git a/landscape/client/broker/tests/test_ping.py b/landscape/client/broker/tests/test_ping.py index b095b5a..843b984 100644 --- a/landscape/client/broker/tests/test_ping.py +++ b/landscape/client/broker/tests/test_ping.py @@ -1,15 +1,15 @@ -from landscape.client.tests.helpers import LandscapeTest - from twisted.internet.defer import fail +from landscape.client.broker.ping import PingClient +from landscape.client.broker.ping import Pinger +from landscape.client.broker.tests.helpers import ExchangeHelper +from landscape.client.tests.helpers import LandscapeTest from landscape.lib import bpickle from landscape.lib.fetch import fetch from landscape.lib.testing import FakeReactor -from landscape.client.broker.ping import PingClient, Pinger -from landscape.client.broker.tests.helpers import ExchangeHelper -class FakePageGetter(object): +class FakePageGetter: """An fake web client.""" def __init__(self, response): @@ -39,9 +39,8 @@ class FakePageGetter(object): class PingClientTest(LandscapeTest): - def setUp(self): - super(PingClientTest, self).setUp() + super().setUp() self.reactor = FakeReactor() def test_default_get_page(self): @@ -64,8 +63,15 @@ class PingClientTest(LandscapeTest): pinger.ping(url, insecure_id) self.assertEqual( client.fetches, - [(url, True, {"Content-Type": "application/x-www-form-urlencoded"}, - "insecure_id=10")]) + [ + ( + url, + True, + {"Content-Type": "application/x-www-form-urlencoded"}, + "insecure_id=10", + ), + ], + ) def test_ping_no_insecure_id(self): """ @@ -100,6 +106,7 @@ class PingClientTest(LandscapeTest): def errback(failure): failures.append(failure) + d.addErrback(errback) self.assertEqual(len(failures), 1) self.assertEqual(failures[0].getErrorMessage(), "That's a failure!") @@ -116,7 +123,7 @@ class PingerTest(LandscapeTest): install_exchanger = False def setUp(self): - super(PingerTest, self).setUp() + super().setUp() self.page_getter = FakePageGetter(None) def factory(reactor): @@ -125,21 +132,25 @@ class PingerTest(LandscapeTest): self.config.ping_url = "http://localhost:8081/whatever" self.config.ping_interval = 10 - self.pinger = Pinger(self.reactor, - self.identity, - self.exchanger, - self.config, - ping_client_factory=factory) + self.pinger = Pinger( + self.reactor, + self.identity, + self.exchanger, + self.config, + ping_client_factory=factory, + ) def test_default_ping_client(self): """ The C{ping_client_factory} argument to L{Pinger} should be optional, and default to L{PingClient}. """ - pinger = Pinger(self.reactor, - self.identity, - self.exchanger, - self.config) + pinger = Pinger( + self.reactor, + self.identity, + self.exchanger, + self.config, + ) self.assertEqual(pinger.ping_client_factory, PingClient) def test_occasional_ping(self): @@ -195,7 +206,7 @@ class PingerTest(LandscapeTest): self.log_helper.ignore_errors(ZeroDivisionError) self.identity.insecure_id = 42 - class BadPingClient(object): + class BadPingClient: def __init__(self, *args, **kwargs): pass @@ -204,11 +215,13 @@ class PingerTest(LandscapeTest): return fail(ZeroDivisionError("Couldn't fetch page")) self.config.ping_url = "http://foo.com/" - pinger = Pinger(self.reactor, - self.identity, - self.exchanger, - self.config, - ping_client_factory=BadPingClient) + pinger = Pinger( + self.reactor, + self.identity, + self.exchanger, + self.config, + ping_client_factory=BadPingClient, + ) pinger.start() self.reactor.advance(30) @@ -238,8 +251,10 @@ class PingerTest(LandscapeTest): self.assertEqual(len(self.page_getter.fetches), 1) def test_get_url(self): - self.assertEqual(self.pinger.get_url(), - "http://localhost:8081/whatever") + self.assertEqual( + self.pinger.get_url(), + "http://localhost:8081/whatever", + ) def test_config_url(self): """ diff --git a/landscape/client/broker/tests/test_registration.py b/landscape/client/broker/tests/test_registration.py index 8ca0cb4..fe1d256 100644 --- a/landscape/client/broker/tests/test_registration.py +++ b/landscape/client/broker/tests/test_registration.py @@ -1,14 +1,14 @@ import json import logging import socket -import mock +from unittest import mock -from landscape.lib.compat import _PY3 - -from landscape.client.broker.registration import RegistrationError, Identity +from landscape.client.broker.registration import Identity +from landscape.client.broker.registration import RegistrationError +from landscape.client.broker.tests.helpers import BrokerConfigurationHelper +from landscape.client.broker.tests.helpers import RegistrationHelper from landscape.client.tests.helpers import LandscapeTest -from landscape.client.broker.tests.helpers import ( - BrokerConfigurationHelper, RegistrationHelper) +from landscape.lib.compat import _PY3 from landscape.lib.persist import Persist @@ -17,33 +17,43 @@ class IdentityTest(LandscapeTest): helpers = [BrokerConfigurationHelper] def setUp(self): - super(IdentityTest, self).setUp() + super().setUp() self.persist = Persist(filename=self.makePersistFile()) self.identity = Identity(self.config, self.persist) def check_persist_property(self, attr, persist_name): value = "VALUE" - self.assertEqual(getattr(self.identity, attr), None, - "%r attribute should default to None, not %r" % - (attr, getattr(self.identity, attr))) + self.assertEqual( + getattr(self.identity, attr), + None, + f"{attr!r} attribute should default to None, " + f"not {getattr(self.identity, attr)!r}", + ) setattr(self.identity, attr, value) - self.assertEqual(getattr(self.identity, attr), value, - "%r attribute should be %r, not %r" % - (attr, value, getattr(self.identity, attr))) self.assertEqual( - self.persist.get(persist_name), value, - "%r not set to %r in persist" % (persist_name, value)) + getattr(self.identity, attr), + value, + f"{attr!r} attribute should be {value!r}, " + f"not {getattr(self.identity, attr)!r}", + ) + self.assertEqual( + self.persist.get(persist_name), + value, + f"{persist_name!r} not set to {value!r} in persist", + ) def check_config_property(self, attr): value = "VALUE" setattr(self.config, attr, value) - self.assertEqual(getattr(self.identity, attr), value, - "%r attribute should be %r, not %r" % - (attr, value, getattr(self.identity, attr))) + self.assertEqual( + getattr(self.identity, attr), + value, + f"{attr!r} attribute should be {value!r}, " + f"not {getattr(self.identity, attr)!r}", + ) def test_secure_id(self): - self.check_persist_property("secure_id", - "registration.secure-id") + self.check_persist_property("secure_id", "registration.secure-id") def test_secure_id_as_unicode(self): """secure-id is expected to be retrieved as unicode.""" @@ -51,8 +61,7 @@ class IdentityTest(LandscapeTest): self.assertEqual(self.identity.secure_id, "spam") def test_insecure_id(self): - self.check_persist_property("insecure_id", - "registration.insecure-id") + self.check_persist_property("insecure_id", "registration.insecure-id") def test_computer_title(self): self.check_config_property("computer_title") @@ -75,7 +84,7 @@ class RegistrationHandlerTestBase(LandscapeTest): helpers = [RegistrationHelper] def setUp(self): - super(RegistrationHandlerTestBase, self).setUp() + super().setUp() logging.getLogger().setLevel(logging.INFO) self.hostname = "ooga.local" self.addCleanup(setattr, socket, "getfqdn", socket.getfqdn) @@ -83,14 +92,14 @@ class RegistrationHandlerTestBase(LandscapeTest): class RegistrationHandlerTest(RegistrationHandlerTestBase): - def test_server_initiated_id_changing(self): """ The server must be able to ask a client to change its secure and insecure ids even if no requests were sent. """ self.exchanger.handle_message( - {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}) + {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}, + ) self.assertEqual(self.identity.secure_id, "abc") self.assertEqual(self.identity.insecure_id, "def") @@ -101,7 +110,8 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): """ reactor_fire_mock = self.reactor.fire = mock.Mock() self.exchanger.handle_message( - {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}) + {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}, + ) reactor_fire_mock.assert_any_call("registration-done") def test_unknown_id(self): @@ -120,9 +130,12 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.config.computer_title = "Wu" self.mstore.set_accepted_types(["register"]) self.exchanger.handle_message( - {"type": b"unknown-id", "clone-of": "Wu"}) - self.assertIn("Client is clone of computer Wu", - self.logfile.getvalue()) + {"type": b"unknown-id", "clone-of": "Wu"}, + ) + self.assertIn( + "Client is clone of computer Wu", + self.logfile.getvalue(), + ) def test_clone_secure_id_saved(self): """ @@ -134,7 +147,8 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.config.computer_title = "Wu" self.mstore.set_accepted_types(["register"]) self.exchanger.handle_message( - {"type": b"unknown-id", "clone-of": "Wu"}) + {"type": b"unknown-id", "clone-of": "Wu"}, + ) self.assertEqual(self.handler._clone_secure_id, secure_id) self.assertIsNone(self.identity.secure_id) @@ -148,7 +162,8 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.mstore.set_accepted_types(["register"]) self.mstore.set_server_api(b"3.3") # Note this is only for later api self.exchanger.handle_message( - {"type": b"unknown-id", "clone-of": "Wu"}) + {"type": b"unknown-id", "clone-of": "Wu"}, + ) self.reactor.fire("pre-exchange") messages = self.mstore.get_pending_messages() self.assertEqual(messages[0]["clone_secure_id"], secure_id) @@ -192,9 +207,11 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): messages = self.mstore.get_pending_messages() self.assertEqual(1, len(messages)) self.assertEqual("register", messages[0]["type"]) - self.assertEqual(self.logfile.getvalue().strip(), - "INFO: Queueing message to register with account " - "'account_name' without a password.") + self.assertEqual( + self.logfile.getvalue().strip(), + "INFO: Queueing message to register with account " + "'account_name' without a password.", + ) @mock.patch("landscape.client.broker.registration.get_vm_info") def test_queue_message_on_exchange_with_vm_info(self, get_vm_info_mock): @@ -210,14 +227,18 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.reactor.fire("pre-exchange") messages = self.mstore.get_pending_messages() self.assertEqual(b"vmware", messages[0]["vm-info"]) - self.assertEqual(self.logfile.getvalue().strip(), - "INFO: Queueing message to register with account " - "'account_name' without a password.") + self.assertEqual( + self.logfile.getvalue().strip(), + "INFO: Queueing message to register with account " + "'account_name' without a password.", + ) get_vm_info_mock.assert_called_once_with() @mock.patch("landscape.client.broker.registration.get_container_info") def test_queue_message_on_exchange_with_lxc_container( - self, get_container_info_mock): + self, + get_container_info_mock, + ): """ If the client is running in an LXC container, the information is included in the registration message. @@ -241,9 +262,11 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): messages = self.mstore.get_pending_messages() password = messages[0]["registration_password"] self.assertEqual("SEKRET", password) - self.assertEqual(self.logfile.getvalue().strip(), - "INFO: Queueing message to register with account " - "'account_name' with a password.") + self.assertEqual( + self.logfile.getvalue().strip(), + "INFO: Queueing message to register with account " + "'account_name' with a password.", + ) def test_queue_message_on_exchange_with_tags(self): """ @@ -254,14 +277,16 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.config.computer_title = "Computer Title" self.config.account_name = "account_name" self.config.registration_key = "SEKRET" - self.config.tags = u"computer,tag" + self.config.tags = "computer,tag" self.reactor.fire("pre-exchange") messages = self.mstore.get_pending_messages() self.assertEqual("computer,tag", messages[0]["tags"]) - self.assertEqual(self.logfile.getvalue().strip(), - "INFO: Queueing message to register with account " - "'account_name' and tags computer,tag with a " - "password.") + self.assertEqual( + self.logfile.getvalue().strip(), + "INFO: Queueing message to register with account " + "'account_name' and tags computer,tag with a " + "password.", + ) def test_queue_message_on_exchange_with_invalid_tags(self): """ @@ -273,14 +298,16 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.config.computer_title = "Computer Title" self.config.account_name = "account_name" self.config.registration_key = "SEKRET" - self.config.tags = u"" + self.config.tags = "" self.reactor.fire("pre-exchange") messages = self.mstore.get_pending_messages() self.assertIs(None, messages[0]["tags"]) - self.assertEqual(self.logfile.getvalue().strip(), - "ERROR: Invalid tags provided for registration.\n " - "INFO: Queueing message to register with account " - "'account_name' with a password.") + self.assertEqual( + self.logfile.getvalue().strip(), + "ERROR: Invalid tags provided for registration.\n " + "INFO: Queueing message to register with account " + "'account_name' with a password.", + ) def test_queue_message_on_exchange_with_unicode_tags(self): """ @@ -291,10 +318,10 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.config.computer_title = "Computer Title" self.config.account_name = "account_name" self.config.registration_key = "SEKRET" - self.config.tags = u"prova\N{LATIN SMALL LETTER J WITH CIRCUMFLEX}o" + self.config.tags = "prova\N{LATIN SMALL LETTER J WITH CIRCUMFLEX}o" self.reactor.fire("pre-exchange") messages = self.mstore.get_pending_messages() - expected = u"prova\N{LATIN SMALL LETTER J WITH CIRCUMFLEX}o" + expected = "prova\N{LATIN SMALL LETTER J WITH CIRCUMFLEX}o" self.assertEqual(expected, messages[0]["tags"]) logs = self.logfile.getvalue().strip() @@ -306,10 +333,12 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): # here, to circumvent that problem. if _PY3: logs = logs.encode("utf-8") - self.assertEqual(logs, - b"INFO: Queueing message to register with account " - b"'account_name' and tags prova\xc4\xb5o " - b"with a password.") + self.assertEqual( + logs, + b"INFO: Queueing message to register with account " + b"'account_name' and tags prova\xc4\xb5o " + b"with a password.", + ) def test_queue_message_on_exchange_with_access_group(self): """ @@ -318,15 +347,17 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): """ self.mstore.set_accepted_types(["register"]) self.config.account_name = "account_name" - self.config.access_group = u"dinosaurs" - self.config.tags = u"server,london" + self.config.access_group = "dinosaurs" + self.config.tags = "server,london" self.reactor.fire("pre-exchange") messages = self.mstore.get_pending_messages() self.assertEqual("dinosaurs", messages[0]["access_group"]) - self.assertEqual(self.logfile.getvalue().strip(), - "INFO: Queueing message to register with account " - "'account_name' in access group 'dinosaurs' and " - "tags server,london without a password.") + self.assertEqual( + self.logfile.getvalue().strip(), + "INFO: Queueing message to register with account " + "'account_name' in access group 'dinosaurs' and " + "tags server,london without a password.", + ) def test_queue_message_on_exchange_with_empty_access_group(self): """ @@ -334,7 +365,7 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): an "access_group" key. """ self.mstore.set_accepted_types(["register"]) - self.config.access_group = u"" + self.config.access_group = "" self.reactor.fire("pre-exchange") messages = self.mstore.get_pending_messages() # Make sure the key does not appear in the outgoing message. @@ -368,8 +399,7 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): self.assertEqual(messages[0]["type"], "register") def test_no_message_when_should_register_is_false(self): - """If we already have a secure id, do not queue a register message. - """ + """If we already have a secure id, do not queue a register message.""" self.mstore.set_accepted_types(["register"]) self.config.computer_title = "Computer Title" self.config.account_name = "account_name" @@ -395,9 +425,12 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): """ reactor_fire_mock = self.reactor.fire = mock.Mock() self.exchanger.handle_message( - {"type": b"registration", "info": b"unknown-account"}) + {"type": b"registration", "info": b"unknown-account"}, + ) reactor_fire_mock.assert_called_with( - "registration-failed", reason="unknown-account") + "registration-failed", + reason="unknown-account", + ) def test_registration_failed_event_max_pending_computers(self): """ @@ -407,9 +440,12 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): """ reactor_fire_mock = self.reactor.fire = mock.Mock() self.exchanger.handle_message( - {"type": b"registration", "info": b"max-pending-computers"}) + {"type": b"registration", "info": b"max-pending-computers"}, + ) reactor_fire_mock.assert_called_with( - "registration-failed", reason="max-pending-computers") + "registration-failed", + reason="max-pending-computers", + ) def test_registration_failed_event_not_fired_when_uncertain(self): """ @@ -418,7 +454,8 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): """ reactor_fire_mock = self.reactor.fire = mock.Mock() self.exchanger.handle_message( - {"type": b"registration", "info": b"blah-blah"}) + {"type": b"registration", "info": b"blah-blah"}, + ) for name, args, kwargs in reactor_fire_mock.mock_calls: self.assertNotEquals("registration-failed", args[0]) @@ -449,13 +486,15 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): # This should somehow callback the deferred. self.exchanger.handle_message( - {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}) + {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}, + ) self.assertEqual(calls, [1]) # Doing it again to ensure that the deferred isn't called twice. self.exchanger.handle_message( - {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}) + {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}, + ) self.assertEqual(calls, [1]) @@ -477,7 +516,8 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): # This should somehow callback the deferred. self.exchanger.handle_message( - {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}) + {"type": b"set-id", "id": b"abc", "insecure-id": b"def"}, + ) self.assertEqual(results, [None]) @@ -502,13 +542,15 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): # This should somehow callback the deferred. self.exchanger.handle_message( - {"type": b"registration", "info": b"unknown-account"}) + {"type": b"registration", "info": b"unknown-account"}, + ) self.assertEqual(calls, [True]) # Doing it again to ensure that the deferred isn't called twice. self.exchanger.handle_message( - {"type": b"registration", "info": b"unknown-account"}) + {"type": b"registration", "info": b"unknown-account"}, + ) self.assertEqual(calls, [True]) @@ -534,13 +576,15 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): d.addErrback(add_call) self.exchanger.handle_message( - {"type": b"registration", "info": b"max-pending-computers"}) + {"type": b"registration", "info": b"max-pending-computers"}, + ) self.assertEqual(calls, [True]) # Doing it again to ensure that the deferred isn't called twice. self.exchanger.handle_message( - {"type": b"registration", "info": b"max-pending-computers"}) + {"type": b"registration", "info": b"max-pending-computers"}, + ) self.assertEqual(calls, [True]) @@ -575,9 +619,13 @@ class RegistrationHandlerTest(RegistrationHandlerTestBase): class JujuRegistrationHandlerTest(RegistrationHandlerTestBase): - juju_contents = json.dumps({"environment-uuid": "DEAD-BEEF", - "machine-id": "1", - "api-addresses": "10.0.3.1:17070"}) + juju_contents = json.dumps( + { + "environment-uuid": "DEAD-BEEF", + "machine-id": "1", + "api-addresses": "10.0.3.1:17070", + }, + ) def test_juju_info_added_when_present(self): """ @@ -593,10 +641,13 @@ class JujuRegistrationHandlerTest(RegistrationHandlerTestBase): messages = self.mstore.get_pending_messages() self.assertEqual( - {"environment-uuid": "DEAD-BEEF", - "machine-id": "1", - "api-addresses": ["10.0.3.1:17070"]}, - messages[0]["juju-info"]) + { + "environment-uuid": "DEAD-BEEF", + "machine-id": "1", + "api-addresses": ["10.0.3.1:17070"], + }, + messages[0]["juju-info"], + ) def test_juju_info_skipped_with_old_server(self): """ diff --git a/landscape/client/broker/tests/test_server.py b/landscape/client/broker/tests/test_server.py index 99fa65a..13461eb 100644 --- a/landscape/client/broker/tests/test_server.py +++ b/landscape/client/broker/tests/test_server.py @@ -1,23 +1,23 @@ import random +from unittest.mock import Mock from configobj import ConfigObj -from mock import Mock -from twisted.internet.defer import succeed, fail +from twisted.internet.defer import fail +from twisted.internet.defer import succeed -from landscape.client.manager.manager import FAILED -from landscape.client.tests.helpers import ( - LandscapeTest, DEFAULT_ACCEPTED_TYPES) -from landscape.client.broker.tests.helpers import ( - BrokerServerHelper, RemoteClientHelper) +from landscape.client.broker.tests.helpers import BrokerServerHelper +from landscape.client.broker.tests.helpers import RemoteClientHelper from landscape.client.broker.tests.test_ping import FakePageGetter +from landscape.client.manager.manager import FAILED +from landscape.client.tests.helpers import DEFAULT_ACCEPTED_TYPES +from landscape.client.tests.helpers import LandscapeTest -class FakeClient(object): +class FakeClient: pass -class FakeCreator(object): - +class FakeCreator: def __init__(self, reactor, config): pass @@ -105,7 +105,11 @@ class BrokerServerTest(LandscapeTest): message = {"type": "test"} self.mstore.set_accepted_types(["test"]) self.assertRaises( - RuntimeError, self.broker.send_message, message, None) + RuntimeError, + self.broker.send_message, + message, + None, + ) def test_send_message_with_old_release_upgrader(self): """ @@ -123,8 +127,11 @@ class BrokerServerTest(LandscapeTest): If we receive a message from an old package-changer process that doesn't know about session IDs, we just let the message in. """ - message = {"type": "change-packages-result", "operation-id": 99, - "result-code": 123} + message = { + "type": "change-packages-result", + "operation-id": 99, + "result-code": 123, + } self.mstore.set_accepted_types(["change-packages-result"]) self.broker.send_message(message, True) self.assertMessages(self.mstore.get_pending_messages(), [message]) @@ -138,14 +145,17 @@ class BrokerServerTest(LandscapeTest): legacy_message = { b"type": b"change-packages-result", b"operation-id": 99, - b"result-code": 123} + b"result-code": 123, + } self.mstore.set_accepted_types(["change-packages-result"]) self.broker.send_message(legacy_message, True) - expected = [{ - "type": "change-packages-result", - "operation-id": 99, - "result-code": 123 - }] + expected = [ + { + "type": "change-packages-result", + "operation-id": 99, + "result-code": 123, + }, + ] self.assertMessages(self.mstore.get_pending_messages(), expected) self.assertTrue(self.exchanger.is_urgent()) @@ -176,9 +186,11 @@ class BrokerServerTest(LandscapeTest): self.assertEqual(len(self.broker.get_clients()), 1) self.assertEqual(len(self.broker.get_connectors()), 1) self.assertTrue( - isinstance(self.broker.get_client("test"), FakeClient)) + isinstance(self.broker.get_client("test"), FakeClient), + ) self.assertTrue( - isinstance(self.broker.get_connector("test"), FakeCreator)) + isinstance(self.broker.get_connector("test"), FakeCreator), + ) self.broker.connectors_registry = {"test": FakeCreator} result = self.broker.register_client("test") @@ -190,8 +202,10 @@ class BrokerServerTest(LandscapeTest): of each registered client, and returns a deferred resulting in C{None} if all C{exit} calls were successful. """ - self.broker.connectors_registry = {"foo": FakeCreator, - "bar": FakeCreator} + self.broker.connectors_registry = { + "foo": FakeCreator, + "bar": FakeCreator, + } self.broker.register_client("foo") self.broker.register_client("bar") for client in self.broker.get_clients(): @@ -203,8 +217,10 @@ class BrokerServerTest(LandscapeTest): The L{BrokerServer.stop_clients} method calls the C{exit} method of each registered client, and raises an exception if any calls fail. """ - self.broker.connectors_registry = {"foo": FakeCreator, - "bar": FakeCreator} + self.broker.connectors_registry = { + "foo": FakeCreator, + "bar": FakeCreator, + } self.broker.register_client("foo") self.broker.register_client("bar") [client1, client2] = self.broker.get_clients() @@ -221,8 +237,12 @@ class BrokerServerTest(LandscapeTest): config_obj["client"]["computer_title"] = "New Title" config_obj.write() result = self.broker.reload_configuration() - result.addCallback(lambda x: self.assertEqual( - self.config.computer_title, "New Title")) + result.addCallback( + lambda x: self.assertEqual( + self.config.computer_title, + "New Title", + ), + ) return result def test_reload_configuration_stops_clients(self): @@ -230,8 +250,10 @@ class BrokerServerTest(LandscapeTest): The L{BrokerServer.reload_configuration} method forces the config file associated with the broker server to be reloaded. """ - self.broker.connectors_registry = {"foo": FakeCreator, - "bar": FakeCreator} + self.broker.connectors_registry = { + "foo": FakeCreator, + "bar": FakeCreator, + } self.broker.register_client("foo") self.broker.register_client("bar") for client in self.broker.get_clients(): @@ -245,8 +267,9 @@ class BrokerServerTest(LandscapeTest): """ registered = self.broker.register() # This should callback the deferred. - self.exchanger.handle_message({"type": "set-id", "id": "abc", - "insecure-id": "def"}) + self.exchanger.handle_message( + {"type": "set-id", "id": "abc", "insecure-id": "def"}, + ) return self.assertSuccess(registered) def test_get_accepted_types_empty(self): @@ -263,8 +286,10 @@ class BrokerServerTest(LandscapeTest): message types accepted by the Landscape server. """ self.mstore.set_accepted_types(["foo", "bar"]) - self.assertEqual(sorted(self.broker.get_accepted_message_types()), - ["bar", "foo"]) + self.assertEqual( + sorted(self.broker.get_accepted_message_types()), + ["bar", "foo"], + ) def test_get_server_uuid_with_unset_uuid(self): """ @@ -288,8 +313,10 @@ class BrokerServerTest(LandscapeTest): """ self.broker.register_client_accepted_message_type("type1") self.broker.register_client_accepted_message_type("type2") - self.assertEqual(self.exchanger.get_client_accepted_message_types(), - sorted(["type1", "type2"] + DEFAULT_ACCEPTED_TYPES)) + self.assertEqual( + self.exchanger.get_client_accepted_message_types(), + sorted(["type1", "type2"] + DEFAULT_ACCEPTED_TYPES), + ) def test_fire_event(self): """ @@ -304,8 +331,10 @@ class BrokerServerTest(LandscapeTest): """ The L{BrokerServer.exit} method stops all registered clients. """ - self.broker.connectors_registry = {"foo": FakeCreator, - "bar": FakeCreator} + self.broker.connectors_registry = { + "foo": FakeCreator, + "bar": FakeCreator, + } self.broker.register_client("foo") self.broker.register_client("bar") for client in self.broker.get_clients(): @@ -430,8 +459,10 @@ class EventTest(LandscapeTest): """ callback = Mock(return_value="foo") self.client_reactor.call_on("resynchronize", callback) - return self.assertSuccess(self.broker.resynchronize(["foo"]), - [["foo"]]) + return self.assertSuccess( + self.broker.resynchronize(["foo"]), + [["foo"]], + ) def test_impending_exchange(self): """ @@ -448,7 +479,9 @@ class EventTest(LandscapeTest): plugin.exchange.assert_called_once_with() deferred = self.assertSuccess( - self.broker.impending_exchange(), [[None]]) + self.broker.impending_exchange(), + [[None]], + ) deferred.addCallback(assert_called) return deferred @@ -464,12 +497,15 @@ class EventTest(LandscapeTest): self.remote.register_client = Mock() def assert_called_made(ignored): - self.remote.register_client_accepted_message_type\ - .assert_called_once_with("type") + self.remote.register_client_accepted_message_type.assert_called_once_with( # noqa: E501 + "type", + ) self.remote.register_client.assert_called_once_with("client") deferred = self.assertSuccess( - self.broker.broker_reconnect(), [[None]]) + self.broker.broker_reconnect(), + [[None]], + ) return deferred.addCallback(assert_called_made) registered = self.client.register_message("type", lambda x: None) @@ -488,7 +524,9 @@ class EventTest(LandscapeTest): self.client_reactor.call_on("server-uuid-changed", callback) deferred = self.assertSuccess( - self.broker.server_uuid_changed(None, "abc"), [[return_value]]) + self.broker.server_uuid_changed(None, "abc"), + [[return_value]], + ) return deferred.addCallback(assert_called) def test_message_type_acceptance_changed(self): @@ -499,7 +537,9 @@ class EventTest(LandscapeTest): return_value = random.randint(1, 100) callback = Mock(return_value=return_value) self.client_reactor.call_on( - ("message-type-acceptance-changed", "type"), callback) + ("message-type-acceptance-changed", "type"), + callback, + ) result = self.broker.message_type_acceptance_changed("type", True) return self.assertSuccess(result, [[return_value]]) @@ -512,7 +552,9 @@ class EventTest(LandscapeTest): callback = Mock(return_value=return_value) self.client_reactor.call_on("package-data-changed", callback) return self.assertSuccess( - self.broker.package_data_changed(), [[return_value]]) + self.broker.package_data_changed(), + [[return_value]], + ) class HandlersTest(LandscapeTest): @@ -520,7 +562,7 @@ class HandlersTest(LandscapeTest): helpers = [BrokerServerHelper] def setUp(self): - super(HandlersTest, self).setUp() + super().setUp() self.broker.connectors_registry = {"test": FakeCreator} self.broker.register_client("test") self.client = self.broker.get_client("test") @@ -549,18 +591,25 @@ class HandlersTest(LandscapeTest): result = self.reactor.fire("message", message) result = [i for i in result if i is not None][0] - class StartsWith(object): - + class StartsWith: def __eq__(self, other): return other.startswith( - "Landscape client failed to handle this request (foobar)") + "Landscape client failed to handle this request (foobar)", + ) def broadcasted(ignored): self.client.message.assert_called_once_with(message) self.assertMessages( self.mstore.get_pending_messages(), - [{"type": "operation-result", "status": FAILED, - "result-text": StartsWith(), "operation-id": 4}]) + [ + { + "type": "operation-result", + "status": FAILED, + "result-text": StartsWith(), + "operation-id": 4, + }, + ], + ) result.addCallback(broadcasted) return result @@ -582,7 +631,10 @@ class HandlersTest(LandscapeTest): self.client.fire_event = Mock(return_value=succeed(None)) self.reactor.fire("message-type-acceptance-changed", "test", True) self.client.fire_event.assert_called_once_with( - "message-type-acceptance-changed", "test", True) + "message-type-acceptance-changed", + "test", + True, + ) def test_server_uuid_changed(self): """ @@ -592,7 +644,10 @@ class HandlersTest(LandscapeTest): self.client.fire_event = Mock(return_value=succeed(None)) self.reactor.fire("server-uuid-changed", None, 123) self.client.fire_event.assert_called_once_with( - "server-uuid-changed", None, 123) + "server-uuid-changed", + None, + 123, + ) def test_package_data_changed(self): """ diff --git a/landscape/client/broker/tests/test_service.py b/landscape/client/broker/tests/test_service.py index 580f64e..e99f9ac 100644 --- a/landscape/client/broker/tests/test_service.py +++ b/landscape/client/broker/tests/test_service.py @@ -1,12 +1,11 @@ import os +from unittest.mock import Mock -from mock import Mock - -from landscape.client.tests.helpers import LandscapeTest -from landscape.client.broker.tests.helpers import BrokerConfigurationHelper +from landscape.client.broker.amp import RemoteBrokerConnector from landscape.client.broker.service import BrokerService +from landscape.client.broker.tests.helpers import BrokerConfigurationHelper from landscape.client.broker.transport import HTTPTransport -from landscape.client.broker.amp import RemoteBrokerConnector +from landscape.client.tests.helpers import LandscapeTest from landscape.lib.testing import FakeReactor @@ -15,7 +14,7 @@ class BrokerServiceTest(LandscapeTest): helpers = [BrokerConfigurationHelper] def setUp(self): - super(BrokerServiceTest, self).setUp() + super().setUp() class FakeBrokerService(BrokerService): reactor_factory = FakeReactor @@ -28,7 +27,8 @@ class BrokerServiceTest(LandscapeTest): """ self.assertEqual( self.service.persist.filename, - os.path.join(self.config.data_path, "broker.bpickle")) + os.path.join(self.config.data_path, "broker.bpickle"), + ) def test_transport(self): """ diff --git a/landscape/client/broker/tests/test_store.py b/landscape/client/broker/tests/test_store.py index 208c75c..b61edf3 100644 --- a/landscape/client/broker/tests/test_store.py +++ b/landscape/client/broker/tests/test_store.py @@ -1,22 +1,22 @@ import os -import mock - +from unittest import mock from twisted.python.compat import intToBytes +from landscape.client.broker.store import MessageStore +from landscape.client.tests.helpers import LandscapeTest from landscape.lib.bpickle import dumps from landscape.lib.persist import Persist -from landscape.lib.schema import InvalidError, Int, Bytes, Unicode +from landscape.lib.schema import Bytes +from landscape.lib.schema import Int +from landscape.lib.schema import InvalidError +from landscape.lib.schema import Unicode from landscape.message_schemas.message import Message -from landscape.client.broker.store import MessageStore - -from landscape.client.tests.helpers import LandscapeTest class MessageStoreTest(LandscapeTest): - def setUp(self): - super(MessageStoreTest, self).setUp() + super().setUp() self.temp_dir = self.makeDir() self.persist_filename = self.makeFile() self.store = self.create_store() @@ -137,13 +137,87 @@ class MessageStoreTest(LandscapeTest): self.assertEqual(self.store.get_pending_offset(), 0) self.assertEqual(self.store.get_pending_messages(), []) + def test_messages_over_limit(self): + """ + Create six messages, two per directory. Since there is a limit of + one directory then only the last 2 messages should be in the queue + """ + + self.store._directory_size = 2 + self.store._max_dirs = 1 + for num in range(6): # 0,1 2,3 4,5 + message = {"type": "data", "data": f"{num}".encode()} + self.store.add(message) + messages = self.store.get_pending_messages(200) + self.assertMessages( + messages, + [{"type": "data", "data": b"4"}, {"type": "data", "data": b"5"}], + ) + + def test_messages_under_limit(self): + """ + Create six messages, all of which should be in the queue, since 3 + directories are active + """ + + self.store._directory_size = 2 + self.store._max_dirs = 3 + messages_sent = [] + for num in range(6): # 0,1 2,3 4,5 + message = {"type": "data", "data": f"{num}".encode()} + messages_sent.append(message) + self.store.add(message) + messages = self.store.get_pending_messages(200) + self.assertMessages(messages, messages_sent) + + def test_messages_over_mb(self): + """ + Create three messages with the second one being very large. The + max size should get triggered, so all should be cleared except for the + last one. + """ + + self.store._directory_size = 2 + self.store._max_size_mb = 0.01 + self.store.add({"type": "data", "data": b"a"}) + self.store.add({"type": "data", "data": b"b" * 15000}) + self.store.add({"type": "data", "data": b"c"}) + messages = self.store.get_pending_messages(200) + self.assertMessages( + messages, + [{"type": "data", "data": b"c"}], + ) + + @mock.patch("shutil.rmtree") + def test_exception_on_message_limit(self, rmtree_mock): + """ + If an exception occurs while deleting it shouldn't affect the next + message sent + """ + rmtree_mock.side_effect = IOError("Error!") + self.store._directory_size = 1 + self.store._max_dirs = 1 + self.store.add({"type": "data", "data": b"a"}) + self.store.add({"type": "data", "data": b"b"}) + self.store.add({"type": "data", "data": b"c"}) + messages = self.store.get_pending_messages(200) + + self.assertMessages( + messages, + [ + {"type": "data", "data": b"a"}, + {"type": "data", "data": b"b"}, + {"type": "data", "data": b"c"}, + ], + ) + def test_one_message(self): self.store.add(dict(type="data", data=b"A thing")) messages = self.store.get_pending_messages(200) - self.assertMessages(messages, - [{"type": "data", - "data": b"A thing", - "api": b"3.2"}]) + self.assertMessages( + messages, + [{"type": "data", "data": b"A thing", "api": b"3.2"}], + ) def test_max_pending(self): for i in range(10): @@ -169,7 +243,7 @@ class MessageStoreTest(LandscapeTest): self.store.add(dict(type="data", data=intToBytes(i))) il = [m["data"] for m in self.store.get_pending_messages(60)] self.assertEqual(il, [intToBytes(i) for i in range(60)]) - self.assertEqual(set(os.listdir(self.temp_dir)), set(["0", "1", "2"])) + self.assertEqual(set(os.listdir(self.temp_dir)), {"0", "1", "2"}) self.store.set_pending_offset(60) self.store.delete_old_messages() @@ -177,23 +251,26 @@ class MessageStoreTest(LandscapeTest): def test_unaccepted(self): for i in range(10): - self.store.add(dict(type=["data", "unaccepted"][i % 2], - data=intToBytes(i))) + self.store.add( + dict(type=["data", "unaccepted"][i % 2], data=intToBytes(i)), + ) il = [m["data"] for m in self.store.get_pending_messages(20)] self.assertEqual(il, [intToBytes(i) for i in [0, 2, 4, 6, 8]]) def test_unaccepted_with_offset(self): for i in range(10): - self.store.add(dict(type=["data", "unaccepted"][i % 2], - data=intToBytes(i))) + self.store.add( + dict(type=["data", "unaccepted"][i % 2], data=intToBytes(i)), + ) self.store.set_pending_offset(2) il = [m["data"] for m in self.store.get_pending_messages(20)] self.assertEqual(il, [intToBytes(i) for i in [4, 6, 8]]) def test_unaccepted_reaccepted(self): for i in range(10): - self.store.add(dict(type=["data", "unaccepted"][i % 2], - data=intToBytes(i))) + self.store.add( + dict(type=["data", "unaccepted"][i % 2], data=intToBytes(i)), + ) self.store.set_pending_offset(2) il = [m["data"] for m in self.store.get_pending_messages(2)] self.store.set_accepted_types(["data", "unaccepted"]) @@ -202,8 +279,9 @@ class MessageStoreTest(LandscapeTest): def test_accepted_unaccepted(self): for i in range(10): - self.store.add(dict(type=["data", "unaccepted"][i % 2], - data=intToBytes(i))) + self.store.add( + dict(type=["data", "unaccepted"][i % 2], data=intToBytes(i)), + ) # Setting pending offset here means that the first two # messages, even though becoming unaccepted now, were already # accepted before, so they shouldn't be marked for hold. @@ -217,8 +295,9 @@ class MessageStoreTest(LandscapeTest): def test_accepted_unaccepted_old(self): for i in range(10): - self.store.add(dict(type=["data", "unaccepted"][i % 2], - data=intToBytes(i))) + self.store.add( + dict(type=["data", "unaccepted"][i % 2], data=intToBytes(i)), + ) self.store.set_pending_offset(2) self.store.set_accepted_types(["unaccepted"]) il = [m["data"] for m in self.store.get_pending_messages(20)] @@ -233,8 +312,10 @@ class MessageStoreTest(LandscapeTest): # messages will also be delivered. self.store.set_accepted_types(["data", "unaccepted"]) il = [m["data"] for m in self.store.get_pending_messages(20)] - self.assertEqual(il, [intToBytes(i) - for i in [1, 3, 5, 7, 9, 0, 2, 4, 6, 8]]) + self.assertEqual( + il, + [intToBytes(i) for i in [1, 3, 5, 7, 9, 0, 2, 4, 6, 8]], + ) def test_wb_handle_broken_messages(self): self.log_helper.ignore_errors(ValueError) @@ -281,8 +362,10 @@ class MessageStoreTest(LandscapeTest): messages = self.store.get_pending_messages() - self.assertEqual(messages, [{"type": "data", "data": b"2", - "api": b"3.2"}]) + self.assertEqual( + messages, + [{"type": "data", "data": b"2", "api": b"3.2"}], + ) self.store.set_pending_offset(len(messages)) @@ -307,11 +390,16 @@ class MessageStoreTest(LandscapeTest): # similar to unplugging the power -- i.e., we're not relying # on special exception-handling in the file-writing code. self.assertRaises( - IOError, self.store.add, {"type": "data", "data": 2}) + IOError, + self.store.add, + {"type": "data", "data": 2}, + ) mock_open.assert_called_with(mock.ANY, "wb") mock_open().write.assert_called_once_with(mock.ANY) - self.assertEqual(self.store.get_pending_messages(), - [{"type": "data", "data": 1, "api": b"3.2"}]) + self.assertEqual( + self.store.get_pending_messages(), + [{"type": "data", "data": 1, "api": b"3.2"}], + ) def test_get_server_api_default(self): """ @@ -331,8 +419,10 @@ class MessageStoreTest(LandscapeTest): By default messages are tagged with the 3.2 server API. """ self.store.add({"type": "empty"}) - self.assertEqual(self.store.get_pending_messages(), - [{"type": "empty", "api": b"3.2"}]) + self.assertEqual( + self.store.get_pending_messages(), + [{"type": "empty", "api": b"3.2"}], + ) def test_custom_api_on_store(self): """ @@ -341,14 +431,18 @@ class MessageStoreTest(LandscapeTest): """ self.store.set_server_api(b"3.3") self.store.add({"type": "empty"}) - self.assertEqual(self.store.get_pending_messages(), - [{"type": "empty", "api": b"3.3"}]) + self.assertEqual( + self.store.get_pending_messages(), + [{"type": "empty", "api": b"3.3"}], + ) def test_custom_api_on_messages(self): self.store.set_server_api(b"3.3") self.store.add({"type": "empty", "api": b"3.2"}) - self.assertEqual(self.store.get_pending_messages(), - [{"type": "empty", "api": b"3.2"}]) + self.assertEqual( + self.store.get_pending_messages(), + [{"type": "empty", "api": b"3.2"}], + ) def test_coercion(self): """ @@ -356,8 +450,11 @@ class MessageStoreTest(LandscapeTest): coerced according to the message schema for the type of the message. """ - self.assertRaises(InvalidError, - self.store.add, {"type": "data", "data": 3}) + self.assertRaises( + InvalidError, + self.store.add, + {"type": "data", "data": 3}, + ) def test_coercion_ignores_custom_api(self): """ @@ -372,12 +469,17 @@ class MessageStoreTest(LandscapeTest): the coercion. """ self.store.add_schema(Message("data", {"data": Unicode()})) - self.store.add({"type": "data", - "data": u"\N{HIRAGANA LETTER A}".encode("utf-8"), - "api": b"3.2"}) - self.assertEqual(self.store.get_pending_messages(), - [{"type": "data", "api": b"3.2", - "data": u"\N{HIRAGANA LETTER A}"}]) + self.store.add( + { + "type": "data", + "data": "\N{HIRAGANA LETTER A}".encode(), + "api": b"3.2", + }, + ) + self.assertEqual( + self.store.get_pending_messages(), + [{"type": "data", "api": b"3.2", "data": "\N{HIRAGANA LETTER A}"}], + ) def test_message_is_coerced_to_its_api_schema(self): """ @@ -392,7 +494,8 @@ class MessageStoreTest(LandscapeTest): self.store.add({"type": "data", "data": 123}) self.assertEqual( self.store.get_pending_messages(), - [{"type": "data", "api": b"3.3", "data": 123}]) + [{"type": "data", "api": b"3.3", "data": 123}], + ) def test_message_is_coerced_to_highest_compatible_api_schema(self): """ @@ -408,7 +511,8 @@ class MessageStoreTest(LandscapeTest): self.store.add({"type": "data", "data": b"foo"}) self.assertEqual( self.store.get_pending_messages(), - [{"type": "data", "api": b"3.2", "data": b"foo"}]) + [{"type": "data", "api": b"3.2", "data": b"foo"}], + ) def test_count_pending_messages(self): """It is possible to get the total number of pending messages.""" @@ -432,8 +536,7 @@ class MessageStoreTest(LandscapeTest): self.assertTrue(os.path.exists(filename)) store = MessageStore(Persist(filename=filename), self.temp_dir) - self.assertEqual(set(store.get_accepted_types()), - set(["foo", "bar"])) + self.assertEqual(set(store.get_accepted_types()), {"foo", "bar"}) def test_is_pending_pre_and_post_message_delivery(self): self.log_helper.ignore_errors(ValueError) @@ -507,8 +610,7 @@ class MessageStoreTest(LandscapeTest): self.assertEqual(global_session_id1, global_session_id2) def test_get_session_id_unique_for_each_scope(self): - """We get a unique session id for differing scopes. - """ + """We get a unique session id for differing scopes.""" session_id1 = self.store.get_session_id() session_id2 = self.store.get_session_id(scope="other") self.assertNotEqual(session_id1, session_id2) @@ -518,14 +620,14 @@ class MessageStoreTest(LandscapeTest): default. """ session_id = self.store.get_session_id() - persisted_ids = self.store._persist.get('session-ids') + persisted_ids = self.store._persist.get("session-ids") scope = persisted_ids[session_id] self.assertIs(None, scope) def test_get_session_id_with_scope(self): """Test that we can generate a session id within a limited scope.""" session_id = self.store.get_session_id(scope="hwinfo") - persisted_ids = self.store._persist.get('session-ids') + persisted_ids = self.store._persist.get("session-ids") scope = persisted_ids[session_id] self.assertEqual("hwinfo", scope) @@ -592,8 +694,7 @@ class MessageStoreTest(LandscapeTest): def test_record_failure_sets_first_failure_time(self): """first-failure-time recorded when calling record_failure().""" self.store.record_failure(123) - self.assertEqual( - 123, self.store._persist.get("first-failure-time")) + self.assertEqual(123, self.store._persist.get("first-failure-time")) def test_messages_rejected_if_failure_older_than_one_week(self): """Messages stop accumulating after one week of not being sent.""" @@ -602,10 +703,12 @@ class MessageStoreTest(LandscapeTest): self.assertIsNot(None, self.store.add({"type": "empty"})) self.store.record_failure((7 * 24 * 60 * 60) + 1) self.assertIs(None, self.store.add({"type": "empty"})) - self.assertIn("WARNING: Unable to succesfully communicate with " - "Landscape server for more than a week. Waiting for " - "resync.", - self.logfile.getvalue()) + self.assertIn( + "WARNING: Unable to succesfully communicate with " + "Landscape server for more than a week. Waiting for " + "resync.", + self.logfile.getvalue(), + ) # Resync message and the first one we added right on the week boundary self.assertEqual(2, len(self.store.get_pending_messages())) @@ -618,8 +721,10 @@ class MessageStoreTest(LandscapeTest): self.store.record_failure((7 * 24 * 60 * 60) + 1) self.store.add({"type": "empty"}) self.assertIs(None, self.store.add({"type": "empty"})) - self.assertIn("DEBUG: Dropped message, awaiting resync.", - self.logfile.getvalue()) + self.assertIn( + "DEBUG: Dropped message, awaiting resync.", + self.logfile.getvalue(), + ) def test_after_clearing_blackhole_messages_are_accepted_again(self): """After a successful exchange, messages are accepted again.""" @@ -644,13 +749,13 @@ class MessageStoreTest(LandscapeTest): filename = os.path.join(self.temp_dir, "0", "0") os.makedirs(os.path.dirname(filename)) with open(filename, "wb") as fh: - fh.write(dumps({b"type": b"data", - b"data": b"A thing", - b"api": b"3.2"})) + fh.write( + dumps({b"type": b"data", b"data": b"A thing", b"api": b"3.2"}), + ) [message] = self.store.get_pending_messages() # message keys are decoded - self.assertIn(u"type", message) - self.assertIn(u"api", message) - self.assertIsInstance(message[u"api"], bytes) # api is bytes - self.assertEqual(u"data", message[u"type"]) # message type is decoded - self.assertEqual(b"A thing", message[u"data"]) # other are kept as-is + self.assertIn("type", message) + self.assertIn("api", message) + self.assertIsInstance(message["api"], bytes) # api is bytes + self.assertEqual("data", message["type"]) # message type is decoded + self.assertEqual(b"A thing", message["data"]) # other are kept as-is diff --git a/landscape/client/broker/tests/test_transport.py b/landscape/client/broker/tests/test_transport.py index 87106a1..3430123 100644 --- a/landscape/client/broker/tests/test_transport.py +++ b/landscape/client/broker/tests/test_transport.py @@ -1,19 +1,18 @@ -# -*- coding: utf-8 -*- import os +from twisted.internet import reactor +from twisted.internet.ssl import DefaultOpenSSLContextFactory +from twisted.internet.threads import deferToThread +from twisted.web import resource +from twisted.web import server + from landscape import VERSION from landscape.client.broker.transport import HTTPTransport +from landscape.client.tests.helpers import LandscapeTest from landscape.lib import bpickle from landscape.lib.fetch import PyCurlError from landscape.lib.testing import LogKeeperHelper -from landscape.client.tests.helpers import LandscapeTest - -from twisted.web import server, resource -from twisted.internet import reactor -from twisted.internet.ssl import DefaultOpenSSLContextFactory -from twisted.internet.threads import deferToThread - def sibpath(path): return os.path.abspath(os.path.join(os.path.dirname(__file__), path)) @@ -29,7 +28,7 @@ class DataCollectingResource(resource.Resource): request = content = None - def getChild(self, request, name): + def getChild(self, request, name): # noqa: N802 return self def render(self, request): @@ -43,23 +42,33 @@ class HTTPTransportTest(LandscapeTest): helpers = [LogKeeperHelper] def setUp(self): - super(HTTPTransportTest, self).setUp() + super().setUp() self.ports = [] def tearDown(self): - super(HTTPTransportTest, self).tearDown() + super().tearDown() for port in self.ports: port.stopListening() def request_with_payload(self, payload): resource = DataCollectingResource() port = reactor.listenTCP( - 0, server.Site(resource), interface="127.0.0.1") + 0, + server.Site(resource), + interface="127.0.0.1", + ) self.ports.append(port) transport = HTTPTransport( - None, "http://localhost:%d/" % (port.getHost().port,)) - result = deferToThread(transport.exchange, payload, computer_id="34", - exchange_token="abcd-efgh", message_api="X.Y") + None, + f"http://localhost:{port.getHost().port:d}/", + ) + result = deferToThread( + transport.exchange, + payload, + computer_id="34", + exchange_token="abcd-efgh", + message_api="X.Y", + ) def got_result(ignored): try: @@ -70,12 +79,15 @@ class HTTPTransportTest(LandscapeTest): def get_header(header): return [resource.request.received_headers[header]] - self.assertEqual(get_header(u"x-computer-id"), ["34"]) + self.assertEqual(get_header("x-computer-id"), ["34"]) self.assertEqual(get_header("x-exchange-token"), ["abcd-efgh"]) self.assertEqual( - get_header("user-agent"), ["landscape-client/%s" % (VERSION,)]) + get_header("user-agent"), + [f"landscape-client/{VERSION}"], + ) self.assertEqual(get_header("x-message-api"), ["X.Y"]) self.assertEqual(bpickle.loads(resource.content), payload) + result.addCallback(got_result) return result @@ -103,7 +115,7 @@ class HTTPTransportTest(LandscapeTest): When a payload contains unicode characters they are properly handled by bpickle. """ - return self.request_with_payload(payload=u"проба") + return self.request_with_payload(payload="проба") def test_ssl_verification_positive(self): """ @@ -113,13 +125,24 @@ class HTTPTransportTest(LandscapeTest): """ resource = DataCollectingResource() context_factory = DefaultOpenSSLContextFactory(PRIVKEY, PUBKEY) - port = reactor.listenSSL(0, server.Site(resource), context_factory, - interface="127.0.0.1") + port = reactor.listenSSL( + 0, + server.Site(resource), + context_factory, + interface="127.0.0.1", + ) self.ports.append(port) transport = HTTPTransport( - None, "https://localhost:%d/" % (port.getHost().port,), PUBKEY) - result = deferToThread(transport.exchange, "HI", computer_id="34", - message_api="X.Y") + None, + f"https://localhost:{port.getHost().port:d}/", + PUBKEY, + ) + result = deferToThread( + transport.exchange, + "HI", + computer_id="34", + message_api="X.Y", + ) def got_result(ignored): try: @@ -129,11 +152,15 @@ class HTTPTransportTest(LandscapeTest): # without requestHeaders def get_header(header): return [resource.request.received_headers[header]] + self.assertEqual(get_header("x-computer-id"), ["34"]) self.assertEqual( - get_header("user-agent"), ["landscape-client/%s" % (VERSION,)]) + get_header("user-agent"), + [f"landscape-client/{VERSION}"], + ) self.assertEqual(get_header("x-message-api"), ["X.Y"]) self.assertEqual(bpickle.loads(resource.content), "HI") + result.addCallback(got_result) return result @@ -145,21 +172,34 @@ class HTTPTransportTest(LandscapeTest): """ self.log_helper.ignore_errors(PyCurlError) r = DataCollectingResource() - context_factory = DefaultOpenSSLContextFactory( - BADPRIVKEY, BADPUBKEY) - port = reactor.listenSSL(0, server.Site(r), context_factory, - interface="127.0.0.1") + context_factory = DefaultOpenSSLContextFactory(BADPRIVKEY, BADPUBKEY) + port = reactor.listenSSL( + 0, + server.Site(r), + context_factory, + interface="127.0.0.1", + ) self.ports.append(port) - transport = HTTPTransport(None, "https://localhost:%d/" - % (port.getHost().port,), pubkey=PUBKEY) - - result = deferToThread(transport.exchange, "HI", computer_id="34", - message_api="X.Y") + transport = HTTPTransport( + None, + f"https://localhost:{port.getHost().port:d}/", + pubkey=PUBKEY, + ) + + result = deferToThread( + transport.exchange, + "HI", + computer_id="34", + message_api="X.Y", + ) def got_result(ignored): self.assertIs(r.request, None) self.assertIs(r.content, None) - self.assertTrue("server certificate verification failed" - in self.logfile.getvalue()) + self.assertTrue( + "server certificate verification failed" + in self.logfile.getvalue(), + ) + result.addErrback(got_result) return result diff --git a/landscape/client/broker/transport.py b/landscape/client/broker/transport.py index 27b70fb..e4c6e00 100644 --- a/landscape/client/broker/transport.py +++ b/landscape/client/broker/transport.py @@ -1,20 +1,21 @@ """Low-level server communication.""" -import time import logging import pprint +import time import uuid import pycurl -from landscape.lib.compat import unicode, _PY3 - +from landscape import SERVER_API +from landscape import VERSION from landscape.lib import bpickle +from landscape.lib.compat import _PY3 +from landscape.lib.compat import unicode from landscape.lib.fetch import fetch from landscape.lib.format import format_delta -from landscape import SERVER_API, VERSION -class HTTPTransport(object): +class HTTPTransport: """Transport makes a request to exchange message data over HTTP. @param url: URL of the remote Landscape server message system. @@ -40,9 +41,11 @@ class HTTPTransport(object): # assigning them to the headers. if _PY3 and isinstance(message_api, bytes): message_api = message_api.decode("ascii") - headers = {"X-Message-API": message_api, - "User-Agent": "landscape-client/%s" % VERSION, - "Content-Type": "application/octet-stream"} + headers = { + "X-Message-API": message_api, + "User-Agent": f"landscape-client/{VERSION}", + "Content-Type": "application/octet-stream", + } if computer_id: if _PY3 and isinstance(computer_id, bytes): computer_id = computer_id.decode("ascii") @@ -52,11 +55,25 @@ class HTTPTransport(object): exchange_token = exchange_token.decode("ascii") headers["X-Exchange-Token"] = str(exchange_token) curl = pycurl.Curl() - return (curl, fetch(self._url, post=True, data=payload, - headers=headers, cainfo=self._pubkey, curl=curl)) - - def exchange(self, payload, computer_id=None, exchange_token=None, - message_api=SERVER_API): + return ( + curl, + fetch( + self._url, + post=True, + data=payload, + headers=headers, + cainfo=self._pubkey, + curl=curl, + ), + ) + + def exchange( + self, + payload, + computer_id=None, + exchange_token=None, + message_api=SERVER_API, + ): """Exchange message data with the server. @param payload: The object to send, it must be L{bpickle}-compatible. @@ -78,30 +95,39 @@ class HTTPTransport(object): if logging.getLogger().getEffectiveLevel() <= logging.DEBUG: logging.debug("Sending payload:\n%s", pprint.pformat(payload)) try: - curly, data = self._curl(spayload, computer_id, exchange_token, - message_api) + curly, data = self._curl( + spayload, + computer_id, + exchange_token, + message_api, + ) except Exception: - logging.exception("Error contacting the server at %s." % self._url) + logging.exception(f"Error contacting the server at {self._url}.") raise else: - logging.info("Sent %d bytes and received %d bytes in %s.", - len(spayload), len(data), - format_delta(time.time() - start_time)) + logging.info( + "Sent %d bytes and received %d bytes in %s.", + len(spayload), + len(data), + format_delta(time.time() - start_time), + ) try: response = bpickle.loads(data) except Exception: - logging.exception("Server returned invalid data: %r" % data) + logging.exception(f"Server returned invalid data: {data!r}") return None else: if logging.getLogger().getEffectiveLevel() <= logging.DEBUG: logging.debug( - "Received payload:\n%s", pprint.pformat(response)) + "Received payload:\n%s", + pprint.pformat(response), + ) return response -class FakeTransport(object): +class FakeTransport: """Fake transport for testing purposes.""" def __init__(self, reactor=None, url=None, pubkey=None): @@ -123,8 +149,13 @@ class FakeTransport(object): def set_url(self, url): self._url = url - def exchange(self, payload, computer_id=None, exchange_token=None, - message_api=SERVER_API): + def exchange( + self, + payload, + computer_id=None, + exchange_token=None, + message_api=SERVER_API, + ): self.payloads.append(payload) self.computer_id = computer_id self.exchange_token = exchange_token @@ -140,8 +171,10 @@ class FakeTransport(object): if isinstance(response, Exception): raise response - result = {"next-expected-sequence": self.next_expected_sequence, - "next-exchange-token": unicode(uuid.uuid4()), - "messages": response} + result = { + "next-expected-sequence": self.next_expected_sequence, + "next-exchange-token": unicode(uuid.uuid4()), + "messages": response, + } result.update(self.extra) return result diff --git a/landscape/client/configuration.py b/landscape/client/configuration.py index e06dd26..621e797 100644 --- a/landscape/client/configuration.py +++ b/landscape/client/configuration.py @@ -3,35 +3,36 @@ This module, and specifically L{LandscapeSetupScript}, implements the support for the C{landscape-config} script. """ - -from __future__ import print_function - -from functools import partial import getpass import io import os import pwd +import shlex import sys import textwrap +from functools import partial +from urllib.parse import urlparse -from landscape.lib.compat import input +from landscape.client.broker.amp import RemoteBrokerConnector +from landscape.client.broker.config import BrokerConfiguration +from landscape.client.broker.registration import Identity +from landscape.client.broker.registration import RegistrationError +from landscape.client.broker.service import BrokerService +from landscape.client.reactor import LandscapeReactor +from landscape.client.serviceconfig import ServiceConfig +from landscape.client.serviceconfig import ServiceConfigException from landscape.lib import base64 - -from landscape.lib.tag import is_valid_tag - -from landscape.client.sysvconfig import SysVConfig, ProcessError from landscape.lib.amp import MethodCallError -from landscape.lib.twisted_util import gather_results -from landscape.lib.fetch import fetch, FetchError +from landscape.lib.bootstrap import BootstrapDirectory +from landscape.lib.bootstrap import BootstrapList +from landscape.lib.compat import input +from landscape.lib.fetch import fetch +from landscape.lib.fetch import FetchError from landscape.lib.fs import create_binary_file -from landscape.lib.bootstrap import BootstrapList, BootstrapDirectory +from landscape.lib.network import get_fqdn from landscape.lib.persist import Persist -from landscape.client.reactor import LandscapeReactor -from landscape.client.broker.registration import RegistrationError -from landscape.client.broker.config import BrokerConfiguration -from landscape.client.broker.amp import RemoteBrokerConnector -from landscape.client.broker.registration import Identity -from landscape.client.broker.service import BrokerService +from landscape.lib.tag import is_valid_tag +from landscape.lib.twisted_util import gather_results EXIT_NOT_REGISTERED = 5 @@ -46,8 +47,8 @@ class ImportOptionError(ConfigurationError): def print_text(text, end="\n", error=False): - """Display the given text to the user, using stderr if flagged as an error. - """ + """Display the given text to the user, using stderr + if flagged as an error.""" if error: stream = sys.stderr else: @@ -66,7 +67,7 @@ def prompt_yes_no(message, default=True): """Prompt for a yes/no question and return the answer as bool.""" default_msg = "[Y/n]" if default else "[y/N]" while True: - value = input("{} {}: ".format(message, default_msg)).lower() + value = input(f"{message} {default_msg}: ").lower() if value: if value.startswith("n"): return False @@ -87,7 +88,8 @@ def get_invalid_users(users): if "ALL" in user_list: if len(user_list) > 1: raise ConfigurationError( - "Extra users specified with ALL users") + "Extra users specified with ALL users", + ) user_list.remove("ALL") invalid_users = [] for user in user_list: @@ -100,8 +102,15 @@ def get_invalid_users(users): class LandscapeSetupConfiguration(BrokerConfiguration): - unsaved_options = ("no_start", "disable", "silent", "ok_no_register", - "import_from") + unsaved_options = ( + "no_start", + "disable", + "silent", + "ok_no_register", + "import_from", + ) + + encoding = "utf-8" def _load_external_options(self): """Handle the --import parameter. @@ -123,18 +132,23 @@ class LandscapeSetupConfiguration(BrokerConfiguration): content = self.fetch_import_url(self.import_from) parser = self._get_config_object( alternative_config=io.StringIO( - content.decode("utf-8"))) + content.decode("utf-8"), + ), + ) elif not os.path.isfile(self.import_from): - raise ImportOptionError("File %s doesn't exist." % - self.import_from) + raise ImportOptionError( + f"File {self.import_from} doesn't exist.", + ) else: try: parser = self._get_config_object( - alternative_config=self.import_from) + alternative_config=self.import_from, + ) except Exception: raise ImportOptionError( - "Couldn't read configuration from %s." % - self.import_from) + "Couldn't read configuration " + f"from {self.import_from}.", + ) except Exception as error: raise ImportOptionError(str(error)) @@ -143,15 +157,16 @@ class LandscapeSetupConfiguration(BrokerConfiguration): if parser and self.config_section in parser: options = parser[self.config_section] if not options: - raise ImportOptionError("Nothing to import at %s." % - self.import_from) + raise ImportOptionError( + f"Nothing to import at {self.import_from}.", + ) options.update(self._command_line_options) self._command_line_options = options def fetch_import_url(self, url): """Handle fetching of URLs passed to --url.""" - print_text("Fetching configuration from %s..." % url) + print_text(f"Fetching configuration from {url}...") error_message = None try: content = fetch(url) @@ -159,52 +174,79 @@ class LandscapeSetupConfiguration(BrokerConfiguration): error_message = str(error) if error_message is not None: raise ImportOptionError( - "Couldn't download configuration from %s: %s" % - (url, error_message)) + f"Couldn't download configuration from {url}: {error_message}", + ) return content def make_parser(self): """ Specialize the parser, adding configure-specific options. """ - parser = super(LandscapeSetupConfiguration, self).make_parser() - - parser.add_option("--import", dest="import_from", - metavar="FILENAME_OR_URL", - help="Filename or URL to import configuration from. " - "Imported options behave as if they were " - "passed in the command line, with precedence " - "being given to real command line options.") - parser.add_option("--script-users", metavar="USERS", - help="A comma-separated list of users to allow " - "scripts to run. To allow scripts to be run " - "by any user, enter: ALL") - parser.add_option("--include-manager-plugins", metavar="PLUGINS", - default="", - help="A comma-separated list of manager plugins to " - "load.") - parser.add_option("-n", "--no-start", action="store_true", - help="Don't start the client automatically.") - parser.add_option("--ok-no-register", action="store_true", - help="Return exit code 0 instead of 2 if the client " - "can't be registered.") - parser.add_option("--silent", action="store_true", default=False, - help="Run without manual interaction.") - parser.add_option("--disable", action="store_true", default=False, - help="Stop running clients and disable start at " - "boot.") - parser.add_option("--init", action="store_true", default=False, - help="Set up the client directories structure " - "and exit.") - parser.add_option("--is-registered", action="store_true", - help="Exit with code 0 (success) if client is " - "registered else returns {}. Displays " - "registration info." - .format(EXIT_NOT_REGISTERED)) + parser = super().make_parser() + + parser.add_option( + "--import", + dest="import_from", + metavar="FILENAME_OR_URL", + help="Filename or URL to import configuration from. " + "Imported options behave as if they were " + "passed in the command line, with precedence " + "being given to real command line options.", + ) + parser.add_option( + "--script-users", + metavar="USERS", + help="A comma-separated list of users to allow " + "scripts to run. To allow scripts to be run " + "by any user, enter: ALL", + ) + parser.add_option( + "--include-manager-plugins", + metavar="PLUGINS", + default="", + help="A comma-separated list of manager plugins to " "load.", + ) + parser.add_option( + "-n", + "--no-start", + action="store_true", + help="Don't start the client automatically.", + ) + parser.add_option( + "--ok-no-register", + action="store_true", + help="Return exit code 0 instead of 2 if the client " + "can't be registered.", + ) + parser.add_option( + "--silent", + action="store_true", + default=False, + help="Run without manual interaction.", + ) + parser.add_option( + "--disable", + action="store_true", + default=False, + help="Stop running clients and disable start at " "boot.", + ) + parser.add_option( + "--init", + action="store_true", + default=False, + help="Set up the client directories structure " "and exit.", + ) + parser.add_option( + "--is-registered", + action="store_true", + help="Exit with code 0 (success) if client is " + "registered else returns {}. Displays " + "registration info.".format(EXIT_NOT_REGISTERED), + ) return parser -class LandscapeSetupScript(object): +class LandscapeSetupScript: """ An interactive procedure which manages the prompting and temporary storage of configuration parameters. @@ -218,6 +260,16 @@ class LandscapeSetupScript(object): def __init__(self, config): self.config = config + self.landscape_domain = None + + def get_domain(self): + domain = self.landscape_domain + if domain: + return domain + url = self.config.url + if url: + return urlparse(url).netloc + return "landscape.canonical.com" def prompt_get_input(self, msg, required): """Prompt the user on the terminal for a value @@ -226,14 +278,14 @@ class LandscapeSetupScript(object): @param required: True if value must be entered """ while True: - value = input(msg) + value = input(msg).strip() if value: return value elif not required: break show_help("This option is required to configure Landscape.") - def prompt(self, option, msg, required=False): + def prompt(self, option, msg, required=False, default=None): """Prompt the user on the terminal for a value. @param option: The attribute of C{self.config} that contains the @@ -241,16 +293,21 @@ class LandscapeSetupScript(object): @param msg: The message to prompt the user with (via C{input}). @param required: If True, the user will be required to enter a value before continuing. + @param default: Default only if set and no current value present """ - default = getattr(self.config, option, None) + cur_value = getattr(self.config, option, None) + if cur_value: + default = cur_value if default: - msg += " [%s]: " % default + msg += f" [{default}]: " else: msg += ": " required = required and not (bool(default)) result = self.prompt_get_input(msg, required) if result: setattr(self.config, option, result) + elif default: + setattr(self.config, option, default) def password_prompt(self, option, msg, required=False): """Prompt the user on the terminal for a password and mask the value. @@ -287,25 +344,31 @@ class LandscapeSetupScript(object): show_help( """ The computer title you provide will be used to represent this - computer in the Landscape user interface. It's important to use - a title that will allow the system to be easily recognized when - it appears on the pending computers page. - """) - - self.prompt("computer_title", "This computer's title", True) + computer in the Landscape dashboard. + """, + ) + self.prompt( + "computer_title", + "This computer's title", + False, + default=get_fqdn(), + ) def query_account_name(self): if "account_name" in self.config.get_command_line_options(): return - show_help( - """ - You must now specify the name of the Landscape account you - want to register this computer with. Your account name is shown - under 'Account name' at https://landscape.canonical.com . - """) - - self.prompt("account_name", "Account name", True) + if not self.landscape_domain: + show_help( + """ + You must now specify the name of the Landscape account you + want to register this computer with. Your account name is shown + under 'Account name' at https://landscape.canonical.com . + """, + ) + self.prompt("account_name", "Account name", True) + else: + self.config.account_name = "standalone" def query_registration_key(self): command_line_options = self.config.get_command_line_options() @@ -313,18 +376,15 @@ class LandscapeSetupScript(object): return show_help( - """ - A registration key may be associated with your Landscape - account to prevent unauthorized registration attempts. This - is not your personal login password. It is optional, and unless - explicitly set on the server, it may be skipped here. + f""" + A Registration Key prevents unauthorized registration attempts. - If you don't remember the registration key you can find it - at https://landscape.canonical.com/account/%s - """ % self.config.account_name) + Provide the Registration Key found at: + https://{self.get_domain()}/account/{self.config.account_name} + """, # noqa: E501 + ) - self.password_prompt("registration_key", - "Account registration key") + self.password_prompt("registration_key", "(Optional) Registration Key") def query_proxies(self): options = self.config.get_command_line_options() @@ -333,11 +393,10 @@ class LandscapeSetupScript(object): show_help( """ - The Landscape client communicates with the server over HTTP and - HTTPS. If your network requires you to use a proxy to access HTTP - and/or HTTPS web sites, please provide the address of these - proxies now. If you don't use a proxy, leave these fields empty. - """) + If your network requires you to use a proxy, provide the address of + these proxies now. + """, + ) if "http_proxy" not in options: self.prompt("http_proxy", "HTTP proxy URL") @@ -349,47 +408,12 @@ class LandscapeSetupScript(object): if "include_manager_plugins" in options and "script_users" in options: invalid_users = get_invalid_users(options["script_users"]) if invalid_users: - raise ConfigurationError("Unknown system users: %s" % - ", ".join(invalid_users)) + raise ConfigurationError( + "Unknown system users: {}".format( + ", ".join(invalid_users), + ), + ) return - show_help( - """ - Landscape has a feature which enables administrators to run - arbitrary scripts on machines under their control. By default this - feature is disabled in the client, disallowing any arbitrary script - execution. If enabled, the set of users that scripts may run as is - also configurable. - """) - msg = "Enable script execution?" - included_plugins = [ - p.strip() for p in self.config.include_manager_plugins.split(",")] - if included_plugins == [""]: - included_plugins = [] - default = "ScriptExecution" in included_plugins - if prompt_yes_no(msg, default=default): - if "ScriptExecution" not in included_plugins: - included_plugins.append("ScriptExecution") - show_help( - """ - By default, scripts are restricted to the 'landscape' and - 'nobody' users. Please enter a comma-delimited list of users - that scripts will be restricted to. To allow scripts to be run - by any user, enter "ALL". - """) - while True: - self.prompt("script_users", "Script users") - invalid_users = get_invalid_users( - self.config.script_users) - if not invalid_users: - break - else: - show_help("Unknown system users: {}".format( - ",".join(invalid_users))) - self.config.script_users = None - else: - if "ScriptExecution" in included_plugins: - included_plugins.remove("ScriptExecution") - self.config.include_manager_plugins = ", ".join(included_plugins) def query_access_group(self): """Query access group from the user.""" @@ -399,7 +423,8 @@ class LandscapeSetupScript(object): show_help( "You may provide an access group for this computer " - "e.g. webservers.") + "e.g. webservers.", + ) self.prompt("access_group", "Access group", False) def _get_invalid_tags(self, tagnames): @@ -419,53 +444,97 @@ class LandscapeSetupScript(object): if "tags" in options: invalid_tags = self._get_invalid_tags(options["tags"]) if invalid_tags: - raise ConfigurationError("Invalid tags: %s" % - ", ".join(invalid_tags)) + raise ConfigurationError( + "Invalid tags: {}".format(", ".join(invalid_tags)), + ) return + def query_landscape_edition(self): show_help( - "You may provide tags for this computer e.g. server,precise.") - while True: - self.prompt("tags", "Tags", False) - if self._get_invalid_tags(self.config.tags): - show_help( - "Tag names may only contain alphanumeric characters.") - self.config.tags = None # Reset for the next prompt - else: - break - - def show_header(self): - show_help( + "Manage this machine with Landscape " + "(https://ubuntu.com/landscape):\n", + ) + options = self.config.get_command_line_options() + if "ping_url" in options and "url" in options: + return + if prompt_yes_no( + "Will you be using your own Self-Hosted Landscape installation?", + default=False, + ): + show_help( + "Provide the fully qualified domain name " + "of your Landscape Server e.g. " + "landscape.yourdomain.com", + ) + self.landscape_domain = self.prompt_get_input( + "Landscape Domain: ", + True, + ) + self.config.ping_url = f"http://{self.landscape_domain}/ping" + self.config.url = f"https://{self.landscape_domain}/message-system" + else: + self.landscape_domain = "" + self.config.ping_url = self.config._command_line_defaults[ + "ping_url" + ] + self.config.url = self.config._command_line_defaults["url"] + if self.config.account_name == "standalone": + self.config.account_name = "" + + def show_summary(self): + + tx = f"""A summary of the provided information: + Computer's Title: {self.config.computer_title} + Account Name: {self.config.account_name} + Landscape FQDN: {self.get_domain()} + Registration Key: {True if self.config.registration_key else False} """ - This script will interactively set up the Landscape client. It will - ask you a few questions about this computer and your Landscape - account, and will submit that information to the Landscape server. - After this computer is registered it will need to be approved by an - account administrator on the pending computers page. - - Please see https://landscape.canonical.com for more information. - """) + show_help(tx) + if self.config.http_proxy or self.config.https_proxy: + show_help( + f"""HTTPS Proxy: {self.config.https_proxy} + HTTP Proxy: {self.config.http_proxy}""", + ) + + params = ( + "account_name", + "url", + "ping_url", + "registration_key", + "https_proxy", + "http_proxy", + ) + cmd = ["sudo", "landscape-config"] + for param in params: + value = self.config.get(param) + if value: + if param in self.config._command_line_defaults: + if value == self.config._command_line_defaults[param]: + continue + cmd.append("--" + param.replace("_", "-")) + if param == "registration_key": + cmd.append("HIDDEN") + else: + cmd.append(shlex.quote(value)) + show_help( + "The landscape-config parameters to repeat this registration" + " on another machine are:", + ) + show_help(" ".join(cmd)) def run(self): """Kick off the interactive process which prompts the user for data. Data will be saved to C{self.config}. """ - self.show_header() + self.query_landscape_edition() self.query_computer_title() self.query_account_name() self.query_registration_key() self.query_proxies() self.query_script_plugin() - self.query_access_group() self.query_tags() - - -def setup_init_script_and_start_client(): - "Configure the init script to start the client on boot." - # XXX This function is misnamed; it doesn't start the client. - sysvconfig = SysVConfig() - sysvconfig.set_start_on_boot(True) + self.show_summary() def stop_client_and_disable_init_script(): @@ -473,9 +542,8 @@ def stop_client_and_disable_init_script(): Stop landscape-client and change configuration to prevent starting landscape-client on boot. """ - sysvconfig = SysVConfig() - sysvconfig.stop_landscape() - sysvconfig.set_start_on_boot(False) + ServiceConfig.stop_landscape() + ServiceConfig.set_start_on_boot(False) def setup_http_proxy(config): @@ -496,8 +564,9 @@ def check_account_name_and_password(config): """ if config.silent and not config.no_start: if not (config.get("account_name") and config.get("computer_title")): - raise ConfigurationError("An account name and computer title are " - "required.") + raise ConfigurationError( + "An account name and computer title are " "required.", + ) def check_script_users(config): @@ -508,8 +577,9 @@ def check_script_users(config): if config.get("script_users"): invalid_users = get_invalid_users(config.get("script_users")) if invalid_users: - raise ConfigurationError("Unknown system users: %s" % - ", ".join(invalid_users)) + raise ConfigurationError( + "Unknown system users: {}".format(", ".join(invalid_users)), + ) if not config.include_manager_plugins: config.include_manager_plugins = "ScriptExecution" @@ -523,9 +593,9 @@ def decode_base64_ssl_public_certificate(config): # certificate, but the actual certificate itself. if config.ssl_public_key and config.ssl_public_key.startswith("base64:"): decoded_cert = base64.decodebytes( - config.ssl_public_key[7:].encode("ascii")) - config.ssl_public_key = store_public_key_data( - config, decoded_cert) + config.ssl_public_key[7:].encode("ascii"), + ) + config.ssl_public_key = store_public_key_data(config, decoded_cert) def setup(config): @@ -538,19 +608,11 @@ def setup(config): """ bootstrap_tree(config) - sysvconfig = SysVConfig() if not config.no_start: if config.silent: - setup_init_script_and_start_client() - elif not sysvconfig.is_configured_to_run(): - answer = prompt_yes_no( - "\nThe Landscape client must be started " - "on boot to operate correctly.\n\n" - "Start Landscape client on boot?") - if answer: - setup_init_script_and_start_client() - else: - sys.exit("Aborting Landscape configuration") + ServiceConfig.set_start_on_boot(True) + elif not ServiceConfig.is_configured_to_run(): + ServiceConfig.set_start_on_boot(True) setup_http_proxy(config) check_account_name_and_password(config) @@ -564,11 +626,14 @@ def setup(config): # Restart the client to ensure that it's using the new configuration. if not config.no_start: try: - sysvconfig.restart_landscape() - except ProcessError: - print_text("Couldn't restart the Landscape client.", error=True) - print_text("This machine will be registered with the provided " - "details when the client runs.", error=True) + ServiceConfig.restart_landscape() + except ServiceConfigException as exc: + print_text(str(exc), error=True) + print_text( + "This machine will be registered with the provided " + "details when the client runs.", + error=True, + ) exit_code = 2 if config.ok_no_register: exit_code = 0 @@ -579,10 +644,17 @@ def bootstrap_tree(config): """Create the client directories tree.""" bootstrap_list = [ BootstrapDirectory("$data_path", "landscape", "root", 0o755), - BootstrapDirectory("$annotations_path", "landscape", "landscape", - 0o755)] + BootstrapDirectory( + "$annotations_path", + "landscape", + "landscape", + 0o755, + ), + ] BootstrapList(bootstrap_list).bootstrap( - data_path=config.data_path, annotations_path=config.annotations_path) + data_path=config.data_path, + annotations_path=config.annotations_path, + ) def store_public_key_data(config, certificate_data): @@ -598,8 +670,9 @@ def store_public_key_data(config, certificate_data): """ key_filename = os.path.join( config.data_path, - os.path.basename(config.get_config_filename() + ".ssl_public_key")) - print_text("Writing SSL CA certificate to %s..." % key_filename) + os.path.basename(config.get_config_filename() + ".ssl_public_key"), + ) + print_text(f"Writing SSL CA certificate to {key_filename}...") create_binary_file(key_filename, certificate_data) return key_filename @@ -647,13 +720,18 @@ def done(ignored_result, connector, reactor): def got_connection(add_result, connector, reactor, remote): """Handle becomming connected to a broker.""" - handlers = {"registration-done": partial(success, add_result), - "registration-failed": partial(failure, add_result), - "exchange-failed": partial(exchange_failure, add_result)} + handlers = { + "registration-done": partial(success, add_result), + "registration-failed": partial(failure, add_result), + "exchange-failed": partial(exchange_failure, add_result), + } deferreds = [ remote.call_on_event(handlers), remote.register().addErrback( - partial(handle_registration_errors, add_result), connector)] + partial(handle_registration_errors, add_result), + connector, + ), + ] results = gather_results(deferreds) results.addCallback(done, connector, reactor) return results @@ -667,9 +745,15 @@ def got_error(failure, reactor, add_result, print=print): reactor.stop() -def register(config, reactor=None, connector_factory=RemoteBrokerConnector, - got_connection=got_connection, max_retries=14, on_error=None, - results=None): +def register( + config, + reactor=None, + connector_factory=RemoteBrokerConnector, + got_connection=got_connection, + max_retries=14, + on_error=None, + results=None, +): """Instruct the Landscape Broker to register the client. The broker will be instructed to reload its configuration and then to @@ -701,9 +785,11 @@ def register(config, reactor=None, connector_factory=RemoteBrokerConnector, connector = connector_factory(reactor, config) connection = connector.connect(max_retries=max_retries, quiet=True) connection.addCallback( - partial(got_connection, add_result, connector, reactor)) + partial(got_connection, add_result, connector, reactor), + ) connection.addErrback( - partial(got_error, reactor=reactor, add_result=add_result)) + partial(got_error, reactor=reactor, add_result=add_result), + ) reactor.run() assert len(results) == 1, "We expect exactly one result." @@ -721,27 +807,30 @@ def register(config, reactor=None, connector_factory=RemoteBrokerConnector, def report_registration_outcome(what_happened, print=print): - """Report the registration interaction outcome to the user in human-readable - form. + """Report the registration interaction outcome to the user in + human-readable form. """ messages = { - "success": "System successfully registered.", + "success": "Registration request sent successfully.", "unknown-account": "Invalid account name or registration key.", "max-pending-computers": ( "Maximum number of computers pending approval reached. ", "Login to your Landscape server account page to manage " - "pending computer approvals."), + "pending computer approvals.", + ), "ssl-error": ( "\nThe server's SSL information is incorrect, or fails " "signature verification!\n" "If the server is using a self-signed certificate, " "please ensure you supply it with the --ssl-public-key " - "parameter."), + "parameter." + ), "non-ssl-error": ( "\nWe were unable to contact the server.\n" "Your internet connection may be down. " "The landscape client will continue to try and contact " - "the server periodically.") + "the server periodically." + ), } message = messages.get(what_happened) if message: @@ -762,28 +851,34 @@ def determine_exit_code(what_happened): def is_registered(config): """Return whether the client is already registered.""" persist_filename = os.path.join( - config.data_path, "{}.bpickle".format(BrokerService.service_name)) + config.data_path, + f"{BrokerService.service_name}.bpickle", + ) persist = Persist(filename=persist_filename) identity = Identity(config, persist) return bool(identity.secure_id) def registration_info_text(config, registration_status): - ''' + """ A simple output displaying whether the client is registered or not, the account name, and config and data paths - ''' + """ config_path = os.path.abspath(config._config_filename) - text = textwrap.dedent(""" + text = textwrap.dedent( + """ Registered: {} Config Path: {} - Data Path {}""" - .format(registration_status, config_path, - config.data_path)) + Data Path {}""".format( + registration_status, + config_path, + config.data_path, + ), + ) if registration_status: - text += '\nAccount Name: {}'.format(config.account_name) + text += f"\nAccount Name: {config.account_name}" return text @@ -829,8 +924,6 @@ def main(args, print=print): print_text(str(e)) sys.exit("Aborting Landscape configuration") - print("Please wait...") - # Attempt to register the client. reactor = LandscapeReactor() if config.silent: @@ -841,7 +934,8 @@ def main(args, print=print): default_answer = not is_registered(config) answer = prompt_yes_no( "\nRequest a new registration for this computer now?", - default=default_answer) + default=default_answer, + ) if answer: result = register(config, reactor) report_registration_outcome(result, print=print) diff --git a/landscape/client/deployment.py b/landscape/client/deployment.py index 7c5464c..db1f110 100644 --- a/landscape/client/deployment.py +++ b/landscape/client/deployment.py @@ -1,32 +1,36 @@ import os.path import sys - from optparse import SUPPRESS_HELP + from twisted.logger import globalLogBeginner from landscape import VERSION +from landscape.client.upgraders import UPGRADE_MANAGERS from landscape.lib import logging from landscape.lib.config import BaseConfiguration as _BaseConfiguration from landscape.lib.persist import Persist -from landscape.client.upgraders import UPGRADE_MANAGERS - def init_logging(configuration, program_name): """Given a basic configuration, set up logging.""" - logging.init_app_logging(configuration.log_dir, configuration.log_level, - progname=program_name, - quiet=configuration.quiet) + logging.init_app_logging( + configuration.log_dir, + configuration.log_level, + progname=program_name, + quiet=configuration.quiet, + ) # Initialize twisted logging, even if we don't explicitly use it, # because of leaky logs https://twistedmatrix.com/trac/ticket/8164 globalLogBeginner.beginLoggingTo( - [lambda _: None], redirectStandardIO=False, discardBuffer=True) + [lambda _: None], + redirectStandardIO=False, + discardBuffer=True, + ) -def _is_script(filename=sys.argv[0], - _scriptdir=os.path.abspath("scripts")): +def _is_script(filename=sys.argv[0], _scriptdir=os.path.abspath("scripts")): filename = os.path.abspath(filename) - return (os.path.dirname(filename) == _scriptdir) + return os.path.dirname(filename) == _scriptdir class BaseConfiguration(_BaseConfiguration): @@ -35,8 +39,10 @@ class BaseConfiguration(_BaseConfiguration): default_config_filename = "/etc/landscape/client.conf" if _is_script(): - default_config_filenames = ("landscape-client.conf", - default_config_filename) + default_config_filenames = ( + "landscape-client.conf", + default_config_filename, + ) else: default_config_filenames = (default_config_filename,) default_data_dir = "/var/lib/landscape/client/" @@ -44,7 +50,7 @@ class BaseConfiguration(_BaseConfiguration): config_section = "client" def __init__(self): - super(BaseConfiguration, self).__init__() + super().__init__() self._command_line_defaults["config"] = None @@ -56,10 +62,10 @@ class BaseConfiguration(_BaseConfiguration): - config - data_path """ - return super(BaseConfiguration, self).make_parser( - cfgfile=self.default_config_filename, - datadir=self.default_data_dir, - ) + return super().make_parser( + cfgfile=self.default_config_filename, + datadir=self.default_data_dir, + ) class Configuration(BaseConfiguration): @@ -84,43 +90,82 @@ class Configuration(BaseConfiguration): - C{ignore_sigint} (C{False}) - C{stagger_launch} (C{0.1}) """ - parser = super(Configuration, self).make_parser() + parser = super().make_parser() logging.add_cli_options(parser, logdir="/var/log/landscape") - parser.add_option("-u", "--url", default=self.DEFAULT_URL, - help="The server URL to connect to.") - parser.add_option("--ping-url", - help="The URL to perform lightweight exchange " - "initiation with.", - default="http://landscape.canonical.com/ping") - parser.add_option("-k", "--ssl-public-key", - help="The public SSL key to verify the server. " - "Only used if the given URL is https.") - parser.add_option("--ignore-sigint", action="store_true", - default=False, help="Ignore interrupt signals.") - parser.add_option("--ignore-sigusr1", action="store_true", - default=False, help="Ignore SIGUSR1 signal to " - "rotate logs.") - parser.add_option("--package-monitor-interval", default=30 * 60, - type="int", - help="The interval between package monitor runs " - "(default: 1800).") - parser.add_option("--apt-update-interval", default=6 * 60 * 60, - type="int", - help="The interval between apt update runs " - "(default: 21600).") - parser.add_option("--flush-interval", default=5 * 60, type="int", - metavar="INTERVAL", - help="The number of seconds between flushes to disk " - "for persistent data.") - parser.add_option("--stagger-launch", metavar="STAGGER_RATIO", - dest="stagger_launch", default=0.1, type=float, - help="Ratio, between 0 and 1, by which to scatter " - "various tasks of landscape.") + parser.add_option( + "-u", + "--url", + default=self.DEFAULT_URL, + help="The server URL to connect to.", + ) + parser.add_option( + "--ping-url", + help="The URL to perform lightweight exchange initiation with.", + default="http://landscape.canonical.com/ping", + ) + parser.add_option( + "-k", + "--ssl-public-key", + help="The public SSL key to verify the server. " + "Only used if the given URL is https.", + ) + parser.add_option( + "--ignore-sigint", + action="store_true", + default=False, + help="Ignore interrupt signals.", + ) + parser.add_option( + "--ignore-sigusr1", + action="store_true", + default=False, + help="Ignore SIGUSR1 signal to " "rotate logs.", + ) + parser.add_option( + "--package-monitor-interval", + default=30 * 60, + type="int", + help="The interval between package monitor runs " + "(default: 1800).", + ) + parser.add_option( + "--apt-update-interval", + default=6 * 60 * 60, + type="int", + help="The interval between apt update runs (default: 21600).", + ) + parser.add_option( + "--flush-interval", + default=5 * 60, + type="int", + metavar="INTERVAL", + help="The number of seconds between flushes to disk " + "for persistent data.", + ) + parser.add_option( + "--stagger-launch", + metavar="STAGGER_RATIO", + dest="stagger_launch", + default=0.1, + type=float, + help="Ratio, between 0 and 1, by which to stagger " + "various tasks of landscape.", + ) + parser.add_option( + "--snap-monitor-interval", + default=30 * 60, + type="int", + help="The interval between snap monitor runs (default 1800).", + ) # Hidden options, used for load-testing to run in-process clones parser.add_option("--clones", default=0, type=int, help=SUPPRESS_HELP) - parser.add_option("--start-clones-over", default=25 * 60, type=int, - help=SUPPRESS_HELP) + parser.add_option( + "--start-clones-over", + default=25 * 60, + type=int, + help=SUPPRESS_HELP, + ) return parser diff --git a/landscape/client/lockfile.py b/landscape/client/lockfile.py index aa0f916..55861ef 100644 --- a/landscape/client/lockfile.py +++ b/landscape/client/lockfile.py @@ -39,14 +39,14 @@ class PatchedFilesystemLock(lockfile.FilesystemLock): # out normally. pass - result = super(PatchedFilesystemLock, self).lock() + result = super().lock() self.clean = self.clean and clean return result def get_process_name(pid): """Return a process name from a pid.""" - stat_path = "/proc/{}/stat".format(pid) + stat_path = f"/proc/{pid}/stat" with open(stat_path) as stat_file: stat = stat_file.read() return stat.partition("(")[2].rpartition(")")[0] diff --git a/landscape/client/manager/aptsources.py b/landscape/client/manager/aptsources.py index 866c6ba..baa641f 100644 --- a/landscape/client/manager/aptsources.py +++ b/landscape/client/manager/aptsources.py @@ -1,17 +1,17 @@ import glob +import grp import os import pwd -import grp import shutil import tempfile import uuid from twisted.internet.defer import succeed -from landscape.lib.twisted_util import spawn_process - from landscape.client.manager.plugin import ManagerPlugin from landscape.client.package.reporter import find_reporter_command +from landscape.constants import FALSE_VALUES +from landscape.lib.twisted_util import spawn_process class ProcessError(Exception): @@ -26,9 +26,11 @@ class AptSources(ManagerPlugin): TRUSTED_GPG_D = "/etc/apt/trusted.gpg.d" def register(self, registry): - super(AptSources, self).register(registry) - registry.register_message("apt-sources-replace", - self._handle_repositories) + super().register(registry) + registry.register_message( + "apt-sources-replace", + self._handle_repositories, + ) def _run_process(self, command, args, uid=None, gid=None): """ @@ -42,7 +44,7 @@ class AptSources(ManagerPlugin): """ out, err, code = result if code: - raise ProcessError("%s\n%s" % (out, err)) + raise ProcessError(f"{out}\n{err}") def _handle_process_failure(self, failure): """ @@ -50,7 +52,7 @@ class AptSources(ManagerPlugin): """ if not failure.check(ProcessError): out, err, signal = failure.value.args - raise ProcessError("%s\n%s" % (out, err)) + raise ProcessError(f"{out}\n{err}") else: return failure @@ -85,9 +87,9 @@ class AptSources(ManagerPlugin): "-----END PGP PUBLIC KEY BLOCK-----"]} """ deferred = succeed(None) - prefix = 'landscape-server-' + prefix = "landscape-server-mirror" for key in message["gpg-keys"]: - filename = prefix + str(uuid.uuid4()) + '.asc' + filename = prefix + str(uuid.uuid4()) + ".asc" key_path = os.path.join(self.TRUSTED_GPG_D, filename) with open(key_path, "w") as key_file: key_file.write(key) @@ -96,8 +98,15 @@ class AptSources(ManagerPlugin): return self.call_with_operation_result(message, lambda: deferred) def _handle_sources(self, ignored, sources): - """Handle sources repositories.""" - saved_sources = "{}.save".format(self.SOURCES_LIST) + """ + Replaces `SOURCES_LIST` with a Landscape-managed version and moves the + original to a ".save" file. + + Configurably does the same with files in `SOURCES_LIST_D`. + """ + + saved_sources = f"{self.SOURCES_LIST}.save" + if sources: fd, path = tempfile.mkstemp() os.close(fd) @@ -106,29 +115,42 @@ class AptSources(ManagerPlugin): new_sources.write( "# Landscape manages repositories for this computer\n" "# Original content of sources.list can be found in " - "sources.list.save\n") + "sources.list.save\n", + ) original_stat = os.stat(self.SOURCES_LIST) if not os.path.isfile(saved_sources): shutil.move(self.SOURCES_LIST, saved_sources) shutil.move(path, self.SOURCES_LIST) os.chmod(self.SOURCES_LIST, original_stat.st_mode) - os.chown(self.SOURCES_LIST, original_stat.st_uid, - original_stat.st_gid) + os.chown( + self.SOURCES_LIST, + original_stat.st_uid, + original_stat.st_gid, + ) else: # Re-instate original sources if os.path.isfile(saved_sources): shutil.move(saved_sources, self.SOURCES_LIST) - for filename in glob.glob(os.path.join(self.SOURCES_LIST_D, "*.list")): - shutil.move(filename, "%s.save" % filename) + manage_sources_list_d = getattr( + self.registry.config, + "manage_sources_list_d", + True, + ) + if manage_sources_list_d not in FALSE_VALUES: + filenames = glob.glob(os.path.join(self.SOURCES_LIST_D, "*.list")) + for filename in filenames: + shutil.move(filename, f"{filename}.save") for source in sources: - filename = os.path.join(self.SOURCES_LIST_D, - "landscape-%s.list" % source["name"]) + filename = os.path.join( + self.SOURCES_LIST_D, + f"landscape-{source['name']}.list", + ) # Servers send unicode, but an upgrade from python2 can get bytes # from stored messages, so we need to handle both. - is_unicode = isinstance(source["content"], type(u"")) + is_unicode = isinstance(source["content"], type("")) with open(filename, ("w" if is_unicode else "wb")) as sources_file: sources_file.write(source["content"]) os.chmod(filename, 0o644) @@ -142,7 +164,7 @@ class AptSources(ManagerPlugin): args = ["--force-apt-update"] if self.registry.config.config is not None: - args.append("--config=%s" % self.registry.config.config) + args.append(f"--config={self.registry.config.config}") if os.getuid() == 0: uid = pwd.getpwnam("landscape").pw_uid diff --git a/landscape/client/manager/config.py b/landscape/client/manager/config.py index bfc0a5d..2f8df8b 100644 --- a/landscape/client/manager/config.py +++ b/landscape/client/manager/config.py @@ -4,9 +4,16 @@ from landscape.client.deployment import Configuration from landscape.client.manager.scriptexecution import ALL_USERS -ALL_PLUGINS = ["ProcessKiller", "PackageManager", "UserManager", - "ShutdownManager", "AptSources", "HardwareInfo", - "KeystoneToken"] +ALL_PLUGINS = [ + "ProcessKiller", + "PackageManager", + "UserManager", + "ShutdownManager", + "AptSources", + "HardwareInfo", + "KeystoneToken", + "SnapManager", +] class ManagerConfiguration(Configuration): @@ -17,26 +24,38 @@ class ManagerConfiguration(Configuration): Specialize L{Configuration.make_parser}, adding many manager-specific options. """ - parser = super(ManagerConfiguration, self).make_parser() + parser = super().make_parser() - parser.add_option("--manager-plugins", metavar="PLUGIN_LIST", - help="Comma-delimited list of manager plugins to " - "use. ALL means use all plugins.", - default="ALL") - parser.add_option("--include-manager-plugins", metavar="PLUGIN_LIST", - help="Comma-delimited list of manager plugins to " - "enable, in addition to the defaults.") - parser.add_option("--script-users", metavar="USERS", - help="Comma-delimited list of usernames that scripts" - " may be run as. Default is to allow all " - "users.") - parser.add_option("--script-output-limit", - metavar="SCRIPT_OUTPUT_LIMIT", - type="int", default=512, - help="Maximum allowed output size that scripts" - " can send. " - "Script output will be truncated at that limit." - " Default is 512 (kB)") + parser.add_option( + "--manager-plugins", + metavar="PLUGIN_LIST", + help="Comma-delimited list of manager plugins to " + "use. ALL means use all plugins.", + default="ALL", + ) + parser.add_option( + "--include-manager-plugins", + metavar="PLUGIN_LIST", + help="Comma-delimited list of manager plugins to " + "enable, in addition to the defaults.", + ) + parser.add_option( + "--script-users", + metavar="USERS", + help="Comma-delimited list of usernames that scripts" + " may be run as. Default is to allow all " + "users.", + ) + parser.add_option( + "--script-output-limit", + metavar="SCRIPT_OUTPUT_LIMIT", + type="int", + default=512, + help="Maximum allowed output size that scripts" + " can send. " + "Script output will be truncated at that limit." + " Default is 512 (kB)", + ) return parser diff --git a/landscape/client/manager/customgraph.py b/landscape/client/manager/customgraph.py index e0e6e1d..5e35310 100644 --- a/landscape/client/manager/customgraph.py +++ b/landscape/client/manager/customgraph.py @@ -1,19 +1,25 @@ +import logging import os import time -import logging -from twisted.internet.defer import fail, DeferredList, succeed +from twisted.internet.defer import DeferredList +from twisted.internet.defer import fail +from twisted.internet.defer import succeed from twisted.python.compat import iteritems -from landscape.lib.scriptcontent import generate_script_hash -from landscape.lib.user import get_user_info, UnknownUserError from landscape.client.accumulate import Accumulator from landscape.client.manager.plugin import ManagerPlugin +from landscape.client.manager.scriptexecution import ProcessFailedError from landscape.client.manager.scriptexecution import ( - ProcessFailedError, ScriptRunnerMixin, ProcessTimeLimitReachedError) + ProcessTimeLimitReachedError, +) +from landscape.client.manager.scriptexecution import ScriptRunnerMixin +from landscape.lib.scriptcontent import generate_script_hash +from landscape.lib.user import get_user_info +from landscape.lib.user import UnknownUserError -class StoreProxy(object): +class StoreProxy: """ Persist-like interface to store graph-points into SQLite store. """ @@ -33,19 +39,17 @@ class StoreProxy(object): class InvalidFormatError(Exception): - def __init__(self, value): self.value = value Exception.__init__(self, self._get_message()) def _get_message(self): - return u"Failed to convert to number: '%s'" % self.value + return f"Failed to convert to number: '{self.value}'" class NoOutputError(Exception): - def __init__(self): - Exception.__init__(self, u"Script did not output any value") + Exception.__init__(self, "Script did not output any value") class ProhibitedUserError(Exception): @@ -60,7 +64,7 @@ class ProhibitedUserError(Exception): Exception.__init__(self, self._get_message()) def _get_message(self): - return (u"Custom graph cannot be run as user %s" % self.username) + return f"Custom graph cannot be run as user {self.username}" class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): @@ -71,23 +75,28 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): @param process_factory: The L{IReactorProcess} provider to run the process with. """ + run_interval = 300 size_limit = 1000 time_limit = 10 message_type = "custom-graph" def __init__(self, process_factory=None, create_time=time.time): - super(CustomGraphPlugin, self).__init__(process_factory) + super().__init__(process_factory) self._create_time = create_time self._data = {} self.do_send = True def register(self, registry): - super(CustomGraphPlugin, self).register(registry) + super().register(registry) registry.register_message( - "custom-graph-add", self._handle_custom_graph_add) + "custom-graph-add", + self._handle_custom_graph_add, + ) registry.register_message( - "custom-graph-remove", self._handle_custom_graph_remove) + "custom-graph-remove", + self._handle_custom_graph_remove, + ) self._persist = StoreProxy(self.registry.store) self._accumulate = Accumulator(self._persist, self.run_interval) @@ -118,8 +127,7 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): data_path = self.registry.config.data_path scripts_directory = os.path.join(data_path, "custom-graph-scripts") - filename = os.path.join( - scripts_directory, "graph-%d" % (graph_id,)) + filename = os.path.join(scripts_directory, f"graph-{graph_id:d}") if os.path.exists(filename): os.unlink(filename) @@ -127,23 +135,31 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): try: uid, gid = get_user_info(user)[:2] except UnknownUserError: - logging.error(u"Attempt to add graph with unknown user %s" % - user) + logging.error(f"Attempt to add graph with unknown user {user}") else: script_file = open(filename, "wb") # file is closed in write_script_file self.write_script_file( - script_file, filename, shell, code, uid, gid) + script_file, + filename, + shell, + code, + uid, + gid, + ) if graph_id in self._data: del self._data[graph_id] self.registry.store.add_graph(graph_id, filename, user) def _format_exception(self, e): - return u"%s: %s" % (e.__class__.__name__, e.args[0]) + return "{}: {}".format(e.__class__.__name__, e.args[0]) def exchange(self, urgent=False): self.registry.broker.call_if_accepted( - self.message_type, self.send_message, urgent) + self.message_type, + self.send_message, + urgent, + ) def send_message(self, urgent): if not self.do_send: @@ -155,7 +171,10 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): if os.path.isfile(filename): script_hash = self._get_script_hash(filename) self._data[graph_id] = { - "values": [], "error": u"", "script-hash": script_hash} + "values": [], + "error": "", + "script-hash": script_hash, + } message = {"type": self.message_type, "data": self._data} @@ -163,11 +182,17 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): for graph_id, item in iteritems(self._data): script_hash = item["script-hash"] new_data[graph_id] = { - "values": [], "error": u"", "script-hash": script_hash} + "values": [], + "error": "", + "script-hash": script_hash, + } self._data = new_data - self.registry.broker.send_message(message, self._session_id, - urgent=urgent) + self.registry.broker.send_message( + message, + self._session_id, + urgent=urgent, + ) def _handle_data(self, output, graph_id, now): if graph_id not in self._data: @@ -190,15 +215,19 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): if failure.check(ProcessFailedError): failure_value = failure.value.data if failure.value.exit_code: - failure_value = ("%s (process exited with code %d)" % - (failure_value, failure.value.exit_code)) + failure_value = ( + f"{failure_value} (process exited with code " + f"{failure.value.exit_code:d})" + ) self._data[graph_id]["error"] = failure_value elif failure.check(ProcessTimeLimitReachedError): - self._data[graph_id]["error"] = ( - u"Process exceeded the %d seconds limit" % (self.time_limit,)) + self._data[graph_id][ + "error" + ] = f"Process exceeded the {self.time_limit:d} seconds limit" else: self._data[graph_id]["error"] = self._format_exception( - failure.value) + failure.value, + ) def _get_script_hash(self, filename): with open(filename) as file_object: @@ -218,7 +247,10 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): return succeed([]) return self.registry.broker.call_if_accepted( - self.message_type, self._continue_run, graphs) + self.message_type, + self._continue_run, + graphs, + ) def _continue_run(self, graphs): deferred_list = [] @@ -231,7 +263,10 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): script_hash = b"" if graph_id not in self._data: self._data[graph_id] = { - "values": [], "error": u"", "script-hash": script_hash} + "values": [], + "error": "", + "script-hash": script_hash, + } else: self._data[graph_id]["script-hash"] = script_hash try: @@ -249,7 +284,13 @@ class CustomGraphPlugin(ManagerPlugin, ScriptRunnerMixin): if not os.path.isfile(filename): continue result = self._run_script( - filename, uid, gid, path, {}, self.time_limit) + filename, + uid, + gid, + path, + {}, + self.time_limit, + ) result.addCallback(self._handle_data, graph_id, now) result.addErrback(self._handle_error, graph_id) deferred_list.append(result) diff --git a/landscape/client/manager/fakepackagemanager.py b/landscape/client/manager/fakepackagemanager.py index 15f085a..c27f2cc 100644 --- a/landscape/client/manager/fakepackagemanager.py +++ b/landscape/client/manager/fakepackagemanager.py @@ -1,7 +1,7 @@ import random -from landscape.client.manager.plugin import ManagerPlugin from landscape.client.manager.manager import SUCCEEDED +from landscape.client.manager.plugin import ManagerPlugin class FakePackageManager(ManagerPlugin): @@ -10,41 +10,57 @@ class FakePackageManager(ManagerPlugin): randint = random.randint def register(self, registry): - super(FakePackageManager, self).register(registry) + super().register(registry) self.config = registry.config - registry.register_message("change-packages", - self.handle_change_packages) - registry.register_message("change-package-locks", - self.handle_change_package_locks) - registry.register_message("release-upgrade", - self.handle_release_upgrade) + registry.register_message( + "change-packages", + self.handle_change_packages, + ) + registry.register_message( + "change-package-locks", + self.handle_change_package_locks, + ) + registry.register_message( + "release-upgrade", + self.handle_release_upgrade, + ) def _handle(self, response): delay = self.randint(30, 300) self.registry.reactor.call_later( - delay, self.manager.broker.send_message, response, - self._session_id, urgent=True) + delay, + self.manager.broker.send_message, + response, + self._session_id, + urgent=True, + ) def handle_change_packages(self, message): - response = {"type": "change-packages-result", - "operation-id": message.get("operation-id"), - "result-code": 1, - "result-text": "OK done."} + response = { + "type": "change-packages-result", + "operation-id": message.get("operation-id"), + "result-code": 1, + "result-text": "OK done.", + } return self._handle(response) def handle_change_package_locks(self, message): - response = {"type": "operation-result", - "operation-id": message.get("operation-id"), - "status": SUCCEEDED, - "result-text": "Package locks successfully changed.", - "result-code": 0} + response = { + "type": "operation-result", + "operation-id": message.get("operation-id"), + "status": SUCCEEDED, + "result-text": "Package locks successfully changed.", + "result-code": 0, + } return self._handle(response) def handle_release_upgrade(self, message): - response = {"type": "operation-result", - "operation-id": message.get("operation-id"), - "status": SUCCEEDED, - "result-text": "Successful release upgrade.", - "result-code": 0} + response = { + "type": "operation-result", + "operation-id": message.get("operation-id"), + "status": SUCCEEDED, + "result-text": "Successful release upgrade.", + "result-code": 0, + } return self._handle(response) diff --git a/landscape/client/manager/hardwareinfo.py b/landscape/client/manager/hardwareinfo.py index 94e290c..b475ee5 100644 --- a/landscape/client/manager/hardwareinfo.py +++ b/landscape/client/manager/hardwareinfo.py @@ -2,8 +2,8 @@ import os from twisted.internet.utils import getProcessOutput -from landscape.lib.encoding import encode_values from landscape.client.manager.plugin import ManagerPlugin +from landscape.lib.encoding import encode_values class HardwareInfo(ManagerPlugin): @@ -15,17 +15,23 @@ class HardwareInfo(ManagerPlugin): command = "/usr/bin/lshw" def register(self, registry): - super(HardwareInfo, self).register(registry) + super().register(registry) self.call_on_accepted(self.message_type, self.send_message) def run(self): return self.registry.broker.call_if_accepted( - self.message_type, self.send_message) + self.message_type, + self.send_message, + ) def send_message(self): environ = encode_values(os.environ) result = getProcessOutput( - self.command, args=["-xml", "-quiet"], env=environ, path=None) + self.command, + args=["-xml", "-quiet"], + env=environ, + path=None, + ) return result.addCallback(self._got_output) def _got_output(self, output): diff --git a/landscape/client/manager/keystonetoken.py b/landscape/client/manager/keystonetoken.py index 687f5c7..5059503 100644 --- a/landscape/client/manager/keystonetoken.py +++ b/landscape/client/manager/keystonetoken.py @@ -1,12 +1,12 @@ -import os import logging +import os -from landscape.lib.compat import _PY3 - -from landscape.lib.compat import ConfigParser, NoOptionError from landscape.client.monitor.plugin import DataWatcher -from landscape.lib.persist import Persist +from landscape.lib.compat import _PY3 +from landscape.lib.compat import ConfigParser +from landscape.lib.compat import NoOptionError from landscape.lib.fs import read_binary_file +from landscape.lib.persist import Persist KEYSTONE_CONFIG_FILE = "/etc/keystone/keystone.conf" @@ -17,6 +17,7 @@ class KeystoneToken(DataWatcher): A plugin which pulls the admin_token from the keystone configuration file and sends it to the landscape server. """ + message_type = "keystone-token" message_key = "data" run_interval = 60 * 15 @@ -26,12 +27,16 @@ class KeystoneToken(DataWatcher): self._keystone_config_file = keystone_config_file def register(self, client): - super(KeystoneToken, self).register(client) - self._persist_filename = os.path.join(self.registry.config.data_path, - "keystone.bpickle") + super().register(client) + self._persist_filename = os.path.join( + self.registry.config.data_path, + "keystone.bpickle", + ) self._persist = Persist(filename=self._persist_filename) - self.registry.reactor.call_every(self.registry.config.flush_interval, - self.flush) + self.registry.reactor.call_every( + self.registry.config.flush_interval, + self.flush, + ) def _reset(self): """ @@ -54,16 +59,20 @@ class KeystoneToken(DataWatcher): # We need to use the surrogateescape error handler as the # admin_token my contain arbitrary bytes. The ConfigParser in # Python 2 on the other hand does not support read_string. - config_str = read_binary_file( - self._keystone_config_file).decode("utf-8", "surrogateescape") + config_str = read_binary_file(self._keystone_config_file).decode( + "utf-8", + "surrogateescape", + ) config.read_string(config_str) else: config.read(self._keystone_config_file) try: admin_token = config.get("DEFAULT", "admin_token") except NoOptionError: - logging.error("KeystoneToken: No admin_token found in %s" - % (self._keystone_config_file)) + logging.error( + "KeystoneToken: No admin_token found in " + f"{self._keystone_config_file}", + ) return None # There is no support for surrogateescape in Python 2, but we actually # have bytes in this case anyway. diff --git a/landscape/client/manager/manager.py b/landscape/client/manager/manager.py index 18d1314..e73fbc7 100644 --- a/landscape/client/manager/manager.py +++ b/landscape/client/manager/manager.py @@ -1,5 +1,5 @@ -from landscape.client.manager.store import ManagerStore from landscape.client.broker.client import BrokerClient +from landscape.client.manager.store import ManagerStore # Protocol messages! Same constants are defined in the server. FAILED = 5 @@ -12,7 +12,7 @@ class Manager(BrokerClient): name = "manager" def __init__(self, reactor, config): - super(Manager, self).__init__(reactor, config) + super().__init__(reactor, config) self.reactor = reactor self.config = config self.store = ManagerStore(self.config.store_filename) diff --git a/landscape/client/manager/packagemanager.py b/landscape/client/manager/packagemanager.py index ce67169..36692a9 100644 --- a/landscape/client/manager/packagemanager.py +++ b/landscape/client/manager/packagemanager.py @@ -1,14 +1,14 @@ import logging import os -from twisted.internet.utils import getProcessOutput from twisted.internet.defer import succeed +from twisted.internet.utils import getProcessOutput -from landscape.lib.encoding import encode_values -from landscape.lib.apt.package.store import PackageStore +from landscape.client.manager.plugin import ManagerPlugin from landscape.client.package.changer import PackageChanger from landscape.client.package.releaseupgrader import ReleaseUpgrader -from landscape.client.manager.plugin import ManagerPlugin +from landscape.lib.apt.package.store import PackageStore +from landscape.lib.encoding import encode_values class PackageManager(ManagerPlugin): @@ -17,20 +17,28 @@ class PackageManager(ManagerPlugin): _package_store = None def register(self, registry): - super(PackageManager, self).register(registry) + super().register(registry) self.config = registry.config if not self._package_store: - filename = os.path.join(registry.config.data_path, - "package/database") + filename = os.path.join( + registry.config.data_path, + "package/database", + ) self._package_store = PackageStore(filename) - registry.register_message("change-packages", - self.handle_change_packages) - registry.register_message("change-package-locks", - self.handle_change_package_locks) - registry.register_message("release-upgrade", - self.handle_release_upgrade) + registry.register_message( + "change-packages", + self.handle_change_packages, + ) + registry.register_message( + "change-package-locks", + self.handle_change_package_locks, + ) + registry.register_message( + "release-upgrade", + self.handle_release_upgrade, + ) # When the package reporter notifies us that something has changed, # we want to run again to see if we can now fulfill tasks that were @@ -74,7 +82,12 @@ class PackageManager(ManagerPlugin): # path is set to None so that getProcessOutput does not # chdir to "." see bug #211373 result = getProcessOutput( - command, args=args, env=environ, errortoo=1, path=None) + command, + args=args, + env=environ, + errortoo=1, + path=None, + ) result.addCallback(self._got_output, cls) else: result = succeed(None) @@ -82,5 +95,6 @@ class PackageManager(ManagerPlugin): def _got_output(self, output, cls): if output: - logging.warning("Package %s output:\n%s" % - (cls.queue_name, output)) + logging.warning( + f"Package {cls.queue_name} output:\n{output}", + ) diff --git a/landscape/client/manager/plugin.py b/landscape/client/manager/plugin.py index 668e207..ea20b8c 100644 --- a/landscape/client/manager/plugin.py +++ b/landscape/client/manager/plugin.py @@ -1,8 +1,8 @@ from twisted.internet.defer import maybeDeferred +from landscape.client.broker.client import BrokerClientPlugin from landscape.lib.format import format_object from landscape.lib.log import log_failure -from landscape.client.broker.client import BrokerClientPlugin # Protocol messages! Same constants are defined in the server. FAILED = 5 @@ -10,7 +10,6 @@ SUCCEEDED = 6 class ManagerPlugin(BrokerClientPlugin): - @property def manager(self): """An alias for the C{client} attribute}.""" @@ -37,21 +36,30 @@ class ManagerPlugin(BrokerClientPlugin): return SUCCEEDED, text def failure(failure): - text = "%s: %s" % (failure.type.__name__, failure.value) - msg = ("Error occured running message handler %s with " - "args %r %r.", format_object(callable), args, kwargs) + text = f"{failure.type.__name__}: {failure.value}" + msg = ( + "Error occured running message handler %s with " "args %r %r.", + format_object(callable), + args, + kwargs, + ) log_failure(failure, msg=msg) return FAILED, text def send(args): status, text = args - result = {"type": "operation-result", - "status": status, - "operation-id": message["operation-id"]} + result = { + "type": "operation-result", + "status": status, + "operation-id": message["operation-id"], + } if text: result["result-text"] = text return self.manager.broker.send_message( - result, self._session_id, urgent=True) + result, + self._session_id, + urgent=True, + ) deferred.addCallback(success) deferred.addErrback(failure) diff --git a/landscape/client/manager/processkiller.py b/landscape/client/manager/processkiller.py index 260ddb8..e5cbac0 100644 --- a/landscape/client/manager/processkiller.py +++ b/landscape/client/manager/processkiller.py @@ -1,10 +1,10 @@ +import logging import os import signal -import logging from datetime import datetime -from landscape.lib.process import ProcessInformation from landscape.client.manager.plugin import ManagerPlugin +from landscape.lib.process import ProcessInformation class ProcessNotFoundError(Exception): @@ -31,24 +31,35 @@ class ProcessKiller(ManagerPlugin): self.process_info = process_info def register(self, registry): - super(ProcessKiller, self).register(registry) - registry.register_message("signal-process", - self._handle_signal_process) + super().register(registry) + registry.register_message( + "signal-process", + self._handle_signal_process, + ) def _handle_signal_process(self, message): - self.call_with_operation_result(message, self.signal_process, - message["pid"], message["name"], - message["start-time"], - message["signal"]) + self.call_with_operation_result( + message, + self.signal_process, + message["pid"], + message["name"], + message["start-time"], + message["signal"], + ) def signal_process(self, pid, name, start_time, signame): - logging.info("Sending %s signal to the process with PID %d.", - signame, pid) + logging.info( + "Sending %s signal to the process with PID %d.", + signame, + pid, + ) process_info = self.process_info.get_process_info(pid) if not process_info: start_time = datetime.utcfromtimestamp(start_time) - message = ("The process %s with PID %d that started at %s UTC was " - "not found") % (name, pid, start_time) + message = ( + f"The process {name} with PID {pid:d} that started " + f"at {start_time} UTC was not found" + ) raise ProcessNotFoundError(message) elif abs(process_info["start-time"] - start_time) > 2: # We don't check that the start time matches precisely because @@ -56,18 +67,21 @@ class ProcessKiller(ManagerPlugin): # cascade into having imprecise process start times. expected_time = datetime.utcfromtimestamp(start_time) actual_time = datetime.utcfromtimestamp(process_info["start-time"]) - message = ("The process %s with PID %d that started at " - "%s UTC was not found. A process with the same " - "PID that started at %s UTC was found and not " - "sent the %s signal") % (name, pid, expected_time, - actual_time, signame) + message = ( + f"The process {name} with PID {pid:d} that started at " + f"{expected_time} UTC was not found. A process with the same " + f"PID that started at {actual_time} UTC was found and not " + f"sent the {signame} signal" + ) raise ProcessMismatchError(message) - signum = getattr(signal, "SIG%s" % (signame,)) + signum = getattr(signal, f"SIG{signame}") try: os.kill(pid, signum) except Exception: # XXX Nothing is indicating what the problem was. - message = ("Attempting to send the %s signal to the process " - "%s with PID %d failed") % (signame, name, pid) + message = ( + f"Attempting to send the {signame} signal to the process " + f"{name} with PID {pid:d} failed" + ) raise SignalProcessError(message) diff --git a/landscape/client/manager/scriptexecution.py b/landscape/client/manager/scriptexecution.py index b20b58a..ac591d7 100644 --- a/landscape/client/manager/scriptexecution.py +++ b/landscape/client/manager/scriptexecution.py @@ -3,25 +3,30 @@ Functionality for running arbitrary shell scripts. @var ALL_USERS: A token indicating all users should be allowed. """ -import os -import sys import os.path -import tempfile import shutil +import sys +import tempfile -from twisted.internet.protocol import ProcessProtocol -from twisted.internet.defer import ( - Deferred, fail, inlineCallbacks, returnValue, succeed) +from twisted.internet.defer import Deferred +from twisted.internet.defer import fail +from twisted.internet.defer import inlineCallbacks +from twisted.internet.defer import returnValue +from twisted.internet.defer import succeed from twisted.internet.error import ProcessDone +from twisted.internet.protocol import ProcessProtocol from twisted.python.compat import unicode from landscape import VERSION +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import ManagerPlugin +from landscape.client.manager.plugin import SUCCEEDED from landscape.constants import UBUNTU_PATH -from landscape.lib.fetch import fetch_async, HTTPCodeError +from landscape.lib.fetch import fetch_async +from landscape.lib.fetch import HTTPCodeError from landscape.lib.persist import Persist from landscape.lib.scriptcontent import build_script from landscape.lib.user import get_user_info -from landscape.client.manager.plugin import ManagerPlugin, SUCCEEDED, FAILED ALL_USERS = object() @@ -65,10 +70,10 @@ class UnknownInterpreterError(Exception): Exception.__init__(self, self._get_message()) def _get_message(self): - return "Unknown interpreter: '%s'" % self.interpreter + return f"Unknown interpreter: '{self.interpreter}'" -class ScriptRunnerMixin(object): +class ScriptRunnerMixin: """ @param process_factory: The L{IReactorProcess} provider to run the process with. @@ -108,46 +113,61 @@ class ScriptRunnerMixin(object): gid = None env = { key: ( - value.encode(sys.getfilesystemencoding(), errors='replace') - if isinstance(value, unicode) else value + value.encode(sys.getfilesystemencoding(), errors="replace") + if isinstance(value, unicode) + else value ) for key, value in env.items() } pp = ProcessAccumulationProtocol( - self.registry.reactor, self.registry.config.script_output_limit, - self.truncation_indicator) + self.registry.reactor, + self.registry.config.script_output_limit, + self.truncation_indicator, + ) args = (filename,) self.process_factory.spawnProcess( - pp, filename, args=args, uid=uid, gid=gid, path=path, env=env) + pp, + filename, + args=args, + uid=uid, + gid=gid, + path=path, + env=env, + ) if time_limit is not None: pp.schedule_cancel(time_limit) return pp.result_deferred class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): - """A plugin which allows execution of arbitrary shell scripts. - - """ + """A plugin which allows execution of arbitrary shell scripts.""" def register(self, registry): - super(ScriptExecutionPlugin, self).register(registry) + super().register(registry) registry.register_message( - "execute-script", self._handle_execute_script) + "execute-script", + self._handle_execute_script, + ) def _respond(self, status, data, opid, result_code=None): if not isinstance(data, unicode): # Let's decode result-text, replacing non-printable # characters data = data.decode("utf-8", "replace") - message = {"type": "operation-result", - "status": status, - "result-text": data, - "operation-id": opid} + message = { + "type": "operation-result", + "status": status, + "result-text": data, + "operation-id": opid, + } if result_code: message["result-code"] = result_code return self.registry.broker.send_message( - message, self._session_id, True) + message, + self._session_id, + True, + ) def _handle_execute_script(self, message): opid = message["operation-id"] @@ -156,14 +176,19 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): if not self.is_user_allowed(user): return self._respond( FAILED, - u"Scripts cannot be run as user %s." % (user,), - opid) + f"Scripts cannot be run as user {user}.", + opid, + ) server_supplied_env = message.get("env", None) - d = self.run_script(message["interpreter"], message["code"], - time_limit=message["time-limit"], user=user, - attachments=message["attachments"], - server_supplied_env=server_supplied_env) + d = self.run_script( + message["interpreter"], + message["code"], + time_limit=message["time-limit"], + user=user, + attachments=message["attachments"], + server_supplied_env=server_supplied_env, + ) d.addCallback(self._respond_success, opid) d.addErrback(self._respond_failure, opid) return d @@ -172,7 +197,7 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): raise def _format_exception(self, e): - return u"%s: %s" % (e.__class__.__name__, e.args[0]) + return "{}: {}".format(e.__class__.__name__, e.args[0]) def _respond_success(self, data, opid): return self._respond(SUCCEEDED, data, opid) @@ -186,8 +211,11 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): elif failure.check(HTTPCodeError): code = FETCH_ATTACHMENTS_FAILED_RESULT return self._respond( - FAILED, str(failure.value), opid, - FETCH_ATTACHMENTS_FAILED_RESULT) + FAILED, + str(failure.value), + opid, + FETCH_ATTACHMENTS_FAILED_RESULT, + ) if code is not None: return self._respond(FAILED, failure.value.data, opid, code) @@ -198,9 +226,11 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): def _save_attachments(self, attachments, uid, gid, computer_id, env): root_path = self.registry.config.url.rsplit("/", 1)[0] + "/attachment/" env["LANDSCAPE_ATTACHMENTS"] = attachment_dir = tempfile.mkdtemp() - headers = {"User-Agent": "landscape-client/%s" % VERSION, - "Content-Type": "application/octet-stream", - "X-Computer-ID": computer_id} + headers = { + "User-Agent": f"landscape-client/{VERSION}", + "Content-Type": "application/octet-stream", + "X-Computer-ID": computer_id, + } for filename, attachment_id in attachments.items(): if isinstance(attachment_id, str): # Backward compatible behavior @@ -208,9 +238,10 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): yield succeed(None) else: data = yield fetch_async( - "%s%d" % (root_path, attachment_id), + f"{root_path}{attachment_id:d}", cainfo=self.registry.config.ssl_public_key, - headers=headers) + headers=headers, + ) full_filename = os.path.join(attachment_dir, filename) with open(full_filename, "wb") as attachment: os.chmod(full_filename, 0o600) @@ -222,8 +253,15 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): os.chown(attachment_dir, uid, gid) returnValue(attachment_dir) - def run_script(self, shell, code, user=None, time_limit=None, - attachments=None, server_supplied_env=None): + def run_script( + self, + shell, + code, + user=None, + time_limit=None, + attachments=None, + server_supplied_env=None, + ): """ Run a script based on a shell and the code. @@ -245,15 +283,13 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): or fail with a L{ProcessTimeLimitReachedError}. """ if not os.path.exists(shell.split()[0]): - return fail( - UnknownInterpreterError(shell)) + return fail(UnknownInterpreterError(shell)) uid, gid, path = get_user_info(user) fd, filename = tempfile.mkstemp() script_file = os.fdopen(fd, "wb") - self.write_script_file( - script_file, filename, shell, code, uid, gid) + self.write_script_file(script_file, filename, shell, code, uid, gid) env = { "PATH": UBUNTU_PATH, @@ -269,8 +305,11 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): if attachments: persist = Persist( - filename=os.path.join(self.registry.config.data_path, - "broker.bpickle")) + filename=os.path.join( + self.registry.config.data_path, + "broker.bpickle", + ), + ) persist = persist.root_at("registration") computer_id = persist.get("secure-id") try: @@ -283,8 +322,7 @@ class ScriptExecutionPlugin(ManagerPlugin, ScriptRunnerMixin): def prepare_script(attachment_dir): - return self._run_script( - filename, uid, gid, path, env, time_limit) + return self._run_script(filename, uid, gid, path, env, time_limit) d.addCallback(prepare_script) return d.addBoth(self._cleanup, filename, env, old_umask) @@ -323,9 +361,11 @@ class ProcessAccumulationProtocol(ProcessProtocol): def schedule_cancel(self, time_limit): self._scheduled_cancel = self.reactor.call_later( - time_limit, self._cancel) + time_limit, + self._cancel, + ) - def childDataReceived(self, fd, data): + def childDataReceived(self, fd, data): # noqa: N802 """Some data was received from the child. Add it to our buffer, as long as it doesn't go over L{size_limit} @@ -334,14 +374,14 @@ class ProcessAccumulationProtocol(ProcessProtocol): if self._size < self.size_limit: data_length = len(data) if (self._size + data_length) >= self._truncated_size_limit: - extent = (self._truncated_size_limit - self._size) + extent = self._truncated_size_limit - self._size self.data.append(data[:extent] + self._truncation_indicator) self._size = self.size_limit else: self.data.append(data) self._size += data_length - def processEnded(self, reason): + def processEnded(self, reason): # noqa: N802 """Fire back the deferred. The deferred will be fired with the string of data received from the @@ -366,7 +406,8 @@ class ProcessAccumulationProtocol(ProcessProtocol): self.result_deferred.callback(data) else: self.result_deferred.errback( - ProcessFailedError(data, exit_code)) + ProcessFailedError(data, exit_code), + ) def _cancel(self): """ @@ -391,11 +432,12 @@ class ScriptExecution(ManagerPlugin): def __init__(self): from landscape.client.manager.customgraph import CustomGraphPlugin + self._script_execution = ScriptExecutionPlugin() self._custom_graph = CustomGraphPlugin() def register(self, registry): - super(ScriptExecution, self).register(registry) + super().register(registry) self._script_execution.register(registry) self._custom_graph.register(registry) diff --git a/landscape/client/manager/service.py b/landscape/client/manager/service.py index ef543d6..7690442 100644 --- a/landscape/client/manager/service.py +++ b/landscape/client/manager/service.py @@ -1,10 +1,11 @@ from twisted.python.reflect import namedClass -from landscape.client.service import LandscapeService, run_landscape_service -from landscape.client.manager.config import ManagerConfiguration -from landscape.client.broker.amp import RemoteBrokerConnector from landscape.client.amp import ComponentPublisher +from landscape.client.broker.amp import RemoteBrokerConnector +from landscape.client.manager.config import ManagerConfiguration from landscape.client.manager.manager import Manager +from landscape.client.service import LandscapeService +from landscape.client.service import run_landscape_service class ManagerService(LandscapeService): @@ -16,19 +17,26 @@ class ManagerService(LandscapeService): service_name = Manager.name def __init__(self, config): - super(ManagerService, self).__init__(config) + super().__init__(config) self.plugins = self.get_plugins() self.manager = Manager(self.reactor, self.config) - self.publisher = ComponentPublisher(self.manager, self.reactor, - self.config) + self.publisher = ComponentPublisher( + self.manager, + self.reactor, + self.config, + ) def get_plugins(self): """Return instances of all the plugins enabled in the configuration.""" - return [namedClass("landscape.client.manager.%s.%s" - % (plugin_name.lower(), plugin_name))() - for plugin_name in self.config.plugin_factories] + return [ + namedClass( + "landscape.client.manager." + f"{plugin_name.lower()}.{plugin_name}", + )() + for plugin_name in self.config.plugin_factories + ] - def startService(self): + def startService(self): # noqa: N802 """Start the manager service. This method does 3 things, in this order: @@ -37,7 +45,7 @@ class ManagerService(LandscapeService): - Connect to the broker. - Add all configured plugins, that will in turn register themselves. """ - super(ManagerService, self).startService() + super().startService() self.publisher.start() def start_plugins(broker): @@ -51,11 +59,11 @@ class ManagerService(LandscapeService): connected = self.connector.connect() return connected.addCallback(start_plugins) - def stopService(self): + def stopService(self): # noqa: N802 """Stop the manager and close the connection with the broker.""" self.connector.disconnect() deferred = self.publisher.stop() - super(ManagerService, self).stopService() + super().stopService() return deferred diff --git a/landscape/client/manager/shutdownmanager.py b/landscape/client/manager/shutdownmanager.py index 8c6f722..eba2be7 100644 --- a/landscape/client/manager/shutdownmanager.py +++ b/landscape/client/manager/shutdownmanager.py @@ -1,10 +1,12 @@ import logging from twisted.internet.defer import Deferred -from twisted.internet.protocol import ProcessProtocol from twisted.internet.error import ProcessDone +from twisted.internet.protocol import ProcessProtocol -from landscape.client.manager.plugin import ManagerPlugin, SUCCEEDED, FAILED +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import ManagerPlugin +from landscape.client.manager.plugin import SUCCEEDED class ShutdownFailedError(Exception): @@ -18,7 +20,6 @@ class ShutdownFailedError(Exception): class ShutdownManager(ManagerPlugin): - def __init__(self, process_factory=None): if process_factory is None: from twisted.internet import reactor as process_factory @@ -30,7 +31,7 @@ class ShutdownManager(ManagerPlugin): The shutdown manager handles C{shutdown} activity messages broadcast from the server. """ - super(ShutdownManager, self).register(registry) + super().register(registry) registry.register_message("shutdown", self.perform_shutdown) def perform_shutdown(self, message): @@ -54,22 +55,24 @@ class ShutdownManager(ManagerPlugin): deferred = self._respond(SUCCEEDED, data, operation_id) # After sending the result to the server, stop accepting messages and # wait for the reboot/shutdown. - deferred.addCallback( - lambda _: self.registry.broker.stop_exchanger()) + deferred.addCallback(lambda _: self.registry.broker.stop_exchanger()) return deferred def _respond_failure(self, failure, operation_id, reboot): logging.info("Shutdown request failed.") - failure_report = '\n'.join([ - failure.value.data, - "", - "Attempting to force {operation}. Please note that if this " - "succeeds, Landscape will have no way of knowing and will still " - "mark this activity as having failed. It is recommended you check " - "the state of the machine manually to determine whether " - "{operation} succeeded.".format( - operation="reboot" if reboot else "shutdown") - ]) + failure_report = "\n".join( + [ + failure.value.data, + "", + "Attempting to force {operation}. Please note that if this " + "succeeds, Landscape will have no way of knowing and will " + "still mark this activity as having failed. It is recommended " + "you check the state of the machine manually to determine " + "whether {operation} succeeded.".format( + operation="reboot" if reboot else "shutdown", + ), + ], + ) deferred = self._respond(FAILED, failure_report, operation_id) # Add another callback spawning the poweroff or reboot command (which # seem more reliable in aberrant situations like a post-trusty release @@ -81,30 +84,45 @@ class ShutdownManager(ManagerPlugin): command, args = self._get_command_and_args(protocol, reboot, True) deferred.addCallback( lambda _: self._process_factory.spawnProcess( - protocol, command, args=args)) + protocol, + command, + args=args, + ), + ) return deferred def _respond(self, status, data, operation_id): - message = {"type": "operation-result", - "status": status, - "result-text": data, - "operation-id": operation_id} + message = { + "type": "operation-result", + "status": status, + "result-text": data, + "operation-id": operation_id, + } return self.registry.broker.send_message( - message, self._session_id, True) + message, + self._session_id, + True, + ) def _get_command_and_args(self, protocol, reboot, force=False): """ Returns a C{command, args} 2-tuple suitable for use with L{IReactorProcess.spawnProcess}. """ - minutes = None if force else "+%d" % (protocol.delay // 60,) + minutes = None if force else f"+{protocol.delay//60:d}" args = { (False, False): [ - "/sbin/shutdown", "-h", minutes, - "Landscape is shutting down the system"], + "/sbin/shutdown", + "-h", + minutes, + "Landscape is shutting down the system", + ], (False, True): [ - "/sbin/shutdown", "-r", minutes, - "Landscape is rebooting the system"], + "/sbin/shutdown", + "-r", + minutes, + "Landscape is rebooting the system", + ], (True, False): ["/sbin/poweroff"], (True, True): ["/sbin/reboot"], }[force, reboot] @@ -148,7 +166,7 @@ class ShutdownProcessProtocol(ProcessProtocol): """ reactor.call_later(timeout, self._succeed) - def childDataReceived(self, fd, data): + def childDataReceived(self, fd, data): # noqa: N802 """Some data was received from the child. Add it to our buffer to pass to C{result} when it's fired. @@ -156,7 +174,7 @@ class ShutdownProcessProtocol(ProcessProtocol): if self._waiting: self._data.append(data) - def processEnded(self, reason): + def processEnded(self, reason): # noqa: N802 """Fire back the C{result} L{Deferred}. C{result}'s callback will be fired with the string of data received diff --git a/landscape/client/manager/snapmanager.py b/landscape/client/manager/snapmanager.py new file mode 100644 index 0000000..d34bafa --- /dev/null +++ b/landscape/client/manager/snapmanager.py @@ -0,0 +1,271 @@ +import logging +from collections import deque + +from twisted.internet import task + +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import ManagerPlugin +from landscape.client.manager.plugin import SUCCEEDED +from landscape.client.snap.http import INCOMPLETE_STATUSES +from landscape.client.snap.http import SnapdHttpException +from landscape.client.snap.http import SnapHttp +from landscape.client.snap.http import SUCCESS_STATUSES + + +class SnapManager(ManagerPlugin): + """ + Plugin that updates the state of snaps on this machine, installing, + removing, refreshing, enabling, and disabling them in response to messages. + + Changes trigger SnapMonitor to send an updated state message immediately. + """ + + def __init__(self): + super().__init__() + + self._snap_http = SnapHttp() + self.SNAP_METHODS = { + "install-snaps": self._snap_http.install_snap, + "install-snaps-batch": self._snap_http.install_snaps, + "remove-snaps": self._snap_http.remove_snap, + "remove-snaps-batch": self._snap_http.remove_snaps, + "refresh-snaps": self._snap_http.refresh_snap, + "refresh-snaps-batch": self._snap_http.refresh_snaps, + "hold-snaps": self._snap_http.hold_snap, + "hold-snaps-batch": self._snap_http.hold_snaps, + "unhold-snaps": self._snap_http.unhold_snap, + "unhold-snaps-batch": self._snap_http.unhold_snaps, + } + + def register(self, registry): + super().register(registry) + self.config = registry.config + + registry.register_message("install-snaps", self._handle_snap_task) + registry.register_message("remove-snaps", self._handle_snap_task) + registry.register_message("refresh-snaps", self._handle_snap_task) + registry.register_message("hold-snaps", self._handle_snap_task) + registry.register_message("unhold-snaps", self._handle_snap_task) + + def _handle_snap_task(self, message): + """ + If there are no per-snap arguments for the targeted + snaps, often the task can be done with a single snapd call, if + we have a handler for the action type, which we call via a kind + of dynamic dispatch. + """ + snaps = message["snaps"] + + if snaps and any(len(s) > 1 for s in snaps): + # "name" key only means no per-snap args. + return self._handle_multiple_snap_tasks(message) + + if f"{message['type']}-batch" not in self.SNAP_METHODS: + return self._handle_multiple_snap_tasks(message) + + return self._handle_batch_snap_task(message) + + def _handle_batch_snap_task(self, message): + logging.debug( + f"Handling message {message} as a single batch snap task", + ) + message_type = message["type"] + snaps = [s["name"] for s in message["snaps"]] + snap_args = message.get("args", {}) + opid = message["operation-id"] + errors = {} + queue = deque() + + logging.info(f"Performing {message_type} action for snaps {snaps}") + + try: + response = self._start_snap_task( + message_type + "-batch", + snaps, + **snap_args, + ) + queue.append((response["change"], "BATCH")) + except SnapdHttpException as e: + result = e.json["result"] + logging.error( + f"Error in {message_type}: {message}", + ) + errors["BATCH"] = result + + deferred = self._check_statuses(queue) + deferred.addCallback(self._respond, opid, errors) + + return deferred + + def _handle_multiple_snap_tasks(self, message): + """ + Performs a generic task, `snap_method`, on a group of snaps + where each task must be performed independently per-snap. + + This is required when we want to provide metadata for refreshes + or installs and also specify the channel, revision, or other + arguments per-snap. + """ + logging.debug(f"Handling message {message} as multiple snap tasks") + message_type = message["type"] + snaps = message["snaps"] + opid = message["operation-id"] + errors = {} + queue = deque() + + logging.info(f"Performing {message_type} action for snaps {snaps}") + + # Naively doing this synchronously because each is an HTTP call to the + # snap REST API that returns basically immediately. We poll for their + # completion statuses once they've all been kicked off. + for snap in snaps: + name = snap["name"] + snap_args = snap.get("args", {}) + + try: + response = self._start_snap_task( + message_type, + name, + **snap_args, + ) + queue.append((response["change"], name)) + except SnapdHttpException as e: + result = e.json["result"] + logging.error( + f"Error in {message_type} for '{name}': {message}", + ) + errors[name] = result + + deferred = self._check_statuses(queue) + deferred.addCallback(self._respond, opid, errors) + + return deferred + + def _check_statuses(self, change_queue): + """ + Repeatedly polls for the status of each change in `change_queue` + until all are no longer in-progress. + """ + completed_changes = [] + interval = getattr(self.registry.config, "snapd_poll_interval", 15) + + def get_status(): + """ + Looping function that polls snapd for the status of + changes, moving them from the queue when they are done. + """ + if not change_queue: + loop.stop() + return + + logging.info("Polling snapd for status of pending snap changes") + + try: + result = self._snap_http.check_changes().get("result", []) + result_dict = {c["id"]: c for c in result} + except SnapdHttpException as e: + logging.error(f"Error checking status of snap changes: {e}") + completed_changes.extend( + [(name, str(e)) for _, name in change_queue], + ) + loop.stop() + return + + for _ in range(len(change_queue)): + cid, name = change_queue.popleft() + + # It's possible (though unlikely) that a change is not in the + # list - snapd could have dropped it for some reason. We need + # to know if that happens, hence this check. + if cid not in result_dict: + completed_changes.append((name, "Unknown")) + continue + + status = result_dict[cid]["status"] + if status in INCOMPLETE_STATUSES: + logging.info( + f"Incomplete status for {name}, waiting...", + ) + change_queue.append((cid, name)) + else: + logging.info(f"Complete status for {name}") + completed_changes.append((name, status)) + + loop = task.LoopingCall(get_status) + loopDeferred = loop.start(interval) + + return loopDeferred.addCallback(lambda _: completed_changes) + + def _start_snap_task(self, action, *args, **kwargs): + """ + Kicks off the appropriate SNAP_METHOD for `action`. + + raises a `SnapdHttpException` in the event of issues. + """ + snap_method = self.SNAP_METHODS[action] + + response = snap_method(*args, **kwargs) + + if "change" not in response: + raise SnapdHttpException(response) + + return response + + def _respond(self, snap_results, opid, errors): + """ + Queues a response to Landscape Server based on the contents of + `results`. + + `completed` and `errored` are lists of snapd change ids. + Text error messages are stored in `errors`. + """ + logging.debug(f"Preparing snap change done response: {snap_results}") + + results = { + "completed": [], + "errored": [], + "errors": errors, + } + + for name, status in snap_results: + if status not in SUCCESS_STATUSES: + results["errored"].append(name) + results["errors"][name] = status + else: + results["completed"].append(name) + + message = { + "type": "operation-result", + "status": FAILED if results["errored"] or errors else SUCCEEDED, + "result-text": str(results), + "operation-id": opid, + } + + logging.debug("Sending snap-action-done response") + + # Kick off an immediate SnapMonitor message as well. + self._send_installed_snap_update() + return self.registry.broker.send_message( + message, + self._session_id, + True, + ) + + def _send_installed_snap_update(self): + try: + installed_snaps = self._snap_http.get_snaps() + except SnapdHttpException as e: + logging.error( + f"Unable to list installed snaps after snap change: {e}", + ) + return + + if installed_snaps: + return self.registry.broker.send_message( + { + "type": "snaps", + "snaps": installed_snaps, + }, + self._session_id, + True, + ) diff --git a/landscape/client/manager/store.py b/landscape/client/manager/store.py index 20f778c..1971b5e 100644 --- a/landscape/client/manager/store.py +++ b/landscape/client/manager/store.py @@ -6,8 +6,7 @@ except ImportError: from landscape.lib.apt.package.store import with_cursor -class ManagerStore(object): - +class ManagerStore: def __init__(self, filename): self._db = sqlite3.connect(filename) ensure_schema(self._db) @@ -16,7 +15,8 @@ class ManagerStore(object): def get_graph(self, cursor, graph_id): cursor.execute( "SELECT graph_id, filename, user FROM graph WHERE graph_id=?", - (graph_id,)) + (graph_id,), + ) return cursor.fetchone() @with_cursor @@ -28,16 +28,19 @@ class ManagerStore(object): def add_graph(self, cursor, graph_id, filename, user): cursor.execute( "SELECT graph_id FROM graph WHERE graph_id=?", - (graph_id,)) + (graph_id,), + ) if cursor.fetchone(): cursor.execute( "UPDATE graph SET filename=?, user=? WHERE graph_id=?", - (filename, user, graph_id)) + (filename, user, graph_id), + ) else: cursor.execute( "INSERT INTO graph (graph_id, filename, user) " "VALUES (?, ?, ?)", - (graph_id, filename, user)) + (graph_id, filename, user), + ) @with_cursor def remove_graph(self, cursor, graph_id): @@ -47,35 +50,46 @@ class ManagerStore(object): def set_graph_accumulate(self, cursor, graph_id, timestamp, value): cursor.execute( "SELECT graph_id, graph_timestamp, graph_value FROM " - "graph_accumulate WHERE graph_id=?", (graph_id,)) + "graph_accumulate WHERE graph_id=?", + (graph_id,), + ) graph_accumulate = cursor.fetchone() if graph_accumulate: cursor.execute( "UPDATE graph_accumulate SET graph_timestamp = ?, " "graph_value = ? WHERE graph_id=?", - (timestamp, value, graph_id)) + (timestamp, value, graph_id), + ) else: cursor.execute( "INSERT INTO graph_accumulate (graph_id, graph_timestamp, " - "graph_value) VALUES (?, ?, ?)", (graph_id, timestamp, value)) + "graph_value) VALUES (?, ?, ?)", + (graph_id, timestamp, value), + ) @with_cursor def get_graph_accumulate(self, cursor, graph_id): cursor.execute( "SELECT graph_id, graph_timestamp, graph_value FROM " - "graph_accumulate WHERE graph_id=?", (graph_id,)) + "graph_accumulate WHERE graph_id=?", + (graph_id,), + ) return cursor.fetchone() def ensure_schema(db): cursor = db.cursor() try: - cursor.execute("CREATE TABLE graph" - " (graph_id INTEGER PRIMARY KEY," - " filename TEXT NOT NULL, user TEXT)") - cursor.execute("CREATE TABLE graph_accumulate" - " (graph_id INTEGER PRIMARY KEY," - " graph_timestamp INTEGER, graph_value FLOAT)") + cursor.execute( + "CREATE TABLE graph" + " (graph_id INTEGER PRIMARY KEY," + " filename TEXT NOT NULL, user TEXT)", + ) + cursor.execute( + "CREATE TABLE graph_accumulate" + " (graph_id INTEGER PRIMARY KEY," + " graph_timestamp INTEGER, graph_value FLOAT)", + ) except sqlite3.OperationalError: cursor.close() db.rollback() diff --git a/landscape/client/manager/tests/test_aptsources.py b/landscape/client/manager/tests/test_aptsources.py index a4d2cd8..4903594 100644 --- a/landscape/client/manager/tests/test_aptsources.py +++ b/landscape/client/manager/tests/test_aptsources.py @@ -1,18 +1,18 @@ import os +from unittest import mock -import mock - -from twisted.internet.defer import Deferred, succeed +from twisted.internet.defer import Deferred +from twisted.internet.defer import succeed from landscape.client.manager.aptsources import AptSources -from landscape.client.manager.plugin import SUCCEEDED, FAILED - -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import SUCCEEDED from landscape.client.package.reporter import find_reporter_command +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper -class FakeStatResult(object): - +class FakeStatResult: def __init__(self, st_mode, st_uid, st_gid): self.st_mode = st_mode self.st_uid = st_uid @@ -23,11 +23,13 @@ class AptSourcesTests(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(AptSourcesTests, self).setUp() + super().setUp() self.sourceslist = AptSources() self.sources_path = self.makeDir() - self.sourceslist.SOURCES_LIST = os.path.join(self.sources_path, - "sources.list") + self.sourceslist.SOURCES_LIST = os.path.join( + self.sources_path, + "sources.list", + ) sources_d = os.path.join(self.sources_path, "sources.list.d") os.mkdir(sources_d) self.sourceslist.SOURCES_LIST_D = sources_d @@ -51,16 +53,21 @@ class AptSourcesTests(LandscapeTest): sources.write("oki\n\ndoki\n#comment\n # other comment\n") self.manager.dispatch_message( - {"type": "apt-sources-replace", - "sources": [{"name": "bla", "content": b""}], - "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [{"name": "bla", "content": b""}], + "gpg-keys": [], + "operation-id": 1, + }, + ) with open(self.sourceslist.SOURCES_LIST) as sources: self.assertEqual( "# Landscape manages repositories for this computer\n" "# Original content of sources.list can be found in " - "sources.list.save\n", sources.read()) + "sources.list.save\n", + sources.read(), + ) def test_save_sources_list(self): """ @@ -71,16 +78,21 @@ class AptSourcesTests(LandscapeTest): sources.write("oki\n\ndoki\n#comment\n # other comment\n") self.manager.dispatch_message( - {"type": "apt-sources-replace", - "sources": [{"name": "bla", "content": b""}], - "gpg-keys": [], - "operation-id": 1}) - - saved_sources_path = "{}.save".format(self.sourceslist.SOURCES_LIST) + { + "type": "apt-sources-replace", + "sources": [{"name": "bla", "content": b""}], + "gpg-keys": [], + "operation-id": 1, + }, + ) + + saved_sources_path = f"{self.sourceslist.SOURCES_LIST}.save" self.assertTrue(os.path.exists(saved_sources_path)) with open(saved_sources_path) as saved_sources: - self.assertEqual("oki\n\ndoki\n#comment\n # other comment\n", - saved_sources.read()) + self.assertEqual( + "oki\n\ndoki\n#comment\n # other comment\n", + saved_sources.read(), + ) def test_existing_saved_sources_list(self): """ @@ -90,15 +102,18 @@ class AptSourcesTests(LandscapeTest): with open(self.sourceslist.SOURCES_LIST, "w") as sources: sources.write("oki\n\ndoki\n#comment\n # other comment\n") - saved_sources_path = "{}.save".format(self.sourceslist.SOURCES_LIST) + saved_sources_path = f"{self.sourceslist.SOURCES_LIST}.save" with open(saved_sources_path, "w") as saved_sources: saved_sources.write("original content\n") self.manager.dispatch_message( - {"type": "apt-sources-replace", - "sources": [{"name": "bla", "content": b""}], - "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [{"name": "bla", "content": b""}], + "gpg-keys": [], + "operation-id": 1, + }, + ) self.assertTrue(os.path.exists(saved_sources_path)) with open(saved_sources_path) as saved_sources: @@ -110,13 +125,18 @@ class AptSourcesTests(LandscapeTest): unicode content correctly. """ self.manager.dispatch_message( - {"type": "apt-sources-replace", - "sources": [{"name": "bla", "content": u"fancy content"}], - "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [{"name": "bla", "content": "fancy content"}], + "gpg-keys": [], + "operation-id": 1, + }, + ) saved_sources_path = os.path.join( - self.sourceslist.SOURCES_LIST_D, "landscape-bla.list") + self.sourceslist.SOURCES_LIST_D, + "landscape-bla.list", + ) self.assertTrue(os.path.exists(saved_sources_path)) with open(saved_sources_path, "rb") as saved_sources: self.assertEqual(b"fancy content", saved_sources.read()) @@ -126,7 +146,7 @@ class AptSourcesTests(LandscapeTest): When getting a repository message without sources, AptSources restores the previous contents of the sources.list file. """ - saved_sources_path = "{}.save".format(self.sourceslist.SOURCES_LIST) + saved_sources_path = f"{self.sourceslist.SOURCES_LIST}.save" with open(saved_sources_path, "w") as old_sources: old_sources.write("original content\n") @@ -134,10 +154,13 @@ class AptSourcesTests(LandscapeTest): sources.write("oki\n\ndoki\n#comment\n # other comment\n") self.manager.dispatch_message( - {"type": "apt-sources-replace", - "sources": [], - "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [], + "gpg-keys": [], + "operation-id": 1, + }, + ) with open(self.sourceslist.SOURCES_LIST) as sources: self.assertEqual("original content\n", sources.read()) @@ -154,8 +177,11 @@ class AptSourcesTests(LandscapeTest): os.chmod(self.sourceslist.SOURCES_LIST, 0o400) sources_stat_orig = os.stat(self.sourceslist.SOURCES_LIST) - fake_stats = FakeStatResult(st_mode=sources_stat_orig.st_mode, - st_uid=30, st_gid=30) + fake_stats = FakeStatResult( + st_mode=sources_stat_orig.st_mode, + st_uid=30, + st_gid=30, + ) orig_stat = os.stat @@ -168,99 +194,202 @@ class AptSourcesTests(LandscapeTest): _mock_chown = mock.patch("os.chown") with _mock_stat as mock_stat, _mock_chown as mock_chown: self.manager.dispatch_message( - {"type": "apt-sources-replace", - "sources": [{"name": "bla", "content": b""}], - "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [{"name": "bla", "content": b""}], + "gpg-keys": [], + "operation-id": 1, + }, + ) service = self.broker_service - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "status": SUCCEEDED, "operation-id": 1}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "operation-id": 1, + }, + ], + ) mock_stat.assert_any_call(self.sourceslist.SOURCES_LIST) mock_chown.assert_any_call( - self.sourceslist.SOURCES_LIST, fake_stats.st_uid, - fake_stats.st_gid) + self.sourceslist.SOURCES_LIST, + fake_stats.st_uid, + fake_stats.st_gid, + ) sources_stat_after = os.stat(self.sourceslist.SOURCES_LIST) - self.assertEqual( - sources_stat_orig.st_mode, sources_stat_after.st_mode) + self.assertEqual(sources_stat_orig.st_mode, sources_stat_after.st_mode) def test_random_failures(self): """ If a failure happens during the manipulation of sources, the activity is reported as FAILED with the error message. """ + def buggy_source_handler(*args): raise RuntimeError("foo") self.sourceslist._handle_sources = buggy_source_handler self.manager.dispatch_message( - {"type": "apt-sources-replace", - "sources": [{"name": "bla", "content": b""}], - "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [{"name": "bla", "content": b""}], + "gpg-keys": [], + "operation-id": 1, + }, + ) msg = "RuntimeError: foo" service = self.broker_service - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "result-text": msg, "status": FAILED, - "operation-id": 1}]) - - def test_rename_sources_list_d(self): + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "result-text": msg, + "status": FAILED, + "operation-id": 1, + }, + ], + ) + + def test_renames_sources_list_d(self): """ The sources files in sources.list.d are renamed to .save when a message - is received. + is received if config says to manage them, which is the default. """ - with open(os.path.join(self.sourceslist.SOURCES_LIST_D, "file1.list"), - "w") as sources1: + with open( + os.path.join(self.sourceslist.SOURCES_LIST_D, "file1.list"), + "w", + ) as sources1: sources1.write("ok\n") - with open(os.path.join(self.sourceslist.SOURCES_LIST_D, - "file2.list.save"), "w") as sources2: + with open( + os.path.join(self.sourceslist.SOURCES_LIST_D, "file2.list.save"), + "w", + ) as sources2: sources2.write("ok\n") self.manager.dispatch_message( - {"type": "apt-sources-replace", "sources": [], "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [], + "gpg-keys": [], + "operation-id": 1, + }, + ) self.assertFalse( os.path.exists( - os.path.join(self.sourceslist.SOURCES_LIST_D, "file1.list"))) + os.path.join(self.sourceslist.SOURCES_LIST_D, "file1.list"), + ), + ) self.assertTrue( os.path.exists( - os.path.join(self.sourceslist.SOURCES_LIST_D, - "file1.list.save"))) + os.path.join( + self.sourceslist.SOURCES_LIST_D, + "file1.list.save", + ), + ), + ) self.assertTrue( os.path.exists( - os.path.join(self.sourceslist.SOURCES_LIST_D, - "file2.list.save"))) + os.path.join( + self.sourceslist.SOURCES_LIST_D, + "file2.list.save", + ), + ), + ) + + def test_does_not_rename_sources_list_d(self): + """ + The sources files in sources.list.d are not renamed to .save when a + message is received if config says not to manage them. + """ + with open( + os.path.join(self.sourceslist.SOURCES_LIST_D, "file1.list"), + "w", + ) as sources1: + sources1.write("ok\n") + + with open( + os.path.join(self.sourceslist.SOURCES_LIST_D, "file2.list.save"), + "w", + ) as sources2: + sources2.write("ok\n") + + self.manager.config.manage_sources_list_d = False + self.manager.dispatch_message( + { + "type": "apt-sources-replace", + "sources": [], + "gpg-keys": [], + "operation-id": 1, + }, + ) + + self.assertTrue( + os.path.exists( + os.path.join(self.sourceslist.SOURCES_LIST_D, "file1.list"), + ), + ) + + self.assertFalse( + os.path.exists( + os.path.join( + self.sourceslist.SOURCES_LIST_D, + "file1.list.save", + ), + ), + ) + + self.assertTrue( + os.path.exists( + os.path.join( + self.sourceslist.SOURCES_LIST_D, + "file2.list.save", + ), + ), + ) def test_create_landscape_sources(self): """ For every sources listed in the sources field of the message, C{AptSources} creates a file with the content in sources.list.d. """ - sources = [{"name": "dev", "content": b"oki\n"}, - {"name": "lucid", "content": b"doki\n"}] + sources = [ + {"name": "dev", "content": b"oki\n"}, + {"name": "lucid", "content": b"doki\n"}, + ] self.manager.dispatch_message( - {"type": "apt-sources-replace", "sources": sources, "gpg-keys": [], - "operation-id": 1}) - - dev_file = os.path.join(self.sourceslist.SOURCES_LIST_D, - "landscape-dev.list") + { + "type": "apt-sources-replace", + "sources": sources, + "gpg-keys": [], + "operation-id": 1, + }, + ) + + dev_file = os.path.join( + self.sourceslist.SOURCES_LIST_D, + "landscape-dev.list", + ) self.assertTrue(os.path.exists(dev_file)) with open(dev_file) as file: result = file.read() self.assertEqual("oki\n", result) - lucid_file = os.path.join(self.sourceslist.SOURCES_LIST_D, - "landscape-lucid.list") + lucid_file = os.path.join( + self.sourceslist.SOURCES_LIST_D, + "landscape-lucid.list", + ) self.assertTrue(os.path.exists(lucid_file)) with open(lucid_file) as file: result = file.read() @@ -276,15 +405,19 @@ class AptSourcesTests(LandscapeTest): gpg_keys = ["key1", "key2"] self.manager.dispatch_message( - {"type": "apt-sources-replace", "sources": [], - "gpg-keys": gpg_keys, - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [], + "gpg-keys": gpg_keys, + "operation-id": 1, + }, + ) keys = [] gpg_dirpath = self.sourceslist.TRUSTED_GPG_D for filename in os.listdir(gpg_dirpath): filepath = os.path.join(gpg_dirpath, filename) - with open(filepath, 'r') as fh: + with open(filepath, "r") as fh: keys.append(fh.read()) self.assertCountEqual(keys, gpg_keys) @@ -298,16 +431,28 @@ class AptSourcesTests(LandscapeTest): def _run_process(command, args, env={}, path=None, uid=None, gid=None): self.assertEqual( - find_reporter_command(self.manager.config), command) - self.assertEqual(["--force-apt-update", "--config=%s" % - self.manager.config.config], args) + find_reporter_command(self.manager.config), + command, + ) + self.assertEqual( + [ + "--force-apt-update", + f"--config={self.manager.config.config}", + ], + args, + ) deferred.callback(("ok", "", 0)) return deferred self.sourceslist._run_process = _run_process self.manager.dispatch_message( - {"type": "apt-sources-replace", "sources": [], "gpg-keys": [], - "operation-id": 1}) + { + "type": "apt-sources-replace", + "sources": [], + "gpg-keys": [], + "operation-id": 1, + }, + ) return deferred diff --git a/landscape/client/manager/tests/test_config.py b/landscape/client/manager/tests/test_config.py index 1cdf709..da0c052 100644 --- a/landscape/client/manager/tests/test_config.py +++ b/landscape/client/manager/tests/test_config.py @@ -1,20 +1,29 @@ -from landscape.client.tests.helpers import LandscapeTest -from landscape.client.manager.config import ManagerConfiguration, ALL_PLUGINS +from landscape.client.manager.config import ALL_PLUGINS +from landscape.client.manager.config import ManagerConfiguration from landscape.client.manager.scriptexecution import ALL_USERS +from landscape.client.tests.helpers import LandscapeTest class ManagerConfigurationTest(LandscapeTest): - def setUp(self): - super(ManagerConfigurationTest, self).setUp() + super().setUp() self.config = ManagerConfiguration() def test_plugin_factories(self): """By default all plugins are enabled.""" - self.assertEqual(["ProcessKiller", "PackageManager", "UserManager", - "ShutdownManager", "AptSources", "HardwareInfo", - "KeystoneToken"], - ALL_PLUGINS) + self.assertEqual( + [ + "ProcessKiller", + "PackageManager", + "UserManager", + "ShutdownManager", + "AptSources", + "HardwareInfo", + "KeystoneToken", + "SnapManager", + ], + ALL_PLUGINS, + ) self.assertEqual(ALL_PLUGINS, self.config.plugin_factories) def test_plugin_factories_with_manager_plugins(self): @@ -31,9 +40,11 @@ class ManagerConfigurationTest(LandscapeTest): command line option. """ self.config.load(["--include-manager-plugins", "ScriptExecution"]) - self.assertEqual(len(self.config.plugin_factories), - len(ALL_PLUGINS) + 1) - self.assertTrue('ScriptExecution' in self.config.plugin_factories) + self.assertEqual( + len(self.config.plugin_factories), + len(ALL_PLUGINS) + 1, + ) + self.assertTrue("ScriptExecution" in self.config.plugin_factories) def test_get_allowed_script_users(self): """ @@ -55,5 +66,7 @@ class ManagerConfigurationTest(LandscapeTest): as. """ self.config.load(["--script-users", "foo, bar,baz"]) - self.assertEqual(self.config.get_allowed_script_users(), - ["foo", "bar", "baz"]) + self.assertEqual( + self.config.get_allowed_script_users(), + ["foo", "bar", "baz"], + ) diff --git a/landscape/client/manager/tests/test_customgraph.py b/landscape/client/manager/tests/test_customgraph.py index 66c8c5b..1c59ccd 100644 --- a/landscape/client/manager/tests/test_customgraph.py +++ b/landscape/client/manager/tests/test_customgraph.py @@ -1,17 +1,17 @@ +import logging import os import pwd -import logging - -import mock +from unittest import mock from twisted.internet.error import ProcessDone from twisted.python.failure import Failure from landscape.client.manager.customgraph import CustomGraphPlugin from landscape.client.manager.store import ManagerStore - -from landscape.lib.testing import StubProcessFactory, DummyProcess -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper +from landscape.lib.testing import DummyProcess +from landscape.lib.testing import StubProcessFactory class CustomGraphManagerTests(LandscapeTest): @@ -19,17 +19,17 @@ class CustomGraphManagerTests(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(CustomGraphManagerTests, self).setUp() + super().setUp() self.store = ManagerStore(":memory:") self.manager.store = self.store - self.broker_service.message_store.set_accepted_types( - ["custom-graph"]) + self.broker_service.message_store.set_accepted_types(["custom-graph"]) self.data_path = self.makeDir() self.manager.config.data_path = self.data_path os.makedirs(os.path.join(self.data_path, "custom-graph-scripts")) self.manager.config.script_users = "ALL" self.graph_manager = CustomGraphPlugin( - create_time=list(range(1500, 0, -300)).pop) + create_time=list(range(1500, 0, -300)).pop, + ) self.manager.add(self.graph_manager) def _exit_process_protocol(self, protocol, stdout): @@ -43,18 +43,29 @@ class CustomGraphManagerTests(LandscapeTest): info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo hi!", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo hi!", + "username": username, + "graph-id": 123, + }, + ) self.assertEqual( self.store.get_graphs(), - [(123, - os.path.join(self.data_path, "custom-graph-scripts", - "graph-123"), - username)]) + [ + ( + 123, + os.path.join( + self.data_path, + "custom-graph-scripts", + "graph-123", + ), + username, + ), + ], + ) @mock.patch("pwd.getpwnam") def test_add_graph_unknown_user(self, mock_getpwnam): @@ -69,14 +80,17 @@ class CustomGraphManagerTests(LandscapeTest): self.logger.setLevel(logging.ERROR) self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo hi!", - "username": "foo", - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo hi!", + "username": "foo", + "graph-id": 123, + }, + ) graph = self.store.get_graph(123) self.assertEqual(graph[0], 123) - self.assertEqual(graph[2], u"foo") + self.assertEqual(graph[2], "foo") self.assertTrue(error_message in self.logfile.getvalue()) mock_getpwnam.assert_called_with("foo") @@ -84,26 +98,36 @@ class CustomGraphManagerTests(LandscapeTest): @mock.patch("os.chmod") @mock.patch("pwd.getpwnam") def test_add_graph_for_user(self, mock_getpwnam, mock_chmod, mock_chown): - - class pwnam(object): + class PwNam: pw_uid = 1234 pw_gid = 5678 pw_dir = self.makeFile() - mock_getpwnam.return_value = pwnam + mock_getpwnam.return_value = PwNam self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo hi!", - "username": "bar", - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo hi!", + "username": "bar", + "graph-id": 123, + }, + ) self.assertEqual( self.store.get_graphs(), - [(123, - os.path.join(self.data_path, "custom-graph-scripts", - "graph-123"), - "bar")]) + [ + ( + 123, + os.path.join( + self.data_path, + "custom-graph-scripts", + "graph-123", + ), + "bar", + ), + ], + ) mock_chown.assert_called_with(mock.ANY, 1234, 5678) mock_chmod.assert_called_with(mock.ANY, 0o700) @@ -111,15 +135,15 @@ class CustomGraphManagerTests(LandscapeTest): def test_remove_unknown_graph(self): self.manager.dispatch_message( - {"type": "custom-graph-remove", - "graph-id": 123}) + {"type": "custom-graph-remove", "graph-id": 123}, + ) def test_remove_graph(self): - filename = self.makeFile(content='foo') - self.store.add_graph(123, filename, u"user") + filename = self.makeFile(content="foo") + self.store.add_graph(123, filename, "user") self.manager.dispatch_message( - {"type": "custom-graph-remove", - "graph-id": 123}) + {"type": "custom-graph-remove", "graph-id": 123}, + ) self.assertFalse(os.path.exists(filename)) def test_run(self): @@ -132,10 +156,20 @@ class CustomGraphManagerTests(LandscapeTest): script_hash = b"483f2304b49063680c75e3c9e09cf6d0" self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": {123: {"error": u"", - "values": [(300, 1.0)], - "script-hash": script_hash}}, - "type": "custom-graph"}]) + [ + { + "data": { + 123: { + "error": "", + "values": [(300, 1.0)], + "script-hash": script_hash, + }, + }, + "type": "custom-graph", + }, + ], + ) + return self.graph_manager.run().addCallback(check) def test_run_multiple(self): @@ -151,17 +185,29 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: {"error": u"", - "values": [(300, 1.0)], - "script-hash": b"483f2304b49063680c75e3c9e09cf6d0", + [ + { + "data": { + 123: { + "error": "", + "values": [(300, 1.0)], + "script-hash": ( + b"483f2304b49063680c75e3c9e09cf6d0" + ), }, - 124: {"error": u"", - "values": [(300, 2.0)], - "script-hash": b"73a74b1530b2256db7edacb9b9cc385e", + 124: { + "error": "", + "values": [(300, 2.0)], + "script-hash": ( + b"73a74b1530b2256db7edacb9b9cc385e" + ), }, - }, - "type": "custom-graph"}]) + }, + "type": "custom-graph", + }, + ], + ) + return self.graph_manager.run().addCallback(check) def test_run_with_nonzero_exit_code(self): @@ -173,14 +219,22 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": u" (process exited with code 1)", - "values": [], - "script-hash": b"eaca3ba1a3bf1948876eba320148c5e9", - } + [ + { + "data": { + 123: { + "error": " (process exited with code 1)", + "values": [], + "script-hash": ( + b"eaca3ba1a3bf1948876eba320148c5e9" + ), + }, + }, + "type": "custom-graph", }, - "type": "custom-graph"}]) + ], + ) + return self.graph_manager.run().addCallback(check) def test_run_cast_result_error(self): @@ -200,15 +254,25 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": (u"InvalidFormatError: Failed to convert to " - u"number: 'foobar'"), - "values": [], - "script-hash": b"baab6c16d9143523b7865d46896e4596", + [ + { + "data": { + 123: { + "error": ( + "InvalidFormatError: Failed to convert to " + "number: 'foobar'" + ), + "values": [], + "script-hash": ( + b"baab6c16d9143523b7865d46896e4596" + ), + }, }, + "type": "custom-graph", }, - "type": "custom-graph"}]) + ], + ) + return result.addCallback(check) def test_run_no_output_error(self): @@ -228,15 +292,25 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": (u"NoOutputError: Script did not output " - u"any value"), - "values": [], - "script-hash": b"baab6c16d9143523b7865d46896e4596", + [ + { + "data": { + 123: { + "error": ( + "NoOutputError: Script did not output " + "any value" + ), + "values": [], + "script-hash": ( + b"baab6c16d9143523b7865d46896e4596" + ), + }, }, + "type": "custom-graph", }, - "type": "custom-graph"}]) + ], + ) + return result.addCallback(check) def test_run_no_output_error_with_other_result(self): @@ -258,18 +332,32 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": (u"NoOutputError: Script did not output " - u"any value"), - "script-hash": b"baab6c16d9143523b7865d46896e4596", - "values": []}, - 124: { - "error": u"", - "script-hash": b"baab6c16d9143523b7865d46896e4596", - "values": [(300, 0.5)]}, - }, - "type": "custom-graph"}]) + [ + { + "data": { + 123: { + "error": ( + "NoOutputError: Script did not output " + "any value" + ), + "script-hash": ( + b"baab6c16d9143523b7865d46896e4596" + ), + "values": [], + }, + 124: { + "error": "", + "script-hash": ( + b"baab6c16d9143523b7865d46896e4596" + ), + "values": [(300, 0.5)], + }, + }, + "type": "custom-graph", + }, + ], + ) + return result.addCallback(check) def test_multiple_errors(self): @@ -291,19 +379,35 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": (u"InvalidFormatError: Failed to convert " - u"to number: 'foo'"), - "script-hash": b"baab6c16d9143523b7865d46896e4596", - "values": []}, - 124: { - "error": (u"NoOutputError: Script did not output " - u"any value"), - "script-hash": b"baab6c16d9143523b7865d46896e4596", - "values": []}, - }, - "type": "custom-graph"}]) + [ + { + "data": { + 123: { + "error": ( + "InvalidFormatError: Failed to convert " + "to number: 'foo'" + ), + "script-hash": ( + b"baab6c16d9143523b7865d46896e4596" + ), + "values": [], + }, + 124: { + "error": ( + "NoOutputError: Script did not output " + "any value" + ), + "script-hash": ( + b"baab6c16d9143523b7865d46896e4596" + ), + "values": [], + }, + }, + "type": "custom-graph", + }, + ], + ) + return result.addCallback(check) @mock.patch("pwd.getpwnam") @@ -313,12 +417,12 @@ class CustomGraphManagerTests(LandscapeTest): factory = StubProcessFactory() self.graph_manager.process_factory = factory - class pwnam(object): + class PwNam: pw_uid = 1234 pw_gid = 5678 pw_dir = self.makeFile() - mock_getpwnam.return_value = pwnam + mock_getpwnam.return_value = PwNam result = self.graph_manager.run() @@ -353,14 +457,22 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": (u"ProhibitedUserError: Custom graph cannot " - u"be run as user %s") % (username,), - "script-hash": b"", - "values": []}, + [ + { + "data": { + 123: { + "error": ( + "ProhibitedUserError: Custom graph cannot " + f"be run as user {username}" + ), + "script-hash": b"", + "values": [], + }, + }, + "type": "custom-graph", }, - "type": "custom-graph"}]) + ], + ) return result.addCallback(check) @@ -381,13 +493,21 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": u"UnknownUserError: Unknown user 'foo'", - "script-hash": b"", - "values": []}, + [ + { + "data": { + 123: { + "error": ( + "UnknownUserError: Unknown user 'foo'" + ), + "script-hash": b"", + "values": [], + }, + }, + "type": "custom-graph", }, - "type": "custom-graph"}]) + ], + ) mock_getpwnam.assert_called_with("foo") return result.addCallback(check) @@ -412,13 +532,23 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": u"Process exceeded the 10 seconds limit", - "script-hash": b"9893532233caff98cd083a116b013c0b", - "values": []}, + [ + { + "data": { + 123: { + "error": ( + "Process exceeded the 10 seconds limit" + ), + "script-hash": ( + b"9893532233caff98cd083a116b013c0b" + ), + "values": [], + }, + }, + "type": "custom-graph", }, - "type": "custom-graph"}]) + ], + ) return result.addCallback(check) @@ -437,8 +567,15 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": {123: {"error": u"", "script-hash": b"", "values": []}}, - "type": "custom-graph"}]) + [ + { + "data": { + 123: {"error": "", "script-hash": b"", "values": []}, + }, + "type": "custom-graph", + }, + ], + ) def test_send_message_add_stored_graph(self): """ @@ -449,21 +586,32 @@ class CustomGraphManagerTests(LandscapeTest): info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo hi!", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo hi!", + "username": username, + "graph-id": 123, + }, + ) self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"api": b"3.2", - "data": {123: {"error": u"", - "script-hash": - b"e00a2f44dbc7b6710ce32af2348aec9b", - "values": []}}, - "timestamp": 0, - "type": "custom-graph"}]) + [ + { + "api": b"3.2", + "data": { + 123: { + "error": "", + "script-hash": b"e00a2f44dbc7b6710ce32af2348aec9b", + "values": [], + }, + }, + "timestamp": 0, + "type": "custom-graph", + }, + ], + ) def test_send_message_check_not_present_graph(self): """C{send_message} checks the presence of the custom-graph script.""" @@ -471,20 +619,28 @@ class CustomGraphManagerTests(LandscapeTest): info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo hi!", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo hi!", + "username": username, + "graph-id": 123, + }, + ) filename = self.store.get_graph(123)[1] os.unlink(filename) self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"api": b"3.2", - "data": {}, - "timestamp": 0, - "type": "custom-graph"}]) + [ + { + "api": b"3.2", + "data": {}, + "timestamp": 0, + "type": "custom-graph", + }, + ], + ) def test_send_message_dont_rehash(self): """ @@ -495,67 +651,102 @@ class CustomGraphManagerTests(LandscapeTest): info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo hi!", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo hi!", + "username": username, + "graph-id": 123, + }, + ) self.graph_manager.exchange() self.graph_manager._get_script_hash = lambda x: 1 / 0 self.graph_manager.do_send = True self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"api": b"3.2", - "data": {123: {"error": u"", - "script-hash": - b"e00a2f44dbc7b6710ce32af2348aec9b", - "values": []}}, - "timestamp": 0, - "type": "custom-graph"}, - {"api": b"3.2", - "data": {123: {"error": u"", - "script-hash": - b"e00a2f44dbc7b6710ce32af2348aec9b", - "values": []}}, - "timestamp": 0, - "type": "custom-graph"}]) + [ + { + "api": b"3.2", + "data": { + 123: { + "error": "", + "script-hash": b"e00a2f44dbc7b6710ce32af2348aec9b", + "values": [], + }, + }, + "timestamp": 0, + "type": "custom-graph", + }, + { + "api": b"3.2", + "data": { + 123: { + "error": "", + "script-hash": b"e00a2f44dbc7b6710ce32af2348aec9b", + "values": [], + }, + }, + "timestamp": 0, + "type": "custom-graph", + }, + ], + ) def test_send_message_rehash_if_necessary(self): uid = os.getuid() info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo hi!", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo hi!", + "username": username, + "graph-id": 123, + }, + ) self.graph_manager.exchange() self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo bye!", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo bye!", + "username": username, + "graph-id": 123, + }, + ) self.graph_manager.do_send = True self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"api": b"3.2", - "data": {123: {"error": u"", - "script-hash": - b"e00a2f44dbc7b6710ce32af2348aec9b", - "values": []}}, - "timestamp": 0, - "type": "custom-graph"}, - {"api": b"3.2", - "data": {123: {"error": u"", - "script-hash": - b"d483816dc0fbb51ede42502a709b0e2a", - "values": []}}, - "timestamp": 0, - "type": "custom-graph"}]) + [ + { + "api": b"3.2", + "data": { + 123: { + "error": "", + "script-hash": b"e00a2f44dbc7b6710ce32af2348aec9b", + "values": [], + }, + }, + "timestamp": 0, + "type": "custom-graph", + }, + { + "api": b"3.2", + "data": { + 123: { + "error": "", + "script-hash": b"d483816dc0fbb51ede42502a709b0e2a", + "values": [], + }, + }, + "timestamp": 0, + "type": "custom-graph", + }, + ], + ) def test_run_with_script_updated(self): """ @@ -567,11 +758,14 @@ class CustomGraphManagerTests(LandscapeTest): info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo 1.0", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo 1.0", + "username": username, + "graph-id": 123, + }, + ) factory = StubProcessFactory() self.graph_manager.process_factory = factory @@ -581,11 +775,14 @@ class CustomGraphManagerTests(LandscapeTest): spawn = factory.spawns[0] self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo 2.0", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo 2.0", + "username": username, + "graph-id": 123, + }, + ) self._exit_process_protocol(spawn[0], b"1.0") @@ -593,13 +790,23 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"api": b"3.2", - "data": {123: {"error": u"", - "script-hash": - b"991e15a81929c79fe1d243b2afd99c62", - "values": []}}, - "timestamp": 0, - "type": "custom-graph"}]) + [ + { + "api": b"3.2", + "data": { + 123: { + "error": "", + "script-hash": ( + b"991e15a81929c79fe1d243b2afd99c62" + ), + "values": [], + }, + }, + "timestamp": 0, + "type": "custom-graph", + }, + ], + ) return result.addCallback(check) @@ -612,11 +819,14 @@ class CustomGraphManagerTests(LandscapeTest): info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo 1.0", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo 1.0", + "username": username, + "graph-id": 123, + }, + ) factory = StubProcessFactory() self.graph_manager.process_factory = factory @@ -626,8 +836,8 @@ class CustomGraphManagerTests(LandscapeTest): spawn = factory.spawns[0] self.manager.dispatch_message( - {"type": "custom-graph-remove", - "graph-id": 123}) + {"type": "custom-graph-remove", "graph-id": 123}, + ) self._exit_process_protocol(spawn[0], b"1.0") @@ -635,8 +845,16 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"api": b"3.2", "data": {}, "timestamp": 0, "type": - "custom-graph"}]) + [ + { + "api": b"3.2", + "data": {}, + "timestamp": 0, + "type": "custom-graph", + }, + ], + ) + return result.addCallback(check) def test_run_not_accepted_types(self): @@ -650,11 +868,14 @@ class CustomGraphManagerTests(LandscapeTest): info = pwd.getpwuid(uid) username = info.pw_name self.manager.dispatch_message( - {"type": "custom-graph-add", - "interpreter": "/bin/sh", - "code": "echo 1.0", - "username": username, - "graph-id": 123}) + { + "type": "custom-graph-add", + "interpreter": "/bin/sh", + "code": "echo 1.0", + "username": username, + "graph-id": 123, + }, + ) factory = StubProcessFactory() self.graph_manager.process_factory = factory @@ -671,7 +892,8 @@ class CustomGraphManagerTests(LandscapeTest): of results. """ self.graph_manager.registry.broker.call_if_accepted = ( - lambda *args: 1 / 0) + lambda *args: 1 / 0 + ) factory = StubProcessFactory() self.graph_manager.process_factory = factory result = self.graph_manager.run() @@ -685,7 +907,7 @@ class CustomGraphManagerTests(LandscapeTest): Using a non-existent user containing unicode characters fails with the appropriate error message. """ - username = u"non-existent-f\N{LATIN SMALL LETTER E WITH ACUTE}e" + username = "non-existent-f\N{LATIN SMALL LETTER E WITH ACUTE}e" self.manager.config.script_users = "ALL" @@ -701,12 +923,21 @@ class CustomGraphManagerTests(LandscapeTest): self.graph_manager.exchange() self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": { - 123: { - "error": - u"UnknownUserError: Unknown user '%s'" % username, - "script-hash": b"9893532233caff98cd083a116b013c0b", - "values": []}}, - "type": "custom-graph"}]) + [ + { + "data": { + 123: { + "error": "UnknownUserError: " + f"Unknown user '{username}'", + "script-hash": ( + b"9893532233caff98cd083a116b013c0b" + ), + "values": [], + }, + }, + "type": "custom-graph", + }, + ], + ) return result.addCallback(check) diff --git a/landscape/client/manager/tests/test_fakepackagemanager.py b/landscape/client/manager/tests/test_fakepackagemanager.py index e4d9b5a..e8bb18f 100644 --- a/landscape/client/manager/tests/test_fakepackagemanager.py +++ b/landscape/client/manager/tests/test_fakepackagemanager.py @@ -1,7 +1,7 @@ -from landscape.client.manager.plugin import SUCCEEDED - from landscape.client.manager.fakepackagemanager import FakePackageManager -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper +from landscape.client.manager.plugin import SUCCEEDED +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper class FakePackageManagerTest(LandscapeTest): @@ -10,7 +10,7 @@ class FakePackageManagerTest(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(FakePackageManagerTest, self).setUp() + super().setUp() self.package_manager = FakePackageManager() self.package_manager.randint = lambda x, y: 0 @@ -26,10 +26,17 @@ class FakePackageManagerTest(LandscapeTest): self.manager.dispatch_message(message) self.manager.reactor.advance(1) - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "change-packages-result", - "result-text": "OK done.", - "result-code": 1, "operation-id": 1}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "change-packages-result", + "result-text": "OK done.", + "result-code": 1, + "operation-id": 1, + }, + ], + ) def test_handle_change_package_locks(self): """ @@ -43,12 +50,18 @@ class FakePackageManagerTest(LandscapeTest): self.manager.dispatch_message(message) self.manager.reactor.advance(1) - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "result-text": - "Package locks successfully changed.", - "result-code": 0, "status": SUCCEEDED, - "operation-id": 1}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "result-text": "Package locks successfully changed.", + "result-code": 0, + "status": SUCCEEDED, + "operation-id": 1, + }, + ], + ) def test_handle_release_upgrade(self): """ @@ -62,9 +75,15 @@ class FakePackageManagerTest(LandscapeTest): self.manager.dispatch_message(message) self.manager.reactor.advance(1) - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "result-text": - "Successful release upgrade.", - "result-code": 0, "status": SUCCEEDED, - "operation-id": 1}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "result-text": "Successful release upgrade.", + "result-code": 0, + "status": SUCCEEDED, + "operation-id": 1, + }, + ], + ) diff --git a/landscape/client/manager/tests/test_hardwareinfo.py b/landscape/client/manager/tests/test_hardwareinfo.py index e309bdc..2f82f16 100644 --- a/landscape/client/manager/tests/test_hardwareinfo.py +++ b/landscape/client/manager/tests/test_hardwareinfo.py @@ -1,13 +1,13 @@ -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper - from landscape.client.manager.hardwareinfo import HardwareInfo +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper class HardwareInfoTests(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(HardwareInfoTests, self).setUp() + super().setUp() self.info = HardwareInfo() self.info.command = "/bin/echo" self.manager.add(self.info) @@ -24,7 +24,8 @@ class HardwareInfoTests(LandscapeTest): def check(ignored): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": u"-xml -quiet\n", "type": "hardware-info"}]) + [{"data": "-xml -quiet\n", "type": "hardware-info"}], + ) return deferred.addCallback(check) @@ -40,7 +41,8 @@ class HardwareInfoTests(LandscapeTest): def check(ignored): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": u"-xml -quiet\n", "type": "hardware-info"}]) + [{"data": "-xml -quiet\n", "type": "hardware-info"}], + ) return deferred.addCallback(check) @@ -57,7 +59,8 @@ class HardwareInfoTests(LandscapeTest): def check(ignored): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"data": u"-xml -quiet\n", "type": "hardware-info"}]) + [{"data": "-xml -quiet\n", "type": "hardware-info"}], + ) self.assertEqual([], calls) return deferred.addCallback(check) diff --git a/landscape/client/manager/tests/test_keystonetoken.py b/landscape/client/manager/tests/test_keystonetoken.py index e5ba0a9..06c7781 100644 --- a/landscape/client/manager/tests/test_keystonetoken.py +++ b/landscape/client/manager/tests/test_keystonetoken.py @@ -1,8 +1,9 @@ import os -from landscape.client.tests.helpers import LandscapeTest from landscape.client.manager.keystonetoken import KeystoneToken -from landscape.client.tests.helpers import ManagerHelper, FakePersist +from landscape.client.tests.helpers import FakePersist +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper class KeystoneTokenTest(LandscapeTest): @@ -10,7 +11,7 @@ class KeystoneTokenTest(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(KeystoneTokenTest, self).setUp() + super().setUp() self.keystone_file = os.path.join(self.makeDir(), "keystone.conf") self.plugin = KeystoneToken(self.keystone_file) @@ -45,7 +46,8 @@ class KeystoneTokenTest(LandscapeTest): """ self.makeFile( path=self.keystone_file, - content="[DEFAULT]\nadmin_token = foobar") + content="[DEFAULT]\nadmin_token = foobar", + ) # As we allow arbitrary bytes, we also need bytes here. self.assertEqual(b"foobar", self.plugin.get_data()) @@ -54,10 +56,7 @@ class KeystoneTokenTest(LandscapeTest): The data can be arbitrary bytes. """ content = b"[DEFAULT]\nadmin_token = \xff" - self.makeFile( - path=self.keystone_file, - content=content, - mode="wb") + self.makeFile(path=self.keystone_file, content=content, mode="wb") self.assertEqual(b"\xff", self.plugin.get_data()) def test_get_message(self): @@ -67,12 +66,14 @@ class KeystoneTokenTest(LandscapeTest): """ self.makeFile( path=self.keystone_file, - content="[DEFAULT]\nadmin_token = foobar") + content="[DEFAULT]\nadmin_token = foobar", + ) self.plugin.register(self.manager) message = self.plugin.get_message() self.assertEqual( - {'type': 'keystone-token', 'data': b'foobar'}, - message) + {"type": "keystone-token", "data": b"foobar"}, + message, + ) message = self.plugin.get_message() self.assertIs(None, message) @@ -82,8 +83,10 @@ class KeystoneTokenTest(LandscapeTest): creates the perists file. """ flush_interval = self.config.flush_interval - persist_filename = os.path.join(self.config.data_path, - "keystone.bpickle") + persist_filename = os.path.join( + self.config.data_path, + "keystone.bpickle", + ) self.assertFalse(os.path.exists(persist_filename)) self.manager.add(self.plugin) @@ -139,8 +142,10 @@ class KeystoneTokenTest(LandscapeTest): If the plugin could not extract the C{admin_token} from the Keystone config file, upon exchange, C{None} is returned. """ - self.makeFile(path=self.keystone_file, - content="[DEFAULT]\nadmin_token =") + self.makeFile( + path=self.keystone_file, + content="[DEFAULT]\nadmin_token =", + ) self.manager.add(self.plugin) def check(result): diff --git a/landscape/client/manager/tests/test_manager.py b/landscape/client/manager/tests/test_manager.py index 7c67bc9..ea90682 100644 --- a/landscape/client/manager/tests/test_manager.py +++ b/landscape/client/manager/tests/test_manager.py @@ -1,6 +1,6 @@ from landscape.client.manager.store import ManagerStore - -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper class ManagerTest(LandscapeTest): diff --git a/landscape/client/manager/tests/test_packagemanager.py b/landscape/client/manager/tests/test_packagemanager.py index 24ab2c0..08cca53 100644 --- a/landscape/client/manager/tests/test_packagemanager.py +++ b/landscape/client/manager/tests/test_packagemanager.py @@ -1,16 +1,15 @@ -import mock -import os import os.path +from unittest import mock from twisted.internet.defer import Deferred +from landscape.client.manager.packagemanager import PackageManager from landscape.client.package.changer import PackageChanger from landscape.client.package.releaseupgrader import ReleaseUpgrader +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper from landscape.lib.apt.package.store import PackageStore - from landscape.lib.testing import EnvironSaverHelper -from landscape.client.manager.packagemanager import PackageManager -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper class PackageManagerTest(LandscapeTest): @@ -20,10 +19,11 @@ class PackageManagerTest(LandscapeTest): def setUp(self): """Initialize test helpers and create a sample package store.""" - super(PackageManagerTest, self).setUp() + super().setUp() self.config = self.broker_service.config - self.package_store = PackageStore(os.path.join(self.data_path, - "package/database")) + self.package_store = PackageStore( + os.path.join(self.data_path, "package/database"), + ) self.package_manager = PackageManager() def test_create_default_store_upon_message_handling(self): @@ -49,8 +49,10 @@ class PackageManagerTest(LandscapeTest): self.manager.add(self.package_manager) with mock.patch.object(self.package_manager, "spawn_handler"): self.package_manager.run() - self.assertNotIn(PackageChanger, - self.package_manager.spawn_handler.call_args_list) + self.assertNotIn( + PackageChanger, + self.package_manager.spawn_handler.call_args_list, + ) self.assertEqual(0, self.package_manager.spawn_handler.call_count) def test_dont_spawn_release_upgrader_if_message_not_accepted(self): @@ -61,8 +63,10 @@ class PackageManagerTest(LandscapeTest): self.manager.add(self.package_manager) with mock.patch.object(self.package_manager, "spawn_handler"): self.package_manager.run() - self.assertNotIn(ReleaseUpgrader, - self.package_manager.spawn_handler.call_args_list) + self.assertNotIn( + ReleaseUpgrader, + self.package_manager.spawn_handler.call_args_list, + ) self.assertEqual(0, self.package_manager.spawn_handler.call_count) def test_spawn_handler_on_registration_when_already_accepted(self): @@ -79,15 +83,20 @@ class PackageManagerTest(LandscapeTest): return run_result_deferred.chainDeferred(deferred) with mock.patch.object(self.package_manager, "spawn_handler"): - with mock.patch.object(self.package_manager, "run", - side_effect=run_has_run): + with mock.patch.object( + self.package_manager, + "run", + side_effect=run_has_run, + ): service = self.broker_service service.message_store.set_accepted_types( - ["change-packages-result"]) + ["change-packages-result"], + ) self.manager.add(self.package_manager) self.successResultOf(deferred) self.package_manager.spawn_handler.assert_called_once_with( - PackageChanger) + PackageChanger, + ) self.package_manager.run.assert_called_once_with() def test_spawn_changer_on_run_if_message_accepted(self): @@ -102,7 +111,8 @@ class PackageManagerTest(LandscapeTest): self.manager.add(self.package_manager) self.package_manager.run() self.package_manager.spawn_handler.assert_called_with( - PackageChanger) + PackageChanger, + ) # Method is called once for registration, then again explicitly. self.assertEquals(2, self.package_manager.spawn_handler.call_count) @@ -119,7 +129,8 @@ class PackageManagerTest(LandscapeTest): self.manager.add(self.package_manager) self.manager.reactor.fire("package-data-changed")[0] self.package_manager.spawn_handler.assert_called_with( - PackageChanger) + PackageChanger, + ) # Method is called once for registration, then again explicitly. self.assertEquals(2, self.package_manager.spawn_handler.call_count) @@ -135,7 +146,8 @@ class PackageManagerTest(LandscapeTest): self.manager.add(self.package_manager) self.package_manager.run() self.package_manager.spawn_handler.assert_called_with( - ReleaseUpgrader) + ReleaseUpgrader, + ) # Method is called once for registration, then again explicitly. self.assertEquals(2, self.package_manager.spawn_handler.call_count) @@ -149,7 +161,8 @@ class PackageManagerTest(LandscapeTest): self.assertTrue(task) self.assertEqual(task.data, message) self.package_manager.spawn_handler.assert_called_once_with( - PackageChanger) + PackageChanger, + ) def test_change_packages_handling_with_reboot(self): self.manager.add(self.package_manager) @@ -161,7 +174,8 @@ class PackageManagerTest(LandscapeTest): self.assertTrue(task) self.assertEqual(task.data, message) self.package_manager.spawn_handler.assert_called_once_with( - PackageChanger) + PackageChanger, + ) def test_release_upgrade_handling(self): """ @@ -178,7 +192,8 @@ class PackageManagerTest(LandscapeTest): self.assertTrue(task) self.assertEqual(task.data, message) self.package_manager.spawn_handler.assert_called_once_with( - ReleaseUpgrader) + ReleaseUpgrader, + ) def test_spawn_changer(self): """ @@ -188,7 +203,8 @@ class PackageManagerTest(LandscapeTest): command = self.write_script( self.config, "landscape-package-changer", - "#!/bin/sh\necho 'I am the changer!' >&2\n") + "#!/bin/sh\necho 'I am the changer!' >&2\n", + ) self.manager.config = self.config self.package_store.add_task("changer", "Do something!") @@ -211,7 +227,8 @@ class PackageManagerTest(LandscapeTest): command = self.write_script( self.config, "landscape-release-upgrader", - "#!/bin/sh\necho 'I am the upgrader!' >&2\n") + "#!/bin/sh\necho 'I am the upgrader!' >&2\n", + ) self.manager.config = self.config self.package_store.add_task("release-upgrader", "Do something!") @@ -229,7 +246,8 @@ class PackageManagerTest(LandscapeTest): self.write_script( self.config, "landscape-package-changer", - "#!/bin/sh\n/bin/true") + "#!/bin/sh\n/bin/true", + ) self.manager.config = self.config self.package_store.add_task("changer", "Do something!") @@ -247,7 +265,8 @@ class PackageManagerTest(LandscapeTest): command = self.write_script( self.config, "landscape-package-changer", - "#!/bin/sh\necho VAR: $VAR\n") + "#!/bin/sh\necho VAR: $VAR\n", + ) self.manager.config = self.config self.manager.add(self.package_manager) @@ -267,7 +286,8 @@ class PackageManagerTest(LandscapeTest): command = self.write_script( self.config, "landscape-package-changer", - "#!/bin/sh\necho OPTIONS: $@\n") + "#!/bin/sh\necho OPTIONS: $@\n", + ) self.manager.config = self.config self.manager.add(self.package_manager) @@ -298,7 +318,8 @@ class PackageManagerTest(LandscapeTest): self.write_script( self.config, "landscape-package-changer", - "#!/bin/sh\necho RUN\n") + "#!/bin/sh\necho RUN\n", + ) self.manager.config = self.config cwd = os.getcwd() @@ -334,4 +355,5 @@ class PackageManagerTest(LandscapeTest): self.assertTrue(task) self.assertEqual(task.data, message) self.package_manager.spawn_handler.assert_called_once_with( - PackageChanger) + PackageChanger, + ) diff --git a/landscape/client/manager/tests/test_plugin.py b/landscape/client/manager/tests/test_plugin.py index f1c925a..1aadd0a 100644 --- a/landscape/client/manager/tests/test_plugin.py +++ b/landscape/client/manager/tests/test_plugin.py @@ -1,8 +1,10 @@ from twisted.internet.defer import Deferred +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import ManagerPlugin +from landscape.client.manager.plugin import SUCCEEDED from landscape.client.tests.helpers import LandscapeTest from landscape.client.tests.helpers import ManagerHelper -from landscape.client.manager.plugin import ManagerPlugin, SUCCEEDED, FAILED class BrokerPluginTest(LandscapeTest): @@ -19,14 +21,22 @@ class BrokerPluginTest(LandscapeTest): broker_service = self.broker_service broker_service.message_store.set_accepted_types(["operation-result"]) message = {"operation-id": 12312} - operation = (lambda: None) + + def operation(): + return None def assert_messages(ignored): messages = broker_service.message_store.get_pending_messages() - self.assertMessages(messages, - [{"type": "operation-result", - "status": SUCCEEDED, - "operation-id": 12312}]) + self.assertMessages( + messages, + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "operation-id": 12312, + }, + ], + ) result = plugin.call_with_operation_result(message, operation) return result.addCallback(assert_messages) @@ -48,10 +58,17 @@ class BrokerPluginTest(LandscapeTest): def assert_messages(ignored): messages = broker_service.message_store.get_pending_messages() - self.assertMessages(messages, - [{"type": "operation-result", "status": FAILED, - "result-text": "RuntimeError: What the " - "crap!", "operation-id": 12312}]) + self.assertMessages( + messages, + [ + { + "type": "operation-result", + "status": FAILED, + "result-text": "RuntimeError: What the crap!", + "operation-id": 12312, + }, + ], + ) logdata = self.logfile.getvalue() self.assertTrue("RuntimeError: What the crap!" in logdata, logdata) @@ -67,7 +84,9 @@ class BrokerPluginTest(LandscapeTest): broker_service = self.broker_service broker_service.message_store.set_accepted_types(["operation-result"]) message = {"operation-id": 123} - operation = (lambda: None) + + def operation(): + return None def assert_urgency(ignored): self.assertTrue(broker_service.exchanger.is_urgent()) @@ -85,15 +104,23 @@ class BrokerPluginTest(LandscapeTest): broker_service.message_store.set_accepted_types(["operation-result"]) message = {"operation-id": 12312} deferred = Deferred() - operation = (lambda: deferred) + + def operation(): + return deferred def assert_messages(ignored): messages = broker_service.message_store.get_pending_messages() - self.assertMessages(messages, - [{"type": "operation-result", - "result-text": "blah", - "status": SUCCEEDED, - "operation-id": 12312}]) + self.assertMessages( + messages, + [ + { + "type": "operation-result", + "result-text": "blah", + "status": SUCCEEDED, + "operation-id": 12312, + }, + ], + ) result = plugin.call_with_operation_result(message, operation) result.addCallback(assert_messages) diff --git a/landscape/client/manager/tests/test_processkiller.py b/landscape/client/manager/tests/test_processkiller.py index ae45b0a..ff906cd 100644 --- a/landscape/client/manager/tests/test_processkiller.py +++ b/landscape/client/manager/tests/test_processkiller.py @@ -1,23 +1,27 @@ -from datetime import datetime -from mock import patch import signal import subprocess +from datetime import datetime +from unittest.mock import patch +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import SUCCEEDED +from landscape.client.manager.processkiller import ProcessKiller +from landscape.client.manager.processkiller import ProcessMismatchError +from landscape.client.manager.processkiller import ProcessNotFoundError +from landscape.client.manager.processkiller import SignalProcessError +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper from landscape.lib.process import ProcessInformation from landscape.lib.testing import ProcessDataBuilder -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper - -from landscape.client.manager.plugin import SUCCEEDED, FAILED -from landscape.client.manager.processkiller import ( - ProcessKiller, ProcessNotFoundError, ProcessMismatchError, - SignalProcessError) def get_active_process(): - return subprocess.Popen(["python3", "-c", "input()"], - stdin=subprocess.PIPE, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE) + return subprocess.Popen( + ["python3", "-c", "input()"], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + ) def get_missing_pid(): @@ -35,8 +39,11 @@ class ProcessKillerTests(LandscapeTest): LandscapeTest.setUp(self) self.sample_dir = self.makeDir() self.builder = ProcessDataBuilder(self.sample_dir) - self.process_info = ProcessInformation(proc_dir=self.sample_dir, - jiffies=1, boot_time=10) + self.process_info = ProcessInformation( + proc_dir=self.sample_dir, + jiffies=1, + boot_time=10, + ) self.signaller = ProcessKiller(process_info=self.process_info) service = self.broker_service service.message_store.set_accepted_types(["operation-result"]) @@ -44,15 +51,25 @@ class ProcessKillerTests(LandscapeTest): @patch("os.kill") def _test_signal_name(self, signame, signum, kill_mock): self.manager.add(self.signaller) - self.builder.create_data(100, self.builder.RUNNING, - uid=1000, gid=1000, started_after_boot=10, - process_name="ooga") + self.builder.create_data( + 100, + self.builder.RUNNING, + uid=1000, + gid=1000, + started_after_boot=10, + process_name="ooga", + ) self.manager.dispatch_message( - {"type": "signal-process", - "operation-id": 1, - "pid": 100, "name": "ooga", - "start-time": 20, "signal": signame}) + { + "type": "signal-process", + "operation-id": 1, + "pid": 100, + "name": "ooga", + "start-time": 20, + "signal": signame, + }, + ) kill_mock.assert_called_once_with(100, signum) def test_kill_process_signal(self): @@ -86,10 +103,15 @@ class ProcessKillerTests(LandscapeTest): start_time = process_info["start-time"] self.manager.dispatch_message( - {"type": "signal-process", - "operation-id": 1, - "pid": popen.pid, "name": "python", - "start-time": start_time, "signal": signame}) + { + "type": "signal-process", + "operation-id": 1, + "pid": popen.pid, + "name": "python", + "start-time": start_time, + "signal": signame, + }, + ) # We're waiting on the child process here so that we (the # parent process) consume it's return code; this prevents it # from becoming a zombie and makes the test do a better job of @@ -102,9 +124,16 @@ class ProcessKillerTests(LandscapeTest): self.assertEqual(process_info, None) service = self.broker_service - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "status": SUCCEEDED, "operation-id": 1}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "operation-id": 1, + }, + ], + ) def test_kill_real_process(self): self._test_signal_real_process("KILL") @@ -122,19 +151,33 @@ class ProcessKillerTests(LandscapeTest): pid = get_missing_pid() self.manager.dispatch_message( - {"operation-id": 1, "type": "signal-process", - "pid": pid, "name": "zsh", "start-time": 110, - "signal": "KILL"}) - expected_text = ("ProcessNotFoundError: The process zsh with PID %d " - "that started at 1970-01-01 00:01:50 UTC was not " - "found" % (pid,)) + { + "operation-id": 1, + "type": "signal-process", + "pid": pid, + "name": "zsh", + "start-time": 110, + "signal": "KILL", + }, + ) + expected_text = ( + f"ProcessNotFoundError: The process zsh with PID {pid:d} " + "that started at 1970-01-01 00:01:50 UTC was not " + "found" + ) service = self.broker_service - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 1, - "status": FAILED, - "result-text": expected_text}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "operation-id": 1, + "status": FAILED, + "result-text": expected_text, + }, + ], + ) self.assertTrue("ProcessNotFoundError" in self.logfile.getvalue()) def test_signal_process_start_time_mismatch(self): @@ -145,29 +188,47 @@ class ProcessKillerTests(LandscapeTest): self.log_helper.ignore_errors(ProcessMismatchError) self.manager.add(self.signaller) pid = get_missing_pid() - self.builder.create_data(pid, self.builder.RUNNING, - uid=1000, gid=1000, started_after_boot=10, - process_name="hostname") + self.builder.create_data( + pid, + self.builder.RUNNING, + uid=1000, + gid=1000, + started_after_boot=10, + process_name="hostname", + ) self.manager.dispatch_message( - {"operation-id": 1, "type": "signal-process", - "pid": pid, "name": "python", - "start-time": 11, "signal": "KILL"}) + { + "operation-id": 1, + "type": "signal-process", + "pid": pid, + "name": "python", + "start-time": 11, + "signal": "KILL", + }, + ) expected_time = datetime.utcfromtimestamp(11) # boot time + proc start time = 20 actual_time = datetime.utcfromtimestamp(20) - expected_text = ("ProcessMismatchError: The process python with " - "PID %d that started at %s UTC was not found. A " - "process with the same PID that started at %s UTC " - "was found and not sent the KILL signal" - % (pid, expected_time, actual_time)) + expected_text = ( + "ProcessMismatchError: The process python with " + f"PID {pid:d} that started at {expected_time} UTC was not " + "found. A process with the same PID that started " + f"at {actual_time} UTC was found and not sent the KILL signal" + ) service = self.broker_service - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 1, - "status": FAILED, - "result-text": expected_text}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "operation-id": 1, + "status": FAILED, + "result-text": expected_text, + }, + ], + ) self.assertTrue("ProcessMismatchError" in self.logfile.getvalue()) def test_signal_process_race(self): @@ -179,28 +240,51 @@ class ProcessKillerTests(LandscapeTest): """ self.log_helper.ignore_errors(SignalProcessError) pid = get_missing_pid() - self.builder.create_data(pid, self.builder.RUNNING, - uid=1000, gid=1000, started_after_boot=10, - process_name="hostname") - self.assertRaises(SignalProcessError, - self.signaller.signal_process, pid, - "hostname", 20, "KILL") + self.builder.create_data( + pid, + self.builder.RUNNING, + uid=1000, + gid=1000, + started_after_boot=10, + process_name="hostname", + ) + self.assertRaises( + SignalProcessError, + self.signaller.signal_process, + pid, + "hostname", + 20, + "KILL", + ) self.manager.add(self.signaller) self.manager.dispatch_message( - {"operation-id": 1, "type": "signal-process", - "pid": pid, "name": "hostname", "start-time": 20, - "signal": "KILL"}) - expected_text = ("SignalProcessError: Attempting to send the KILL " - "signal to the process hostname with PID %d failed" - % (pid,)) + { + "operation-id": 1, + "type": "signal-process", + "pid": pid, + "name": "hostname", + "start-time": 20, + "signal": "KILL", + }, + ) + expected_text = ( + "SignalProcessError: Attempting to send the KILL " + f"signal to the process hostname with PID {pid:d} failed" + ) service = self.broker_service - self.assertMessages(service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 1, - "status": FAILED, - "result-text": expected_text}]) + self.assertMessages( + service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "operation-id": 1, + "status": FAILED, + "result-text": expected_text, + }, + ], + ) self.assertTrue("SignalProcessError" in self.logfile.getvalue()) @patch("os.kill") @@ -210,13 +294,23 @@ class ProcessKillerTests(LandscapeTest): computed process start time. """ self.manager.add(self.signaller) - self.builder.create_data(100, self.builder.RUNNING, - uid=1000, gid=1000, started_after_boot=10, - process_name="ooga") + self.builder.create_data( + 100, + self.builder.RUNNING, + uid=1000, + gid=1000, + started_after_boot=10, + process_name="ooga", + ) self.manager.dispatch_message( - {"type": "signal-process", - "operation-id": 1, - "pid": 100, "name": "ooga", - "start-time": 21, "signal": "KILL"}) + { + "type": "signal-process", + "operation-id": 1, + "pid": 100, + "name": "ooga", + "start-time": 21, + "signal": "KILL", + }, + ) kill_mock.assert_called_once_with(100, signal.SIGKILL) diff --git a/landscape/client/manager/tests/test_scriptexecution.py b/landscape/client/manager/tests/test_scriptexecution.py index db3ef12..e560f49 100644 --- a/landscape/client/manager/tests/test_scriptexecution.py +++ b/landscape/client/manager/tests/test_scriptexecution.py @@ -1,25 +1,37 @@ -import pwd import os +import pwd +import stat import sys import tempfile -import stat +from unittest import mock -import mock - -from twisted.internet.defer import gatherResults, succeed, fail +from twisted.internet.defer import fail +from twisted.internet.defer import gatherResults +from twisted.internet.defer import succeed from twisted.internet.error import ProcessDone from twisted.python.failure import Failure from landscape import VERSION +from landscape.client.manager.manager import FAILED +from landscape.client.manager.manager import SUCCEEDED +from landscape.client.manager.scriptexecution import ( + FETCH_ATTACHMENTS_FAILED_RESULT, +) +from landscape.client.manager.scriptexecution import PROCESS_FAILED_RESULT +from landscape.client.manager.scriptexecution import ( + ProcessTimeLimitReachedError, +) +from landscape.client.manager.scriptexecution import ScriptExecutionPlugin +from landscape.client.manager.scriptexecution import UBUNTU_PATH +from landscape.client.manager.scriptexecution import UnknownInterpreterError +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper from landscape.lib.fetch import HTTPCodeError from landscape.lib.persist import Persist -from landscape.lib.testing import StubProcessFactory, DummyProcess -from landscape.lib.user import get_user_info, UnknownUserError -from landscape.client.manager.scriptexecution import ( - ScriptExecutionPlugin, ProcessTimeLimitReachedError, PROCESS_FAILED_RESULT, - UBUNTU_PATH, UnknownInterpreterError, FETCH_ATTACHMENTS_FAILED_RESULT) -from landscape.client.manager.manager import SUCCEEDED, FAILED -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper +from landscape.lib.testing import DummyProcess +from landscape.lib.testing import StubProcessFactory +from landscape.lib.user import get_user_info +from landscape.lib.user import UnknownUserError def get_default_environment(): @@ -38,7 +50,7 @@ def get_default_environment(): def encoded_default_environment(): return { - key: value.encode('ascii', 'replace') + key: value.encode("ascii", "replace") for key, value in get_default_environment().items() } @@ -48,7 +60,7 @@ class RunScriptTests(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(RunScriptTests, self).setUp() + super().setUp() self.plugin = ScriptExecutionPlugin() self.manager.add(self.plugin) @@ -64,8 +76,7 @@ class RunScriptTests(LandscapeTest): def test_snap_path(self): """The bin path for snaps is included in the PATH.""" deferred = self.plugin.run_script("/bin/sh", "echo $PATH") - deferred.addCallback( - lambda result: self.assertIn("/snap/bin", result)) + deferred.addCallback(lambda result: self.assertIn("/snap/bin", result)) return deferred def test_other_interpreter(self): @@ -81,7 +92,8 @@ class RunScriptTests(LandscapeTest): """ result = self.plugin.run_script( sys.executable, - "import os\nprint(os.environ)") + "import os\nprint(os.environ)", + ) def check_environment(results): for string in get_default_environment(): @@ -99,7 +111,8 @@ class RunScriptTests(LandscapeTest): result = self.plugin.run_script( sys.executable, "import os\nprint(os.environ)", - server_supplied_env=server_supplied_env) + server_supplied_env=server_supplied_env, + ) def check_environment(results): for string in get_default_environment(): @@ -118,20 +131,23 @@ class RunScriptTests(LandscapeTest): (encoding from Python's sys.getfilesystemencoding). """ patch_fs_encoding = mock.patch( - 'sys.getfilesystemencoding', return_value='UTF-8' + "sys.getfilesystemencoding", + return_value="UTF-8", ) patch_fs_encoding.start() server_supplied_env = { - "LEMMY": u"Mot\N{LATIN SMALL LETTER O WITH DIAERESIS}rhead", + "LEMMY": "Mot\N{LATIN SMALL LETTER O WITH DIAERESIS}rhead", # Somehow it's just not as cool... } result = self.plugin.run_script( - "/bin/sh", "echo $LEMMY", - server_supplied_env=server_supplied_env) + "/bin/sh", + "echo $LEMMY", + server_supplied_env=server_supplied_env, + ) def check_environment(results): - self.assertEqual(server_supplied_env["LEMMY"] + u'\n', results) + self.assertEqual(server_supplied_env["LEMMY"] + "\n", results) def cleanup(result): patch_fs_encoding.stop() @@ -145,12 +161,16 @@ class RunScriptTests(LandscapeTest): Server-supplied environment variables override client default values if the server provides them. """ - server_supplied_env = {"PATH": "server-path", "USER": "server-user", - "HOME": "server-home"} + server_supplied_env = { + "PATH": "server-path", + "USER": "server-user", + "HOME": "server-home", + } result = self.plugin.run_script( sys.executable, "import os\nprint(os.environ)", - server_supplied_env=server_supplied_env) + server_supplied_env=server_supplied_env, + ) def check_environment(results): for name, value in server_supplied_env.items(): @@ -181,26 +201,28 @@ class RunScriptTests(LandscapeTest): Scripts can contain accented data both in the code and in the result. """ - accented_content = u"\N{LATIN SMALL LETTER E WITH ACUTE}" + accented_content = "\N{LATIN SMALL LETTER E WITH ACUTE}" result = self.plugin.run_script( - u"/bin/sh", u"echo %s" % (accented_content,)) + "/bin/sh", + f"echo {accented_content}", + ) # self.assertEqual gets the result as first argument and that's what we # compare against. - result.addCallback( - self.assertEqual, "%s\n" % (accented_content,)) + result.addCallback(self.assertEqual, f"{accented_content}\n") return result def test_accented_run_in_interpreter(self): """ Scripts can also contain accents in the interpreter. """ - accented_content = u"\N{LATIN SMALL LETTER E WITH ACUTE}" + accented_content = "\N{LATIN SMALL LETTER E WITH ACUTE}" result = self.plugin.run_script( - u"/bin/echo %s" % (accented_content,), u"") + f"/bin/echo {accented_content}", + "", + ) def check(result): - self.assertTrue( - "%s " % (accented_content,) in result) + self.assertTrue(f"{accented_content} " in result) result.addCallback(check) return result @@ -220,9 +242,10 @@ class RunScriptTests(LandscapeTest): result = self.plugin.run_script("/bin/sh", "umask") def check(result): - self.assertEqual("%04o\n" % old_umask, result) + self.assertEqual(f"{old_umask:04o}\n", result) mock_umask.assert_has_calls( - [mock.call(0o22), mock.call(old_umask)]) + [mock.call(0o22), mock.call(old_umask)], + ) result.addCallback(check) return result.addCallback(lambda _: patch_umask.stop()) @@ -236,11 +259,16 @@ class RunScriptTests(LandscapeTest): mock_umask = patch_umask.start() patch_mkdtemp = mock.patch( - "tempfile.mkdtemp", side_effect=OSError("Fail!")) + "tempfile.mkdtemp", + side_effect=OSError("Fail!"), + ) mock_mkdtemp = patch_mkdtemp.start() result = self.plugin.run_script( - "/bin/sh", "umask", attachments={u"file1": "some data"}) + "/bin/sh", + "umask", + attachments={"file1": "some data"}, + ) def check(error): self.assertIsInstance(error.value, OSError) @@ -257,9 +285,10 @@ class RunScriptTests(LandscapeTest): def test_run_with_attachments(self): result = self.plugin.run_script( - u"/bin/sh", - u"ls $LANDSCAPE_ATTACHMENTS && cat $LANDSCAPE_ATTACHMENTS/file1", - attachments={u"file1": "some data"}) + "/bin/sh", + "ls $LANDSCAPE_ATTACHMENTS && cat $LANDSCAPE_ATTACHMENTS/file1", + attachments={"file1": "some data"}, + ) def check(result): self.assertEqual(result, "file1\nsome data") @@ -275,30 +304,37 @@ class RunScriptTests(LandscapeTest): """ self.manager.config.url = "https://localhost/message-system" persist = Persist( - filename=os.path.join(self.config.data_path, "broker.bpickle")) + filename=os.path.join(self.config.data_path, "broker.bpickle"), + ) registration_persist = persist.root_at("registration") registration_persist.set("secure-id", "secure_id") persist.save() patch_fetch = mock.patch( - "landscape.client.manager.scriptexecution.fetch_async") + "landscape.client.manager.scriptexecution.fetch_async", + ) mock_fetch = patch_fetch.start() mock_fetch.return_value = succeed(b"some other data") - headers = {"User-Agent": "landscape-client/%s" % VERSION, - "Content-Type": "application/octet-stream", - "X-Computer-ID": "secure_id"} + headers = { + "User-Agent": f"landscape-client/{VERSION}", + "Content-Type": "application/octet-stream", + "X-Computer-ID": "secure_id", + } result = self.plugin.run_script( - u"/bin/sh", - u"ls $LANDSCAPE_ATTACHMENTS && cat $LANDSCAPE_ATTACHMENTS/file1", - attachments={u"file1": 14}) + "/bin/sh", + "ls $LANDSCAPE_ATTACHMENTS && cat $LANDSCAPE_ATTACHMENTS/file1", + attachments={"file1": 14}, + ) def check(result): self.assertEqual(result, "file1\nsome other data") mock_fetch.assert_called_with( - "https://localhost/attachment/14", headers=headers, - cainfo=None) + "https://localhost/attachment/14", + headers=headers, + cainfo=None, + ) def cleanup(result): patch_fetch.stop() @@ -315,30 +351,37 @@ class RunScriptTests(LandscapeTest): self.manager.config.url = "https://localhost/message-system" self.manager.config.ssl_public_key = "/some/key" persist = Persist( - filename=os.path.join(self.config.data_path, "broker.bpickle")) + filename=os.path.join(self.config.data_path, "broker.bpickle"), + ) registration_persist = persist.root_at("registration") registration_persist.set("secure-id", b"secure_id") persist.save() patch_fetch = mock.patch( - "landscape.client.manager.scriptexecution.fetch_async") + "landscape.client.manager.scriptexecution.fetch_async", + ) mock_fetch = patch_fetch.start() mock_fetch.return_value = succeed(b"some other data") - headers = {"User-Agent": "landscape-client/%s" % VERSION, - "Content-Type": "application/octet-stream", - "X-Computer-ID": "secure_id"} + headers = { + "User-Agent": f"landscape-client/{VERSION}", + "Content-Type": "application/octet-stream", + "X-Computer-ID": "secure_id", + } result = self.plugin.run_script( - u"/bin/sh", - u"ls $LANDSCAPE_ATTACHMENTS && cat $LANDSCAPE_ATTACHMENTS/file1", - attachments={u"file1": 14}) + "/bin/sh", + "ls $LANDSCAPE_ATTACHMENTS && cat $LANDSCAPE_ATTACHMENTS/file1", + attachments={"file1": 14}, + ) def check(result): self.assertEqual(result, "file1\nsome other data") mock_fetch.assert_called_with( - "https://localhost/attachment/14", headers=headers, - cainfo="/some/key") + "https://localhost/attachment/14", + headers=headers, + cainfo="/some/key", + ) def cleanup(result): patch_fetch.stop() @@ -361,9 +404,10 @@ class RunScriptTests(LandscapeTest): the script execution plugin tries to remove the attachments directory. """ result = self.plugin.run_script( - u"/bin/sh", - u"ls $LANDSCAPE_ATTACHMENTS && rm -r $LANDSCAPE_ATTACHMENTS", - attachments={u"file1": "some data"}) + "/bin/sh", + "ls $LANDSCAPE_ATTACHMENTS && rm -r $LANDSCAPE_ATTACHMENTS", + attachments={"file1": "some data"}, + ) def check(result): self.assertEqual(result, "file1\n") @@ -429,12 +473,12 @@ class RunScriptTests(LandscapeTest): patch_getpwnam = mock.patch("pwd.getpwnam") mock_getpwnam = patch_getpwnam.start() - class pwnam(object): + class PwNam: pw_uid = 1234 pw_gid = 5678 pw_dir = self.makeFile() - mock_getpwnam.return_value = pwnam + mock_getpwnam.return_value = PwNam result = self._run_script("user", 1234, 5678, "/") @@ -459,13 +503,17 @@ class RunScriptTests(LandscapeTest): factory = StubProcessFactory() self.plugin.process_factory = factory - result = self.plugin.run_script("/bin/sh", "echo hi", user=username, - attachments={u"file 1": "some data"}) + result = self.plugin.run_script( + "/bin/sh", + "echo hi", + user=username, + attachments={"file 1": "some data"}, + ) self.assertEqual(len(factory.spawns), 1) spawn = factory.spawns[0] self.assertIn("LANDSCAPE_ATTACHMENTS", spawn[3]) - attachment_dir = spawn[3]["LANDSCAPE_ATTACHMENTS"].decode('ascii') + attachment_dir = spawn[3]["LANDSCAPE_ATTACHMENTS"].decode("ascii") self.assertEqual(stat.S_IMODE(os.stat(attachment_dir).st_mode), 0o700) filename = os.path.join(attachment_dir, "file 1") self.assertEqual(stat.S_IMODE(os.stat(filename).st_mode), 0o600) @@ -480,7 +528,8 @@ class RunScriptTests(LandscapeTest): self.assertEqual(data, "foobar") self.assertFalse(os.path.exists(attachment_dir)) mock_chown.assert_has_calls( - [mock.call(mock.ANY, uid, gid) for x in range(3)]) + [mock.call(mock.ANY, uid, gid) for x in range(3)], + ) def cleanup(result): patch_chown.stop() @@ -497,13 +546,15 @@ class RunScriptTests(LandscapeTest): # Ultimately we assert that the resulting output is limited to # 1024 bytes and indicates its truncation. - result.addCallback(self.assertEqual, - ("x" * (1024 - 21)) + "\n**OUTPUT TRUNCATED**") + result.addCallback( + self.assertEqual, + ("x" * (1024 - 21)) + "\n**OUTPUT TRUNCATED**", + ) protocol = factory.spawns[0][0] # Push 2kB of output, so we trigger truncation. - protocol.childDataReceived(1, b"x" * (2*1024)) + protocol.childDataReceived(1, b"x" * (2 * 1024)) for fd in (0, 1, 2): protocol.childConnectionLost(fd) @@ -520,8 +571,10 @@ class RunScriptTests(LandscapeTest): # Ultimately we assert that the resulting output is limited to # 1024 bytes and indicates its truncation. - result.addCallback(self.assertEqual, - ("x" * (1024 - 21)) + "\n**OUTPUT TRUNCATED**") + result.addCallback( + self.assertEqual, + ("x" * (1024 - 21)) + "\n**OUTPUT TRUNCATED**", + ) protocol = factory.spawns[0][0] # Push 1024 bytes of output, so we trigger truncation. @@ -606,8 +659,13 @@ class RunScriptTests(LandscapeTest): @mock.patch("os.chmod") @mock.patch("tempfile.mkstemp") @mock.patch("os.fdopen") - def test_script_is_owned_by_user(self, mock_fdopen, mock_mkstemp, - mock_chmod, mock_chown): + def test_script_is_owned_by_user( + self, + mock_fdopen, + mock_mkstemp, + mock_chmod, + mock_chown, + ): """ This is a very white-box test. When a script is generated, it must be created such that data NEVER gets into it before the file has the @@ -636,7 +694,15 @@ class RunScriptTests(LandscapeTest): mock_fdopen.side_effect = mock_fdopen_side_effect - def spawnProcess(protocol, filename, args, env, path, uid, gid): + def spawnProcess( # noqa: N802 + protocol, + filename, + args, + env, + path, + uid, + gid, + ): self.assertIsNone(uid) self.assertIsNone(gid) self.assertEqual(encoded_default_environment(), env) @@ -646,8 +712,11 @@ class RunScriptTests(LandscapeTest): process_factory.spawnProcess = spawnProcess self.plugin.process_factory = process_factory - result = self.plugin.run_script("/bin/sh", "code", - user=pwd.getpwuid(uid)[0]) + result = self.plugin.run_script( + "/bin/sh", + "code", + user=pwd.getpwuid(uid)[0], + ) def check(_): mock_fdopen.assert_called_with(99, "wb") @@ -657,7 +726,8 @@ class RunScriptTests(LandscapeTest): script_file.close.assert_called_with() self.assertEqual( [mock_mkstemp, mock_fdopen, mock_chmod, mock_chown], - called_mocks) + called_mocks, + ) return result.addCallback(check) @@ -671,7 +741,8 @@ class RunScriptTests(LandscapeTest): mock_mkstemp.return_value = (fd, filename) d = self.plugin.run_script("/bin/sh", "true") return d.addCallback( - lambda _: self.assertFalse(os.path.exists(filename))) + lambda _: self.assertFalse(os.path.exists(filename)), + ) def test_unknown_interpreter(self): """ @@ -687,7 +758,9 @@ class RunScriptTests(LandscapeTest): failure.trap(UnknownInterpreterError) self.assertEqual( failure.value.interpreter, - "/bin/cantpossiblyexist") + "/bin/cantpossiblyexist", + ) + return d.addCallback(cb).addErrback(eb) @@ -695,9 +768,10 @@ class ScriptExecutionMessageTests(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(ScriptExecutionMessageTests, self).setUp() + super().setUp() self.broker_service.message_store.set_accepted_types( - ["operation-result"]) + ["operation-result"], + ) self.manager.config.script_users = "ALL" def _verify_script(self, executable, interp, code): @@ -706,19 +780,27 @@ class ScriptExecutionMessageTests(LandscapeTest): script has the correct content. """ data = open(executable, "r").read() - self.assertEqual(data, "#!%s\n%s" % (interp, code)) - - def _send_script(self, interpreter, code, operation_id=123, - user=pwd.getpwuid(os.getuid())[0], - time_limit=None, attachments={}, - server_supplied_env=None): - message = {"type": "execute-script", - "interpreter": interpreter, - "code": code, - "operation-id": operation_id, - "username": user, - "time-limit": time_limit, - "attachments": dict(attachments)} + self.assertEqual(data, f"#!{interp}\n{code}") + + def _send_script( + self, + interpreter, + code, + operation_id=123, + user=pwd.getpwuid(os.getuid())[0], + time_limit=None, + attachments={}, + server_supplied_env=None, + ): + message = { + "type": "execute-script", + "interpreter": interpreter, + "code": code, + "operation-id": operation_id, + "username": user, + "time-limit": time_limit, + "attachments": dict(attachments), + } if server_supplied_env: message["env"] = server_supplied_env return self.manager.dispatch_message(message) @@ -739,7 +821,9 @@ class ScriptExecutionMessageTests(LandscapeTest): self._verify_script(factory.spawns[0][1], sys.executable, "print 'hi'") self.assertMessages( - self.broker_service.message_store.get_pending_messages(), []) + self.broker_service.message_store.get_pending_messages(), + [], + ) # Now let's simulate the completion of the process factory.spawns[0][0].childDataReceived(1, b"hi!\n") @@ -748,10 +832,15 @@ class ScriptExecutionMessageTests(LandscapeTest): def got_result(r): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "status": SUCCEEDED, - "result-text": u"hi!\n"}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "status": SUCCEEDED, + "result-text": "hi!\n", + }, + ], + ) result.addCallback(got_result) return result @@ -768,11 +857,13 @@ class ScriptExecutionMessageTests(LandscapeTest): self.manager.add(ScriptExecutionPlugin(process_factory=factory)) - result = self._send_script(sys.executable, "print 'hi'", - server_supplied_env={"Dog": "Woof"}) + result = self._send_script( + sys.executable, + "print 'hi'", + server_supplied_env={"Dog": "Woof"}, + ) - self._verify_script( - factory.spawns[0][1], sys.executable, "print 'hi'") + self._verify_script(factory.spawns[0][1], sys.executable, "print 'hi'") # Verify environment was passed self.assertIn("HOME", factory.spawns[0][3]) self.assertIn("USER", factory.spawns[0][3]) @@ -781,7 +872,9 @@ class ScriptExecutionMessageTests(LandscapeTest): self.assertEqual(b"Woof", factory.spawns[0][3]["Dog"]) self.assertMessages( - self.broker_service.message_store.get_pending_messages(), []) + self.broker_service.message_store.get_pending_messages(), + [], + ) # Now let's simulate the completion of the process factory.spawns[0][0].childDataReceived(1, b"Woof\n") @@ -790,10 +883,15 @@ class ScriptExecutionMessageTests(LandscapeTest): def got_result(r): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "status": SUCCEEDED, - "result-text": u"Woof\n"}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "status": SUCCEEDED, + "result-text": "Woof\n", + }, + ], + ) result.addCallback(got_result) return result @@ -803,7 +901,15 @@ class ScriptExecutionMessageTests(LandscapeTest): username = pwd.getpwuid(os.getuid())[0] uid, gid, home = get_user_info(username) - def spawnProcess(protocol, filename, args, env, path, uid, gid): + def spawnProcess( # noqa: N802 + protocol, + filename, + args, + env, + path, + uid, + gid, + ): protocol.childDataReceived(1, "hi!\n") protocol.processEnded(Failure(ProcessDone(0))) self._verify_script(filename, sys.executable, "print 'hi'") @@ -811,14 +917,21 @@ class ScriptExecutionMessageTests(LandscapeTest): process_factory = mock.Mock() process_factory.spawnProcess = mock.Mock(side_effect=spawnProcess) self.manager.add( - ScriptExecutionPlugin(process_factory=process_factory)) + ScriptExecutionPlugin(process_factory=process_factory), + ) result = self._send_script(sys.executable, "print 'hi'", user=username) def check(_): process_factory.spawnProcess.assert_called_with( - mock.ANY, mock.ANY, args=mock.ANY, uid=None, gid=None, - path=mock.ANY, env=encoded_default_environment()) + mock.ANY, + mock.ANY, + args=mock.ANY, + uid=None, + gid=None, + path=mock.ANY, + env=encoded_default_environment(), + ) return result.addCallback(check) @@ -832,17 +945,22 @@ class ScriptExecutionMessageTests(LandscapeTest): unknown user as an easy way to test it. """ self.log_helper.ignore_errors(UnknownUserError) - username = u"non-existent-f\N{LATIN SMALL LETTER E WITH ACUTE}e" - self.manager.add( - ScriptExecutionPlugin()) + username = "non-existent-f\N{LATIN SMALL LETTER E WITH ACUTE}e" + self.manager.add(ScriptExecutionPlugin()) self._send_script(sys.executable, "print 'hi'", user=username) self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "result-text": u"UnknownUserError: Unknown user '%s'" % username, - "status": FAILED}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "result-text": "UnknownUserError: " + f"Unknown user '{username}'", + "status": FAILED, + }, + ], + ) def test_timeout(self): """ @@ -864,11 +982,16 @@ class ScriptExecutionMessageTests(LandscapeTest): def got_result(r): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "status": FAILED, - "result-text": u"ONOEZ", - "result-code": 102}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "status": FAILED, + "result-text": "ONOEZ", + "result-code": 102, + }, + ], + ) result.addCallback(got_result) return result @@ -885,10 +1008,17 @@ class ScriptExecutionMessageTests(LandscapeTest): def got_result(r): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "status": FAILED, - "result-text": u"Scripts cannot be run as user whatever."}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "status": FAILED, + "result-text": ( + "Scripts cannot be run as user whatever." + ), + }, + ], + ) result.addCallback(got_result) return result @@ -896,7 +1026,15 @@ class ScriptExecutionMessageTests(LandscapeTest): def test_urgent_response(self): """Responses to script execution messages are urgent.""" - def spawnProcess(protocol, filename, args, env, path, uid, gid): + def spawnProcess( # noqa: N802 + protocol, + filename, + args, + env, + path, + uid, + gid, + ): protocol.childDataReceived(1, b"hi!\n") protocol.processEnded(Failure(ProcessDone(0))) self._verify_script(filename, sys.executable, "print 'hi'") @@ -905,19 +1043,31 @@ class ScriptExecutionMessageTests(LandscapeTest): process_factory.spawnProcess = mock.Mock(side_effect=spawnProcess) self.manager.add( - ScriptExecutionPlugin(process_factory=process_factory)) + ScriptExecutionPlugin(process_factory=process_factory), + ) def got_result(r): self.assertTrue(self.broker_service.exchanger.is_urgent()) self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "result-text": u"hi!\n", - "status": SUCCEEDED}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "result-text": "hi!\n", + "status": SUCCEEDED, + }, + ], + ) process_factory.spawnProcess.assert_called_with( - mock.ANY, mock.ANY, args=mock.ANY, uid=None, gid=None, - path=mock.ANY, env=encoded_default_environment()) + mock.ANY, + mock.ANY, + args=mock.ANY, + uid=None, + gid=None, + path=mock.ANY, + env=encoded_default_environment(), + ) result = self._send_script(sys.executable, "print 'hi'") return result.addCallback(got_result) @@ -927,9 +1077,20 @@ class ScriptExecutionMessageTests(LandscapeTest): If a script outputs non-printable characters not handled by utf-8, they are replaced during the encoding phase but the script succeeds. """ - def spawnProcess(protocol, filename, args, env, path, uid, gid): + + def spawnProcess( # noqa: N802 + protocol, + filename, + args, + env, + path, + uid, + gid, + ): protocol.childDataReceived( - 1, b"\x7fELF\x01\x01\x01\x00\x00\x00\x95\x01") + 1, + b"\x7fELF\x01\x01\x01\x00\x00\x00\x95\x01", + ) protocol.processEnded(Failure(ProcessDone(0))) self._verify_script(filename, sys.executable, "print 'hi'") @@ -937,18 +1098,27 @@ class ScriptExecutionMessageTests(LandscapeTest): process_factory.spawnProcess = mock.Mock(side_effect=spawnProcess) self.manager.add( - ScriptExecutionPlugin(process_factory=process_factory)) + ScriptExecutionPlugin(process_factory=process_factory), + ) def got_result(r): self.assertTrue(self.broker_service.exchanger.is_urgent()) - [message] = ( - self.broker_service.message_store.get_pending_messages()) + [ + message, + ] = self.broker_service.message_store.get_pending_messages() self.assertEqual( message["result-text"], - u"\x7fELF\x01\x01\x01\x00\x00\x00\ufffd\x01") + "\x7fELF\x01\x01\x01\x00\x00\x00\ufffd\x01", + ) process_factory.spawnProcess.assert_called_with( - mock.ANY, mock.ANY, args=mock.ANY, uid=None, gid=None, - path=mock.ANY, env=encoded_default_environment()) + mock.ANY, + mock.ANY, + args=mock.ANY, + uid=None, + gid=None, + path=mock.ANY, + env=encoded_default_environment(), + ) result = self._send_script(sys.executable, "print 'hi'") return result.addCallback(got_result) @@ -962,16 +1132,22 @@ class ScriptExecutionMessageTests(LandscapeTest): self.manager.add(ScriptExecutionPlugin()) self.manager.dispatch_message( - {"type": "execute-script", "operation-id": 444}) + {"type": "execute-script", "operation-id": 444}, + ) - expected_message = [{"type": "operation-result", - "operation-id": 444, - "result-text": u"KeyError: username", - "status": FAILED}] + expected_message = [ + { + "type": "operation-result", + "operation-id": 444, + "result-text": "KeyError: username", + "status": FAILED, + }, + ] self.assertMessages( self.broker_service.message_store.get_pending_messages(), - expected_message) + expected_message, + ) self.assertTrue("KeyError: 'username'" in self.logfile.getvalue()) @@ -986,11 +1162,16 @@ class ScriptExecutionMessageTests(LandscapeTest): def got_result(ignored): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "result-text": "hi\n", - "result-code": PROCESS_FAILED_RESULT, - "status": FAILED}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "result-text": "hi\n", + "result-code": PROCESS_FAILED_RESULT, + "status": FAILED, + }, + ], + ) return result.addCallback(got_result) @@ -1008,7 +1189,9 @@ class ScriptExecutionMessageTests(LandscapeTest): self._verify_script(factory.spawns[0][1], sys.executable, "print 'hi'") self.assertMessages( - self.broker_service.message_store.get_pending_messages(), []) + self.broker_service.message_store.get_pending_messages(), + [], + ) failure = Failure(RuntimeError("Oh noes!")) factory.spawns[0][0].result_deferred.errback(failure) @@ -1016,10 +1199,15 @@ class ScriptExecutionMessageTests(LandscapeTest): def got_result(r): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "status": FAILED, - "result-text": str(failure)}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "status": FAILED, + "result-text": str(failure), + }, + ], + ) result.addCallback(got_result) return result @@ -1032,30 +1220,43 @@ class ScriptExecutionMessageTests(LandscapeTest): """ self.manager.config.url = "https://localhost/message-system" persist = Persist( - filename=os.path.join(self.config.data_path, "broker.bpickle")) + filename=os.path.join(self.config.data_path, "broker.bpickle"), + ) registration_persist = persist.root_at("registration") registration_persist.set("secure-id", "secure_id") persist.save() - headers = {"User-Agent": "landscape-client/%s" % VERSION, - "Content-Type": "application/octet-stream", - "X-Computer-ID": "secure_id"} + headers = { + "User-Agent": f"landscape-client/{VERSION}", + "Content-Type": "application/octet-stream", + "X-Computer-ID": "secure_id", + } mock_fetch.return_value = fail(HTTPCodeError(404, "Not found")) self.manager.add(ScriptExecutionPlugin()) result = self._send_script( - "/bin/sh", "echo hi", attachments={u"file1": 14}) + "/bin/sh", + "echo hi", + attachments={"file1": 14}, + ) def got_result(ignored): self.assertMessages( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", - "operation-id": 123, - "result-text": "Server returned HTTP code 404", - "result-code": FETCH_ATTACHMENTS_FAILED_RESULT, - "status": FAILED}]) + [ + { + "type": "operation-result", + "operation-id": 123, + "result-text": "Server returned HTTP code 404", + "result-code": FETCH_ATTACHMENTS_FAILED_RESULT, + "status": FAILED, + }, + ], + ) mock_fetch.assert_called_with( - "https://localhost/attachment/14", headers=headers, - cainfo=None) + "https://localhost/attachment/14", + headers=headers, + cainfo=None, + ) return result.addCallback(got_result) diff --git a/landscape/client/manager/tests/test_service.py b/landscape/client/manager/tests/test_service.py index a5a60bc..91b0211 100644 --- a/landscape/client/manager/tests/test_service.py +++ b/landscape/client/manager/tests/test_service.py @@ -1,9 +1,10 @@ -from landscape.client.tests.helpers import ( - LandscapeTest, FakeBrokerServiceHelper) -from landscape.lib.testing import FakeReactor -from landscape.client.manager.config import ManagerConfiguration, ALL_PLUGINS -from landscape.client.manager.service import ManagerService +from landscape.client.manager.config import ALL_PLUGINS +from landscape.client.manager.config import ManagerConfiguration from landscape.client.manager.processkiller import ProcessKiller +from landscape.client.manager.service import ManagerService +from landscape.client.tests.helpers import FakeBrokerServiceHelper +from landscape.client.tests.helpers import LandscapeTest +from landscape.lib.testing import FakeReactor class ManagerServiceTest(LandscapeTest): @@ -11,7 +12,7 @@ class ManagerServiceTest(LandscapeTest): helpers = [FakeBrokerServiceHelper] def setUp(self): - super(ManagerServiceTest, self).setUp() + super().setUp() config = ManagerConfiguration() config.load(["-c", self.config_filename]) @@ -41,6 +42,7 @@ class ManagerServiceTest(LandscapeTest): The L{ManagerService.startService} method connects to the broker, starts the plugins and register the manager as broker client. """ + def stop_service(ignored): for plugin in self.service.plugins: if getattr(plugin, "stop", None) is not None: diff --git a/landscape/client/manager/tests/test_shutdownmanager.py b/landscape/client/manager/tests/test_shutdownmanager.py index 54ff4f0..4d2a28e 100644 --- a/landscape/client/manager/tests/test_shutdownmanager.py +++ b/landscape/client/manager/tests/test_shutdownmanager.py @@ -1,12 +1,15 @@ -from twisted.python.failure import Failure -from twisted.internet.error import ProcessTerminated, ProcessDone +from twisted.internet.error import ProcessDone +from twisted.internet.error import ProcessTerminated from twisted.internet.protocol import ProcessProtocol +from twisted.python.failure import Failure +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import SUCCEEDED +from landscape.client.manager.shutdownmanager import ShutdownManager +from landscape.client.manager.shutdownmanager import ShutdownProcessProtocol +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper from landscape.lib.testing import StubProcessFactory -from landscape.client.manager.plugin import SUCCEEDED, FAILED -from landscape.client.manager.shutdownmanager import ( - ShutdownManager, ShutdownProcessProtocol) -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper class ShutdownManagerTest(LandscapeTest): @@ -14,9 +17,10 @@ class ShutdownManagerTest(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(ShutdownManagerTest, self).setUp() + super().setUp() self.broker_service.message_store.set_accepted_types( - ["shutdown", "operation-result"]) + ["shutdown", "operation-result"], + ) self.broker_service.pinger.start() self.process_factory = StubProcessFactory() self.plugin = ShutdownManager(process_factory=self.process_factory) @@ -37,16 +41,32 @@ class ShutdownManagerTest(LandscapeTest): self.assertTrue(isinstance(protocol, ShutdownProcessProtocol)) self.assertEqual( arguments[1:3], - ("/sbin/shutdown", ["/sbin/shutdown", "-r", "+4", - "Landscape is rebooting the system"])) + ( + "/sbin/shutdown", + [ + "/sbin/shutdown", + "-r", + "+4", + "Landscape is rebooting the system", + ], + ), + ) def restart_performed(ignore): self.assertTrue(self.broker_service.exchanger.is_urgent()) self.assertEqual( self.broker_service.message_store.get_pending_messages(), - [{"type": "operation-result", "api": b"3.2", - "operation-id": 100, "timestamp": 10, "status": SUCCEEDED, - "result-text": u"Data may arrive in batches."}]) + [ + { + "type": "operation-result", + "api": b"3.2", + "operation-id": 100, + "timestamp": 10, + "status": SUCCEEDED, + "result-text": "Data may arrive in batches.", + }, + ], + ) protocol.result.addCallback(restart_performed) protocol.childDataReceived(0, b"Data may arrive ") @@ -68,8 +88,16 @@ class ShutdownManagerTest(LandscapeTest): [arguments] = self.process_factory.spawns self.assertEqual( arguments[1:3], - ("/sbin/shutdown", ["/sbin/shutdown", "-h", "+4", - "Landscape is shutting down the system"])) + ( + "/sbin/shutdown", + [ + "/sbin/shutdown", + "-h", + "+4", + "Landscape is shutting down the system", + ], + ), + ) def test_restart_fails(self): """ @@ -90,15 +118,17 @@ class ShutdownManagerTest(LandscapeTest): self.assertEqual(message["operation-id"], 100) self.assertEqual(message["timestamp"], 0) self.assertEqual(message["status"], FAILED) - self.assertIn(u"Failure text is reported.", message["result-text"]) + self.assertIn("Failure text is reported.", message["result-text"]) # Check that after failing, we attempt to force the shutdown by # switching the binary called [spawn1_args, spawn2_args] = self.process_factory.spawns protocol = spawn2_args[0] self.assertIsInstance(protocol, ProcessProtocol) - self.assertEqual(spawn2_args[1:3], - ("/sbin/reboot", ["/sbin/reboot"])) + self.assertEqual( + spawn2_args[1:3], + ("/sbin/reboot", ["/sbin/reboot"]), + ) [arguments] = self.process_factory.spawns protocol = arguments[0] diff --git a/landscape/client/manager/tests/test_snapmanager.py b/landscape/client/manager/tests/test_snapmanager.py new file mode 100644 index 0000000..1ce52f1 --- /dev/null +++ b/landscape/client/manager/tests/test_snapmanager.py @@ -0,0 +1,259 @@ +from unittest import mock + +from landscape.client.manager.manager import FAILED +from landscape.client.manager.manager import SUCCEEDED +from landscape.client.manager.snapmanager import SnapManager +from landscape.client.snap.http import SnapdHttpException +from landscape.client.snap.http import SnapHttp as OrigSnapHttp +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper + + +class SnapManagerTest(LandscapeTest): + helpers = [ManagerHelper] + + def setUp(self): + super().setUp() + + self.snap_http = mock.Mock(spec_set=OrigSnapHttp) + self.SnapHttp = mock.patch( + "landscape.client.manager.snapmanager.SnapHttp", + ).start() + + self.SnapHttp.return_value = self.snap_http + + self.broker_service.message_store.set_accepted_types( + ["operation-result"], + ) + self.plugin = SnapManager() + self.manager.add(self.plugin) + + self.manager.config.snapd_poll_attempts = 2 + self.manager.config.snapd_poll_interval = 0.1 + + def tearDown(self): + mock.patch.stopall() + + def test_install_snaps(self): + """ + When at least one channel or revision is specified, snaps are + installed via one call to snapd per snap. + """ + + def install_snap(name, revision=None, channel=None, classic=False): + if name == "hello": + return {"change": "1"} + + if name == "goodbye": + return {"change": "2"} + + return mock.DEFAULT + + self.snap_http.install_snap.side_effect = install_snap + self.snap_http.check_changes.return_value = { + "result": [ + {"id": "1", "status": "Done"}, + {"id": "2", "status": "Done"}, + ], + } + self.snap_http.get_snaps.return_value = {"installed": []} + + result = self.manager.dispatch_message( + { + "type": "install-snaps", + "operation-id": 123, + "snaps": [ + {"name": "hello", "revision": 9001}, + {"name": "goodbye"}, + ], + }, + ) + + def got_result(r): + self.assertMessages( + self.broker_service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "result-text": "{'completed': ['hello', 'goodbye'], " + "'errored': [], 'errors': {}}", + "operation-id": 123, + }, + ], + ) + + return result.addCallback(got_result) + + def test_install_snaps_batch(self): + """ + When no channels or revisions are specified, snaps are installed + via a single call to snapd. + """ + self.snap_http.install_snaps.return_value = {"change": "1"} + self.snap_http.check_changes.return_value = { + "result": [{"id": "1", "status": "Done"}], + } + self.snap_http.get_snaps.return_value = { + "installed": [ + { + "name": "hello", + "id": "test", + "confinement": "strict", + "tracking-channel": "latest/stable", + "revision": "100", + "publisher": {"validation": "yep", "username": "me"}, + "version": "1.2.3", + }, + ], + } + + result = self.manager.dispatch_message( + { + "type": "install-snaps", + "operation-id": 123, + "snaps": [ + {"name": "hello"}, + {"name": "goodbye"}, + ], + }, + ) + + def got_result(r): + self.assertMessages( + self.broker_service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "result-text": "{'completed': ['BATCH'], 'errored': " + "[], 'errors': {}}", + "operation-id": 123, + }, + ], + ) + + return result.addCallback(got_result) + + def test_install_snap_immediate_error(self): + self.snap_http.install_snaps.side_effect = SnapdHttpException( + b'{"result": "whoops"}', + ) + self.snap_http.get_snaps.return_value = {"installed": []} + + result = self.manager.dispatch_message( + { + "type": "install-snaps", + "operation-id": 123, + "snaps": [{"name": "hello"}], + }, + ) + + self.log_helper.ignore_errors(r".+whoops$") + + def got_result(r): + self.assertMessages( + self.broker_service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": FAILED, + "result-text": "{'completed': [], 'errored': [], " + "'errors': {'BATCH': 'whoops'}}", + "operation-id": 123, + }, + ], + ) + + return result.addCallback(got_result) + + def test_install_snap_no_status(self): + self.snap_http.install_snaps.return_value = {"change": "1"} + self.snap_http.check_changes.return_value = {"result": []} + self.snap_http.get_snaps.return_value = {"installed": []} + + result = self.manager.dispatch_message( + { + "type": "install-snaps", + "operation-id": 123, + "snaps": [{"name": "hello"}], + }, + ) + + def got_result(r): + self.assertMessages( + self.broker_service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": FAILED, + "result-text": "{'completed': [], 'errored': ['BATCH']" + ", 'errors': {'BATCH': 'Unknown'}}", + "operation-id": 123, + }, + ], + ) + + return result.addCallback(got_result) + + def test_install_snap_check_error(self): + self.snap_http.install_snaps.return_value = {"change": "1"} + self.snap_http.check_changes.side_effect = SnapdHttpException("whoops") + self.snap_http.get_snaps.return_value = {"installed": []} + + result = self.manager.dispatch_message( + { + "type": "install-snaps", + "operation-id": 123, + "snaps": [{"name": "hello"}], + }, + ) + + self.log_helper.ignore_errors(r".+whoops$") + + def got_result(r): + self.assertMessages( + self.broker_service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": FAILED, + "result-text": "{'completed': [], 'errored': ['BATCH']" + ", 'errors': {'BATCH': 'whoops'}}", + "operation-id": 123, + }, + ], + ) + + return result.addCallback(got_result) + + def test_remove_snap(self): + self.snap_http.remove_snaps.return_value = {"change": "1"} + self.snap_http.check_changes.return_value = { + "result": [{"id": "1", "status": "Done"}], + } + self.snap_http.get_snaps.return_value = {"installed": []} + + result = self.manager.dispatch_message( + { + "type": "remove-snaps", + "operation-id": 123, + "snaps": [{"name": "hello"}], + }, + ) + + def got_result(r): + self.assertMessages( + self.broker_service.message_store.get_pending_messages(), + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "result-text": "{'completed': ['BATCH'], 'errored': []" + ", 'errors': {}}", + "operation-id": 123, + }, + ], + ) + + return result.addCallback(got_result) diff --git a/landscape/client/manager/tests/test_store.py b/landscape/client/manager/tests/test_store.py index 2856818..49907d7 100644 --- a/landscape/client/manager/tests/test_store.py +++ b/landscape/client/manager/tests/test_store.py @@ -1,15 +1,13 @@ -from landscape.client.tests.helpers import LandscapeTest - from landscape.client.manager.store import ManagerStore +from landscape.client.tests.helpers import LandscapeTest class ManagerStoreTest(LandscapeTest): - def setUp(self): - super(ManagerStoreTest, self).setUp() + super().setUp() self.filename = self.makeFile() self.store = ManagerStore(self.filename) - self.store.add_graph(1, u"file 1", u"user1") + self.store.add_graph(1, "file 1", "user1") self.store.set_graph_accumulate(1, 1234, 1.0) def test_get_unknown_graph(self): @@ -18,11 +16,11 @@ class ManagerStoreTest(LandscapeTest): def test_get_graph(self): graph = self.store.get_graph(1) - self.assertEqual(graph, (1, u"file 1", u"user1")) + self.assertEqual(graph, (1, "file 1", "user1")) def test_get_graphs(self): graphs = self.store.get_graphs() - self.assertEqual(graphs, [(1, u"file 1", u"user1")]) + self.assertEqual(graphs, [(1, "file 1", "user1")]) def test_get_no_graphs(self): self.store.remove_graph(1) @@ -30,14 +28,14 @@ class ManagerStoreTest(LandscapeTest): self.assertEqual(graphs, []) def test_add_graph(self): - self.store.add_graph(2, u"file 2", u"user2") + self.store.add_graph(2, "file 2", "user2") graph = self.store.get_graph(2) - self.assertEqual(graph, (2, u"file 2", u"user2")) + self.assertEqual(graph, (2, "file 2", "user2")) def test_add_update_graph(self): - self.store.add_graph(1, u"file 2", u"user2") + self.store.add_graph(1, "file 2", "user2") graph = self.store.get_graph(1) - self.assertEqual(graph, (1, u"file 2", u"user2")) + self.assertEqual(graph, (1, "file 2", "user2")) def test_remove_graph(self): self.store.remove_graph(1) @@ -47,7 +45,7 @@ class ManagerStoreTest(LandscapeTest): def test_remove_unknown_graph(self): self.store.remove_graph(2) graphs = self.store.get_graphs() - self.assertEqual(graphs, [(1, u"file 1", u"user1")]) + self.assertEqual(graphs, [(1, "file 1", "user1")]) def test_get_accumulate_unknown_graph(self): accumulate = self.store.get_graph_accumulate(2) diff --git a/landscape/client/manager/tests/test_usermanager.py b/landscape/client/manager/tests/test_usermanager.py index e3c154b..fe02d03 100644 --- a/landscape/client/manager/tests/test_usermanager.py +++ b/landscape/client/manager/tests/test_usermanager.py @@ -1,17 +1,18 @@ -# -*- coding: utf-8 -*- import os -from mock import Mock +from unittest.mock import Mock -from landscape.lib.persist import Persist -from landscape.lib.twisted_util import gather_results -from landscape.client.manager.plugin import SUCCEEDED, FAILED +from landscape.client.manager.plugin import FAILED +from landscape.client.manager.plugin import SUCCEEDED +from landscape.client.manager.usermanager import RemoteUserManagerConnector +from landscape.client.manager.usermanager import UserManager from landscape.client.monitor.usermonitor import UserMonitor -from landscape.client.manager.usermanager import ( - UserManager, RemoteUserManagerConnector) -from landscape.client.user.tests.helpers import ( - FakeUserProvider, FakeUserManagement) -from landscape.client.tests.helpers import LandscapeTest, ManagerHelper +from landscape.client.tests.helpers import LandscapeTest +from landscape.client.tests.helpers import ManagerHelper from landscape.client.user.provider import UserManagementError +from landscape.client.user.tests.helpers import FakeUserManagement +from landscape.client.user.tests.helpers import FakeUserProvider +from landscape.lib.persist import Persist +from landscape.lib.twisted_util import gather_results class UserGroupTestBase(LandscapeTest): @@ -19,27 +20,34 @@ class UserGroupTestBase(LandscapeTest): helpers = [ManagerHelper] def setUp(self): - super(UserGroupTestBase, self).setUp() - self.shadow_file = self.makeFile("""\ + super().setUp() + self.shadow_file = self.makeFile( + """\ jdoe:$1$xFlQvTqe$cBtrNEDOIKMy/BuJoUdeG0:13348:0:99999:7::: psmith:!:13348:0:99999:7::: sbarnes:$1$q7sz09uw$q.A3526M/SHu8vUb.Jo1A/:13349:0:99999:7::: -""") +""", + ) accepted_types = ["operation-result", "users"] self.broker_service.message_store.set_accepted_types(accepted_types) def tearDown(self): - super(UserGroupTestBase, self).tearDown() + super().tearDown() for plugin in self.plugins: plugin.stop() def setup_environment(self, users, groups, shadow_file): - provider = FakeUserProvider(users=users, groups=groups, - shadow_file=shadow_file) + provider = FakeUserProvider( + users=users, + groups=groups, + shadow_file=shadow_file, + ) user_monitor = UserMonitor(provider=provider) management = FakeUserManagement(provider=provider) - user_manager = UserManager(management=management, - shadow_file=shadow_file) + user_manager = UserManager( + management=management, + shadow_file=shadow_file, + ) self.manager.persist = Persist() user_monitor.register(self.manager) user_manager.register(self.manager) @@ -48,7 +56,6 @@ sbarnes:$1$q7sz09uw$q.A3526M/SHu8vUb.Jo1A/:13349:0:99999:7::: class UserOperationsMessagingTest(UserGroupTestBase): - def test_add_user_event(self): """ When an C{add-user} event is received the user should be @@ -59,30 +66,52 @@ class UserOperationsMessagingTest(UserGroupTestBase): def handle_callback(result): messages = self.broker_service.message_store.get_pending_messages() - self.assertMessages(messages, - [{"type": "operation-result", - "status": SUCCEEDED, - "operation-id": 123, "timestamp": 0, - "result-text": "add_user succeeded"}, - {"timestamp": 0, "type": "users", - "operation-id": 123, - "create-users": [{"home-phone": None, - "username": "jdoe", - "uid": 1000, - "enabled": True, - "location": "Room 101", - "work-phone": "+12345", - "name": u"John Doe", - "primary-gid": 1000}]}]) + self.assertMessages( + messages, + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "operation-id": 123, + "timestamp": 0, + "result-text": "add_user succeeded", + }, + { + "timestamp": 0, + "type": "users", + "operation-id": 123, + "create-users": [ + { + "home-phone": None, + "username": "jdoe", + "uid": 1000, + "enabled": True, + "location": "Room 101", + "work-phone": "+12345", + "name": "John Doe", + "primary-gid": 1000, + }, + ], + }, + ], + ) self.setup_environment([], [], None) result = self.manager.dispatch_message( - {"username": "jdoe", "name": "John Doe", "password": "password", - "operation-id": 123, "require-password-reset": False, - "primary-group-name": None, "location": "Room 101", - "work-number": "+12345", "home-number": None, - "type": "add-user"}) + { + "username": "jdoe", + "name": "John Doe", + "password": "password", + "operation-id": 123, + "require-password-reset": False, + "primary-group-name": None, + "location": "Room 101", + "work-number": "+12345", + "home-number": None, + "type": "add-user", + }, + ) result.addCallback(handle_callback) return result @@ -92,32 +121,55 @@ class UserOperationsMessagingTest(UserGroupTestBase): When an C{add-user} event with utf-8 unicode strings is received the user should be added. """ + def handle_callback(result): messages = self.broker_service.message_store.get_pending_messages() - self.assertMessages(messages, - [{"type": "operation-result", - "status": SUCCEEDED, - "operation-id": 123, "timestamp": 0, - "result-text": "add_user succeeded"}, - {"timestamp": 0, "type": "users", - "operation-id": 123, - "create-users": [{"home-phone": None, - "username": "jdoe", - "uid": 1000, - "enabled": True, - "location": "Room 101", - "work-phone": "+12345", - "name": u"請不要刪除", - "primary-gid": 1000}]}]) + self.assertMessages( + messages, + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "operation-id": 123, + "timestamp": 0, + "result-text": "add_user succeeded", + }, + { + "timestamp": 0, + "type": "users", + "operation-id": 123, + "create-users": [ + { + "home-phone": None, + "username": "jdoe", + "uid": 1000, + "enabled": True, + "location": "Room 101", + "work-phone": "+12345", + "name": "請不要刪除", + "primary-gid": 1000, + }, + ], + }, + ], + ) self.setup_environment([], [], None) result = self.manager.dispatch_message( - {"username": "jdoe", "name": "請不要刪除", "password": "password", - "operation-id": 123, "require-password-reset": False, - "primary-group-name": None, "location": "Room 101", - "work-number": "+12345", "home-number": None, - "type": "add-user"}) + { + "username": "jdoe", + "name": "請不要刪除", + "password": "password", + "operation-id": 123, + "require-password-reset": False, + "primary-group-name": None, + "location": "Room 101", + "work-number": "+12345", + "home-number": None, + "type": "add-user", + }, + ) result.addCallback(handle_callback) return result @@ -128,38 +180,55 @@ class UserOperationsMessagingTest(UserGroupTestBase): received the user should be added. This is what the server is sending over the wire in the real-world. """ + def handle_callback(result): messages = self.broker_service.message_store.get_pending_messages() - self.assertMessages(messages, - [{"type": "operation-result", - "status": SUCCEEDED, - "operation-id": 123, "timestamp": 0, - "result-text": "add_user succeeded"}, - {"timestamp": 0, "type": "users", - "operation-id": 123, - "create-users": [ - {"home-phone": u"請不要刪除", - "username": u"請不要刪除", - "uid": 1000, - "enabled": True, - "location": u"請不要刪除", - "work-phone": u"請不要刪除", - "name": u"請不要刪除", - "primary-gid": 1000}]}]) + self.assertMessages( + messages, + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "operation-id": 123, + "timestamp": 0, + "result-text": "add_user succeeded", + }, + { + "timestamp": 0, + "type": "users", + "operation-id": 123, + "create-users": [ + { + "home-phone": "請不要刪除", + "username": "請不要刪除", + "uid": 1000, + "enabled": True, + "location": "請不要刪除", + "work-phone": "請不要刪除", + "name": "請不要刪除", + "primary-gid": 1000, + }, + ], + }, + ], + ) self.setup_environment([], [], None) result = self.manager.dispatch_message( - {'username': u'\u8acb\u4e0d\u8981\u522a\u9664', - 'work-number': u'\u8acb\u4e0d\u8981\u522a\u9664', - 'home-number': u'\u8acb\u4e0d\u8981\u522a\u9664', - 'name': u'\u8acb\u4e0d\u8981\u522a\u9664', - 'operation-id': 123, - 'require-password-reset': False, - 'password': u'\u8acb\u4e0d\u8981\u522a\u9664', - 'type': 'add-user', - 'primary-group-name': u'\u8acb\u4e0d\u8981\u522a\u9664', - 'location': u'\u8acb\u4e0d\u8981\u522a\u9664'}) + { + "username": "\u8acb\u4e0d\u8981\u522a\u9664", + "work-number": "\u8acb\u4e0d\u8981\u522a\u9664", + "home-number": "\u8acb\u4e0d\u8981\u522a\u9664", + "name": "\u8acb\u4e0d\u8981\u522a\u9664", + "operation-id": 123, + "require-password-reset": False, + "password": "\u8acb\u4e0d\u8981\u522a\u9664", + "type": "add-user", + "primary-group-name": "\u8acb\u4e0d\u8981\u522a\u9664", + "location": "\u8acb\u4e0d\u8981\u522a\u9664", + }, + ) result.addCallback(handle_callback) return result @@ -174,16 +243,30 @@ class UserOperationsMessagingTest(UserGroupTestBase): def handle_callback(result): messages = self.broker_service.message_store.get_pending_messages() - self.assertMessages(messages, - [{"type": "operation-result", "status": FAILED, - "operation-id": 123, "timestamp": 0, - "result-text": "KeyError: 'username'"}]) + self.assertMessages( + messages, + [ + { + "type": "operation-result", + "status": FAILED, + "operation-id": 123, + "timestamp": 0, + "result-text": "KeyError: 'username'", + }, + ], + ) self.setup_environment([], [], None) result = self.manager.dispatch_message( - {"name": "John Doe", "password": "password", "operation-id": 123, - "require-password-reset": False, "type": "add-user"}) + { + "name": "John Doe", + "password": "password", + "operation-id": 123, + "require-password-reset": False, + "type": "add-user", + }, + ) result.addCallback(handle_callback) return result @@ -210,10 +293,19 @@ class UserOperationsMessagingTest(UserGroupTestBase): plugin = self.setup_environment([], [], None) result = self.manager.dispatch_message( - {"username": "jdoe", "name": "John Doe", "password": "password", - "operation-id": 123, "require-password-reset": False, - "primary-group-name": None, "type": "add-user", - "location": None, "home-number": "+123456", "work-number": None}) + { + "username": "jdoe", + "name": "John Doe", + "password": "password", + "operation-id": 123, + "require-password-reset": False, + "primary-group-name": None, + "type": "add-user", + "location": None, + "home-number": "+123456", + "work-number": None, + }, + ) result.addCallback(handle_callback1) return result @@ -230,33 +322,59 @@ class UserOperationsMessagingTest(UserGroupTestBase): messages = self.broker_service.message_store.get_pending_messages() self.assertEqual(len(messages), 3) messages = [messages[0], messages[2]] - self.assertMessages(messages, - [{"type": "users", - "create-users": [{"home-phone": None, - "name": "Bo", - "username": "bo", - "uid": 1000, - "enabled": True, - "location": None, - "primary-gid": 1000, - "work-phone": None}]}, - {"type": "users", "operation-id": 123, - "create-users": [{"home-phone": "+123456", - "username": "jdoe", - "uid": 1001, - "enabled": True, - "location": None, - "work-phone": None, - "name": "John Doe", - "primary-gid": 1001}]}]) + self.assertMessages( + messages, + [ + { + "type": "users", + "create-users": [ + { + "home-phone": None, + "name": "Bo", + "username": "bo", + "uid": 1000, + "enabled": True, + "location": None, + "primary-gid": 1000, + "work-phone": None, + }, + ], + }, + { + "type": "users", + "operation-id": 123, + "create-users": [ + { + "home-phone": "+123456", + "username": "jdoe", + "uid": 1001, + "enabled": True, + "location": None, + "work-phone": None, + "name": "John Doe", + "primary-gid": 1001, + }, + ], + }, + ], + ) users = [("bo", "x", 1000, 1000, "Bo,,,,", "/home/bo", "/bin/zsh")] self.setup_environment(users, [], None) result = self.manager.dispatch_message( - {"username": "jdoe", "name": "John Doe", "password": "password", - "operation-id": 123, "require-password-reset": False, - "type": "add-user", "primary-group-name": None, - "location": None, "work-number": None, "home-number": "+123456"}) + { + "username": "jdoe", + "name": "John Doe", + "password": "password", + "operation-id": 123, + "require-password-reset": False, + "type": "add-user", + "primary-group-name": None, + "location": None, + "work-number": None, + "home-number": "+123456", + }, + ) result.addCallback(handle_callback) return result @@ -272,32 +390,55 @@ class UserOperationsMessagingTest(UserGroupTestBase): messages = self.broker_service.message_store.get_pending_messages() self.assertEqual(len(messages), 3) # Ignore the message created by plugin.run. - self.assertMessages(messages[1:], - [{"type": "operation-result", - "status": SUCCEEDED, - "operation-id": 99, "timestamp": 0, - "result-text": "set_user_details succeeded"}, - {"update-users": [{"username": "jdoe", - "uid": 1001, - "enabled": True, - "work-phone": "789WORK", - "home-phone": "123HOME", - "location": "Everywhere", - "name": "John Doe", - "primary-gid": 1001}], - "timestamp": 0, "type": "users", - "operation-id": 99}]) - - users = [("jdoe", "x", 1001, 1000, "John Doe,,,,", - "/home/bo", "/bin/zsh")] + self.assertMessages( + messages[1:], + [ + { + "type": "operation-result", + "status": SUCCEEDED, + "operation-id": 99, + "timestamp": 0, + "result-text": "set_user_details succeeded", + }, + { + "update-users": [ + { + "username": "jdoe", + "uid": 1001, + "enabled": True, + "work-phone": "789WORK", + "home-phone": "123HOME", + "location": "Everywhere", + "name": "John Doe", + "primary-gid": 1001, + }, + ], + "timestamp": 0, + "type": "users", + "operation-id": 99, + }, + ], + ) + + users = [ + ("jdoe", "x", 1001, 1000, "John Doe,,,,", "/home/bo", "/bin/zsh"), + ] groups = [("users", "x", 1001, [])] self.setup_environment(users, groups, None) result = self.manager.dispatch_message( - {"uid": 1001, "username": "jdoe", "password": "password", - "name": "John Doe", "location": "Everywhere", - "work-number": "789WORK", "home-number": "123HOME", - "operation-id": 99, "primary-group-name": u"users", - "type": "edit-user"}) + { + "uid": 1001, + "username": "jdoe", + "password": "password", + "name": "John Doe", + "location": "Everywhere", + "work-number": "789WORK", + "home-number": "123HOME", + "operation-id": 99, + "primary-group-name": "users", + "type": "edit-user", + }, + ) result.addCallback(handle_callback) return result @@ -321,14 +462,23 @@ class UserOperationsMessagingTest(UserGroupTestBase): self.assertEqual(messages, new_messages) return result - users = [("jdoe", "x", 1000, 1000, "John Doe,,,,", - "/home/bo", "/bin/zsh")] + users = [ + ("jdoe", "x", 1000, 1000, "John Doe,,,,", "/home/bo", "/bin/zsh"), + ] plugin = self.setup_environment(users, [], None) result = self.manager.dispatch_message( - {"username": "jdoe", "password": "password", "name": "John Doe", - "location": "Everywhere", "work-number": "789WORK", - "home-number": "123HOME", "primary-group-name": None, - "type": "edit-user", "operation-id": 99}) + { + "username": "jdoe", + "password": "password", + "name": "John Doe", + "location": "Everywhere", + "work-number": "789WORK", + "home-number": "123HOME", + "primary-group-name": None, + "type": "edit-user", + "operation-id": 99, + }, + ) result.addCallback(handle_callback1) return result @@ -343,39 +493,63 @@ class UserOperationsMessagingTest(UserGroupTestBase): def handle_callback(result): messages = self.broker_service.message_store.get_pending_messages() self.assertEqual(len(messages), 3) - self.assertMessages([messages[0], messages[2]], - [{"type": "users", - "create-group-members": {u"users": - [u"jdoe"]}, - "create-groups": [{"gid": 1001, - "name": u"users"}], - "create-users": [{"home-phone": None, - "work-phone": None, - "username": "jdoe", - "uid": 1000, - "enabled": True, - "location": None, - "name": "John Doe", - "primary-gid": 1000}]}, - {"type": "users", "operation-id": 99, - "update-users": [{"username": "jdoe", - "uid": 1000, - "enabled": True, - "work-phone": "789WORK", - "home-phone": "123HOME", - "location": "Everywhere", - "primary-gid": 1001, - "name": "John Doe"}]}]) - - users = [("jdoe", "x", 1000, 1000, "John Doe,,,,", "/home/bo", - "/bin/zsh")] + self.assertMessages( + [messages[0], messages[2]], + [ + { + "type": "users", + "create-group-members": {"users": ["jdoe"]}, + "create-groups": [{"gid": 1001, "name": "users"}], + "create-users": [ + { + "home-phone": None, + "work-phone": None, + "username": "jdoe", + "uid": 1000, + "enabled": True, + "location": None, + "name": "John Doe", + "primary-gid": 1000, + }, + ], + }, + { + "type": "users", + "operation-id": 99, + "update-users": [ + { + "username": "jdoe", + "uid": 1000, + "enabled": True, + "work-phone": "789WORK", + "home-phone": "123HOME", + "location": "Everywhere", + "primary-gid": 1001, + "name": "John Doe", + }, + ], + }, + ], + ) + + users = [ + ("jdoe", "x", 1000, 1000, "John Doe,,,,", "/home/bo", "/bin/zsh"), + ] groups = [("users", "x", 1001, ["jdoe"])] self.setup_environment(users, groups, None) result = self.manager.dispatch_message( - {"username": "jdoe", "password": "password", "name": "John Doe", - "location": "Everywhere", "work-number": "789WORK", - "home-number": "123HOME", "primary-group-name": u"users", - "type": "edit-user", "operation-id": 99}) + { + "username": "jdoe", + "password": "password", + "name": "John Doe", + "location": "Everywhere", + "work-number": "789WORK", + "home-number": "123HOME", + "primary-group-name": "users", + "type": "edit-user", + "operation-id": 99, + }, + ) result.addCallback(handle_callback) return result @@ -393,22 +567,37 @@ class UserOperationsMessagingTest(UserGroupTestBase): messages = self.broker_service.message_store.get_pending_messages() self.assertEqual(len(messages), 3) # Ignore the message created by plugin.run. - self.assertMessages([messages[2], messages[1]], - [{"timestamp": 0, "delete-users": ["jdoe"], - "type": "users", "operation-id": 39}, - {"type": "operation-result", - "status": SUCCEEDED, - "operation-id": 39, "timestamp": 0, - "result-text": "remove_user succeeded"}]) - - users = [("jdoe", "x", 1000, 1000, "John Doe,,,,", "/home/bo", - "/bin/zsh")] + self.assertMessages( + [mess