Update docstrings + type annotations in Collector

This commit is contained in:
Vladan Popovic 2020-09-04 19:56:41 +02:00
parent eaf0b51fe9
commit 99785ab69c
2 changed files with 16 additions and 12 deletions

View file

@ -2,7 +2,6 @@
Checks status of web servers and sends them to a configured Kafka topic. Checks status of web servers and sends them to a configured Kafka topic.
""" """
import asyncio import asyncio
import json
import re import re
from typing import Optional from typing import Optional
from urllib.parse import urlparse from urllib.parse import urlparse
@ -11,7 +10,7 @@ import aiokafka # type: ignore
import requests import requests
from chweb.base import Application from chweb.base import Application
from chweb.models import Check from chweb.models import Check, SiteConfig
class Collector(Application): class Collector(Application):
@ -26,7 +25,8 @@ class Collector(Application):
:param url: The URL of the site that needs to be checked. :param url: The URL of the site that needs to be checked.
:param regex: An optional regex to match on the response body. :param regex: An optional regex to match on the response body.
:returns: A dict ready to be sent to the queue for further processing. :returns: ``chweb.models.Check``, sent to a queue in
``chweb.collector.Collector.check_forever``.
""" """
res = await self.loop.run_in_executor(None, requests.get, url) res = await self.loop.run_in_executor(None, requests.get, url)
matches = None # The matches value should be None since the regex can matches = None # The matches value should be None since the regex can
@ -42,12 +42,16 @@ class Collector(Application):
url=url, url=url,
) )
async def check_forever(self, site): async def check_forever(self, site: SiteConfig):
""" """
A void function that gets the status of a site and sends it to an A void function that checks a site and sends the result to an
``asyncio.Queue`` for further processing (sending to a Kafka topic). :class:`asyncio.Queue` for further processing, i.e. sends to a Kafka
topic when it's read from the queue in
:meth:`chweb.collector.Collector.produce` (as in produce data for the
Kafka consumers, defined in :class:`chweb.consumer.Consumer.consume`.
:param site: A site object from the config. :param site: A :py:class:`chweb.models.SiteConfig` object from the
config.
""" """
while True: while True:
try: try:
@ -61,9 +65,9 @@ class Collector(Application):
async def produce(self): async def produce(self):
""" """
Creates and starts an ``aiokafka.AIOKafkaProducer`` and runs a loop that Creates and starts an ``aiokafka.AIOKafkaProducer`` and runs a loop
reads from the ``queue`` and sends the messages to the topic from the that reads from the queue and sends the messages to the topic from the
``config``. config.
""" """
producer = aiokafka.AIOKafkaProducer( producer = aiokafka.AIOKafkaProducer(
loop=self.loop, loop=self.loop,

View file

@ -2,8 +2,8 @@
""" """
Schemas that are used in all modules. This module contains classes for: Schemas that are used in all modules. This module contains classes for:
- Configuring the ``chweb.collector.Collector``. - Configuring the :class:`chweb.collector.Collector`.
- Configuring the ``chweb.consumer.Consumer``. - Configuring the :class:`chweb.consumer.Consumer`.
- The schema for the stats being sent in the Kafka topic. - The schema for the stats being sent in the Kafka topic.
""" """
from datetime import datetime from datetime import datetime