vladan/webstat-collector
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Move the kafka consumer to the object

Browse source
This commit is contained in:
Vladan Popovic 2020-09-06 00:56:52 +02:00
parent af6d59ca97
commit 8ca5d145bd
1 changed files with 7 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

16
src/chweb/consumer.py
View file

@ -21,24 +21,22 @@ class Consumer(Service):
queue: asyncio.Queue):
super().__init__(config, logger, event_loop, queue)
self.db = Db(self.loop, self.logger, self.config.postgres)
self.consumer = aiokafka.AIOKafkaConsumer(
self.config.kafka.topic,
loop=self.loop,
bootstrap_servers=self.config.kafka.servers)
async def consume(self):
"""
Consumes messages from a kafka topic and writes them in the database.
"""
consumer = aiokafka.AIOKafkaConsumer(
self.config.kafka.topic,
loop=self.loop,
bootstrap_servers=self.config.kafka.servers)
await consumer.start()
await self.consumer.start()
try:
async with self.db as db:
await db.setup()
async for msg in consumer:
async for msg in self.consumer:
try:
check = Check(**json.loads(msg.value))
self.queue.put_nowait(check)
self.logger.debug(check)
await db.save(check)
except Exception as exc:
@ -47,7 +45,7 @@ class Consumer(Service):
except Exception as exc:
self.logger.error(exc)
finally:
await consumer.stop()
await self.consumer.stop()
def __call__(self) -> asyncio.Future:
return self.consume()