vladan
/
webstat-collector
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Move the kafka consumer to the object

This commit is contained in:
Vladan Popovic 2020-09-06 00:56:52 +02:00
parent af6d59ca97
commit 8ca5d145bd
1 changed files with 7 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

16
src/chweb/consumer.py
View File

@ -21,24 +21,22 @@ class Consumer(Service):
queue: asyncio.Queue): queue: asyncio.Queue):
super().__init__(config, logger, event_loop, queue) super().__init__(config, logger, event_loop, queue)
self.db = Db(self.loop, self.logger, self.config.postgres) self.db = Db(self.loop, self.logger, self.config.postgres)
self.consumer = aiokafka.AIOKafkaConsumer(
self.config.kafka.topic,
loop=self.loop,
bootstrap_servers=self.config.kafka.servers)
async def consume(self): async def consume(self):
""" """
Consumes messages from a kafka topic and writes them in the database. Consumes messages from a kafka topic and writes them in the database.
""" """
consumer = aiokafka.AIOKafkaConsumer( await self.consumer.start()
self.config.kafka.topic,
loop=self.loop,
bootstrap_servers=self.config.kafka.servers)
await consumer.start()
try: try:
async with self.db as db: async with self.db as db:
await db.setup() await db.setup()
async for msg in consumer: async for msg in self.consumer:
try: try:
check = Check(**json.loads(msg.value)) check = Check(**json.loads(msg.value))
self.queue.put_nowait(check)
self.logger.debug(check) self.logger.debug(check)
await db.save(check) await db.save(check)
except Exception as exc: except Exception as exc:
@ -47,7 +45,7 @@ class Consumer(Service):
except Exception as exc: except Exception as exc:
self.logger.error(exc) self.logger.error(exc)
finally: finally:
await consumer.stop() await self.consumer.stop()
def __call__(self) -> asyncio.Future: def __call__(self) -> asyncio.Future:
return self.consume() return self.consume()