From d9b04b94687de8ed4365287ec10de753b642c388 Mon Sep 17 00:00:00 2001 From: Vladan Popovic Date: Fri, 4 Sep 2020 18:21:54 +0200 Subject: [PATCH] Add pydantic models for config and check info --- setup.py | 3 ++- src/webstat/models.py | 62 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 1 deletion(-) create mode 100644 src/webstat/models.py diff --git a/setup.py b/setup.py index c2c71af..ae04a5a 100644 --- a/setup.py +++ b/setup.py @@ -1,5 +1,5 @@ """ -Setup file for the web stats collector. +Setup file for the website checker. """ from setuptools import setup # type: ignore @@ -11,6 +11,7 @@ setup( install_requires=[ 'aiokafka==0.6.0', 'asyncpg==0.21.0', + 'pydantic==1.6.1', 'PyYAML==5.3.1', 'requests==2.24.0', ], diff --git a/src/webstat/models.py b/src/webstat/models.py new file mode 100644 index 0000000..4fa3448 --- /dev/null +++ b/src/webstat/models.py @@ -0,0 +1,62 @@ +# pylint: disable=too-few-public-methods +""" +Schemas that are used in all modules. This module contains classes for: + +- Configuring the ``chweb.collector.Collector``. +- Configuring the ``chweb.consumer.Consumer``. +- The schema for the stats being sent in the Kafka topic. +""" +from datetime import datetime +from typing import List, Optional +from pydantic import BaseModel + + +class Check(BaseModel): + """ + Information for a website check request. + """ + domain: str = "" + regex: Optional[str] = None + regex_matches: Optional[bool] = None + request_time: datetime = datetime.now() + response_time: int = 0 + status: int = 0 + url: str = "" + + +class KafkaConfig(BaseModel): + """ + Kafka broker configuration. + """ + servers: List[str] = ["localhost:9992"] + topic: str = "sample" + + +class PostgresConfig(BaseModel): + """ + PostgreSQL server configuration. + """ + dbhost: str = "localhost" + dbport: int = 5432 + dbname: str = "chweb" + dbuser: str = "vladan" + dbpass: str = "" + + +class SiteConfig(BaseModel): + """ + Single website configuration. + """ + url: str = "https://example.com" + regex: str = "domain" + check_interval: int = 5 + + +class Config(BaseModel): + """ + Main application configuration. Same for the checker and the kafka + consumer / postgres writer for simplicity while deploying. + """ + kafka: KafkaConfig = KafkaConfig() + postgres: PostgresConfig = PostgresConfig() + sites: List[SiteConfig] = []