chrono24Scraper/crontab.py at main · CodeOfRob/chrono24Scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
import json

import bs4
import requests

from app import (DB_URI, TG_API_KEY, TG_CHAT_ID, URL, create_watchTimestamp,
                 extract_prices)
from DBClient import DBClient
from telegram.telegram import TelegramService
from WatchOffer import WatchOffer


def fetch_timestamps(db, tg, doc):
    # extract watchTimestamps
    [prices, currency] = extract_prices(doc)

    # create watchTimestamp
    watchTimestamp = create_watchTimestamp(prices, currency)

    # push data to database
    db.push_watchTimestamp(watchTimestamp)

    # print success
    print("successful upload:", watchTimestamp)
    tg.send_msg(f"new watchtimestamp:\n {watchTimestamp}")

def fetch_offers(db, tg, doc):

    soup = bs4.BeautifulSoup(doc, "lxml")
    offers_container = json.loads(soup.find("script", type="application/ld+json").contents[0])
    aggregate_offers = offers_container["@graph"][1]
    currency = aggregate_offers["priceCurrency"]
    offers = aggregate_offers["offers"]

    db.drop_watchOfferSnapshots()

    count = 0
    for offer in offers:
        new_offer = WatchOffer(offer["url"], offer["name"], float(offer["price"]),currency)
        db.push_watchOfferSnapshot(new_offer)
        count += 1

    print("pushed", count, "new offer snapshots")
    tg.send_msg(f"pushed {count} new offer snapshots")


def main():

    db = DBClient(DB_URI)

    tg = TelegramService(TG_API_KEY, TG_CHAT_ID)

    res = None
    try:
        res = requests.get(URL)
    except Exception as e:
        print("Error while fetching website content")
        tg.send_msg("WTs:\nError while fetching website content:\n{e}")

    if not res or not res.ok:
        return

    doc = res.content

    fetch_timestamps(db, tg, doc)

    fetch_offers(db, tg, doc)

if __name__ == "__main__":
    main()