VideoGamesInstruct/ReviewCollector.py at master · LastShekel/VideoGamesInstruct · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
from abc import ABC

import requests
from bs4 import BeautifulSoup as BS
from bs4.element import Tag, NavigableString
import re
import time
from typing import Union


class ReviewCollector(ABC):

    def __init__(self, headers=None, host=None):
        self.games = None
        if not headers:
            self.headers = {
                'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
                'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/97.0.4692.99 Safari/537.36'}

        if not host:
            self.host = 'https://stopgame.ru'

    def add_text(self, main: str, text: Union[str, NavigableString]):
        separators = [' ', '\n']
        if len(main) == 0:
            return text
        if len(text) == 0:
            return main

        if main[-1] in separators and text[0] in separators:
            return self.add_text(main[:-1], text[1:])
        if main[-1] in separators and not (text[0] in separators):
            return self.add_text(main[:-1], text)
        if not (main[-1] in separators) and text[0] in separators:
            return self.add_text(main, text[1:])
        if not (main[-1] in separators) and not (text[0] in separators):
            return main + ' ' + text

    def filter_text(self, text: Union[Tag, NavigableString, str]):
        result = ''
        if isinstance(text, NavigableString) or isinstance(text, str):
            return text
        if text.attrs.get('class', None):
            if sum(map(lambda c: True if re.match(".*gallery.*", c) else False,
                       text.attrs['class'])):  # skip pictures
                return result
        if isinstance(text, Tag) and text.name == 'abbr':
            return text.attrs['title']
        for part in text:
            addition = ''
            if isinstance(part, NavigableString):
                addition = self.filter_text(part)
            if part.name == 'p':
                addition = self.filter_text(part.contents[0])
            if str(part) == '<br>' or str(part) == '<br/>':
                continue
            if part.name == 'b' or part.name == 'u':
                if len(part.contents) == 0:
                    continue
                addition = self.filter_text(part.contents[-1])
            if part.name == 'i' or part.name == 'em':
                addition = self.filter_text(part)
            if part.name == 'ol' or part.name == 'ul':
                list_counter = 1
                for li in part.contents:
                    addition = str(list_counter) + '. ' + self.filter_text(li)
                    list_counter += 1
            if part.name == 'blockquote':
                addition = '"' + str(part.contents[0]) + '"'
            if part.name == 'span':
                addition = self.filter_text(part.contents[0])
            if part.name == 'strong':
                addition = part.contents[0]
            if part.name == 'sg-spoiler':
                addition = '"' + self.filter_text(part) + '"'
            if part.name == 'abbr':
                if '█' in self.filter_text(part.contents[0]):
                    addition = part.attrs['title']
                elif isinstance(part.contents[0], Tag):
                    addition = self.filter_text(part.contents[0])
                else:
                    addition = part.contents[0]
            if part.name == 'a':
                addition = self.filter_text(part.contents[-1])
            if part.name == 'lite-youtube':
                addition = '\n' + str(part.contents[1]['href']) + '\n'
            if part.name == 'div':
                if hasattr(part, 'attrs') and 'class' in part.attrs:
                    if 'caption' in part.attrs['class'][0]:
                        addition = self.filter_text(part.contents[0])
                    if 'content' in part.attrs['class'][0]:
                        for content in part.contents:
                            result = self.add_text(result,
                                                   self.filter_text(
                                                       content))
                    if 'stop-choice' in part.attrs['class'][0]:
                        addition = 'Наш выбор.'
                    elif 'review-rating' in part.attrs['class'][0]:
                        addition = part.text
                        if 'Оценка игры' in part.contents[0]:
                            addition += ':'
                        else:
                            review_rating = list(
                                filter(lambda x: 'active' in
                                                 ' '.join(x['class']),
                                       part.contents))[0]
                            rus_rating = review_rating.contents[0]['href'][9:]
                            if rus_rating == 'musor':
                                rating = 'мусор'
                            if rus_rating == 'prohodnyak':
                                rating = 'проходняк'
                            if rus_rating == 'pohvalno':
                                rating = 'похвально'
                            if rus_rating == 'izum':
                                rating = 'изумительно'
                            addition = rating
                else:
                    result = self.add_text(result, self.filter_text(part))
            result = self.add_text(result, addition)

        return result + '\n'

    def link_to_review(self, link: str):
        soup = BS(self._get_html(self.host + link).text, 'html.parser')
        content = soup.find('div', class_=re.compile(".*_content_.*"))
        review = ''
        for text in content.contents:
            if isinstance(text, NavigableString):
                review += '\n'
                continue
            if not isinstance(text, Tag):
                continue
            review += self.filter_text(text)
        review = review.replace(' ', ' ')
        review = review.replace('<br/>', '')
        review = review.rstrip()
        review = review.lstrip()
        return review

    def _get_html(self, url: str, params=''):
        while True:
            req = requests.get(url, headers=self.headers, params=params)
            if req.status_code == 200:
                break
            else:
                time.sleep(60)
        return req

    async def get_review_page_content(self, html: str):
        soup = BS(html, 'html.parser')
        # "_default-grid_b9a3a_208" find all reviews on page
        grid = soup.find('div', class_=re.compile(".*default-grid.*"))
        items = []
        for i in grid.descendants:
            if not isinstance(i, Tag):
                continue
            items.append(i)

        games = []
        for item in items:
            article = item.find('article')
            if article is None:
                continue
            href = [a['href'] for a in article.find_all('a')][0]
            review = self.link_to_review(href)
            games.append(
                {
                    'title': article.attrs['aria-label'][7:],
                    'review_link': self.host + href,
                    'review': review
                }
            )
        self.games = games
        return games