WebCrawler/module.py at master · UtkucanBykl/WebCrawler · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
#!/usr/bin/env python
# -*- coding: utf-8 -*-
import re
import bs4 as bs
import mechanize
import sys
import urllib2
reload(sys)
sys.setdefaultencoding('utf-8')

class WebCrawler():
    def __init__(self):

        self.b = mechanize.Browser()
        self.b.set_handle_refresh(False)
        self.b.set_handle_robots(False)
        self.b.addheaders = [('User-agent',
                     "Mozilla/5.0 (Windows NT 6.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/41.0.2228.0 Safari/537.36")]

    def FindLinks(self,URL):

        try:
            self.b.open(URL, timeout=2.0)
        except mechanize.HTTPError,e:
            return e
        links=[]
        for link in self.b.links():
            if(link.text != ""):
                links.append(str(link.text)+"-"+str(link.url))

        if not links:
            return False
        else:
            return links

    def WebCrawlerSearch(self,URL,inputName,search):

        try:
            self.b.open(URL, timeout=2.0)
        except mechanize.HTTPError, e:
            return e
        self.b.select_form(nr=0)
        self.b.form[inputName] = search
        self.b.method = "post"
        response = self.b.submit()
        return response

    def CrawleWithId(self,response,tag,id):
        find=[]
        response=urllib2.urlopen(response).read()
        soup = bs.BeautifulSoup(response, "lxml")
        for search in soup.find_all(tag, id=re.compile("^"+id+"$")):
            if(search.text !=""):
                find.append(search.text)
        if not find:
            return False
        else:
            return find

    def CrawleWithClass(self,response,tag,class_):
        find = []
        try:
            response=urllib2.urlopen(response).read()
            soup = bs.BeautifulSoup(response, "lxml")
        except mechanize.HTTPError, e:
            return e

        for search in soup.find_all(tag,class_=class_):
            if(search.text !=""):
                find.append(search.text)
        if not find:
            return False
        else:
            return find