web-scraping-ML/scraping-code.py at master · Mario-RJunior/web-scraping-ML · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
import requests
from bs4 import BeautifulSoup
import re
import numpy as np
from time import sleep
import pandas as pd

zonas = ['norte', 'sul', 'leste', 'oeste']
paginas = ['', '_Desde_49', '_Desde_97', '_Desde_145', '_Desde_193', '_Desde_241', '_Desde_289']
url_ml = 'https://imoveis.mercadolivre.com.br/casas/aluguel/sao-paulo/sao-paulo-zona-{}/{}'

re_precos = r'<span class="price-tag-fraction">(.*)<'
re_quartos = r'([1-9]{1,2}) quarto[s]?'
re_areas = r'<li class="ui-search-card-attributes__attribute">(.*) m²'


class Scraper:
    zonas = []
    areas = []
    quartos = []
    precos = []
    dados = {}

    def __init__(self, url, zona, pagina):

        self.zona = zona
        self.url = url.format(zona, pagina)

    def get_atributes(self):

        c = requests.get(self.url).content
        soup = BeautifulSoup(c, 'html.parser')

        for dado in soup.select('.ui-search-result__content-wrapper'):
            preco = dado.select_one('.price-tag-fraction')
            metros = dado.select_one('.ui-search-card-attributes__attribute ')

            precos = re.findall(re_precos, str(preco))
            areas = re.findall(re_areas, str(metros))
            quartos = re.findall(re_quartos, dado.text)

            for p in precos:
                if isinstance(p, str):
                    p = int(p.replace('.', '').replace(',', '').strip())
                    self.precos.append(p)

            if len(areas) == 0:
                areas.append(np.nan)
            for a in areas:
                if isinstance(a, str):
                    a = int(a.replace('.', '').replace(',', '').strip())
                self.areas.append(a)

            if len(quartos) == 0:
                quartos.append(np.nan)
            for q in quartos:
                self.quartos.append(q)

            self.zonas.append(self.zona)

            self.dados['zona'] = self.zonas
            self.dados['quartos'] = self.quartos
            self.dados['area'] = self.areas
            self.dados['preco'] = self.precos

        sleep(2)
        return self.dados

    def create_csv(self):
        df = pd.DataFrame(self.dados)
        df.to_csv('dados_imoveis.csv', index=False)


if __name__ == '__main__':
    for zona in zonas:
        for pagina in paginas:
            sleep(2)
            scrap = Scraper(url_ml, zona, pagina)
            scrap.get_atributes()
            scrap.create_csv()