Qual é a melhor maneira de raspar os dados de um site?

Se você conhece python - I'd recomenda os módulos beautiful soup, splinter, and pandas.

splinter automatiza a entrada e recuperação da página web (ele usa um navegador real e assim pode trabalhar com páginas que precisam rodar javascript), e então beautifulsoup pode ser usado para analisar os dados, então pandas dataframe para escrever os dados no formato csv ou xls.

Código escrito para fazer isso - foi't muito difícil. Você'precisará instalar o python e os módulos pandas, beautifulsoup, e splinter para usá-lo. E você precisa do navegador firefox instalado. Usage is (from command line)

1. python script.py datestart dateend 
2. python script.py 16-Sep-2014 20-Sep-2014 

datestart and dateend are optional

It should run by just doubleclicking if you have your paths setup correct.

There isn't any error handling and other things that would be done in production code, but you get what you pay for :)

It runs a bit slow since I've added long delays since the site has slow response time and it is using a real browser due to having to execute javascript.

1. # -*- coding: cp1252 -*- 
2. #!/usr/bin/python 
3.  
4. from splinter import Browser 
5. from bs4 import BeautifulSoup 
6. from time import sleep 
7. from datetime import datetime, timedelta 
8. import pandas as pd 
9. import sys, os 
10.  
11. def get_header_and_columns_from_html(soup): 
12. table = soup.find('table') 
13. head = table.findAll('th') 
14. header = [h.getText().strip() for h in head] 
15. all_columns = [] 
16. for row in table.findAll('tr'): 
17. col = row.findAll('td') 
18. columns = [c.getText().strip() for c in col] 
19. if(columns): 
20. all_columns.append(columns) 
21.  
22. return header, all_columns 
23.  
24. def get_flight_table_for_city_on_date(citycode, date): 
25. url = r"http://bristowgroup.com/clients/flight-status" 
26. df = None 
27. with Browser() as browser: 
28. browser.visit(url) 
29. sleep(3) 
30. city = '//select[@id="id_base"]/option[@value="{}"]'.format(citycode) 
31. browser.find_by_xpath(city)._element.click() 
32. sleep(3) 
33. browser.find_by_id('id_request_date').fill(date+"\t") 
34. sleep(3) 
35. browser.find_by_name('submit').click() 
36. if browser.is_text_present('Important Information', wait_time=7): 
37. html_source = browser.html 
38. soup = BeautifulSoup(html_source, 'html.parser') 
39. try: 
40. header, all_columns = get_header_and_columns_from_html(soup) 
41. df = pd.DataFrame(data=all_columns, columns=header) 
42. return df 
43. except: 
44. return None 
45. else: 
46. return None 
47.  
48. def get_dates(): 
49. months = ['', 'Jan', 'Feb', 'Mar', 'Apr', 'May', 'Jun', 
50. 'Jul', 'Aug', 'Sep', 'Nov', 'Dec'] 
51.  
52. now = datetime.now() 
53. date = "{}-{}-{}".format(now.day, months[now.month], now.year) 
54.  
55. if len(sys.argv) == 1: 
56. return [date] 
57.  
58. elif len(sys.argv) == 2: 
59. date = sys.argv[1] 
60. return [date] 
61.  
62. elif len(sys.argv) == 3: 
63. dates = [] 
64. d1 = datetime.strptime(sys.argv[1], "%d-%b-%Y").date() 
65. d2 = datetime.strptime(sys.argv[2], "%d-%b-%Y").date() 
66.  
67. delta = d2 - d1 
68.  
69. for i in range(delta.days + 1): 
70. d = d1 + timedelta(days=i) 
71. dates.append(d.strftime("%d-%b-%Y").lstrip('0')) 
72. return dates 
73. else: 
74. return None 
75.  
76. def create_folder(directory, foldername): 
77. if not directory: 
78. directory = os.path.curdir 
79.  
80. outdir = directory+r"/{}".format(foldername) 
81. if not os.path.exists(outdir): 
82. os.makedirs(outdir) 
83. return outdir 
84.  
85. if __name__ == '__main__': 
86. #for a specific cities you can comment out the cities that you don't want 
87.  
88. cities ={ 
89. "20":"Bergen", 
90. "47":"Brønnøysund", 
91. "19":"Den Helder", 
92. "21":"Hammerfest", 
93. "30":"Humberside", 
94. "18":"Norwich", 
95. "2":"Scatsta", 
96. "13":"Sola" 
97. } 
98.  
99. dates = get_dates() 
100.  
101. directory = r"" #replace with a string containing your prefered path, default will be the directory the script is ran from 
102. for citycode in cities: 
103. for date in dates: 
104. outdir = create_folder(directory, date) 
105. flights_table = get_flight_table_for_city_on_date(citycode, date) 
106. if type(flights_table) != type(None): 
107. out_path = r"{}\{}_{}.csv".format(outdir, date, cities[citycode])  
108. flights_table.to_csv(out_path) 
109. else: 
110. out_path = "{}\{}_{}_log.txt".format(outdir, date, cities[citycode])  
111.  
112. with open(out_path, 'w') as f: 
113. f.write("failed {} {}".format(date, cities[citycode])) 
114.