EA-Scraper/fetch_river_levels.py at master · crouchingbadger/EA-Scraper · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
#!/usr/bin/python

# Forked from https://github.com/russss/ea_scrape

import requests
import datetime
import eeml
from lxml.html.soupparser import fromstring
from lxml.etree import tostring
from time import sleep

# area_id -> [station_id]
SOURCES = [[136495, 7074, '88571'], # Seacourt Stream at Minns Estate [88571]
           [136495, 7075, '88603'], # Bulstake Stream at New Botley
           [136495, 7057, '88605'], # Thames - Upstream at Osney Lock
           [136495, 7076, '88607'], # Hinksey Stream at Cold Harbour
           [136497, 7071, '88610'], # Cherwell - Upstream at Kings Mill
           [136497, 7072, '88611'], # Thames - Upstream at Iffley Lock
           [136497, '7072&Sensor=D', '88615'], # Thames - Downstream at Iffley Lock
                ]


FEED = '88571'
API_KEY = '{xivelykey}' # rivers
API_URL = '/v2/feeds/{feednum}.xml' .format(feednum = FEED)


def get_data(area_id, station_id):
    url = 'http://www.environment-agency.gov.uk/homeandleisure/floods/riverlevels/%s.aspx?stationId=%s'
    try:
        r = requests.get(url % (area_id, station_id))
#       print vars(r)

        data = fromstring(r.content)

        date_text = data.find('.//div[@id="content"]/div/div/p').text

        time, date = date_text.strip('Last updated ').split(' on ')

        day, month, year = map(int, date.split('/'))
        hour, minute = map(int, time.split(':'))

        height = float(data.find(".//div[@id='station-detail-left']//div[@class='plain_text']/p"
                                                                ).text.split(' is ')[1].split(' ')[0])

        return datetime.datetime(year, month, day, hour, minute), height
    except Exception:
        print "couldn't scrape"
        pass


for stationinfo in SOURCES:
        print stationinfo

        area_id = stationinfo[0]
        station_id = stationinfo[1]
        station_feed = stationinfo[2]

        try:
                readdate, level = get_data(area_id, station_id)
                print "Returned:", readdate, level
                api_url = "/v2/feeds/"+ station_feed + ".xml"

                print api_url
                try:
                        # Open up Xively feed
                        pac = eeml.Pachube(api_url, API_KEY)

                        # Update feed
                        pac.update([eeml.Data("Level", level, unit=eeml.Unit("Metre",'basicSI',"m") )])

                        try:
                                pac.put()

                        except:
                                print "couldn't connect to Xively"
                                print "Xively error:", sys.exc_info()[0]
                                pass
                except:
                        print "problem creating url"
                        pass
        except Exception:
                print "Feed unavailable"

        sleep(2)