1 | ''' |
---|
2 | common XBMC Module |
---|
3 | Copyright (C) 2011 t0mm0 |
---|
4 | |
---|
5 | This program is free software: you can redistribute it and/or modify |
---|
6 | it under the terms of the GNU General Public License as published by |
---|
7 | the Free Software Foundation, either version 3 of the License, or |
---|
8 | (at your option) any later version. |
---|
9 | |
---|
10 | This program is distributed in the hope that it will be useful, |
---|
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of |
---|
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the |
---|
13 | GNU General Public License for more details. |
---|
14 | |
---|
15 | You should have received a copy of the GNU General Public License |
---|
16 | along with this program. If not, see <http://www.gnu.org/licenses/>. |
---|
17 | ''' |
---|
18 | |
---|
19 | import cookielib |
---|
20 | import gzip |
---|
21 | import re |
---|
22 | import StringIO |
---|
23 | import urllib |
---|
24 | import urllib2 |
---|
25 | import socket |
---|
26 | |
---|
27 | # Set Global timeout - Useful for slow connections and Putlocker. |
---|
28 | socket.setdefaulttimeout(60) |
---|
29 | |
---|
30 | class HeadRequest(urllib2.Request): |
---|
31 | '''A Request class that sends HEAD requests''' |
---|
32 | |
---|
33 | def get_method(self): |
---|
34 | return 'HEAD' |
---|
35 | |
---|
36 | class Net: |
---|
37 | ''' |
---|
38 | This class wraps :mod:`urllib2` and provides an easy way to make http |
---|
39 | requests while taking care of cookies, proxies, gzip compression and |
---|
40 | character encoding. |
---|
41 | |
---|
42 | Example:: |
---|
43 | |
---|
44 | from addon.common.net import Net |
---|
45 | net = Net() |
---|
46 | response = net.http_GET('http://xbmc.org') |
---|
47 | print response.content |
---|
48 | ''' |
---|
49 | |
---|
50 | _cj = cookielib.LWPCookieJar() |
---|
51 | _proxy = None |
---|
52 | _user_agent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0' |
---|
53 | _http_debug = False |
---|
54 | |
---|
55 | def __init__(self, cookie_file='', proxy='', user_agent='', http_debug=False): |
---|
56 | ''' |
---|
57 | Kwargs: |
---|
58 | cookie_file (str): Full path to a file to be used to load and save |
---|
59 | cookies to. |
---|
60 | |
---|
61 | proxy (str): Proxy setting (eg. |
---|
62 | ``'http://user:pass@example.com:1234'``) |
---|
63 | |
---|
64 | user_agent (str): String to use as the User Agent header. If not |
---|
65 | supplied the class will use a default user agent (chrome) |
---|
66 | |
---|
67 | http_debug (bool): Set ``True`` to have HTTP header info written to |
---|
68 | the XBMC log for all requests. |
---|
69 | ''' |
---|
70 | if cookie_file: |
---|
71 | self.set_cookies(cookie_file) |
---|
72 | if proxy: |
---|
73 | self.set_proxy(proxy) |
---|
74 | if user_agent: |
---|
75 | self.set_user_agent(user_agent) |
---|
76 | self._http_debug = http_debug |
---|
77 | self._update_opener() |
---|
78 | |
---|
79 | def set_cookies(self, cookie_file): |
---|
80 | ''' |
---|
81 | Set the cookie file and try to load cookies from it if it exists. |
---|
82 | |
---|
83 | Args: |
---|
84 | cookie_file (str): Full path to a file to be used to load and save |
---|
85 | cookies to. |
---|
86 | ''' |
---|
87 | try: |
---|
88 | self._cj.load(cookie_file, ignore_discard=True) |
---|
89 | self._update_opener() |
---|
90 | return True |
---|
91 | except: |
---|
92 | return False |
---|
93 | |
---|
94 | def get_cookies(self): |
---|
95 | '''Returns A dictionary containing all cookie information by domain.''' |
---|
96 | return self._cj._cookies |
---|
97 | |
---|
98 | def save_cookies(self, cookie_file): |
---|
99 | ''' |
---|
100 | Saves cookies to a file. |
---|
101 | |
---|
102 | Args: |
---|
103 | cookie_file (str): Full path to a file to save cookies to. |
---|
104 | ''' |
---|
105 | self._cj.save(cookie_file, ignore_discard=True) |
---|
106 | |
---|
107 | def set_proxy(self, proxy): |
---|
108 | ''' |
---|
109 | Args: |
---|
110 | proxy (str): Proxy setting (eg. |
---|
111 | ``'http://user:pass@example.com:1234'``) |
---|
112 | ''' |
---|
113 | self._proxy = proxy |
---|
114 | self._update_opener() |
---|
115 | |
---|
116 | def get_proxy(self): |
---|
117 | '''Returns string containing proxy details.''' |
---|
118 | return self._proxy |
---|
119 | |
---|
120 | def set_user_agent(self, user_agent): |
---|
121 | ''' |
---|
122 | Args: |
---|
123 | user_agent (str): String to use as the User Agent header. |
---|
124 | ''' |
---|
125 | self._user_agent = user_agent |
---|
126 | |
---|
127 | def get_user_agent(self): |
---|
128 | '''Returns user agent string.''' |
---|
129 | return self._user_agent |
---|
130 | |
---|
131 | def _update_opener(self): |
---|
132 | ''' |
---|
133 | Builds and installs a new opener to be used by all future calls to |
---|
134 | :func:`urllib2.urlopen`. |
---|
135 | ''' |
---|
136 | if self._http_debug: |
---|
137 | http = urllib2.HTTPHandler(debuglevel=1) |
---|
138 | else: |
---|
139 | http = urllib2.HTTPHandler() |
---|
140 | |
---|
141 | if self._proxy: |
---|
142 | opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self._cj), |
---|
143 | urllib2.ProxyHandler({'http': |
---|
144 | self._proxy}), |
---|
145 | urllib2.HTTPBasicAuthHandler(), |
---|
146 | http) |
---|
147 | |
---|
148 | else: |
---|
149 | opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self._cj), |
---|
150 | urllib2.HTTPBasicAuthHandler(), |
---|
151 | http) |
---|
152 | urllib2.install_opener(opener) |
---|
153 | |
---|
154 | def http_GET(self, url, headers={}, compression=True): |
---|
155 | ''' |
---|
156 | Perform an HTTP GET request. |
---|
157 | |
---|
158 | Args: |
---|
159 | url (str): The URL to GET. |
---|
160 | |
---|
161 | Kwargs: |
---|
162 | headers (dict): A dictionary describing any headers you would like |
---|
163 | to add to the request. (eg. ``{'X-Test': 'testing'}``) |
---|
164 | |
---|
165 | compression (bool): If ``True`` (default), try to use gzip |
---|
166 | compression. |
---|
167 | |
---|
168 | Returns: |
---|
169 | An :class:`HttpResponse` object containing headers and other |
---|
170 | meta-information about the page and the page content. |
---|
171 | ''' |
---|
172 | return self._fetch(url, headers=headers, compression=compression) |
---|
173 | |
---|
174 | def http_POST(self, url, form_data, headers={}, compression=True): |
---|
175 | ''' |
---|
176 | Perform an HTTP POST request. |
---|
177 | |
---|
178 | Args: |
---|
179 | url (str): The URL to POST. |
---|
180 | |
---|
181 | form_data (dict): A dictionary of form data to POST. |
---|
182 | |
---|
183 | Kwargs: |
---|
184 | headers (dict): A dictionary describing any headers you would like |
---|
185 | to add to the request. (eg. ``{'X-Test': 'testing'}``) |
---|
186 | |
---|
187 | compression (bool): If ``True`` (default), try to use gzip |
---|
188 | compression. |
---|
189 | |
---|
190 | Returns: |
---|
191 | An :class:`HttpResponse` object containing headers and other |
---|
192 | meta-information about the page and the page content. |
---|
193 | ''' |
---|
194 | return self._fetch(url, form_data, headers=headers, compression=compression) |
---|
195 | |
---|
196 | def http_HEAD(self, url, headers={}): |
---|
197 | ''' |
---|
198 | Perform an HTTP HEAD request. |
---|
199 | |
---|
200 | Args: |
---|
201 | url (str): The URL to GET. |
---|
202 | |
---|
203 | Kwargs: |
---|
204 | headers (dict): A dictionary describing any headers you would like |
---|
205 | to add to the request. (eg. ``{'X-Test': 'testing'}``) |
---|
206 | |
---|
207 | Returns: |
---|
208 | An :class:`HttpResponse` object containing headers and other |
---|
209 | meta-information about the page. |
---|
210 | ''' |
---|
211 | req = HeadRequest(url) |
---|
212 | req.add_header('User-Agent', self._user_agent) |
---|
213 | for key in headers: |
---|
214 | req.add_header(key, headers[key]) |
---|
215 | response = urllib2.urlopen(req) |
---|
216 | return HttpResponse(response) |
---|
217 | |
---|
218 | def _fetch(self, url, form_data={}, headers={}, compression=True): |
---|
219 | ''' |
---|
220 | Perform an HTTP GET or POST request. |
---|
221 | |
---|
222 | Args: |
---|
223 | url (str): The URL to GET or POST. |
---|
224 | |
---|
225 | form_data (dict): A dictionary of form data to POST. If empty, the |
---|
226 | request will be a GET, if it contains form data it will be a POST. |
---|
227 | |
---|
228 | Kwargs: |
---|
229 | headers (dict): A dictionary describing any headers you would like |
---|
230 | to add to the request. (eg. ``{'X-Test': 'testing'}``) |
---|
231 | |
---|
232 | compression (bool): If ``True`` (default), try to use gzip |
---|
233 | compression. |
---|
234 | |
---|
235 | Returns: |
---|
236 | An :class:`HttpResponse` object containing headers and other |
---|
237 | meta-information about the page and the page content. |
---|
238 | ''' |
---|
239 | req = urllib2.Request(url) |
---|
240 | if form_data: |
---|
241 | if isinstance(form_data, basestring): |
---|
242 | form_data = form_data |
---|
243 | else: |
---|
244 | form_data = urllib.urlencode(form_data, True) |
---|
245 | req = urllib2.Request(url, form_data) |
---|
246 | req.add_header('User-Agent', self._user_agent) |
---|
247 | for key in headers: |
---|
248 | req.add_header(key, headers[key]) |
---|
249 | if compression: |
---|
250 | req.add_header('Accept-Encoding', 'gzip') |
---|
251 | req.add_unredirected_header('Host', req.get_host()) |
---|
252 | response = urllib2.urlopen(req) |
---|
253 | return HttpResponse(response) |
---|
254 | |
---|
255 | class HttpResponse: |
---|
256 | ''' |
---|
257 | This class represents a resoponse from an HTTP request. |
---|
258 | |
---|
259 | The content is examined and every attempt is made to properly encode it to |
---|
260 | Unicode. |
---|
261 | |
---|
262 | .. seealso:: |
---|
263 | :meth:`Net.http_GET`, :meth:`Net.http_HEAD` and :meth:`Net.http_POST` |
---|
264 | ''' |
---|
265 | |
---|
266 | content = '' |
---|
267 | '''Unicode encoded string containing the body of the reposne.''' |
---|
268 | |
---|
269 | def __init__(self, response): |
---|
270 | ''' |
---|
271 | Args: |
---|
272 | response (:class:`mimetools.Message`): The object returned by a call |
---|
273 | to :func:`urllib2.urlopen`. |
---|
274 | ''' |
---|
275 | self._response = response |
---|
276 | html = response.read() |
---|
277 | try: |
---|
278 | if response.headers['content-encoding'].lower() == 'gzip': |
---|
279 | html = gzip.GzipFile(fileobj=StringIO.StringIO(html)).read() |
---|
280 | except: |
---|
281 | pass |
---|
282 | |
---|
283 | try: |
---|
284 | content_type = response.headers['content-type'] |
---|
285 | if 'charset=' in content_type: |
---|
286 | encoding = content_type.split('charset=')[-1] |
---|
287 | except: |
---|
288 | pass |
---|
289 | |
---|
290 | r = re.search('<meta\s+http-equiv="Content-Type"\s+content="(?:.+?);\s+charset=(.+?)"', html, re.IGNORECASE) |
---|
291 | if r: encoding = r.group(1) |
---|
292 | try: html = unicode(html, encoding) |
---|
293 | except: pass |
---|
294 | |
---|
295 | self.content = html |
---|
296 | |
---|
297 | def get_headers(self): |
---|
298 | '''Returns a List of headers returned by the server.''' |
---|
299 | return self._response.info().headers |
---|
300 | |
---|
301 | def get_url(self): |
---|
302 | ''' |
---|
303 | Return the URL of the resource retrieved, commonly used to determine if |
---|
304 | a redirect was followed. |
---|
305 | ''' |
---|
306 | return self._response.geturl() |
---|