1 | """
|
---|
2 | common XBMC Module
|
---|
3 | Copyright (C) 2011 t0mm0
|
---|
4 |
|
---|
5 | This program is free software: you can redistribute it and/or modify
|
---|
6 | it under the terms of the GNU General Public License as published by
|
---|
7 | the Free Software Foundation, either version 3 of the License, or
|
---|
8 | (at your option) any later version.
|
---|
9 |
|
---|
10 | This program is distributed in the hope that it will be useful,
|
---|
11 | but WITHOUT ANY WARRANTY; without even the implied warranty of
|
---|
12 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
---|
13 | GNU General Public License for more details.
|
---|
14 |
|
---|
15 | You should have received a copy of the GNU General Public License
|
---|
16 | along with this program. If not, see <http://www.gnu.org/licenses/>.
|
---|
17 | """
|
---|
18 | import random
|
---|
19 | from six.moves import http_cookiejar
|
---|
20 | import gzip
|
---|
21 | import re
|
---|
22 | import json
|
---|
23 | import six
|
---|
24 | from six.moves import urllib_request, urllib_parse
|
---|
25 | import socket
|
---|
26 | import time
|
---|
27 | #from urlresolver.lib import kodi
|
---|
28 |
|
---|
29 | # Set Global timeout - Useful for slow connections and Putlocker.
|
---|
30 | socket.setdefaulttimeout(10)
|
---|
31 |
|
---|
32 | BR_VERS = [
|
---|
33 | ['%s.0' % i for i in range(18, 50)],
|
---|
34 | ['37.0.2062.103', '37.0.2062.120', '37.0.2062.124', '38.0.2125.101', '38.0.2125.104', '38.0.2125.111', '39.0.2171.71', '39.0.2171.95', '39.0.2171.99', '40.0.2214.93', '40.0.2214.111',
|
---|
35 | '40.0.2214.115', '42.0.2311.90', '42.0.2311.135', '42.0.2311.152', '43.0.2357.81', '43.0.2357.124', '44.0.2403.155', '44.0.2403.157', '45.0.2454.101', '45.0.2454.85', '46.0.2490.71',
|
---|
36 | '46.0.2490.80', '46.0.2490.86', '47.0.2526.73', '47.0.2526.80', '48.0.2564.116', '49.0.2623.112', '50.0.2661.86'],
|
---|
37 | ['11.0'],
|
---|
38 | ['8.0', '9.0', '10.0', '10.6']]
|
---|
39 | WIN_VERS = ['Windows NT 10.0', 'Windows NT 7.0', 'Windows NT 6.3', 'Windows NT 6.2', 'Windows NT 6.1', 'Windows NT 6.0', 'Windows NT 5.1', 'Windows NT 5.0']
|
---|
40 | FEATURES = ['; WOW64', '; Win64; IA64', '; Win64; x64', '']
|
---|
41 | RAND_UAS = ['Mozilla/5.0 ({win_ver}{feature}; rv:{br_ver}) Gecko/20100101 Firefox/{br_ver}',
|
---|
42 | 'Mozilla/5.0 ({win_ver}{feature}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{br_ver} Safari/537.36',
|
---|
43 | 'Mozilla/5.0 ({win_ver}{feature}; Trident/7.0; rv:{br_ver}) like Gecko',
|
---|
44 | 'Mozilla/5.0 (compatible; MSIE {br_ver}; {win_ver}{feature}; Trident/6.0)']
|
---|
45 |
|
---|
46 |
|
---|
47 | def get_ua():
|
---|
48 | # try: last_gen = int(kodi.get_setting('last_ua_create'))
|
---|
49 | try: last_gen = 0
|
---|
50 | except: last_gen = 0
|
---|
51 | # if not kodi.get_setting('current_ua') or last_gen < (time.time() - (7 * 24 * 60 * 60)):
|
---|
52 | index = random.randrange(len(RAND_UAS))
|
---|
53 | versions = {'win_ver': random.choice(WIN_VERS), 'feature': random.choice(FEATURES), 'br_ver': random.choice(BR_VERS[index])}
|
---|
54 | user_agent = RAND_UAS[index].format(**versions)
|
---|
55 | # logger.log('Creating New User Agent: %s' % (user_agent), log_utils.LOGDEBUG)
|
---|
56 | # kodi.set_setting('current_ua', user_agent)
|
---|
57 | # kodi.set_setting('last_ua_create', str(int(time.time())))
|
---|
58 | # else:
|
---|
59 | # user_agent = kodi.get_setting('current_ua')
|
---|
60 | return user_agent
|
---|
61 |
|
---|
62 | class Net:
|
---|
63 | """
|
---|
64 | This class wraps :mod:`urllib2` and provides an easy way to make http
|
---|
65 | requests while taking care of cookies, proxies, gzip compression and
|
---|
66 | character encoding.
|
---|
67 |
|
---|
68 | Example::
|
---|
69 |
|
---|
70 | from addon.common.net import Net
|
---|
71 | net = Net()
|
---|
72 | response = net.http_GET('http://xbmc.org')
|
---|
73 | print response.content
|
---|
74 | """
|
---|
75 |
|
---|
76 | _cj = http_cookiejar.LWPCookieJar()
|
---|
77 | _proxy = None
|
---|
78 | _user_agent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0'
|
---|
79 | _http_debug = False
|
---|
80 |
|
---|
81 | def __init__(self, cookie_file='', proxy='', user_agent='', ssl_verify=True, http_debug=False):
|
---|
82 | """
|
---|
83 | Kwargs:
|
---|
84 | cookie_file (str): Full path to a file to be used to load and save
|
---|
85 | cookies to.
|
---|
86 |
|
---|
87 | proxy (str): Proxy setting (eg.
|
---|
88 | ``'http://user:pass@example.com:1234'``)
|
---|
89 |
|
---|
90 | user_agent (str): String to use as the User Agent header. If not
|
---|
91 | supplied the class will use a default user agent (chrome)
|
---|
92 |
|
---|
93 | http_debug (bool): Set ``True`` to have HTTP header info written to
|
---|
94 | the XBMC log for all requests.
|
---|
95 | """
|
---|
96 | if cookie_file:
|
---|
97 | self.set_cookies(cookie_file)
|
---|
98 | if proxy:
|
---|
99 | self.set_proxy(proxy)
|
---|
100 | if user_agent:
|
---|
101 | self.set_user_agent(user_agent)
|
---|
102 | self._ssl_verify = ssl_verify
|
---|
103 | szzzelf._http_debug = http_debug
|
---|
104 | self._update_opener()
|
---|
105 |
|
---|
106 | def set_cookies(self, cookie_file):
|
---|
107 | """
|
---|
108 | Set the cookie file and try to load cookies from it if it exists.
|
---|
109 |
|
---|
110 | Args:
|
---|
111 | cookie_file (str): Full path to a file to be used to load and save
|
---|
112 | cookies to.
|
---|
113 | """
|
---|
114 | try:
|
---|
115 | self._cj.load(cookie_file, ignore_discard=True)
|
---|
116 | self._update_opener()
|
---|
117 | return True
|
---|
118 | except:
|
---|
119 | return False
|
---|
120 |
|
---|
121 | def get_cookies(self, as_dict=False):
|
---|
122 | """Returns A dictionary containing all cookie information by domain."""
|
---|
123 | if as_dict:
|
---|
124 | return dict((cookie.name, cookie.value) for cookie in self._cj)
|
---|
125 | else:
|
---|
126 | return self._cj._cookies
|
---|
127 |
|
---|
128 | def save_cookies(self, cookie_file):
|
---|
129 | """
|
---|
130 | Saves cookies to a file.
|
---|
131 |
|
---|
132 | Args:
|
---|
133 | cookie_file (str): Full path to a file to save cookies to.
|
---|
134 | """
|
---|
135 | self._cj.save(cookie_file, ignore_discard=True)
|
---|
136 |
|
---|
137 | def set_proxy(self, proxy):
|
---|
138 | """
|
---|
139 | Args:
|
---|
140 | proxy (str): Proxy setting (eg.
|
---|
141 | ``'http://user:pass@example.com:1234'``)
|
---|
142 | """
|
---|
143 | self._proxy = proxy
|
---|
144 | self._update_opener()
|
---|
145 |
|
---|
146 | def get_proxy(self):
|
---|
147 | """Returns string containing proxy details."""
|
---|
148 | return self._proxy
|
---|
149 |
|
---|
150 | def set_user_agent(self, user_agent):
|
---|
151 | """
|
---|
152 | Args:
|
---|
153 | user_agent (str): String to use as the User Agent header.
|
---|
154 | """
|
---|
155 | self._user_agent = user_agent
|
---|
156 |
|
---|
157 | def get_user_agent(self):
|
---|
158 | """Returns user agent string."""
|
---|
159 | return self._user_agent
|
---|
160 |
|
---|
161 | def _update_opener(self):
|
---|
162 | """
|
---|
163 | Builds and installs a new opener to be used by all future calls to
|
---|
164 | :func:`urllib2.urlopen`.
|
---|
165 | """
|
---|
166 | handlers = [urllib_request.HTTPCookieProcessor(self._cj), urllib_request.HTTPBasicAuthHandler()]
|
---|
167 |
|
---|
168 | if self._http_debug:
|
---|
169 | handlers += [urllib_request.HTTPHandler(debuglevel=1)]
|
---|
170 | else:
|
---|
171 | handlers += [urllib_request.HTTPHandler()]
|
---|
172 |
|
---|
173 | if self._proxy:
|
---|
174 | handlers += [urllib_request.ProxyHandler({'http': self._proxy})]
|
---|
175 |
|
---|
176 | try:
|
---|
177 | import platform
|
---|
178 | node = platform.node().lower()
|
---|
179 | except:
|
---|
180 | node = ''
|
---|
181 |
|
---|
182 | if not self._ssl_verify or node == 'xboxone':
|
---|
183 | try:
|
---|
184 | import ssl
|
---|
185 | ctx = ssl.create_default_context()
|
---|
186 | ctx.check_hostname = False
|
---|
187 | ctx.verify_mode = ssl.CERT_NONE
|
---|
188 | if self._http_debug:
|
---|
189 | handlers += [urllib_request.HTTPSHandler(context=ctx, debuglevel=1)]
|
---|
190 | else:
|
---|
191 | handlers += [urllib_request.HTTPSHandler(context=ctx)]
|
---|
192 | except:
|
---|
193 | pass
|
---|
194 |
|
---|
195 | opener = urllib_request.build_opener(*handlers)
|
---|
196 | urllib_request.install_opener(opener)
|
---|
197 |
|
---|
198 | def http_GET(self, url, headers={}, compression=True):
|
---|
199 | """
|
---|
200 | Perform an HTTP GET request.
|
---|
201 |
|
---|
202 | Args:
|
---|
203 | url (str): The URL to GET.
|
---|
204 |
|
---|
205 | Kwargs:
|
---|
206 | headers (dict): A dictionary describing any headers you would like
|
---|
207 | to add to the request. (eg. ``{'X-Test': 'testing'}``)
|
---|
208 |
|
---|
209 | compression (bool): If ``True`` (default), try to use gzip
|
---|
210 | compression.
|
---|
211 |
|
---|
212 | Returns:
|
---|
213 | An :class:`HttpResponse` object containing headers and other
|
---|
214 | meta-information about the page and the page content.
|
---|
215 | """
|
---|
216 | return self._fetch(url, headers=headers, compression=compression)
|
---|
217 |
|
---|
218 | def http_POST(self, url, form_data, headers={}, compression=True, jdata=False):
|
---|
219 | """
|
---|
220 | Perform an HTTP POST request.
|
---|
221 |
|
---|
222 | Args:
|
---|
223 | url (str): The URL to POST.
|
---|
224 |
|
---|
225 | form_data (dict): A dictionary of form data to POST.
|
---|
226 |
|
---|
227 | Kwargs:
|
---|
228 | headers (dict): A dictionary describing any headers you would like
|
---|
229 | to add to the request. (eg. ``{'X-Test': 'testing'}``)
|
---|
230 |
|
---|
231 | compression (bool): If ``True`` (default), try to use gzip
|
---|
232 | compression.
|
---|
233 |
|
---|
234 | Returns:
|
---|
235 | An :class:`HttpResponse` object containing headers and other
|
---|
236 | meta-information about the page and the page content.
|
---|
237 | """
|
---|
238 | return self._fetch(url, form_data, headers=headers, compression=compression, jdata=jdata)
|
---|
239 |
|
---|
240 | def http_HEAD(self, url, headers={}):
|
---|
241 | """
|
---|
242 | Perform an HTTP HEAD request.
|
---|
243 |
|
---|
244 | Args:
|
---|
245 | url (str): The URL to GET.
|
---|
246 |
|
---|
247 | Kwargs:
|
---|
248 | headers (dict): A dictionary describing any headers you would like
|
---|
249 | to add to the request. (eg. ``{'X-Test': 'testing'}``)
|
---|
250 |
|
---|
251 | Returns:
|
---|
252 | An :class:`HttpResponse` object containing headers and other
|
---|
253 | meta-information about the page.
|
---|
254 | """
|
---|
255 | request = urllib_request.Request(url)
|
---|
256 | request.get_method = lambda: 'HEAD'
|
---|
257 | request.add_header('User-Agent', self._user_agent)
|
---|
258 | for key in headers:
|
---|
259 | request.add_header(key, headers[key])
|
---|
260 | response = urllib_request.urlopen(request)
|
---|
261 | return HttpResponse(response)
|
---|
262 |
|
---|
263 | def http_DELETE(self, url, headers={}):
|
---|
264 | """
|
---|
265 | Perform an HTTP DELETE request.
|
---|
266 |
|
---|
267 | Args:
|
---|
268 | url (str): The URL to GET.
|
---|
269 |
|
---|
270 | Kwargs:
|
---|
271 | headers (dict): A dictionary describing any headers you would like
|
---|
272 | to add to the request. (eg. ``{'X-Test': 'testing'}``)
|
---|
273 |
|
---|
274 | Returns:
|
---|
275 | An :class:`HttpResponse` object containing headers and other
|
---|
276 | meta-information about the page.
|
---|
277 | """
|
---|
278 | request = urllib_request.Request(url)
|
---|
279 | request.get_method = lambda: 'DELETE'
|
---|
280 | request.add_header('User-Agent', self._user_agent)
|
---|
281 | for key in headers:
|
---|
282 | request.add_header(key, headers[key])
|
---|
283 | response = urllib_request.urlopen(request)
|
---|
284 | return HttpResponse(response)
|
---|
285 |
|
---|
286 | def _fetch(self, url, form_data={}, headers={}, compression=True, jdata=False):
|
---|
287 | """
|
---|
288 | Perform an HTTP GET or POST request.
|
---|
289 |
|
---|
290 | Args:
|
---|
291 | url (str): The URL to GET or POST.
|
---|
292 |
|
---|
293 | form_data (dict): A dictionary of form data to POST. If empty, the
|
---|
294 | request will be a GET, if it contains form data it will be a POST.
|
---|
295 |
|
---|
296 | Kwargs:
|
---|
297 | headers (dict): A dictionary describing any headers you would like
|
---|
298 | to add to the request. (eg. ``{'X-Test': 'testing'}``)
|
---|
299 |
|
---|
300 | compression (bool): If ``True`` (default), try to use gzip
|
---|
301 | compression.
|
---|
302 |
|
---|
303 | Returns:
|
---|
304 | An :class:`HttpResponse` object containing headers and other
|
---|
305 | meta-information about the page and the page content.
|
---|
306 | """
|
---|
307 | req = urllib_request.Request(url)
|
---|
308 | if form_data:
|
---|
309 | if jdata:
|
---|
310 | form_data = json.dumps(form_data)
|
---|
311 | elif isinstance(form_data, six.string_types):
|
---|
312 | form_data = form_data
|
---|
313 | else:
|
---|
314 | form_data = urllib_parse.urlencode(form_data, True)
|
---|
315 | form_data = form_data.encode('utf-8') if six.PY3 else form_data
|
---|
316 | req = urllib_request.Request(url, form_data)
|
---|
317 | req.add_header('User-Agent', self._user_agent)
|
---|
318 | for key in headers:
|
---|
319 | req.add_header(key, headers[key])
|
---|
320 | if compression:
|
---|
321 | req.add_header('Accept-Encoding', 'gzip')
|
---|
322 | if jdata:
|
---|
323 | req.add_header('Content-Type', 'application/json')
|
---|
324 | host = req.host if six.PY3 else req.get_host()
|
---|
325 | req.add_unredirected_header('Host', host)
|
---|
326 | response = urllib_request.urlopen(req, timeout=15)
|
---|
327 | return HttpResponse(response)
|
---|
328 |
|
---|
329 |
|
---|
330 | class HttpResponse:
|
---|
331 | """
|
---|
332 | This class represents a resoponse from an HTTP request.
|
---|
333 |
|
---|
334 | The content is examined and every attempt is made to properly decode it to
|
---|
335 | Unicode unless the nodecode property flag is set to True.
|
---|
336 |
|
---|
337 | .. seealso::
|
---|
338 | :meth:`Net.http_GET`, :meth:`Net.http_HEAD` and :meth:`Net.http_POST`
|
---|
339 | """
|
---|
340 |
|
---|
341 | # content = ''
|
---|
342 | """Unicode encoded string containing the body of the reponse."""
|
---|
343 |
|
---|
344 | def __init__(self, response):
|
---|
345 | """
|
---|
346 | Args:
|
---|
347 | response (:class:`mimetools.Message`): The object returned by a call
|
---|
348 | to :func:`urllib2.urlopen`.
|
---|
349 | """
|
---|
350 | self._response = response
|
---|
351 | self._nodecode = False
|
---|
352 |
|
---|
353 | @property
|
---|
354 | def content(self):
|
---|
355 | html = self._response.read()
|
---|
356 | encoding = None
|
---|
357 | try:
|
---|
358 | if self._response.headers['content-encoding'].lower() == 'gzip':
|
---|
359 | html = gzip.GzipFile(fileobj=six.BytesIO(html)).read()
|
---|
360 | except:
|
---|
361 | pass
|
---|
362 |
|
---|
363 | if self._nodecode:
|
---|
364 | return html
|
---|
365 |
|
---|
366 | try:
|
---|
367 | content_type = self._response.headers['content-type']
|
---|
368 | if 'charset=' in content_type:
|
---|
369 | encoding = content_type.split('charset=')[-1]
|
---|
370 | except:
|
---|
371 | pass
|
---|
372 |
|
---|
373 | if encoding is None:
|
---|
374 | epattern = r'<meta\s+http-equiv="Content-Type"\s+content="(?:.+?);\s+charset=(.+?)"'
|
---|
375 | epattern = epattern.encode('utf8') if six.PY3 else epattern
|
---|
376 | r = re.search(epattern, html, re.IGNORECASE)
|
---|
377 | if r:
|
---|
378 | encoding = r.group(1).decode('utf8') if six.PY3 else r.group(1)
|
---|
379 |
|
---|
380 | if encoding is not None:
|
---|
381 | html = html.decode(encoding, errors='ignore')
|
---|
382 | else:
|
---|
383 | html = html.decode('ascii', errors='ignore') if six.PY3 else html
|
---|
384 | return html
|
---|
385 |
|
---|
386 | def get_headers(self, as_dict=False):
|
---|
387 | """Returns headers returned by the server.
|
---|
388 | If as_dict is True, headers are returned as a dictionary otherwise a list"""
|
---|
389 | if as_dict:
|
---|
390 | hdrs = {}
|
---|
391 | for item in list(self._response.info().items()):
|
---|
392 | if item[0].title() not in list(hdrs.keys()):
|
---|
393 | hdrs.update({item[0].title(): item[1]})
|
---|
394 | else:
|
---|
395 | hdrs.update({item[0].title(): ','.join([hdrs[item[0].title()], item[1]])})
|
---|
396 | # return dict([(item[0].title(), item[1]) for item in list(self._response.info().items())])
|
---|
397 | return hdrs
|
---|
398 | else:
|
---|
399 | return self._response.info()._headers if six.PY3 else [(x.split(':')[0].strip(), x.split(':')[1].strip()) for x in self._response.info().headers]
|
---|
400 |
|
---|
401 | def get_url(self):
|
---|
402 | """
|
---|
403 | Return the URL of the resource retrieved, commonly used to determine if
|
---|
404 | a redirect was followed.
|
---|
405 | """
|
---|
406 | return self._response.geturl()
|
---|
407 |
|
---|
408 | def nodecode(self, nodecode):
|
---|
409 | """
|
---|
410 | Sets whether or not content returns decoded text
|
---|
411 | nodecode (bool): Set to ``True`` to allow content to return undecoded data
|
---|
412 | suitable to write to a binary file
|
---|
413 | """
|
---|
414 | self._nodecode = bool(nodecode)
|
---|
415 | return self
|
---|