source: titan/mediathek/localhoster/lib/net.py @ 38985

Last change on this file since 38985 was 38985, checked in by obi, 8 years ago

localhoster add allmyvideos

File size: 9.7 KB
Line 
1'''
2    common XBMC Module
3    Copyright (C) 2011 t0mm0
4
5    This program is free software: you can redistribute it and/or modify
6    it under the terms of the GNU General Public License as published by
7    the Free Software Foundation, either version 3 of the License, or
8    (at your option) any later version.
9
10    This program is distributed in the hope that it will be useful,
11    but WITHOUT ANY WARRANTY; without even the implied warranty of
12    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13    GNU General Public License for more details.
14
15    You should have received a copy of the GNU General Public License
16    along with this program.  If not, see <http://www.gnu.org/licenses/>.
17'''
18
19import cookielib
20import gzip
21import re
22import StringIO
23import urllib
24import urllib2
25import socket
26
27# Set Global timeout - Useful for slow connections and Putlocker.
28socket.setdefaulttimeout(60)
29
30class HeadRequest(urllib2.Request):
31    '''A Request class that sends HEAD requests'''
32
33    def get_method(self):
34        return 'HEAD'
35
36class Net:
37    '''
38    This class wraps :mod:`urllib2` and provides an easy way to make http
39    requests while taking care of cookies, proxies, gzip compression and
40    character encoding.
41
42    Example::
43
44        from addon.common.net import Net
45        net = Net()
46        response = net.http_GET('http://xbmc.org')
47        print response.content
48    '''
49
50    _cj = cookielib.LWPCookieJar()
51    _proxy = None
52    _user_agent = 'Mozilla/5.0 (Windows NT 6.3; rv:36.0) Gecko/20100101 Firefox/36.0'
53    _http_debug = False
54
55    def __init__(self, cookie_file='', proxy='', user_agent='', http_debug=False):
56        '''
57        Kwargs:
58            cookie_file (str): Full path to a file to be used to load and save
59            cookies to.
60
61            proxy (str): Proxy setting (eg.
62            ``'http://user:pass@example.com:1234'``)
63
64            user_agent (str): String to use as the User Agent header. If not
65            supplied the class will use a default user agent (chrome)
66
67            http_debug (bool): Set ``True`` to have HTTP header info written to
68            the XBMC log for all requests.
69        '''
70        if cookie_file:
71            self.set_cookies(cookie_file)
72        if proxy:
73            self.set_proxy(proxy)
74        if user_agent:
75            self.set_user_agent(user_agent)
76        self._http_debug = http_debug
77        self._update_opener()
78
79    def set_cookies(self, cookie_file):
80        '''
81        Set the cookie file and try to load cookies from it if it exists.
82
83        Args:
84            cookie_file (str): Full path to a file to be used to load and save
85            cookies to.
86        '''
87        try:
88            self._cj.load(cookie_file, ignore_discard=True)
89            self._update_opener()
90            return True
91        except:
92            return False
93
94    def get_cookies(self):
95        '''Returns A dictionary containing all cookie information by domain.'''
96        return self._cj._cookies
97
98    def save_cookies(self, cookie_file):
99        '''
100        Saves cookies to a file.
101
102        Args:
103            cookie_file (str): Full path to a file to save cookies to.
104        '''
105        self._cj.save(cookie_file, ignore_discard=True)
106
107    def set_proxy(self, proxy):
108        '''
109        Args:
110            proxy (str): Proxy setting (eg.
111            ``'http://user:pass@example.com:1234'``)
112        '''
113        self._proxy = proxy
114        self._update_opener()
115
116    def get_proxy(self):
117        '''Returns string containing proxy details.'''
118        return self._proxy
119
120    def set_user_agent(self, user_agent):
121        '''
122        Args:
123            user_agent (str): String to use as the User Agent header.
124        '''
125        self._user_agent = user_agent
126
127    def get_user_agent(self):
128        '''Returns user agent string.'''
129        return self._user_agent
130
131    def _update_opener(self):
132        '''
133        Builds and installs a new opener to be used by all future calls to
134        :func:`urllib2.urlopen`.
135        '''
136        if self._http_debug:
137            http = urllib2.HTTPHandler(debuglevel=1)
138        else:
139            http = urllib2.HTTPHandler()
140
141        if self._proxy:
142            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self._cj),
143                                          urllib2.ProxyHandler({'http':
144                                                                self._proxy}),
145                                          urllib2.HTTPBasicAuthHandler(),
146                                          http)
147
148        else:
149            opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(self._cj),
150                                          urllib2.HTTPBasicAuthHandler(),
151                                          http)
152        urllib2.install_opener(opener)
153
154    def http_GET(self, url, headers={}, compression=True):
155        '''
156        Perform an HTTP GET request.
157
158        Args:
159            url (str): The URL to GET.
160
161        Kwargs:
162            headers (dict): A dictionary describing any headers you would like
163            to add to the request. (eg. ``{'X-Test': 'testing'}``)
164
165            compression (bool): If ``True`` (default), try to use gzip
166            compression.
167
168        Returns:
169            An :class:`HttpResponse` object containing headers and other
170            meta-information about the page and the page content.
171        '''
172        return self._fetch(url, headers=headers, compression=compression)
173
174    def http_POST(self, url, form_data, headers={}, compression=True):
175        '''
176        Perform an HTTP POST request.
177
178        Args:
179            url (str): The URL to POST.
180
181            form_data (dict): A dictionary of form data to POST.
182
183        Kwargs:
184            headers (dict): A dictionary describing any headers you would like
185            to add to the request. (eg. ``{'X-Test': 'testing'}``)
186
187            compression (bool): If ``True`` (default), try to use gzip
188            compression.
189
190        Returns:
191            An :class:`HttpResponse` object containing headers and other
192            meta-information about the page and the page content.
193        '''
194        return self._fetch(url, form_data, headers=headers, compression=compression)
195
196    def http_HEAD(self, url, headers={}):
197        '''
198        Perform an HTTP HEAD request.
199
200        Args:
201            url (str): The URL to GET.
202
203        Kwargs:
204            headers (dict): A dictionary describing any headers you would like
205            to add to the request. (eg. ``{'X-Test': 'testing'}``)
206
207        Returns:
208            An :class:`HttpResponse` object containing headers and other
209            meta-information about the page.
210        '''
211        req = HeadRequest(url)
212        req.add_header('User-Agent', self._user_agent)
213        for key in headers:
214            req.add_header(key, headers[key])
215        response = urllib2.urlopen(req)
216        return HttpResponse(response)
217
218    def _fetch(self, url, form_data={}, headers={}, compression=True):
219        '''
220        Perform an HTTP GET or POST request.
221
222        Args:
223            url (str): The URL to GET or POST.
224
225            form_data (dict): A dictionary of form data to POST. If empty, the
226            request will be a GET, if it contains form data it will be a POST.
227
228        Kwargs:
229            headers (dict): A dictionary describing any headers you would like
230            to add to the request. (eg. ``{'X-Test': 'testing'}``)
231
232            compression (bool): If ``True`` (default), try to use gzip
233            compression.
234
235        Returns:
236            An :class:`HttpResponse` object containing headers and other
237            meta-information about the page and the page content.
238        '''
239        req = urllib2.Request(url)
240        if form_data:
241            if isinstance(form_data, basestring):
242                form_data = form_data
243            else:
244                form_data = urllib.urlencode(form_data, True)
245            req = urllib2.Request(url, form_data)
246        req.add_header('User-Agent', self._user_agent)
247        for key in headers:
248            req.add_header(key, headers[key])
249        if compression:
250            req.add_header('Accept-Encoding', 'gzip')
251        req.add_unredirected_header('Host', req.get_host())
252        response = urllib2.urlopen(req)
253        return HttpResponse(response)
254
255class HttpResponse:
256    '''
257    This class represents a resoponse from an HTTP request.
258
259    The content is examined and every attempt is made to properly encode it to
260    Unicode.
261
262    .. seealso::
263        :meth:`Net.http_GET`, :meth:`Net.http_HEAD` and :meth:`Net.http_POST`
264    '''
265
266    content = ''
267    '''Unicode encoded string containing the body of the reposne.'''
268
269    def __init__(self, response):
270        '''
271        Args:
272            response (:class:`mimetools.Message`): The object returned by a call
273            to :func:`urllib2.urlopen`.
274        '''
275        self._response = response
276        html = response.read()
277        try:
278            if response.headers['content-encoding'].lower() == 'gzip':
279                html = gzip.GzipFile(fileobj=StringIO.StringIO(html)).read()
280        except:
281            pass
282
283        try:
284            content_type = response.headers['content-type']
285            if 'charset=' in content_type:
286                encoding = content_type.split('charset=')[-1]
287        except:
288            pass
289
290        r = re.search('<meta\s+http-equiv="Content-Type"\s+content="(?:.+?);\s+charset=(.+?)"', html, re.IGNORECASE)
291        if r: encoding = r.group(1)
292        try: html = unicode(html, encoding)
293        except: pass
294       
295        self.content = html
296
297    def get_headers(self):
298        '''Returns a List of headers returned by the server.'''
299        return self._response.info().headers
300
301    def get_url(self):
302        '''
303        Return the URL of the resource retrieved, commonly used to determine if
304        a redirect was followed.
305        '''
306        return self._response.geturl()
Note: See TracBrowser for help on using the repository browser.