+ return urlsafe_b64encode(account['email'])
+
+ def _session_post (self, component, type='document', data={}, headers={}, params={}):
+ """Executes a get request using requests for the current session & measures the duration of that request
+
+ Parameters
+ ----------
+ component : :obj:`str`
+ Component to query
+
+ type : :obj:`str`
+ Is it a document or API request ('document' is default)
+
+ data : :obj:`dict` of :obj:`str`
+ Payload body as dict
+
+ header : :obj:`dict` of :obj:`str`
+ Additional headers as dict
+
+ params : :obj:`dict` of :obj:`str`
+ Request params
+
+ Returns
+ -------
+ :obj:`str`
+ Contents of the field to match
+ """
+ url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
+ start = time()
+ response = self.session.post(url=url, data=data, params=params, headers=headers, verify=self.verify_ssl)
+ end = time()
+ self.log(msg='[POST] Request for "' + url + '" took ' + str(end - start) + ' seconds')
+ return response
+
+ def _session_get (self, component, type='document', params={}):
+ """Executes a get request using requests for the current session & measures the duration of that request
+
+ Parameters
+ ----------
+ component : :obj:`str`
+ Component to query
+
+ type : :obj:`str`
+ Is it a document or API request ('document' is default)
+
+ params : :obj:`dict` of :obj:`str`
+ Request params
+
+ Returns
+ -------
+ :obj:`str`
+ Contents of the field to match
+ """
+ url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
+ start = time()
+ response = self.session.get(url=url, verify=self.verify_ssl, params=params)
+ end = time()
+ self.log(msg='[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')
+ return response
+
+ def _sloppy_parse_user_and_api_data (self, key, contents):
+ """Try to find the user & API data from the inline js by using a string parser
+
+ Parameters
+ ----------
+ key : :obj:`str`
+ Key to match in the inline js
+
+ contents : :obj:`str`
+ Inline JS contents
+
+ Returns
+ -------
+ :obj:`str`
+ Contents of the field to match
+ """
+ key_start = contents.find(key + '"')
+ if int(key_start) == -1:
+ return None
+ sub_contents = contents[int(key_start):]
+ l = sub_contents.find('",')
+ return contents[(int(key_start)+len(key)+3):int(key_start)+l].decode('string_escape')
+
+ def _sloppy_parse_profiles (self, contents):
+ """Try to find the profile data from the inline js by using a string parser & parse/convert the result to JSON
+
+ Parameters
+ ----------
+ contents : :obj:`str`
+ Inline JS contents
+
+ Returns
+ -------
+ :obj:`dict` of :obj:`str` or None
+ Profile data
+ """
+ profile_start = contents.find('profiles":')
+ profile_list_start = contents.find('profilesList')
+ if int(profile_start) > -1 and int(profile_list_start) > -1:
+ try:
+ try:
+ return json.loads('{"a":{"' + contents[profile_start:profile_list_start-2].decode('string_escape') + '}}').get('a').get('profiles')
+ except ValueError, e:
+ return None
+ except TypeError, e:
+ return None
+ return None
+
+ def _sloppy_parse_avatars (self, contents):
+ """Try to find the avatar data from the inline js by using a string parser & parse/convert the result to JSON
+
+ Parameters
+ ----------
+ contents : :obj:`str`
+ Inline JS contents
+
+ Returns
+ -------
+ :obj:`dict` of :obj:`str` or None
+ Avatar data
+ """
+ avatars_start = contents.find('"nf":')
+ avatars_list_start = contents.find('"profiles"')
+ if int(avatars_start) > -1 and int(avatars_list_start) > -1:
+ try:
+ try:
+ return json.loads('{' + contents[avatars_start:avatars_list_start-2].decode('string_escape') + '}')
+ except ValueError, e:
+ return None
+ except TypeError, e:
+ return None
+ return None
+
+ def _verfify_auth_and_profiles_data (self, data):
+ """Checks if the authURL has at least a certain length & doesn't overrule a certain length & if the profiles dict exists
+ Simple validity check for the sloppy data parser
+
+ Parameters
+ ----------
+ data : :obj:`dict` of :obj:`str`
+ Parsed JS contents
+
+ Returns
+ -------
+ bool
+ Data is valid
+ """
+ if type(data.get('profiles')) == dict:
+ if len(str(data.get('authURL', ''))) > 10 and len(str(data.get('authURL', ''))) < 50:
+ return True
+ return False
+
+ def _sloppy_parse_inline_data (self, scripts):
+ """Strips out all the needed user, api & profile data from the inline JS by string parsing
+ Might fail, so if this doesn't succeed, a proper JS parser will chime in
+
+ Note: This has been added for performance reasons only
+
+ Parameters
+ ----------
+ scripts : :obj:`list` of :obj:`BeautifoulSoup`
+ Script tags & contents from the Netflix browse page
+
+ Returns
+ -------
+ :obj:`dict` of :obj:`str`
+ Dict containijg user, api & profile data
+ """
+ inline_data = {};
+ for script in scripts:
+ contents = str(script.contents[0])
+ important_data = ['authURL', 'API_BASE_URL', 'API_ROOT', 'BUILD_IDENTIFIER', 'ICHNAEA_ROOT', 'gpsModel', 'guid', 'esn']
+ res = {}
+ for key in important_data:
+ _res = self._sloppy_parse_user_and_api_data(key, contents)
+ if _res != None:
+ res.update({key: _res})
+ if res != {}:
+ inline_data.update(res)
+
+ # parse profiles
+ profiles = self._sloppy_parse_profiles(contents)
+ avatars = self._sloppy_parse_avatars(contents)
+ if profiles != None:
+ inline_data.update({'profiles': profiles})
+ if avatars != None:
+ inline_data.update(avatars)
+ return inline_data
+
+ def _accurate_parse_inline_data (self, scripts):
+ """Uses a proper JS parser to fetch all the api, iser & profile data from within the inline JS
+
+ Note: This is slow but accurate
+
+ Parameters
+ ----------
+ scripts : :obj:`list` of :obj:`BeautifoulSoup`
+ Script tags & contents from the Netflix browse page
+
+ Returns
+ -------
+ :obj:`dict` of :obj:`str`
+ Dict containing user, api & profile data
+ """
+ inline_data = []
+ from pyjsparser import PyJsParser
+ parser = PyJsParser()
+ for script in scripts:
+ data = {}
+ # unicode escape that incoming script stuff
+ contents = self._to_unicode(str(script.contents[0]))
+ # parse the JS & load the declarations we´re interested in
+ parsed = parser.parse(contents)
+ if len(parsed['body']) > 1 and parsed['body'][1]['expression']['right'].get('properties', None) != None:
+ declarations = parsed['body'][1]['expression']['right']['properties']
+ for declaration in declarations:
+ for key in declaration:
+ # we found the correct path if the declaration is a dict & of type 'ObjectExpression'
+ if type(declaration[key]) is dict:
+ if declaration[key]['type'] == 'ObjectExpression':
+ # add all static data recursivly
+ for expression in declaration[key]['properties']:
+ data[expression['key']['value']] = self._parse_rec(expression['value'])
+ inline_data.append(data)
+ return inline_data
+
+ def _parse_rec (self, node):
+ """Iterates over a JavaScript AST and return values found
+
+ Parameters
+ ----------
+ value : :obj:`dict`
+ JS AST Expression
+ Returns
+ -------
+ :obj:`dict` of :obj:`dict` or :obj:`str`
+ Parsed contents of the node
+ """
+ if node['type'] == 'ObjectExpression':
+ _ret = {}
+ for prop in node['properties']:
+ _ret.update({prop['key']['value']: self._parse_rec(prop['value'])})
+ return _ret
+ if node['type'] == 'Literal':
+ return node['value']
+
+ def _parse_user_data (self, netflix_page_data):
+ """Parse out the user data from the big chunk of dicts we got from
+ parsing the JSON-ish data from the netflix homepage
+
+ Parameters
+ ----------
+ netflix_page_data : :obj:`list`
+ List of all the JSON-ish data that has been extracted from the Netflix homepage
+ see: extract_inline_netflix_page_data
+
+ Returns
+ -------
+ :obj:`dict` of :obj:`str`
+
+ {
+ "guid": "72ERT45...",
+ "authURL": "145637....",
+ "gpsModel": "harris"
+ }
+ """
+ user_data = {};
+ important_fields = [
+ 'authURL',
+ 'gpsModel',
+ 'guid'
+ ]
+
+ # values are accessible via dict (sloppy parsing successfull)
+ if type(netflix_page_data) == dict:
+ for important_field in important_fields:
+ user_data.update({important_field: netflix_page_data.get(important_field, '')})
+ return user_data
+
+ # values are stored in lists (returned from JS parser)
+ for item in netflix_page_data:
+ if 'memberContext' in dict(item).keys():
+ for important_field in important_fields:
+ user_data.update({important_field: item['memberContext']['data']['userInfo'][important_field]})
+
+ return user_data
+
+ def _parse_profile_data (self, netflix_page_data):
+ """Parse out the profile data from the big chunk of dicts we got from
+ parsing the JSON-ish data from the netflix homepage
+
+ Parameters
+ ----------
+ netflix_page_data : :obj:`list`
+ List of all the JSON-ish data that has been extracted from the Netflix homepage
+ see: extract_inline_netflix_page_data
+
+ Returns
+ -------
+ :obj:`dict` of :obj:`dict
+
+ {
+ "72ERT45...": {
+ "profileName": "username",
+ "avatar": "http://..../avatar.png",
+ "id": "72ERT45...",
+ "isAccountOwner": False,
+ "isActive": True,
+ "isFirstUse": False
+ }
+ }
+ """
+ profiles = {};
+ important_fields = [
+ 'profileName',
+ 'isActive',
+ 'isAccountOwner',
+ 'isKids'
+ ]
+ # values are accessible via dict (sloppy parsing successfull)
+ if type(netflix_page_data) == dict:
+ for profile_id in netflix_page_data.get('profiles'):
+ if self._is_size_key(key=profile_id) == False and type(netflix_page_data['profiles'][profile_id]) == dict and netflix_page_data['profiles'][profile_id].get('avatar', False) != False:
+ profile = {'id': profile_id}
+ for important_field in important_fields:
+ profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
+ avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
+ avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
+ profile.update({'avatar': avatar, 'isFirstUse': False})
+ profiles.update({profile_id: profile})
+ return profiles
+
+ # values are stored in lists (returned from JS parser)
+ # TODO: get rid of this christmas tree of doom
+ for item in netflix_page_data:
+ if 'hasViewedRatingWelcomeModal' in dict(item).keys():
+ for profile_id in item:
+ if self._is_size_key(key=profile_id) == False and type(item[profile_id]) == dict and item[profile_id].get('avatar', False) != False:
+ profile = {'id': profile_id}
+ for important_field in important_fields:
+ profile.update({important_field: item[profile_id]['summary'][important_field]})
+ avatar_base = item['nf'].get(item[profile_id]['summary']['avatarName'], False);
+ avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
+ profile.update({'avatar': avatar})
+ profiles.update({profile_id: profile})
+ return profiles
+
+ def _parse_api_base_data (self, netflix_page_data):
+ """Parse out the api url data from the big chunk of dicts we got from
+ parsing the JSOn-ish data from the netflix homepage
+
+ Parameters
+ ----------
+ netflix_page_data : :obj:`list`
+ List of all the JSON-ish data that has been extracted from the Netflix homepage
+ see: extract_inline_netflix_page_data
+
+ Returns
+ -------
+ :obj:`dict` of :obj:`str
+
+ {
+ "API_BASE_URL": "/shakti",
+ "API_ROOT": "https://www.netflix.com/api",
+ "BUILD_IDENTIFIER": "113b89c9",
+ "ICHNAEA_ROOT": "/ichnaea"
+ }
+ """
+ api_data = {};
+ important_fields = [
+ 'API_BASE_URL',
+ 'API_ROOT',
+ 'BUILD_IDENTIFIER',
+ 'ICHNAEA_ROOT'
+ ]
+
+ # values are accessible via dict (sloppy parsing successfull)
+ if type(netflix_page_data) == dict:
+ for important_field in important_fields:
+ api_data.update({important_field: netflix_page_data.get(important_field, '')})
+ return api_data
+
+ for item in netflix_page_data:
+ if 'serverDefs' in dict(item).keys():
+ for important_field in important_fields:
+ api_data.update({important_field: item['serverDefs']['data'][important_field]})
+ return api_data
+
+ def _parse_esn_data (self, netflix_page_data):
+ """Parse out the esn id data from the big chunk of dicts we got from
+ parsing the JSOn-ish data from the netflix homepage
+
+ Parameters
+ ----------
+ netflix_page_data : :obj:`list`
+ List of all the JSON-ish data that has been extracted from the Netflix homepage
+ see: extract_inline_netflix_page_data
+
+ Returns
+ -------
+ :obj:`str` of :obj:`str
+ ESN, something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME
+ """
+ # we generate an esn from device strings for android
+ import subprocess
+ try:
+ manufacturer = subprocess.check_output(["/system/bin/getprop", "ro.product.manufacturer"])
+ if manufacturer:
+ esn = 'NFANDROID1-PRV-'
+ input = subprocess.check_output(["/system/bin/getprop", "ro.nrdp.modelgroup"])
+ if not input:
+ esn = esn + 'T-L3-'
+ else:
+ esn = esn + input.strip(' \t\n\r') + '-'
+ esn = esn + '{:5}'.format(manufacturer.strip(' \t\n\r').upper())
+ input = subprocess.check_output(["/system/bin/getprop" ,"ro.product.model"])
+ esn = esn + input.strip(' \t\n\r').replace(' ', '=').upper()
+ self.log(msg='Android generated ESN:' + esn)
+ return esn
+ except OSError as e:
+ self.log(msg='Ignoring exception for non Android devices')
+
+ # values are accessible via dict (sloppy parsing successfull)
+ if type(netflix_page_data) == dict:
+ return netflix_page_data.get('esn', '')
+
+ esn = ''
+
+ # values are stored in lists (returned from JS parser)
+ for item in netflix_page_data:
+ if 'esnGeneratorModel' in dict(item).keys():
+ esn = item['esnGeneratorModel']['data']['esn']
+ return esn
+
+ def _parse_page_contents (self, page_soup):
+ """Call all the parsers we need to extract all the session relevant data from the HTML page
+ Directly assigns it to the NetflixSession instance
+
+ Parameters
+ ----------
+ page_soup : :obj:`BeautifulSoup`
+ Instance of an BeautifulSoup document or node containing the complete page contents
+ """
+ netflix_page_data = self.extract_inline_netflix_page_data(page_soup=page_soup)
+ self.user_data = self._parse_user_data(netflix_page_data=netflix_page_data)
+ self.esn = self._parse_esn_data(netflix_page_data=netflix_page_data)
+ self.api_data = self._parse_api_base_data(netflix_page_data=netflix_page_data)
+ self.profiles = self._parse_profile_data(netflix_page_data=netflix_page_data)
+ self.log(msg='Found ESN "' + self.esn + '"')
+ return netflix_page_data
projects / plugin.video.netflix.git / blobdiff