fix(esn): Fixes unicode problems

[plugin.video.netflix.git] / resources / lib / NetflixSession.py

diff --git a/resources/lib/NetflixSession.py b/resources/lib/NetflixSession.py
index d3334bcfb596065d35551d996f3e9b3528335b06..e5d32191bcb993adcf599f64d26897f14153fd02 100644 (file)
--- a/resources/lib/NetflixSession.py
+++ b/resources/lib/NetflixSession.py
@@ -6,7 +6,7 @@
 import os
 import json
 from requests import session, cookies
 import os
 import json
 from requests import session, cookies

-from urllib import quote
+from urllib import quote, unquote

 from time import time
 from base64 import urlsafe_b64encode
 from bs4 import BeautifulSoup, SoupStrainer
 from time import time
 from base64 import urlsafe_b64encode
 from bs4 import BeautifulSoup, SoupStrainer


@@ -24,15 +24,16 @@ class NetflixSession:
 
     urls = {
         'login': '/login',
 
     urls = {
         'login': '/login',

-        'browse': '/browse',
+        'browse': '/profiles/manage',

         'video_list_ids': '/preflight',
         'shakti': '/pathEvaluator',
         'video_list_ids': '/preflight',
         'shakti': '/pathEvaluator',

-        'profiles':  '/browse',
+        'profiles':  '/profiles/manage',

         'switch_profiles': '/profiles/switch',
         'adult_pin': '/pin/service',
         'metadata': '/metadata',
         'set_video_rating': '/setVideoRating',
         'switch_profiles': '/profiles/switch',
         'adult_pin': '/pin/service',
         'metadata': '/metadata',
         'set_video_rating': '/setVideoRating',

-        'update_my_list': '/playlistop'
+        'update_my_list': '/playlistop',
+        'kids': '/Kids'

     }
     """:obj:`dict` of :obj:`str` List of all static endpoints for HTML/JSON POST/GET requests"""
 
     }
     """:obj:`dict` of :obj:`str` List of all static endpoints for HTML/JSON POST/GET requests"""
 


@@ -149,12 +150,12 @@ class NetflixSession:
                 List of all the serialized data pulled out of the pagws <script/> tags
         """
         scripts = page_soup.find_all('script', attrs={'src': None});
                 List of all the serialized data pulled out of the pagws <script/> tags
         """
         scripts = page_soup.find_all('script', attrs={'src': None});

-        self.log('Trying sloppy inline data parser')
+        self.log(msg='Trying sloppy inline data parser')

         inline_data = self._sloppy_parse_inline_data(scripts=scripts)
         if self._verfify_auth_and_profiles_data(data=inline_data) != False:
         inline_data = self._sloppy_parse_inline_data(scripts=scripts)
         if self._verfify_auth_and_profiles_data(data=inline_data) != False:

-            self.log('Sloppy inline data parsing successfull')
+            self.log(msg='Sloppy inline data parsing successfull')

             return inline_data
             return inline_data

-        self.log('Sloppy inline parser failed, trying JS parser')
+        self.log(msg='Sloppy inline parser failed, trying JS parser')

         return self._accurate_parse_inline_data(scripts=scripts)
 
     def is_logged_in (self, account):
         return self._accurate_parse_inline_data(scripts=scripts)
 
     def is_logged_in (self, account):


@@ -280,14 +281,9 @@ class NetflixSession:
         if response.status_code != 200:
             return False
 
         if response.status_code != 200:
             return False
 

-        # fetch the index page again, so that we can fetch the corresponding user data
-        browse_response = self._session_get(component='browse')
-        only_script_tags = SoupStrainer('script')
-        browse_soup = BeautifulSoup(browse_response.text, 'html.parser', parse_only=only_script_tags)

         account_hash = self._generate_account_hash(account=account)
         self.user_data['guid'] = profile_id;
         account_hash = self._generate_account_hash(account=account)
         self.user_data['guid'] = profile_id;

-        self._save_data(filename=self.data_path + '_' + account_hash)
-        return True
+        return self._save_data(filename=self.data_path + '_' + account_hash)

 
     def send_adult_pin (self, pin):
         """Send the adult pin to Netflix in case an adult rated video requests it
 
     def send_adult_pin (self, pin):
         """Send the adult pin to Netflix in case an adult rated video requests it


@@ -450,17 +446,22 @@ class NetflixSession:
         for key in self.video_list_keys:
             video_list_ids[key] = {}
 
         for key in self.video_list_keys:
             video_list_ids[key] = {}
 

+        # check if the list items are hidden behind a `value` sub key
+        # this is the case when we fetch the lists via POST, not via a GET preflight request
+        if 'value' in response_data.keys():
+            response_data = response_data['value']
+

         # subcatogorize the lists by their context
         video_lists = response_data['lists']
         for video_list_id in video_lists.keys():
             video_list = video_lists[video_list_id]
         # subcatogorize the lists by their context
         video_lists = response_data['lists']
         for video_list_id in video_lists.keys():
             video_list = video_lists[video_list_id]

-            if video_list['context'] == 'genre':
-                video_list_ids['genres'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
-            elif video_list['context'] == 'similars' or video_list['context'] == 'becauseYouAdded':
-                video_list_ids['recommendations'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
-            else:
-                video_list_ids['user'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
-
+            if video_list.get('context', False) != False:
+                if video_list['context'] == 'genre':
+                    video_list_ids['genres'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
+                elif video_list['context'] == 'similars' or video_list['context'] == 'becauseYouAdded':
+                    video_list_ids['recommendations'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
+                else:
+                    video_list_ids['user'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))

         return video_list_ids
 
     def parse_video_list_ids_entry (self, id, entry):
         return video_list_ids
 
     def parse_video_list_ids_entry (self, id, entry):


@@ -773,7 +774,7 @@ class NetflixSession:
                 'synopsis': video['synopsis'],
                 'regular_synopsis': video['regularSynopsis'],
                 'type': video['summary']['type'],
                 'synopsis': video['synopsis'],
                 'regular_synopsis': video['regularSynopsis'],
                 'type': video['summary']['type'],

-                'rating': video['userRating']['average'],
+                'rating': video['userRating'].get('average', 0) if video['userRating'].get('average', None) != None else video['userRating'].get('predicted', 0),

                 'episode_count': season_info['episode_count'],
                 'seasons_label': season_info['seasons_label'],
                 'seasons_count': season_info['seasons_count'],
                 'episode_count': season_info['episode_count'],
                 'seasons_label': season_info['seasons_label'],
                 'seasons_count': season_info['seasons_count'],


@@ -1249,10 +1250,10 @@ class NetflixSession:
                 'title': episode['info']['title'],
                 'year': episode['info']['releaseYear'],
                 'genres': self.parse_genres_for_video(video=episode, genres=genres),
                 'title': episode['info']['title'],
                 'year': episode['info']['releaseYear'],
                 'genres': self.parse_genres_for_video(video=episode, genres=genres),

-                'mpaa': str(episode['maturity']['rating']['board']) + ' ' + str(episode['maturity']['rating']['value']),
+                'mpaa': str(episode['maturity']['rating']['board']).encode('utf-8') + ' ' + str(episode['maturity']['rating']['value']).encode('utf-8'),

                 'maturity': episode['maturity'],
                 'playcount': (0, 1)[episode['watched']],
                 'maturity': episode['maturity'],
                 'playcount': (0, 1)[episode['watched']],

-                'rating': episode['userRating']['average'],
+                'rating': episode['userRating'].get('average', 0) if episode['userRating'].get('average', None) != None else episode['userRating'].get('predicted', 0),

                 'thumb': episode['info']['interestingMoments']['url'],
                 'fanart': episode['interestingMoment']['_1280x720']['jpg']['url'],
                 'poster': episode['boxarts']['_1280x720']['jpg']['url'],
                 'thumb': episode['info']['interestingMoments']['url'],
                 'fanart': episode['interestingMoment']['_1280x720']['jpg']['url'],
                 'poster': episode['boxarts']['_1280x720']['jpg']['url'],


@@ -1274,8 +1275,9 @@ class NetflixSession:
         response = self._session_get(component='browse')
         return BeautifulSoup(response.text, 'html.parser')
 
         response = self._session_get(component='browse')
         return BeautifulSoup(response.text, 'html.parser')
 

-    def fetch_video_list_ids (self, list_from=0, list_to=50):
+    def fetch_video_list_ids_via_preflight (self, list_from=0, list_to=50):

         """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
         """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix

+           via the preflight (GET) request

 
         Parameters
         ----------
 
         Parameters
         ----------


@@ -1298,9 +1300,33 @@ class NetflixSession:
             '_': int(time()),
             'authURL': self.user_data['authURL']
         }
             '_': int(time()),
             'authURL': self.user_data['authURL']
         }

+

         response = self._session_get(component='video_list_ids', params=payload, type='api')
         return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
 
         response = self._session_get(component='video_list_ids', params=payload, type='api')
         return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
 

+    def fetch_video_list_ids (self, list_from=0, list_to=50):
+        """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
+
+        Parameters
+        ----------
+        list_from : :obj:`int`
+            Start entry for pagination
+
+        list_to : :obj:`int`
+            Last entry for pagination
+
+        Returns
+        -------
+        :obj:`dict` of :obj:`dict` of :obj:`str`
+            Raw Netflix API call response or api call error
+        """
+        paths = [
+            ['lolomo', {'from': list_from, 'to': list_to}, ['displayName', 'context', 'id', 'index', 'length']]
+        ]
+
+        response = self._path_request(paths=paths)
+        return self._process_response(response=response, component='Video list ids')
+

     def fetch_search_results (self, search_str, list_from=0, list_to=10):
         """Fetches the JSON which contains the results for the given search query
 
     def fetch_search_results (self, search_str, list_from=0, list_to=10):
         """Fetches the JSON which contains the results for the given search query
 


@@ -1568,8 +1594,6 @@ class NetflixSession:
         })
 
         params = {
         })
 
         params = {

-            'withSize': True,
-            'materialize': True,

             'model': self.user_data['gpsModel']
         }
 
             'model': self.user_data['gpsModel']
         }
 


@@ -1860,14 +1884,14 @@ class NetflixSession:
             User Agent for platform
         """
         import platform
             User Agent for platform
         """
         import platform

-        if platform == 'linux' or platform == 'linux2':
-            return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        elif platform == 'darwin':
+        self.log(msg='Building User Agent for platform: ' + str(platform.system()) + ' - ' + str(platform.machine()))
+        if platform.system() == 'Darwin':

             return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
             return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

-        elif platform == 'win32':
+        if platform.system() == 'Windows':

             return 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
             return 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

-        else:
-            return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
+        if platform.machine().startswith('arm'):
+            return 'Mozilla/5.0 (X11; CrOS armv7l 7647.78.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/48.0.2564.109 Safari/537.36'
+        return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'

 
     def _session_post (self, component, type='document', data={}, headers={}, params={}):
         """Executes a get request using requests for the current session & measures the duration of that request
 
     def _session_post (self, component, type='document', data={}, headers={}, params={}):
         """Executes a get request using requests for the current session & measures the duration of that request


@@ -1898,7 +1922,7 @@ class NetflixSession:
         start = time()
         response = self.session.post(url=url, data=data, params=params, headers=headers, verify=self.verify_ssl)
         end = time()
         start = time()
         response = self.session.post(url=url, data=data, params=params, headers=headers, verify=self.verify_ssl)
         end = time()

-        self.log('[POST] Request for "' + url + '" took ' + str(end - start) + ' seconds')
+        self.log(msg='[POST] Request for "' + url + '" took ' + str(end - start) + ' seconds')

         return response
 
     def _session_get (self, component, type='document', params={}):
         return response
 
     def _session_get (self, component, type='document', params={}):


@@ -1924,7 +1948,7 @@ class NetflixSession:
         start = time()
         response = self.session.get(url=url, verify=self.verify_ssl, params=params)
         end = time()
         start = time()
         response = self.session.get(url=url, verify=self.verify_ssl, params=params)
         end = time()

-        self.log('[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')
+        self.log(msg='[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')

         return response
 
     def _sloppy_parse_user_and_api_data (self, key, contents):
         return response
 
     def _sloppy_parse_user_and_api_data (self, key, contents):


@@ -2183,10 +2207,9 @@ class NetflixSession:
         important_fields = [
             'profileName',
             'isActive',
         important_fields = [
             'profileName',
             'isActive',

-            'isFirstUse',
-            'isAccountOwner'
+            'isAccountOwner',
+            'isKids'

         ]
         ]

-

         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             for profile_id in netflix_page_data.get('profiles'):
         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             for profile_id in netflix_page_data.get('profiles'):


@@ -2196,7 +2219,7 @@ class NetflixSession:
                         profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
                     avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
                     avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
                         profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
                     avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
                     avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']

-                    profile.update({'avatar': avatar})
+                    profile.update({'avatar': avatar, 'isFirstUse': False})

                     profiles.update({profile_id: profile})
             return profiles
 
                     profiles.update({profile_id: profile})
             return profiles
 


@@ -2296,5 +2319,5 @@ class NetflixSession:
         self.esn = self._parse_esn_data(netflix_page_data=netflix_page_data)
         self.api_data = self._parse_api_base_data(netflix_page_data=netflix_page_data)
         self.profiles = self._parse_profile_data(netflix_page_data=netflix_page_data)
         self.esn = self._parse_esn_data(netflix_page_data=netflix_page_data)
         self.api_data = self._parse_api_base_data(netflix_page_data=netflix_page_data)
         self.profiles = self._parse_profile_data(netflix_page_data=netflix_page_data)

-        self.log('Found ESN "' + self.esn + '"')
+        self.log(msg='Found ESN "' + self.esn + '"')

         return netflix_page_data
         return netflix_page_data