Rename NetflixSession.parse_season_entry() to _parse_season_entry()...

[plugin.video.netflix.git] / resources / lib / NetflixSession.py

diff --git a/resources/lib/NetflixSession.py b/resources/lib/NetflixSession.py
index d3334bcfb596065d35551d996f3e9b3528335b06..a4c48744ef7a933cd6e0cc18ef2dc8b9a0dc4929 100644 (file)
--- a/resources/lib/NetflixSession.py
+++ b/resources/lib/NetflixSession.py
@@ -6,11 +6,11 @@
 import os
 import json
 from requests import session, cookies
 import os
 import json
 from requests import session, cookies

-from urllib import quote
+from urllib import quote, unquote

 from time import time
 from base64 import urlsafe_b64encode
 from bs4 import BeautifulSoup, SoupStrainer
 from time import time
 from base64 import urlsafe_b64encode
 from bs4 import BeautifulSoup, SoupStrainer

-from utils import noop
+from utils import noop, get_user_agent_for_current_platform

 try:
    import cPickle as pickle
 except:
 try:
    import cPickle as pickle
 except:


@@ -24,15 +24,16 @@ class NetflixSession:
 
     urls = {
         'login': '/login',
 
     urls = {
         'login': '/login',

-        'browse': '/browse',
+        'browse': '/profiles/manage',

         'video_list_ids': '/preflight',
         'shakti': '/pathEvaluator',
         'video_list_ids': '/preflight',
         'shakti': '/pathEvaluator',

-        'profiles':  '/browse',
+        'profiles':  '/profiles/manage',

         'switch_profiles': '/profiles/switch',
         'adult_pin': '/pin/service',
         'metadata': '/metadata',
         'set_video_rating': '/setVideoRating',
         'switch_profiles': '/profiles/switch',
         'adult_pin': '/pin/service',
         'metadata': '/metadata',
         'set_video_rating': '/setVideoRating',

-        'update_my_list': '/playlistop'
+        'update_my_list': '/playlistop',
+        'kids': '/Kids'

     }
     """:obj:`dict` of :obj:`str` List of all static endpoints for HTML/JSON POST/GET requests"""
 
     }
     """:obj:`dict` of :obj:`str` List of all static endpoints for HTML/JSON POST/GET requests"""
 


@@ -102,7 +103,7 @@ class NetflixSession:
         # start session, fake chrome on the current platform (so that we get a proper widevine esn) & enable gzip
         self.session = session()
         self.session.headers.update({
         # start session, fake chrome on the current platform (so that we get a proper widevine esn) & enable gzip
         self.session = session()
         self.session.headers.update({

-            'User-Agent': self._get_user_agent_for_current_platform(),
+            'User-Agent': get_user_agent_for_current_platform(),

             'Accept-Encoding': 'gzip'
         })
 
             'Accept-Encoding': 'gzip'
         })
 


@@ -148,13 +149,13 @@ class NetflixSession:
             :obj:`list` of :obj:`dict`
                 List of all the serialized data pulled out of the pagws <script/> tags
         """
             :obj:`list` of :obj:`dict`
                 List of all the serialized data pulled out of the pagws <script/> tags
         """

-        scripts = page_soup.find_all('script', attrs={'src': None});
-        self.log('Trying sloppy inline data parser')
+        scripts = page_soup.find_all('script', attrs={'src': None})
+        self.log(msg='Trying sloppy inline data parser')

         inline_data = self._sloppy_parse_inline_data(scripts=scripts)
         if self._verfify_auth_and_profiles_data(data=inline_data) != False:
         inline_data = self._sloppy_parse_inline_data(scripts=scripts)
         if self._verfify_auth_and_profiles_data(data=inline_data) != False:

-            self.log('Sloppy inline data parsing successfull')
+            self.log(msg='Sloppy inline data parsing successfull')

             return inline_data
             return inline_data

-        self.log('Sloppy inline parser failed, trying JS parser')
+        self.log(msg='Sloppy inline parser failed, trying JS parser')

         return self._accurate_parse_inline_data(scripts=scripts)
 
     def is_logged_in (self, account):
         return self._accurate_parse_inline_data(scripts=scripts)
 
     def is_logged_in (self, account):


@@ -280,14 +281,9 @@ class NetflixSession:
         if response.status_code != 200:
             return False
 
         if response.status_code != 200:
             return False
 

-        # fetch the index page again, so that we can fetch the corresponding user data
-        browse_response = self._session_get(component='browse')
-        only_script_tags = SoupStrainer('script')
-        browse_soup = BeautifulSoup(browse_response.text, 'html.parser', parse_only=only_script_tags)

         account_hash = self._generate_account_hash(account=account)
         self.user_data['guid'] = profile_id;
         account_hash = self._generate_account_hash(account=account)
         self.user_data['guid'] = profile_id;

-        self._save_data(filename=self.data_path + '_' + account_hash)
-        return True
+        return self._save_data(filename=self.data_path + '_' + account_hash)

 
     def send_adult_pin (self, pin):
         """Send the adult pin to Netflix in case an adult rated video requests it
 
     def send_adult_pin (self, pin):
         """Send the adult pin to Netflix in case an adult rated video requests it


@@ -450,17 +446,22 @@ class NetflixSession:
         for key in self.video_list_keys:
             video_list_ids[key] = {}
 
         for key in self.video_list_keys:
             video_list_ids[key] = {}
 

+        # check if the list items are hidden behind a `value` sub key
+        # this is the case when we fetch the lists via POST, not via a GET preflight request
+        if 'value' in response_data.keys():
+            response_data = response_data['value']
+

         # subcatogorize the lists by their context
         video_lists = response_data['lists']
         for video_list_id in video_lists.keys():
             video_list = video_lists[video_list_id]
         # subcatogorize the lists by their context
         video_lists = response_data['lists']
         for video_list_id in video_lists.keys():
             video_list = video_lists[video_list_id]

-            if video_list['context'] == 'genre':
-                video_list_ids['genres'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
-            elif video_list['context'] == 'similars' or video_list['context'] == 'becauseYouAdded':
-                video_list_ids['recommendations'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
-            else:
-                video_list_ids['user'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
-
+            if video_list.get('context', False) != False:
+                if video_list['context'] == 'genre':
+                    video_list_ids['genres'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
+                elif video_list['context'] == 'similars' or video_list['context'] == 'becauseYouAdded':
+                    video_list_ids['recommendations'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
+                else:
+                    video_list_ids['user'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))

         return video_list_ids
 
     def parse_video_list_ids_entry (self, id, entry):
         return video_list_ids
 
     def parse_video_list_ids_entry (self, id, entry):


@@ -773,7 +774,7 @@ class NetflixSession:
                 'synopsis': video['synopsis'],
                 'regular_synopsis': video['regularSynopsis'],
                 'type': video['summary']['type'],
                 'synopsis': video['synopsis'],
                 'regular_synopsis': video['regularSynopsis'],
                 'type': video['summary']['type'],

-                'rating': video['userRating']['average'],
+                'rating': video['userRating'].get('average', 0) if video['userRating'].get('average', None) != None else video['userRating'].get('predicted', 0),

                 'episode_count': season_info['episode_count'],
                 'seasons_label': season_info['seasons_label'],
                 'seasons_count': season_info['seasons_count'],
                 'episode_count': season_info['episode_count'],
                 'seasons_label': season_info['seasons_label'],
                 'seasons_count': season_info['seasons_count'],


@@ -1073,14 +1074,29 @@ class NetflixSession:
                 }
             }
         """
                 }
             }
         """

-        seasons = {}

         raw_seasons = response_data['value']
         raw_seasons = response_data['value']

+        videos = raw_seasons['videos']
+
+        # get art video key
+        video = {}
+        for key, video_candidate in videos.iteritems():
+            if not self._is_size_key(key):
+                video = video_candidate
+
+        # get season index
+        sorting = {}
+        for idx, season_list_entry in video['seasonList'].iteritems():
+            if self._is_size_key(key=idx) == False and idx != 'summary':
+                sorting[int(season_list_entry[1])] = int(idx)
+
+        seasons = {}
+

         for season in raw_seasons['seasons']:
             if self._is_size_key(key=season) == False:
         for season in raw_seasons['seasons']:
             if self._is_size_key(key=season) == False:

-                seasons.update(self.parse_season_entry(season=raw_seasons['seasons'][season], videos=raw_seasons['videos']))
+                seasons.update(self._parse_season_entry(season=raw_seasons['seasons'][season], video=video, sorting=sorting))

         return seasons
 
         return seasons
 

-    def parse_season_entry (self, season, videos):
+    def _parse_season_entry (self, season, video, sorting):

         """Parse a season list entry e.g. rip out the parts we need
 
         Parameters
         """Parse a season list entry e.g. rip out the parts we need
 
         Parameters


@@ -1106,16 +1122,6 @@ class NetflixSession:
                 }
             }
         """
                 }
             }
         """

-        # get art video key
-        video_key = ''
-        for key in videos.keys():
-            if self._is_size_key(key=key) == False:
-                video_key = key
-        # get season index
-        sorting = {}
-        for idx in videos[video_key]['seasonList']:
-            if self._is_size_key(key=idx) == False and idx != 'summary':
-                sorting[int(videos[video_key]['seasonList'][idx][1])] = int(idx)

         return {
             season['summary']['id']: {
                 'idx': sorting[season['summary']['id']],
         return {
             season['summary']['id']: {
                 'idx': sorting[season['summary']['id']],


@@ -1123,10 +1129,10 @@ class NetflixSession:
                 'text': season['summary']['name'],
                 'shortName': season['summary']['shortName'],
                 'boxarts': {
                 'text': season['summary']['name'],
                 'shortName': season['summary']['shortName'],
                 'boxarts': {

-                    'small': videos[video_key]['boxarts']['_342x192']['jpg']['url'],
-                    'big': videos[video_key]['boxarts']['_1280x720']['jpg']['url']
+                    'small': video['boxarts']['_342x192']['jpg']['url'],
+                    'big': video['boxarts']['_1280x720']['jpg']['url']

                 },
                 },

-                'interesting_moment': videos[video_key]['interestingMoment']['_665x375']['jpg']['url'],
+                'interesting_moment': video['interestingMoment']['_665x375']['jpg']['url'],

             }
         }
 
             }
         }
 


@@ -1239,6 +1245,11 @@ class NetflixSession:
           },
         }
         """
           },
         }
         """

+        mpaa = ''
+        if episode.get('maturity', None) is not None:
+            if episode['maturity'].get('board', None) is not None and episode['maturity'].get('value', None) is not None:
+                mpaa = str(episode['maturity'].get('board', '').encode('utf-8')) + '-' + str(episode['maturity'].get('value', '').encode('utf-8'))
+

         return {
             episode['summary']['id']: {
                 'id': episode['summary']['id'],
         return {
             episode['summary']['id']: {
                 'id': episode['summary']['id'],


@@ -1249,10 +1260,10 @@ class NetflixSession:
                 'title': episode['info']['title'],
                 'year': episode['info']['releaseYear'],
                 'genres': self.parse_genres_for_video(video=episode, genres=genres),
                 'title': episode['info']['title'],
                 'year': episode['info']['releaseYear'],
                 'genres': self.parse_genres_for_video(video=episode, genres=genres),

-                'mpaa': str(episode['maturity']['rating']['board']) + ' ' + str(episode['maturity']['rating']['value']),
+                'mpaa': mpaa,

                 'maturity': episode['maturity'],
                 'playcount': (0, 1)[episode['watched']],
                 'maturity': episode['maturity'],
                 'playcount': (0, 1)[episode['watched']],

-                'rating': episode['userRating']['average'],
+                'rating': episode['userRating'].get('average', 0) if episode['userRating'].get('average', None) != None else episode['userRating'].get('predicted', 0),

                 'thumb': episode['info']['interestingMoments']['url'],
                 'fanart': episode['interestingMoment']['_1280x720']['jpg']['url'],
                 'poster': episode['boxarts']['_1280x720']['jpg']['url'],
                 'thumb': episode['info']['interestingMoments']['url'],
                 'fanart': episode['interestingMoment']['_1280x720']['jpg']['url'],
                 'poster': episode['boxarts']['_1280x720']['jpg']['url'],


@@ -1274,8 +1285,9 @@ class NetflixSession:
         response = self._session_get(component='browse')
         return BeautifulSoup(response.text, 'html.parser')
 
         response = self._session_get(component='browse')
         return BeautifulSoup(response.text, 'html.parser')
 

-    def fetch_video_list_ids (self, list_from=0, list_to=50):
+    def fetch_video_list_ids_via_preflight (self, list_from=0, list_to=50):

         """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
         """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix

+           via the preflight (GET) request

 
         Parameters
         ----------
 
         Parameters
         ----------


@@ -1298,9 +1310,33 @@ class NetflixSession:
             '_': int(time()),
             'authURL': self.user_data['authURL']
         }
             '_': int(time()),
             'authURL': self.user_data['authURL']
         }

+

         response = self._session_get(component='video_list_ids', params=payload, type='api')
         return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
 
         response = self._session_get(component='video_list_ids', params=payload, type='api')
         return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
 

+    def fetch_video_list_ids (self, list_from=0, list_to=50):
+        """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
+
+        Parameters
+        ----------
+        list_from : :obj:`int`
+            Start entry for pagination
+
+        list_to : :obj:`int`
+            Last entry for pagination
+
+        Returns
+        -------
+        :obj:`dict` of :obj:`dict` of :obj:`str`
+            Raw Netflix API call response or api call error
+        """
+        paths = [
+            ['lolomo', {'from': list_from, 'to': list_to}, ['displayName', 'context', 'id', 'index', 'length']]
+        ]
+
+        response = self._path_request(paths=paths)
+        return self._process_response(response=response, component='Video list ids')
+

     def fetch_search_results (self, search_str, list_from=0, list_to=10):
         """Fetches the JSON which contains the results for the given search query
 
     def fetch_search_results (self, search_str, list_from=0, list_to=10):
         """Fetches the JSON which contains the results for the given search query
 


@@ -1334,7 +1370,7 @@ class NetflixSession:
         response = self._path_request(paths=paths)
         return self._process_response(response=response, component='Search results')
 
         response = self._path_request(paths=paths)
         return self._process_response(response=response, component='Search results')
 

-    def fetch_video_list (self, list_id, list_from=0, list_to=20):
+    def fetch_video_list (self, list_id, list_from=0, list_to=26):

         """Fetches the JSON which contains the contents of a given video list
 
         Parameters
         """Fetches the JSON which contains the contents of a given video list
 
         Parameters


@@ -1568,8 +1604,6 @@ class NetflixSession:
         })
 
         params = {
         })
 
         params = {

-            'withSize': True,
-            'materialize': True,

             'model': self.user_data['gpsModel']
         }
 
             'model': self.user_data['gpsModel']
         }
 


@@ -1851,24 +1885,6 @@ class NetflixSession:
         """
         return urlsafe_b64encode(account['email'])
 
         """
         return urlsafe_b64encode(account['email'])
 

-    def _get_user_agent_for_current_platform (self):
-        """Determines the user agent string for the current platform (to retrieve a valid ESN)
-
-        Returns
-        -------
-        :obj:`str`
-            User Agent for platform
-        """
-        import platform
-        if platform == 'linux' or platform == 'linux2':
-            return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        elif platform == 'darwin':
-            return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        elif platform == 'win32':
-            return 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        else:
-            return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-

     def _session_post (self, component, type='document', data={}, headers={}, params={}):
         """Executes a get request using requests for the current session & measures the duration of that request
 
     def _session_post (self, component, type='document', data={}, headers={}, params={}):
         """Executes a get request using requests for the current session & measures the duration of that request
 


@@ -1898,7 +1914,7 @@ class NetflixSession:
         start = time()
         response = self.session.post(url=url, data=data, params=params, headers=headers, verify=self.verify_ssl)
         end = time()
         start = time()
         response = self.session.post(url=url, data=data, params=params, headers=headers, verify=self.verify_ssl)
         end = time()

-        self.log('[POST] Request for "' + url + '" took ' + str(end - start) + ' seconds')
+        self.log(msg='[POST] Request for "' + url + '" took ' + str(end - start) + ' seconds')

         return response
 
     def _session_get (self, component, type='document', params={}):
         return response
 
     def _session_get (self, component, type='document', params={}):


@@ -1924,7 +1940,7 @@ class NetflixSession:
         start = time()
         response = self.session.get(url=url, verify=self.verify_ssl, params=params)
         end = time()
         start = time()
         response = self.session.get(url=url, verify=self.verify_ssl, params=params)
         end = time()

-        self.log('[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')
+        self.log(msg='[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')

         return response
 
     def _sloppy_parse_user_and_api_data (self, key, contents):
         return response
 
     def _sloppy_parse_user_and_api_data (self, key, contents):


@@ -2183,10 +2199,9 @@ class NetflixSession:
         important_fields = [
             'profileName',
             'isActive',
         important_fields = [
             'profileName',
             'isActive',

-            'isFirstUse',
-            'isAccountOwner'
+            'isAccountOwner',
+            'isKids'

         ]
         ]

-

         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             for profile_id in netflix_page_data.get('profiles'):
         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             for profile_id in netflix_page_data.get('profiles'):


@@ -2196,7 +2211,7 @@ class NetflixSession:
                         profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
                     avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
                     avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
                         profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
                     avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
                     avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']

-                    profile.update({'avatar': avatar})
+                    profile.update({'avatar': avatar, 'isFirstUse': False})

                     profiles.update({profile_id: profile})
             return profiles
 
                     profiles.update({profile_id: profile})
             return profiles
 


@@ -2271,11 +2286,31 @@ class NetflixSession:
             :obj:`str` of :obj:`str
             ESN, something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME
         """
             :obj:`str` of :obj:`str
             ESN, something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME
         """

-        esn = ''
+        # we generate an esn from device strings for android
+        import subprocess
+        try:
+            manufacturer = subprocess.check_output(["/system/bin/getprop", "ro.product.manufacturer"])
+            if manufacturer:
+                esn = 'NFANDROID1-PRV-'
+                input = subprocess.check_output(["/system/bin/getprop", "ro.nrdp.modelgroup"])
+                if not input:
+                    esn = esn + 'T-L3-'
+                else:
+                    esn = esn + input.strip(' \t\n\r') + '-'
+                esn = esn + '{:5}'.format(manufacturer.strip(' \t\n\r').upper())
+                input = subprocess.check_output(["/system/bin/getprop" ,"ro.product.model"])
+                esn = esn + input.strip(' \t\n\r').replace(' ', '=').upper()
+                self.log(msg='Android generated ESN:' + esn)
+                return esn
+        except OSError as e:
+            self.log(msg='Ignoring exception for non Android devices')
+

         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             return netflix_page_data.get('esn', '')
 
         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             return netflix_page_data.get('esn', '')
 

+        esn = ''
+

         # values are stored in lists (returned from JS parser)
         for item in netflix_page_data:
             if 'esnGeneratorModel' in dict(item).keys():
         # values are stored in lists (returned from JS parser)
         for item in netflix_page_data:
             if 'esnGeneratorModel' in dict(item).keys():


@@ -2296,5 +2331,5 @@ class NetflixSession:
         self.esn = self._parse_esn_data(netflix_page_data=netflix_page_data)
         self.api_data = self._parse_api_base_data(netflix_page_data=netflix_page_data)
         self.profiles = self._parse_profile_data(netflix_page_data=netflix_page_data)
         self.esn = self._parse_esn_data(netflix_page_data=netflix_page_data)
         self.api_data = self._parse_api_base_data(netflix_page_data=netflix_page_data)
         self.profiles = self._parse_profile_data(netflix_page_data=netflix_page_data)

-        self.log('Found ESN "' + self.esn + '"')
+        self.log(msg='Found ESN "' + self.esn + '"')

         return netflix_page_data
         return netflix_page_data