Simplify NetflixSession.parse_genres_for_video().

[plugin.video.netflix.git] / resources / lib / NetflixSession.py

diff --git a/resources/lib/NetflixSession.py b/resources/lib/NetflixSession.py
index 7fcea384d88b330b07cc1c9658b201f387071f9e..62e58fd0d76fc05ba78667774a420e61b50b7f79 100644 (file)
--- a/resources/lib/NetflixSession.py
+++ b/resources/lib/NetflixSession.py
@@ -10,7 +10,7 @@ from urllib import quote, unquote
 from time import time
 from base64 import urlsafe_b64encode
 from bs4 import BeautifulSoup, SoupStrainer
 from time import time
 from base64 import urlsafe_b64encode
 from bs4 import BeautifulSoup, SoupStrainer

-from utils import noop
+from utils import noop, get_user_agent_for_current_platform

 try:
    import cPickle as pickle
 except:
 try:
    import cPickle as pickle
 except:


@@ -103,7 +103,7 @@ class NetflixSession:
         # start session, fake chrome on the current platform (so that we get a proper widevine esn) & enable gzip
         self.session = session()
         self.session.headers.update({
         # start session, fake chrome on the current platform (so that we get a proper widevine esn) & enable gzip
         self.session = session()
         self.session.headers.update({

-            'User-Agent': self._get_user_agent_for_current_platform(),
+            'User-Agent': get_user_agent_for_current_platform(),

             'Accept-Encoding': 'gzip'
         })
 
             'Accept-Encoding': 'gzip'
         })
 


@@ -149,7 +149,7 @@ class NetflixSession:
             :obj:`list` of :obj:`dict`
                 List of all the serialized data pulled out of the pagws <script/> tags
         """
             :obj:`list` of :obj:`dict`
                 List of all the serialized data pulled out of the pagws <script/> tags
         """

-        scripts = page_soup.find_all('script', attrs={'src': None});
+        scripts = page_soup.find_all('script', attrs={'src': None})

         self.log(msg='Trying sloppy inline data parser')
         inline_data = self._sloppy_parse_inline_data(scripts=scripts)
         if self._verfify_auth_and_profiles_data(data=inline_data) != False:
         self.log(msg='Trying sloppy inline data parser')
         inline_data = self._sloppy_parse_inline_data(scripts=scripts)
         if self._verfify_auth_and_profiles_data(data=inline_data) != False:


@@ -446,6 +446,11 @@ class NetflixSession:
         for key in self.video_list_keys:
             video_list_ids[key] = {}
 
         for key in self.video_list_keys:
             video_list_ids[key] = {}
 

+        # check if the list items are hidden behind a `value` sub key
+        # this is the case when we fetch the lists via POST, not via a GET preflight request
+        if 'value' in response_data.keys():
+            response_data = response_data['value']
+

         # subcatogorize the lists by their context
         video_lists = response_data['lists']
         for video_list_id in video_lists.keys():
         # subcatogorize the lists by their context
         video_lists = response_data['lists']
         for video_list_id in video_lists.keys():


@@ -890,12 +895,14 @@ class NetflixSession:
             List of genres
         """
         video_genres = []
             List of genres
         """
         video_genres = []

-        for genre_key in dict(genres).keys():
-            if self._is_size_key(key=genre_key) == False and genre_key != 'summary':
-                for show_genre_key in dict(video['genres']).keys():
-                    if self._is_size_key(key=show_genre_key) == False and show_genre_key != 'summary':
-                        if video['genres'][show_genre_key][1] == genre_key:
-                            video_genres.append(genres[genre_key]['name'])
+
+        for video_genre_key, video_genre in video['genres'].iteritems():
+            if self._is_size_key(video_genre_key) == False and video_genre_key != 'summary':
+                name = genres.get(video_genre[1], {}).get('name')
+
+                if name:
+                    video_genres.append(name)
+

         return video_genres
 
     def parse_tags_for_video (self, video):
         return video_genres
 
     def parse_tags_for_video (self, video):


@@ -1069,14 +1076,29 @@ class NetflixSession:
                 }
             }
         """
                 }
             }
         """

-        seasons = {}

         raw_seasons = response_data['value']
         raw_seasons = response_data['value']

+        videos = raw_seasons['videos']
+
+        # get art video key
+        video = {}
+        for key, video_candidate in videos.iteritems():
+            if not self._is_size_key(key):
+                video = video_candidate
+
+        # get season index
+        sorting = {}
+        for idx, season_list_entry in video['seasonList'].iteritems():
+            if self._is_size_key(key=idx) == False and idx != 'summary':
+                sorting[int(season_list_entry[1])] = int(idx)
+
+        seasons = {}
+

         for season in raw_seasons['seasons']:
             if self._is_size_key(key=season) == False:
         for season in raw_seasons['seasons']:
             if self._is_size_key(key=season) == False:

-                seasons.update(self.parse_season_entry(season=raw_seasons['seasons'][season], videos=raw_seasons['videos']))
+                seasons.update(self._parse_season_entry(season=raw_seasons['seasons'][season], video=video, sorting=sorting))

         return seasons
 
         return seasons
 

-    def parse_season_entry (self, season, videos):
+    def _parse_season_entry (self, season, video, sorting):

         """Parse a season list entry e.g. rip out the parts we need
 
         Parameters
         """Parse a season list entry e.g. rip out the parts we need
 
         Parameters


@@ -1102,16 +1124,6 @@ class NetflixSession:
                 }
             }
         """
                 }
             }
         """

-        # get art video key
-        video_key = ''
-        for key in videos.keys():
-            if self._is_size_key(key=key) == False:
-                video_key = key
-        # get season index
-        sorting = {}
-        for idx in videos[video_key]['seasonList']:
-            if self._is_size_key(key=idx) == False and idx != 'summary':
-                sorting[int(videos[video_key]['seasonList'][idx][1])] = int(idx)

         return {
             season['summary']['id']: {
                 'idx': sorting[season['summary']['id']],
         return {
             season['summary']['id']: {
                 'idx': sorting[season['summary']['id']],


@@ -1119,10 +1131,10 @@ class NetflixSession:
                 'text': season['summary']['name'],
                 'shortName': season['summary']['shortName'],
                 'boxarts': {
                 'text': season['summary']['name'],
                 'shortName': season['summary']['shortName'],
                 'boxarts': {

-                    'small': videos[video_key]['boxarts']['_342x192']['jpg']['url'],
-                    'big': videos[video_key]['boxarts']['_1280x720']['jpg']['url']
+                    'small': video['boxarts']['_342x192']['jpg']['url'],
+                    'big': video['boxarts']['_1280x720']['jpg']['url']

                 },
                 },

-                'interesting_moment': videos[video_key]['interestingMoment']['_665x375']['jpg']['url'],
+                'interesting_moment': video['interestingMoment']['_665x375']['jpg']['url'],

             }
         }
 
             }
         }
 


@@ -1235,6 +1247,11 @@ class NetflixSession:
           },
         }
         """
           },
         }
         """

+        mpaa = ''
+        if episode.get('maturity', None) is not None:
+            if episode['maturity'].get('board', None) is not None and episode['maturity'].get('value', None) is not None:
+                mpaa = str(episode['maturity'].get('board', '').encode('utf-8')) + '-' + str(episode['maturity'].get('value', '').encode('utf-8'))
+

         return {
             episode['summary']['id']: {
                 'id': episode['summary']['id'],
         return {
             episode['summary']['id']: {
                 'id': episode['summary']['id'],


@@ -1245,7 +1262,7 @@ class NetflixSession:
                 'title': episode['info']['title'],
                 'year': episode['info']['releaseYear'],
                 'genres': self.parse_genres_for_video(video=episode, genres=genres),
                 'title': episode['info']['title'],
                 'year': episode['info']['releaseYear'],
                 'genres': self.parse_genres_for_video(video=episode, genres=genres),

-                'mpaa': str(episode['maturity']['rating']['board']) + ' ' + str(episode['maturity']['rating']['value']),
+                'mpaa': mpaa,

                 'maturity': episode['maturity'],
                 'playcount': (0, 1)[episode['watched']],
                 'rating': episode['userRating'].get('average', 0) if episode['userRating'].get('average', None) != None else episode['userRating'].get('predicted', 0),
                 'maturity': episode['maturity'],
                 'playcount': (0, 1)[episode['watched']],
                 'rating': episode['userRating'].get('average', 0) if episode['userRating'].get('average', None) != None else episode['userRating'].get('predicted', 0),


@@ -1270,8 +1287,9 @@ class NetflixSession:
         response = self._session_get(component='browse')
         return BeautifulSoup(response.text, 'html.parser')
 
         response = self._session_get(component='browse')
         return BeautifulSoup(response.text, 'html.parser')
 

-    def fetch_video_list_ids (self, list_from=0, list_to=50):
+    def fetch_video_list_ids_via_preflight (self, list_from=0, list_to=50):

         """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
         """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix

+           via the preflight (GET) request

 
         Parameters
         ----------
 
         Parameters
         ----------


@@ -1298,14 +1316,11 @@ class NetflixSession:
         response = self._session_get(component='video_list_ids', params=payload, type='api')
         return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
 
         response = self._session_get(component='video_list_ids', params=payload, type='api')
         return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
 

-    def fetch_search_results (self, search_str, list_from=0, list_to=10):
-        """Fetches the JSON which contains the results for the given search query
+    def fetch_video_list_ids (self, list_from=0, list_to=50):
+        """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix

 
         Parameters
         ----------
 
         Parameters
         ----------

-        search_str : :obj:`str`
-            String to query Netflix search for
-

         list_from : :obj:`int`
             Start entry for pagination
 
         list_from : :obj:`int`
             Start entry for pagination
 


@@ -1317,42 +1332,20 @@ class NetflixSession:
         :obj:`dict` of :obj:`dict` of :obj:`str`
             Raw Netflix API call response or api call error
         """
         :obj:`dict` of :obj:`dict` of :obj:`str`
             Raw Netflix API call response or api call error
         """

-        # properly encode the search string
-        encoded_search_string = quote(search_str)
-

         paths = [
         paths = [

-            ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, ['summary', 'title']],
-            ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
-            ['search', encoded_search_string, 'titles', ['id', 'length', 'name', 'trackIds', 'requestId']],
-            ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, ['summary', 'title']],
-            ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
-            ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', ['id', 'length', 'name', 'trackIds', 'requestId']]
+            ['lolomo', {'from': list_from, 'to': list_to}, ['displayName', 'context', 'id', 'index', 'length']]

         ]
         ]

-        response = self._path_request(paths=paths)
-        return self._process_response(response=response, component='Search results')

 
 

-    def get_lolomo_for_kids (self):
-        """Fetches the lolomo ID for Kids profiles
-
-        Returns
-        -------
-        :obj:`str`
-            Kids Lolomo ID
-        """
-        response = self._session_get(component='kids')
-        for cookie in response.cookies:
-            if cookie.name.find('lhpuuidh-browse-' + self.user_data['guid']) != -1 and cookie.name.rfind('-T') == -1:
-                start = unquote(cookie.value).rfind(':')
-                return unquote(cookie.value)[start+1:]
-        return None
+        response = self._path_request(paths=paths)
+        return self._process_response(response=response, component='Video list ids')

 
 

-    def fetch_lists_for_kids (self, lolomo, list_from=0, list_to=50):
-        """Fetches the JSON which contains the contents of a the video list for kids users
+    def fetch_search_results (self, search_str, list_from=0, list_to=10):
+        """Fetches the JSON which contains the results for the given search query

 
         Parameters
         ----------
 
         Parameters
         ----------

-        lolomo : :obj:`str`
-            Lolomo ID for the Kids profile
+        search_str : :obj:`str`
+            String to query Netflix search for

 
         list_from : :obj:`int`
             Start entry for pagination
 
         list_from : :obj:`int`
             Start entry for pagination


@@ -1365,15 +1358,21 @@ class NetflixSession:
         :obj:`dict` of :obj:`dict` of :obj:`str`
             Raw Netflix API call response or api call error
         """
         :obj:`dict` of :obj:`dict` of :obj:`str`
             Raw Netflix API call response or api call error
         """

+        # properly encode the search string
+        encoded_search_string = quote(search_str)
+

         paths = [
         paths = [

-            ['lists', lolomo, {'from': list_from, 'to': list_to}, ['displayName', 'context', 'genreId', 'id', 'index', 'length']]
+            ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, ['summary', 'title']],
+            ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
+            ['search', encoded_search_string, 'titles', ['id', 'length', 'name', 'trackIds', 'requestId']],
+            ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, ['summary', 'title']],
+            ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
+            ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', ['id', 'length', 'name', 'trackIds', 'requestId']]

         ]
         ]

-

         response = self._path_request(paths=paths)
         response = self._path_request(paths=paths)

-        res = self._process_response(response=response, component='Kids lists')
-        return self.parse_video_list_ids(response_data=res['value'])
+        return self._process_response(response=response, component='Search results')

 
 

-    def fetch_video_list (self, list_id, list_from=0, list_to=20):
+    def fetch_video_list (self, list_id, list_from=0, list_to=26):

         """Fetches the JSON which contains the contents of a given video list
 
         Parameters
         """Fetches the JSON which contains the contents of a given video list
 
         Parameters


@@ -1888,24 +1887,6 @@ class NetflixSession:
         """
         return urlsafe_b64encode(account['email'])
 
         """
         return urlsafe_b64encode(account['email'])
 

-    def _get_user_agent_for_current_platform (self):
-        """Determines the user agent string for the current platform (to retrieve a valid ESN)
-
-        Returns
-        -------
-        :obj:`str`
-            User Agent for platform
-        """
-        import platform
-        if platform == 'linux' or platform == 'linux2':
-            return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        elif platform == 'darwin':
-            return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        elif platform == 'win32':
-            return 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-        else:
-            return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
-

     def _session_post (self, component, type='document', data={}, headers={}, params={}):
         """Executes a get request using requests for the current session & measures the duration of that request
 
     def _session_post (self, component, type='document', data={}, headers={}, params={}):
         """Executes a get request using requests for the current session & measures the duration of that request
 


@@ -1960,10 +1941,6 @@ class NetflixSession:
         url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
         start = time()
         response = self.session.get(url=url, verify=self.verify_ssl, params=params)
         url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
         start = time()
         response = self.session.get(url=url, verify=self.verify_ssl, params=params)

-        for cookie in response.cookies:
-            if cookie.name.find('lhpuuidh-browse-' + self.user_data['guid']) != -1 and cookie.name.rfind('-T') == -1:
-                start = unquote(cookie.value).rfind(':')
-                return unquote(cookie.value)[start+1:]

         end = time()
         self.log(msg='[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')
         return response
         end = time()
         self.log(msg='[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')
         return response


@@ -2224,7 +2201,6 @@ class NetflixSession:
         important_fields = [
             'profileName',
             'isActive',
         important_fields = [
             'profileName',
             'isActive',

-            'isFirstUse',

             'isAccountOwner',
             'isKids'
         ]
             'isAccountOwner',
             'isKids'
         ]


@@ -2237,7 +2213,7 @@ class NetflixSession:
                         profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
                     avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
                     avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
                         profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
                     avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
                     avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']

-                    profile.update({'avatar': avatar})
+                    profile.update({'avatar': avatar, 'isFirstUse': False})

                     profiles.update({profile_id: profile})
             return profiles
 
                     profiles.update({profile_id: profile})
             return profiles
 


@@ -2312,11 +2288,31 @@ class NetflixSession:
             :obj:`str` of :obj:`str
             ESN, something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME
         """
             :obj:`str` of :obj:`str
             ESN, something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME
         """

-        esn = ''
+        # we generate an esn from device strings for android
+        import subprocess
+        try:
+            manufacturer = subprocess.check_output(["/system/bin/getprop", "ro.product.manufacturer"])
+            if manufacturer:
+                esn = 'NFANDROID1-PRV-'
+                input = subprocess.check_output(["/system/bin/getprop", "ro.nrdp.modelgroup"])
+                if not input:
+                    esn = esn + 'T-L3-'
+                else:
+                    esn = esn + input.strip(' \t\n\r') + '-'
+                esn = esn + '{:5}'.format(manufacturer.strip(' \t\n\r').upper())
+                input = subprocess.check_output(["/system/bin/getprop" ,"ro.product.model"])
+                esn = esn + input.strip(' \t\n\r').replace(' ', '=').upper()
+                self.log(msg='Android generated ESN:' + esn)
+                return esn
+        except OSError as e:
+            self.log(msg='Ignoring exception for non Android devices')
+

         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             return netflix_page_data.get('esn', '')
 
         # values are accessible via dict (sloppy parsing successfull)
         if type(netflix_page_data) == dict:
             return netflix_page_data.get('esn', '')
 

+        esn = ''
+

         # values are stored in lists (returned from JS parser)
         for item in netflix_page_data:
             if 'esnGeneratorModel' in dict(item).keys():
         # values are stored in lists (returned from JS parser)
         for item in netflix_page_data:
             if 'esnGeneratorModel' in dict(item).keys():