2 # -*- coding: utf-8 -*-
3 # Module: NetflixSession
4 # Created on: 13.01.2017
8 from requests import session, cookies
9 from urllib import quote, unquote
11 from base64 import urlsafe_b64encode
12 from bs4 import BeautifulSoup, SoupStrainer
13 from utils import noop, get_user_agent_for_current_platform
15 import cPickle as pickle
20 """Helps with login/session management of Netflix users & API data fetching"""
22 base_url = 'https://www.netflix.com'
23 """str: Secure Netflix url"""
27 'browse': '/profiles/manage',
28 'video_list_ids': '/preflight',
29 'shakti': '/pathEvaluator',
30 'profiles': '/profiles/manage',
31 'switch_profiles': '/profiles/switch',
32 'adult_pin': '/pin/service',
33 'metadata': '/metadata',
34 'set_video_rating': '/setVideoRating',
35 'update_my_list': '/playlistop',
38 """:obj:`dict` of :obj:`str` List of all static endpoints for HTML/JSON POST/GET requests"""
40 video_list_keys = ['user', 'genres', 'recommendations']
41 """:obj:`list` of :obj:`str` Divide the users video lists into 3 different categories (for easier digestion)"""
45 Dict of user profiles, user id is the key:
48 "profileName": "username",
49 "avatar": "http://..../avatar.png",
51 "isAccountOwner": False,
59 dict of user data (used for authentication):
63 "authURL": "145637....",
70 dict of api data (used to build up the api urls):
73 "API_BASE_URL": "/shakti",
74 "API_ROOT": "https://www.netflix.com/api",
75 "BUILD_IDENTIFIER": "113b89c9", "
76 ICHNAEA_ROOT": "/ichnaea"
81 """str: ESN - something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME"""
83 def __init__(self, cookie_path, data_path, verify_ssl=True, log_fn=noop):
84 """Stores the cookie path for later use & instanciates a requests
85 session with a proper user agent & stored cookies/data if available
89 cookie_path : :obj:`str`
92 data_path : :obj:`str`
93 User data cache location
98 self.cookie_path = cookie_path
99 self.data_path = data_path
100 self.verify_ssl = verify_ssl
103 # start session, fake chrome on the current platform (so that we get a proper widevine esn) & enable gzip
104 self.session = session()
105 self.session.headers.update({
106 'User-Agent': get_user_agent_for_current_platform(),
107 'Accept-Encoding': 'gzip'
110 def parse_login_form_fields (self, form_soup):
111 """Fetches all the inputfields from the login form, so that we
112 can build a request with all the fields needed besides the known email & password ones
116 form_soup : :obj:`BeautifulSoup`
117 Instance of an BeautifulSoup documet or node containing the login form
121 :obj:`dict` of :obj:`str`
122 Dictionary of all input fields with their name as the key & the default
123 value from the form field
125 login_input_fields = {}
126 login_inputs = form_soup.find_all('input')
127 # gather all form fields, set an empty string as the default value
128 for item in login_inputs:
129 keys = dict(item.attrs).keys()
130 if 'name' in keys and 'value' not in keys:
131 login_input_fields[item['name']] = ''
132 elif 'name' in keys and 'value' in keys:
133 login_input_fields[item['name']] = item['value']
134 return login_input_fields
136 def extract_inline_netflix_page_data (self, page_soup):
137 """Extracts all <script/> tags from the given document and parses the contents of each one of `em.
138 The contents of the parsable tags looks something like this:
139 <script>window.netflix = window.netflix || {} ; netflix.notification = {"constants":{"sessionLength":30,"ownerToken":"ZDD...};</script>
140 We use a JS parser to generate an AST of the code given & then parse that AST into a python dict.
141 This should be okay, as we´re only interested in a few static values & put the rest aside
145 page_soup : :obj:`BeautifulSoup`
146 Instance of an BeautifulSoup document or node containing the complete page contents
149 :obj:`list` of :obj:`dict`
150 List of all the serialized data pulled out of the pagws <script/> tags
152 scripts = page_soup.find_all('script', attrs={'src': None})
153 self.log(msg='Trying sloppy inline data parser')
154 inline_data = self._sloppy_parse_inline_data(scripts=scripts)
155 if self._verfify_auth_and_profiles_data(data=inline_data) != False:
156 self.log(msg='Sloppy inline data parsing successfull')
158 self.log(msg='Sloppy inline parser failed, trying JS parser')
159 return self._accurate_parse_inline_data(scripts=scripts)
161 def is_logged_in (self, account):
162 """Determines if a user is already logged in (with a valid cookie),
163 by fetching the index page with the current cookie & checking for the
164 `membership status` user data
168 account : :obj:`dict` of :obj:`str`
169 Dict containing an email, country & a password property
174 User is already logged in (e.g. Cookie is valid) or not
178 account_hash = self._generate_account_hash(account=account)
179 if self._load_cookies(filename=self.cookie_path + '_' + account_hash) == False:
181 if self._load_data(filename=self.data_path + '_' + account_hash) == False:
182 # load the profiles page (to verify the user)
183 response = self._session_get(component='profiles')
185 # parse out the needed inline information
186 only_script_tags = SoupStrainer('script')
187 page_soup = BeautifulSoup(response.text, 'html.parser', parse_only=only_script_tags)
188 page_data = self._parse_page_contents(page_soup=page_soup)
190 # check if the cookie is still valid
191 for item in page_data:
192 if 'profilesList' in dict(item).keys():
193 if item['profilesList']['summary']['length'] >= 1:
199 """Delete all cookies and session data
203 account : :obj:`dict` of :obj:`str`
204 Dict containing an email, country & a password property
207 self._delete_cookies(path=self.cookie_path)
208 self._delete_data(path=self.data_path)
210 def login (self, account):
211 """Try to log in a user with its credentials & stores the cookies if the action is successfull
213 Note: It fetches the HTML of the login page to extract the fields of the login form,
214 again, this is dirty, but as the fields & their values could change at any time, this
215 should be the most reliable way of retrieving the information
219 account : :obj:`dict` of :obj:`str`
220 Dict containing an email, country & a password property
225 User could be logged in or not
227 response = self._session_get(component='login')
228 if response.status_code != 200:
231 # collect all the login fields & their contents and add the user credentials
232 page_soup = BeautifulSoup(response.text, 'html.parser')
233 login_form = page_soup.find(attrs={'class' : 'ui-label-text'}).findPrevious('form')
234 login_payload = self.parse_login_form_fields(form_soup=login_form)
235 if 'email' in login_payload:
236 login_payload['email'] = account['email']
237 if 'emailOrPhoneNumber' in login_payload:
238 login_payload['emailOrPhoneNumber'] = account['email']
239 login_payload['password'] = account['password']
242 login_response = self._session_post(component='login', data=login_payload)
243 login_soup = BeautifulSoup(login_response.text, 'html.parser')
245 # we know that the login was successfull if we find an HTML element with the class of 'profile-name'
246 if login_soup.find(attrs={'class' : 'profile-name'}) or login_soup.find(attrs={'class' : 'profile-icon'}):
247 # parse the needed inline information & store cookies for later requests
248 self._parse_page_contents(page_soup=login_soup)
249 account_hash = self._generate_account_hash(account=account)
250 self._save_cookies(filename=self.cookie_path + '_' + account_hash)
251 self._save_data(filename=self.data_path + '_' + account_hash)
256 def switch_profile (self, profile_id, account):
257 """Switch the user profile based on a given profile id
259 Note: All available profiles & their ids can be found in the ´profiles´ property after a successfull login
263 profile_id : :obj:`str`
266 account : :obj:`dict` of :obj:`str`
267 Dict containing an email, country & a password property
272 User could be switched or not
275 'switchProfileGuid': profile_id,
277 'authURL': self.user_data['authURL']
280 response = self._session_get(component='switch_profiles', type='api', params=payload)
281 if response.status_code != 200:
284 account_hash = self._generate_account_hash(account=account)
285 self.user_data['guid'] = profile_id;
286 return self._save_data(filename=self.data_path + '_' + account_hash)
288 def send_adult_pin (self, pin):
289 """Send the adult pin to Netflix in case an adult rated video requests it
291 Note: Once entered, it should last for the complete session (Not so sure about this)
301 Pin was accepted or not
303 :obj:`dict` of :obj:`str`
308 'authURL': self.user_data['authURL']
310 response = self._session_get(component='adult_pin', params=payload)
311 pin_response = self._process_response(response=response, component=self._get_api_url_for(component='adult_pin'))
312 keys = pin_response.keys()
313 if 'success' in keys:
319 def add_to_list (self, video_id):
320 """Adds a video to "my list" on Netflix
324 video_id : :obj:`str`
325 ID of th show/video/movie to be added
330 Adding was successfull
332 return self._update_my_list(video_id=video_id, operation='add')
334 def remove_from_list (self, video_id):
335 """Removes a video from "my list" on Netflix
339 video_id : :obj:`str`
340 ID of th show/video/movie to be removed
345 Removing was successfull
347 return self._update_my_list(video_id=video_id, operation='remove')
349 def rate_video (self, video_id, rating):
350 """Rate a video on Netflix
354 video_id : :obj:`str`
355 ID of th show/video/movie to be rated
358 Rating, must be between 0 & 10
363 Rating successfull or not
366 # dirty rating validation
368 if rating > 10 or rating < 0:
371 # In opposition to Kodi, Netflix uses a rating from 0 to in 0.5 steps
376 'Content-Type': 'application/json',
377 'Accept': 'application/json, text/javascript, */*',
385 payload = json.dumps({
386 'authURL': self.user_data['authURL']
389 response = self._session_post(component='set_video_rating', type='api', params=params, headers=headers, data=payload)
390 return response.status_code == 200
392 def parse_video_list_ids (self, response_data):
393 """Parse the list of video ids e.g. rip out the parts we need
397 response_data : :obj:`dict` of :obj:`str`
398 Parsed response JSON from the ´fetch_video_list_ids´ call
402 :obj:`dict` of :obj:`dict`
403 Video list ids in the format:
407 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568367": {
408 "displayName": "US-Serien",
409 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568367",
414 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568368": {
419 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568364": {
420 "displayName": "Meine Liste",
421 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568364",
426 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568365": {
431 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382": {
432 "displayName": "Passend zu Family Guy",
433 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382",
438 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568397": {
444 # prepare the return dictionary
446 for key in self.video_list_keys:
447 video_list_ids[key] = {}
449 # check if the list items are hidden behind a `value` sub key
450 # this is the case when we fetch the lists via POST, not via a GET preflight request
451 if 'value' in response_data.keys():
452 response_data = response_data['value']
454 # subcatogorize the lists by their context
455 video_lists = response_data['lists']
456 for video_list_id in video_lists.keys():
457 video_list = video_lists[video_list_id]
458 if video_list.get('context', False) != False:
459 if video_list['context'] == 'genre':
460 video_list_ids['genres'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
461 elif video_list['context'] == 'similars' or video_list['context'] == 'becauseYouAdded':
462 video_list_ids['recommendations'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
464 video_list_ids['user'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
465 return video_list_ids
467 def parse_video_list_ids_entry (self, id, entry):
468 """Parse a video id entry e.g. rip out the parts we need
472 response_data : :obj:`dict` of :obj:`str`
473 Dictionary entry from the ´fetch_video_list_ids´ call
478 Unique id of the video list
480 entry : :obj:`dict` of :obj:`str`
481 Video list entry in the format:
483 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382": {
484 "displayName": "Passend zu Family Guy",
485 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382",
494 'index': entry['index'],
495 'name': entry['context'],
496 'displayName': entry['displayName'],
497 'size': entry['length']
501 def parse_search_results (self, response_data):
502 """Parse the list of search results, rip out the parts we need
503 and extend it with detailed show informations
507 response_data : :obj:`dict` of :obj:`str`
508 Parsed response JSON from the `fetch_search_results` call
512 :obj:`dict` of :obj:`dict` of :obj:`str`
513 Search results in the format:
517 "boxarts": "https://art-s.nflximg.net/0d7af/d5c72668c35d3da65ae031302bd4ae1bcc80d7af.jpg",
518 "detail_text": "Die legend\u00e4re und mit 13 Emmys nominierte Serie von Gene Roddenberry inspirierte eine ganze Generation.",
520 "season_id": "70109435",
521 "synopsis": "Unter Befehl von Captain Kirk begibt sich die Besatzung des Raumschiffs Enterprise in die Tiefen des Weltraums, wo sie fremde Galaxien und neue Zivilisationen erforscht.",
522 "title": "Star Trek",
531 raw_search_results = response_data['value']['videos']
532 for entry_id in raw_search_results:
533 if self._is_size_key(key=entry_id) == False:
534 # fetch information about each show & build up a proper search results dictionary
535 show = self.parse_show_list_entry(id=entry_id, entry=raw_search_results[entry_id])
536 show[entry_id].update(self.parse_show_information(id=entry_id, response_data=self.fetch_show_information(id=entry_id, type=show[entry_id]['type'])))
537 search_results.update(show)
538 return search_results
540 def parse_show_list_entry (self, id, entry):
541 """Parse a show entry e.g. rip out the parts we need
545 response_data : :obj:`dict` of :obj:`str`
546 Dictionary entry from the ´fetch_show_information´ call
549 Unique id of the video list
553 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
554 Show list entry in the format:
557 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382": {
558 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382",
559 "title": "Enterprise",
560 "boxarts": "https://art-s.nflximg.net/.../smth.jpg",
568 'title': entry['title'],
569 'boxarts': entry['boxarts']['_342x192']['jpg']['url'],
570 'type': entry['summary']['type']
574 def parse_video_list (self, response_data):
575 """Parse a list of videos
579 response_data : :obj:`dict` of :obj:`str`
580 Parsed response JSON from the `fetch_video_list` call
584 :obj:`dict` of :obj:`dict`
585 Video list in the format:
591 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
592 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
607 "episode_count": null,
613 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg",
614 "list_id": "9588df32-f957-40e4-9055-1f6f33b60103_46891306",
617 "description": "Nur f\u00fcr Erwachsene geeignet.",
623 "regular_synopsis": "Ein Spielzeughersteller erweckt aus Versehen die Seele der M\u00f6rderpuppe Chucky erneut zum Leben, die sich unmittelbar wieder ihren m\u00f6rderischen Aktivit\u00e4ten zuwendet.",
625 "seasons_count": null,
626 "seasons_label": null,
627 "synopsis": "Die allseits beliebte, von D\u00e4monen besessene M\u00f6rderpuppe ist wieder da und verbreitet erneut Horror und Schrecken.",
632 "title": "Chucky 2 \u2013 Die M\u00f6rderpuppe ist wieder da",
640 "big": "https://art-s.nflximg.net/7c10d/5dcc3fc8f08487e92507627068cfe26ef727c10d.jpg",
641 "small": "https://art-s.nflximg.net/5bc0e/f3be361b8c594929062f90a8d9c6eb57fb75bc0e.jpg"
658 "interesting_moment": "https://art-s.nflximg.net/0188e/19cd705a71ee08c8d2609ae01cd8a97a86c0188e.jpg",
659 "list_id": "9588df32-f957-40e4-9055-1f6f33b60103_46891306",
662 "description": "Geeignet ab 12 Jahren.",
668 "regular_synopsis": "Comedy-Serie \u00fcber die Erlebnisse eines Tatortreinigers, der seine schmutzige Arbeit erst beginnen kann, wenn die Polizei die Tatortanalyse abgeschlossen hat.",
671 "seasons_label": "5 Staffeln",
672 "synopsis": "In den meisten Krimiserien werden Mordf\u00e4lle auf faszinierende und spannende Weise gel\u00f6st. Diese Serie ist anders.",
676 "title": "Der Tatortreiniger",
684 raw_video_list = response_data['value']
685 netflix_list_id = self.parse_netflix_list_id(video_list=raw_video_list);
686 for video_id in raw_video_list['videos']:
687 if self._is_size_key(key=video_id) == False:
688 video_list.update(self.parse_video_list_entry(id=video_id, list_id=netflix_list_id, video=raw_video_list['videos'][video_id], persons=raw_video_list['person'], genres=raw_video_list['genres']))
691 def parse_video_list_entry (self, id, list_id, video, persons, genres):
692 """Parse a video list entry e.g. rip out the parts we need
697 Unique id of the video
700 Unique id of the containing list
702 video : :obj:`dict` of :obj:`str`
703 Video entry from the ´fetch_video_list´ call
705 persons : :obj:`dict` of :obj:`dict` of :obj:`str`
706 List of persons with reference ids
708 persons : :obj:`dict` of :obj:`dict` of :obj:`str`
709 List of genres with reference ids
713 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
714 Video list entry in the format:
720 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
721 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
736 "episode_count": null,
742 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg",
743 "list_id": "9588df32-f957-40e4-9055-1f6f33b60103_46891306",
746 "description": "Nur f\u00fcr Erwachsene geeignet.",
752 "regular_synopsis": "Ein Spielzeughersteller erweckt aus Versehen die Seele der M\u00f6rderpuppe Chucky erneut zum Leben, die sich unmittelbar wieder ihren m\u00f6rderischen Aktivit\u00e4ten zuwendet.",
754 "seasons_count": null,
755 "seasons_label": null,
756 "synopsis": "Die allseits beliebte, von D\u00e4monen besessene M\u00f6rderpuppe ist wieder da und verbreitet erneut Horror und Schrecken.",
761 "title": "Chucky 2 \u2013 Die M\u00f6rderpuppe ist wieder da",
768 season_info = self.parse_season_information_for_video(video=video)
773 'title': video['title'],
774 'synopsis': video['synopsis'],
775 'regular_synopsis': video['regularSynopsis'],
776 'type': video['summary']['type'],
777 'rating': video['userRating'].get('average', 0) if video['userRating'].get('average', None) != None else video['userRating'].get('predicted', 0),
778 'episode_count': season_info['episode_count'],
779 'seasons_label': season_info['seasons_label'],
780 'seasons_count': season_info['seasons_count'],
781 'in_my_list': video['queue']['inQueue'],
782 'year': video['releaseYear'],
783 'runtime': self.parse_runtime_for_video(video=video),
784 'watched': video['watched'],
785 'tags': self.parse_tags_for_video(video=video),
786 'genres': self.parse_genres_for_video(video=video, genres=genres),
787 'quality': self.parse_quality_for_video(video=video),
788 'cast': self.parse_cast_for_video(video=video, persons=persons),
789 'directors': self.parse_directors_for_video(video=video, persons=persons),
790 'creators': self.parse_creators_for_video(video=video, persons=persons),
792 'board': None if 'board' not in video['maturity']['rating'].keys() else video['maturity']['rating']['board'],
793 'value': None if 'value' not in video['maturity']['rating'].keys() else video['maturity']['rating']['value'],
794 'description': None if 'maturityDescription' not in video['maturity']['rating'].keys() else video['maturity']['rating']['maturityDescription'],
795 'level': None if 'maturityLevel' not in video['maturity']['rating'].keys() else video['maturity']['rating']['maturityLevel']
798 'small': video['boxarts']['_342x192']['jpg']['url'],
799 'big': video['boxarts']['_1280x720']['jpg']['url']
801 'interesting_moment': None if 'interestingMoment' not in video.keys() else video['interestingMoment']['_665x375']['jpg']['url'],
802 'artwork': video['artWorkByType']['BILLBOARD']['_1280x720']['jpg']['url'],
806 def parse_creators_for_video (self, video, persons):
807 """Matches ids with person names to generate a list of creators
811 video : :obj:`dict` of :obj:`str`
812 Dictionary entry for one video entry
814 persons : :obj:`dict` of :obj:`str`
815 Raw resposne of all persons delivered by the API call
819 :obj:`list` of :obj:`str`
823 for person_key in dict(persons).keys():
824 if self._is_size_key(key=person_key) == False and person_key != 'summary':
825 for creator_key in dict(video['creators']).keys():
826 if self._is_size_key(key=creator_key) == False and creator_key != 'summary':
827 if video['creators'][creator_key][1] == person_key:
828 creators.append(persons[person_key]['name'])
831 def parse_directors_for_video (self, video, persons):
832 """Matches ids with person names to generate a list of directors
836 video : :obj:`dict` of :obj:`str`
837 Dictionary entry for one video entry
839 persons : :obj:`dict` of :obj:`str`
840 Raw resposne of all persons delivered by the API call
844 :obj:`list` of :obj:`str`
848 for person_key in dict(persons).keys():
849 if self._is_size_key(key=person_key) == False and person_key != 'summary':
850 for director_key in dict(video['directors']).keys():
851 if self._is_size_key(key=director_key) == False and director_key != 'summary':
852 if video['directors'][director_key][1] == person_key:
853 directors.append(persons[person_key]['name'])
856 def parse_cast_for_video (self, video, persons):
857 """Matches ids with person names to generate a list of cast members
861 video : :obj:`dict` of :obj:`str`
862 Dictionary entry for one video entry
864 persons : :obj:`dict` of :obj:`str`
865 Raw resposne of all persons delivered by the API call
869 :obj:`list` of :obj:`str`
873 for person_key in dict(persons).keys():
874 if self._is_size_key(key=person_key) == False and person_key != 'summary':
875 for cast_key in dict(video['cast']).keys():
876 if self._is_size_key(key=cast_key) == False and cast_key != 'summary':
877 if video['cast'][cast_key][1] == person_key:
878 cast.append(persons[person_key]['name'])
881 def parse_genres_for_video (self, video, genres):
882 """Matches ids with genre names to generate a list of genres for a video
886 video : :obj:`dict` of :obj:`str`
887 Dictionary entry for one video entry
889 genres : :obj:`dict` of :obj:`str`
890 Raw resposne of all genres delivered by the API call
894 :obj:`list` of :obj:`str`
899 for video_genre_key, video_genre in video['genres'].iteritems():
900 if self._is_size_key(video_genre_key) == False and video_genre_key != 'summary':
901 name = genres.get(video_genre[1], {}).get('name')
904 video_genres.append(name)
908 def parse_tags_for_video (self, video):
909 """Parses a nested list of tags, removes the not needed meta information & returns a raw string list
913 video : :obj:`dict` of :obj:`str`
914 Dictionary entry for one video entry
918 :obj:`list` of :obj:`str`
922 for tag_key in dict(video['tags']).keys():
923 if self._is_size_key(key=tag_key) == False and tag_key != 'summary':
924 tags.append(video['tags'][tag_key]['name'])
927 def parse_season_information_for_video (self, video):
928 """Checks if the fiven video is a show (series) and returns season & episode information
932 video : :obj:`dict` of :obj:`str`
933 Dictionary entry for one video entry
937 :obj:`dict` of :obj:`str`
938 Episode count / Season Count & Season label if given
941 'episode_count': None,
942 'seasons_label': None,
943 'seasons_count': None
945 if video['summary']['type'] == 'show':
947 'episode_count': video['episodeCount'],
948 'seasons_label': video['numSeasonsLabel'],
949 'seasons_count': video['seasonCount']
953 def parse_quality_for_video (self, video):
954 """Transforms Netflix quality information in video resolution info
958 video : :obj:`dict` of :obj:`str`
959 Dictionary entry for one video entry
967 if video['videoQuality']['hasHD']:
969 if video['videoQuality']['hasUltraHD']:
973 def parse_runtime_for_video (self, video):
974 """Checks if the video is a movie & returns the runtime if given
978 video : :obj:`dict` of :obj:`str`
979 Dictionary entry for one video entry
984 Runtime of the video (in seconds)
987 if video['summary']['type'] != 'show':
988 runtime = video['runtime']
991 def parse_netflix_list_id (self, video_list):
992 """Parse a video list and extract the list id
996 video_list : :obj:`dict` of :obj:`str`
1001 entry : :obj:`str` or None
1004 netflix_list_id = None
1005 if 'lists' in video_list.keys():
1006 for video_id in video_list['lists']:
1007 if self._is_size_key(key=video_id) == False:
1008 netflix_list_id = video_id;
1009 return netflix_list_id
1011 def parse_show_information (self, id, response_data):
1012 """Parse extended show information (synopsis, seasons, etc.)
1019 response_data : :obj:`dict` of :obj:`str`
1020 Parsed response JSON from the `fetch_show_information` call
1024 entry : :obj:`dict` of :obj:`str`
1025 Show information in the format:
1027 "season_id": "80113084",
1028 "synopsis": "Aus verzweifelter Geldnot versucht sich der Familienvater und Drucker Jochen als Geldf\u00e4lscher und rutscht dabei immer mehr in die dunkle Welt des Verbrechens ab."
1029 "detail_text": "I´m optional"
1033 raw_show = response_data['value']['videos'][id]
1034 show.update({'synopsis': raw_show['regularSynopsis']})
1035 if 'evidence' in raw_show:
1036 show.update({'detail_text': raw_show['evidence']['value']['text']})
1037 if 'seasonList' in raw_show:
1038 show.update({'season_id': raw_show['seasonList']['current'][1]})
1041 def parse_seasons (self, id, response_data):
1042 """Parse a list of seasons for a given show
1049 response_data : :obj:`dict` of :obj:`str`
1050 Parsed response JSON from the `fetch_seasons_for_show` call
1054 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1055 Season information in the format:
1060 "shortName": "St. 1",
1062 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
1063 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
1065 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg"
1070 "shortName": "St. 2",
1072 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
1073 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
1075 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg"
1079 raw_seasons = response_data['value']
1080 videos = raw_seasons['videos']
1084 for key, video_candidate in videos.iteritems():
1085 if not self._is_size_key(key):
1086 video = video_candidate
1090 for idx, season_list_entry in video['seasonList'].iteritems():
1091 if self._is_size_key(key=idx) == False and idx != 'summary':
1092 sorting[int(season_list_entry[1])] = int(idx)
1096 for season in raw_seasons['seasons']:
1097 if self._is_size_key(key=season) == False:
1098 seasons.update(self._parse_season_entry(season=raw_seasons['seasons'][season], video=video, sorting=sorting))
1101 def _parse_season_entry (self, season, video, sorting):
1102 """Parse a season list entry e.g. rip out the parts we need
1106 season : :obj:`dict` of :obj:`str`
1107 Season entry from the `fetch_seasons_for_show` call
1111 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1112 Season list entry in the format:
1118 "shortName": "St. 1",
1120 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
1121 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
1123 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg"
1128 season['summary']['id']: {
1129 'idx': sorting[season['summary']['id']],
1130 'id': season['summary']['id'],
1131 'text': season['summary']['name'],
1132 'shortName': season['summary']['shortName'],
1134 'small': video['boxarts']['_342x192']['jpg']['url'],
1135 'big': video['boxarts']['_1280x720']['jpg']['url']
1137 'interesting_moment': video['interestingMoment']['_665x375']['jpg']['url'],
1141 def parse_episodes_by_season (self, response_data):
1142 """Parse episodes for a given season/episode list
1146 response_data : :obj:`dict` of :obj:`str`
1147 Parsed response JSON from the `fetch_seasons_for_show` call
1151 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1152 Season information in the format:
1156 "banner": "https://art-s.nflximg.net/63a36/c7fdfe6604ef2c22d085ac5dca5f69874e363a36.jpg",
1159 "fanart": "https://art-s.nflximg.net/74e02/e7edcc5cc7dcda1e94d505df2f0a2f0d22774e02.jpg",
1165 "mediatype": "episode",
1169 "plot": "Als die Griffins und andere Einwohner von Quahog in die Villa von James Woods eingeladen werden, muss pl\u00f6tzlich ein Mord aufgekl\u00e4rt werden.",
1170 "poster": "https://art-s.nflximg.net/72fd6/57088715e8d436fdb6986834ab39124b0a972fd6.jpg",
1171 "rating": 3.9111512,
1173 "thumb": "https://art-s.nflximg.net/be686/07680670a68da8749eba607efb1ae37f9e3be686.jpg",
1174 "title": "Und dann gab es weniger (Teil 1)",
1179 "banner": "https://art-s.nflximg.net/63a36/c7fdfe6604ef2c22d085ac5dca5f69874e363a36.jpg",
1182 "fanart": "https://art-s.nflximg.net/c472c/6c10f9578bf2c1d0a183c2ccb382931efcbc472c.jpg",
1188 "mediatype": "episode",
1192 "plot": "Wer ist der M\u00f6rder? Nach zahlreichen Morden wird immer wieder jemand anderes verd\u00e4chtigt.",
1193 "poster": "https://art-s.nflximg.net/72fd6/57088715e8d436fdb6986834ab39124b0a972fd6.jpg",
1194 "rating": 3.9111512,
1196 "thumb": "https://art-s.nflximg.net/15a08/857d59126641987bec302bb147a802a00d015a08.jpg",
1197 "title": "Und dann gab es weniger (Teil 2)",
1204 raw_episodes = response_data['value']['videos']
1205 for episode_id in raw_episodes:
1206 if self._is_size_key(key=episode_id) == False:
1207 if (raw_episodes[episode_id]['summary']['type'] == 'episode'):
1208 episodes.update(self.parse_episode(episode=raw_episodes[episode_id], genres=response_data['value']['genres']))
1211 def parse_episode (self, episode, genres=None):
1212 """Parse episode from an list of episodes by season
1216 episode : :obj:`dict` of :obj:`str`
1217 Episode entry from the `fetch_episodes_by_season` call
1221 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1222 Episode information in the format:
1226 "banner": "https://art-s.nflximg.net/63a36/c7fdfe6604ef2c22d085ac5dca5f69874e363a36.jpg",
1229 "fanart": "https://art-s.nflximg.net/74e02/e7edcc5cc7dcda1e94d505df2f0a2f0d22774e02.jpg",
1235 "mediatype": "episode",
1239 "plot": "Als die Griffins und andere Einwohner von Quahog in die Villa von James Woods eingeladen werden, muss pl\u00f6tzlich ein Mord aufgekl\u00e4rt werden.",
1240 "poster": "https://art-s.nflximg.net/72fd6/57088715e8d436fdb6986834ab39124b0a972fd6.jpg",
1241 "rating": 3.9111512,
1243 "thumb": "https://art-s.nflximg.net/be686/07680670a68da8749eba607efb1ae37f9e3be686.jpg",
1244 "title": "Und dann gab es weniger (Teil 1)",
1251 if episode.get('maturity', None) is not None:
1252 if episode['maturity'].get('board', None) is not None and episode['maturity'].get('value', None) is not None:
1253 mpaa = str(episode['maturity'].get('board', '').encode('utf-8')) + '-' + str(episode['maturity'].get('value', '').encode('utf-8'))
1256 episode['summary']['id']: {
1257 'id': episode['summary']['id'],
1258 'episode': episode['summary']['episode'],
1259 'season': episode['summary']['season'],
1260 'plot': episode['info']['synopsis'],
1261 'duration': episode['info']['runtime'],
1262 'title': episode['info']['title'],
1263 'year': episode['info']['releaseYear'],
1264 'genres': self.parse_genres_for_video(video=episode, genres=genres),
1266 'maturity': episode['maturity'],
1267 'playcount': (0, 1)[episode['watched']],
1268 'rating': episode['userRating'].get('average', 0) if episode['userRating'].get('average', None) != None else episode['userRating'].get('predicted', 0),
1269 'thumb': episode['info']['interestingMoments']['url'],
1270 'fanart': episode['interestingMoment']['_1280x720']['jpg']['url'],
1271 'poster': episode['boxarts']['_1280x720']['jpg']['url'],
1272 'banner': episode['boxarts']['_342x192']['jpg']['url'],
1273 'mediatype': {'episode': 'episode', 'movie': 'movie'}[episode['summary']['type']],
1274 'my_list': episode['queue']['inQueue'],
1275 'bookmark': episode['bookmarkPosition']
1279 def fetch_browse_list_contents (self):
1280 """Fetches the HTML data for the lists on the landing page (browse page) of Netflix
1284 :obj:`BeautifulSoup`
1285 Instance of an BeautifulSoup document containing the complete page contents
1287 response = self._session_get(component='browse')
1288 return BeautifulSoup(response.text, 'html.parser')
1290 def fetch_video_list_ids_via_preflight (self, list_from=0, list_to=50):
1291 """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
1292 via the preflight (GET) request
1296 list_from : :obj:`int`
1297 Start entry for pagination
1299 list_to : :obj:`int`
1300 Last entry for pagination
1304 :obj:`dict` of :obj:`dict` of :obj:`str`
1305 Raw Netflix API call response or api call error
1308 'fromRow': list_from,
1310 'opaqueImageExtension': 'jpg',
1311 'transparentImageExtension': 'png',
1313 'authURL': self.user_data['authURL']
1316 response = self._session_get(component='video_list_ids', params=payload, type='api')
1317 return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
1319 def fetch_video_list_ids (self, list_from=0, list_to=50):
1320 """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
1324 list_from : :obj:`int`
1325 Start entry for pagination
1327 list_to : :obj:`int`
1328 Last entry for pagination
1332 :obj:`dict` of :obj:`dict` of :obj:`str`
1333 Raw Netflix API call response or api call error
1336 ['lolomo', {'from': list_from, 'to': list_to}, ['displayName', 'context', 'id', 'index', 'length']]
1339 response = self._path_request(paths=paths)
1340 return self._process_response(response=response, component='Video list ids')
1342 def fetch_search_results (self, search_str, list_from=0, list_to=10):
1343 """Fetches the JSON which contains the results for the given search query
1347 search_str : :obj:`str`
1348 String to query Netflix search for
1350 list_from : :obj:`int`
1351 Start entry for pagination
1353 list_to : :obj:`int`
1354 Last entry for pagination
1358 :obj:`dict` of :obj:`dict` of :obj:`str`
1359 Raw Netflix API call response or api call error
1361 # properly encode the search string
1362 encoded_search_string = quote(search_str)
1365 ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, ['summary', 'title']],
1366 ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1367 ['search', encoded_search_string, 'titles', ['id', 'length', 'name', 'trackIds', 'requestId']],
1368 ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, ['summary', 'title']],
1369 ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1370 ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', ['id', 'length', 'name', 'trackIds', 'requestId']]
1372 response = self._path_request(paths=paths)
1373 return self._process_response(response=response, component='Search results')
1375 def fetch_video_list (self, list_id, list_from=0, list_to=26):
1376 """Fetches the JSON which contains the contents of a given video list
1380 list_id : :obj:`str`
1381 Unique list id to query Netflix for
1383 list_from : :obj:`int`
1384 Start entry for pagination
1386 list_to : :obj:`int`
1387 Last entry for pagination
1391 :obj:`dict` of :obj:`dict` of :obj:`str`
1392 Raw Netflix API call response or api call error
1395 ['lists', list_id, {'from': list_from, 'to': list_to}, ['summary', 'title', 'synopsis', 'regularSynopsis', 'evidence', 'queue', 'episodeCount', 'info', 'maturity', 'runtime', 'seasonCount', 'releaseYear', 'userRating', 'numSeasonsLabel', 'bookmarkPosition', 'watched', 'videoQuality']],
1396 ['lists', list_id, {'from': list_from, 'to': list_to}, 'cast', {'from': 0, 'to': 15}, ['id', 'name']],
1397 ['lists', list_id, {'from': list_from, 'to': list_to}, 'cast', 'summary'],
1398 ['lists', list_id, {'from': list_from, 'to': list_to}, 'genres', {'from': 0, 'to': 5}, ['id', 'name']],
1399 ['lists', list_id, {'from': list_from, 'to': list_to}, 'genres', 'summary'],
1400 ['lists', list_id, {'from': list_from, 'to': list_to}, 'tags', {'from': 0, 'to': 9}, ['id', 'name']],
1401 ['lists', list_id, {'from': list_from, 'to': list_to}, 'tags', 'summary'],
1402 ['lists', list_id, {'from': list_from, 'to': list_to}, ['creators', 'directors'], {'from': 0, 'to': 49}, ['id', 'name']],
1403 ['lists', list_id, {'from': list_from, 'to': list_to}, ['creators', 'directors'], 'summary'],
1404 ['lists', list_id, {'from': list_from, 'to': list_to}, 'bb2OGLogo', '_400x90', 'png'],
1405 ['lists', list_id, {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1406 ['lists', list_id, {'from': list_from, 'to': list_to}, 'boxarts', '_1280x720', 'jpg'],
1407 ['lists', list_id, {'from': list_from, 'to': list_to}, 'storyarts', '_1632x873', 'jpg'],
1408 ['lists', list_id, {'from': list_from, 'to': list_to}, 'interestingMoment', '_665x375', 'jpg'],
1409 ['lists', list_id, {'from': list_from, 'to': list_to}, 'artWorkByType', 'BILLBOARD', '_1280x720', 'jpg']
1412 response = self._path_request(paths=paths)
1413 return self._process_response(response=response, component='Video list')
1415 def fetch_video_list_information (self, video_ids):
1416 """Fetches the JSON which contains the detail information of a list of given video ids
1420 video_ids : :obj:`list` of :obj:`str`
1421 List of video ids to fetch detail data for
1425 :obj:`dict` of :obj:`dict` of :obj:`str`
1426 Raw Netflix API call response or api call error
1429 for video_id in video_ids:
1430 paths.append(['videos', video_id, ['summary', 'title', 'synopsis', 'regularSynopsis', 'evidence', 'queue', 'episodeCount', 'info', 'maturity', 'runtime', 'seasonCount', 'releaseYear', 'userRating', 'numSeasonsLabel', 'bookmarkPosition', 'watched', 'videoQuality']])
1431 paths.append(['videos', video_id, 'cast', {'from': 0, 'to': 15}, ['id', 'name']])
1432 paths.append(['videos', video_id, 'cast', 'summary'])
1433 paths.append(['videos', video_id, 'genres', {'from': 0, 'to': 5}, ['id', 'name']])
1434 paths.append(['videos', video_id, 'genres', 'summary'])
1435 paths.append(['videos', video_id, 'tags', {'from': 0, 'to': 9}, ['id', 'name']])
1436 paths.append(['videos', video_id, 'tags', 'summary'])
1437 paths.append(['videos', video_id, ['creators', 'directors'], {'from': 0, 'to': 49}, ['id', 'name']])
1438 paths.append(['videos', video_id, ['creators', 'directors'], 'summary'])
1439 paths.append(['videos', video_id, 'bb2OGLogo', '_400x90', 'png'])
1440 paths.append(['videos', video_id, 'boxarts', '_342x192', 'jpg'])
1441 paths.append(['videos', video_id, 'boxarts', '_1280x720', 'jpg'])
1442 paths.append(['videos', video_id, 'storyarts', '_1632x873', 'jpg'])
1443 paths.append(['videos', video_id, 'interestingMoment', '_665x375', 'jpg'])
1444 paths.append(['videos', video_id, 'artWorkByType', 'BILLBOARD', '_1280x720', 'jpg'])
1446 response = self._path_request(paths=paths)
1447 return self._process_response(response=response, component='fetch_video_list_information')
1449 def fetch_metadata (self, id):
1450 """Fetches the JSON which contains the metadata for a given show/movie or season id
1455 Show id, movie id or season id
1459 :obj:`dict` of :obj:`dict` of :obj:`str`
1460 Raw Netflix API call response or api call error
1464 'imageformat': 'jpg',
1467 response = self._session_get(component='metadata', params=payload, type='api')
1468 return self._process_response(response=response, component=self._get_api_url_for(component='metadata'))
1470 def fetch_show_information (self, id, type):
1471 """Fetches the JSON which contains the detailed contents of a show
1476 Unique show id to query Netflix for
1479 Can be 'movie' or 'show'
1483 :obj:`dict` of :obj:`dict` of :obj:`str`
1484 Raw Netflix API call response or api call error
1486 # check if we have a show or a movie, the request made depends on this
1489 ['videos', id, ['requestId', 'regularSynopsis', 'evidence']],
1490 ['videos', id, 'seasonList', 'current', 'summary']
1493 paths = [['videos', id, ['requestId', 'regularSynopsis', 'evidence']]]
1494 response = self._path_request(paths=paths)
1495 return self._process_response(response=response, component='Show information')
1497 def fetch_seasons_for_show (self, id, list_from=0, list_to=30):
1498 """Fetches the JSON which contains the seasons of a given show
1503 Unique show id to query Netflix for
1505 list_from : :obj:`int`
1506 Start entry for pagination
1508 list_to : :obj:`int`
1509 Last entry for pagination
1513 :obj:`dict` of :obj:`dict` of :obj:`str`
1514 Raw Netflix API call response or api call error
1517 ['videos', id, 'seasonList', {'from': list_from, 'to': list_to}, 'summary'],
1518 ['videos', id, 'seasonList', 'summary'],
1519 ['videos', id, 'boxarts', '_342x192', 'jpg'],
1520 ['videos', id, 'boxarts', '_1280x720', 'jpg'],
1521 ['videos', id, 'storyarts', '_1632x873', 'jpg'],
1522 ['videos', id, 'interestingMoment', '_665x375', 'jpg']
1524 response = self._path_request(paths=paths)
1525 return self._process_response(response=response, component='Seasons')
1527 def fetch_episodes_by_season (self, season_id, list_from=-1, list_to=40):
1528 """Fetches the JSON which contains the episodes of a given season
1530 TODO: Add more metadata
1534 season_id : :obj:`str`
1535 Unique season_id id to query Netflix for
1537 list_from : :obj:`int`
1538 Start entry for pagination
1540 list_to : :obj:`int`
1541 Last entry for pagination
1545 :obj:`dict` of :obj:`dict` of :obj:`str`
1546 Raw Netflix API call response or api call error
1549 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, ['summary', 'queue', 'info', 'maturity', 'userRating', 'bookmarkPosition', 'creditOffset', 'watched', 'videoQuality']],
1550 #['videos', season_id, 'cast', {'from': 0, 'to': 15}, ['id', 'name']],
1551 #['videos', season_id, 'cast', 'summary'],
1552 #['videos', season_id, 'genres', {'from': 0, 'to': 5}, ['id', 'name']],
1553 #['videos', season_id, 'genres', 'summary'],
1554 #['videos', season_id, 'tags', {'from': 0, 'to': 9}, ['id', 'name']],
1555 #['videos', season_id, 'tags', 'summary'],
1556 #['videos', season_id, ['creators', 'directors'], {'from': 0, 'to': 49}, ['id', 'name']],
1557 #['videos', season_id, ['creators', 'directors'], 'summary'],
1558 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'genres', {'from': 0, 'to': 1}, ['id', 'name']],
1559 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'genres', 'summary'],
1560 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'interestingMoment', '_1280x720', 'jpg'],
1561 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'interestingMoment', '_665x375', 'jpg'],
1562 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1563 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'boxarts', '_1280x720', 'jpg']
1565 response = self._path_request(paths=paths)
1566 return self._process_response(response=response, component='fetch_episodes_by_season')
1568 def refresh_session_data (self, account):
1569 """Reload the session data (profiles, user_data, api_data)
1573 account : :obj:`dict` of :obj:`str`
1574 Dict containing an email, country & a password property
1576 # load the profiles page (to verify the user)
1577 response = self._session_get(component='profiles')
1578 # parse out the needed inline information
1579 only_script_tags = SoupStrainer('script')
1580 page_soup = BeautifulSoup(response.text, 'html.parser', parse_only=only_script_tags)
1581 page_data = self._parse_page_contents(page_soup=page_soup)
1582 account_hash = self._generate_account_hash(account=account)
1583 self._save_data(filename=self.data_path + '_' + account_hash)
1585 def _path_request (self, paths):
1586 """Executes a post request against the shakti endpoint with Falcor style payload
1590 paths : :obj:`list` of :obj:`list`
1591 Payload with path querys for the Netflix Shakti API in Falcor style
1595 :obj:`requests.response`
1596 Response from a POST call made with Requests
1599 'Content-Type': 'application/json',
1600 'Accept': 'application/json, text/javascript, */*',
1605 'authURL': self.user_data['authURL']
1609 'model': self.user_data['gpsModel']
1612 return self._session_post(component='shakti', type='api', params=params, headers=headers, data=data)
1614 def _is_size_key (self, key):
1615 """Tiny helper that checks if a given key is called $size or size, as we need to check this often
1620 Key to check the value for
1625 Key has a size value or not
1627 return key == '$size' or key == 'size'
1629 def _get_api_url_for (self, component):
1630 """Tiny helper that builds the url for a requested API endpoint component
1634 component : :obj:`str`
1635 Component endpoint to build the URL for
1642 if self.api_data['API_ROOT'].find(self.api_data['API_BASE_URL']) > -1:
1643 return self.api_data['API_ROOT'] + '/' + self.api_data['BUILD_IDENTIFIER'] + self.urls[component]
1645 return self.api_data['API_ROOT'] + self.api_data['API_BASE_URL'] + '/' + self.api_data['BUILD_IDENTIFIER'] + self.urls[component]
1647 def _get_document_url_for (self, component):
1648 """Tiny helper that builds the url for a requested document endpoint component
1652 component : :obj:`str`
1653 Component endpoint to build the URL for
1660 return self.base_url + self.urls[component]
1662 def _process_response (self, response, component):
1663 """Tiny helper to check responses for API requests
1667 response : :obj:`requests.response`
1668 Response from a requests instance
1670 component : :obj:`str`
1675 :obj:`dict` of :obj:`dict` of :obj:`str` or :obj:`dict` of :obj:`str`
1676 Raw Netflix API call response or api call error
1678 # check if we´re not authorized to make thios call
1679 if response.status_code == 401:
1682 'message': 'Session invalid',
1685 # check if somethign else failed
1686 if response.status_code != 200:
1689 'message': 'API call for "' + component + '" failed',
1690 'code': response.status_code
1692 # return the parsed response & everything´s fine
1693 return response.json()
1695 def _to_unicode(self, str):
1696 '''Attempt to fix non uft-8 string into utf-8, using a limited set of encodings
1708 # fuller list of encodings at http://docs.python.org/library/codecs.html#standard-encodings
1709 if not str: return u''
1711 # we could add more encodings here, as warranted.
1712 encodings = ('ascii', 'utf8', 'latin1')
1713 for enc in encodings:
1716 u = unicode(str,enc)
1717 except UnicodeDecodeError:
1720 u = unicode(str, errors='replace')
1723 def _update_my_list (self, video_id, operation):
1724 """Tiny helper to add & remove items from "my list"
1728 video_id : :obj:`str`
1729 ID of the show/movie to be added
1731 operation : :obj:`str`
1732 Either "add" or "remove"
1737 Operation successfull
1740 'Content-Type': 'application/json',
1741 'Accept': 'application/json, text/javascript, */*',
1744 payload = json.dumps({
1745 'operation': operation,
1746 'videoId': int(video_id),
1747 'authURL': self.user_data['authURL']
1750 response = self._session_post(component='update_my_list', type='api', headers=headers, data=payload)
1751 return response.status_code == 200
1753 def _save_data(self, filename):
1754 """Tiny helper that stores session data from the session in a given file
1758 filename : :obj:`str`
1759 Complete path incl. filename that determines where to store the cookie
1764 Storage procedure was successfull
1766 if not os.path.isdir(os.path.dirname(filename)):
1768 with open(filename, 'w') as f:
1771 'user_data': self.user_data,
1772 'api_data': self.api_data,
1773 'profiles': self.profiles
1776 def _load_data(self, filename):
1777 """Tiny helper that loads session data into the active session from a given file
1781 filename : :obj:`str`
1782 Complete path incl. filename that determines where to load the data from
1787 Load procedure was successfull
1789 if not os.path.isfile(filename):
1792 with open(filename) as f:
1793 data = pickle.load(f)
1795 self.profiles = data['profiles']
1796 self.user_data = data['user_data']
1797 self.api_data = data['api_data']
1801 def _delete_data (self, path):
1802 """Tiny helper that deletes session data
1806 filename : :obj:`str`
1807 Complete path incl. filename that determines where to delete the files
1810 head, tail = os.path.split(path)
1811 for subdir, dirs, files in os.walk(head):
1814 os.remove(os.path.join(subdir, file))
1816 def _save_cookies(self, filename):
1817 """Tiny helper that stores cookies from the session in a given file
1821 filename : :obj:`str`
1822 Complete path incl. filename that determines where to store the cookie
1827 Storage procedure was successfull
1829 if not os.path.isdir(os.path.dirname(filename)):
1831 with open(filename, 'w') as f:
1833 pickle.dump(self.session.cookies._cookies, f)
1835 def _load_cookies(self, filename):
1836 """Tiny helper that loads cookies into the active session from a given file
1840 filename : :obj:`str`
1841 Complete path incl. filename that determines where to load the cookie from
1846 Load procedure was successfull
1848 if not os.path.isfile(filename):
1851 with open(filename) as f:
1852 _cookies = pickle.load(f)
1854 jar = cookies.RequestsCookieJar()
1855 jar._cookies = _cookies
1856 self.session.cookies = jar
1860 def _delete_cookies (self, path):
1861 """Tiny helper that deletes cookie data
1865 filename : :obj:`str`
1866 Complete path incl. filename that determines where to delete the files
1869 head, tail = os.path.split(path)
1870 for subdir, dirs, files in os.walk(head):
1873 os.remove(os.path.join(subdir, file))
1875 def _generate_account_hash (self, account):
1876 """Generates a has for the given account (used for cookie verification)
1880 account : :obj:`dict` of :obj:`str`
1881 Dict containing an email, country & a password property
1888 return urlsafe_b64encode(account['email'])
1890 def _session_post (self, component, type='document', data={}, headers={}, params={}):
1891 """Executes a get request using requests for the current session & measures the duration of that request
1895 component : :obj:`str`
1899 Is it a document or API request ('document' is default)
1901 data : :obj:`dict` of :obj:`str`
1902 Payload body as dict
1904 header : :obj:`dict` of :obj:`str`
1905 Additional headers as dict
1907 params : :obj:`dict` of :obj:`str`
1913 Contents of the field to match
1915 url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
1917 response = self.session.post(url=url, data=data, params=params, headers=headers, verify=self.verify_ssl)
1919 self.log(msg='[POST] Request for "' + url + '" took ' + str(end - start) + ' seconds')
1922 def _session_get (self, component, type='document', params={}):
1923 """Executes a get request using requests for the current session & measures the duration of that request
1927 component : :obj:`str`
1931 Is it a document or API request ('document' is default)
1933 params : :obj:`dict` of :obj:`str`
1939 Contents of the field to match
1941 url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
1943 response = self.session.get(url=url, verify=self.verify_ssl, params=params)
1945 self.log(msg='[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')
1948 def _sloppy_parse_user_and_api_data (self, key, contents):
1949 """Try to find the user & API data from the inline js by using a string parser
1954 Key to match in the inline js
1956 contents : :obj:`str`
1962 Contents of the field to match
1964 key_start = contents.find(key + '"')
1965 if int(key_start) == -1:
1967 sub_contents = contents[int(key_start):]
1968 l = sub_contents.find('",')
1969 return contents[(int(key_start)+len(key)+3):int(key_start)+l].decode('string_escape')
1971 def _sloppy_parse_profiles (self, contents):
1972 """Try to find the profile data from the inline js by using a string parser & parse/convert the result to JSON
1976 contents : :obj:`str`
1981 :obj:`dict` of :obj:`str` or None
1984 profile_start = contents.find('profiles":')
1985 profile_list_start = contents.find('profilesList')
1986 if int(profile_start) > -1 and int(profile_list_start) > -1:
1989 return json.loads('{"a":{"' + contents[profile_start:profile_list_start-2].decode('string_escape') + '}}').get('a').get('profiles')
1990 except ValueError, e:
1992 except TypeError, e:
1996 def _sloppy_parse_avatars (self, contents):
1997 """Try to find the avatar data from the inline js by using a string parser & parse/convert the result to JSON
2001 contents : :obj:`str`
2006 :obj:`dict` of :obj:`str` or None
2009 avatars_start = contents.find('"nf":')
2010 avatars_list_start = contents.find('"profiles"')
2011 if int(avatars_start) > -1 and int(avatars_list_start) > -1:
2014 return json.loads('{' + contents[avatars_start:avatars_list_start-2].decode('string_escape') + '}')
2015 except ValueError, e:
2017 except TypeError, e:
2021 def _verfify_auth_and_profiles_data (self, data):
2022 """Checks if the authURL has at least a certain length & doesn't overrule a certain length & if the profiles dict exists
2023 Simple validity check for the sloppy data parser
2027 data : :obj:`dict` of :obj:`str`
2035 if type(data.get('profiles')) == dict:
2036 if len(str(data.get('authURL', ''))) > 10 and len(str(data.get('authURL', ''))) < 50:
2040 def _sloppy_parse_inline_data (self, scripts):
2041 """Strips out all the needed user, api & profile data from the inline JS by string parsing
2042 Might fail, so if this doesn't succeed, a proper JS parser will chime in
2044 Note: This has been added for performance reasons only
2048 scripts : :obj:`list` of :obj:`BeautifoulSoup`
2049 Script tags & contents from the Netflix browse page
2053 :obj:`dict` of :obj:`str`
2054 Dict containijg user, api & profile data
2057 for script in scripts:
2058 contents = str(script.contents[0])
2059 important_data = ['authURL', 'API_BASE_URL', 'API_ROOT', 'BUILD_IDENTIFIER', 'ICHNAEA_ROOT', 'gpsModel', 'guid', 'esn']
2061 for key in important_data:
2062 _res = self._sloppy_parse_user_and_api_data(key, contents)
2064 res.update({key: _res})
2066 inline_data.update(res)
2069 profiles = self._sloppy_parse_profiles(contents)
2070 avatars = self._sloppy_parse_avatars(contents)
2071 if profiles != None:
2072 inline_data.update({'profiles': profiles})
2074 inline_data.update(avatars)
2077 def _accurate_parse_inline_data (self, scripts):
2078 """Uses a proper JS parser to fetch all the api, iser & profile data from within the inline JS
2080 Note: This is slow but accurate
2084 scripts : :obj:`list` of :obj:`BeautifoulSoup`
2085 Script tags & contents from the Netflix browse page
2089 :obj:`dict` of :obj:`str`
2090 Dict containing user, api & profile data
2093 from pyjsparser import PyJsParser
2094 parser = PyJsParser()
2095 for script in scripts:
2097 # unicode escape that incoming script stuff
2098 contents = self._to_unicode(str(script.contents[0]))
2099 # parse the JS & load the declarations we´re interested in
2100 parsed = parser.parse(contents)
2101 if len(parsed['body']) > 1 and parsed['body'][1]['expression']['right'].get('properties', None) != None:
2102 declarations = parsed['body'][1]['expression']['right']['properties']
2103 for declaration in declarations:
2104 for key in declaration:
2105 # we found the correct path if the declaration is a dict & of type 'ObjectExpression'
2106 if type(declaration[key]) is dict:
2107 if declaration[key]['type'] == 'ObjectExpression':
2108 # add all static data recursivly
2109 for expression in declaration[key]['properties']:
2110 data[expression['key']['value']] = self._parse_rec(expression['value'])
2111 inline_data.append(data)
2114 def _parse_rec (self, node):
2115 """Iterates over a JavaScript AST and return values found
2123 :obj:`dict` of :obj:`dict` or :obj:`str`
2124 Parsed contents of the node
2126 if node['type'] == 'ObjectExpression':
2128 for prop in node['properties']:
2129 _ret.update({prop['key']['value']: self._parse_rec(prop['value'])})
2131 if node['type'] == 'Literal':
2132 return node['value']
2134 def _parse_user_data (self, netflix_page_data):
2135 """Parse out the user data from the big chunk of dicts we got from
2136 parsing the JSON-ish data from the netflix homepage
2140 netflix_page_data : :obj:`list`
2141 List of all the JSON-ish data that has been extracted from the Netflix homepage
2142 see: extract_inline_netflix_page_data
2146 :obj:`dict` of :obj:`str`
2149 "guid": "72ERT45...",
2150 "authURL": "145637....",
2151 "gpsModel": "harris"
2155 important_fields = [
2161 # values are accessible via dict (sloppy parsing successfull)
2162 if type(netflix_page_data) == dict:
2163 for important_field in important_fields:
2164 user_data.update({important_field: netflix_page_data.get(important_field, '')})
2167 # values are stored in lists (returned from JS parser)
2168 for item in netflix_page_data:
2169 if 'memberContext' in dict(item).keys():
2170 for important_field in important_fields:
2171 user_data.update({important_field: item['memberContext']['data']['userInfo'][important_field]})
2175 def _parse_profile_data (self, netflix_page_data):
2176 """Parse out the profile data from the big chunk of dicts we got from
2177 parsing the JSON-ish data from the netflix homepage
2181 netflix_page_data : :obj:`list`
2182 List of all the JSON-ish data that has been extracted from the Netflix homepage
2183 see: extract_inline_netflix_page_data
2187 :obj:`dict` of :obj:`dict
2191 "profileName": "username",
2192 "avatar": "http://..../avatar.png",
2194 "isAccountOwner": False,
2201 important_fields = [
2207 # values are accessible via dict (sloppy parsing successfull)
2208 if type(netflix_page_data) == dict:
2209 for profile_id in netflix_page_data.get('profiles'):
2210 if self._is_size_key(key=profile_id) == False and type(netflix_page_data['profiles'][profile_id]) == dict and netflix_page_data['profiles'][profile_id].get('avatar', False) != False:
2211 profile = {'id': profile_id}
2212 for important_field in important_fields:
2213 profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
2214 avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
2215 avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
2216 profile.update({'avatar': avatar, 'isFirstUse': False})
2217 profiles.update({profile_id: profile})
2220 # values are stored in lists (returned from JS parser)
2221 # TODO: get rid of this christmas tree of doom
2222 for item in netflix_page_data:
2223 if 'hasViewedRatingWelcomeModal' in dict(item).keys():
2224 for profile_id in item:
2225 if self._is_size_key(key=profile_id) == False and type(item[profile_id]) == dict and item[profile_id].get('avatar', False) != False:
2226 profile = {'id': profile_id}
2227 for important_field in important_fields:
2228 profile.update({important_field: item[profile_id]['summary'][important_field]})
2229 avatar_base = item['nf'].get(item[profile_id]['summary']['avatarName'], False);
2230 avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
2231 profile.update({'avatar': avatar})
2232 profiles.update({profile_id: profile})
2235 def _parse_api_base_data (self, netflix_page_data):
2236 """Parse out the api url data from the big chunk of dicts we got from
2237 parsing the JSOn-ish data from the netflix homepage
2241 netflix_page_data : :obj:`list`
2242 List of all the JSON-ish data that has been extracted from the Netflix homepage
2243 see: extract_inline_netflix_page_data
2247 :obj:`dict` of :obj:`str
2250 "API_BASE_URL": "/shakti",
2251 "API_ROOT": "https://www.netflix.com/api",
2252 "BUILD_IDENTIFIER": "113b89c9",
2253 "ICHNAEA_ROOT": "/ichnaea"
2257 important_fields = [
2264 # values are accessible via dict (sloppy parsing successfull)
2265 if type(netflix_page_data) == dict:
2266 for important_field in important_fields:
2267 api_data.update({important_field: netflix_page_data.get(important_field, '')})
2270 for item in netflix_page_data:
2271 if 'serverDefs' in dict(item).keys():
2272 for important_field in important_fields:
2273 api_data.update({important_field: item['serverDefs']['data'][important_field]})
2276 def _parse_esn_data (self, netflix_page_data):
2277 """Parse out the esn id data from the big chunk of dicts we got from
2278 parsing the JSOn-ish data from the netflix homepage
2282 netflix_page_data : :obj:`list`
2283 List of all the JSON-ish data that has been extracted from the Netflix homepage
2284 see: extract_inline_netflix_page_data
2288 :obj:`str` of :obj:`str
2289 ESN, something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME
2291 # we generate an esn from device strings for android
2294 manufacturer = subprocess.check_output(["/system/bin/getprop", "ro.product.manufacturer"])
2296 esn = 'NFANDROID1-PRV-'
2297 input = subprocess.check_output(["/system/bin/getprop", "ro.nrdp.modelgroup"])
2301 esn = esn + input.strip(' \t\n\r') + '-'
2302 esn = esn + '{:5}'.format(manufacturer.strip(' \t\n\r').upper())
2303 input = subprocess.check_output(["/system/bin/getprop" ,"ro.product.model"])
2304 esn = esn + input.strip(' \t\n\r').replace(' ', '=').upper()
2305 self.log(msg='Android generated ESN:' + esn)
2307 except OSError as e:
2308 self.log(msg='Ignoring exception for non Android devices')
2310 # values are accessible via dict (sloppy parsing successfull)
2311 if type(netflix_page_data) == dict:
2312 return netflix_page_data.get('esn', '')
2316 # values are stored in lists (returned from JS parser)
2317 for item in netflix_page_data:
2318 if 'esnGeneratorModel' in dict(item).keys():
2319 esn = item['esnGeneratorModel']['data']['esn']
2322 def _parse_page_contents (self, page_soup):
2323 """Call all the parsers we need to extract all the session relevant data from the HTML page
2324 Directly assigns it to the NetflixSession instance
2328 page_soup : :obj:`BeautifulSoup`
2329 Instance of an BeautifulSoup document or node containing the complete page contents
2331 netflix_page_data = self.extract_inline_netflix_page_data(page_soup=page_soup)
2332 self.user_data = self._parse_user_data(netflix_page_data=netflix_page_data)
2333 self.esn = self._parse_esn_data(netflix_page_data=netflix_page_data)
2334 self.api_data = self._parse_api_base_data(netflix_page_data=netflix_page_data)
2335 self.profiles = self._parse_profile_data(netflix_page_data=netflix_page_data)
2336 self.log(msg='Found ESN "' + self.esn + '"')
2337 return netflix_page_data