2 # -*- coding: utf-8 -*-
3 # Module: NetflixSession
4 # Created on: 13.01.2017
8 from requests import session, cookies
9 from urllib import quote, unquote
11 from base64 import urlsafe_b64encode
12 from bs4 import BeautifulSoup, SoupStrainer
13 from utils import noop
15 import cPickle as pickle
20 """Helps with login/session management of Netflix users & API data fetching"""
22 base_url = 'https://www.netflix.com'
23 """str: Secure Netflix url"""
27 'browse': '/profiles/manage',
28 'video_list_ids': '/preflight',
29 'shakti': '/pathEvaluator',
30 'profiles': '/profiles/manage',
31 'switch_profiles': '/profiles/switch',
32 'adult_pin': '/pin/service',
33 'metadata': '/metadata',
34 'set_video_rating': '/setVideoRating',
35 'update_my_list': '/playlistop'
37 """:obj:`dict` of :obj:`str` List of all static endpoints for HTML/JSON POST/GET requests"""
39 video_list_keys = ['user', 'genres', 'recommendations']
40 """:obj:`list` of :obj:`str` Divide the users video lists into 3 different categories (for easier digestion)"""
44 Dict of user profiles, user id is the key:
47 "profileName": "username",
48 "avatar": "http://..../avatar.png",
50 "isAccountOwner": False,
58 dict of user data (used for authentication):
62 "authURL": "145637....",
69 dict of api data (used to build up the api urls):
72 "API_BASE_URL": "/shakti",
73 "API_ROOT": "https://www.netflix.com/api",
74 "BUILD_IDENTIFIER": "113b89c9", "
75 ICHNAEA_ROOT": "/ichnaea"
80 """str: ESN - something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME"""
82 def __init__(self, cookie_path, data_path, verify_ssl=True, log_fn=noop):
83 """Stores the cookie path for later use & instanciates a requests
84 session with a proper user agent & stored cookies/data if available
88 cookie_path : :obj:`str`
91 data_path : :obj:`str`
92 User data cache location
97 self.cookie_path = cookie_path
98 self.data_path = data_path
99 self.verify_ssl = verify_ssl
102 # start session, fake chrome on the current platform (so that we get a proper widevine esn) & enable gzip
103 self.session = session()
104 self.session.headers.update({
105 'User-Agent': self._get_user_agent_for_current_platform(),
106 'Accept-Encoding': 'gzip'
109 def parse_login_form_fields (self, form_soup):
110 """Fetches all the inputfields from the login form, so that we
111 can build a request with all the fields needed besides the known email & password ones
115 form_soup : :obj:`BeautifulSoup`
116 Instance of an BeautifulSoup documet or node containing the login form
120 :obj:`dict` of :obj:`str`
121 Dictionary of all input fields with their name as the key & the default
122 value from the form field
124 login_input_fields = {}
125 login_inputs = form_soup.find_all('input')
126 # gather all form fields, set an empty string as the default value
127 for item in login_inputs:
128 keys = dict(item.attrs).keys()
129 if 'name' in keys and 'value' not in keys:
130 login_input_fields[item['name']] = ''
131 elif 'name' in keys and 'value' in keys:
132 login_input_fields[item['name']] = item['value']
133 return login_input_fields
135 def extract_inline_netflix_page_data (self, page_soup):
136 """Extracts all <script/> tags from the given document and parses the contents of each one of `em.
137 The contents of the parsable tags looks something like this:
138 <script>window.netflix = window.netflix || {} ; netflix.notification = {"constants":{"sessionLength":30,"ownerToken":"ZDD...};</script>
139 We use a JS parser to generate an AST of the code given & then parse that AST into a python dict.
140 This should be okay, as we´re only interested in a few static values & put the rest aside
144 page_soup : :obj:`BeautifulSoup`
145 Instance of an BeautifulSoup document or node containing the complete page contents
148 :obj:`list` of :obj:`dict`
149 List of all the serialized data pulled out of the pagws <script/> tags
151 scripts = page_soup.find_all('script', attrs={'src': None});
152 self.log('Trying sloppy inline data parser')
153 inline_data = self._sloppy_parse_inline_data(scripts=scripts)
154 if self._verfify_auth_and_profiles_data(data=inline_data) != False:
155 self.log('Sloppy inline data parsing successfull')
157 self.log('Sloppy inline parser failed, trying JS parser')
158 return self._accurate_parse_inline_data(scripts=scripts)
160 def is_logged_in (self, account):
161 """Determines if a user is already logged in (with a valid cookie),
162 by fetching the index page with the current cookie & checking for the
163 `membership status` user data
167 account : :obj:`dict` of :obj:`str`
168 Dict containing an email, country & a password property
173 User is already logged in (e.g. Cookie is valid) or not
177 account_hash = self._generate_account_hash(account=account)
178 if self._load_cookies(filename=self.cookie_path + '_' + account_hash) == False:
180 if self._load_data(filename=self.data_path + '_' + account_hash) == False:
181 # load the profiles page (to verify the user)
182 response = self._session_get(component='profiles')
184 # parse out the needed inline information
185 only_script_tags = SoupStrainer('script')
186 page_soup = BeautifulSoup(response.text, 'html.parser', parse_only=only_script_tags)
187 page_data = self._parse_page_contents(page_soup=page_soup)
189 # check if the cookie is still valid
190 for item in page_data:
191 if 'profilesList' in dict(item).keys():
192 if item['profilesList']['summary']['length'] >= 1:
198 """Delete all cookies and session data
202 account : :obj:`dict` of :obj:`str`
203 Dict containing an email, country & a password property
206 self._delete_cookies(path=self.cookie_path)
207 self._delete_data(path=self.data_path)
209 def login (self, account):
210 """Try to log in a user with its credentials & stores the cookies if the action is successfull
212 Note: It fetches the HTML of the login page to extract the fields of the login form,
213 again, this is dirty, but as the fields & their values could change at any time, this
214 should be the most reliable way of retrieving the information
218 account : :obj:`dict` of :obj:`str`
219 Dict containing an email, country & a password property
224 User could be logged in or not
226 response = self._session_get(component='login')
227 if response.status_code != 200:
230 # collect all the login fields & their contents and add the user credentials
231 page_soup = BeautifulSoup(response.text, 'html.parser')
232 login_form = page_soup.find(attrs={'class' : 'ui-label-text'}).findPrevious('form')
233 login_payload = self.parse_login_form_fields(form_soup=login_form)
234 if 'email' in login_payload:
235 login_payload['email'] = account['email']
236 if 'emailOrPhoneNumber' in login_payload:
237 login_payload['emailOrPhoneNumber'] = account['email']
238 login_payload['password'] = account['password']
241 login_response = self._session_post(component='login', data=login_payload)
242 login_soup = BeautifulSoup(login_response.text, 'html.parser')
244 # we know that the login was successfull if we find an HTML element with the class of 'profile-name'
245 if login_soup.find(attrs={'class' : 'profile-name'}) or login_soup.find(attrs={'class' : 'profile-icon'}):
246 # parse the needed inline information & store cookies for later requests
247 self._parse_page_contents(page_soup=login_soup)
248 account_hash = self._generate_account_hash(account=account)
249 self._save_cookies(filename=self.cookie_path + '_' + account_hash)
250 self._save_data(filename=self.data_path + '_' + account_hash)
255 def switch_profile (self, profile_id, account):
256 """Switch the user profile based on a given profile id
258 Note: All available profiles & their ids can be found in the ´profiles´ property after a successfull login
262 profile_id : :obj:`str`
265 account : :obj:`dict` of :obj:`str`
266 Dict containing an email, country & a password property
271 User could be switched or not
274 'switchProfileGuid': profile_id,
276 'authURL': self.user_data['authURL']
279 response = self._session_get(component='switch_profiles', type='api', params=payload)
280 if response.status_code != 200:
283 account_hash = self._generate_account_hash(account=account)
284 self.user_data['guid'] = profile_id;
285 return self._save_data(filename=self.data_path + '_' + account_hash)
287 def send_adult_pin (self, pin):
288 """Send the adult pin to Netflix in case an adult rated video requests it
290 Note: Once entered, it should last for the complete session (Not so sure about this)
300 Pin was accepted or not
302 :obj:`dict` of :obj:`str`
307 'authURL': self.user_data['authURL']
309 response = self._session_get(component='adult_pin', params=payload)
310 pin_response = self._process_response(response=response, component=self._get_api_url_for(component='adult_pin'))
311 keys = pin_response.keys()
312 if 'success' in keys:
318 def add_to_list (self, video_id):
319 """Adds a video to "my list" on Netflix
323 video_id : :obj:`str`
324 ID of th show/video/movie to be added
329 Adding was successfull
331 return self._update_my_list(video_id=video_id, operation='add')
333 def remove_from_list (self, video_id):
334 """Removes a video from "my list" on Netflix
338 video_id : :obj:`str`
339 ID of th show/video/movie to be removed
344 Removing was successfull
346 return self._update_my_list(video_id=video_id, operation='remove')
348 def rate_video (self, video_id, rating):
349 """Rate a video on Netflix
353 video_id : :obj:`str`
354 ID of th show/video/movie to be rated
357 Rating, must be between 0 & 10
362 Rating successfull or not
365 # dirty rating validation
367 if rating > 10 or rating < 0:
370 # In opposition to Kodi, Netflix uses a rating from 0 to in 0.5 steps
375 'Content-Type': 'application/json',
376 'Accept': 'application/json, text/javascript, */*',
384 payload = json.dumps({
385 'authURL': self.user_data['authURL']
388 response = self._session_post(component='set_video_rating', type='api', params=params, headers=headers, data=payload)
389 return response.status_code == 200
391 def parse_video_list_ids (self, response_data):
392 """Parse the list of video ids e.g. rip out the parts we need
396 response_data : :obj:`dict` of :obj:`str`
397 Parsed response JSON from the ´fetch_video_list_ids´ call
401 :obj:`dict` of :obj:`dict`
402 Video list ids in the format:
406 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568367": {
407 "displayName": "US-Serien",
408 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568367",
413 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568368": {
418 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568364": {
419 "displayName": "Meine Liste",
420 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568364",
425 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568365": {
430 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382": {
431 "displayName": "Passend zu Family Guy",
432 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382",
437 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568397": {
443 # prepare the return dictionary
445 for key in self.video_list_keys:
446 video_list_ids[key] = {}
448 # subcatogorize the lists by their context
449 video_lists = response_data['lists']
450 for video_list_id in video_lists.keys():
451 video_list = video_lists[video_list_id]
452 if video_list['context'] == 'genre':
453 video_list_ids['genres'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
454 elif video_list['context'] == 'similars' or video_list['context'] == 'becauseYouAdded':
455 video_list_ids['recommendations'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
457 video_list_ids['user'].update(self.parse_video_list_ids_entry(id=video_list_id, entry=video_list))
459 return video_list_ids
461 def parse_video_list_ids_entry (self, id, entry):
462 """Parse a video id entry e.g. rip out the parts we need
466 response_data : :obj:`dict` of :obj:`str`
467 Dictionary entry from the ´fetch_video_list_ids´ call
472 Unique id of the video list
474 entry : :obj:`dict` of :obj:`str`
475 Video list entry in the format:
477 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382": {
478 "displayName": "Passend zu Family Guy",
479 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382",
488 'index': entry['index'],
489 'name': entry['context'],
490 'displayName': entry['displayName'],
491 'size': entry['length']
495 def parse_search_results (self, response_data):
496 """Parse the list of search results, rip out the parts we need
497 and extend it with detailed show informations
501 response_data : :obj:`dict` of :obj:`str`
502 Parsed response JSON from the `fetch_search_results` call
506 :obj:`dict` of :obj:`dict` of :obj:`str`
507 Search results in the format:
511 "boxarts": "https://art-s.nflximg.net/0d7af/d5c72668c35d3da65ae031302bd4ae1bcc80d7af.jpg",
512 "detail_text": "Die legend\u00e4re und mit 13 Emmys nominierte Serie von Gene Roddenberry inspirierte eine ganze Generation.",
514 "season_id": "70109435",
515 "synopsis": "Unter Befehl von Captain Kirk begibt sich die Besatzung des Raumschiffs Enterprise in die Tiefen des Weltraums, wo sie fremde Galaxien und neue Zivilisationen erforscht.",
516 "title": "Star Trek",
525 raw_search_results = response_data['value']['videos']
526 for entry_id in raw_search_results:
527 if self._is_size_key(key=entry_id) == False:
528 # fetch information about each show & build up a proper search results dictionary
529 show = self.parse_show_list_entry(id=entry_id, entry=raw_search_results[entry_id])
530 show[entry_id].update(self.parse_show_information(id=entry_id, response_data=self.fetch_show_information(id=entry_id, type=show[entry_id]['type'])))
531 search_results.update(show)
532 return search_results
534 def parse_show_list_entry (self, id, entry):
535 """Parse a show entry e.g. rip out the parts we need
539 response_data : :obj:`dict` of :obj:`str`
540 Dictionary entry from the ´fetch_show_information´ call
543 Unique id of the video list
547 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
548 Show list entry in the format:
551 "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382": {
552 "id": "3589e2c6-ca3b-48b4-a72d-34f2c09ffbf4_11568382",
553 "title": "Enterprise",
554 "boxarts": "https://art-s.nflximg.net/.../smth.jpg",
562 'title': entry['title'],
563 'boxarts': entry['boxarts']['_342x192']['jpg']['url'],
564 'type': entry['summary']['type']
568 def parse_video_list (self, response_data):
569 """Parse a list of videos
573 response_data : :obj:`dict` of :obj:`str`
574 Parsed response JSON from the `fetch_video_list` call
578 :obj:`dict` of :obj:`dict`
579 Video list in the format:
585 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
586 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
601 "episode_count": null,
607 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg",
608 "list_id": "9588df32-f957-40e4-9055-1f6f33b60103_46891306",
611 "description": "Nur f\u00fcr Erwachsene geeignet.",
617 "regular_synopsis": "Ein Spielzeughersteller erweckt aus Versehen die Seele der M\u00f6rderpuppe Chucky erneut zum Leben, die sich unmittelbar wieder ihren m\u00f6rderischen Aktivit\u00e4ten zuwendet.",
619 "seasons_count": null,
620 "seasons_label": null,
621 "synopsis": "Die allseits beliebte, von D\u00e4monen besessene M\u00f6rderpuppe ist wieder da und verbreitet erneut Horror und Schrecken.",
626 "title": "Chucky 2 \u2013 Die M\u00f6rderpuppe ist wieder da",
634 "big": "https://art-s.nflximg.net/7c10d/5dcc3fc8f08487e92507627068cfe26ef727c10d.jpg",
635 "small": "https://art-s.nflximg.net/5bc0e/f3be361b8c594929062f90a8d9c6eb57fb75bc0e.jpg"
652 "interesting_moment": "https://art-s.nflximg.net/0188e/19cd705a71ee08c8d2609ae01cd8a97a86c0188e.jpg",
653 "list_id": "9588df32-f957-40e4-9055-1f6f33b60103_46891306",
656 "description": "Geeignet ab 12 Jahren.",
662 "regular_synopsis": "Comedy-Serie \u00fcber die Erlebnisse eines Tatortreinigers, der seine schmutzige Arbeit erst beginnen kann, wenn die Polizei die Tatortanalyse abgeschlossen hat.",
665 "seasons_label": "5 Staffeln",
666 "synopsis": "In den meisten Krimiserien werden Mordf\u00e4lle auf faszinierende und spannende Weise gel\u00f6st. Diese Serie ist anders.",
670 "title": "Der Tatortreiniger",
678 raw_video_list = response_data['value']
679 netflix_list_id = self.parse_netflix_list_id(video_list=raw_video_list);
680 for video_id in raw_video_list['videos']:
681 if self._is_size_key(key=video_id) == False:
682 video_list.update(self.parse_video_list_entry(id=video_id, list_id=netflix_list_id, video=raw_video_list['videos'][video_id], persons=raw_video_list['person'], genres=raw_video_list['genres']))
685 def parse_video_list_entry (self, id, list_id, video, persons, genres):
686 """Parse a video list entry e.g. rip out the parts we need
691 Unique id of the video
694 Unique id of the containing list
696 video : :obj:`dict` of :obj:`str`
697 Video entry from the ´fetch_video_list´ call
699 persons : :obj:`dict` of :obj:`dict` of :obj:`str`
700 List of persons with reference ids
702 persons : :obj:`dict` of :obj:`dict` of :obj:`str`
703 List of genres with reference ids
707 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
708 Video list entry in the format:
714 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
715 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
730 "episode_count": null,
736 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg",
737 "list_id": "9588df32-f957-40e4-9055-1f6f33b60103_46891306",
740 "description": "Nur f\u00fcr Erwachsene geeignet.",
746 "regular_synopsis": "Ein Spielzeughersteller erweckt aus Versehen die Seele der M\u00f6rderpuppe Chucky erneut zum Leben, die sich unmittelbar wieder ihren m\u00f6rderischen Aktivit\u00e4ten zuwendet.",
748 "seasons_count": null,
749 "seasons_label": null,
750 "synopsis": "Die allseits beliebte, von D\u00e4monen besessene M\u00f6rderpuppe ist wieder da und verbreitet erneut Horror und Schrecken.",
755 "title": "Chucky 2 \u2013 Die M\u00f6rderpuppe ist wieder da",
762 season_info = self.parse_season_information_for_video(video=video)
767 'title': video['title'],
768 'synopsis': video['synopsis'],
769 'regular_synopsis': video['regularSynopsis'],
770 'type': video['summary']['type'],
771 'rating': video['userRating']['average'],
772 'episode_count': season_info['episode_count'],
773 'seasons_label': season_info['seasons_label'],
774 'seasons_count': season_info['seasons_count'],
775 'in_my_list': video['queue']['inQueue'],
776 'year': video['releaseYear'],
777 'runtime': self.parse_runtime_for_video(video=video),
778 'watched': video['watched'],
779 'tags': self.parse_tags_for_video(video=video),
780 'genres': self.parse_genres_for_video(video=video, genres=genres),
781 'quality': self.parse_quality_for_video(video=video),
782 'cast': self.parse_cast_for_video(video=video, persons=persons),
783 'directors': self.parse_directors_for_video(video=video, persons=persons),
784 'creators': self.parse_creators_for_video(video=video, persons=persons),
786 'board': None if 'board' not in video['maturity']['rating'].keys() else video['maturity']['rating']['board'],
787 'value': None if 'value' not in video['maturity']['rating'].keys() else video['maturity']['rating']['value'],
788 'description': None if 'maturityDescription' not in video['maturity']['rating'].keys() else video['maturity']['rating']['maturityDescription'],
789 'level': None if 'maturityLevel' not in video['maturity']['rating'].keys() else video['maturity']['rating']['maturityLevel']
792 'small': video['boxarts']['_342x192']['jpg']['url'],
793 'big': video['boxarts']['_1280x720']['jpg']['url']
795 'interesting_moment': None if 'interestingMoment' not in video.keys() else video['interestingMoment']['_665x375']['jpg']['url'],
796 'artwork': video['artWorkByType']['BILLBOARD']['_1280x720']['jpg']['url'],
800 def parse_creators_for_video (self, video, persons):
801 """Matches ids with person names to generate a list of creators
805 video : :obj:`dict` of :obj:`str`
806 Dictionary entry for one video entry
808 persons : :obj:`dict` of :obj:`str`
809 Raw resposne of all persons delivered by the API call
813 :obj:`list` of :obj:`str`
817 for person_key in dict(persons).keys():
818 if self._is_size_key(key=person_key) == False and person_key != 'summary':
819 for creator_key in dict(video['creators']).keys():
820 if self._is_size_key(key=creator_key) == False and creator_key != 'summary':
821 if video['creators'][creator_key][1] == person_key:
822 creators.append(persons[person_key]['name'])
825 def parse_directors_for_video (self, video, persons):
826 """Matches ids with person names to generate a list of directors
830 video : :obj:`dict` of :obj:`str`
831 Dictionary entry for one video entry
833 persons : :obj:`dict` of :obj:`str`
834 Raw resposne of all persons delivered by the API call
838 :obj:`list` of :obj:`str`
842 for person_key in dict(persons).keys():
843 if self._is_size_key(key=person_key) == False and person_key != 'summary':
844 for director_key in dict(video['directors']).keys():
845 if self._is_size_key(key=director_key) == False and director_key != 'summary':
846 if video['directors'][director_key][1] == person_key:
847 directors.append(persons[person_key]['name'])
850 def parse_cast_for_video (self, video, persons):
851 """Matches ids with person names to generate a list of cast members
855 video : :obj:`dict` of :obj:`str`
856 Dictionary entry for one video entry
858 persons : :obj:`dict` of :obj:`str`
859 Raw resposne of all persons delivered by the API call
863 :obj:`list` of :obj:`str`
867 for person_key in dict(persons).keys():
868 if self._is_size_key(key=person_key) == False and person_key != 'summary':
869 for cast_key in dict(video['cast']).keys():
870 if self._is_size_key(key=cast_key) == False and cast_key != 'summary':
871 if video['cast'][cast_key][1] == person_key:
872 cast.append(persons[person_key]['name'])
875 def parse_genres_for_video (self, video, genres):
876 """Matches ids with genre names to generate a list of genres for a video
880 video : :obj:`dict` of :obj:`str`
881 Dictionary entry for one video entry
883 genres : :obj:`dict` of :obj:`str`
884 Raw resposne of all genres delivered by the API call
888 :obj:`list` of :obj:`str`
892 for genre_key in dict(genres).keys():
893 if self._is_size_key(key=genre_key) == False and genre_key != 'summary':
894 for show_genre_key in dict(video['genres']).keys():
895 if self._is_size_key(key=show_genre_key) == False and show_genre_key != 'summary':
896 if video['genres'][show_genre_key][1] == genre_key:
897 video_genres.append(genres[genre_key]['name'])
900 def parse_tags_for_video (self, video):
901 """Parses a nested list of tags, removes the not needed meta information & returns a raw string list
905 video : :obj:`dict` of :obj:`str`
906 Dictionary entry for one video entry
910 :obj:`list` of :obj:`str`
914 for tag_key in dict(video['tags']).keys():
915 if self._is_size_key(key=tag_key) == False and tag_key != 'summary':
916 tags.append(video['tags'][tag_key]['name'])
919 def parse_season_information_for_video (self, video):
920 """Checks if the fiven video is a show (series) and returns season & episode information
924 video : :obj:`dict` of :obj:`str`
925 Dictionary entry for one video entry
929 :obj:`dict` of :obj:`str`
930 Episode count / Season Count & Season label if given
933 'episode_count': None,
934 'seasons_label': None,
935 'seasons_count': None
937 if video['summary']['type'] == 'show':
939 'episode_count': video['episodeCount'],
940 'seasons_label': video['numSeasonsLabel'],
941 'seasons_count': video['seasonCount']
945 def parse_quality_for_video (self, video):
946 """Transforms Netflix quality information in video resolution info
950 video : :obj:`dict` of :obj:`str`
951 Dictionary entry for one video entry
959 if video['videoQuality']['hasHD']:
961 if video['videoQuality']['hasUltraHD']:
965 def parse_runtime_for_video (self, video):
966 """Checks if the video is a movie & returns the runtime if given
970 video : :obj:`dict` of :obj:`str`
971 Dictionary entry for one video entry
976 Runtime of the video (in seconds)
979 if video['summary']['type'] != 'show':
980 runtime = video['runtime']
983 def parse_netflix_list_id (self, video_list):
984 """Parse a video list and extract the list id
988 video_list : :obj:`dict` of :obj:`str`
993 entry : :obj:`str` or None
996 netflix_list_id = None
997 if 'lists' in video_list.keys():
998 for video_id in video_list['lists']:
999 if self._is_size_key(key=video_id) == False:
1000 netflix_list_id = video_id;
1001 return netflix_list_id
1003 def parse_show_information (self, id, response_data):
1004 """Parse extended show information (synopsis, seasons, etc.)
1011 response_data : :obj:`dict` of :obj:`str`
1012 Parsed response JSON from the `fetch_show_information` call
1016 entry : :obj:`dict` of :obj:`str`
1017 Show information in the format:
1019 "season_id": "80113084",
1020 "synopsis": "Aus verzweifelter Geldnot versucht sich der Familienvater und Drucker Jochen als Geldf\u00e4lscher und rutscht dabei immer mehr in die dunkle Welt des Verbrechens ab."
1021 "detail_text": "I´m optional"
1025 raw_show = response_data['value']['videos'][id]
1026 show.update({'synopsis': raw_show['regularSynopsis']})
1027 if 'evidence' in raw_show:
1028 show.update({'detail_text': raw_show['evidence']['value']['text']})
1029 if 'seasonList' in raw_show:
1030 show.update({'season_id': raw_show['seasonList']['current'][1]})
1033 def parse_seasons (self, id, response_data):
1034 """Parse a list of seasons for a given show
1041 response_data : :obj:`dict` of :obj:`str`
1042 Parsed response JSON from the `fetch_seasons_for_show` call
1046 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1047 Season information in the format:
1052 "shortName": "St. 1",
1054 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
1055 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
1057 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg"
1062 "shortName": "St. 2",
1064 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
1065 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
1067 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg"
1072 raw_seasons = response_data['value']
1073 for season in raw_seasons['seasons']:
1074 if self._is_size_key(key=season) == False:
1075 seasons.update(self.parse_season_entry(season=raw_seasons['seasons'][season], videos=raw_seasons['videos']))
1078 def parse_season_entry (self, season, videos):
1079 """Parse a season list entry e.g. rip out the parts we need
1083 season : :obj:`dict` of :obj:`str`
1084 Season entry from the `fetch_seasons_for_show` call
1088 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1089 Season list entry in the format:
1095 "shortName": "St. 1",
1097 "big": "https://art-s.nflximg.net/5e7d3/b3b48749843fd3a36db11c319ffa60f96b55e7d3.jpg",
1098 "small": "https://art-s.nflximg.net/57543/a039845c2eb9186dc26019576d895bf5a1957543.jpg"
1100 "interesting_moment": "https://art-s.nflximg.net/09544/ed4b3073394b4469fb6ec22b9df81a4f5cb09544.jpg"
1106 for key in videos.keys():
1107 if self._is_size_key(key=key) == False:
1111 for idx in videos[video_key]['seasonList']:
1112 if self._is_size_key(key=idx) == False and idx != 'summary':
1113 sorting[int(videos[video_key]['seasonList'][idx][1])] = int(idx)
1115 season['summary']['id']: {
1116 'idx': sorting[season['summary']['id']],
1117 'id': season['summary']['id'],
1118 'text': season['summary']['name'],
1119 'shortName': season['summary']['shortName'],
1121 'small': videos[video_key]['boxarts']['_342x192']['jpg']['url'],
1122 'big': videos[video_key]['boxarts']['_1280x720']['jpg']['url']
1124 'interesting_moment': videos[video_key]['interestingMoment']['_665x375']['jpg']['url'],
1128 def parse_episodes_by_season (self, response_data):
1129 """Parse episodes for a given season/episode list
1133 response_data : :obj:`dict` of :obj:`str`
1134 Parsed response JSON from the `fetch_seasons_for_show` call
1138 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1139 Season information in the format:
1143 "banner": "https://art-s.nflximg.net/63a36/c7fdfe6604ef2c22d085ac5dca5f69874e363a36.jpg",
1146 "fanart": "https://art-s.nflximg.net/74e02/e7edcc5cc7dcda1e94d505df2f0a2f0d22774e02.jpg",
1152 "mediatype": "episode",
1156 "plot": "Als die Griffins und andere Einwohner von Quahog in die Villa von James Woods eingeladen werden, muss pl\u00f6tzlich ein Mord aufgekl\u00e4rt werden.",
1157 "poster": "https://art-s.nflximg.net/72fd6/57088715e8d436fdb6986834ab39124b0a972fd6.jpg",
1158 "rating": 3.9111512,
1160 "thumb": "https://art-s.nflximg.net/be686/07680670a68da8749eba607efb1ae37f9e3be686.jpg",
1161 "title": "Und dann gab es weniger (Teil 1)",
1166 "banner": "https://art-s.nflximg.net/63a36/c7fdfe6604ef2c22d085ac5dca5f69874e363a36.jpg",
1169 "fanart": "https://art-s.nflximg.net/c472c/6c10f9578bf2c1d0a183c2ccb382931efcbc472c.jpg",
1175 "mediatype": "episode",
1179 "plot": "Wer ist der M\u00f6rder? Nach zahlreichen Morden wird immer wieder jemand anderes verd\u00e4chtigt.",
1180 "poster": "https://art-s.nflximg.net/72fd6/57088715e8d436fdb6986834ab39124b0a972fd6.jpg",
1181 "rating": 3.9111512,
1183 "thumb": "https://art-s.nflximg.net/15a08/857d59126641987bec302bb147a802a00d015a08.jpg",
1184 "title": "Und dann gab es weniger (Teil 2)",
1191 raw_episodes = response_data['value']['videos']
1192 for episode_id in raw_episodes:
1193 if self._is_size_key(key=episode_id) == False:
1194 if (raw_episodes[episode_id]['summary']['type'] == 'episode'):
1195 episodes.update(self.parse_episode(episode=raw_episodes[episode_id], genres=response_data['value']['genres']))
1198 def parse_episode (self, episode, genres=None):
1199 """Parse episode from an list of episodes by season
1203 episode : :obj:`dict` of :obj:`str`
1204 Episode entry from the `fetch_episodes_by_season` call
1208 entry : :obj:`dict` of :obj:`dict` of :obj:`str`
1209 Episode information in the format:
1213 "banner": "https://art-s.nflximg.net/63a36/c7fdfe6604ef2c22d085ac5dca5f69874e363a36.jpg",
1216 "fanart": "https://art-s.nflximg.net/74e02/e7edcc5cc7dcda1e94d505df2f0a2f0d22774e02.jpg",
1222 "mediatype": "episode",
1226 "plot": "Als die Griffins und andere Einwohner von Quahog in die Villa von James Woods eingeladen werden, muss pl\u00f6tzlich ein Mord aufgekl\u00e4rt werden.",
1227 "poster": "https://art-s.nflximg.net/72fd6/57088715e8d436fdb6986834ab39124b0a972fd6.jpg",
1228 "rating": 3.9111512,
1230 "thumb": "https://art-s.nflximg.net/be686/07680670a68da8749eba607efb1ae37f9e3be686.jpg",
1231 "title": "Und dann gab es weniger (Teil 1)",
1238 episode['summary']['id']: {
1239 'id': episode['summary']['id'],
1240 'episode': episode['summary']['episode'],
1241 'season': episode['summary']['season'],
1242 'plot': episode['info']['synopsis'],
1243 'duration': episode['info']['runtime'],
1244 'title': episode['info']['title'],
1245 'year': episode['info']['releaseYear'],
1246 'genres': self.parse_genres_for_video(video=episode, genres=genres),
1247 'mpaa': str(episode['maturity']['rating']['board']) + ' ' + str(episode['maturity']['rating']['value']),
1248 'maturity': episode['maturity'],
1249 'playcount': (0, 1)[episode['watched']],
1250 'rating': episode['userRating']['average'],
1251 'thumb': episode['info']['interestingMoments']['url'],
1252 'fanart': episode['interestingMoment']['_1280x720']['jpg']['url'],
1253 'poster': episode['boxarts']['_1280x720']['jpg']['url'],
1254 'banner': episode['boxarts']['_342x192']['jpg']['url'],
1255 'mediatype': {'episode': 'episode', 'movie': 'movie'}[episode['summary']['type']],
1256 'my_list': episode['queue']['inQueue'],
1257 'bookmark': episode['bookmarkPosition']
1261 def fetch_browse_list_contents (self):
1262 """Fetches the HTML data for the lists on the landing page (browse page) of Netflix
1266 :obj:`BeautifulSoup`
1267 Instance of an BeautifulSoup document containing the complete page contents
1269 response = self._session_get(component='browse')
1270 return BeautifulSoup(response.text, 'html.parser')
1272 def fetch_video_list_ids (self, list_from=0, list_to=50):
1273 """Fetches the JSON with detailed information based on the lists on the landing page (browse page) of Netflix
1277 list_from : :obj:`int`
1278 Start entry for pagination
1280 list_to : :obj:`int`
1281 Last entry for pagination
1285 :obj:`dict` of :obj:`dict` of :obj:`str`
1286 Raw Netflix API call response or api call error
1289 'fromRow': list_from,
1291 'opaqueImageExtension': 'jpg',
1292 'transparentImageExtension': 'png',
1294 'authURL': self.user_data['authURL']
1297 # check if we have a root lolomo for that user within our cookies
1298 for cookie in self.session.cookies:
1299 if cookie.name == 'lhpuuidh-browse-' + self.user_data['guid']:
1300 value = unquote(cookie.value)
1301 payload['lolomoid'] = value[value.rfind(':')+1:];
1303 response = self._session_get(component='video_list_ids', params=payload, type='api')
1304 return self._process_response(response=response, component=self._get_api_url_for(component='video_list_ids'))
1306 def fetch_search_results (self, search_str, list_from=0, list_to=10):
1307 """Fetches the JSON which contains the results for the given search query
1311 search_str : :obj:`str`
1312 String to query Netflix search for
1314 list_from : :obj:`int`
1315 Start entry for pagination
1317 list_to : :obj:`int`
1318 Last entry for pagination
1322 :obj:`dict` of :obj:`dict` of :obj:`str`
1323 Raw Netflix API call response or api call error
1325 # properly encode the search string
1326 encoded_search_string = quote(search_str)
1329 ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, ['summary', 'title']],
1330 ['search', encoded_search_string, 'titles', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1331 ['search', encoded_search_string, 'titles', ['id', 'length', 'name', 'trackIds', 'requestId']],
1332 ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, ['summary', 'title']],
1333 ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1334 ['search', encoded_search_string, 'suggestions', 0, 'relatedvideos', ['id', 'length', 'name', 'trackIds', 'requestId']]
1336 response = self._path_request(paths=paths)
1337 return self._process_response(response=response, component='Search results')
1339 def fetch_video_list (self, list_id, list_from=0, list_to=20):
1340 """Fetches the JSON which contains the contents of a given video list
1344 list_id : :obj:`str`
1345 Unique list id to query Netflix for
1347 list_from : :obj:`int`
1348 Start entry for pagination
1350 list_to : :obj:`int`
1351 Last entry for pagination
1355 :obj:`dict` of :obj:`dict` of :obj:`str`
1356 Raw Netflix API call response or api call error
1359 ['lists', list_id, {'from': list_from, 'to': list_to}, ['summary', 'title', 'synopsis', 'regularSynopsis', 'evidence', 'queue', 'episodeCount', 'info', 'maturity', 'runtime', 'seasonCount', 'releaseYear', 'userRating', 'numSeasonsLabel', 'bookmarkPosition', 'watched', 'videoQuality']],
1360 ['lists', list_id, {'from': list_from, 'to': list_to}, 'cast', {'from': 0, 'to': 15}, ['id', 'name']],
1361 ['lists', list_id, {'from': list_from, 'to': list_to}, 'cast', 'summary'],
1362 ['lists', list_id, {'from': list_from, 'to': list_to}, 'genres', {'from': 0, 'to': 5}, ['id', 'name']],
1363 ['lists', list_id, {'from': list_from, 'to': list_to}, 'genres', 'summary'],
1364 ['lists', list_id, {'from': list_from, 'to': list_to}, 'tags', {'from': 0, 'to': 9}, ['id', 'name']],
1365 ['lists', list_id, {'from': list_from, 'to': list_to}, 'tags', 'summary'],
1366 ['lists', list_id, {'from': list_from, 'to': list_to}, ['creators', 'directors'], {'from': 0, 'to': 49}, ['id', 'name']],
1367 ['lists', list_id, {'from': list_from, 'to': list_to}, ['creators', 'directors'], 'summary'],
1368 ['lists', list_id, {'from': list_from, 'to': list_to}, 'bb2OGLogo', '_400x90', 'png'],
1369 ['lists', list_id, {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1370 ['lists', list_id, {'from': list_from, 'to': list_to}, 'boxarts', '_1280x720', 'jpg'],
1371 ['lists', list_id, {'from': list_from, 'to': list_to}, 'storyarts', '_1632x873', 'jpg'],
1372 ['lists', list_id, {'from': list_from, 'to': list_to}, 'interestingMoment', '_665x375', 'jpg'],
1373 ['lists', list_id, {'from': list_from, 'to': list_to}, 'artWorkByType', 'BILLBOARD', '_1280x720', 'jpg']
1376 response = self._path_request(paths=paths)
1377 return self._process_response(response=response, component='Video list')
1379 def fetch_video_list_information (self, video_ids):
1380 """Fetches the JSON which contains the detail information of a list of given video ids
1384 video_ids : :obj:`list` of :obj:`str`
1385 List of video ids to fetch detail data for
1389 :obj:`dict` of :obj:`dict` of :obj:`str`
1390 Raw Netflix API call response or api call error
1393 for video_id in video_ids:
1394 paths.append(['videos', video_id, ['summary', 'title', 'synopsis', 'regularSynopsis', 'evidence', 'queue', 'episodeCount', 'info', 'maturity', 'runtime', 'seasonCount', 'releaseYear', 'userRating', 'numSeasonsLabel', 'bookmarkPosition', 'watched', 'videoQuality']])
1395 paths.append(['videos', video_id, 'cast', {'from': 0, 'to': 15}, ['id', 'name']])
1396 paths.append(['videos', video_id, 'cast', 'summary'])
1397 paths.append(['videos', video_id, 'genres', {'from': 0, 'to': 5}, ['id', 'name']])
1398 paths.append(['videos', video_id, 'genres', 'summary'])
1399 paths.append(['videos', video_id, 'tags', {'from': 0, 'to': 9}, ['id', 'name']])
1400 paths.append(['videos', video_id, 'tags', 'summary'])
1401 paths.append(['videos', video_id, ['creators', 'directors'], {'from': 0, 'to': 49}, ['id', 'name']])
1402 paths.append(['videos', video_id, ['creators', 'directors'], 'summary'])
1403 paths.append(['videos', video_id, 'bb2OGLogo', '_400x90', 'png'])
1404 paths.append(['videos', video_id, 'boxarts', '_342x192', 'jpg'])
1405 paths.append(['videos', video_id, 'boxarts', '_1280x720', 'jpg'])
1406 paths.append(['videos', video_id, 'storyarts', '_1632x873', 'jpg'])
1407 paths.append(['videos', video_id, 'interestingMoment', '_665x375', 'jpg'])
1408 paths.append(['videos', video_id, 'artWorkByType', 'BILLBOARD', '_1280x720', 'jpg'])
1410 response = self._path_request(paths=paths)
1411 return self._process_response(response=response, component='fetch_video_list_information')
1413 def fetch_metadata (self, id):
1414 """Fetches the JSON which contains the metadata for a given show/movie or season id
1419 Show id, movie id or season id
1423 :obj:`dict` of :obj:`dict` of :obj:`str`
1424 Raw Netflix API call response or api call error
1428 'imageformat': 'jpg',
1431 response = self._session_get(component='metadata', params=payload, type='api')
1432 return self._process_response(response=response, component=self._get_api_url_for(component='metadata'))
1434 def fetch_show_information (self, id, type):
1435 """Fetches the JSON which contains the detailed contents of a show
1440 Unique show id to query Netflix for
1443 Can be 'movie' or 'show'
1447 :obj:`dict` of :obj:`dict` of :obj:`str`
1448 Raw Netflix API call response or api call error
1450 # check if we have a show or a movie, the request made depends on this
1453 ['videos', id, ['requestId', 'regularSynopsis', 'evidence']],
1454 ['videos', id, 'seasonList', 'current', 'summary']
1457 paths = [['videos', id, ['requestId', 'regularSynopsis', 'evidence']]]
1458 response = self._path_request(paths=paths)
1459 return self._process_response(response=response, component='Show information')
1461 def fetch_seasons_for_show (self, id, list_from=0, list_to=30):
1462 """Fetches the JSON which contains the seasons of a given show
1467 Unique show id to query Netflix for
1469 list_from : :obj:`int`
1470 Start entry for pagination
1472 list_to : :obj:`int`
1473 Last entry for pagination
1477 :obj:`dict` of :obj:`dict` of :obj:`str`
1478 Raw Netflix API call response or api call error
1481 ['videos', id, 'seasonList', {'from': list_from, 'to': list_to}, 'summary'],
1482 ['videos', id, 'seasonList', 'summary'],
1483 ['videos', id, 'boxarts', '_342x192', 'jpg'],
1484 ['videos', id, 'boxarts', '_1280x720', 'jpg'],
1485 ['videos', id, 'storyarts', '_1632x873', 'jpg'],
1486 ['videos', id, 'interestingMoment', '_665x375', 'jpg']
1488 response = self._path_request(paths=paths)
1489 return self._process_response(response=response, component='Seasons')
1491 def fetch_episodes_by_season (self, season_id, list_from=-1, list_to=40):
1492 """Fetches the JSON which contains the episodes of a given season
1494 TODO: Add more metadata
1498 season_id : :obj:`str`
1499 Unique season_id id to query Netflix for
1501 list_from : :obj:`int`
1502 Start entry for pagination
1504 list_to : :obj:`int`
1505 Last entry for pagination
1509 :obj:`dict` of :obj:`dict` of :obj:`str`
1510 Raw Netflix API call response or api call error
1513 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, ['summary', 'queue', 'info', 'maturity', 'userRating', 'bookmarkPosition', 'creditOffset', 'watched', 'videoQuality']],
1514 #['videos', season_id, 'cast', {'from': 0, 'to': 15}, ['id', 'name']],
1515 #['videos', season_id, 'cast', 'summary'],
1516 #['videos', season_id, 'genres', {'from': 0, 'to': 5}, ['id', 'name']],
1517 #['videos', season_id, 'genres', 'summary'],
1518 #['videos', season_id, 'tags', {'from': 0, 'to': 9}, ['id', 'name']],
1519 #['videos', season_id, 'tags', 'summary'],
1520 #['videos', season_id, ['creators', 'directors'], {'from': 0, 'to': 49}, ['id', 'name']],
1521 #['videos', season_id, ['creators', 'directors'], 'summary'],
1522 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'genres', {'from': 0, 'to': 1}, ['id', 'name']],
1523 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'genres', 'summary'],
1524 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'interestingMoment', '_1280x720', 'jpg'],
1525 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'interestingMoment', '_665x375', 'jpg'],
1526 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'boxarts', '_342x192', 'jpg'],
1527 ['seasons', season_id, 'episodes', {'from': list_from, 'to': list_to}, 'boxarts', '_1280x720', 'jpg']
1529 response = self._path_request(paths=paths)
1530 return self._process_response(response=response, component='fetch_episodes_by_season')
1532 def refresh_session_data (self, account):
1533 """Reload the session data (profiles, user_data, api_data)
1537 account : :obj:`dict` of :obj:`str`
1538 Dict containing an email, country & a password property
1540 # load the profiles page (to verify the user)
1541 response = self._session_get(component='profiles')
1542 # parse out the needed inline information
1543 only_script_tags = SoupStrainer('script')
1544 page_soup = BeautifulSoup(response.text, 'html.parser', parse_only=only_script_tags)
1545 page_data = self._parse_page_contents(page_soup=page_soup)
1546 account_hash = self._generate_account_hash(account=account)
1547 self._save_data(filename=self.data_path + '_' + account_hash)
1549 def _path_request (self, paths):
1550 """Executes a post request against the shakti endpoint with Falcor style payload
1554 paths : :obj:`list` of :obj:`list`
1555 Payload with path querys for the Netflix Shakti API in Falcor style
1559 :obj:`requests.response`
1560 Response from a POST call made with Requests
1563 'Content-Type': 'application/json',
1564 'Accept': 'application/json, text/javascript, */*',
1569 'authURL': self.user_data['authURL']
1573 'model': self.user_data['gpsModel']
1576 return self._session_post(component='shakti', type='api', params=params, headers=headers, data=data)
1578 def _is_size_key (self, key):
1579 """Tiny helper that checks if a given key is called $size or size, as we need to check this often
1584 Key to check the value for
1589 Key has a size value or not
1591 return key == '$size' or key == 'size'
1593 def _get_api_url_for (self, component):
1594 """Tiny helper that builds the url for a requested API endpoint component
1598 component : :obj:`str`
1599 Component endpoint to build the URL for
1606 if self.api_data['API_ROOT'].find(self.api_data['API_BASE_URL']) > -1:
1607 return self.api_data['API_ROOT'] + '/' + self.api_data['BUILD_IDENTIFIER'] + self.urls[component]
1609 return self.api_data['API_ROOT'] + self.api_data['API_BASE_URL'] + '/' + self.api_data['BUILD_IDENTIFIER'] + self.urls[component]
1611 def _get_document_url_for (self, component):
1612 """Tiny helper that builds the url for a requested document endpoint component
1616 component : :obj:`str`
1617 Component endpoint to build the URL for
1624 return self.base_url + self.urls[component]
1626 def _process_response (self, response, component):
1627 """Tiny helper to check responses for API requests
1631 response : :obj:`requests.response`
1632 Response from a requests instance
1634 component : :obj:`str`
1639 :obj:`dict` of :obj:`dict` of :obj:`str` or :obj:`dict` of :obj:`str`
1640 Raw Netflix API call response or api call error
1642 # check if we´re not authorized to make thios call
1643 if response.status_code == 401:
1646 'message': 'Session invalid',
1649 # check if somethign else failed
1650 if response.status_code != 200:
1653 'message': 'API call for "' + component + '" failed',
1654 'code': response.status_code
1656 # return the parsed response & everything´s fine
1657 return response.json()
1659 def _to_unicode(self, str):
1660 '''Attempt to fix non uft-8 string into utf-8, using a limited set of encodings
1672 # fuller list of encodings at http://docs.python.org/library/codecs.html#standard-encodings
1673 if not str: return u''
1675 # we could add more encodings here, as warranted.
1676 encodings = ('ascii', 'utf8', 'latin1')
1677 for enc in encodings:
1680 u = unicode(str,enc)
1681 except UnicodeDecodeError:
1684 u = unicode(str, errors='replace')
1687 def _update_my_list (self, video_id, operation):
1688 """Tiny helper to add & remove items from "my list"
1692 video_id : :obj:`str`
1693 ID of the show/movie to be added
1695 operation : :obj:`str`
1696 Either "add" or "remove"
1701 Operation successfull
1704 'Content-Type': 'application/json',
1705 'Accept': 'application/json, text/javascript, */*',
1708 payload = json.dumps({
1709 'operation': operation,
1710 'videoId': int(video_id),
1711 'authURL': self.user_data['authURL']
1714 response = self._session_post(component='update_my_list', type='api', headers=headers, data=payload)
1715 return response.status_code == 200
1717 def _save_data(self, filename):
1718 """Tiny helper that stores session data from the session in a given file
1722 filename : :obj:`str`
1723 Complete path incl. filename that determines where to store the cookie
1728 Storage procedure was successfull
1730 if not os.path.isdir(os.path.dirname(filename)):
1732 with open(filename, 'w') as f:
1735 'user_data': self.user_data,
1736 'api_data': self.api_data,
1737 'profiles': self.profiles
1740 def _load_data(self, filename):
1741 """Tiny helper that loads session data into the active session from a given file
1745 filename : :obj:`str`
1746 Complete path incl. filename that determines where to load the data from
1751 Load procedure was successfull
1753 if not os.path.isfile(filename):
1756 with open(filename) as f:
1757 data = pickle.load(f)
1759 self.profiles = data['profiles']
1760 self.user_data = data['user_data']
1761 self.api_data = data['api_data']
1765 def _delete_data (self, path):
1766 """Tiny helper that deletes session data
1770 filename : :obj:`str`
1771 Complete path incl. filename that determines where to delete the files
1774 head, tail = os.path.split(path)
1775 for subdir, dirs, files in os.walk(head):
1778 os.remove(os.path.join(subdir, file))
1780 def _save_cookies(self, filename):
1781 """Tiny helper that stores cookies from the session in a given file
1785 filename : :obj:`str`
1786 Complete path incl. filename that determines where to store the cookie
1791 Storage procedure was successfull
1793 if not os.path.isdir(os.path.dirname(filename)):
1795 with open(filename, 'w') as f:
1797 pickle.dump(self.session.cookies._cookies, f)
1799 def _load_cookies(self, filename):
1800 """Tiny helper that loads cookies into the active session from a given file
1804 filename : :obj:`str`
1805 Complete path incl. filename that determines where to load the cookie from
1810 Load procedure was successfull
1812 if not os.path.isfile(filename):
1815 with open(filename) as f:
1816 _cookies = pickle.load(f)
1818 jar = cookies.RequestsCookieJar()
1819 jar._cookies = _cookies
1820 self.session.cookies = jar
1824 def _delete_cookies (self, path):
1825 """Tiny helper that deletes cookie data
1829 filename : :obj:`str`
1830 Complete path incl. filename that determines where to delete the files
1833 head, tail = os.path.split(path)
1834 for subdir, dirs, files in os.walk(head):
1837 os.remove(os.path.join(subdir, file))
1839 def _generate_account_hash (self, account):
1840 """Generates a has for the given account (used for cookie verification)
1844 account : :obj:`dict` of :obj:`str`
1845 Dict containing an email, country & a password property
1852 return urlsafe_b64encode(account['email'])
1854 def _get_user_agent_for_current_platform (self):
1855 """Determines the user agent string for the current platform (to retrieve a valid ESN)
1860 User Agent for platform
1863 if platform == 'linux' or platform == 'linux2':
1864 return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
1865 elif platform == 'darwin':
1866 return 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_10_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
1867 elif platform == 'win32':
1868 return 'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
1870 return 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36'
1872 def _session_post (self, component, type='document', data={}, headers={}, params={}):
1873 """Executes a get request using requests for the current session & measures the duration of that request
1877 component : :obj:`str`
1881 Is it a document or API request ('document' is default)
1883 data : :obj:`dict` of :obj:`str`
1884 Payload body as dict
1886 header : :obj:`dict` of :obj:`str`
1887 Additional headers as dict
1889 params : :obj:`dict` of :obj:`str`
1895 Contents of the field to match
1897 url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
1899 response = self.session.post(url=url, data=data, params=params, headers=headers, verify=self.verify_ssl)
1901 self.log('[POST] Request for "' + url + '" took ' + str(end - start) + ' seconds')
1904 def _session_get (self, component, type='document', params={}):
1905 """Executes a get request using requests for the current session & measures the duration of that request
1909 component : :obj:`str`
1913 Is it a document or API request ('document' is default)
1915 params : :obj:`dict` of :obj:`str`
1921 Contents of the field to match
1923 url = self._get_document_url_for(component=component) if type == 'document' else self._get_api_url_for(component=component)
1925 response = self.session.get(url=url, verify=self.verify_ssl, params=params)
1927 self.log('[GET] Request for "' + url + '" took ' + str(end - start) + ' seconds')
1930 def _sloppy_parse_user_and_api_data (self, key, contents):
1931 """Try to find the user & API data from the inline js by using a string parser
1936 Key to match in the inline js
1938 contents : :obj:`str`
1944 Contents of the field to match
1946 key_start = contents.find(key + '"')
1947 if int(key_start) == -1:
1949 sub_contents = contents[int(key_start):]
1950 l = sub_contents.find('",')
1951 return contents[(int(key_start)+len(key)+3):int(key_start)+l].decode('string_escape')
1953 def _sloppy_parse_profiles (self, contents):
1954 """Try to find the profile data from the inline js by using a string parser & parse/convert the result to JSON
1958 contents : :obj:`str`
1963 :obj:`dict` of :obj:`str` or None
1966 profile_start = contents.find('profiles":')
1967 profile_list_start = contents.find('profilesList')
1968 if int(profile_start) > -1 and int(profile_list_start) > -1:
1971 return json.loads('{"a":{"' + contents[profile_start:profile_list_start-2].decode('string_escape') + '}}').get('a').get('profiles')
1972 except ValueError, e:
1974 except TypeError, e:
1978 def _sloppy_parse_avatars (self, contents):
1979 """Try to find the avatar data from the inline js by using a string parser & parse/convert the result to JSON
1983 contents : :obj:`str`
1988 :obj:`dict` of :obj:`str` or None
1991 avatars_start = contents.find('"nf":')
1992 avatars_list_start = contents.find('"profiles"')
1993 if int(avatars_start) > -1 and int(avatars_list_start) > -1:
1996 return json.loads('{' + contents[avatars_start:avatars_list_start-2].decode('string_escape') + '}')
1997 except ValueError, e:
1999 except TypeError, e:
2003 def _verfify_auth_and_profiles_data (self, data):
2004 """Checks if the authURL has at least a certain length & doesn't overrule a certain length & if the profiles dict exists
2005 Simple validity check for the sloppy data parser
2009 data : :obj:`dict` of :obj:`str`
2017 if type(data.get('profiles')) == dict:
2018 if len(str(data.get('authURL', ''))) > 10 and len(str(data.get('authURL', ''))) < 50:
2022 def _sloppy_parse_inline_data (self, scripts):
2023 """Strips out all the needed user, api & profile data from the inline JS by string parsing
2024 Might fail, so if this doesn't succeed, a proper JS parser will chime in
2026 Note: This has been added for performance reasons only
2030 scripts : :obj:`list` of :obj:`BeautifoulSoup`
2031 Script tags & contents from the Netflix browse page
2035 :obj:`dict` of :obj:`str`
2036 Dict containijg user, api & profile data
2039 for script in scripts:
2040 contents = str(script.contents[0])
2041 important_data = ['authURL', 'API_BASE_URL', 'API_ROOT', 'BUILD_IDENTIFIER', 'ICHNAEA_ROOT', 'gpsModel', 'guid', 'esn']
2043 for key in important_data:
2044 _res = self._sloppy_parse_user_and_api_data(key, contents)
2046 res.update({key: _res})
2048 inline_data.update(res)
2051 profiles = self._sloppy_parse_profiles(contents)
2052 avatars = self._sloppy_parse_avatars(contents)
2053 if profiles != None:
2054 inline_data.update({'profiles': profiles})
2056 inline_data.update(avatars)
2059 def _accurate_parse_inline_data (self, scripts):
2060 """Uses a proper JS parser to fetch all the api, iser & profile data from within the inline JS
2062 Note: This is slow but accurate
2066 scripts : :obj:`list` of :obj:`BeautifoulSoup`
2067 Script tags & contents from the Netflix browse page
2071 :obj:`dict` of :obj:`str`
2072 Dict containing user, api & profile data
2075 from pyjsparser import PyJsParser
2076 parser = PyJsParser()
2077 for script in scripts:
2079 # unicode escape that incoming script stuff
2080 contents = self._to_unicode(str(script.contents[0]))
2081 # parse the JS & load the declarations we´re interested in
2082 parsed = parser.parse(contents)
2083 if len(parsed['body']) > 1 and parsed['body'][1]['expression']['right'].get('properties', None) != None:
2084 declarations = parsed['body'][1]['expression']['right']['properties']
2085 for declaration in declarations:
2086 for key in declaration:
2087 # we found the correct path if the declaration is a dict & of type 'ObjectExpression'
2088 if type(declaration[key]) is dict:
2089 if declaration[key]['type'] == 'ObjectExpression':
2090 # add all static data recursivly
2091 for expression in declaration[key]['properties']:
2092 data[expression['key']['value']] = self._parse_rec(expression['value'])
2093 inline_data.append(data)
2096 def _parse_rec (self, node):
2097 """Iterates over a JavaScript AST and return values found
2105 :obj:`dict` of :obj:`dict` or :obj:`str`
2106 Parsed contents of the node
2108 if node['type'] == 'ObjectExpression':
2110 for prop in node['properties']:
2111 _ret.update({prop['key']['value']: self._parse_rec(prop['value'])})
2113 if node['type'] == 'Literal':
2114 return node['value']
2116 def _parse_user_data (self, netflix_page_data):
2117 """Parse out the user data from the big chunk of dicts we got from
2118 parsing the JSON-ish data from the netflix homepage
2122 netflix_page_data : :obj:`list`
2123 List of all the JSON-ish data that has been extracted from the Netflix homepage
2124 see: extract_inline_netflix_page_data
2128 :obj:`dict` of :obj:`str`
2131 "guid": "72ERT45...",
2132 "authURL": "145637....",
2133 "gpsModel": "harris"
2137 important_fields = [
2143 # values are accessible via dict (sloppy parsing successfull)
2144 if type(netflix_page_data) == dict:
2145 for important_field in important_fields:
2146 user_data.update({important_field: netflix_page_data.get(important_field, '')})
2149 # values are stored in lists (returned from JS parser)
2150 for item in netflix_page_data:
2151 if 'memberContext' in dict(item).keys():
2152 for important_field in important_fields:
2153 user_data.update({important_field: item['memberContext']['data']['userInfo'][important_field]})
2157 def _parse_profile_data (self, netflix_page_data):
2158 """Parse out the profile data from the big chunk of dicts we got from
2159 parsing the JSON-ish data from the netflix homepage
2163 netflix_page_data : :obj:`list`
2164 List of all the JSON-ish data that has been extracted from the Netflix homepage
2165 see: extract_inline_netflix_page_data
2169 :obj:`dict` of :obj:`dict
2173 "profileName": "username",
2174 "avatar": "http://..../avatar.png",
2176 "isAccountOwner": False,
2183 important_fields = [
2190 # values are accessible via dict (sloppy parsing successfull)
2191 if type(netflix_page_data) == dict:
2192 for profile_id in netflix_page_data.get('profiles'):
2193 if self._is_size_key(key=profile_id) == False and type(netflix_page_data['profiles'][profile_id]) == dict and netflix_page_data['profiles'][profile_id].get('avatar', False) != False:
2194 profile = {'id': profile_id}
2195 for important_field in important_fields:
2196 profile.update({important_field: netflix_page_data['profiles'][profile_id]['summary'][important_field]})
2197 avatar_base = netflix_page_data['nf'].get(netflix_page_data['profiles'][profile_id]['summary']['avatarName'], False);
2198 avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
2199 profile.update({'avatar': avatar})
2200 profiles.update({profile_id: profile})
2203 # values are stored in lists (returned from JS parser)
2204 # TODO: get rid of this christmas tree of doom
2205 for item in netflix_page_data:
2206 if 'hasViewedRatingWelcomeModal' in dict(item).keys():
2207 for profile_id in item:
2208 if self._is_size_key(key=profile_id) == False and type(item[profile_id]) == dict and item[profile_id].get('avatar', False) != False:
2209 profile = {'id': profile_id}
2210 for important_field in important_fields:
2211 profile.update({important_field: item[profile_id]['summary'][important_field]})
2212 avatar_base = item['nf'].get(item[profile_id]['summary']['avatarName'], False);
2213 avatar = 'https://secure.netflix.com/ffe/profiles/avatars_v2/320x320/PICON_029.png' if avatar_base == False else avatar_base['images']['byWidth']['320']['value']
2214 profile.update({'avatar': avatar})
2215 profiles.update({profile_id: profile})
2218 def _parse_api_base_data (self, netflix_page_data):
2219 """Parse out the api url data from the big chunk of dicts we got from
2220 parsing the JSOn-ish data from the netflix homepage
2224 netflix_page_data : :obj:`list`
2225 List of all the JSON-ish data that has been extracted from the Netflix homepage
2226 see: extract_inline_netflix_page_data
2230 :obj:`dict` of :obj:`str
2233 "API_BASE_URL": "/shakti",
2234 "API_ROOT": "https://www.netflix.com/api",
2235 "BUILD_IDENTIFIER": "113b89c9",
2236 "ICHNAEA_ROOT": "/ichnaea"
2240 important_fields = [
2247 # values are accessible via dict (sloppy parsing successfull)
2248 if type(netflix_page_data) == dict:
2249 for important_field in important_fields:
2250 api_data.update({important_field: netflix_page_data.get(important_field, '')})
2253 for item in netflix_page_data:
2254 if 'serverDefs' in dict(item).keys():
2255 for important_field in important_fields:
2256 api_data.update({important_field: item['serverDefs']['data'][important_field]})
2259 def _parse_esn_data (self, netflix_page_data):
2260 """Parse out the esn id data from the big chunk of dicts we got from
2261 parsing the JSOn-ish data from the netflix homepage
2265 netflix_page_data : :obj:`list`
2266 List of all the JSON-ish data that has been extracted from the Netflix homepage
2267 see: extract_inline_netflix_page_data
2271 :obj:`str` of :obj:`str
2272 ESN, something like: NFCDCH-MC-D7D6F54LOPY8J416T72MQXX3RD20ME
2275 # values are accessible via dict (sloppy parsing successfull)
2276 if type(netflix_page_data) == dict:
2277 return netflix_page_data.get('esn', '')
2279 # values are stored in lists (returned from JS parser)
2280 for item in netflix_page_data:
2281 if 'esnGeneratorModel' in dict(item).keys():
2282 esn = item['esnGeneratorModel']['data']['esn']
2285 def _parse_page_contents (self, page_soup):
2286 """Call all the parsers we need to extract all the session relevant data from the HTML page
2287 Directly assigns it to the NetflixSession instance
2291 page_soup : :obj:`BeautifulSoup`
2292 Instance of an BeautifulSoup document or node containing the complete page contents
2294 netflix_page_data = self.extract_inline_netflix_page_data(page_soup=page_soup)
2295 self.user_data = self._parse_user_data(netflix_page_data=netflix_page_data)
2296 self.esn = self._parse_esn_data(netflix_page_data=netflix_page_data)
2297 self.api_data = self._parse_api_base_data(netflix_page_data=netflix_page_data)
2298 self.profiles = self._parse_profile_data(netflix_page_data=netflix_page_data)
2299 self.log('Found ESN "' + self.esn + '"')
2300 return netflix_page_data