U
    gC                     @   s   d dl Z d dlZddlmZ ddlmZmZmZmZm	Z	m
Z
mZmZmZmZmZmZmZmZmZ G dd deZG dd	 d	eZG d
d deZG dd deZG dd deZdS )    N   )InfoExtractor   )ExtractorError
clean_htmldetermine_extextract_attributesfloat_or_noneget_elements_html_by_classint_or_nonemerge_dictsmimetype2extparse_iso8601
remove_endremove_startstr_or_nonetraverse_objurl_or_nonec                   @   sB   e Zd ZedZdZdZdZdd Z	dd Z
d	d
 Zdd ZdS )NYTimesBaseIEz$36dd619a-56dc-595b-9e09-37f4152c7b5da  MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAuNIzKBOFB77aT/jN/FQ+/QVKWq5V1ka1AYmCR9hstz1pGNPH5ajOU9gAqta0T89iPnhjwla+3oec/Z3kGjxbpv6miQXufHFq3u2RC6HyU458cLat5kVPSOQCe3VVB5NRpOlRuwKHqn0txfxnwSSj8mqzstR997d3gKB//RO9zE16y3PoWlDQXkASngNJEWvL19iob/xwAkfEWCjyRILWFY0JYX3AvLMSbq7wsqOCE5srJpo7rRU32zsByhsp1D5W9OYqqwDmflsgCEQy2vqTsJjrJohuNg+urMXNNZ7Y3naMoqttsGDrWVxtPBafKMI8pM2ReNZBbGQsQXRzQNo7+QIDAQABz/https://samizdat-graphql.nytimes.com/graphql/v2a  query VideoQuery($id: String!) {
  video(id: $id) {
    ... on Video {
      bylines {
        renderedRepresentation
      }
      duration
      firstPublished
      promotionalHeadline
      promotionalMedia {
        ... on Image {
          crops {
            name
            renditions {
              name
              width
              height
              url
            }
          }
        }
      }
      renditions {
        type
        width
        height
        url
        bitrate
      }
      summary
    }
  }
}c                 C   sr   t | jd}t ||}t| j| j|dtj| jdd| iddd	 dd	d
| j
ddddddthfppi S )Nvideoz!Downloading JSON from GraphQL APIidznyt://video/)query	variables),:)
separatorszapplication/jsonZvhszv3.52.21zhttps://nytimes.com)zContent-TypezNyt-App-TypezNyt-App-Versionz	Nyt-TokenZOriginF)dataheadersfatalr   )uuidZuuid5_DNS_NAMESPACEr   Z_download_json_GRAPHQL_APIjsondumps_GRAPHQL_QUERYencode_TOKENdict)selfmedia_idZ
video_uuidZ
media_uuid r*   </tmp/pip-unpacked-wheel-q5ljy6pj/yt_dlp/extractor/nytimes.py	_call_api>   s0       


zNYTimesBaseIE._call_apic                 C   s*   t |dd ddthfdthfdfd dS )Nc                 S   s   t |d S )Nurl)r   _vr*   r*   r+   <lambda>P       z3NYTimesBaseIE._extract_thumbnails.<locals>.<lambda>r-   widthheight)r-   r3   r4   default)r   r   )r(   thumbsr*   r*   r+   _extract_thumbnailsO   s    
z!NYTimesBaseIE._extract_thumbnailsc                 C   s&  g }g }i }t |ddthfD ]}|d}|d}|r|dks||krLq|| t|dpjt|}	|	dkr| j||dd	|pd
dd\}
}||
 | j||d q|	dkrqq||||dp|dt	|dt	|dt |ddt	hfddt	|ddpd |	d q||fS )N
renditions.r-   typer7   mimetypem3u8mp4Zm3u8_nativehlsF)m3u8_idr   )targetZmpdZvideoencodingZvideo_codecr3   r4   )	file_sizeZfileSize)Nvalueget_allZbitrate  )r-   	format_idvcodecr3   r4   filesizeZtbrext)
r   r'   getappendr   r   #_extract_m3u8_formats_and_subtitlesextendZ_merge_subtitlesr   )r(   video_idZcontent_media_jsonurlsformats	subtitlesr   Z	video_urlrF   rI   Z	m3u8_fmtsZ	m3u8_subsr*   r*   r+   _extract_formats_and_subtitlesV   sP    


    

  
z,NYTimesBaseIE._extract_formats_and_subtitlesc                 C   s|   |  |}| ||\}}||d|dt|dt|ddddt|dd	d
dd hf||| t|dd	S )NZpromotionalHeadlinesummaryfirstPublisheddurationrE   Zscale, bylines.renderedRepresentationc                 S   s
   t | dS )NzBy )r   xr*   r*   r+   r1      r2   z.NYTimesBaseIE._extract_video.<locals>.<lambda>promotionalMediacrops.r9   .)	r   titledescription	timestamprU   creatorrP   rQ   
thumbnails)r,   rR   rJ   r   r	   joinr   r8   )r(   r)   r   rP   rQ   r*   r*   r+   _extract_videox   s&    
   zNYTimesBaseIE._extract_videoN)__name__
__module____qualname__r   UUIDr    r&   r!   r$   r,   r8   rR   re   r*   r*   r*   r+   r      s   
""r   c                   @   sH   e Zd ZdZdgZddddddd	d
dddd	ddddgZdd ZdS )	NYTimesIEzhttps?://(?:(?:www\.)?nytimes\.com/video/(?:[^/]+/)+?|graphics8\.nytimes\.com/bcvideo/\d+(?:\.\d+)?/iframe/embed\.html\?videoId=)(?P<id>\d+)zl<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//graphics8\.nytimes\.com/bcvideo/[^/]+/iframe/embed\.html.+?)\1>zshttp://www.nytimes.com/video/opinion/100000002847155/verbatim-what-is-a-photocopier.html?playlistId=100000001150263Z a553aa344014e3723d33893d89d4defcZ100000002847155r=   z Verbatim: What Is a Photocopier?z$md5:93603dada88ddbda9395632fdc5da260it]SZ20140428zBrett Weinerz're:https?://\w+\.nyt.com/images/.+\.jpgi  	r   rI   r_   r`   ra   upload_daterb   	thumbnailrU   r-   md5	info_dictzJhttp://www.nytimes.com/video/travel/100000003550828/36-hours-in-dubai.htmlTr-   Zonly_matchingc                 C   s   |  |}| |S )N)	_match_idre   )r(   r-   rN   r*   r*   r+   _real_extract   s    
zNYTimesIE._real_extractN)rf   rg   rh   
_VALID_URLZ_EMBED_REGEX_TESTSrs   r*   r*   r*   r+   rj      s(   rj   c                   @   s   e Zd ZdZdddddddd	d
dddd
ddddddddddddd	ddddddddd d!d"dd#d
d$d%id&d'd(d)d*d+d,d-d.d/d0d1d2d3gZd4d5 Zd6d7 Zd8S )9NYTimesArticleIEzphttps?://(?:www\.)?nytimes\.com/\d{4}/\d{2}/\d{2}/(?!books|podcasts)[^/?#]+/(?:\w+/)?(?P<id>[^./?#]+)(?:\.html)?zhttp://www.nytimes.com/2015/04/14/business/owner-of-gravity-payments-a-credit-card-processor-is-setting-a-new-minimum-wage-70000-a-year.html?_r=0Z 3eb5ddb1d6f86254fe4f233826778737Z100000003628438r=   u0   One Company’s New Minimum Wage: $70,000 a Yearz$md5:89ba9ab67ca767bb92bf823d1f138433i-UZ20150414zMatthew WilliamszPatricia Cohen're:https?://\w+\.nyt.com/images/.*\.jpgg     ]@)
r   rI   r_   r`   ra   rl   uploaderrb   rm   rU   rn   zMhttps://www.nytimes.com/2023/09/29/health/mosquitoes-genetic-engineering.htmlZ 2365b3555c8aa7f4dd34ca735ad02e6aZ100000009110381Zmp3z<The Gamble: Can Genetically Modified Mosquitoes End Disease?z$md5:9ff8b47acbaf7f3ca8c732f5c815be2ei|NeZ20230929z#Stephanie Nolen, Natalija Gormalovai*  rk   zThttps://www.nytimes.com/2023/11/29/business/dealbook/kamala-harris-biden-voters.htmlZ100000009202270uU   Kamala Harris Defends Biden Policies, but Says ‘More Work’ Needed to Reach Votersz$md5:de4212a7e19bb89e4fb14210ca915f1figeZ20231129zBy The New York TimeszKatie RogersgMbhX@Zskip_downloadr<   )r-   ro   rp   paramszOhttps://www.nytimes.com/2023/12/02/business/air-traffic-controllers-safety.htmlzair-traffic-controllers-safetyzHDrunk and Asleep on the Job: Air Traffic Controllers Pushed to the Brinkz$md5:549e5a5e935bf7d048be53ba3d2c863dZ20231202zEmily Steel, Sydney Emberi` ke)r   r_   r`   rl   rb   ra      r-   rp   playlist_countzShttps://www.nytimes.com/2023/12/02/business/media/netflix-squid-game-challenge.htmlTrq   c                 C   s   t |dthfdddthfd ddd hfdthfffd	thfd
thfddd}| |d|\}}t |dthf}|s|r||dd || t |d||dS )NZsourceIdrX   .rY   rU   c                 S   s   t | ddS )NrE   rV   )r	   rZ   r*   r*   r+   r1      r2   z>NYTimesArticleIE._extract_content_from_block.<locals>.<lambda>lengthrT   ZpodcastSeries)r   rx   rU   ra   ZseriesFrC   r   ZfileUrlnone)r-   rG   r\   )rc   rP   rQ   )	r   strr   r   rR   rJ   r   rK   r8   )r(   blockdetailsrP   rQ   r-   r*   r*   r+   _extract_content_from_block   s*     z,NYTimesArticleIE._extract_content_from_blockc           	      C   s  |  |}| ||}| jd|d|dd dd d d }t|d	d
dddd f}|s^tdt| |dt|d	d
ddd
ddthfddp| ddg|t|dt	hfd
t|d| t|dd}g }|D ]}|t| || qt|dkr| j||f|S d|i|d S )Nzwindow\.__preloadedData\s*=zmedia detailsc                 S   s   |  ddS )NZ	undefinednull)replacerZ   r*   r*   r+   r1     r2   z0NYTimesArticleIE._real_extract.<locals>.<lambda>)Ztransform_sourceZinitialDatar   articleZsprinkledBodycontent.)Z	ledeMediaNc                 S   s   |d dkS )NZ
__typename)ZVideoZAudior*   r.   r*   r*   r+   r1   
  r2   z/Unable to extract any media blocks from webpagez - The New York TimesrS   textFrC   og:descriptiontwitter:descriptionrT   rW   )rX   .Zcreators.ZdisplayName)r]   Z
assetCrops.r9   .)r_   r`   ra   rb   rc   r   r   r   )rr   _download_webpageZ_search_jsonr   r   r   Z_html_extract_titler   _html_search_metar   rd   r8   rK   r   r   lenplaylist_result)	r(   r-   page_idwebpageZart_jsonblocksZcommon_infoentriesr   r*   r*   r+   rs     sr    
               zNYTimesArticleIE._real_extractN)rf   rg   rh   rt   ru   r   rs   r*   r*   r*   r+   rv      sz    Arv   c                   @   sd   e Zd ZdZdZddddddd	d
ddddddddddd	dddddddddgZdd ZdS ) NYTimesCookingIEZNYTimesCookingGuidez4https?://cooking\.nytimes\.com/guides/(?P<id>[\w-]+)z:https://cooking.nytimes.com/guides/13-how-to-cook-a-turkeyz13-how-to-cook-a-turkeyzHow to Cook a Turkeyz$md5:726cfd3f9b161bdf5c279879e8050ca0)r   r_   r`   r   r{   z@https://cooking.nytimes.com/guides/50-how-to-make-mac-and-cheeseZ 64415805fe0b8640fce6b0b9def5989aZ100000005835845r=   zHow to Make Mac and Cheesez$md5:b8f2f33ec1fb7523b21367147c9594f1i`ZZ20180405gQ#@zAlison Romanrw   )	r   rI   r_   r`   ra   rl   rU   rb   rm   rn   z9https://cooking.nytimes.com/guides/20-how-to-frost-a-cakez20-how-to-frost-a-cakezHow to Frost a Cakez$md5:a31fe3b98a8ce7b98aae097730c269cd   )r-   ro   rp   r|   c              	      s     |} ||} ddg|} ddg|} d|d}ttd|dthd	f}|r||   fd
d|D |||S  	||| jd|dd ddS )Nzog:titleztwitter:titler   r   z#data-video-player-id="(\d+)"></div>z
lead videoz
video-item.zdata-video-idc                    s   g | ]}  |qS r*   )re   ).0r)   r(   r*   r+   
<listcomp>Z  s     z2NYTimesCookingIE._real_extract.<locals>.<listcomp>z*<span itemprop="author">([^<]+)</span></p>authorr5   )r_   r`   rb   )
rr   r   r   Z_search_regexr   r
   r   rK   r   re   )r(   r-   r   r   r_   r`   Zlead_video_idZ	media_idsr*   r   r+   rs   L  s>    
   

      zNYTimesCookingIE._real_extractN)rf   rg   rh   ZIE_NAMErt   ru   rs   r*   r*   r*   r+   r   '  s@   
"r   c                   @   sp   e Zd ZdZdddddddd	d
dddddddddddddddddddddddddddgZdd  Zd!S )"NYTimesCookingRecipeIEz2https?://cooking\.nytimes\.com/recipes/(?P<id>\d+)z?https://cooking.nytimes.com/recipes/1017817-cranberry-curd-tartZ 579e83bbe8e61e9de67f80edba8a78a8Z1017817r=   zCranberry Curd Tartz$md5:ad77a3fc321db636256d4343c5742152iKVZ20151118zDavid Tanisrw   )r   rI   r_   r`   ra   rl   rb   rm   rn   zKhttps://cooking.nytimes.com/recipes/1024781-neapolitan-checkerboard-cookiesZ 58df35998241dcf0620e99e646331b42Z1024781zNeapolitan Checkerboard Cookiesz$md5:ba12394c585ababea951cb6d2fcc6631igeZ20231130zSue Liz:https://cooking.nytimes.com/recipes/1019516-overnight-oatsZ 2fe7965a3adc899913b8e25ada360823Z1019516i +\z$md5:8856ce10239161bd2596ac335b9f9bfbZ20190102zOvernight OatszGenevieve Ko)r   rI   ra   r`   rl   r_   rb   rm   c                 C   s   |  |}| ||}| ||d d d }| j|d |ddd\}}t|dthfd	thfd
thfdthfddthfd||dd t|ddddt	hfD dS )NpropsZ	pagePropsZrecipeZvideoSrcr=   r>   )r?   r   r_   ZtopnoteZpublishedAtZcontentAttributionZ
cardByline)r   r_   r`   ra   rb   c                 S   s   g | ]}d |iqS )r-   r*   )r   Z	thumb_urlr*   r*   r+   r     s     z8NYTimesCookingRecipeIE._real_extract.<locals>.<listcomp>imager^   .)rP   rQ   rc   )
rr   r   Z_search_nextjs_datarL   r   r   r   r   r   r   )r(   r-   r   r   Zrecipe_datarP   rQ   r*   r*   r+   rs     s.    
   

 z$NYTimesCookingRecipeIE._real_extractN)rf   rg   rh   rt   ru   rs   r*   r*   r*   r+   r   e  sN   )r   )r"   r   commonr   utilsr   r   r   r   r	   r
   r   r   r   r   r   r   r   r   r   r   rj   rv   r   r   r*   r*   r*   r+   <module>   s   Ds >