U
    gQ                     @  s   d dl mZ d dlZd dlZd dlZddlmZ ddlm	Z	m
Z
 ddlmZ ddlmZ dd	lmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#m$Z$m%Z%m&Z&m'Z' G d
d deZ(G dd deZ)dS )    )annotationsN   )InfoExtractor)YoutubeBaseInfoExtractor	YoutubeIE   )HEADRequest)	HTTPError)KNOWN_EXTENSIONSExtractorErrorbug_reports_message
clean_htmldict_getextract_attributesget_element_by_idint_or_nonejoin_nonempty
js_to_jsonmerge_dictsmimetype2ext
orderedSetparse_durationparse_qsstr_or_none
str_to_inttraverse_objtry_getunified_strdateunified_timestampurl_or_noneurlhandle_detect_extvariadicc                   @  s  e Zd ZdZdZdZdddddd	d
dddgddddddddddddddddddd
dd d!d"d#d$d%d&d'dd(d&d)dd*d+d,d-d'd.d/d0d1d-d2d3dd4dd5d6d7d8d9d8d:gd;d<d=d>d?	dd@dAdBdCd9dDdEdFdGdHdIdCd:gd=dJdKdLdMddNdOdPdQdRdSdTdUdVdWdXdYdRdZd[d\d]d^dTdUd_d`dadbdcdddedfgdgdhdidjddkdldmdkdndodpdqddrdsdsdtdudodvdwddxdydzdyd{dogd|g
Zed}d~ Zdd Z	dS )ArchiveOrgIEzarchive.orgzarchive.org video and audiozKhttps?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^?#]+)(?:[?].*)?$zMhttp://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellectZ 8af1d4cf447933ed3c7f4871162602dbz2XD300-23_68HighlightsAResearchCntAugHumanIntellectZogvz01968 Demo - FJCC Conference Presentation Reel #1z$md5:da45c349df039f1cc8075268eb1b5c25Z19681210iKZ20100315zSRI Internationalzlaura@archive.orgz(re:https://archive\.org/download/.*\.jpgz6XD300-23_68HighlightsAResearchCntAugHumanIntellect.cdrz2XD300-23 68HighlightsAResearchCntAugHumanIntellect)idexttitledescriptionrelease_date	timestampupload_datecreatorsuploader	thumbnail
display_idtrack)urlmd5	info_dictz$https://archive.org/details/Cops1922Z 0869000b4ce265e8ca62738b336b268aZCops1922mp4zBuster Keaton's "Cops" (1922)z$md5:cd6f9910c35aedd5fc237dbc3957e2cazyorkmba99@hotmail.comiRZ20131222zCops-v2.mp4gp=
@)
r#   r$   r%   r&   r+   r(   r)   r-   r,   durationzKhttp://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellectTr/   Zonly_matchingz(https://archive.org/details/Election_AdsZ eec5cddebd4793c6a653b69c3b11f2e6z;Election_Ads/Commercial-JFK1960ElectionAdCampaignJingle.mpgz.Commercial-JFK1960ElectionAdCampaignJingle.mpgZmpgg(\M@)r#   r%   r$   r,   r3   r-   zYhttps://archive.org/details/Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpgZ ea1eed8234e7d4165f38c8c769edef38z=Election_Ads/Commercial-Nixon1960ElectionAdToughonDefense.mpgz0Commercial-Nixon1960ElectionAdToughonDefense.mpgiMGzmikedavisstripmaster@yahoo.comzO1960 Presidential Campaign Election Commercials John F Kennedy, Richard M NixonZ20080315gzGM@z1http://creativecommons.org/licenses/publicdomain/)r#   r%   r$   r(   r+   r&   r)   r-   r3   licenser,   zGhttps://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16Z 7d07ffb42aba6537c28e053efa4b54c9zCgd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t01.flacZTurningZflaczGrateful Deadzgd1977-05-08d01t01.flacr   z-1977-05-08 - Barton Hall - Cornell UniversitygfffffC@)	r#   r%   r$   r.   r*   r-   track_numberalbumr3   z_https://archive.org/details/gd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t07.flacZ a07cd8c6ab4ee1560f8a0021717130f3zCgd1977-05-08.shure57.stevenson.29303.flac16/gd1977-05-08d01t07.flacZDealiȁGzmvernon54@yahoo.comz$md5:6c921464414814720c6593810a5c7e3dZ20080319z Barton Hall - Cornell Universityg{Gj{@Z19770508zgd1977-05-08d01t07.flac   )r#   r%   r$   r(   r+   r&   r)   locationr3   r.   r*   r7   r'   r-   r6   z]https://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melikZ 7cb019baa9b332e82ea7c10403acd180zblp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/01.01. Bells Of Rostov.mp3zBells Of RostovZmp3)r#   r%   r$   Z
restricted)r/   r0   r1   skipzhttps://archive.org/details/lp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/02.02.+Song+And+Chorus+In+The+Polovetsian+Camp+From+%22Prince+Igor%22+(Act+2%2C+Scene+1).mp3Z 1d0aabe03edca83ca58d9ed3b493a3c3zlp_the-music-of-russia_various-artists-a-askaryan-alexander-melik/disc1/02.02. Song And Chorus In The Polovetsian Camp From "Prince Igor" (Act 2, Scene 1).mp3zKSong And Chorus In The Polovetsian Camp From "Prince Igor" (Act 2, Scene 1)i{&]z%associate-joygen-odiongan@archive.orgz$md5:012b2d668ae753be36896f343d12a236Z20190928)r#   r%   r$   r(   r+   r&   r)   z7https://archive.org/details/irelandthemakingofarepublicZirelandthemakingofarepublicz!Ireland: The Making of a RepublicZ20160610z$md5:f70956a156645a658a0dc9513d9e78b7zdimitrios@archive.orgz British Broadcasting CorporationzTime-Life FilmsiC4[W)r#   r%   r)   r&   r+   r*   r(   Z 0b211261b26590d49df968f71b90690dzCirelandthemakingofarepublic/irelandthemakingofarepublicreel1_01.movz'irelandthemakingofarepublicreel1_01.movgQN`@zhttps://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_01_000117.jpg)r#   r$   r%   r3   r,   r-   )r0   r1   Z 67335ee3b23a0da930841981c1e79b02zCirelandthemakingofarepublic/irelandthemakingofarepublicreel1_02.movgQ̕@z'irelandthemakingofarepublicreel1_02.movzhttps://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel1_02_001374.jpg)r#   r$   r3   r%   r-   r,   Z e470e86787893603f4a341a16c281eb5z@irelandthemakingofarepublic/irelandthemakingofarepublicreel2.movgHz
@z$irelandthemakingofarepublicreel2.movzhttps://archive.org/download/irelandthemakingofarepublic/irelandthemakingofarepublic.thumbs/irelandthemakingofarepublicreel2_001554.jpg)r#   r$   r3   r%   r,   r-   )r/   r1   playlistc                 C  s"   t d| d }tt|d S )Na   (?xs)
            <input
            (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
            \s+class=['"]?js-play8-playlist['"]?
            (?:\s+[a-zA-Z0-9:._-]+(?:=[a-zA-Z0-9:._-]*|="[^"]*"|='[^']*'|))*?
            \s*/>
        r   value)refindalljsonloadsr   )webpageelement rC   ?/tmp/pip-unpacked-wheel-q5ljy6pj/yt_dlp/extractor/archiveorg.py_playlist_data   s    zArchiveOrgIE._playlist_datac                 C  sR  t j| |}|d\}}}| d| |}| |}i }|D ]}	|r\|	d |kr\qFg g |	d|	di d||	d < |	dg D ]8}
|
d d	krqd
d|
d d i||	d  |
d < qqF| 	d| |}|d }|d }||d t
|dt|ddgt|dthdd hf|dt|dtt|ddgd| |dt|dd}|d D ]}|d  |kr*t||d   |d |d   |dp|d  |d  t
|dt|dthd!d hft|d"t|d#|d$t|d%t|dd&
||d  < ||d   }n&t|d'td(|krl||d'  }nql|d)d*kr|d+ |d  d,| d |d   t|d-t|d-t|d.d/ |d  d0\}}}|sd }t| d1d2}|tkrl|d3r|rl|d4 d,| d t j|d   |d)t|d-t|d5t|d.d6|d7d'kr^d8nd9|d7d: ql| D ]}d;|d<< q|t|d=krtt| }|rt||}n
t||}nd>|d?< t| |d@< |dArNg |dB< |dA D ]P}|dB |dC|dDt|dEdFdG |dH t|dIdJdK q|S )LN/zhttps://archive.org/embed/origartistr%   )formats
thumbnailsrH   r.   	subtitlesZtrackskindrK   r/   zhttps://archive.org/filelabelzhttp://archive.org/metadata/metadata
identifierr&   r+   Zaddercreatorc                 S  s   | d ot | S Nr   listxrC   rC   rD   <lambda>      z,ArchiveOrgIE._real_extract.<locals>.<lambda>Z
licenseurldateZ
publicdateZ	addeddatezhttps://archive.org/details/Zvenueyear)r#   r%   r&   r+   r*   r5   r'   r(   Zwebpage_urlr9   release_yearfilesnamec                 S  s   | d ot | S rR   rS   rU   rC   rC   rD   rW     rX   lengthr.   r7   Zdisc)
r#   r%   r-   r&   r*   r3   r6   r7   Z
discnumberr[   original)expected_typeformatZ	ThumbnailrJ   zhttps://archive.org/download/widthsize)r#   r/   rb   heightfilesize.zhttps://archive.orgzlogged-in-sigprivaterI   rd   httpssourcer   )r/   ra   rb   rd   re   protocolZsource_preferenceZformat_note)ri   Z_format_sort_fieldsr   r;   _typeentriesZreviewscommentsZ	review_idZreviewerZreviewtitle z

Z
reviewbodyZ
createdateroot)r#   authortextr(   parent) urllibparseunquote_plusZ	_match_id	partition_download_webpagerE   getlstrip_download_jsonr   r   r   r!   r   r   r   r   r   strappend
rpartitionboolZ_get_cookiesr
   quotevalueslennextiterrT   r   )selfr/   video_idrP   _Zentry_idZ
embed_pager;   rm   pr.   rO   minfofentryhas_ext	extensionZis_logged_inZ
only_videoZreviewrC   rC   rD   _real_extract   s    
 zArchiveOrgIE._real_extractN)
__name__
__module____qualname__IE_NAMEIE_DESC
_VALID_URL_TESTSstaticmethodrE   r   rC   rC   rC   rD   r"   '   s8  	 (
r"   c                   @  s  e Zd ZdZdZdZdddddd	d
ddd
dddddddddddddddddddddddd d!d"d#d$d%dd$d&
dd'd(d)d*d+d,d-d.d/d0d1dd/d2dd3d4d)d5d6d7d8d9d:d;d<d=dd>dd?d@ddAdBdCdDdEdFdGdHddId2ddJdKddLdMdCdNdOdFdGdHddId2dPgdQdRdSd)dTdUddVdWddXdYdZd[d\d]dd^d_
dd`daddbdcdddedfdgdhdhdiddjddkdld)dmdndodpdpddqdrds
ddtdud)dvdwdxdydwddzd{d|
dd}d~dddpdpdqddddd
dddddddddddd	dddddddddddddddddddddddddddddddddddddddddd
ddddddddddddd
ddddddddddddddddddddddddddddgZejZdej	 dZ	dgZ
ee
dddń dD ZdZdZdZdddddΜddЄZdd҄ ZddԄ Zddք Zdd؄ Zddڄ ZdS )YoutubeWebArchiveIEzweb.archive:youtubez9web.archive.org saved youtube videos, "ytarchive:" prefixaE  (?x)(?:(?P<prefix>ytarchive:)|
            (?:https?://)?web\.archive\.org/
            (?:web/)?(?:(?P<date>[0-9]{14})?[0-9A-Za-z_*]*/)?  # /web and the version index is optional
            (?:https?(?::|%3[Aa])//)?(?:
                (?:\w+\.)?youtube\.com(?::(?:80|443))?/watch(?:\.php)?(?:\?|%3[fF])(?:[^\#]+(?:&|%26))?v(?:=|%3[dD])  # Youtube URL
                |(?:wayback-fakeurl\.archive\.org/yt/)  # Or the internal fake url
            )
        )(?P<id>[0-9A-Za-z_-]{11})
        (?(prefix)
            (?::(?P<date2>[0-9]{14}))?$|
            (?:%26|[#&]|$)
        )zVhttps://web.archive.org/web/20150415002341/https://www.youtube.com/watch?v=aYAGB11YrSsZaYAGB11YrSsZwebmzTeam Fortress 2 - Sandviches!z$md5:4984c0f9a07f349fc5d8e82ab7af4eafZ20110926ZZeurelzUCukCyHaD-bK3in_pKpfH9Eg    z#https://www.youtube.com/user/Zeurelzre:https?://.*\.(jpg|webp)z8https://www.youtube.com/channel/UCukCyHaD-bK3in_pKpfH9Eg)r#   r$   r%   r&   r)   r+   
channel_idr3   uploader_iduploader_urlr,   channel_url)r/   r1   zQhttps://web.archive.org/web/2oe/http://wayback-fakeurl.archive.org/yt/97t7Xj_iBv0Z97t7Xj_iBv0r2   z!Why Machines That Bend Are Betterz$md5:00404df2c632d16a674ff8df1ecfbb6cZ20190312Z
VeritasiumzUCHnyfMqiRRG1u-2MsSQLbXAi  Z1veritasiumz(https://www.youtube.com/user/1veritasiumz8https://www.youtube.com/channel/UCHnyfMqiRRG1u-2MsSQLbXAzahttps://web.archive.org/web/20120712231619/http://www.youtube.com/watch?v=AkhihxRKcrs&gl=US&hl=enZAkhihxRKcrsz6Limited Run: Mondo's Modern Classic 1 of 3 (SDCC 2012)Z20120712i  z$md5:ff4de6a7980cb65d951c2f6966a4f2f3Z	machinimaz&https://www.youtube.com/user/machinima)
r#   r$   r%   r)   r3   r&   r   r   r,   r+   zUhttps://web.archive.org/web/20081211103536/http://www.youtube.com/watch?v=jNQXAC9IVRwZjNQXAC9IVRwZflvzMe at the zooZ20050423ZUC4QobU6STFB0P71PMvOGN5A   z$md5:10436b12e07ac43ff8df65287a56efb4Zjawedz"https://www.youtube.com/user/jawedz8https://www.youtube.com/channel/UC4QobU6STFB0P71PMvOGN5A)r#   r$   r%   r)   r   r3   r&   r   r   r   r,   r+   zUhttps://web.archive.org/web/20110712231407/http://www.youtube.com/watch?v=lTx3G6h2xyAZlTx3G6h2xyAz"Madeon - Pop Culture (live mashup)Z20110711ZMadeonZUCqMDNf3Pn5L7pcNkuSEeO3w   z$md5:f7535343b6eda34a314eff8b85444680Z	itsmadeonz&https://www.youtube.com/user/itsmadeonz8https://www.youtube.com/channel/UCqMDNf3Pn5L7pcNkuSEeO3w)r#   r$   r%   r)   r+   r   r3   r&   r   r   r   r,   zChttps://web.archive.org/https://www.youtube.com/watch?v=1JYutPM8O6EZ1JYutPM8O6Ez3Fake Teen Doctor Strikes AGAIN! - Weekly Weird NewsZ20160218ZUCdIaNUarhzLSXGoItz7BHVAi  z$md5:21032bae736421e89c2edf36d1936947ZMachinimaETCz)https://www.youtube.com/user/MachinimaETCz8https://www.youtube.com/channel/UCdIaNUarhzLSXGoItz7BHVAzETC NewszVhttps://web.archive.org/web/20180803221945/https://www.youtube.com/watch?v=6FPhZJGvf4EZ6FPhZJGvf4Ez1WTF: Video Games Still Launch BROKEN?! - T.U.G.S.Z20160219i  z$md5:a1dbf12d9a3bd7cb4c5e33b27d77ffe7z=unable to download capture webpage \(it may not be archived\))r/   r1   Zexpected_warningszPhttp://web.archive.org/web/20070302011044/http://youtube.com/watch?v=-06-KB9XTzgz-06-KB9XTzgzNew Coin Hack!! 100% Safe!!)r#   r$   r%   z;web.archive.org/https://www.youtube.com/watch?v=dWW7qP423y8ZdWW7qP423y8zIt's Bootleg AirPods Time.Z20211021ZUC7Jwj9fkrf1adN4fMmTkpugz8https://www.youtube.com/channel/UC7Jwj9fkrf1adN4fMmTkpugi*  z$md5:7b567f898d8237b256f36c1a07d6d7bcZDankPods)
r#   r$   r%   r)   r   r   r3   r&   r,   r+   zXhttps://web.archive.org/web/20200827003909if_/http://www.youtube.com/watch?v=6Dh-RL__uN4z6Dh-RL__uN4zbitch lasagnaZ20181005zUC-lHJZR3Gqxm24_Vd_AJ5Ywz8https://www.youtube.com/channel/UC-lHJZR3Gqxm24_Vd_AJ5Yw   z$md5:2dbe4051feeff2dab5f41f82bb6d11d0Z	PewDiePiez&https://www.youtube.com/user/PewDiePie)r#   r$   r%   r)   r   r   r3   r&   r+   r   r   r,   zIhttps://web.archive.org/web/0/https://www.youtube.com/watch?v=8XeW5ilk-9Yz8XeW5ilk-9Yz(Story of Stuff, The Critique Part 4 of 4i  z$md5:28157da06f2c5e94c97f7f3072509972ZHowTheWorldWorksz-https://www.youtube.com/user/HowTheWorldWorksZ20090520)
r#   r$   r%   r3   r&   r+   r   r,   r   r)   zUhttps://web.archive.org/web/20110126141719/http://www.youtube.com/watch?v=Q_yjX80U7YcZQ_yjX80U7Ycz5Spray Paint Art by Clay Butler: Purple Fantasy ForestZclaybutlermusicz$md5:4595264559e3d0a0ceb3f011f6334543Z20090803   z,https://www.youtube.com/user/claybutlermusic)
r#   r$   r%   r   r&   r)   r+   r,   r3   r   zIhttps://web.archive.org/web/0/https://www.youtube.com/watch?v=c5uJgG05xUYZc5uJgG05xUYz(Story of Stuff, The Critique Part 1 of 4Z20090513z$md5:4ca77d79538064e41e4cc464e93f44f0i  )
r#   r$   r%   r   r+   r   r)   r&   r,   r3   zUhttps://web.archive.org/web/20120607174520/http://www.youtube.com/watch?v=xWTLLl-dQaAzxWTLLl-dQaAz,Black Nerd eHarmony Video Bio Parody (SPOOF)z,https://www.youtube.com/user/BlackNerdComedyz$md5:e25f0133aaf9e6793fb81c18021d193eZBlackNerdComedy   )	r#   r$   r%   r   r&   r   r+   r3   r,   zIhttps://web.archive.org/web/*/https://www.youtube.com/watch?v=9eO1aasHyTMZ9eO1aasHyTMz	Polar-oidz Cameras and bears are dangerous!z&https://www.youtube.com/user/punkybirdZ	punkybird   ZUC62R2cBezNBOqxSerfb1nMQz8https://www.youtube.com/channel/UC62R2cBezNBOqxSerfb1nMQZ20060428)r#   r$   r%   r&   r   r   r3   r   r   r)   r+   zbhttps://web.archive.org/web/20200416034815/https://www.youtube.com/watch?v=Cf7vS8jc7dY&gl=US&hl=enZCf7vS8jc7dYzDA Dramatic Pool Story (by Jamie Spicer-Lewis) - Game Grumps Animated@   Z20200408Z
GameGrumpsz8https://www.youtube.com/channel/UC9CuvdOVfMPvKCiwdGKL3cQZUC9CuvdOVfMPvKCiwdGKL3cQz$md5:c625bb3c02c4f5fb4205971e468fa341z'https://www.youtube.com/user/GameGrumps)r#   r$   r%   r3   r)   r   r+   r   r   r,   r&   r   z$ytarchive:kbh4T_b4Ixw:20160307085057Zkbh4T_b4IxwzXShovel Knight OST - Strike the Earth! Plains of Passage 16 bit SNES style remake / remixz8https://www.youtube.com/channel/UCnTaGvsHmMy792DWeT6HbGAzNelward music   z$md5:804b4a9ce37b050a5fefdbb23aeba54dZ20150503ZUCnTaGvsHmMy792DWeT6HbGA)
r#   r$   r%   r   r+   r3   r&   r,   r)   r   zIhttps://web.archive.org/web/0/https://www.youtube.com/watch?v=SOm7mPoPskUZSOm7mPoPskUz Boyfriend - Justin Bieber Parodyz.https://www.youtube.com/user/thecomputernerd01Zthecomputernerd01z$md5:dd7fa635519c2a5b4d566beaecad7491   Z20120407)
r#   r$   r%   r   r+   r,   r&   r3   r)   r   zFhttps://web.archive.org/web/http://www.youtube.com/watch?v=kH-G_aIBlFwTr4   zXhttps://web.archive.org/web/20050214000000_if/http://www.youtube.com/watch?v=0altSZ96U4Mzchttps://web.archive.org/web/20210530071008/https://www.youtube.com/watch?v=lHJTf93HL1s&spfreload=10zohttps://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fgl%3DUS%26v%3DAkhihxRKcrs%26hl%3Denzohttps://web.archive.org/web/20120712231619/http%3A//www.youtube.com/watch%3Fv%3DAkhihxRKcrs%26gl%3DUS%26hl%3Denznhttps://web.archive.org/web/20060527081937/http://www.youtube.com:80/watch.php?v=ELTFsLT73fA&amp;search=soccerzEhttps://web.archive.org/http://www.youtube.com:80/watch?v=-05VVye-ffgz$ytarchive:BaW_jenozKc:20050214000000zytarchive:BaW_jenozKczq(?x:
        (?:window\s*\[\s*["\']ytInitialPlayerResponse["\']\s*\]|ytInitialPlayerResponse)\s*=[(\s]*|
        z
    )zi.ytimg.comimg.youtube.comc                 C  s0   g | ](}t d dD ]}| |p"d dqqS )   	   ro   z
.ytimg.com)r   )range).0cnrC   rC   rD   
<listcomp>  s       zYoutubeWebArchiveIE.<listcomp>)isz"https://web.archive.org/web/%sif_/l   Hl   @kyJNFzlist | Nonezdict | Nonefilterscollapsequeryc                   s   |ddddg|pg  |pg d|p$i }| j d||p6d||d t trpt d	krp fd
d dd  D S t trt dkr| dt   d S )Nr?   z"original,mimetype,length,timestampi  zstatuscode:200)r/   outputfllimitfilterr   z&https://web.archive.org/cdx/search/cdxzDownloading CDX API JSON)r   fatalr   c                   s   g | ]}t t d  |qS )r   )dictzip)r   vresrC   rD   r     s     z5YoutubeWebArchiveIE._call_cdx_api.<locals>.<listcomp>r   r   z$Error while parsing CDX API response)r{   
isinstancerT   r   Zreport_warningr   )r   Zitem_idr/   r   r   r   noter   rC   r   rD   _call_cdx_api  s(    	   z!YoutubeWebArchiveIE._call_cdx_apic                 C  s    | j |dd}| jd|dddS )Nro   defaultz/(?:YouTube\s*-\s*(.*)$)|(?:(.*)\s*-\s*YouTube$)r%   )Z_html_extract_title_html_search_regex)r   rA   Z
page_titlerC   rC   rD   _extract_webpage_title  s      z*YoutubeWebArchiveIE._extract_webpage_titlec                   s|  r fddndd } j  jd|i d} j  jd|i d}i }tdD ]$}| j||dd	td
dpvi  qV j dd|d dp|dpi } j dd|i d}	|	rt	|ds|	|d< |s jt	|dpd|dd}t	|dt
di d}
t	|dt
di d}t	|dt
di d}|dpht|dpht|
dpht	|dph ph|dddg} fdd}d } jd!| d"d#| gd$d d}t|d%}||d&p||d&p|d'}|rd(| nd } jd)d*d+d,gd-d dpJ jd.d/gtd0d-d dpJ jd1td2d-d dpJt	|d3pJ|d4}t|d5p|d6p|d5p jd7d8d d9d:p||d;p||d;pt	|d<}|rd=| nd }t|d>p|d>pt	|d?dd@pt|dA}|dBp,t|dCp,ttdDp,|dCdEdFg}tt|dGp`|dHdIgp` jdJdKdLgdMd d}|||||||||dN	S )ONc                   s    j | d dS )Nr   )Z_html_search_metarU   r   rA   rC   rD   rW     rX   z7YoutubeWebArchiveIE._extract_metadata.<locals>.<lambda>c                 S  s   d S )NrC   rU   rC   rC   rD   rW     rX   zinitial player responser   zinitial dataz2yt\.setConfig\(\s*(?P<json>{\s*(?s:.+?)\s*})\s*\);FTro   )r   Zignore_extraZtransform_sourceerrnotez3(?:yt\.playerConfig|ytplayer\.config|swfConfig)\s*=zplayer configZPLAYER_CONFIGzswfArgs\s*=z
swf config)argsr   )r   player_responsez{}r   )contentsZtwoColumnWatchNextResultsresultsr   r   .ZvideoPrimaryInfoRenderer)r`   get_allr   ZvideoDetails)ZmicroformatZplayerMicroformatRendererr%   )r   r%   zog:titleztwitter:titlec                   s$    j d| d| pd| dd dS )Nz(?:z)/([^/#&?]+)ro   z idr   )_search_regex)r/   type_r   rC   rD   id_from_url  s    
   z:YoutubeWebArchiveIE._extract_metadata.<locals>.id_from_urlzFhref="[^"]*(?P<url>https?://www\.youtube\.com/(?:user|channel)/[^"]+)"z;<(?:link\s*itemprop=\"url\"|a\s*id=\"watch-username\").*?\b>zP<div\s*id=\"(?:watch-channel-stats|watch-headline-user-info)\"[^>]*>\s*<a[^>]*\bzuploader or channel urlZownerProfileUrluserZVIDEO_USERNAMEzhttps://www.youtube.com/user/z9<a\s*id="watch-username"[^>]*>\s*<strong>([^<]+)</strong>z$var\s*watchUsername\s*=\s*\'(.+?)\';zA<div\s*\bid=\"watch-channel-stats"[^>]*>\s*<a[^>]*>\s*(.+?)\s*</az2<a\s*id="watch-userbanner"[^>]*title="\s*(.+?)\s*"r+   z=(?s)<div\s*class="yt-user-info".*?<a[^>]*[^>]*>\s*(.*?)\s*</az+(?s)<a[^>]*yt-user-name[^>]*>\s*(.*?)\s*</azwatch7-user-headerz6<button\s*href="/user/[^>]*>\s*<span[^>]*>\s*(.+?)\s*<zwatch-headline-user-info)r   rQ   rq   Z	channelIdZexternalChannelIdz6data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1z
channel idr#   )r   groupZchannel)r   Zucidz https://www.youtube.com/channel/ZlengthSeconds)r   )Zlength_secondsl)r   r3   ZshortDescriptionr&   zeow-descriptionzog:descriptionztwitter:description)
uploadDateZpublishDater   ZdatePublishedz&(?s)id="eow-date.*?>\s*(.*?)\s*</span>z(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']zSclass\s*=\s*"(?:watch-video-date|watch-video-added post-date)"[^>]*>\s*([^<]+?)\s*<zupload date)	r%   r&   r)   r+   r   r   r3   r   r   )Z_search_json_YT_INITIAL_PLAYER_RESPONSE_RE_YT_INITIAL_DATA_REr=   r>   updateZ_parse_jsonr   ry   r   r   r   Z	_get_textr   r   r   r   r   r   r   r   r   r   r   )r   r   rA   Zsearch_metar   Zinitial_dataZytcfgjZplayer_configZswf_argsZinitial_data_videoZvideo_detailsZmicroformatsZvideo_titler   Z_CHANNEL_URL_HREF_REZuploader_or_channel_urlZowner_profile_urlr   r   r+   r   r   r3   r&   r)   rC   r   rD   _extract_metadata  sb          "                


      
     


  
z%YoutubeWebArchiveIE._extract_metadatac                   s   d  dkfddr$ jn jD }g }|D ]F} j|dgdgddid	}|s\q6| fd
d|D  s6 q~q6 | |S )NrJ   	check_allc                   s:   g | ]2} rd ndD ] }dj |dkr(dnd|dqqS ))jpgwebp)r   z#http://{server}/vi{webp}/{video_id}r   Z_webpro   )r   r   server)ra   )r   r   r$   )try_allr   rC   rD   r   K  s       z;YoutubeWebArchiveIE._extract_thumbnails.<locals>.<listcomp>zmimetype:image/(?:webp|jpeg)Zurlkey	matchTypeprefixr   c                 3  sN   | ]F} j t|d p j |d t|dt|ddV  qdS )r(   r_   r^   )r/   re   Z
preferenceN)_WAYBACK_BASE_URLr   ry   _OLDEST_CAPTURE_DATE)r   Zthumbnail_dictr   rC   rD   	<genexpr>V  s
   "z:YoutubeWebArchiveIE._extract_thumbnails.<locals>.<genexpr>)_configuration_arg_YT_ALL_THUMB_SERVERS_YT_DEFAULT_THUMB_SERVERSr   extendZ_remove_duplicate_formats)r   r   Zthumbnail_base_urlsrJ   r/   responserC   )r   r   r   rD   _extract_thumbnailsI  s,       
z'YoutubeWebArchiveIE._extract_thumbnailsc                 C  s   g }| j |d| dgddgddidp*g }tdd	 |D }d
d |D }|r^||d  || |rz||d  d| dkr|||  || j| jg ttd |S )Nz https://www.youtube.com/watch?v=zmimetype:text/htmlztimestamp:6digestr   r   r   c                 s  s*   | ]"}t |d  dk	rt |d  V  qdS )r(   N)r   )r   rrC   rC   rD   r   h  s      z9YoutubeWebArchiveIE._get_capture_dates.<locals>.<genexpr>c                 S  s   g | ]}|d kr|qS )l   @y6/}IrC   )r   rV   rC   rC   rD   r   l  s      z:YoutubeWebArchiveIE._get_capture_dates.<locals>.<listcomp>r   capturesr   )	r   sortedr}   r   r   r   _NEWEST_CAPTURE_DATEr   r   )r   r   url_datecapture_datesr   Zall_capturesZmodern_capturesrC   rC   rD   _get_capture_datesb  s*       
z&YoutubeWebArchiveIE._get_capture_datesc                 C  s`  |  |ddd\}}}|p |}d }| jdd}|D ]v}z | jtd| |ddd	}W q6 tk
r } z2t|jtr|jj	d
kr| j
ddd n||_W 5 d }~X Y q6X q6|jr| j
|jd|d | |t|}	| dt|	ddi  d|i}
|	D ]`}| j| jd ||f |dddd}| ||p.d}|drt|
|}
d| dkr q\q| ||
d< |rHtj|j}t|}|t|jdd}t|dd }|r|tjkr| tj|  | d|i n@t|dd }t!|pt"|pt!|jd }| d!|i |g|
d"< |
d#sHt#t|d$d |
d#< |
ds\||
d< |
S )%Nr#   rY   Zdate2Fr   zGhttps://web.archive.org/web/2oe_/http://wayback-fakeurl.archive.org/yt/z Fetching archived video file urlT)r   Zexpected_statusi  ziThe requested video is not archived, indexed, or there is an issue with web.archive.org (try again later))expected)r   r   zCaptures to try: delimz, z!http://www.youtube.com/watch?v=%sz;unable to download capture webpage (it may not be archived)zDownloading capture webpage)r   r   r   r   ro   r%   r   r   rJ   zx-archive-orig-content-length)r/   re   c                 S  s   | d d S )Nitagr   rC   rU   rC   rC   rD   rW     rX   z3YoutubeWebArchiveIE._real_extract.<locals>.<lambda>Z	format_idc                 S  s   | d d S )Nmimer   rC   rU   rC   rC   rD   rW     rX   zx-archive-guessed-content-typer$   rI   r3   c                 S  s   | d d S )NZdurr   rC   rU   rC   rC   rD   rW     rX   )$Z_match_valid_urlr   ZRetryManagerZ_request_webpager   r   r   causer	   statusZraise_no_formatserrorr   r   Zwrite_debugr   rx   r   r   ry   r   r   r   rt   ru   unquoter/   r   headersr   r   _formatsr   r   r    r   )r   r/   r   r   Z
url_date_2ZurlhZretry_managerretryer   r   capturerA   Zcurrent_infoZvideo_file_url_qsfmtr   r   r$   rC   rC   rD   r   z  sv      
   



z!YoutubeWebArchiveIE._real_extract)r   )NNNNF)r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   rC   rC   rC   rD   r   Y  sT      8 r   )*
__future__r   r?   r=   urllib.parsert   commonr   Zyoutuber   r   Z
networkingr   Znetworking.exceptionsr	   utilsr
   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r   r    r!   r"   r   rC   rC   rC   rD   <module>   s   h  4