a ʿd@sdZddlZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddlZddlZddlZddlmZddlZddlmZmZmZmZmZmZmZmZmZmZm Z m!Z!m"Z"m#Z#ddl$m%Z%ddl&m'Z'ddl(m)Z)ddl*m+Z+dd l,m-Z-e.d Z/e.d ej0Z1e.d Z2e.d ej0j3Z4d5Z6gdZ7dZ8dZ9e9j:dj:ej;edZddZ?dBddZ@dCddZAdDddZBdedfd d!ZCd"d#ZDe.d$ej0ZEeDd%d&ZFGd'd(d(ZGGd)d*d*eGZHGd+d,d,eZIe.d-jJZKd.d/ZLd0d1ZMdEd2d3ZNd4d5ZOGd6d7d7ZPGd8d9d9e jQZRejSjTfd:d;ZUdd?ZWd@dAZXdS)Fz$PyPI and direct package downloading.Nwraps) CHECKOUT_DIST Distribution BINARY_DISTnormalize_path SOURCE_DIST Environmentfind_distributions safe_name safe_version to_filename Requirement DEVELOP_DISTEGG_DIST parse_version)logDistutilsError) translate)Wheelunique_everseenz^egg=([-A-Za-z0-9_.+!]+)$zhref\s*=\s*['"]?([^'"> ]+)z([^<]+)\n\s+\(md5\)z([-+.a-z0-9]{2,}):z.tar.gz .tar.bz2 .tar .zip .tgz) PackageIndexdistros_for_urlparse_bdist_wininstinterpret_distro_namezz(interpret_distro_name..Nr3) py_versionrArP)r7anyrangerNisdigitrjoin)rJrOrFrZrArPr9rVr#r#r$rs rcstfdd}|S)zs Wrap a function returning an iterable such that the resulting iterable only ever yields unique items. cst|i|SNr)argskwargsfuncr#r$wrapperszunique_values..wrapperr)rcrdr#rbr$ unique_valuessrez3<([^>]*\srel\s{0,10}=\s{0,10}['"]?([^'" >]+)[^>]*)>c cst|D]d}|\}}tttj|d}d|vsDd|vr t |D]}t j |t |dVqNq dD]@}||}|dkrtt ||}|rtt j |t |dVqtdS)zEFind rel="homepage" and rel="download" links in `page`, yielding URLs,Zhomepager0r3)z Home PagezDownload URLr/N)RELfinditergroupssetmapstrstripr(r7HREFr4rurljoin htmldecoderEfindsearch)r8pagerDtagZrelZrelsposr#r#r$find_external_linkss   rvc@s(eZdZdZddZddZddZdS) ContentCheckerzP A null content checker that defines the interface for checking content cCsdS)z3 Feed a block of data to the hash. Nr#selfblockr#r#r$feedszContentChecker.feedcCsdS)zC Check the hash. Return False if validation fails. Tr#ryr#r#r$is_validszContentChecker.is_validcCsdS)zu Call reporter with information about the checker (hash name) substituted into the template. Nr#)ryreportertemplater#r#r$reportszContentChecker.reportN)__name__ __module__ __qualname____doc__r{r}rr#r#r#r$rwsrwc@sBeZdZedZddZeddZddZ dd Z d d Z d S) HashCheckerzK(?Psha1|sha224|sha384|sha256|sha512|md5)=(?P[a-f0-9]+)cCs||_t||_||_dSr_) hash_namehashlibnewhashexpected)ryrrr#r#r$__init__s zHashChecker.__init__cCsBtj|d}|stS|j|}|s0tS|fi|S)z5Construct a (possibly null) ContentChecker from a URLr/)r4rr5rwpatternrr groupdict)clsr8r?rDr#r#r$from_urls zHashChecker.from_urlcCs|j|dSr_)rupdaterxr#r#r$r{szHashChecker.feedcCs|j|jkSr_)rZ hexdigestrr|r#r#r$r}"szHashChecker.is_validcCs||j}||Sr_)r)ryr~rmsgr#r#r$r%s zHashChecker.reportN) rrrrTcompilerr classmethodrr{r}rr#r#r#r$r s rcsTeZdZdZdNfdd Zfdd ZdOd d ZdPd dZdQddZddZ ddZ ddZ ddZ ddZ dRddZddZdSfdd Zd!d"Zd#d$Zd%d&Zd'd(Zd)d*ZdTd+d,ZdUd-d.Zd/d0Zd1Zd2d3Zd4d5ZdVd6d7Zd8d9Zd:d;Zdd?Z d@dAZ!e"dWdBdCZ#dDdEZ$dFdGZ%dHdIZ&dJdKZ'dLdMZ(Z)S)Xrz;A distribution index that scans web pages for download URLshttps://pypi.org/simple/*NTcsltj|i||dd|d |_i|_i|_i|_td t t |j |_ g|_tjj|_dS)Nr.|)superrr) index_url scanned_urls fetched_urls package_pagesrTrr^rkrrDallowsto_scanr4requesturlopenopener)ryrZhostsZ ca_bundleZ verify_sslr`kw __class__r#r$r-s zPackageIndex.__init__cs0zt|jWnty"YdS0t|Sr_)rrL Exceptionradd)ryrGrr#r$r?s  zPackageIndex.addFc Cstdrtd|||jvr(|s(dSd|j|<t|sH||dStt|}|rr||sfdS| d||s|r||j vrtt |j |dS||sd|j |<dS| d|d|j |<d}||||}|durdSt|tjjr|jdkr| d |jd|j |j<d |jd d vrB|dS|j}|}t|tst|tjjrrd }n|jdpd }||d}|t|D](} tj !|t"| #d} |$| q|%|j&rt'|dddkr|(||}dS)z.)filterrQr<rr itertoolsstarmap scan_egg_link)ry search_pathdirsZ egg_linksr#r#r$scan_egg_linkss zPackageIndex.scan_egg_linkscCsttj||&}ttdttj|}Wdn1s>0Yt |dkrXdS|\}}t tj||D]*}tjj|g|R|_ t |_ ||qrdS)NrY)openrQr<r^rrrkrlrmrNr rJrrAr)ryr<rZ raw_lineslinesZegg_pathZ setup_pathrGr#r#r$rs4 zPackageIndex.scan_egg_linkcCsd}||js|Stttjj|t|jdd}t|dksRd|dvrV|St |d}t |d}d|j | i|<t|t|fS)N)NNr.rYr2r3rT)r*rrrkr4rr6rNr7r r r setdefaultr(r )ryrZNO_MATCH_SENTINELr9pkgverr#r#r$_scans &  zPackageIndex._scanc Cst|D]:}z"|tj|t|dWq tyBYq 0q ||\}}|s\dSt ||D]H}t |\}}| dr|s|r|d||f7}n | || |qftdd|S)z#Process the contents of a PyPI pager3r.pyz #egg=%s-%scSsd|dddS)Nz%sr3rY)rE)mr#r#r$rXz,PackageIndex.process_index..)rnrhrr4rrorprEr rvr@r)need_version_infoscan_urlPYPI_MD5sub) ryr8rsrDrrnew_urlr,fragr#r#r$rs$"    zPackageIndex.process_indexcCs|d|dS)NzPPage at %s links to .py file(s) without version info; an index scan is required.)scan_allryr8r#r#r$rszPackageIndex.need_version_infocGs<|j|jvr,|r"|j|g|R|d||jdS)Nz6Scanning index of all packages (this may take a while))rrrrrryrr`r#r#r$rs   zPackageIndex.scan_allcCsz||j|jd|j|js:||j|jd|j|jsR||t|j|jdD]}||qfdS)Nr.r#) rr unsafe_namerrkeyrKnot_found_in_indexr)ry requirementr8r#r#r$ find_packagess zPackageIndex.find_packagescsR|||||jD]"}||vr0|S|d||qtt|||S)Nz%s does not match %s)prescanrrrrrobtain)ryrZ installerrGrr#r$rs zPackageIndex.obtaincCsL||jd||sH|t|td|jjtj |fdS)z- checker is a ContentChecker zValidating %%s checksum for %sz7%s validation failed for %s; possible download problem?N) rrr}rrQunlinkrrr+r<rO)rycheckerrRtfpr#r#r$ check_hash s zPackageIndex.check_hashcCsN|D]D}|jdus0t|r0|ds0tt|r<||q|j|qdS)z;Add `urls` to the list that will be prescanned for searchesNfile:)rrr*rrrappend)ryZurlsr8r#r#r$add_find_linkss  zPackageIndex.add_find_linkscCs"|jrtt|j|jd|_dS)z7Scan urls scheduled for prescanning (e.g. --find-links)N)rrrkrr|r#r#r$r*szPackageIndex.prescancCs<||jr|jd}}n |jd}}|||j|dS)Nz#Couldn't retrieve index page for %rz3Couldn't find index page for %r (maybe misspelled?))rrrrr)ryrmethrr#r#r$r0s  zPackageIndex.not_found_in_indexcCs~t|tsjt|}|rR||d||}t|\}}|drN||||}|Stj |rb|St |}t | ||ddS)aLocate and/or download `spec` to `tmpdir`, returning a local path `spec` may be a ``Requirement`` object, or a string containing a URL, an existing local filename, or a project/version requirement spec (i.e. the string form of a ``Requirement`` object). If it is the URL of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a project/version requirement spec, this method returns the location of a matching distribution (possibly after downloading it to `tmpdir`). If `spec` is a locally existing file or directory name, it is simply returned unchanged. If `spec` is a URL, it is downloaded to a subpath of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. r3rrJN)rrr _download_urlrEr@r) gen_setuprQr<rr%rfetch_distribution)ryr!tmpdirr:foundr,r?r#r#r$r0;s    zPackageIndex.downloadc sd|id}d fdd }|rH|||}|s^|dur^|||}|durjdurx||}|dur|s|||}|durdrdpd|nd||j|jd SdS) a|Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. If necessary, or if the `force_scan` flag is set, the requirement is searched for in the (online) package index as well as the locally installed packages. If a distribution matching `requirement` is found, the returned distribution's ``location`` is the value you would have gotten from calling the ``download()`` method with the matching distribution's URL or filename. If no matching distribution is found, ``None`` is returned. If the `source` flag is set, only source distributions and source checkout links will be considered. Unless the `develop_ok` flag is set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. zSearching for %sNcs|dur }||jD]v}|jtkrFsF|vrd|d|<q||vo\|jtkp\ }|r|j}||_tj |jr|SqdS)Nz&Skipping development or system egg: %sr3) rrArrrr0rJdownload_locationrQr<r)ZreqenvrGtestZloc develop_okryZskippedsourcerr#r$rq{s"z-PackageIndex.fetch_distribution..findz:No local packages or working download links found for %s%sza source distribution of rzBest match: %s)rJ)N)rrrrrZcloner) ryrr force_scanrrZ local_indexrGrqr#rr$r]s2         zPackageIndex.fetch_distributioncCs"|||||}|dur|jSdS)a3Obtain a file suitable for fulfilling `requirement` DEPRECATED; use the ``fetch_distribution()`` method now instead. For backward compatibility, this routine is identical but returns the ``location`` of the downloaded distribution instead of a distribution object. N)rrJ)ryrrrrrGr#r#r$fetchszPackageIndex.fetchc Cst|}|r*ddt||ddDp,g}t|dkrtj|}tj||krtj ||}tj |r~tj ||st |||}ttj |dd<}|d|dj|djtj|dfWdn1s0Y|S|r td||fntd dS) NcSsg|]}|jr|qSr#)rL)rUdr#r#r$ sz*PackageIndex.gen_setup..r3zsetup.pywzIfrom setuptools import setup setup(name=%r, version=%r, py_modules=[%r]) rzCan't unambiguously interpret project/version identifier %r; any dashes in the name or version should be escaped using underscores. %rzpCan't process plain .py files without an '#egg=name-version' suffix to enable automatic setup script generation.)rCrDrrErNrQr<rOdirnamer^rsamefileshutilZcopy2rwriterKrLsplitextr) ryrRr?rrDrrOZdstrr#r#r$rsD    " zPackageIndex.gen_setupi c Cs:|d|d}zt|}||}t|tjjrLtd||j |j f|}d}|j }d}d|vr| d} t tt| }||||||t|d`} ||} | r|| | | |d7}||||||qqq|||| Wdn1s0Y|W|r"|Sn|r4|0dS) NzDownloading %szCan't download %s: %s %srr/zcontent-lengthzContent-Lengthwbr3)rrrrrr4rrrrr dl_blocksizeZget_allmaxrkint reporthookrrr{rrr) ryr8rRfprrblocknumZbssizeZsizesrrzr#r#r$ _download_tosD        .zPackageIndex._download_tocCsdSr_r#)ryr8rRrZblksizerr#r#r$rszPackageIndex.reporthookc Cs|drt|Szt||jWSttjjfy}zHddd|j D}|r`| ||nt d||f|WYd}~n*d}~0t j jy}z|WYd}~Sd}~0t j jy}z4|r| ||jnt d||jf|WYd}~nd}~0tjjyT}z6|r,| ||jnt d||jf|WYd}~n^d}~0tjjtj fy}z2|r| ||nt d||f|WYd}~n d}~00dS)Nr cSsg|] }t|qSr#)rl)rUargr#r#r$r rXz)PackageIndex.open_url..z%s %szDownload error for %s: %sz;%s returned a bad status line. The server might be down, %s)r* local_openopen_with_authrr httpclient InvalidURLr^r`rrr4rrZURLErrorreasonZ BadStatusLinelineZ HTTPExceptionsocket)ryr8Zwarningvrr#r#r$rsB ( zPackageIndex.open_urlcCst|\}}|r0d|vr4|dddd}qnd}|drJ|dd}tj||}|dksj|d rv|||S|d ks|d r|||S|d r| ||S|d krt j t j |dS||d|||SdS)Nz...\_Z__downloaded__rHr'Zsvnzsvn+Zgitzgit+zhg+rrYT)r@replacer)rQr<r^r* _download_svn _download_git _download_hgr4r url2pathnamerr5r_attempt_download)ryr:r8rr+r?rRr#r#r$r4s$        zPackageIndex._download_urlcCs||ddS)NT)rrr#r#r$rQszPackageIndex.scan_urlcCs6|||}d|ddvr.||||S|SdS)Nrrr)rrr(_invalid_download_html)ryr8rRrr#r#r$r,Ts zPackageIndex._attempt_downloadcCst|td|dS)NzUnexpected HTML page found at )rQrr)ryr8rrRr#r#r$r-[s z#PackageIndex._invalid_download_htmlcCstd|dS)Nz/Invalid config, SVN download is not supported: r)ryr8Z _filenamer#r#r$r(_szPackageIndex._download_svncCsptj|\}}}}}|ddd}|ddd}d}d|vrR|dd\}}tj||||df}||fS)N+r3r/r2r@r)r4rZurlsplitr7rsplitZ urlunsplit)r8 pop_prefixr:netlocr<r>rrevr#r#r$_vcs_split_rev_from_urlbsz$PackageIndex._vcs_split_rev_from_urlcCsl|ddd}|j|dd\}}|d||td||f|durh|d|td ||f|S) Nr2r3rTr1zDoing git clone from %s to %szgit clone --quiet %s %szChecking out %szgit -C %s checkout --quiet %sr7r4rrQsystemryr8rRr3r#r#r$r)ts zPackageIndex._download_gitcCsl|ddd}|j|dd\}}|d||td||f|durh|d|td ||f|S) Nr2r3rTr5zDoing hg clone from %s to %szhg clone --quiet %s %szUpdating to %szhg --cwd %s up -C -r %s -qr6r8r#r#r$r*s zPackageIndex._download_hgcGstj|g|RdSr_)rrrr#r#r$rszPackageIndex.debugcGstj|g|RdSr_)rrrr#r#r$rszPackageIndex.infocGstj|g|RdSr_)rrrr#r#r$rszPackageIndex.warn)rrNT)F)F)F)N)N)FFFN)FF)N)F)*rrrrrrrrrrrrrrrrrrrrrr0rrrrrrrrrr,r-r( staticmethodr4r)r*rrr __classcell__r#r#rr$r*s\ 9      & P .$ " rz!&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?cCs|d}t|S)Nr)rErZunescape)rDZwhatr#r#r$ decode_entitys r;cCs tt|S)a Decode HTML entities in the given text. >>> htmldecode( ... 'https://../package_name-0.1.2.tar.gz' ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' ) entity_subr;)textr#r#r$rps rpcsfdd}|S)Ncsfdd}|S)Nc s>t}tz|i|Wt|St|0dSr_)r"ZgetdefaulttimeoutZsetdefaulttimeout)r`raZ old_timeout)rctimeoutr#r$_socket_timeouts  z@socket_timeout.._socket_timeout.._socket_timeoutr#)rcr?r>rbr$r?sz'socket_timeout.._socket_timeoutr#)r>r?r#r@r$socket_timeouts rAcCs2tj|}|}t|}|}|ddS)a9 Encode auth from a URL suitable for an HTTP header. >>> str(_encode_auth('username%3Apassword')) 'dXNlcm5hbWU6cGFzc3dvcmQ=' Long auth strings should not cause a newline to be inserted. >>> long_auth = 'username:' + 'password'*10 >>> chr(10) in str(_encode_auth(long_auth)) False  r)r4rr6encodebase64Z b64encoderr')authZauth_sZ auth_bytesZ encoded_bytesZencodedr#r#r$ _encode_auths  rFc@s(eZdZdZddZddZddZdS) Credentialz: A username/password pair. Use like a namedtuple. cCs||_||_dSr_usernamepassword)ryrIrJr#r#r$rszCredential.__init__ccs|jV|jVdSr_rHr|r#r#r$__iter__szCredential.__iter__cCs dt|S)Nz%(username)s:%(password)s)varsr|r#r#r$__str__szCredential.__str__N)rrrrrrKrMr#r#r#r$rGsrGcs8eZdZfddZeddZddZddZZS) PyPIConfigcsLtgdd}t|tjtjdd}tj|rH| |dS)z% Load from ~/.pypirc )rIrJ repositoryr~z.pypircN) dictfromkeysrrrQr<r^ expanduserrr)rydefaultsZrcrr#r$rs   zPyPIConfig.__init__cs&fddD}ttj|S)Ncs g|]}|dr|qS)rO)rrm)rUsectionr|r#r$r sz2PyPIConfig.creds_by_repository..)ZsectionsrQrk_get_repo_cred)ryZsections_with_repositoriesr#r|r$creds_by_repositorys zPyPIConfig.creds_by_repositorycCs6||d}|t||d||dfS)NrOrIrJ)rrmrG)ryrUZrepor#r#r$rVs zPyPIConfig._get_repo_credcCs*|jD]\}}||r |Sq dS)z If the URL indicated appears to be a repository defined in this config, return the credential for that repository. N)rWitemsr*)ryr8rOcredr#r#r$find_credential s zPyPIConfig.find_credential) rrrrpropertyrWrVrZr:r#r#rr$rNs   rNcCs@tj|}|\}}}}}}|dr2tjd|dvrHt|\} } nd} | st |} | rt | } | j |f} t j dg| R| rdt| } || ||||f} tj| }tj|}|d| n tj|}|dt||}| rrrEZaddressrYrr9rrrs2Zh2Zpath2Zparam2Zquery2Zfrag2r#r#r$rs8          rcCs |d\}}}|r|nd|fS)zNsplituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.r/N) rpartition)ZhostuserZdelimr#r#r$r]Esr]cCs|Sr_r#)r8r#r#r$ fix_sf_urlPsrcc Cs,tj|\}}}}}}tj|}tj|r{name})r+zB{url}{files}rB)r8files)ZOK)rzPath not foundz Not foundrz text/html)r4rr5rr+rQr<isfilerr)rrr^rrrformatioStringIOrr)r8r:r;r<Zparamr>rrRrerfilepathrbodyrZstatusmessagerZ body_streamr#r#r$rTs.    &    r)N)N)N)r)YrsysrQrTrir r"rDrrZ configparserrZ http.clientrZ urllib.parser4Zurllib.requestZ urllib.error functoolsrrZ pkg_resourcesrrrrrr r r r r rrrr distutilsrZdistutils.errorsrZfnmatchrZsetuptools.wheelrZ setuptools.extern.more_itertoolsrrrCIrnrrDrr7rM__all__Z_SOCKET_TIMEOUTZ_tmplrh version_infor^r%rr@rrBrSrrergrvrwrrrr<r;rprArFrGZRawConfigParserrNrrrr]rcrr#r#r#r$s @          #  "  !  '/