3
��gLQ � @ sF d dl mZ d dl mZ d dlmZmZ d dlZd dlZd dl Zd dl
Zd dl Zd dl Zd dl
Zd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dlZd dl Z d dl Z d dl Z d dl Z dZ dZ!e j"e j# d Z$dej%e$�e j&f Z'd e'e!f d
e'e f d d �Z(ej)d
�Z*dd� Z+dd� Z,d-dd�Z-dd� Z.G dd� de/�Z0G dd� de1�Z2G dd � d ej3j4�Z5G d d � d e/�Z6G d d � d ej7j8�Z9G d!d"� d"ej3j:�Z;G d#d$� d$e;�Zej7j?j@ZAej7j?jBZCej7j?jDZEG d)d*� d*ej7jF�ZGG d+d,� d,ejHjI�Z?dS ).� )�absolute_import)�unicode_literals)�ucd�_N�packagesZ
mirrorlistz-_.:z(?P[%s]+)\-[%s]{16}z>^%s\/.*((xml|yaml)(\.gz|\.xz|\.bz2|.zck)?|asc|cachecookie|%s)$z^%s\/%s\/.+rpm$z^.+(solv|solvx)$)�metadatar Zdbcache�dnfc C s t jjj| �}|dk rdS |S )zAReturn index of an invalid character in the repo ID (if present).r N)�libdnf�repo�RepoZverifyId)Zrepo_idZ
first_invalid� r �/usr/lib/python3.6/repo.py�repo_id_invalidH s r c G s8 x"|D ]}|| |�}|d k r|S qW t td�| ��d S )Nz"no matching payload factory for %s)�
ValueErrorr )�pkg�progressZ factories�fn�ploadr r r
� _pkg2payloadO s
r Tc C s� dd� }|j j� dd� t| |d�D �}t� }y tjjjtjj|�|� W n, t k
rv } zt
|�|_ W Y d d }~X nX |j � |j j
� |_xj|D ]b}|j� }|d ks�|jd�r�q�|j� } | j}
|
j} |dkr�|jj| � q�| jjj� |g|j| < q�W |S )Nc S s t | d� S )NZdelta)�hasattr)�payloadr r r
�_download_sort_keyY s z._download_payloads.._download_sort_keyc S s g | ] }|j � �qS r )�_librepo_target)�.0r r r r
�
] s z&_download_payloads..)�keyz Not finishedzAlready downloaded)�err�clear�sorted�_DownloadErrorsr r
�
PackageTargetZdownloadPackagesZVectorPPackageTarget� RuntimeError�str�_fatal�wait�copy� _recoverableZgetErr�
startswithZ getCallbacks�
package_ploadr �_skipped�add�_repoZexpire�_pkg_irrecoverable) �payloadsZdrpmZ fail_fastr Ztargets�errs�eZtgtr � callbacksr r r r r
�_download_payloadsW s0
r1 c C sL | \}}x:|D ]2}|j }||kr,||j7 }q||j7 }||j7 }qW ||fS )N)r �
download_size�
_full_size)Zsavingr- r. �realZfullr r r r r
�_update_savingx s
r5 c @ s> e Zd Zdd� Zdd� Zedd� �Zejdd� �Zdd � Zd
S ) r c C s i | _ i | _d | _t� | _d S )N)r, �_val_recoverabler# �setr) )�selfr r r
�__init__� s z_DownloadErrors.__init__c C s" | j r | j S | jr d| jgiS i S )N� )r, r# )r8 r r r
�_irrecoverable� s
z _DownloadErrors._irrecoverablec C s | j S )N)r6 )r8 r r r
r& � s z _DownloadErrors._recoverablec C s
|| _ d S )N)r6 )r8 Znew_dctr r r
r& � s c C s |j | jkrdS |jS )Nr )r r) r2 )r8 r r r r
�_bandwidth_used� s z _DownloadErrors._bandwidth_usedN) �__name__�
__module__� __qualname__r9 r; �propertyr& �setterr< r r r r
r � s
r c @ s e Zd Zdd� ZdS )�_DetailedLibrepoErrorc C s, t j| � |jd | _|jd | _|| _d S )Nr � )� Exceptionr9 �argsZ librepo_codeZ librepo_msg�
source_url)r8 Z librepo_errrF r r r
r9 � s
z _DetailedLibrepoError.__init__N)r= r> r? r9 r r r r
rB � s rB c @ s e Zd Zdd� ZdS )�_NullKeyImportc C s dS )NTr )r8 �id�userid� fingerprint�url� timestampr r r
�_confirm� s z_NullKeyImport._confirmN)r= r> r? rM r r r r
rG � s rG c @ s e Zd Zdd� Zedd� �ZdS )�Metadatac C s
|| _ d S )N)r+ )r8 r
r r r
r9 � s zMetadata.__init__c C s
| j j� S )N)r+ �fresh)r8 r r r
rO � s zMetadata.freshN)r= r> r? r9 r@ rO r r r r
rN � s rN c s4 e Zd Z� fdd�Zdd� Zdd� Zdd� Z� ZS ) �PackageTargetCallbacksc s t t| �j� || _d S )N)�superrP r9 r( )r8 r( )� __class__r r
r9 � s z PackageTargetCallbacks.__init__c C s | j jd ||� dS )Nr )r( �_end_cb)r8 �status�msgr r r
�end� s zPackageTargetCallbacks.endc C s | j jd ||� dS )Nr )r( � _progress_cb)r8 �totalToDownload�
downloadedr r r
r � s z PackageTargetCallbacks.progressc C s | j jd ||� dS )Nr )r( �_mirrorfail_cb)r8 rU rK r r r
�
mirrorFailure� s z$PackageTargetCallbacks.mirrorFailure)r= r> r? r9 rV r r[ �
__classcell__r r )rR r
rP � s rP c sH e Zd Z� fdd�Zdd� Zdd� Zdd� Zed d
� �Zd d � Z � Z
S )
�PackagePayloadc s$ t t| �j|� t| �| _|| _d S )N)rQ r] r9 rP r0 r )r8 r r )rR r r
r9 � s
zPackagePayload.__init__c C sR t jj}|dkrt jj}n$|jd�r(dS |tjjjkr>t jj }| j
j | ||� dS )z"End callback to librepo operation.Nz Not finished) r �callbackZ
STATUS_FAILEDZ STATUS_OKr' r r
�PackageTargetCBZ TransferStatus_ALREADYEXISTSZSTATUS_ALREADY_EXISTSr rV )r8 �cbdataZ lr_statusrU rT r r r
rS � s
zPackagePayload._end_cbc C s | j j| tjj|� d S )N)r rV r r^ Z
STATUS_MIRROR)r8 r` r rK r r r
rZ � s z PackagePayload._mirrorfail_cbc C sX y| j j | |� W n@ tk
rR tj� \}}}tj|||�}tjdj|�� Y nX d S )Nr: ) r rD �sys�exc_info� traceback�format_exception�logger�critical�join)r8 r` �total�done�exc_type� exc_value�
exc_traceback� except_listr r r
rW � s zPackagePayload._progress_cbc C s | j S )N)r2 )r8 r r r
r3 � s zPackagePayload._full_sizec C s� | j }|j}tjj|� |d| | j| j| jd�}|j| j � � t
j j |j j
|d |d |d |d |d |d |d d
d
| j� S ) NT)�dest�resumer` Z
progresscbZendcbZmirrorfailurecb� relative_urlrn �
checksum_type�checksum� expectedsize�base_urlro r )r �pkgdirr �util�
ensure_dirrW rS rZ �update�_target_paramsr r
r r+ r0 )r8 r ru Z
target_dctr r r
r � s
z PackagePayload._librepo_target) r= r> r? r9 rS rZ rW r@ r3 r r\ r r )rR r
r] � s r] c @ s( e Zd Zdd� Zdd� Zedd� �ZdS )�
RPMPayloadc C s t jj| jj�S )N)�os�path�basenamer �location)r8 r r r
�__str__ s zRPMPayload.__str__c C sT | j }|j� \}}tjjj|�}|tjjjkr>tjt d�|� |j
|||j |j d�S )Nz unsupported checksum type: %s)rp rq rr rs rt )
r Z returnIdSumr r
r Z checksumTypeZChecksumType_UNKNOWNre �warningr r~ � downloadsize�baseurl)r8 r ZctypeZcsumZ
ctype_coder r r
ry s zRPMPayload._target_paramsc C s | j jS )zTotal size of the download.)r r� )r8 r r r
r2 s zRPMPayload.download_sizeN)r= r> r? r ry r@ r2 r r r r
rz s rz c s@ e Zd Z� fdd�Zdd� Zdd� Zdd� Zed d
� �Z� Z S ) �RemoteRPMPayloadc s� t t| �jd|� || _d| _|| _| jjp.d| jjjd� }t j
|j d��j � d d� }d| }t
jj| jj|d�| _tjj| j� t
jj| j| j� jd ��| _d S )
NZ
unused_objectr r: Zbasearch�utf8� z commandline-r �/)rQ r� r9 �remote_location� remote_size�confZ
releasever�
substitutions�get�hashlibZsha256�encodeZ hexdigestr{ r| rg Zcachedirru r rv rw r �lstripZ
local_path)r8 r� r� r �sZdigestZrepodir)rR r r
r9 s zRemoteRPMPayload.__init__c C s t jj| j�S )N)r{ r| r} r� )r8 r r r
r ) s zRemoteRPMPayload.__str__c C s^ || _ y| jj| |� W n@ tk
rX tj� \}}}tj|||�}tjdj |�� Y nX d S )Nr: )
r� r rD ra rb rc rd re rf rg )r8 r` rh ri rj rk rl rm r r r
rW , s z RemoteRPMPayload._progress_cbc C s< t jj| jjtjj| j�| j dd dtjj
| j�ddd| j � S )Nr T) r r
r r� �_configr{ r| r} r� ru �dirnamer0 )r8 r r r
r 5 s z RemoteRPMPayload._librepo_targetc C s | j S )zTotal size of the download.)r� )r8 r r r
r2 ; s z RemoteRPMPayload.download_size)
r= r> r? r9 r rW r r@ r2 r\ r r )rR r
r� s
r� c sz e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d
� Zd d � Ze d
d� �Z
e dd� �Z e j dd� �Z dd� Z
dd� Z� ZS )� MDPayloadc s. t t| �j|� d| _d| _d| _t� | _d S )Nr: r F)rQ r� r9 �_text�_download_size�fastest_mirror_runningr7 �mirror_failures)r8 r )rR r r
r9 C s
zMDPayload.__init__c C s t jjr| jS | jjd�S d S )Nzutf-8)r �pycompZPY3r� r� )r8 r r r
r J s zMDPayload.__str__c C s | j S )N)r� )r8 r r r
� __unicode__P s zMDPayload.__unicode__c C s || _ | jj| |� d S )N)r� r )r8 r` rh ri r r r
rW S s zMDPayload._progress_cbc C s\ |t jjjkr"td�| }d| _n*|t jjjkrH| jrH|rBd| nd}nd S | jj|� d S )Nz,determining the fastest mirror (%s hosts).. Tz
error: %s
zdone.
) r r
�RepoCBZ FastestMirrorStage_DETECTIONr r� ZFastestMirrorStage_STATUSr �message)r8 r` �stage�datarU r r r
�_fastestmirror_cbW s zMDPayload._fastestmirror_cbc C s& | j j|� d||f }tj|� d S )Nzerror: %s (%s).)r� r* re �debug)r8 r` rU rK r r r r
�_mirror_failure_cbc s z MDPayload._mirror_failure_cbc C s | j S )N)r� )r8 r r r
r2 h s zMDPayload.download_sizec C s | j S )N)� _progress)r8 r r r
r l s zMDPayload.progressc C s |d krt jj� }|| _d S )N)r r^ �NullDownloadProgressr� )r8 r r r r
r p s
c C s || _ | jjdd� d S )NrC r )r� r �start)r8 �textr r r
r� v s zMDPayload.startc C s d| _ | jj| d d � d S )Nr )r� r rV )r8 r r r
rV z s z
MDPayload.end)r= r> r? r9 r r� rW r� r� r@ r2 r rA r� rV r\ r r )rR r
r� A s r� c sL e Zd Z� fdd�Zdd� Zdd� Zdd� Zd d
� Zd d � Zd
d� Z � Z
S )�
RepoCallbacksc s t t| �j� || _|j| _d S )N)rQ r� r9 r+ � _md_pload)r8 r
)rR r r
r9 � s zRepoCallbacks.__init__c C s | j j|� d S )N)r� r� )r8 Zwhatr r r
r� � s zRepoCallbacks.startc C s | j j� d S )N)r� rV )r8 r r r
rV � s zRepoCallbacks.endc C s | j jd ||� dS )Nr )r� rW )r8 rX rY r r r
r � s zRepoCallbacks.progressc C s | j jd ||� d S )N)r� r� )r8 r� Zptrr r r
�
fastestMirror� s zRepoCallbacks.fastestMirrorc C s | j jd |||� dS )Nr )r� r� )r8 rU rK r r r r
�handleMirrorFailure� s z!RepoCallbacks.handleMirrorFailurec C s | j jj|||||�S )N)r+ � _key_importrM )r8 rH rI rJ rK rL r r r
�
repokeyImport� s zRepoCallbacks.repokeyImport) r= r> r? r9 r� rV r r� r� r� r\ r r )rR r
r� � s r� c s e Zd ZeZd7� fdd� Zedd� �Zedd� �Zej dd� �Zed d
� �Z
d d � Z e
j d
d
� �Z
edd� �Z edd� �Z
e
j dd� �Z
dd� Zdd� Z� fdd�Zdd� Zdd � Zd d � Zd d � Zd!d"� Zd#d$� Zd%d&� Zd'd(� Zd)d*� Zd+d,� Zd-d.� Zd/d0� Z d9d5d6�Z � Z S ):r Nc s� t t| �j||d� | jjj� tjj|r.|nd| j�| _t t
j j � �| _
t| �| _| jjj� | jj| j� d | _t� | _d | _| jj|r�|jr�tn| j� |r�| jj|j� t
jjj � | _ |d k r�|j nd| _ d S )N)Zsection�parentr: T) rQ r r9 r� �thisZdisownr r
r+ r� r r^ r� r� r� Z
_callbacksZ setCallbacks�_pkgdirrG r� r ZsetSyncStrategyZ cacheonly�SYNC_ONLY_CACHE� DEFAULT_SYNCZsetSubstitutionsr� r� Z
SubstitutionsZ_substitutionsZcheck_config_file_ageZ_check_config_file_age)r8 �nameZ parent_conf)rR r r
r9 � s
z
Repo.__init__c C s
| j j� S )N)r+ ZgetId)r8 r r r
rH � s zRepo.idc C s
| j j� S )N)r+ ZgetRepoFilePath)r8 r r r
�repofile� s z
Repo.repofilec C s | j j|� d S )N)r+ ZsetRepoFilePath)r8 �valuer r r
r� � s c C s | j j� r| j j� S | j� S )N)r+ ZisLocalZgetLocalBaseurl� cache_pkgdir)r8 r r r
ru � s
z Repo.pkgdirc C s$ | j d k r| j S tjj| jj� t�S )N)r� r{ r| rg r+ � getCachedir�_PACKAGES_RELATIVE_DIR)r8 r r r
r� � s
zRepo.cache_pkgdirc C s
|| _ d S )N)r� )r8 �valr r r
ru � s c C s t jj| jj� d�S )NZpubring)r{ r| rg r+ r� )r8 r r r
� _pubring_dir� s zRepo._pubring_dirc C s
| j j� S )N)r+ ZgetLoadMetadataOther)r8 r r r
�load_metadata_other� s zRepo.load_metadata_otherc C s | j j|� d S )N)r+ ZsetLoadMetadataOther)r8 r� r r r
r� � s c C s | j |j k S )N)rH )r8 �otherr r r
�__lt__� s z Repo.__lt__c C s d| j j| jf S )Nz<%s %s>)rR r= rH )r8 r r r
�__repr__� s z
Repo.__repr__c s t t| �j||� d S )N)rQ r � __setattr__)r8 r� r� )rR r r
r� � s zRepo.__setattr__c C s | j j� d S )N)r+ �disable)r8 r r r
r� � s z Repo.disablec C s | j j� d S )N)r+ �enable)r8 r r r
r� � s z Repo.enablec C s | j j|� dS )a/ Ask for additional repository metadata type to download.
Given metadata_type is appended to the default metadata set when
repository is downloaded.
Parameters
----------
metadata_type: string
Example: add_metadata_type_to_download("productid")
N)r+ ZaddMetadataTypeToDownload)r8 �
metadata_typer r r
� add_metadata_type_to_download� s
z"Repo.add_metadata_type_to_downloadc C s | j j|� dS )aI Stop asking for this additional repository metadata type
in download.
Given metadata_type is no longer downloaded by default
when this repository is downloaded.
Parameters
----------
metadata_type: string
Example: remove_metadata_type_from_download("productid")
N)r+ Z removeMetadataTypeFromDownload)r8 r� r r r
�"remove_metadata_type_from_download s z'Repo.remove_metadata_type_from_downloadc C s | j j|�S )z�Return path to the file with downloaded repository metadata of given type.
Parameters
----------
metadata_type: string
)r+ ZgetMetadataPath)r8 r� r r r
�get_metadata_path s zRepo.get_metadata_pathc C s | j j|�S )z�Return content of the file with downloaded repository metadata of given type.
Content of compressed metadata file is returned uncompressed.
Parameters
----------
metadata_type: string
)r+ ZgetMetadataContent)r8 r� r r r
�get_metadata_content! s
zRepo.get_metadata_contentc C s� d}z�y| j j� }W nt tjjtfk
r� } zP| jjrhd| j }x | jjD ]}|d| 7 }qJW t j
|� t j j
t|���W Y dd}~X nX W dt� | j_X t| j �| _|S )a� Load the metadata for this repo.
Depending on the configuration and the age and consistence of data
available on the disk cache, either loads the metadata from the cache or
downloads them from the mirror, baseurl or metalink.
This method will by default not try to refresh already loaded data if
called repeatedly.
Returns True if this call to load() caused a fresh metadata download.
Fz7Errors during downloading metadata for repository '%s':z
- %sN)r+ �loadr �error�Errorr! r� r� rH re r� r �
exceptionsZ RepoErrorr" r7 rN r )r8 �retr/ rU Zfailurer r r
r� - s
& z Repo.loadc C sP | j s| jjd� | j rL| jdkr&dS | jj� }| jj� rDtd|�}d|fS dS ) a) Get the number of seconds after which the cached metadata will expire.
Returns a tuple, boolean whether there even is cached metadata and the
number of seconds it will expire in. Negative number means the metadata
has expired already, None that it never expires.
FrC TNr �����)TN)Fr )r r+ Z loadCacheZmetadata_expireZ getExpiresInZ isExpired�min)r8 Z
expirationr r r
�_metadata_expire_inJ s
zRepo._metadata_expire_inc C s
|| _ d S )N)r� )r8 Z
key_importr r r
�_set_key_import] s zRepo._set_key_importc C s || j _d S )N)r� r )r8 r r r r
�set_progress_bar` s zRepo.set_progress_barc C s
| j j� S )zoReturns user defined http headers.
Returns
-------
headers : tuple of strings
)r+ ZgetHttpHeaders)r8 r r r
�get_http_headersd s zRepo.get_http_headersc C s | j j|� dS )a Sets http headers.
Sets new http headers and rewrites existing ones.
Parameters
----------
headers : tuple or list of strings
Example: set_http_headers(["User-Agent: Agent007", "MyFieldName: MyFieldValue"])
N)r+ ZsetHttpHeaders)r8 Zheadersr r r
�set_http_headersn s zRepo.set_http_headers�http�ftp�file�httpsc s@ � �fdd�}� sdS | j j� }|r,||�S | jr<|| j�S dS )z�
:param location: relative location inside the repo
:param schemes: list of allowed protocols. Default is ('http', 'ftp', 'file', 'https')
:return: absolute url (string) or None
c sZ xT| D ]L}�r>t jjj|�d }|�krRtjj|� jd��S qtjj|� jd��S qW d S )Nr r� )r r� Zurlparser{ r| rg r� )Zurl_listrK r� )r~ �schemesr r
�schemes_filter� s
z,Repo.remote_location..schemes_filterN)r+ Z
getMirrorsr� )r8 r~ r� r� Zmirrorsr )r~ r� r
r� { s
zRepo.remote_location)NN�r� r� r� r� )r� ) r= r> r? �SYNC_TRY_CACHEr� r9 r@ rH r� rA ru r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r� r\ r r )rR r
r � s6
r )T)JZ
__future__r r Zdnf.i18nr r Z dnf.callbackr Zdnf.confZdnf.conf.substitutionsZ dnf.constZ
dnf.cryptoZdnf.exceptionsZ dnf.loggingZ
dnf.pycompZdnf.utilZ dnf.yum.miscZ libdnf.errorr Z libdnf.repo� functoolsr� ZhawkeyZlogging�operatorr{ �reZshutil�stringra Ztimerc r� Z_MIRRORLIST_FILENAMEZ
ascii_lettersZdigitsZ
_REPOID_CHARS�escapeZ hexdigitsZ _CACHEDIR_REZ CACHE_FILESZ getLoggerre r r r1 r5 �objectr rD rB r^ Z KeyImportrG rN r
r_ rP ZPayloadr] rz r� r� r ZSyncStrategy_LAZYZ SYNC_LAZYZSyncStrategy_ONLY_CACHEr� ZSyncStrategy_TRY_CACHEr� r� r� r� ZRepoConfr r r r
� sl
!
8&?