33#include <zypp-curl/parser/MetaLinkParser>
36#include <zypp-curl/auth/CurlAuthData>
43#undef CURLVERSION_AT_LEAST
44#define CURLVERSION_AT_LEAST(M,N,O) LIBCURL_VERSION_NUM >= ((((M)<<8)+(N))<<8)+(O)
187 void adddnsfd( std::vector<GPollFD> &waitFds );
188 void dnsevent(
const std::vector<GPollFD> &waitFds );
219 size_t writefunction (
char *ptr, std::optional<off_t> offset,
size_t bytes )
override;
221 bool beginRange ( off_t range, std::string &cancelReason )
override;
222 bool finishedRange ( off_t range,
bool validated, std::string &cancelReason )
override;
235 Url baseurl, CURLM *multi, FILE *fp,
244 void run(std::vector<Url> &urllist);
266 std::list< std::unique_ptr<multifetchworker> >
_workers;
319#if _POSIX_C_SOURCE >= 199309L
321 if ( clock_gettime( CLOCK_MONOTONIC, &ts) )
323 return ts.tv_sec + ts.tv_nsec / 1000000000.;
326 if (gettimeofday(&tv, NULL))
328 return tv.tv_sec + tv.tv_usec / 1000000.;
336 return bytes ? 0 : 1;
358 const auto &blk =
_blocks[*currRange];
359 off_t seekTo = blk.start + blk.bytesWritten;
364 return bytes ? 0 : 1;
376 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
379 cancelReason =
"Cancelled because stripe block is already finalized";
381 WAR <<
"#" <<
_workerno <<
": trying to start a range ("<<stripeRangeOff<<
"["<<
_blocks[workerRangeOff].start <<
" : "<<
_blocks[workerRangeOff].len<<
"]) that was already finalized, cancelling. Stealing was: " <<
_request->
_stealing << endl;
392 const auto &currRangeState = stripeDesc.blockStates[stripeRangeOff];
396 cancelReason =
"Block failed to validate";
423 if (l > 9 && !strncasecmp(p,
"Location:", 9)) {
424 std::string line(p + 9, l - 9);
425 if (line[l - 10] ==
'\r')
426 line.erase(l - 10, 1);
427 XXX <<
"#" <<
_workerno <<
": redirecting to" << line << endl;
433 XXX <<
"#" <<
_workerno <<
": filesize mismatch" << endl;
445, _maxspeed( request._maxspeed )
446, _request ( &request )
452 XXX <<
"reused worker from pool" << endl;
472 curl_easy_cleanup(
_curl);
478 curl_easy_setopt(
_curl, CURLOPT_PRIVATE,
this);
491 if (use_auth.empty())
492 use_auth =
"digest,basic";
494 if( auth != CURLAUTH_NONE)
496 XXX <<
"#" <<
_workerno <<
": Enabling HTTP authentication methods: " << use_auth
497 <<
" (CURLOPT_HTTPAUTH=" << auth <<
")" << std::endl;
498 curl_easy_setopt(
_curl, CURLOPT_HTTPAUTH, auth);
513#if CURLVERSION_AT_LEAST(7,15,5)
514 curl_easy_setopt(
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)0);
516 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
517 curl_easy_setopt(
_curl, CURLOPT_WRITEFUNCTION, (
void *)0);
518 curl_easy_setopt(
_curl, CURLOPT_WRITEDATA, (
void *)0);
519 curl_easy_setopt(
_curl, CURLOPT_HEADERFUNCTION, (
void *)0);
520 curl_easy_setopt(
_curl, CURLOPT_HEADERDATA, (
void *)0);
524 curl_easy_cleanup(
_curl);
546 const char *s = getenv(name.c_str());
547 return s && *s ? true :
false;
563 if (inet_pton(AF_INET, host.c_str(), addrbuf) == 1)
565 if (inet_pton(AF_INET6, host.c_str(), addrbuf) == 1)
576 if (schemeproxy !=
"http_proxy")
578 std::transform(schemeproxy.begin(), schemeproxy.end(), schemeproxy.begin(), ::toupper);
583 XXX <<
"checking DNS lookup of " << host << endl;
592 if (
_pid == pid_t(-1))
605 struct addrinfo *ai =
nullptr, aihints;
606 memset(&aihints, 0,
sizeof(aihints));
607 aihints.ai_family = PF_UNSPEC;
608 int tstsock = socket(PF_INET6, SOCK_DGRAM | SOCK_CLOEXEC, 0);
610 aihints.ai_family = PF_INET;
613 aihints.ai_socktype = SOCK_STREAM;
614 aihints.ai_flags = AI_CANONNAME;
617 alarm(connecttimeout);
618 signal(SIGALRM, SIG_DFL);
619 if (getaddrinfo(host.c_str(), NULL, &aihints, &ai))
637 .events = G_IO_IN | G_IO_HUP | G_IO_ERR,
645 bool hasEvent = std::any_of( waitFds.begin (), waitFds.end(),[
this](
const GPollFD &waitfd ){
646 return ( waitfd.fd == _dnspipe && waitfd.revents != 0 );
659 if (!WIFEXITED(status))
666 int exitcode = WEXITSTATUS(status);
667 XXX <<
"#" <<
_workerno <<
": DNS lookup returned " << exitcode << endl;
685 auto &blk =
_blocks[workerRangeIdx];
701 size_t cnt = l >
sizeof(buf) ?
sizeof(buf) : l;
711 blk._digest = std::move(newDig);
726 std::optional<zypp::Digest> digest;
727 std::optional<size_t> relDigLen;
728 std::optional<size_t> blkSumPad;
733 relDigLen = sum.size( );
739 return MultiByteHandler::Range::make(
745 std::move(relDigLen),
746 std::move(blkSumPad) );
753 XXX <<
"start stealing!" << endl;
767 if (worker->
_pass == -1)
834 XXX <<
"#" <<
_workerno <<
": going to sleep for " << sl * 1000 <<
" ms" << endl;
891 for ( uint i = 0; i < stripeDesc.blocks.size(); i++ ) {
911 DBG <<
"#" <<
_workerno <<
"Done adding blocks to download, going to download: " <<
_blocks.size() <<
" nr of block with " <<
_datasize <<
" nr of bytes" << std::endl;
922 bool hadRangeFail =
_multiByteHandler->lastError() == MultiByteHandler::Code::RangeFail;
928 if ( hadRangeFail ) {
935 curl_easy_reset(
_curl );
973 :
internal::CurlPollHelper::CurlPoll{ multi }
975 , _filename(
std::move(filename))
976 , _baseurl(
std::move(baseurl))
979 , _blklist(
std::move(blklist))
980 , _filesize(filesize)
982 , _timeout(context->_settings.timeout())
983 , _connect_timeout(context->_settings.connectTimeout())
984 , _maxspeed(context->_settings.maxDownloadSpeed())
985 , _maxworkers(context->_settings.maxConcurrentConnections())
1012 currStripeSize += blk.
size;
1027 std::vector<Url>::iterator urliter = urllist.begin();
1033 if (mcode != CURLM_OK)
1039 std::vector<GPollFD> waitFds;
1044 XXX <<
"finished!" << endl;
1051 _workers.push_back(std::make_unique<multifetchworker>(workerno++, *
this, *urliter));
1068 WAR <<
"No more active workers!" << endl;
1070 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1080 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1081 (*workeriter)->adddnsfd( waitFds );
1088 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter) {
1102 timeoutMs = sl * 1000;
1106 timeoutMs = std::min<long>( timeoutMs, _curlHelper.
timeout_ms.value() );
1108 dnsFdCount = waitFds.size();
1109 waitFds.insert( waitFds.end(), _curlHelper.
socks.begin(), _curlHelper.
socks.end() );
1115 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1120 (*workeriter)->
dnsevent( waitFds );
1130 if (mcode != CURLM_OK)
1134 if (mcode != CURLM_OK)
1154 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1163 XXX <<
"#" << worker->
_workerno <<
": sleep done, wake up" << endl;
1171 CURLMsg *msg =
nullptr;
1173 while ((msg = curl_multi_info_read(
_multi, &nqueue)) != 0)
1175 if (msg->msg != CURLMSG_DONE)
1177 CURL *easy = msg->easy_handle;
1178 CURLcode cc = msg->data.result;
1181 if (curl_easy_getinfo(easy, CURLINFO_PRIVATE, &worker) != CURLE_OK)
1192 curl_multi_remove_handle(
_multi, easy);
1194 const auto &setWorkerBroken = [&](
const std::string &
str = {} ){
1196 if ( !
str.empty () )
1207 WAR <<
"#" << worker->
_workerno <<
": has no multibyte handler, this is a bug" << endl;
1208 setWorkerBroken(
"Multibyte handler error");
1217 WAR <<
"#" << worker->
_workerno <<
": still has work to do or can recover from a error, continuing the job!" << endl;
1228 if ( cc != CURLE_OK ) {
1234 WAR <<
"#" << worker->
_workerno <<
": failed, but was set to discard, reusing for new requests" << endl;
1253 bool done = std::all_of( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), [](
const Stripe::RState s ) { return s == Stripe::FINALIZED; } );
1256 std::for_each( wrkerStripe.blockStates.begin(), wrkerStripe.blockStates.begin(), [](
Stripe::RState &s ) {
1257 if ( s != Stripe::FINALIZED)
1258 s = Stripe::PENDING;
1269 int maxworkerno = 0;
1271 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1286 double ratio = worker->
_avgspeed / maxavg;
1289 ratio = ratio * ratio;
1292 XXX <<
"#" << worker->
_workerno <<
": too slow ("<< ratio <<
", " << worker->
_avgspeed <<
", #" << maxworkerno <<
": " << maxavg <<
"), going to sleep for " << ratio * 1000 <<
" ms" << endl;
1313#if CURLVERSION_AT_LEAST(7,15,5)
1314 curl_easy_setopt(worker->
_curl, CURLOPT_MAX_RECV_SPEED_LARGE, (curl_off_t)(avg));
1346 WAR <<
"overall result" << endl;
1347 for (
auto workeriter =
_workers.begin(); workeriter !=
_workers.end(); ++workeriter)
1367 MIL <<
"MediaMultiCurl::MediaMultiCurl(" << url_r <<
", " << attach_point_hint_r <<
")" << endl;
1381 curl_multi_cleanup(
_multi);
1384 std::map<std::string, CURL *>::iterator it;
1387 CURL *easy = it->second;
1390 curl_easy_cleanup(easy);
1406 for (; sl; sl = sl->next)
1424 if ( curl_easy_getinfo(
_curl, CURLINFO_PRIVATE, &fp ) != CURLE_OK || !fp )
1426 if ( ftell( fp ) == 0 )
1431 long httpReturnCode = 0;
1432 if (curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0)
1436 bool ismetalink =
false;
1437 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1439 std::string ct = std::string(ptr);
1440 if (ct.find(
"application/x-zsync") == 0 || ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1443 if (!ismetalink && dlnow < 256)
1452 DBG <<
"looks_like_meta_file: " << ismetalink << endl;
1469 if( assert_dir( dest.
dirname() ) )
1471 DBG <<
"assert_dir " << dest.
dirname() <<
" failed" << endl;
1481 ERR <<
"out of memory for temp file name" << endl;
1485 AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1488 ERR <<
"mkstemp failed for file '" << destNew <<
"'" << endl;
1493 file = ::fdopen( tmp_fd,
"we" );
1496 ERR <<
"fopen failed for file '" << destNew <<
"'" << endl;
1499 tmp_fd.resetDispose();
1502 DBG <<
"dest: " << dest << endl;
1503 DBG <<
"temp: " << destNew << endl;
1508 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1509 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, (
long)
PathInfo(target).mtime());
1513 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1514 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1520 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (*file) );
1527 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1528 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1530 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1533 curl_easy_setopt(
_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1534 curl_easy_setopt(
_curl, CURLOPT_TIMEVALUE, 0L);
1536 curl_easy_setopt(
_curl, CURLOPT_PRIVATE, (
void *)0);
1537 long httpReturnCode = 0;
1538 CURLcode infoRet = curl_easy_getinfo(
_curl, CURLINFO_RESPONSE_CODE, &httpReturnCode);
1539 if (infoRet == CURLE_OK)
1542 if ( httpReturnCode == 304
1545 DBG <<
"not modified: " <<
PathInfo(dest) << endl;
1551 WAR <<
"Could not get the response code." << endl;
1557 if (curl_easy_getinfo(
_curl, CURLINFO_CONTENT_TYPE, &ptr) == CURLE_OK && ptr)
1559 std::string ct = std::string(ptr);
1560 if (ct.find(
"application/x-zsync") == 0 )
1562 else if (ct.find(
"application/metalink+xml") == 0 || ct.find(
"application/metalink4+xml") == 0)
1576 bool userabort =
false;
1582 std::vector<Url> urls;
1585 parser.
parse( destNew );
1602 XXX <<
"With no blocks" << std::endl;
1609 XXX <<
"No filesize in metalink file and no expected filesize, aborting multicurl." << std::endl;
1614 Disabling
this workaround
for now, since we now
do zip ranges into bigger requests
1625 file = fopen((*destNew).c_str(),
"w+e");
1630 XXX <<
"reusing blocks from file " << target << endl;
1636 XXX <<
"reusing blocks from file " << failedFile << endl;
1644 XXX <<
"reusing blocks from file " << df << endl;
1654 userabort = ex.
errstr() ==
"User abort";
1665 WAR<<
"Failed to multifetch file " << ex <<
" falling back to single Curl download!" << std::endl;
1666 if (
PathInfo(destNew).size() >= 63336)
1668 ::unlink(failedFile.
asString().c_str());
1675 file = fopen((*destNew).c_str(),
"w+e");
1687 ERR <<
"Failed to chmod file " << destNew << endl;
1694 ERR <<
"Fclose failed for file '" << destNew <<
"'" << endl;
1698 if ( rename( destNew, dest ) != 0 )
1700 ERR <<
"Rename failed" << endl;
1703 destNew.resetDispose();
1711 if (filesize == off_t(-1) && blklist.haveFilesize())
1712 filesize = blklist.getFilesize();
1713 if (!blklist.haveBlocks() && filesize != 0) {
1714 if ( filesize == -1 ) {
1719 MIL <<
"Generate blocklist, since there was none in the metalink file." << std::endl;
1724 while ( currOff < filesize ) {
1726 auto blksize = filesize - currOff ;
1727 if ( blksize > prefSize )
1730 blklist.addBlock( currOff, blksize );
1734 XXX <<
"Generated blocklist: " << std::endl << blklist << std::endl <<
" End blocklist " << std::endl;
1737 if (filesize == 0 || !blklist.numBlocks()) {
1746 _multi = curl_multi_init();
1752 std::vector<Url> myurllist;
1753 for (std::vector<Url>::iterator urliter = urllist->begin(); urliter != urllist->end(); ++urliter)
1757 std::string scheme = urliter->getScheme();
1758 if (scheme ==
"http" || scheme ==
"https" || scheme ==
"ftp" || scheme ==
"tftp")
1768 if (!myurllist.size())
1769 myurllist.push_back(baseurl);
1778 if (fseeko(fp, off_t(0), SEEK_SET))
1784 while ((l = fread(buf, 1,
sizeof(buf), fp)) > 0)
1792 return _dnsok.find(host) ==
_dnsok.end() ? false :
true;
1814 curl_easy_cleanup(oldeasy);
void resetDispose()
Set no dispose function.
Store and operate with byte count.
static const Unit MB
1000^2 Byte
SizeType blocks(ByteCount blocksize_r=K) const
Return number of blocks of size blocksize_r (default 1K).
static const Unit K
1024 Byte
std::string asString(unsigned field_width_r=0, unsigned unit_width_r=1) const
Auto selected Unit and precision.
Compute Message Digests (MD5, SHA1 etc)
bool update(const char *bytes, size_t len)
feed data into digest computation algorithm
Digest clone() const
Returns a clone of the current Digest and returns it.
Base class for Exception.
std::string getScheme() const
Returns the scheme name of the URL.
std::string asString() const
Returns a default string representation of the Url object.
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Pathname repoCachePath() const
Path where the caches are kept (/var/cache/zypp)
static ZConfig & instance()
Singleton ctor.
Wrapper class for stat/lstat.
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Pathname dirname() const
Return all but the last component od this path.
const std::string & asString() const
String representation.
Pathname absolutename() const
Return this path, adding a leading '/' if relative.
The CurlMultiPartHandler class.
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
String related utilities and Regular expression matching.
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
int hardlinkCopy(const Pathname &oldpath, const Pathname &newpath)
Create newpath as hardlink or copy of oldpath.
int unlink(const Pathname &path)
Like 'unlink'.
std::string numstring(char n, int w=0)
int zypp_poll(std::vector< GPollFD > &fds, int timeout)
Small wrapper around g_poll that additionally listens to the shutdown FD returned by ZYpp::shutdownSi...
Easy-to use interface to the ZYPP dependency resolver.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
auto eintrSafeCall(Fun &&function, Args &&... args)
CURLMcode handleSocketActions(const std::vector< GPollFD > &actionsFds, int first=0)
std::vector< GPollFD > socks
std::optional< long > timeout_ms
AutoDispose<int> calling close
AutoDispose<FILE*> calling fclose
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.