libzypp  17.31.13
request.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
13 #include <zypp-core/zyppng/base/EventDispatcher>
14 #include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
15 #include <zypp-core/zyppng/core/String>
17 #include <zypp-curl/CurlConfig>
18 #include <zypp-curl/auth/CurlAuthData>
19 #include <zypp-media/MediaConfig>
20 #include <zypp-core/base/String.h>
21 #include <zypp-core/base/StringV.h>
22 #include <zypp-core/Pathname.h>
23 #include <curl/curl.h>
24 #include <stdio.h>
25 #include <fcntl.h>
26 #include <sstream>
27 #include <utility>
28 
29 #include <iostream>
30 #include <boost/variant.hpp>
31 #include <boost/variant/polymorphic_get.hpp>
32 
33 
34 namespace zyppng {
35 
36  namespace {
37  static size_t nwr_headerCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
38  if ( !userdata )
39  return 0;
40 
41  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
42  return that->headerCallback( ptr, size, nmemb );
43  }
44  static size_t nwr_writeCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
45  if ( !userdata )
46  return 0;
47 
48  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
49  return that->writeCallback( ptr, size, nmemb );
50  }
51 
52  //helper for std::visit
53  template<class T> struct always_false : std::false_type {};
54  }
55 
56  std::vector<char> peek_data_fd( FILE *fd, off_t offset, size_t count )
57  {
58  if ( !fd )
59  return {};
60 
61  fflush( fd );
62 
63  std::vector<char> data( count + 1 , '\0' );
64 
65  ssize_t l = -1;
66  while ((l = pread( fileno( fd ), data.data(), count, offset ) ) == -1 && errno == EINTR)
67  ;
68  if (l == -1)
69  return {};
70 
71  return data;
72  }
73 
74  NetworkRequest::Range NetworkRequest::Range::make(size_t start, size_t len, zyppng::NetworkRequest::DigestPtr &&digest, zyppng::NetworkRequest::CheckSumBytes &&expectedChkSum, std::any &&userData, std::optional<size_t> digestCompareLen, std::optional<size_t> dataBlockPadding )
75  {
76  return NetworkRequest::Range {
77  .start = start,
78  .len = len,
79  .bytesWritten = 0,
80  ._digest = std::move( digest ),
81  ._checksum = std::move( expectedChkSum ),
82  ._relevantDigestLen = std::move( digestCompareLen ),
83  ._chksumPad = std::move( dataBlockPadding ),
84  .userData = std::move( userData ),
85  ._rangeState = State::Pending
86  };
87  }
88 
90  : _outFile( std::move(prevState._outFile) )
91  , _downloaded( prevState._downloaded )
92  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
93  { }
94 
96  : _requireStatusPartial( prevState._requireStatusPartial )
97  { }
98 
100  : _outFile( std::move(prevState._outFile) )
101  , _requireStatusPartial( true )
102  , _downloaded( prevState._downloaded )
103  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
104  { }
105 
107  : BasePrivate(p)
108  , _url ( std::move(url) )
109  , _targetFile ( std::move( targetFile) )
110  , _fMode ( std::move(fMode) )
111  , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
112  { }
113 
115  {
116  if ( _easyHandle ) {
117  //clean up for now, later we might reuse handles
118  curl_easy_cleanup( _easyHandle );
119  //reset in request but make sure the request was not enqueued again and got a new handle
120  _easyHandle = nullptr;
121  }
122  }
123 
124  bool NetworkRequestPrivate::initialize( std::string &errBuf )
125  {
126  reset();
127 
128  if ( _easyHandle )
129  //will reset to defaults but keep live connections, session ID and DNS caches
130  curl_easy_reset( _easyHandle );
131  else
132  _easyHandle = curl_easy_init();
133  return setupHandle ( errBuf );
134  }
135 
136  bool NetworkRequestPrivate::setupHandle( std::string &errBuf )
137  {
139  curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
140 
141  const std::string urlScheme = _url.getScheme();
142  if ( urlScheme == "http" || urlScheme == "https" )
144 
145  try {
146 
147  setCurlOption( CURLOPT_PRIVATE, this );
148  setCurlOption( CURLOPT_XFERINFOFUNCTION, NetworkRequestPrivate::curlProgressCallback );
149  setCurlOption( CURLOPT_XFERINFODATA, this );
150  setCurlOption( CURLOPT_NOPROGRESS, 0L);
151  setCurlOption( CURLOPT_FAILONERROR, 1L);
152  setCurlOption( CURLOPT_NOSIGNAL, 1L);
153 
154  std::string urlBuffer( _url.asString() );
155  setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
156 
157  setCurlOption( CURLOPT_WRITEFUNCTION, nwr_writeCallback );
158  setCurlOption( CURLOPT_WRITEDATA, this );
159 
161  setCurlOption( CURLOPT_CONNECT_ONLY, 1L );
162  setCurlOption( CURLOPT_FRESH_CONNECT, 1L );
163  }
165  // instead of returning no data with NOBODY, we return
166  // little data, that works with broken servers, and
167  // works for ftp as well, because retrieving only headers
168  // ftp will return always OK code ?
169  // See http://curl.haxx.se/docs/knownbugs.html #58
171  setCurlOption( CURLOPT_NOBODY, 1L );
172  else
173  setCurlOption( CURLOPT_RANGE, "0-1" );
174  }
175 
177  if ( _requestedRanges.size() ) {
178  if ( ! prepareNextRangeBatch ( errBuf ))
179  return false;
180  } else {
181  std::visit( [&]( auto &arg ){
182  using T = std::decay_t<decltype(arg)>;
183  if constexpr ( std::is_same_v<T, pending_t> ) {
184  arg._requireStatusPartial = false;
185  } else {
186  DBG << _easyHandle << " " << "NetworkRequestPrivate::setupHandle called in unexpected state" << std::endl;
187  }
188  }, _runningMode );
190  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
191  }
192  }
193 
194  //make a local copy of the settings, so headers are not added multiple times
195  TransferSettings locSet = _settings;
196 
197  if ( _dispatcher ) {
198  locSet.setUserAgentString( _dispatcher->agentString().c_str() );
199 
200  // add custom headers as configured (bsc#955801)
201  const auto &cHeaders = _dispatcher->hostSpecificHeaders();
202  if ( auto i = cHeaders.find(_url.getHost()); i != cHeaders.end() ) {
203  for ( const auto &[key, value] : i->second ) {
205  "%s: %s", key.c_str(), value.c_str() )
206  ));
207  }
208  }
209  }
210 
211  locSet.addHeader("Pragma:");
212 
215  {
216  case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
217  case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
218  default: break;
219  }
220 
221  setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
222  setCurlOption( CURLOPT_HEADERDATA, this );
223 
227  setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
228  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
229  // just in case curl does not trigger its progress callback frequently
230  // enough.
231  if ( locSet.timeout() )
232  {
233  setCurlOption( CURLOPT_TIMEOUT, 3600L );
234  }
235 
236  if ( urlScheme == "https" )
237  {
238 #if CURLVERSION_AT_LEAST(7,19,4)
239  // restrict following of redirections from https to https only
240  if ( _url.getHost() == "download.opensuse.org" )
241  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
242  else
243  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
244 #endif
245 
246  if( locSet.verifyPeerEnabled() ||
247  locSet.verifyHostEnabled() )
248  {
249  setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
250  }
251 
252  if( ! locSet.clientCertificatePath().empty() )
253  {
254  setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
255  }
256  if( ! locSet.clientKeyPath().empty() )
257  {
258  setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
259  }
260 
261 #ifdef CURLSSLOPT_ALLOW_BEAST
262  // see bnc#779177
263  setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
264 #endif
265  setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
266  setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
267  // bnc#903405 - POODLE: libzypp should only talk TLS
268  setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
269  }
270 
271  // follow any Location: header that the server sends as part of
272  // an HTTP header (#113275)
273  setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
274  // 3 redirects seem to be too few in some cases (bnc #465532)
275  setCurlOption( CURLOPT_MAXREDIRS, 6L );
276 
277  //set the user agent
278  setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
279 
280 
281  /*---------------------------------------------------------------*
282  CURLOPT_USERPWD: [user name]:[password]
283  Url::username/password -> CURLOPT_USERPWD
284  If not provided, anonymous FTP identification
285  *---------------------------------------------------------------*/
286  if ( locSet.userPassword().size() )
287  {
288  setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
289  std::string use_auth = _settings.authType();
290  if (use_auth.empty())
291  use_auth = "digest,basic"; // our default
292  long auth = zypp::media::CurlAuthData::auth_type_str2long(use_auth);
293  if( auth != CURLAUTH_NONE)
294  {
295  DBG << _easyHandle << " " << "Enabling HTTP authentication methods: " << use_auth
296  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
297  setCurlOption(CURLOPT_HTTPAUTH, auth);
298  }
299  }
300 
301  if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
302  {
303  DBG << _easyHandle << " " << "Proxy: '" << locSet.proxy() << "'" << std::endl;
304  setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
305  setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
306 
307  /*---------------------------------------------------------------*
308  * CURLOPT_PROXYUSERPWD: [user name]:[password]
309  *
310  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
311  * If not provided, $HOME/.curlrc is evaluated
312  *---------------------------------------------------------------*/
313 
314  std::string proxyuserpwd = locSet.proxyUserPassword();
315 
316  if ( proxyuserpwd.empty() )
317  {
318  zypp::media::CurlConfig curlconf;
319  zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
320  if ( curlconf.proxyuserpwd.empty() )
321  DBG << _easyHandle << " " << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
322  else
323  {
324  proxyuserpwd = curlconf.proxyuserpwd;
325  DBG << _easyHandle << " " << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
326  }
327  }
328  else
329  {
330  DBG << _easyHandle << " " << _easyHandle << " " << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
331  }
332 
333  if ( ! proxyuserpwd.empty() )
334  {
335  setCurlOption(CURLOPT_PROXYUSERPWD, ::internal::curlUnEscape( proxyuserpwd ).c_str());
336  }
337  }
338 #if CURLVERSION_AT_LEAST(7,19,4)
339  else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
340  {
341  // Explicitly disabled in URL (see fillSettingsFromUrl()).
342  // This should also prevent libcurl from looking into the environment.
343  DBG << _easyHandle << " " << "Proxy: explicitly NOPROXY" << std::endl;
344  setCurlOption(CURLOPT_NOPROXY, "*");
345  }
346 
347 #endif
348  // else: Proxy: not explicitly set; libcurl may look into the environment
349 
351  if ( locSet.minDownloadSpeed() != 0 )
352  {
353  setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
354  // default to 10 seconds at low speed
355  setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
356  }
357 
358 #if CURLVERSION_AT_LEAST(7,15,5)
359  if ( locSet.maxDownloadSpeed() != 0 )
360  setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
361 #endif
362 
363  if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
364  setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
365  else
366  MIL << _easyHandle << " " << "No cookies requested" << std::endl;
367  setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
368 
369 #if CURLVERSION_AT_LEAST(7,18,0)
370  // bnc #306272
371  setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
372 #endif
373 
374  // append settings custom headers to curl
375  for ( const auto &header : locSet.headers() ) {
376  if ( !z_func()->addRequestHeader( header.c_str() ) )
378  }
379 
380  if ( _headers )
381  setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
382 
383  return true;
384 
385  } catch ( const zypp::Exception &excp ) {
386  ZYPP_CAUGHT(excp);
387  errBuf = excp.asString();
388  }
389  return false;
390  }
391 
393  {
394  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
395  if ( !rmode ) {
396  DBG << _easyHandle << "Can only create output file in running mode" << std::endl;
397  return false;
398  }
399  // if we have no open file create or open it
400  if ( !rmode->_outFile ) {
401  std::string openMode = "w+b";
403  openMode = "r+b";
404 
405  rmode->_outFile = fopen( _targetFile.asString().c_str() , openMode.c_str() );
406 
407  //if the file does not exist create a new one
408  if ( !rmode->_outFile && _fMode == NetworkRequest::WriteShared ) {
409  rmode->_outFile = fopen( _targetFile.asString().c_str() , "w+b" );
410  }
411 
412  if ( !rmode->_outFile ) {
414  ,zypp::str::Format("Unable to open target file (%1%). Errno: (%2%:%3%)") % _targetFile.asString() % errno % strerr_cxx() );
415  return false;
416  }
417  }
418 
419  return true;
420  }
421 
423  {
424  // We can recover from RangeFail errors if we have more batch sizes to try
425  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
426  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail )
427  return ( rmode->_rangeAttemptIdx + 1 < sizeof( _rangeAttempt ) ) && hasMoreWork();
428  return false;
429  }
430 
431  bool NetworkRequestPrivate::prepareToContinue( std::string &errBuf )
432  {
433  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
434 
435  if ( hasMoreWork() ) {
436  // go to the next range batch level if we are restarted due to a failed range request
437  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail ) {
438  if ( rmode->_rangeAttemptIdx + 1 >= sizeof( _rangeAttempt ) ) {
439  errBuf = "No more range batch sizes available";
440  return false;
441  }
442  rmode->_rangeAttemptIdx++;
443  }
444 
445  _runningMode = prepareNextRangeBatch_t( std::move(std::get<running_t>( _runningMode )) );
446 
447  // we reset the handle to default values. We do this to not run into
448  // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
449  // we cancel a connection because of a range error to request a smaller batch.
450  // The error will still happen but much less frequently than without resetting the handle.
451  //
452  // Note: Even creating a new handle will NOT fix the issue
453  curl_easy_reset( _easyHandle );
454  if ( !setupHandle (errBuf) )
455  return false;
456  return true;
457  }
458  errBuf = "Request has no more work";
459  return false;
460 
461  }
462 
464  {
465  if ( _requestedRanges.size() == 0 ) {
466  errBuf = "Calling the prepareNextRangeBatch function without a range to download is not supported.";
467  return false;
468  }
469 
470  std::string rangeDesc;
471  uint rangesAdded = 0;
472  if ( _requestedRanges.size() > 1 && _protocolMode != ProtocolMode::HTTP ) {
473  errBuf = "Using more than one range is not supported with protocols other than HTTP/HTTPS";
474  return false;
475  }
476 
477  // check if we have one big range convering the whole file
478  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 ) {
479  if ( !std::holds_alternative<pending_t>( _runningMode ) ) {
480  errBuf = zypp::str::Str() << "Unexpected state when calling prepareNextRangeBatch " << _runningMode.index ();
481  return false;
482  }
483 
484  _requestedRanges[0]._rangeState = NetworkRequest::Running;
485  std::get<pending_t>( _runningMode )._requireStatusPartial = false;
486 
487  } else {
488  std::sort( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &elem1, const auto &elem2 ){
489  return ( elem1.start < elem2.start );
490  });
491 
492  if ( std::holds_alternative<pending_t>( _runningMode ) )
493  std::get<pending_t>( _runningMode )._requireStatusPartial = true;
494 
495  auto maxRanges = _rangeAttempt[0];
496  if ( std::holds_alternative<prepareNextRangeBatch_t>( _runningMode ) )
497  maxRanges = _rangeAttempt[std::get<prepareNextRangeBatch_t>( _runningMode )._rangeAttemptIdx];
498 
499  // helper function to build up the request string for the range
500  auto addRangeString = [ &rangeDesc, &rangesAdded ]( const std::pair<size_t, size_t> &range ) {
501  std::string rangeD = zypp::str::form("%llu-", static_cast<unsigned long long>( range.first ) );
502  if( range.second > 0 )
503  rangeD.append( zypp::str::form( "%llu", static_cast<unsigned long long>( range.second ) ) );
504 
505  if ( rangeDesc.size() )
506  rangeDesc.append(",").append( rangeD );
507  else
508  rangeDesc = std::move( rangeD );
509 
510  rangesAdded++;
511  };
512 
513  std::optional<std::pair<size_t, size_t>> currentZippedRange;
514  bool closedRange = true;
515  for ( auto &range : _requestedRanges ) {
516 
517  if ( range._rangeState != NetworkRequest::Pending )
518  continue;
519 
520  //reset the download results
521  range.bytesWritten = 0;
522 
523  //when we have a open range in the list of ranges we will get from start of range to end of file,
524  //all following ranges would never be marked as valid, so we have to fail early
525  if ( !closedRange ) {
526  errBuf = "It is not supported to request more ranges after a open range.";
527  return false;
528  }
529 
530  const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
531  closedRange = (rangeEnd > 0);
532 
533  // remember this range was already requested
534  range._rangeState = NetworkRequest::Running;
535  range.bytesWritten = 0;
536  if ( range._digest )
537  range._digest->reset();
538 
539  // we try to compress the requested ranges into as big chunks as possible for the request,
540  // when receiving we still track the original ranges so we can collect and test their checksums
541  if ( !currentZippedRange ) {
542  currentZippedRange = std::make_pair( range.start, rangeEnd );
543  } else {
544  //range is directly consecutive to the previous range
545  if ( currentZippedRange->second + 1 == range.start ) {
546  currentZippedRange->second = rangeEnd;
547  } else {
548  //this range does not directly follow the previous one, we build the string and start a new one
549  addRangeString( *currentZippedRange );
550  currentZippedRange = std::make_pair( range.start, rangeEnd );
551  }
552  }
553 
554  if ( rangesAdded >= maxRanges ) {
555  MIL << _easyHandle << " " << "Reached max nr of ranges (" << maxRanges << "), batching the request to not break the server" << std::endl;
556  break;
557  }
558  }
559 
560  // add the last range too
561  if ( currentZippedRange )
562  addRangeString( *currentZippedRange );
563 
564  MIL << _easyHandle << " " << "Requesting Ranges: " << rangeDesc << std::endl;
565 
566  setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
567  }
568 
569  return true;
570  }
571 
573  {
574  // check if we have ranges that have never been requested
575  return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == NetworkRequest::Pending; });
576  }
577 
579  {
580  bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( _runningMode );
581  if ( isRangeContinuation ) {
582  MIL << _easyHandle << " " << "Continuing a previously started range batch." << std::endl;
583  _runningMode = running_t( std::move(std::get<prepareNextRangeBatch_t>( _runningMode )) );
584  } else {
585  auto mode = running_t( std::move(std::get<pending_t>( _runningMode )) );
586  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 )
587  mode._currentRange = 0;
588 
589  _runningMode = std::move(mode);
590  }
591 
592  auto &m = std::get<running_t>( _runningMode );
593 
594  if ( m._activityTimer ) {
595  DBG_MEDIA << _easyHandle << " Setting activity timeout to: " << _settings.timeout() << std::endl;
596  m._activityTimer->connect( &Timer::sigExpired, *this, &NetworkRequestPrivate::onActivityTimeout );
597  m._activityTimer->start( static_cast<uint64_t>( _settings.timeout() * 1000 ) );
598  }
599 
600  if ( !isRangeContinuation )
601  _sigStarted.emit( *z_func() );
602  }
603 
605  {
606  if ( std::holds_alternative<running_t>(_runningMode) ) {
607  auto &rmode = std::get<running_t>( _runningMode );
608  // if we still have a current range set it valid by checking the checksum
609  if ( rmode._currentRange >= 0 ) {
610  auto &currR = _requestedRanges[rmode._currentRange];
611  rmode._currentRange = -1;
612  validateRange( currR );
613  }
614  }
615  }
616 
618  {
619 
620  finished_t resState;
621  resState._result = std::move(err);
622 
623  if ( std::holds_alternative<running_t>(_runningMode) ) {
624 
625  auto &rmode = std::get<running_t>( _runningMode );
626  rmode._outFile.reset();
627  resState._downloaded = rmode._downloaded;
628  resState._contentLenght = rmode._contentLenght;
629 
631  //we have a successful download lets see if we got everything we needed
632  for ( const auto &r : _requestedRanges ) {
633  if ( r._rangeState != NetworkRequest::Finished ) {
634  if ( r.len > 0 && r.bytesWritten != r.len )
635  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::MissingData, (zypp::str::Format("Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
636  else if ( r._digest && r._checksum.size() && ! checkIfRangeChkSumIsValid(r) ) {
637  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % r._digest->digest() % zypp::Digest::digestVectorToString( r._checksum ) ) );
638  } else {
640  }
641  //we only report the first error
642  break;
643  }
644  }
645  }
646  }
647 
648  _runningMode = std::move( resState );
649  _sigFinished.emit( *z_func(), std::get<finished_t>(_runningMode)._result );
650  }
651 
653  {
655  _headers.reset( nullptr );
656  _errorBuf.fill( 0 );
658  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
659  range._rangeState = NetworkRequest::Pending;
660  });
661  }
662 
664  {
665  auto &m = std::get<running_t>( _runningMode );
666 
667  MIL_MEDIA << _easyHandle << " Request timeout interval: " << t.interval()<< " remaining: " << t.remaining() << std::endl;
668  std::map<std::string, boost::any> extraInfo;
669  extraInfo.insert( {"requestUrl", _url } );
670  extraInfo.insert( {"filepath", _targetFile } );
671  _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
672  }
673 
675  {
676  if ( rng._digest && rng._checksum.size() ) {
677  auto bytesHashed = rng._digest->bytesHashed ();
678  if ( rng._chksumPad && *rng._chksumPad > bytesHashed ) {
679  MIL_MEDIA << _easyHandle << " " << "Padding the digest to required block size" << std::endl;
680  zypp::ByteArray padding( *rng._chksumPad - bytesHashed, '\0' );
681  rng._digest->update( padding.data(), padding.size() );
682  }
683  auto digVec = rng._digest->digestVector();
684  if ( rng._relevantDigestLen ) {
685  digVec.resize( *rng._relevantDigestLen );
686  }
687  return ( digVec == rng._checksum );
688  }
689 
690  // no checksum required
691  return true;
692  }
693 
695  {
696  if ( rng._digest && rng._checksum.size() ) {
697  if ( ( rng.len == 0 || rng.bytesWritten == rng.len ) && checkIfRangeChkSumIsValid(rng) )
699  else
701  } else {
702  if ( rng.len == 0 ? true : rng.bytesWritten == rng.len )
704  else
706  }
707  }
708 
709  bool NetworkRequestPrivate::parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len )
710  { //content-range: bytes 10485760-19147879/19147880
711  static const zypp::str::regex regex("^Content-Range:[[:space:]]+bytes[[:space:]]+([0-9]+)-([0-9]+)\\/([0-9]+)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
712 
713  zypp::str::smatch what;
714  if( !zypp::str::regex_match( std::string(line), what, regex ) || what.size() != 4 ) {
715  DBG << _easyHandle << " " << "Invalid Content-Range Header format: '" << std::string(line) << std::endl;
716  return false;
717  }
718 
719  size_t s = zypp::str::strtonum<size_t>( what[1]);
720  size_t e = zypp::str::strtonum<size_t>( what[2]);
721  start = std::move(s);
722  len = ( e - s ) + 1;
723  return true;
724  }
725 
726  bool NetworkRequestPrivate::parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
727  {
728  static const zypp::str::regex regex("^Content-Type:[[:space:]]+multipart\\/byteranges;[[:space:]]+boundary=(.*)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
729 
730  zypp::str::smatch what;
731  if( zypp::str::regex_match( std::string(line), what, regex ) ) {
732  if ( what.size() >= 2 ) {
733  boundary = what[1];
734  return true;
735  }
736  }
737  return false;
738  }
739 
741  {
742  return std::string( _errorBuf.data() );
743  }
744 
746  {
747  if ( std::holds_alternative<running_t>( _runningMode ) ){
748  auto &rmode = std::get<running_t>( _runningMode );
749  if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
750  rmode._activityTimer->start();
751  }
752  }
753 
754  int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
755  {
756  if ( !clientp )
757  return CURLE_OK;
758  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
759 
760  if ( !std::holds_alternative<running_t>(that->_runningMode) ){
761  DBG << that->_easyHandle << " " << "Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
762  return -1;
763  }
764 
765  auto &rmode = std::get<running_t>( that->_runningMode );
766 
767  //reset the timer
768  that->resetActivityTimer();
769 
770  rmode._isInCallback = true;
771  if ( rmode._lastProgressNow != dlnow ) {
772  rmode._lastProgressNow = dlnow;
773  that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
774  }
775  rmode._isInCallback = false;
776 
777  return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
778  }
779 
780  size_t NetworkRequestPrivate::headerCallback(char *ptr, size_t size, size_t nmemb)
781  {
782  //it is valid to call this function with no data to write, just return OK
783  if ( size * nmemb == 0)
784  return 0;
785 
787 
789 
790  std::string_view hdr( ptr, size*nmemb );
791 
792  hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
793  const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
794  if ( lastNonWhitespace != hdr.npos )
795  hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
796  else
797  hdr = std::string_view();
798 
799  auto &rmode = std::get<running_t>( _runningMode );
800  if ( !hdr.size() ) {
801  return ( size * nmemb );
802  }
803  if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
804 
805  long statuscode = 0;
806  (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
807 
808  const auto &doRangeFail = [&](){
809  WAR << _easyHandle << " " << "Range FAIL, trying with a smaller batch" << std::endl;
810  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::RangeFail, "Expected range status code 206, but got none." );
811 
812  // reset all ranges we requested to pending, we never got the data for them
813  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
814  if ( range._rangeState == NetworkRequest::Running )
815  range._rangeState = NetworkRequest::Pending;
816  });
817  return 0;
818  };
819 
820  // if we have a status 204 we need to create a empty file
821  if( statuscode == 204 && !( _options & NetworkRequest::ConnectionTest ) && !( _options & NetworkRequest::HeadRequest ) )
823 
824  if ( rmode._requireStatusPartial ) {
825  // ignore other status codes, maybe we are redirected etc.
826  if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
827  || statuscode == 416 ) {
828  return doRangeFail();
829  }
830  }
831 
832  } else if ( zypp::strv::hasPrefixCI( hdr, "Location:" ) ) {
833  _lastRedirect = hdr.substr( 9 );
834  DBG << _easyHandle << " " << "redirecting to " << _lastRedirect << std::endl;
835 
836  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Type:") ) {
837  std::string sep;
838  if ( parseContentTypeMultiRangeHeader( hdr, sep ) ) {
839  rmode._gotMultiRangeHeader = true;
840  rmode._seperatorString = "--"+sep;
841  }
842  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Range:") ) {
844  if ( !parseContentRangeHeader( hdr, r.start, r.len) ) {
845  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
846  return 0;
847  }
848  DBG << _easyHandle << " " << "Got content range :" << r.start << " len " << r.len << std::endl;
849  rmode._gotContentRangeHeader = true;
850  rmode._currentSrvRange = r;
851 
852  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Length:") ) {
853  auto lenStr = str::trim( hdr.substr( 15 ), zypp::str::TRIM );
854  auto str = std::string ( lenStr.data(), lenStr.length() );
855  auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>( str.data() );
856  if ( len > 0 ) {
857  DBG << _easyHandle << " " << "Got Content-Length Header: " << len << std::endl;
858  rmode._contentLenght = zypp::ByteCount(len, zypp::ByteCount::B);
859  }
860  }
861  }
862 
863  return ( size * nmemb );
864  }
865 
866  size_t NetworkRequestPrivate::writeCallback(char *ptr, size_t size, size_t nmemb)
867  {
868  const auto max = ( size * nmemb );
869 
871 
872  //it is valid to call this function with no data to write, just return OK
873  if ( max == 0)
874  return 0;
875 
876  //in case of a HEAD request, we do not write anything
878  return ( size * nmemb );
879  }
880 
881  auto &rmode = std::get<running_t>( _runningMode );
882 
883  auto writeDataToFile = [ this, &rmode ]( off_t offset, const char *data, size_t len ) -> off_t {
884 
885  if ( rmode._currentRange < 0 ) {
886  DBG << _easyHandle << " " << "Current range is zero in write request" << std::endl;
887  return 0;
888  }
889 
890  // if we have no open file create or open it
891  if ( !assertOutputFile() )
892  return 0;
893 
894  // seek to the given offset
895  if ( offset >= 0 ) {
896  if ( fseek( rmode._outFile, offset, SEEK_SET ) != 0 ) {
898  "Unable to set output file pointer." );
899  return 0;
900  }
901  }
902 
903  auto &rng = _requestedRanges[ rmode._currentRange ];
904  const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, len ) : len;
905 
906  //make sure we do not write after the expected file size
907  if ( _expectedFileSize && _expectedFileSize <= static_cast<zypp::ByteCount::SizeType>(rng.start + rng.bytesWritten + bytesToWrite) ) {
908  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Downloaded data exceeds expected length." );
909  return 0;
910  }
911 
912  auto written = fwrite( data, 1, bytesToWrite, rmode._outFile );
913  if ( written == 0 )
914  return 0;
915 
916  if ( rng._digest && rng._checksum.size() ) {
917  if ( !rng._digest->update( data, written ) )
918  return 0;
919  }
920 
921  rng.bytesWritten += written;
922  if ( rmode._currentSrvRange ) rmode._currentSrvRange->bytesWritten += written;
923 
924  if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
925  rmode._currentRange = -1;
926  validateRange( rng );
927  }
928 
929  if ( rmode._currentSrvRange && rmode._currentSrvRange->len > 0 && rmode._currentSrvRange->bytesWritten >= rmode._currentSrvRange->len ) {
930  rmode._currentSrvRange.reset();
931  // we ran out of data in the current chunk, reset the target range as well because next data will be
932  // a chunk header again
933  rmode._currentRange = -1;
934  }
935 
936  // count the number of real bytes we have downloaded so far
937  rmode._downloaded += written;
938  _sigBytesDownloaded.emit( *z_func(), rmode._downloaded );
939 
940  return written;
941  };
942 
943  // we are currenty writing a range, continue until we hit the end of the requested chunk, or if we hit end of data
944  size_t bytesWrittenSoFar = 0;
945 
946  while ( bytesWrittenSoFar != max ) {
947 
948  off_t seekTo = -1;
949 
950  // this is called after all headers have been processed
951  if ( !rmode._allHeadersReceived ) {
952  rmode._allHeadersReceived = true;
953 
954  // no ranges at all, must be a normal download
955  if ( !rmode._gotMultiRangeHeader && !rmode._gotContentRangeHeader ) {
956 
957  if ( rmode._requireStatusPartial ) {
958  //we got a invalid response, the status code pointed to being partial but we got no range definition
960  "Invalid data from server, range respone was announced but there was no range definiton." );
961  return 0;
962  }
963 
964  //we always download a range even if it is not explicitly requested
965  if ( _requestedRanges.empty() ) {
967  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
968  }
969 
970  rmode._currentRange = 0;
971  seekTo = _requestedRanges[0].start;
972  }
973  }
974 
975  if ( rmode._currentSrvRange && rmode._currentRange == -1 ) {
976  //if we enter this branch, we just have finished writing a requested chunk but
977  //are still inside a chunk that was sent by the server, due to the std the server can coalesce requested ranges
978  //to optimize downloads we need to find the best match ( because the current offset might not even be in our requested ranges )
979  //Or we just parsed a Content-Lenght header and start a new block
980 
981  std::optional<uint> foundRange;
982  const size_t beginRange = rmode._currentSrvRange->start + rmode._currentSrvRange->bytesWritten;
983  const size_t endRange = beginRange + (rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten);
984  auto currDist = ULONG_MAX;
985  for ( uint i = 0; i < _requestedRanges.size(); i++ ) {
986  const auto &currR = _requestedRanges[i];
987 
988  // do not allow double ranges
989  if ( currR._rangeState == NetworkRequest::Finished || currR._rangeState == NetworkRequest::Error )
990  continue;
991 
992  // check if the range was already written
993  if ( currR.len == currR.bytesWritten )
994  continue;
995 
996  const auto currRBegin = currR.start + currR.bytesWritten;
997  if ( !( beginRange <= currRBegin && endRange >= currRBegin ) )
998  continue;
999 
1000  // calculate the distance of the current ranges offset+data written to the range we got back from the server
1001  const auto newDist = currRBegin - beginRange;
1002 
1003  if ( !foundRange ) {
1004  foundRange = i;
1005  currDist = newDist;
1006  } else {
1007  //pick the range with the closest distance
1008  if ( newDist < currDist ) {
1009  foundRange = i;
1010  currDist = newDist;
1011  }
1012  }
1013  }
1014  if ( !foundRange ) {
1016  , "Unable to find a matching range for data returned by the server." );
1017  return 0;
1018  }
1019 
1020  //set the found range as the current one
1021  rmode._currentRange = *foundRange;
1022 
1023  //continue writing where we stopped
1024  seekTo = _requestedRanges[*foundRange].start + _requestedRanges[*foundRange].bytesWritten;
1025 
1026  //if we skip bytes we need to advance our written bytecount
1027  const auto skipBytes = seekTo - beginRange;
1028  bytesWrittenSoFar += skipBytes;
1029  rmode._currentSrvRange->bytesWritten += skipBytes;
1030  }
1031 
1032  if ( rmode._currentRange >= 0 ) {
1033  auto availableData = max - bytesWrittenSoFar;
1034  if ( rmode._currentSrvRange ) {
1035  availableData = std::min( availableData, rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten );
1036  }
1037  auto bw = writeDataToFile( seekTo, ptr + bytesWrittenSoFar, availableData );
1038  if ( bw <= 0 )
1039  return 0;
1040 
1041  bytesWrittenSoFar += bw;
1042  }
1043 
1044  if ( bytesWrittenSoFar == max )
1045  return max;
1046 
1047  if ( rmode._currentRange == -1 ) {
1048 
1049  // we still are inside the current range from the server
1050  if ( rmode._currentSrvRange )
1051  continue;
1052 
1053  std::string_view incoming( ptr + bytesWrittenSoFar, max - bytesWrittenSoFar );
1054  auto hdrEnd = incoming.find("\r\n\r\n");
1055  if ( hdrEnd == incoming.npos ) {
1056  //no header end in the data yet, push to buffer and return
1057  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.end() );
1058  return max;
1059  }
1060 
1061  //append the data of the current header to the buffer and parse it
1062  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.begin() + ( hdrEnd + 4 ) );
1063  bytesWrittenSoFar += ( hdrEnd + 4 ); //header data plus header end
1064 
1065  std::string_view data( rmode._rangePrefaceBuffer.data(), rmode._rangePrefaceBuffer.size() );
1066  auto sepStrIndex = data.find( rmode._seperatorString );
1067  if ( sepStrIndex == data.npos ) {
1069  "Invalid multirange header format, seperator string missing." );
1070  return 0;
1071  }
1072 
1073  auto startOfHeader = sepStrIndex + rmode._seperatorString.length();
1074  std::vector<std::string_view> lines;
1075  zypp::strv::split( data.substr( startOfHeader ), "\r\n", zypp::strv::Trim::trim, [&]( std::string_view strv ) { lines.push_back(strv); } );
1076  for ( const auto &hdrLine : lines ) {
1077  if ( zypp::strv::hasPrefixCI(hdrLine, "Content-Range:") ) {
1079  //if we can not parse the header the message must be broken
1080  if(! parseContentRangeHeader( hdrLine, r.start, r.len ) ) {
1081  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
1082  return 0;
1083  }
1084  rmode._currentSrvRange = r;
1085  break;
1086  }
1087  }
1088  //clear the buffer again
1089  rmode._rangePrefaceBuffer.clear();
1090  }
1091  }
1092  return bytesWrittenSoFar;
1093  }
1094 
1096 
1097  NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, zyppng::NetworkRequest::FileMode fMode)
1098  : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(fMode), *this ) )
1099  {
1100  }
1101 
1103  {
1104  Z_D();
1105 
1106  if ( d->_dispatcher )
1107  d->_dispatcher->cancel( *this, "Request destroyed while still running" );
1108  }
1109 
1111  {
1112  d_func()->_expectedFileSize = std::move( expectedFileSize );
1113  }
1114 
1115  void NetworkRequest::setPriority( NetworkRequest::Priority prio, bool triggerReschedule )
1116  {
1117  Z_D();
1118  d->_priority = prio;
1119  if ( state() == Pending && triggerReschedule && d->_dispatcher )
1120  d->_dispatcher->reschedule();
1121  }
1122 
1124  {
1125  return d_func()->_priority;
1126  }
1127 
1128  void NetworkRequest::setOptions( Options opt )
1129  {
1130  d_func()->_options = opt;
1131  }
1132 
1133  NetworkRequest::Options NetworkRequest::options() const
1134  {
1135  return d_func()->_options;
1136  }
1137 
1138  void NetworkRequest::addRequestRange( size_t start, size_t len, DigestPtr digest, CheckSumBytes expectedChkSum , std::any userData, std::optional<size_t> digestCompareLen, std::optional<size_t> chksumpad )
1139  {
1140  Z_D();
1141  if ( state() == Running )
1142  return;
1143 
1144  d->_requestedRanges.push_back( Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
1145  }
1146 
1148  {
1149  Z_D();
1150  if ( state() == Running )
1151  return;
1152 
1153  d->_requestedRanges.push_back( range );
1154  auto &rng = d->_requestedRanges.back();
1155  rng._rangeState = NetworkRequest::Pending;
1156  rng.bytesWritten = 0;
1157  if ( rng._digest )
1158  rng._digest->reset();
1159  }
1160 
1162  {
1163  Z_D();
1164  if ( state() == Running )
1165  return;
1166  d->_requestedRanges.clear();
1167  }
1168 
1169  std::vector<NetworkRequest::Range> NetworkRequest::failedRanges() const
1170  {
1171  const auto mystate = state();
1172  if ( mystate != Finished && mystate != Error )
1173  return {};
1174 
1175  Z_D();
1176 
1177  std::vector<Range> failed;
1178  for ( const auto &r : d->_requestedRanges ) {
1179  if ( r._rangeState != NetworkRequest::Finished )
1180  failed.push_back( r );
1181  }
1182  return failed;
1183  }
1184 
1185  const std::vector<NetworkRequest::Range> &NetworkRequest::requestedRanges() const
1186  {
1187  return d_func()->_requestedRanges;
1188  }
1189 
1190  const std::string &NetworkRequest::lastRedirectInfo() const
1191  {
1192  return d_func()->_lastRedirect;
1193  }
1194 
1196  {
1197  return d_func()->_easyHandle;
1198  }
1199 
1200  std::optional<zyppng::NetworkRequest::Timings> NetworkRequest::timings() const
1201  {
1202  const auto myerr = error();
1203  const auto mystate = state();
1204  if ( mystate != Finished )
1205  return {};
1206 
1207  Timings t;
1208 
1209  auto getMeasurement = [ this ]( const CURLINFO info, std::chrono::microseconds &target ){
1210  using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
1211  double val = 0;
1212  const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
1213  if ( CURLE_OK == res ) {
1214  target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
1215  }
1216  };
1217 
1218  getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.namelookup );
1219  getMeasurement( CURLINFO_CONNECT_TIME, t.connect);
1220  getMeasurement( CURLINFO_APPCONNECT_TIME, t.appconnect);
1221  getMeasurement( CURLINFO_PRETRANSFER_TIME , t.pretransfer);
1222  getMeasurement( CURLINFO_TOTAL_TIME, t.total);
1223  getMeasurement( CURLINFO_REDIRECT_TIME, t.redirect);
1224 
1225  return t;
1226  }
1227 
1228  std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
1229  {
1230  Z_D();
1231 
1232  if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
1233  return {};
1234 
1235  const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
1236  return peek_data_fd( rmode._outFile, offset, count );
1237  }
1238 
1240  {
1241  return d_func()->_url;
1242  }
1243 
1244  void NetworkRequest::setUrl(const Url &url)
1245  {
1246  Z_D();
1247  if ( state() == NetworkRequest::Running )
1248  return;
1249 
1250  d->_url = url;
1251  }
1252 
1254  {
1255  return d_func()->_targetFile;
1256  }
1257 
1259  {
1260  Z_D();
1261  if ( state() == NetworkRequest::Running )
1262  return;
1263  d->_targetFile = path;
1264  }
1265 
1267  {
1268  return d_func()->_fMode;
1269  }
1270 
1272  {
1273  Z_D();
1274  if ( state() == NetworkRequest::Running )
1275  return;
1276  d->_fMode = std::move( mode );
1277  }
1278 
1279  std::string NetworkRequest::contentType() const
1280  {
1281  char *ptr = NULL;
1282  if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
1283  return std::string(ptr);
1284  return std::string();
1285  }
1286 
1288  {
1289  return std::visit([](auto& arg) -> zypp::ByteCount {
1290  using T = std::decay_t<decltype(arg)>;
1291  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1292  return zypp::ByteCount(0);
1293  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1294  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1295  return arg._contentLenght;
1296  else
1297  static_assert(always_false<T>::value, "Unhandled state type");
1298  }, d_func()->_runningMode);
1299  }
1300 
1302  {
1303  return std::visit([](auto& arg) -> zypp::ByteCount {
1304  using T = std::decay_t<decltype(arg)>;
1305  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1306  return zypp::ByteCount();
1307  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1308  || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
1309  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1310  return arg._downloaded;
1311  else
1312  static_assert(always_false<T>::value, "Unhandled state type");
1313  }, d_func()->_runningMode);
1314  }
1315 
1317  {
1318  return d_func()->_settings;
1319  }
1320 
1322  {
1323  return std::visit([this](auto& arg) {
1324  using T = std::decay_t<decltype(arg)>;
1325  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1326  return Pending;
1327  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1328  return Running;
1329  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
1330  if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
1331  return Error;
1332  else
1333  return Finished;
1334  }
1335  else
1336  static_assert(always_false<T>::value, "Unhandled state type");
1337  }, d_func()->_runningMode);
1338  }
1339 
1341  {
1342  const auto s = state();
1343  if ( s != Error && s != Finished )
1344  return NetworkRequestError();
1345  return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
1346  }
1347 
1349  {
1350  if ( !hasError() )
1351  return std::string();
1352 
1353  return error().nativeErrorString();
1354  }
1355 
1357  {
1358  return error().isError();
1359  }
1360 
1361  bool NetworkRequest::addRequestHeader( const std::string &header )
1362  {
1363  Z_D();
1364 
1365  curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
1366  if ( !res )
1367  return false;
1368 
1369  if ( !d->_headers )
1370  d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1371 
1372  return true;
1373  }
1374 
1375  SignalProxy<void (NetworkRequest &req)> NetworkRequest::sigStarted()
1376  {
1377  return d_func()->_sigStarted;
1378  }
1379 
1380  SignalProxy<void (NetworkRequest &req, zypp::ByteCount count)> NetworkRequest::sigBytesDownloaded()
1381  {
1382  return d_func()->_sigBytesDownloaded;
1383  }
1384 
1385  SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
1386  {
1387  return d_func()->_sigProgress;
1388  }
1389 
1390  SignalProxy<void (zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)> NetworkRequest::sigFinished()
1391  {
1392  return d_func()->_sigFinished;
1393  }
1394 
1395 }
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:132
long timeout() const
transfer timeout
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
std::string errorMessage() const
Definition: request.cc:740
bool isError() const
isError Will return true if this is a actual error
#define MIL
Definition: Logger.h:96
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:107
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:1200
void * nativeHandle() const
Definition: request.cc:1195
std::optional< size_t > _chksumPad
Definition: request.h:88
#define DBG_MEDIA
Definition: mediadebug_p.h:28
unsigned size() const
Definition: Regex.cc:106
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size...
Definition: request.cc:1287
const std::vector< Range > & requestedRanges() const
Definition: request.cc:1185
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::chrono::microseconds connect
Definition: request.h:98
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:104
void addRequestRange(size_t start, size_t len=0, DigestPtr digest=nullptr, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
Definition: request.cc:1138
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar"
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Regular expression.
Definition: Regex.h:94
ZYPP_IMPL_PRIVATE(Provide)
std::optional< size_t > _relevantDigestLen
Definition: request.h:87
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1380
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
Definition: String.h:1030
NetworkRequest::FileMode _fMode
Definition: request_p.h:122
bool checkIfRangeChkSumIsValid(const NetworkRequest::Range &rng)
Definition: request.cc:674
Store and operate with byte count.
Definition: ByteCount.h:30
const std::string & lastRedirectInfo() const
Definition: request.cc:1190
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
const std::string _currentCookieFile
Definition: request_p.h:126
std::chrono::microseconds pretransfer
Definition: request.h:100
Holds transfer setting.
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
Definition: request.cc:1301
const std::string & authType() const
get the allowed authentication types
NetworkRequest::Options _options
Definition: request_p.h:118
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
const char * c_str() const
String representation.
Definition: Pathname.h:110
String related utilities and Regular expression matching.
Definition: Arch.h:357
std::chrono::microseconds appconnect
Definition: request.h:99
bool prepareNextRangeBatch(std::string &errBuf)
Definition: request.cc:463
constexpr bool always_false
Definition: PathInfo.cc:544
running_t(pending_t &&prevState)
Definition: request.cc:95
std::string nativeErrorString() const
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:133
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
void setOptions(Options opt)
Definition: request.cc:1128
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
TransferSettings & transferSettings()
Definition: request.cc:1316
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
Definition: request.cc:1110
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
Definition: request.cc:1271
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1356
void onActivityTimeout(Timer &)
Definition: request.cc:663
const Headers & headers() const
returns a list of all added headers
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition: Digest.cc:184
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
zypp::Pathname _targetFile
Definition: request_p.h:116
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
void setUrl(const Url &url)
This will change the URL of the request.
Definition: request.cc:1244
std::chrono::microseconds namelookup
Definition: request.h:97
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
Do not differentiate case.
Definition: Regex.h:99
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
size_t headerCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:780
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:211
bool addRequestHeader(const std::string &header)
Definition: request.cc:1361
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
const std::string & asString() const
String representation.
Definition: Pathname.h:91
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:134
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition: request.cc:726
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
long connectTimeout() const
connection timeout
bool initialize(std::string &errBuf)
Definition: request.cc:124
#define WAR
Definition: Logger.h:97
#define nullptr
Definition: Easy.h:55
The NetworkRequestError class Represents a error that occured in.
std::vector< char > peekData(off_t offset, size_t count) const
Definition: request.cc:1228
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:1340
zypp::ByteCount _expectedFileSize
Definition: request_p.h:119
static constexpr int _rangeAttempt[]
Definition: request_p.h:148
UByteArray CheckSumBytes
Definition: request.h:47
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1348
Priority priority() const
Definition: request.cc:1123
std::string proxyuserpwd
Definition: curlconfig.h:49
bool setupHandle(std::string &errBuf)
Definition: request.cc:136
const Pathname & clientKeyPath() const
SSL client key file.
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
Definition: request.cc:1253
void validateRange(NetworkRequest::Range &rng)
Definition: request.cc:694
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:141
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
#define MIL_MEDIA
Definition: mediadebug_p.h:29
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len)
Definition: request.cc:709
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: request.cc:56
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
bool proxyEnabled() const
proxy is enabled
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
Definition: request.cc:1258
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:754
Regular expression match result.
Definition: Regex.h:167
std::string contentType() const
Returns the content type as reported from the server.
Definition: request.cc:1279
static const Unit B
1 Byte
Definition: ByteCount.h:42
Base class for Exception.
Definition: Exception.h:145
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:125
std::chrono::microseconds total
Definition: request.h:101
bool any_of(const Container &c, Fnc &&cb)
Definition: Algorithm.h:76
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
Definition: request.h:86
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
static Range make(size_t start, size_t len=0, DigestPtr &&digest=nullptr, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
Definition: request.cc:74
void setPriority(Priority prio, bool triggerReschedule=true)
Definition: request.cc:1115
State state() const
Returns the current state the HttpDownloadRequest is in.
Definition: request.cc:1321
TransferSettings _settings
Definition: request_p.h:117
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:129
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
virtual ~NetworkRequest()
Definition: request.cc:1102
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3"
Options options() const
Definition: request.cc:1133
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:120
size_t writeCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:866
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
Definition: Regex.h:70
std::chrono::microseconds redirect
Definition: request.h:102
std::shared_ptr< zypp::Digest > DigestPtr
Definition: request.h:46
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1390
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:135
Type type() const
type Returns the type of the error
These are enforced even if you don&#39;t pass them as flag argument.
Definition: Regex.h:103
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1375
std::string userPassword() const
returns the user and password as a user:pass string
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
FileMode fileOpenMode() const
Returns the currently configured file open mode.
Definition: request.cc:1266
std::vector< Range > failedRanges() const
Definition: request.cc:1169
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:106
void setResult(NetworkRequestError &&err)
Definition: request.cc:617
const std::string & proxy() const
proxy host
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
Definition: request.cc:1385
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:431
const std::string & userAgentString() const
user agent string
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
ZYppCommitResult & _result
Definition: TargetImpl.cc:1596
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:211