libzypp  17.31.14
request.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 ----------------------------------------------------------------------*/
13 #include <zypp-core/zyppng/base/EventDispatcher>
14 #include <zypp-core/zyppng/base/private/linuxhelpers_p.h>
15 #include <zypp-core/zyppng/core/String>
17 #include <zypp-curl/CurlConfig>
18 #include <zypp-curl/auth/CurlAuthData>
19 #include <zypp-media/MediaConfig>
20 #include <zypp-core/base/String.h>
21 #include <zypp-core/base/StringV.h>
22 #include <zypp-core/Pathname.h>
23 #include <curl/curl.h>
24 #include <stdio.h>
25 #include <fcntl.h>
26 #include <sstream>
27 #include <utility>
28 
29 #include <iostream>
30 #include <boost/variant.hpp>
31 #include <boost/variant/polymorphic_get.hpp>
32 
33 
34 namespace zyppng {
35 
36  namespace {
37  static size_t nwr_headerCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
38  if ( !userdata )
39  return 0;
40 
41  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
42  return that->headerCallback( ptr, size, nmemb );
43  }
44  static size_t nwr_writeCallback ( char *ptr, size_t size, size_t nmemb, void *userdata ) {
45  if ( !userdata )
46  return 0;
47 
48  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( userdata );
49  return that->writeCallback( ptr, size, nmemb );
50  }
51 
52  //helper for std::visit
53  template<class T> struct always_false : std::false_type {};
54  }
55 
56  std::vector<char> peek_data_fd( FILE *fd, off_t offset, size_t count )
57  {
58  if ( !fd )
59  return {};
60 
61  fflush( fd );
62 
63  std::vector<char> data( count + 1 , '\0' );
64 
65  ssize_t l = -1;
66  while ((l = pread( fileno( fd ), data.data(), count, offset ) ) == -1 && errno == EINTR)
67  ;
68  if (l == -1)
69  return {};
70 
71  return data;
72  }
73 
74  NetworkRequest::Range NetworkRequest::Range::make(size_t start, size_t len, zyppng::NetworkRequest::DigestPtr &&digest, zyppng::NetworkRequest::CheckSumBytes &&expectedChkSum, std::any &&userData, std::optional<size_t> digestCompareLen, std::optional<size_t> dataBlockPadding )
75  {
76  return NetworkRequest::Range {
77  .start = start,
78  .len = len,
79  .bytesWritten = 0,
80  ._digest = std::move( digest ),
81  ._checksum = std::move( expectedChkSum ),
82  ._relevantDigestLen = std::move( digestCompareLen ),
83  ._chksumPad = std::move( dataBlockPadding ),
84  .userData = std::move( userData ),
85  ._rangeState = State::Pending
86  };
87  }
88 
90  : _outFile( std::move(prevState._outFile) )
91  , _downloaded( prevState._downloaded )
92  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
93  { }
94 
96  : _requireStatusPartial( prevState._requireStatusPartial )
97  { }
98 
100  : _outFile( std::move(prevState._outFile) )
101  , _requireStatusPartial( true )
102  , _downloaded( prevState._downloaded )
103  , _rangeAttemptIdx( prevState._rangeAttemptIdx )
104  { }
105 
107  : BasePrivate(p)
108  , _url ( std::move(url) )
109  , _targetFile ( std::move( targetFile) )
110  , _fMode ( std::move(fMode) )
111  , _headers( std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( nullptr, &curl_slist_free_all ) )
112  { }
113 
115  {
116  if ( _easyHandle ) {
117  //clean up for now, later we might reuse handles
118  curl_easy_cleanup( _easyHandle );
119  //reset in request but make sure the request was not enqueued again and got a new handle
120  _easyHandle = nullptr;
121  }
122  }
123 
124  bool NetworkRequestPrivate::initialize( std::string &errBuf )
125  {
126  reset();
127 
128  if ( _easyHandle )
129  //will reset to defaults but keep live connections, session ID and DNS caches
130  curl_easy_reset( _easyHandle );
131  else
132  _easyHandle = curl_easy_init();
133  return setupHandle ( errBuf );
134  }
135 
136  bool NetworkRequestPrivate::setupHandle( std::string &errBuf )
137  {
139  curl_easy_setopt( _easyHandle, CURLOPT_ERRORBUFFER, this->_errorBuf.data() );
140 
141  const std::string urlScheme = _url.getScheme();
142  if ( urlScheme == "http" || urlScheme == "https" )
144 
145  try {
146 
147  setCurlOption( CURLOPT_PRIVATE, this );
148  setCurlOption( CURLOPT_XFERINFOFUNCTION, NetworkRequestPrivate::curlProgressCallback );
149  setCurlOption( CURLOPT_XFERINFODATA, this );
150  setCurlOption( CURLOPT_NOPROGRESS, 0L);
151  setCurlOption( CURLOPT_FAILONERROR, 1L);
152  setCurlOption( CURLOPT_NOSIGNAL, 1L);
153 
154  std::string urlBuffer( _url.asString() );
155  setCurlOption( CURLOPT_URL, urlBuffer.c_str() );
156 
157  setCurlOption( CURLOPT_WRITEFUNCTION, nwr_writeCallback );
158  setCurlOption( CURLOPT_WRITEDATA, this );
159 
161  setCurlOption( CURLOPT_CONNECT_ONLY, 1L );
162  setCurlOption( CURLOPT_FRESH_CONNECT, 1L );
163  }
165  // instead of returning no data with NOBODY, we return
166  // little data, that works with broken servers, and
167  // works for ftp as well, because retrieving only headers
168  // ftp will return always OK code ?
169  // See http://curl.haxx.se/docs/knownbugs.html #58
171  setCurlOption( CURLOPT_NOBODY, 1L );
172  else
173  setCurlOption( CURLOPT_RANGE, "0-1" );
174  }
175 
177  if ( _requestedRanges.size() ) {
178  if ( ! prepareNextRangeBatch ( errBuf ))
179  return false;
180  } else {
181  std::visit( [&]( auto &arg ){
182  using T = std::decay_t<decltype(arg)>;
183  if constexpr ( std::is_same_v<T, pending_t> ) {
184  arg._requireStatusPartial = false;
185  } else {
186  DBG << _easyHandle << " " << "NetworkRequestPrivate::setupHandle called in unexpected state" << std::endl;
187  }
188  }, _runningMode );
190  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
191  }
192  }
193 
194  //make a local copy of the settings, so headers are not added multiple times
195  TransferSettings locSet = _settings;
196 
197  if ( _dispatcher ) {
198  locSet.setUserAgentString( _dispatcher->agentString().c_str() );
199 
200  // add custom headers as configured (bsc#955801)
201  const auto &cHeaders = _dispatcher->hostSpecificHeaders();
202  if ( auto i = cHeaders.find(_url.getHost()); i != cHeaders.end() ) {
203  for ( const auto &[key, value] : i->second ) {
205  "%s: %s", key.c_str(), value.c_str() )
206  ));
207  }
208  }
209  }
210 
211  locSet.addHeader("Pragma:");
212 
215  {
216  case 4: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4 ); break;
217  case 6: setCurlOption( CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6 ); break;
218  default: break;
219  }
220 
221  setCurlOption( CURLOPT_HEADERFUNCTION, &nwr_headerCallback );
222  setCurlOption( CURLOPT_HEADERDATA, this );
223 
227  setCurlOption( CURLOPT_CONNECTTIMEOUT, locSet.connectTimeout() );
228  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
229  // just in case curl does not trigger its progress callback frequently
230  // enough.
231  if ( locSet.timeout() )
232  {
233  setCurlOption( CURLOPT_TIMEOUT, 3600L );
234  }
235 
236  if ( urlScheme == "https" )
237  {
238 #if CURLVERSION_AT_LEAST(7,19,4)
239  // restrict following of redirections from https to https only
240  if ( _url.getHost() == "download.opensuse.org" )
241  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
242  else
243  setCurlOption( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
244 #endif
245 
246  if( locSet.verifyPeerEnabled() ||
247  locSet.verifyHostEnabled() )
248  {
249  setCurlOption(CURLOPT_CAPATH, locSet.certificateAuthoritiesPath().c_str());
250  }
251 
252  if( ! locSet.clientCertificatePath().empty() )
253  {
254  setCurlOption(CURLOPT_SSLCERT, locSet.clientCertificatePath().c_str());
255  }
256  if( ! locSet.clientKeyPath().empty() )
257  {
258  setCurlOption(CURLOPT_SSLKEY, locSet.clientKeyPath().c_str());
259  }
260 
261 #ifdef CURLSSLOPT_ALLOW_BEAST
262  // see bnc#779177
263  setCurlOption( CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
264 #endif
265  setCurlOption(CURLOPT_SSL_VERIFYPEER, locSet.verifyPeerEnabled() ? 1L : 0L);
266  setCurlOption(CURLOPT_SSL_VERIFYHOST, locSet.verifyHostEnabled() ? 2L : 0L);
267  // bnc#903405 - POODLE: libzypp should only talk TLS
268  setCurlOption(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
269  }
270 
271  // follow any Location: header that the server sends as part of
272  // an HTTP header (#113275)
273  setCurlOption( CURLOPT_FOLLOWLOCATION, 1L);
274  // 3 redirects seem to be too few in some cases (bnc #465532)
275  setCurlOption( CURLOPT_MAXREDIRS, 6L );
276 
277  //set the user agent
278  setCurlOption(CURLOPT_USERAGENT, locSet.userAgentString().c_str() );
279 
280 
281  /*---------------------------------------------------------------*
282  CURLOPT_USERPWD: [user name]:[password]
283  Url::username/password -> CURLOPT_USERPWD
284  If not provided, anonymous FTP identification
285  *---------------------------------------------------------------*/
286  if ( locSet.userPassword().size() )
287  {
288  setCurlOption(CURLOPT_USERPWD, locSet.userPassword().c_str());
289  std::string use_auth = _settings.authType();
290  if (use_auth.empty())
291  use_auth = "digest,basic"; // our default
292  long auth = zypp::media::CurlAuthData::auth_type_str2long(use_auth);
293  if( auth != CURLAUTH_NONE)
294  {
295  DBG << _easyHandle << " " << "Enabling HTTP authentication methods: " << use_auth
296  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
297  setCurlOption(CURLOPT_HTTPAUTH, auth);
298  }
299  }
300 
301  if ( locSet.proxyEnabled() && ! locSet.proxy().empty() )
302  {
303  DBG << _easyHandle << " " << "Proxy: '" << locSet.proxy() << "'" << std::endl;
304  setCurlOption(CURLOPT_PROXY, locSet.proxy().c_str());
305  setCurlOption(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
306 
307  /*---------------------------------------------------------------*
308  * CURLOPT_PROXYUSERPWD: [user name]:[password]
309  *
310  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
311  * If not provided, $HOME/.curlrc is evaluated
312  *---------------------------------------------------------------*/
313 
314  std::string proxyuserpwd = locSet.proxyUserPassword();
315 
316  if ( proxyuserpwd.empty() )
317  {
318  zypp::media::CurlConfig curlconf;
319  zypp::media::CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
320  if ( curlconf.proxyuserpwd.empty() )
321  DBG << _easyHandle << " " << "Proxy: ~/.curlrc does not contain the proxy-user option" << std::endl;
322  else
323  {
324  proxyuserpwd = curlconf.proxyuserpwd;
325  DBG << _easyHandle << " " << "Proxy: using proxy-user from ~/.curlrc" << std::endl;
326  }
327  }
328  else
329  {
330  DBG << _easyHandle << " " << _easyHandle << " " << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << std::endl;
331  }
332 
333  if ( ! proxyuserpwd.empty() )
334  {
335  setCurlOption(CURLOPT_PROXYUSERPWD, ::internal::curlUnEscape( proxyuserpwd ).c_str());
336  }
337  }
338 #if CURLVERSION_AT_LEAST(7,19,4)
339  else if ( locSet.proxy() == EXPLICITLY_NO_PROXY )
340  {
341  // Explicitly disabled in URL (see fillSettingsFromUrl()).
342  // This should also prevent libcurl from looking into the environment.
343  DBG << _easyHandle << " " << "Proxy: explicitly NOPROXY" << std::endl;
344  setCurlOption(CURLOPT_NOPROXY, "*");
345  }
346 
347 #endif
348  // else: Proxy: not explicitly set; libcurl may look into the environment
349 
351  if ( locSet.minDownloadSpeed() != 0 )
352  {
353  setCurlOption(CURLOPT_LOW_SPEED_LIMIT, locSet.minDownloadSpeed());
354  // default to 10 seconds at low speed
355  setCurlOption(CURLOPT_LOW_SPEED_TIME, 60L);
356  }
357 
358 #if CURLVERSION_AT_LEAST(7,15,5)
359  if ( locSet.maxDownloadSpeed() != 0 )
360  setCurlOption(CURLOPT_MAX_RECV_SPEED_LARGE, locSet.maxDownloadSpeed());
361 #endif
362 
363  if ( zypp::str::strToBool( _url.getQueryParam( "cookies" ), true ) )
364  setCurlOption( CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
365  else
366  MIL << _easyHandle << " " << "No cookies requested" << std::endl;
367  setCurlOption(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
368 
369 #if CURLVERSION_AT_LEAST(7,18,0)
370  // bnc #306272
371  setCurlOption(CURLOPT_PROXY_TRANSFER_MODE, 1L );
372 #endif
373 
374  // Append settings custom headers to curl.
375  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
376  for ( const auto &header : locSet.headers() ) {
377  if ( !z_func()->addRequestHeader( header.c_str() ) )
379  }
380 
381  if ( _headers )
382  setCurlOption( CURLOPT_HTTPHEADER, _headers.get() );
383 
384  return true;
385 
386  } catch ( const zypp::Exception &excp ) {
387  ZYPP_CAUGHT(excp);
388  errBuf = excp.asString();
389  }
390  return false;
391  }
392 
394  {
395  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
396  if ( !rmode ) {
397  DBG << _easyHandle << "Can only create output file in running mode" << std::endl;
398  return false;
399  }
400  // if we have no open file create or open it
401  if ( !rmode->_outFile ) {
402  std::string openMode = "w+b";
404  openMode = "r+b";
405 
406  rmode->_outFile = fopen( _targetFile.asString().c_str() , openMode.c_str() );
407 
408  //if the file does not exist create a new one
409  if ( !rmode->_outFile && _fMode == NetworkRequest::WriteShared ) {
410  rmode->_outFile = fopen( _targetFile.asString().c_str() , "w+b" );
411  }
412 
413  if ( !rmode->_outFile ) {
415  ,zypp::str::Format("Unable to open target file (%1%). Errno: (%2%:%3%)") % _targetFile.asString() % errno % strerr_cxx() );
416  return false;
417  }
418  }
419 
420  return true;
421  }
422 
424  {
425  // We can recover from RangeFail errors if we have more batch sizes to try
426  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
427  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail )
428  return ( rmode->_rangeAttemptIdx + 1 < sizeof( _rangeAttempt ) ) && hasMoreWork();
429  return false;
430  }
431 
432  bool NetworkRequestPrivate::prepareToContinue( std::string &errBuf )
433  {
434  auto rmode = std::get_if<NetworkRequestPrivate::running_t>( &_runningMode );
435 
436  if ( hasMoreWork() ) {
437  // go to the next range batch level if we are restarted due to a failed range request
438  if ( rmode->_cachedResult && rmode->_cachedResult->type() == NetworkRequestError::RangeFail ) {
439  if ( rmode->_rangeAttemptIdx + 1 >= sizeof( _rangeAttempt ) ) {
440  errBuf = "No more range batch sizes available";
441  return false;
442  }
443  rmode->_rangeAttemptIdx++;
444  }
445 
446  _runningMode = prepareNextRangeBatch_t( std::move(std::get<running_t>( _runningMode )) );
447 
448  // we reset the handle to default values. We do this to not run into
449  // "transfer closed with outstanding read data remaining" error CURL sometimes returns when
450  // we cancel a connection because of a range error to request a smaller batch.
451  // The error will still happen but much less frequently than without resetting the handle.
452  //
453  // Note: Even creating a new handle will NOT fix the issue
454  curl_easy_reset( _easyHandle );
455  if ( !setupHandle (errBuf) )
456  return false;
457  return true;
458  }
459  errBuf = "Request has no more work";
460  return false;
461 
462  }
463 
465  {
466  if ( _requestedRanges.size() == 0 ) {
467  errBuf = "Calling the prepareNextRangeBatch function without a range to download is not supported.";
468  return false;
469  }
470 
471  std::string rangeDesc;
472  uint rangesAdded = 0;
473  if ( _requestedRanges.size() > 1 && _protocolMode != ProtocolMode::HTTP ) {
474  errBuf = "Using more than one range is not supported with protocols other than HTTP/HTTPS";
475  return false;
476  }
477 
478  // check if we have one big range convering the whole file
479  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 ) {
480  if ( !std::holds_alternative<pending_t>( _runningMode ) ) {
481  errBuf = zypp::str::Str() << "Unexpected state when calling prepareNextRangeBatch " << _runningMode.index ();
482  return false;
483  }
484 
485  _requestedRanges[0]._rangeState = NetworkRequest::Running;
486  std::get<pending_t>( _runningMode )._requireStatusPartial = false;
487 
488  } else {
489  std::sort( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &elem1, const auto &elem2 ){
490  return ( elem1.start < elem2.start );
491  });
492 
493  if ( std::holds_alternative<pending_t>( _runningMode ) )
494  std::get<pending_t>( _runningMode )._requireStatusPartial = true;
495 
496  auto maxRanges = _rangeAttempt[0];
497  if ( std::holds_alternative<prepareNextRangeBatch_t>( _runningMode ) )
498  maxRanges = _rangeAttempt[std::get<prepareNextRangeBatch_t>( _runningMode )._rangeAttemptIdx];
499 
500  // helper function to build up the request string for the range
501  auto addRangeString = [ &rangeDesc, &rangesAdded ]( const std::pair<size_t, size_t> &range ) {
502  std::string rangeD = zypp::str::form("%llu-", static_cast<unsigned long long>( range.first ) );
503  if( range.second > 0 )
504  rangeD.append( zypp::str::form( "%llu", static_cast<unsigned long long>( range.second ) ) );
505 
506  if ( rangeDesc.size() )
507  rangeDesc.append(",").append( rangeD );
508  else
509  rangeDesc = std::move( rangeD );
510 
511  rangesAdded++;
512  };
513 
514  std::optional<std::pair<size_t, size_t>> currentZippedRange;
515  bool closedRange = true;
516  for ( auto &range : _requestedRanges ) {
517 
518  if ( range._rangeState != NetworkRequest::Pending )
519  continue;
520 
521  //reset the download results
522  range.bytesWritten = 0;
523 
524  //when we have a open range in the list of ranges we will get from start of range to end of file,
525  //all following ranges would never be marked as valid, so we have to fail early
526  if ( !closedRange ) {
527  errBuf = "It is not supported to request more ranges after a open range.";
528  return false;
529  }
530 
531  const auto rangeEnd = range.len > 0 ? range.start + range.len - 1 : 0;
532  closedRange = (rangeEnd > 0);
533 
534  // remember this range was already requested
535  range._rangeState = NetworkRequest::Running;
536  range.bytesWritten = 0;
537  if ( range._digest )
538  range._digest->reset();
539 
540  // we try to compress the requested ranges into as big chunks as possible for the request,
541  // when receiving we still track the original ranges so we can collect and test their checksums
542  if ( !currentZippedRange ) {
543  currentZippedRange = std::make_pair( range.start, rangeEnd );
544  } else {
545  //range is directly consecutive to the previous range
546  if ( currentZippedRange->second + 1 == range.start ) {
547  currentZippedRange->second = rangeEnd;
548  } else {
549  //this range does not directly follow the previous one, we build the string and start a new one
550  addRangeString( *currentZippedRange );
551  currentZippedRange = std::make_pair( range.start, rangeEnd );
552  }
553  }
554 
555  if ( rangesAdded >= maxRanges ) {
556  MIL << _easyHandle << " " << "Reached max nr of ranges (" << maxRanges << "), batching the request to not break the server" << std::endl;
557  break;
558  }
559  }
560 
561  // add the last range too
562  if ( currentZippedRange )
563  addRangeString( *currentZippedRange );
564 
565  MIL << _easyHandle << " " << "Requesting Ranges: " << rangeDesc << std::endl;
566 
567  setCurlOption( CURLOPT_RANGE, rangeDesc.c_str() );
568  }
569 
570  return true;
571  }
572 
574  {
575  // check if we have ranges that have never been requested
576  return std::any_of( _requestedRanges.begin(), _requestedRanges.end(), []( const auto &range ){ return range._rangeState == NetworkRequest::Pending; });
577  }
578 
580  {
581  bool isRangeContinuation = std::holds_alternative<prepareNextRangeBatch_t>( _runningMode );
582  if ( isRangeContinuation ) {
583  MIL << _easyHandle << " " << "Continuing a previously started range batch." << std::endl;
584  _runningMode = running_t( std::move(std::get<prepareNextRangeBatch_t>( _runningMode )) );
585  } else {
586  auto mode = running_t( std::move(std::get<pending_t>( _runningMode )) );
587  if ( _requestedRanges.size() == 1 && _requestedRanges.front().start == 0 && _requestedRanges.front().len == 0 )
588  mode._currentRange = 0;
589 
590  _runningMode = std::move(mode);
591  }
592 
593  auto &m = std::get<running_t>( _runningMode );
594 
595  if ( m._activityTimer ) {
596  DBG_MEDIA << _easyHandle << " Setting activity timeout to: " << _settings.timeout() << std::endl;
597  m._activityTimer->connect( &Timer::sigExpired, *this, &NetworkRequestPrivate::onActivityTimeout );
598  m._activityTimer->start( static_cast<uint64_t>( _settings.timeout() * 1000 ) );
599  }
600 
601  if ( !isRangeContinuation )
602  _sigStarted.emit( *z_func() );
603  }
604 
606  {
607  if ( std::holds_alternative<running_t>(_runningMode) ) {
608  auto &rmode = std::get<running_t>( _runningMode );
609  // if we still have a current range set it valid by checking the checksum
610  if ( rmode._currentRange >= 0 ) {
611  auto &currR = _requestedRanges[rmode._currentRange];
612  rmode._currentRange = -1;
613  validateRange( currR );
614  }
615  }
616  }
617 
619  {
620 
621  finished_t resState;
622  resState._result = std::move(err);
623 
624  if ( std::holds_alternative<running_t>(_runningMode) ) {
625 
626  auto &rmode = std::get<running_t>( _runningMode );
627  rmode._outFile.reset();
628  resState._downloaded = rmode._downloaded;
629  resState._contentLenght = rmode._contentLenght;
630 
632  //we have a successful download lets see if we got everything we needed
633  for ( const auto &r : _requestedRanges ) {
634  if ( r._rangeState != NetworkRequest::Finished ) {
635  if ( r.len > 0 && r.bytesWritten != r.len )
636  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::MissingData, (zypp::str::Format("Did not receive all requested data from the server ( off: %1%, req: %2%, recv: %3% ).") % r.start % r.len % r.bytesWritten ) );
637  else if ( r._digest && r._checksum.size() && ! checkIfRangeChkSumIsValid(r) ) {
638  resState._result = NetworkRequestErrorPrivate::customError( NetworkRequestError::InvalidChecksum, (zypp::str::Format("Invalid checksum %1%, expected checksum %2%") % r._digest->digest() % zypp::Digest::digestVectorToString( r._checksum ) ) );
639  } else {
641  }
642  //we only report the first error
643  break;
644  }
645  }
646  }
647  }
648 
649  _runningMode = std::move( resState );
650  _sigFinished.emit( *z_func(), std::get<finished_t>(_runningMode)._result );
651  }
652 
654  {
656  _headers.reset( nullptr );
657  _errorBuf.fill( 0 );
659  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
660  range._rangeState = NetworkRequest::Pending;
661  });
662  }
663 
665  {
666  auto &m = std::get<running_t>( _runningMode );
667 
668  MIL_MEDIA << _easyHandle << " Request timeout interval: " << t.interval()<< " remaining: " << t.remaining() << std::endl;
669  std::map<std::string, boost::any> extraInfo;
670  extraInfo.insert( {"requestUrl", _url } );
671  extraInfo.insert( {"filepath", _targetFile } );
672  _dispatcher->cancel( *z_func(), NetworkRequestErrorPrivate::customError( NetworkRequestError::Timeout, "Download timed out", std::move(extraInfo) ) );
673  }
674 
676  {
677  if ( rng._digest && rng._checksum.size() ) {
678  auto bytesHashed = rng._digest->bytesHashed ();
679  if ( rng._chksumPad && *rng._chksumPad > bytesHashed ) {
680  MIL_MEDIA << _easyHandle << " " << "Padding the digest to required block size" << std::endl;
681  zypp::ByteArray padding( *rng._chksumPad - bytesHashed, '\0' );
682  rng._digest->update( padding.data(), padding.size() );
683  }
684  auto digVec = rng._digest->digestVector();
685  if ( rng._relevantDigestLen ) {
686  digVec.resize( *rng._relevantDigestLen );
687  }
688  return ( digVec == rng._checksum );
689  }
690 
691  // no checksum required
692  return true;
693  }
694 
696  {
697  if ( rng._digest && rng._checksum.size() ) {
698  if ( ( rng.len == 0 || rng.bytesWritten == rng.len ) && checkIfRangeChkSumIsValid(rng) )
700  else
702  } else {
703  if ( rng.len == 0 ? true : rng.bytesWritten == rng.len )
705  else
707  }
708  }
709 
710  bool NetworkRequestPrivate::parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len )
711  { //content-range: bytes 10485760-19147879/19147880
712  static const zypp::str::regex regex("^Content-Range:[[:space:]]+bytes[[:space:]]+([0-9]+)-([0-9]+)\\/([0-9]+)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
713 
714  zypp::str::smatch what;
715  if( !zypp::str::regex_match( std::string(line), what, regex ) || what.size() != 4 ) {
716  DBG << _easyHandle << " " << "Invalid Content-Range Header format: '" << std::string(line) << std::endl;
717  return false;
718  }
719 
720  size_t s = zypp::str::strtonum<size_t>( what[1]);
721  size_t e = zypp::str::strtonum<size_t>( what[2]);
722  start = std::move(s);
723  len = ( e - s ) + 1;
724  return true;
725  }
726 
727  bool NetworkRequestPrivate::parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
728  {
729  static const zypp::str::regex regex("^Content-Type:[[:space:]]+multipart\\/byteranges;[[:space:]]+boundary=(.*)$", zypp::str::regex::rxdefault | zypp::str::regex::icase );
730 
731  zypp::str::smatch what;
732  if( zypp::str::regex_match( std::string(line), what, regex ) ) {
733  if ( what.size() >= 2 ) {
734  boundary = what[1];
735  return true;
736  }
737  }
738  return false;
739  }
740 
742  {
743  return std::string( _errorBuf.data() );
744  }
745 
747  {
748  if ( std::holds_alternative<running_t>( _runningMode ) ){
749  auto &rmode = std::get<running_t>( _runningMode );
750  if ( rmode._activityTimer && rmode._activityTimer->isRunning() )
751  rmode._activityTimer->start();
752  }
753  }
754 
755  int NetworkRequestPrivate::curlProgressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
756  {
757  if ( !clientp )
758  return CURLE_OK;
759  NetworkRequestPrivate *that = reinterpret_cast<NetworkRequestPrivate *>( clientp );
760 
761  if ( !std::holds_alternative<running_t>(that->_runningMode) ){
762  DBG << that->_easyHandle << " " << "Curl progress callback was called in invalid state "<< that->z_func()->state() << std::endl;
763  return -1;
764  }
765 
766  auto &rmode = std::get<running_t>( that->_runningMode );
767 
768  //reset the timer
769  that->resetActivityTimer();
770 
771  rmode._isInCallback = true;
772  if ( rmode._lastProgressNow != dlnow ) {
773  rmode._lastProgressNow = dlnow;
774  that->_sigProgress.emit( *that->z_func(), dltotal, dlnow, ultotal, ulnow );
775  }
776  rmode._isInCallback = false;
777 
778  return rmode._cachedResult ? CURLE_ABORTED_BY_CALLBACK : CURLE_OK;
779  }
780 
781  size_t NetworkRequestPrivate::headerCallback(char *ptr, size_t size, size_t nmemb)
782  {
783  //it is valid to call this function with no data to write, just return OK
784  if ( size * nmemb == 0)
785  return 0;
786 
788 
790 
791  std::string_view hdr( ptr, size*nmemb );
792 
793  hdr.remove_prefix( std::min( hdr.find_first_not_of(" \t\r\n"), hdr.size() ) );
794  const auto lastNonWhitespace = hdr.find_last_not_of(" \t\r\n");
795  if ( lastNonWhitespace != hdr.npos )
796  hdr.remove_suffix( hdr.size() - (lastNonWhitespace + 1) );
797  else
798  hdr = std::string_view();
799 
800  auto &rmode = std::get<running_t>( _runningMode );
801  if ( !hdr.size() ) {
802  return ( size * nmemb );
803  }
804  if ( zypp::strv::hasPrefixCI( hdr, "HTTP/" ) ) {
805 
806  long statuscode = 0;
807  (void)curl_easy_getinfo( _easyHandle, CURLINFO_RESPONSE_CODE, &statuscode);
808 
809  const auto &doRangeFail = [&](){
810  WAR << _easyHandle << " " << "Range FAIL, trying with a smaller batch" << std::endl;
811  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::RangeFail, "Expected range status code 206, but got none." );
812 
813  // reset all ranges we requested to pending, we never got the data for them
814  std::for_each( _requestedRanges.begin (), _requestedRanges.end(), []( auto &range ) {
815  if ( range._rangeState == NetworkRequest::Running )
816  range._rangeState = NetworkRequest::Pending;
817  });
818  return 0;
819  };
820 
821  // if we have a status 204 we need to create a empty file
822  if( statuscode == 204 && !( _options & NetworkRequest::ConnectionTest ) && !( _options & NetworkRequest::HeadRequest ) )
824 
825  if ( rmode._requireStatusPartial ) {
826  // ignore other status codes, maybe we are redirected etc.
827  if ( ( statuscode >= 200 && statuscode <= 299 && statuscode != 206 )
828  || statuscode == 416 ) {
829  return doRangeFail();
830  }
831  }
832 
833  } else if ( zypp::strv::hasPrefixCI( hdr, "Location:" ) ) {
834  _lastRedirect = hdr.substr( 9 );
835  DBG << _easyHandle << " " << "redirecting to " << _lastRedirect << std::endl;
836 
837  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Type:") ) {
838  std::string sep;
839  if ( parseContentTypeMultiRangeHeader( hdr, sep ) ) {
840  rmode._gotMultiRangeHeader = true;
841  rmode._seperatorString = "--"+sep;
842  }
843  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Range:") ) {
845  if ( !parseContentRangeHeader( hdr, r.start, r.len) ) {
846  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
847  return 0;
848  }
849  DBG << _easyHandle << " " << "Got content range :" << r.start << " len " << r.len << std::endl;
850  rmode._gotContentRangeHeader = true;
851  rmode._currentSrvRange = r;
852 
853  } else if ( zypp::strv::hasPrefixCI( hdr, "Content-Length:") ) {
854  auto lenStr = str::trim( hdr.substr( 15 ), zypp::str::TRIM );
855  auto str = std::string ( lenStr.data(), lenStr.length() );
856  auto len = zypp::str::strtonum<typename zypp::ByteCount::SizeType>( str.data() );
857  if ( len > 0 ) {
858  DBG << _easyHandle << " " << "Got Content-Length Header: " << len << std::endl;
859  rmode._contentLenght = zypp::ByteCount(len, zypp::ByteCount::B);
860  }
861  }
862  }
863 
864  return ( size * nmemb );
865  }
866 
867  size_t NetworkRequestPrivate::writeCallback(char *ptr, size_t size, size_t nmemb)
868  {
869  const auto max = ( size * nmemb );
870 
872 
873  //it is valid to call this function with no data to write, just return OK
874  if ( max == 0)
875  return 0;
876 
877  //in case of a HEAD request, we do not write anything
879  return ( size * nmemb );
880  }
881 
882  auto &rmode = std::get<running_t>( _runningMode );
883 
884  auto writeDataToFile = [ this, &rmode ]( off_t offset, const char *data, size_t len ) -> off_t {
885 
886  if ( rmode._currentRange < 0 ) {
887  DBG << _easyHandle << " " << "Current range is zero in write request" << std::endl;
888  return 0;
889  }
890 
891  // if we have no open file create or open it
892  if ( !assertOutputFile() )
893  return 0;
894 
895  // seek to the given offset
896  if ( offset >= 0 ) {
897  if ( fseek( rmode._outFile, offset, SEEK_SET ) != 0 ) {
899  "Unable to set output file pointer." );
900  return 0;
901  }
902  }
903 
904  auto &rng = _requestedRanges[ rmode._currentRange ];
905  const auto bytesToWrite = rng.len > 0 ? std::min( rng.len - rng.bytesWritten, len ) : len;
906 
907  //make sure we do not write after the expected file size
908  if ( _expectedFileSize && _expectedFileSize <= static_cast<zypp::ByteCount::SizeType>(rng.start + rng.bytesWritten + bytesToWrite) ) {
909  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Downloaded data exceeds expected length." );
910  return 0;
911  }
912 
913  auto written = fwrite( data, 1, bytesToWrite, rmode._outFile );
914  if ( written == 0 )
915  return 0;
916 
917  if ( rng._digest && rng._checksum.size() ) {
918  if ( !rng._digest->update( data, written ) )
919  return 0;
920  }
921 
922  rng.bytesWritten += written;
923  if ( rmode._currentSrvRange ) rmode._currentSrvRange->bytesWritten += written;
924 
925  if ( rng.len > 0 && rng.bytesWritten >= rng.len ) {
926  rmode._currentRange = -1;
927  validateRange( rng );
928  }
929 
930  if ( rmode._currentSrvRange && rmode._currentSrvRange->len > 0 && rmode._currentSrvRange->bytesWritten >= rmode._currentSrvRange->len ) {
931  rmode._currentSrvRange.reset();
932  // we ran out of data in the current chunk, reset the target range as well because next data will be
933  // a chunk header again
934  rmode._currentRange = -1;
935  }
936 
937  // count the number of real bytes we have downloaded so far
938  rmode._downloaded += written;
939  _sigBytesDownloaded.emit( *z_func(), rmode._downloaded );
940 
941  return written;
942  };
943 
944  // we are currenty writing a range, continue until we hit the end of the requested chunk, or if we hit end of data
945  size_t bytesWrittenSoFar = 0;
946 
947  while ( bytesWrittenSoFar != max ) {
948 
949  off_t seekTo = -1;
950 
951  // this is called after all headers have been processed
952  if ( !rmode._allHeadersReceived ) {
953  rmode._allHeadersReceived = true;
954 
955  // no ranges at all, must be a normal download
956  if ( !rmode._gotMultiRangeHeader && !rmode._gotContentRangeHeader ) {
957 
958  if ( rmode._requireStatusPartial ) {
959  //we got a invalid response, the status code pointed to being partial but we got no range definition
961  "Invalid data from server, range respone was announced but there was no range definiton." );
962  return 0;
963  }
964 
965  //we always download a range even if it is not explicitly requested
966  if ( _requestedRanges.empty() ) {
968  _requestedRanges.back()._rangeState = NetworkRequest::State::Running;
969  }
970 
971  rmode._currentRange = 0;
972  seekTo = _requestedRanges[0].start;
973  }
974  }
975 
976  if ( rmode._currentSrvRange && rmode._currentRange == -1 ) {
977  //if we enter this branch, we just have finished writing a requested chunk but
978  //are still inside a chunk that was sent by the server, due to the std the server can coalesce requested ranges
979  //to optimize downloads we need to find the best match ( because the current offset might not even be in our requested ranges )
980  //Or we just parsed a Content-Lenght header and start a new block
981 
982  std::optional<uint> foundRange;
983  const size_t beginRange = rmode._currentSrvRange->start + rmode._currentSrvRange->bytesWritten;
984  const size_t endRange = beginRange + (rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten);
985  auto currDist = ULONG_MAX;
986  for ( uint i = 0; i < _requestedRanges.size(); i++ ) {
987  const auto &currR = _requestedRanges[i];
988 
989  // do not allow double ranges
990  if ( currR._rangeState == NetworkRequest::Finished || currR._rangeState == NetworkRequest::Error )
991  continue;
992 
993  // check if the range was already written
994  if ( currR.len == currR.bytesWritten )
995  continue;
996 
997  const auto currRBegin = currR.start + currR.bytesWritten;
998  if ( !( beginRange <= currRBegin && endRange >= currRBegin ) )
999  continue;
1000 
1001  // calculate the distance of the current ranges offset+data written to the range we got back from the server
1002  const auto newDist = currRBegin - beginRange;
1003 
1004  if ( !foundRange ) {
1005  foundRange = i;
1006  currDist = newDist;
1007  } else {
1008  //pick the range with the closest distance
1009  if ( newDist < currDist ) {
1010  foundRange = i;
1011  currDist = newDist;
1012  }
1013  }
1014  }
1015  if ( !foundRange ) {
1017  , "Unable to find a matching range for data returned by the server." );
1018  return 0;
1019  }
1020 
1021  //set the found range as the current one
1022  rmode._currentRange = *foundRange;
1023 
1024  //continue writing where we stopped
1025  seekTo = _requestedRanges[*foundRange].start + _requestedRanges[*foundRange].bytesWritten;
1026 
1027  //if we skip bytes we need to advance our written bytecount
1028  const auto skipBytes = seekTo - beginRange;
1029  bytesWrittenSoFar += skipBytes;
1030  rmode._currentSrvRange->bytesWritten += skipBytes;
1031  }
1032 
1033  if ( rmode._currentRange >= 0 ) {
1034  auto availableData = max - bytesWrittenSoFar;
1035  if ( rmode._currentSrvRange ) {
1036  availableData = std::min( availableData, rmode._currentSrvRange->len - rmode._currentSrvRange->bytesWritten );
1037  }
1038  auto bw = writeDataToFile( seekTo, ptr + bytesWrittenSoFar, availableData );
1039  if ( bw <= 0 )
1040  return 0;
1041 
1042  bytesWrittenSoFar += bw;
1043  }
1044 
1045  if ( bytesWrittenSoFar == max )
1046  return max;
1047 
1048  if ( rmode._currentRange == -1 ) {
1049 
1050  // we still are inside the current range from the server
1051  if ( rmode._currentSrvRange )
1052  continue;
1053 
1054  std::string_view incoming( ptr + bytesWrittenSoFar, max - bytesWrittenSoFar );
1055  auto hdrEnd = incoming.find("\r\n\r\n");
1056  if ( hdrEnd == incoming.npos ) {
1057  //no header end in the data yet, push to buffer and return
1058  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.end() );
1059  return max;
1060  }
1061 
1062  //append the data of the current header to the buffer and parse it
1063  rmode._rangePrefaceBuffer.insert( rmode._rangePrefaceBuffer.end(), incoming.begin(), incoming.begin() + ( hdrEnd + 4 ) );
1064  bytesWrittenSoFar += ( hdrEnd + 4 ); //header data plus header end
1065 
1066  std::string_view data( rmode._rangePrefaceBuffer.data(), rmode._rangePrefaceBuffer.size() );
1067  auto sepStrIndex = data.find( rmode._seperatorString );
1068  if ( sepStrIndex == data.npos ) {
1070  "Invalid multirange header format, seperator string missing." );
1071  return 0;
1072  }
1073 
1074  auto startOfHeader = sepStrIndex + rmode._seperatorString.length();
1075  std::vector<std::string_view> lines;
1076  zypp::strv::split( data.substr( startOfHeader ), "\r\n", zypp::strv::Trim::trim, [&]( std::string_view strv ) { lines.push_back(strv); } );
1077  for ( const auto &hdrLine : lines ) {
1078  if ( zypp::strv::hasPrefixCI(hdrLine, "Content-Range:") ) {
1080  //if we can not parse the header the message must be broken
1081  if(! parseContentRangeHeader( hdrLine, r.start, r.len ) ) {
1082  rmode._cachedResult = NetworkRequestErrorPrivate::customError( NetworkRequestError::InternalError, "Invalid Content-Range header format." );
1083  return 0;
1084  }
1085  rmode._currentSrvRange = r;
1086  break;
1087  }
1088  }
1089  //clear the buffer again
1090  rmode._rangePrefaceBuffer.clear();
1091  }
1092  }
1093  return bytesWrittenSoFar;
1094  }
1095 
1097 
1098  NetworkRequest::NetworkRequest(zyppng::Url url, zypp::filesystem::Pathname targetFile, zyppng::NetworkRequest::FileMode fMode)
1099  : Base ( *new NetworkRequestPrivate( std::move(url), std::move(targetFile), std::move(fMode), *this ) )
1100  {
1101  }
1102 
1104  {
1105  Z_D();
1106 
1107  if ( d->_dispatcher )
1108  d->_dispatcher->cancel( *this, "Request destroyed while still running" );
1109  }
1110 
1112  {
1113  d_func()->_expectedFileSize = std::move( expectedFileSize );
1114  }
1115 
1116  void NetworkRequest::setPriority( NetworkRequest::Priority prio, bool triggerReschedule )
1117  {
1118  Z_D();
1119  d->_priority = prio;
1120  if ( state() == Pending && triggerReschedule && d->_dispatcher )
1121  d->_dispatcher->reschedule();
1122  }
1123 
1125  {
1126  return d_func()->_priority;
1127  }
1128 
1129  void NetworkRequest::setOptions( Options opt )
1130  {
1131  d_func()->_options = opt;
1132  }
1133 
1134  NetworkRequest::Options NetworkRequest::options() const
1135  {
1136  return d_func()->_options;
1137  }
1138 
1139  void NetworkRequest::addRequestRange( size_t start, size_t len, DigestPtr digest, CheckSumBytes expectedChkSum , std::any userData, std::optional<size_t> digestCompareLen, std::optional<size_t> chksumpad )
1140  {
1141  Z_D();
1142  if ( state() == Running )
1143  return;
1144 
1145  d->_requestedRanges.push_back( Range::make( start, len, std::move(digest), std::move( expectedChkSum ), std::move( userData ), digestCompareLen, chksumpad ) );
1146  }
1147 
1149  {
1150  Z_D();
1151  if ( state() == Running )
1152  return;
1153 
1154  d->_requestedRanges.push_back( range );
1155  auto &rng = d->_requestedRanges.back();
1156  rng._rangeState = NetworkRequest::Pending;
1157  rng.bytesWritten = 0;
1158  if ( rng._digest )
1159  rng._digest->reset();
1160  }
1161 
1163  {
1164  Z_D();
1165  if ( state() == Running )
1166  return;
1167  d->_requestedRanges.clear();
1168  }
1169 
1170  std::vector<NetworkRequest::Range> NetworkRequest::failedRanges() const
1171  {
1172  const auto mystate = state();
1173  if ( mystate != Finished && mystate != Error )
1174  return {};
1175 
1176  Z_D();
1177 
1178  std::vector<Range> failed;
1179  for ( const auto &r : d->_requestedRanges ) {
1180  if ( r._rangeState != NetworkRequest::Finished )
1181  failed.push_back( r );
1182  }
1183  return failed;
1184  }
1185 
1186  const std::vector<NetworkRequest::Range> &NetworkRequest::requestedRanges() const
1187  {
1188  return d_func()->_requestedRanges;
1189  }
1190 
1191  const std::string &NetworkRequest::lastRedirectInfo() const
1192  {
1193  return d_func()->_lastRedirect;
1194  }
1195 
1197  {
1198  return d_func()->_easyHandle;
1199  }
1200 
1201  std::optional<zyppng::NetworkRequest::Timings> NetworkRequest::timings() const
1202  {
1203  const auto myerr = error();
1204  const auto mystate = state();
1205  if ( mystate != Finished )
1206  return {};
1207 
1208  Timings t;
1209 
1210  auto getMeasurement = [ this ]( const CURLINFO info, std::chrono::microseconds &target ){
1211  using FPSeconds = std::chrono::duration<double, std::chrono::seconds::period>;
1212  double val = 0;
1213  const auto res = curl_easy_getinfo( d_func()->_easyHandle, info, &val );
1214  if ( CURLE_OK == res ) {
1215  target = std::chrono::duration_cast<std::chrono::microseconds>( FPSeconds(val) );
1216  }
1217  };
1218 
1219  getMeasurement( CURLINFO_NAMELOOKUP_TIME, t.namelookup );
1220  getMeasurement( CURLINFO_CONNECT_TIME, t.connect);
1221  getMeasurement( CURLINFO_APPCONNECT_TIME, t.appconnect);
1222  getMeasurement( CURLINFO_PRETRANSFER_TIME , t.pretransfer);
1223  getMeasurement( CURLINFO_TOTAL_TIME, t.total);
1224  getMeasurement( CURLINFO_REDIRECT_TIME, t.redirect);
1225 
1226  return t;
1227  }
1228 
1229  std::vector<char> NetworkRequest::peekData( off_t offset, size_t count ) const
1230  {
1231  Z_D();
1232 
1233  if ( !std::holds_alternative<NetworkRequestPrivate::running_t>( d->_runningMode) )
1234  return {};
1235 
1236  const auto &rmode = std::get<NetworkRequestPrivate::running_t>( d->_runningMode );
1237  return peek_data_fd( rmode._outFile, offset, count );
1238  }
1239 
1241  {
1242  return d_func()->_url;
1243  }
1244 
1245  void NetworkRequest::setUrl(const Url &url)
1246  {
1247  Z_D();
1248  if ( state() == NetworkRequest::Running )
1249  return;
1250 
1251  d->_url = url;
1252  }
1253 
1255  {
1256  return d_func()->_targetFile;
1257  }
1258 
1260  {
1261  Z_D();
1262  if ( state() == NetworkRequest::Running )
1263  return;
1264  d->_targetFile = path;
1265  }
1266 
1268  {
1269  return d_func()->_fMode;
1270  }
1271 
1273  {
1274  Z_D();
1275  if ( state() == NetworkRequest::Running )
1276  return;
1277  d->_fMode = std::move( mode );
1278  }
1279 
1280  std::string NetworkRequest::contentType() const
1281  {
1282  char *ptr = NULL;
1283  if ( curl_easy_getinfo( d_func()->_easyHandle, CURLINFO_CONTENT_TYPE, &ptr ) == CURLE_OK && ptr )
1284  return std::string(ptr);
1285  return std::string();
1286  }
1287 
1289  {
1290  return std::visit([](auto& arg) -> zypp::ByteCount {
1291  using T = std::decay_t<decltype(arg)>;
1292  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1293  return zypp::ByteCount(0);
1294  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1295  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1296  return arg._contentLenght;
1297  else
1298  static_assert(always_false<T>::value, "Unhandled state type");
1299  }, d_func()->_runningMode);
1300  }
1301 
1303  {
1304  return std::visit([](auto& arg) -> zypp::ByteCount {
1305  using T = std::decay_t<decltype(arg)>;
1306  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1307  return zypp::ByteCount();
1308  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t>
1309  || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t>
1310  || std::is_same_v<T, NetworkRequestPrivate::finished_t>)
1311  return arg._downloaded;
1312  else
1313  static_assert(always_false<T>::value, "Unhandled state type");
1314  }, d_func()->_runningMode);
1315  }
1316 
1318  {
1319  return d_func()->_settings;
1320  }
1321 
1323  {
1324  return std::visit([this](auto& arg) {
1325  using T = std::decay_t<decltype(arg)>;
1326  if constexpr (std::is_same_v<T, NetworkRequestPrivate::pending_t>)
1327  return Pending;
1328  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::running_t> || std::is_same_v<T, NetworkRequestPrivate::prepareNextRangeBatch_t> )
1329  return Running;
1330  else if constexpr (std::is_same_v<T, NetworkRequestPrivate::finished_t>) {
1331  if ( std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode )._result.isError() )
1332  return Error;
1333  else
1334  return Finished;
1335  }
1336  else
1337  static_assert(always_false<T>::value, "Unhandled state type");
1338  }, d_func()->_runningMode);
1339  }
1340 
1342  {
1343  const auto s = state();
1344  if ( s != Error && s != Finished )
1345  return NetworkRequestError();
1346  return std::get<NetworkRequestPrivate::finished_t>( d_func()->_runningMode)._result;
1347  }
1348 
1350  {
1351  if ( !hasError() )
1352  return std::string();
1353 
1354  return error().nativeErrorString();
1355  }
1356 
1358  {
1359  return error().isError();
1360  }
1361 
1362  bool NetworkRequest::addRequestHeader( const std::string &header )
1363  {
1364  Z_D();
1365 
1366  curl_slist *res = curl_slist_append( d->_headers ? d->_headers.get() : nullptr, header.c_str() );
1367  if ( !res )
1368  return false;
1369 
1370  if ( !d->_headers )
1371  d->_headers = std::unique_ptr< curl_slist, decltype (&curl_slist_free_all) >( res, &curl_slist_free_all );
1372 
1373  return true;
1374  }
1375 
1376  SignalProxy<void (NetworkRequest &req)> NetworkRequest::sigStarted()
1377  {
1378  return d_func()->_sigStarted;
1379  }
1380 
1381  SignalProxy<void (NetworkRequest &req, zypp::ByteCount count)> NetworkRequest::sigBytesDownloaded()
1382  {
1383  return d_func()->_sigBytesDownloaded;
1384  }
1385 
1386  SignalProxy<void (NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> NetworkRequest::sigProgress()
1387  {
1388  return d_func()->_sigProgress;
1389  }
1390 
1391  SignalProxy<void (zyppng::NetworkRequest &req, const zyppng::NetworkRequestError &err)> NetworkRequest::sigFinished()
1392  {
1393  return d_func()->_sigFinished;
1394  }
1395 
1396 }
Signal< void(NetworkRequest &req)> _sigStarted
Definition: request_p.h:132
long timeout() const
transfer timeout
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
std::string errorMessage() const
Definition: request.cc:741
bool isError() const
isError Will return true if this is a actual error
#define MIL
Definition: Logger.h:96
void setCurlOption(CURLoption opt, T data)
Definition: request_p.h:107
std::optional< Timings > timings() const
After the request is finished query the timings that were collected during download.
Definition: request.cc:1201
void * nativeHandle() const
Definition: request.cc:1196
std::optional< size_t > _chksumPad
Definition: request.h:88
#define DBG_MEDIA
Definition: mediadebug_p.h:28
unsigned size() const
Definition: Regex.cc:106
zypp::ByteCount reportedByteCount() const
Returns the number of bytes that are reported from the backend as the full download size...
Definition: request.cc:1288
const std::vector< Range > & requestedRanges() const
Definition: request.cc:1186
const Pathname & clientCertificatePath() const
SSL client certificate file.
std::chrono::microseconds connect
Definition: request.h:98
std::array< char, CURL_ERROR_SIZE+1 > _errorBuf
Definition: request_p.h:104
void addRequestRange(size_t start, size_t len=0, DigestPtr digest=nullptr, CheckSumBytes expectedChkSum=CheckSumBytes(), std::any userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > chksumpad={})
Definition: request.cc:1139
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Regular expression.
Definition: Regex.h:94
ZYPP_IMPL_PRIVATE(Provide)
std::optional< size_t > _relevantDigestLen
Definition: request.h:87
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
SignalProxy< void(NetworkRequest &req, zypp::ByteCount count)> sigBytesDownloaded()
Signals that new data has been downloaded, this is only the payload and does not include control data...
Definition: request.cc:1381
bool hasPrefixCI(const C_Str &str_r, const C_Str &prefix_r)
Definition: String.h:1030
NetworkRequest::FileMode _fMode
Definition: request_p.h:122
bool checkIfRangeChkSumIsValid(const NetworkRequest::Range &rng)
Definition: request.cc:675
Store and operate with byte count.
Definition: ByteCount.h:30
const std::string & lastRedirectInfo() const
Definition: request.cc:1191
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
const std::string _currentCookieFile
Definition: request_p.h:126
std::chrono::microseconds pretransfer
Definition: request.h:100
Holds transfer setting.
zypp::ByteCount downloadedByteCount() const
Returns the number of already downloaded bytes as reported by the backend.
Definition: request.cc:1302
const std::string & authType() const
get the allowed authentication types
NetworkRequest::Options _options
Definition: request_p.h:118
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
const char * c_str() const
String representation.
Definition: Pathname.h:110
String related utilities and Regular expression matching.
Definition: Arch.h:357
std::chrono::microseconds appconnect
Definition: request.h:99
bool prepareNextRangeBatch(std::string &errBuf)
Definition: request.cc:464
constexpr bool always_false
Definition: PathInfo.cc:544
running_t(pending_t &&prevState)
Definition: request.cc:95
std::string nativeErrorString() const
Signal< void(NetworkRequest &req, zypp::ByteCount count)> _sigBytesDownloaded
Definition: request_p.h:133
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
void setOptions(Options opt)
Definition: request.cc:1129
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
TransferSettings & transferSettings()
Definition: request.cc:1317
enum zyppng::NetworkRequestPrivate::ProtocolMode _protocolMode
void setExpectedFileSize(zypp::ByteCount expectedFileSize)
Definition: request.cc:1111
void setFileOpenMode(FileMode mode)
Sets the file open mode to mode.
Definition: request.cc:1272
bool hasError() const
Checks if there was a error with the request.
Definition: request.cc:1357
void onActivityTimeout(Timer &)
Definition: request.cc:664
const Headers & headers() const
returns a list of all added headers (trimmed)
static std::string digestVectorToString(const UByteArray &vec)
get hex string representation of the digest vector given as parameter
Definition: Digest.cc:184
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
zypp::Pathname _targetFile
Definition: request_p.h:116
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
void setUrl(const Url &url)
This will change the URL of the request.
Definition: request.cc:1245
std::chrono::microseconds namelookup
Definition: request.h:97
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
Do not differentiate case.
Definition: Regex.h:99
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
size_t headerCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:781
Convenient building of std::string via std::ostringstream Basically a std::ostringstream autoconverti...
Definition: String.h:211
bool addRequestHeader(const std::string &header)
Definition: request.cc:1362
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
const std::string & asString() const
String representation.
Definition: Pathname.h:91
Signal< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> _sigProgress
Definition: request_p.h:134
bool parseContentTypeMultiRangeHeader(const std::string_view &line, std::string &boundary)
Definition: request.cc:727
std::string asString() const
Error message provided by dumpOn as string.
Definition: Exception.cc:75
long connectTimeout() const
connection timeout
bool initialize(std::string &errBuf)
Definition: request.cc:124
#define WAR
Definition: Logger.h:97
#define nullptr
Definition: Easy.h:55
The NetworkRequestError class Represents a error that occured in.
std::vector< char > peekData(off_t offset, size_t count) const
Definition: request.cc:1229
NetworkRequestError error() const
Returns the last set Error.
Definition: request.cc:1341
zypp::ByteCount _expectedFileSize
Definition: request_p.h:119
static constexpr int _rangeAttempt[]
Definition: request_p.h:148
UByteArray CheckSumBytes
Definition: request.h:47
std::string extendedErrorString() const
In some cases, curl can provide extended error information collected at runtime.
Definition: request.cc:1349
Priority priority() const
Definition: request.cc:1124
std::string proxyuserpwd
Definition: curlconfig.h:49
bool setupHandle(std::string &errBuf)
Definition: request.cc:136
const Pathname & clientKeyPath() const
SSL client key file.
const zypp::Pathname & targetFilePath() const
Returns the target filename path.
Definition: request.cc:1254
void validateRange(NetworkRequest::Range &rng)
Definition: request.cc:695
std::unique_ptr< curl_slist, decltype(&curl_slist_free_all) > _headers
Definition: request_p.h:141
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
#define MIL_MEDIA
Definition: mediadebug_p.h:29
bool parseContentRangeHeader(const std::string_view &line, size_t &start, size_t &len)
Definition: request.cc:710
std::vector< char > peek_data_fd(FILE *fd, off_t offset, size_t count)
Definition: request.cc:56
#define ZYPP_CAUGHT(EXCPT)
Drops a logline telling the Exception was caught (in order to handle it).
Definition: Exception.h:436
bool proxyEnabled() const
proxy is enabled
void setTargetFilePath(const zypp::Pathname &path)
Changes the target file path of the download.
Definition: request.cc:1259
static int curlProgressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Definition: request.cc:755
Regular expression match result.
Definition: Regex.h:167
std::string contentType() const
Returns the content type as reported from the server.
Definition: request.cc:1280
static const Unit B
1 Byte
Definition: ByteCount.h:42
Base class for Exception.
Definition: Exception.h:145
std::string _lastRedirect
to log/report redirections
Definition: request_p.h:125
std::chrono::microseconds total
Definition: request.h:101
bool any_of(const Container &c, Fnc &&cb)
Definition: Algorithm.h:76
CheckSumBytes _checksum
Enables automated checking of downloaded contents against a checksum.
Definition: request.h:86
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
static Range make(size_t start, size_t len=0, DigestPtr &&digest=nullptr, CheckSumBytes &&expectedChkSum=CheckSumBytes(), std::any &&userData=std::any(), std::optional< size_t > digestCompareLen={}, std::optional< size_t > _dataBlockPadding={})
Definition: request.cc:74
void setPriority(Priority prio, bool triggerReschedule=true)
Definition: request.cc:1116
State state() const
Returns the current state the HttpDownloadRequest is in.
Definition: request.cc:1322
TransferSettings _settings
Definition: request_p.h:117
NetworkRequestDispatcher * _dispatcher
Definition: request_p.h:129
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
virtual ~NetworkRequest()
Definition: request.cc:1103
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Options options() const
Definition: request.cc:1134
std::vector< NetworkRequest::Range > _requestedRanges
the requested ranges that need to be downloaded
Definition: request_p.h:120
size_t writeCallback(char *ptr, size_t size, size_t nmemb)
Definition: request.cc:867
bool regex_match(const std::string &s, smatch &matches, const regex &regex)
regex ZYPP_STR_REGEX regex ZYPP_STR_REGEX
Definition: Regex.h:70
std::chrono::microseconds redirect
Definition: request.h:102
std::shared_ptr< zypp::Digest > DigestPtr
Definition: request.h:46
SignalProxy< void(NetworkRequest &req, const NetworkRequestError &err)> sigFinished()
Signals that the download finished.
Definition: request.cc:1391
Signal< void(NetworkRequest &req, const NetworkRequestError &err)> _sigFinished
Definition: request_p.h:135
Type type() const
type Returns the type of the error
These are enforced even if you don&#39;t pass them as flag argument.
Definition: Regex.h:103
SignalProxy< void(NetworkRequest &req)> sigStarted()
Signals that the dispatcher dequeued the request and actually starts downloading data.
Definition: request.cc:1376
std::string userPassword() const
returns the user and password as a user:pass string
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
FileMode fileOpenMode() const
Returns the currently configured file open mode.
Definition: request.cc:1267
std::vector< Range > failedRanges() const
Definition: request.cc:1170
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
NetworkRequestPrivate(Url &&url, zypp::Pathname &&targetFile, NetworkRequest::FileMode fMode, NetworkRequest &p)
Definition: request.cc:106
void setResult(NetworkRequestError &&err)
Definition: request.cc:618
const std::string & proxy() const
proxy host
static zyppng::NetworkRequestError customError(NetworkRequestError::Type t, std::string &&errorMsg="", std::map< std::string, boost::any > &&extraInfo={})
SignalProxy< void(NetworkRequest &req, off_t dltotal, off_t dlnow, off_t ultotal, off_t ulnow)> sigProgress()
Signals if there was data read from the download.
Definition: request.cc:1386
bool prepareToContinue(std::string &errBuf)
Definition: request.cc:432
const std::string & userAgentString() const
user agent string (trimmed)
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
ZYppCommitResult & _result
Definition: TargetImpl.cc:1596
std::variant< pending_t, running_t, prepareNextRangeBatch_t, finished_t > _runningMode
Definition: request_p.h:211