libzypp  17.31.27
MediaCurl.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
13 #include <iostream>
14 #include <chrono>
15 #include <list>
16 
17 #include <zypp/base/Logger.h>
18 #include <zypp/ExternalProgram.h>
19 #include <zypp/base/String.h>
20 #include <zypp/base/Gettext.h>
21 #include <zypp-core/parser/Sysconfig>
22 #include <zypp/base/Gettext.h>
23 
24 #include <zypp/media/MediaCurl.h>
25 #include <zypp-curl/ProxyInfo>
26 #include <zypp-curl/auth/CurlAuthData>
27 #include <zypp-media/auth/CredentialManager>
28 #include <zypp-curl/CurlConfig>
30 #include <zypp/Target.h>
31 #include <zypp/ZYppFactory.h>
32 #include <zypp/ZConfig.h>
33 
34 #include <cstdlib>
35 #include <sys/types.h>
36 #include <sys/stat.h>
37 #include <sys/mount.h>
38 #include <errno.h>
39 #include <dirent.h>
40 #include <unistd.h>
41 
42 using std::endl;
43 
44 namespace internal {
45  using namespace zypp;
49  struct OptionalDownloadProgressReport : public callback::ReceiveReport<media::DownloadProgressReport>
50  {
51  using TimePoint = std::chrono::steady_clock::time_point;
52 
53  OptionalDownloadProgressReport( bool isOptional=false )
54  : _oldRec { Distributor::instance().getReceiver() }
55  , _isOptional { isOptional }
56  { connect(); }
57 
59  { if ( _oldRec ) Distributor::instance().setReceiver( *_oldRec ); else Distributor::instance().noReceiver(); }
60 
61  void reportbegin() override
62  { if ( _oldRec ) _oldRec->reportbegin(); }
63 
64  void reportend() override
65  { if ( _oldRec ) _oldRec->reportend(); }
66 
67  void report( const UserData & userData_r = UserData() ) override
68  { if ( _oldRec ) _oldRec->report( userData_r ); }
69 
70 
71  void start( const Url & file_r, Pathname localfile_r ) override
72  {
73  if ( not _oldRec ) return;
74  if ( _isOptional ) {
75  // delay start until first data are received.
76  _startFile = file_r;
77  _startLocalfile = std::move(localfile_r);
78  return;
79  }
80  _oldRec->start( file_r, localfile_r );
81  }
82 
83  bool progress( int value_r, const Url & file_r, double dbps_avg_r = -1, double dbps_current_r = -1 ) override
84  {
85  if ( not _oldRec ) return true;
86  if ( notStarted() ) {
87  if ( not ( value_r || dbps_avg_r || dbps_current_r ) )
88  return true;
89  sendStart();
90  }
91 
92  //static constexpr std::chrono::milliseconds minfequency { 1000 }; only needed if we'd avoid sending reports without change
93  static constexpr std::chrono::milliseconds maxfequency { 100 };
94  TimePoint now { TimePoint::clock::now() };
95  TimePoint::duration elapsed { now - _lastProgressSent };
96  if ( elapsed < maxfequency )
97  return true; // continue
98  _lastProgressSent = now;
99  return _oldRec->progress( value_r, file_r, dbps_avg_r, dbps_current_r );
100  }
101 
102  Action problem( const Url & file_r, Error error_r, const std::string & description_r ) override
103  {
104  if ( not _oldRec || notStarted() ) return ABORT;
105  return _oldRec->problem( file_r, error_r, description_r );
106  }
107 
108  void finish( const Url & file_r, Error error_r, const std::string & reason_r ) override
109  {
110  if ( not _oldRec || notStarted() ) return;
111  _oldRec->finish( file_r, error_r, reason_r );
112  }
113 
114  private:
115  // _isOptional also indicates the delayed start
116  bool notStarted() const
117  { return _isOptional; }
118 
119  void sendStart()
120  {
121  if ( _isOptional ) {
122  // we know _oldRec is valid...
123  _oldRec->start( std::move(_startFile), std::move(_startLocalfile) );
124  _isOptional = false;
125  }
126  }
127 
128  private:
129  Receiver *const _oldRec;
132  Pathname _startLocalfile;
134  };
135 
137  {
138  ProgressData( CURL *curl, time_t timeout = 0, const zypp::Url & url = zypp::Url(),
139  zypp::ByteCount expectedFileSize_r = 0,
141 
142  void updateStats( curl_off_t dltotal = 0.0, curl_off_t dlnow = 0.0 );
143 
144  int reportProgress() const;
145 
146  CURL * curl()
147  { return _curl; }
148 
149  bool timeoutReached() const
150  { return _timeoutReached; }
151 
152  bool fileSizeExceeded() const
153  { return _fileSizeExceeded; }
154 
156  { return _expectedFileSize; }
157 
158  void expectedFileSize( ByteCount newval_r )
159  { _expectedFileSize = newval_r; }
160 
161  private:
162  CURL * _curl;
164  time_t _timeout;
169 
170  time_t _timeStart = 0;
171  time_t _timeLast = 0;
172  time_t _timeRcv = 0;
173  time_t _timeNow = 0;
174 
175  curl_off_t _dnlTotal = 0.0;
176  curl_off_t _dnlLast = 0.0;
177  curl_off_t _dnlNow = 0.0;
178 
179  int _dnlPercent= 0;
180 
181  double _drateTotal= 0.0;
182  double _drateLast = 0.0;
183  };
184 
185 
186 
187  ProgressData::ProgressData(CURL *curl, time_t timeout, const Url &url, ByteCount expectedFileSize_r, zypp::callback::SendReport< zypp::media::DownloadProgressReport> *_report)
188  : _curl( curl )
189  , _url( url )
190  , _timeout( timeout )
191  , _timeoutReached( false )
192  , _fileSizeExceeded ( false )
193  , _expectedFileSize( expectedFileSize_r )
194  , report( _report )
195  {}
196 
197  void ProgressData::updateStats( curl_off_t dltotal, curl_off_t dlnow )
198  {
199  time_t now = _timeNow = time(0);
200 
201  // If called without args (0.0), recompute based on the last values seen
202  if ( dltotal && dltotal != _dnlTotal )
203  _dnlTotal = dltotal;
204 
205  if ( dlnow && dlnow != _dnlNow )
206  {
207  _timeRcv = now;
208  _dnlNow = dlnow;
209  }
210 
211  // init or reset if time jumps back
212  if ( !_timeStart || _timeStart > now )
213  _timeStart = _timeLast = _timeRcv = now;
214 
215  // timeout condition
216  if ( _timeout )
217  _timeoutReached = ( (now - _timeRcv) > _timeout );
218 
219  // check if the downloaded data is already bigger than what we expected
220  _fileSizeExceeded = _expectedFileSize > 0 && _expectedFileSize < static_cast<ByteCount::SizeType>(_dnlNow);
221 
222  // percentage:
223  if ( _dnlTotal )
224  _dnlPercent = int( _dnlNow * 100 / _dnlTotal );
225 
226  // download rates:
227  _drateTotal = double(_dnlNow) / std::max( int(now - _timeStart), 1 );
228 
229  if ( _timeLast < now )
230  {
231  _drateLast = double(_dnlNow - _dnlLast) / int(now - _timeLast);
232  // start new period
233  _timeLast = now;
234  _dnlLast = _dnlNow;
235  }
236  else if ( _timeStart == _timeLast )
238  }
239 
241  {
242  if ( _fileSizeExceeded )
243  return 1;
244  if ( _timeoutReached )
245  return 1; // no-data timeout
246  if ( report && !(*report)->progress( _dnlPercent, _url, _drateTotal, _drateLast ) )
247  return 1; // user requested abort
248  return 0;
249  }
250 
251  const char * anonymousIdHeader()
252  {
253  // we need to add the release and identifier to the
254  // agent string.
255  // The target could be not initialized, and then this information
256  // is guessed.
257  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
258  static const std::string _value( str::trim( str::form(
259  "X-ZYpp-AnonymousId: %s",
260  Target::anonymousUniqueId( Pathname()/*guess root*/ ).c_str()
261  )));
262  return _value.c_str();
263  }
264 
266  {
267  // we need to add the release and identifier to the
268  // agent string.
269  // The target could be not initialized, and then this information
270  // is guessed.
271  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
272  static const std::string _value( str::trim( str::form(
273  "X-ZYpp-DistributionFlavor: %s",
274  Target::distributionFlavor( Pathname()/*guess root*/ ).c_str()
275  )));
276  return _value.c_str();
277  }
278 
279  const char * agentString()
280  {
281  // we need to add the release and identifier to the
282  // agent string.
283  // The target could be not initialized, and then this information
284  // is guessed.
285  // bsc#1212187: HTTP/2 RFC 9113 forbids fields ending with a space
286  static const std::string _value( str::trim( str::form(
287  "ZYpp " LIBZYPP_VERSION_STRING " (curl %s) %s"
288  , curl_version_info(CURLVERSION_NOW)->version
289  , Target::targetDistribution( Pathname()/*guess root*/ ).c_str()
290  )));
291  return _value.c_str();
292  }
293 
298  {
299  public:
301  const std::string & err_r,
302  const std::string & msg_r )
303  : media::MediaCurlException( url_r, err_r, msg_r )
304  {}
305  //~MediaCurlExceptionMayRetryInternaly() noexcept {}
306  };
307 
308 }
309 
310 
311 using namespace internal;
312 using namespace zypp::base;
313 
314 namespace zypp {
315 
316  namespace media {
317 
318 Pathname MediaCurl::_cookieFile = "/var/lib/YaST2/cookies";
319 
320 // we use this define to unbloat code as this C setting option
321 // and catching exception is done frequently.
323 #define SET_OPTION(opt,val) do { \
324  ret = curl_easy_setopt ( _curl, opt, val ); \
325  if ( ret != 0) { \
326  ZYPP_THROW(MediaCurlSetOptException(_url, _curlError)); \
327  } \
328  } while ( false )
329 
330 #define SET_OPTION_OFFT(opt,val) SET_OPTION(opt,(curl_off_t)val)
331 #define SET_OPTION_LONG(opt,val) SET_OPTION(opt,(long)val)
332 #define SET_OPTION_VOID(opt,val) SET_OPTION(opt,(void*)val)
333 
334 MediaCurl::MediaCurl( const Url & url_r,
335  const Pathname & attach_point_hint_r )
336  : MediaNetworkCommonHandler( url_r, attach_point_hint_r,
337  "/", // urlpath at attachpoint
338  true ), // does_download
339  _curl( NULL ),
340  _customHeaders(0L)
341 {
342  _curlError[0] = '\0';
343 
344  MIL << "MediaCurl::MediaCurl(" << url_r << ", " << attach_point_hint_r << ")" << endl;
345 
347 
348  if( !attachPoint().empty())
349  {
350  PathInfo ainfo(attachPoint());
351  Pathname apath(attachPoint() + "XXXXXX");
352  char *atemp = ::strdup( apath.asString().c_str());
353  char *atest = NULL;
354  if( !ainfo.isDir() || !ainfo.userMayRWX() ||
355  atemp == NULL || (atest=::mkdtemp(atemp)) == NULL)
356  {
357  WAR << "attach point " << ainfo.path()
358  << " is not useable for " << url_r.getScheme() << endl;
359  setAttachPoint("", true);
360  }
361  else if( atest != NULL)
362  ::rmdir(atest);
363 
364  if( atemp != NULL)
365  ::free(atemp);
366  }
367 }
368 
370 {
372 }
373 
374 void MediaCurl::setCookieFile( const Pathname &fileName )
375 {
376  _cookieFile = fileName;
377 }
378 
380 
381 void MediaCurl::checkProtocol(const Url &url) const
382 {
383  curl_version_info_data *curl_info = NULL;
384  curl_info = curl_version_info(CURLVERSION_NOW);
385  // curl_info does not need any free (is static)
386  if (curl_info->protocols)
387  {
388  const char * const *proto;
389  std::string scheme( url.getScheme());
390  bool found = false;
391  for(proto=curl_info->protocols; !found && *proto; ++proto)
392  {
393  if( scheme == std::string((const char *)*proto))
394  found = true;
395  }
396  if( !found)
397  {
398  std::string msg("Unsupported protocol '");
399  msg += scheme;
400  msg += "'";
402  }
403  }
404 }
405 
407 {
409 
410  curl_easy_setopt(_curl, CURLOPT_HEADERFUNCTION, log_redirects_curl);
411  curl_easy_setopt(_curl, CURLOPT_HEADERDATA, &_lastRedirect);
412  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_ERRORBUFFER, _curlError );
413  if ( ret != 0 ) {
414  ZYPP_THROW(MediaCurlSetOptException(_url, "Error setting error buffer"));
415  }
416 
417  SET_OPTION(CURLOPT_FAILONERROR, 1L);
418  SET_OPTION(CURLOPT_NOSIGNAL, 1L);
419 
420  // create non persistant settings
421  // so that we don't add headers twice
422  TransferSettings vol_settings(_settings);
423 
424  // add custom headers for download.opensuse.org (bsc#955801)
425  if ( _url.getHost() == "download.opensuse.org" )
426  {
427  vol_settings.addHeader(anonymousIdHeader());
428  vol_settings.addHeader(distributionFlavorHeader());
429  }
430  vol_settings.addHeader("Pragma:");
431 
433 
434  // fill some settings from url query parameters
435  try
436  {
438  }
439  catch ( const MediaException &e )
440  {
441  disconnectFrom();
442  ZYPP_RETHROW(e);
443  }
444  // if the proxy was not set (or explicitly unset) by url, then look...
445  if ( _settings.proxy().empty() )
446  {
447  // ...at the system proxy settings
449  }
450 
452  switch ( env::ZYPP_MEDIA_CURL_IPRESOLVE() )
453  {
454  case 4: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V4); break;
455  case 6: SET_OPTION(CURLOPT_IPRESOLVE, CURL_IPRESOLVE_V6); break;
456  }
457 
461  SET_OPTION(CURLOPT_CONNECTTIMEOUT, _settings.connectTimeout());
462  // If a transfer timeout is set, also set CURLOPT_TIMEOUT to an upper limit
463  // just in case curl does not trigger its progress callback frequently
464  // enough.
465  if ( _settings.timeout() )
466  {
467  SET_OPTION(CURLOPT_TIMEOUT, 3600L);
468  }
469 
470  // follow any Location: header that the server sends as part of
471  // an HTTP header (#113275)
472  SET_OPTION(CURLOPT_FOLLOWLOCATION, 1L);
473  // 3 redirects seem to be too few in some cases (bnc #465532)
474  SET_OPTION(CURLOPT_MAXREDIRS, 6L);
475 
476  if ( _url.getScheme() == "https" )
477  {
478 #if CURLVERSION_AT_LEAST(7,19,4)
479  // restrict following of redirections from https to https only
480  if ( _url.getHost() == "download.opensuse.org" ) {
481 #if CURLVERSION_AT_LEAST(7,85,0)
482  SET_OPTION( CURLOPT_REDIR_PROTOCOLS_STR, "http,https" );
483 #else
484  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTP | CURLPROTO_HTTPS );
485 #endif
486  } else {
487 #if CURLVERSION_AT_LEAST(7,85,0)
488  SET_OPTION( CURLOPT_REDIR_PROTOCOLS_STR, "https" );
489 #else
490  SET_OPTION( CURLOPT_REDIR_PROTOCOLS, CURLPROTO_HTTPS );
491 #endif
492  }
493 #endif // if CURLVERSION_AT_LEAST(7,19,4)
494 
497  {
499  }
500 
502  {
503  SET_OPTION(CURLOPT_SSLCERT, _settings.clientCertificatePath().c_str());
504  }
505  if( ! _settings.clientKeyPath().empty() )
506  {
507  SET_OPTION(CURLOPT_SSLKEY, _settings.clientKeyPath().c_str());
508  }
509 
510 #ifdef CURLSSLOPT_ALLOW_BEAST
511  // see bnc#779177
512  ret = curl_easy_setopt( _curl, CURLOPT_SSL_OPTIONS, CURLSSLOPT_ALLOW_BEAST );
513  if ( ret != 0 ) {
514  disconnectFrom();
516  }
517 #endif
518  SET_OPTION(CURLOPT_SSL_VERIFYPEER, _settings.verifyPeerEnabled() ? 1L : 0L);
519  SET_OPTION(CURLOPT_SSL_VERIFYHOST, _settings.verifyHostEnabled() ? 2L : 0L);
520  // bnc#903405 - POODLE: libzypp should only talk TLS
521  SET_OPTION(CURLOPT_SSLVERSION, CURL_SSLVERSION_TLSv1);
522  }
523 
524  SET_OPTION(CURLOPT_USERAGENT, _settings.userAgentString().c_str() );
525 
526  /* Fixes bsc#1174011 "auth=basic ignored in some cases"
527  * We should proactively add the password to the request if basic auth is configured
528  * and a password is available in the credentials but not in the URL.
529  *
530  * We will be a bit paranoid here and require that the URL has a user embedded, otherwise we go the default route
531  * and ask the server first about the auth method
532  */
533  if ( _settings.authType() == "basic"
534  && _settings.username().size()
535  && !_settings.password().size() ) {
536 
537  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
538  const auto cred = cm.getCred( _url );
539  if ( cred && cred->valid() ) {
540  if ( !_settings.username().size() )
541  _settings.setUsername(cred->username());
542  _settings.setPassword(cred->password());
543  }
544  }
545 
546  /*---------------------------------------------------------------*
547  CURLOPT_USERPWD: [user name]:[password]
548 
549  Url::username/password -> CURLOPT_USERPWD
550  If not provided, anonymous FTP identification
551  *---------------------------------------------------------------*/
552 
553  if ( _settings.userPassword().size() )
554  {
555  SET_OPTION(CURLOPT_USERPWD, _settings.userPassword().c_str());
556  std::string use_auth = _settings.authType();
557  if (use_auth.empty())
558  use_auth = "digest,basic"; // our default
559  long auth = CurlAuthData::auth_type_str2long(use_auth);
560  if( auth != CURLAUTH_NONE)
561  {
562  DBG << "Enabling HTTP authentication methods: " << use_auth
563  << " (CURLOPT_HTTPAUTH=" << auth << ")" << std::endl;
564  SET_OPTION(CURLOPT_HTTPAUTH, auth);
565  }
566  }
567 
568  if ( _settings.proxyEnabled() && ! _settings.proxy().empty() )
569  {
570  DBG << "Proxy: '" << _settings.proxy() << "'" << endl;
571  SET_OPTION(CURLOPT_PROXY, _settings.proxy().c_str());
572  SET_OPTION(CURLOPT_PROXYAUTH, CURLAUTH_BASIC|CURLAUTH_DIGEST|CURLAUTH_NTLM );
573  /*---------------------------------------------------------------*
574  * CURLOPT_PROXYUSERPWD: [user name]:[password]
575  *
576  * Url::option(proxyuser and proxypassword) -> CURLOPT_PROXYUSERPWD
577  * If not provided, $HOME/.curlrc is evaluated
578  *---------------------------------------------------------------*/
579 
580  std::string proxyuserpwd = _settings.proxyUserPassword();
581 
582  if ( proxyuserpwd.empty() )
583  {
584  CurlConfig curlconf;
585  CurlConfig::parseConfig(curlconf); // parse ~/.curlrc
586  if ( curlconf.proxyuserpwd.empty() )
587  DBG << "Proxy: ~/.curlrc does not contain the proxy-user option" << endl;
588  else
589  {
590  proxyuserpwd = curlconf.proxyuserpwd;
591  DBG << "Proxy: using proxy-user from ~/.curlrc" << endl;
592  }
593  }
594  else
595  {
596  DBG << "Proxy: using provided proxy-user '" << _settings.proxyUsername() << "'" << endl;
597  }
598 
599  if ( ! proxyuserpwd.empty() )
600  {
601  SET_OPTION(CURLOPT_PROXYUSERPWD, curlUnEscape( proxyuserpwd ).c_str());
602  }
603  }
604 #if CURLVERSION_AT_LEAST(7,19,4)
605  else if ( _settings.proxy() == EXPLICITLY_NO_PROXY )
606  {
607  // Explicitly disabled in URL (see fillSettingsFromUrl()).
608  // This should also prevent libcurl from looking into the environment.
609  DBG << "Proxy: explicitly NOPROXY" << endl;
610  SET_OPTION(CURLOPT_NOPROXY, "*");
611  }
612 #endif
613  else
614  {
615  DBG << "Proxy: not explicitly set" << endl;
616  DBG << "Proxy: libcurl may look into the environment" << endl;
617  }
618 
620  if ( _settings.minDownloadSpeed() != 0 )
621  {
622  SET_OPTION(CURLOPT_LOW_SPEED_LIMIT, _settings.minDownloadSpeed());
623  // default to 10 seconds at low speed
624  SET_OPTION(CURLOPT_LOW_SPEED_TIME, 60L);
625  }
626 
627 #if CURLVERSION_AT_LEAST(7,15,5)
628  if ( _settings.maxDownloadSpeed() != 0 )
629  SET_OPTION_OFFT(CURLOPT_MAX_RECV_SPEED_LARGE, _settings.maxDownloadSpeed());
630 #endif
631 
632  /*---------------------------------------------------------------*
633  *---------------------------------------------------------------*/
634 
636  if ( ::geteuid() == 0 || PathInfo(_currentCookieFile).owner() == ::geteuid() )
638  if ( str::strToBool( _url.getQueryParam( "cookies" ), true ) )
639  SET_OPTION(CURLOPT_COOKIEFILE, _currentCookieFile.c_str() );
640  else
641  MIL << "No cookies requested" << endl;
642  SET_OPTION(CURLOPT_COOKIEJAR, _currentCookieFile.c_str() );
643  SET_OPTION(CURLOPT_XFERINFOFUNCTION, &progressCallback );
644  SET_OPTION(CURLOPT_NOPROGRESS, 0L);
645 
646 #if CURLVERSION_AT_LEAST(7,18,0)
647  // bnc #306272
648  SET_OPTION(CURLOPT_PROXY_TRANSFER_MODE, 1L );
649 #endif
650  // Append settings custom headers to curl.
651  // TransferSettings assert strings are trimmed (HTTP/2 RFC 9113)
652  for ( const auto &header : vol_settings.headers() ) {
653  _customHeaders = curl_slist_append(_customHeaders, header.c_str());
654  if ( !_customHeaders )
656  }
657  SET_OPTION(CURLOPT_HTTPHEADER, _customHeaders);
658 }
659 
661 
662 
663 void MediaCurl::attachTo (bool next)
664 {
665  if ( next )
667 
668  if ( !_url.isValid() )
670 
673  {
675  }
676 
677  disconnectFrom(); // clean _curl if needed
678  _curl = curl_easy_init();
679  if ( !_curl ) {
681  }
682  try
683  {
684  setupEasy();
685  }
686  catch (Exception & ex)
687  {
688  disconnectFrom();
689  ZYPP_RETHROW(ex);
690  }
691 
692  // FIXME: need a derived class to propelly compare url's
694  setMediaSource(media);
695 }
696 
697 bool
699 {
700  return MediaHandler::checkAttachPoint( apoint, true, true);
701 }
702 
704 
706 {
707  if ( _customHeaders )
708  {
709  curl_slist_free_all(_customHeaders);
710  _customHeaders = 0L;
711  }
712 
713  if ( _curl )
714  {
715  // bsc#1201092: Strange but within global_dtors we may exceptions here.
716  try { curl_easy_cleanup( _curl ); }
717  catch (...) { ; }
718  _curl = NULL;
719  }
720 }
721 
723 
724 void MediaCurl::releaseFrom( const std::string & ejectDev )
725 {
726  disconnect();
727 }
728 
730 
731 void MediaCurl::getFile( const OnMediaLocation &file ) const
732 {
733  // Use absolute file name to prevent access of files outside of the
734  // hierarchy below the attach point.
735  getFileCopy( file, localPath(file.filename()).absolutename() );
736 }
737 
739 
740 void MediaCurl::getFileCopy( const OnMediaLocation & srcFile , const Pathname & target ) const
741 {
742 
743  const auto &filename = srcFile.filename();
744 
745  // Optional files will send no report until data are actually received (we know it exists).
746  OptionalDownloadProgressReport reportfilter( srcFile.optional() );
748 
749  Url fileurl(getFileUrl(filename));
750 
751  bool firstAuth = true; // bsc#1210870: authenticate must not return stored credentials more than once.
752  unsigned internalTry = 0;
753  static constexpr unsigned maxInternalTry = 3;
754 
755  do
756  {
757  try
758  {
759  doGetFileCopy( srcFile, target, report );
760  break; // success!
761  }
762  // retry with proper authentication data
763  catch (MediaUnauthorizedException & ex_r)
764  {
765  if ( authenticate(ex_r.hint(), firstAuth) ) {
766  firstAuth = false; // must not return stored credentials again
767  continue; // retry
768  }
769 
770  report->finish(fileurl, zypp::media::DownloadProgressReport::ACCESS_DENIED, ex_r.asUserHistory());
771  ZYPP_RETHROW(ex_r);
772  }
773  // unexpected exception
774  catch (MediaException & excpt_r)
775  {
776  if ( typeid(excpt_r) == typeid( MediaCurlExceptionMayRetryInternaly ) ) {
777  ++internalTry;
778  if ( internalTry < maxInternalTry ) {
779  // just report (NO_ERROR); no interactive request to the user
780  report->problem(fileurl, media::DownloadProgressReport::NO_ERROR, excpt_r.asUserHistory()+_("Will try again..."));
781  continue; // retry
782  }
783  excpt_r.addHistory( str::Format(_("Giving up after %1% attempts.")) % maxInternalTry );
784  }
785 
787  if( typeid(excpt_r) == typeid( media::MediaFileNotFoundException ) ||
788  typeid(excpt_r) == typeid( media::MediaNotAFileException ) )
789  {
791  }
792  report->finish(fileurl, reason, excpt_r.asUserHistory());
793  ZYPP_RETHROW(excpt_r);
794  }
795  }
796  while ( true );
797  report->finish(fileurl, zypp::media::DownloadProgressReport::NO_ERROR, "");
798 }
799 
801 
802 bool MediaCurl::getDoesFileExist( const Pathname & filename ) const
803 {
804  bool retry = false;
805 
806  do
807  {
808  try
809  {
810  return doGetDoesFileExist( filename );
811  }
812  // authentication problem, retry with proper authentication data
813  catch (MediaUnauthorizedException & ex_r)
814  {
815  if(authenticate(ex_r.hint(), !retry))
816  retry = true;
817  else
818  ZYPP_RETHROW(ex_r);
819  }
820  // unexpected exception
821  catch (MediaException & excpt_r)
822  {
823  ZYPP_RETHROW(excpt_r);
824  }
825  }
826  while (retry);
827 
828  return false;
829 }
830 
832 
834  CURLcode code,
835  bool timeout_reached) const
836 {
837  if ( code != 0 )
838  {
839  Url url;
840  if (filename.empty())
841  url = _url;
842  else
843  url = getFileUrl(filename);
844 
845  std::string err;
846  {
847  switch ( code )
848  {
849  case CURLE_UNSUPPORTED_PROTOCOL:
850  err = " Unsupported protocol";
851  if ( !_lastRedirect.empty() )
852  {
853  err += " or redirect (";
854  err += _lastRedirect;
855  err += ")";
856  }
857  break;
858  case CURLE_URL_MALFORMAT:
859  case CURLE_URL_MALFORMAT_USER:
860  err = " Bad URL";
861  break;
862  case CURLE_LOGIN_DENIED:
863  ZYPP_THROW(
864  MediaUnauthorizedException(url, "Login failed.", _curlError, ""));
865  break;
866  case CURLE_HTTP_RETURNED_ERROR:
867  {
868  long httpReturnCode = 0;
869  CURLcode infoRet = curl_easy_getinfo( _curl,
870  CURLINFO_RESPONSE_CODE,
871  &httpReturnCode );
872  if ( infoRet == CURLE_OK )
873  {
874  std::string msg = "HTTP response: " + str::numstring( httpReturnCode );
875  switch ( httpReturnCode )
876  {
877  case 401:
878  {
879  std::string auth_hint = getAuthHint();
880 
881  DBG << msg << " Login failed (URL: " << url.asString() << ")" << std::endl;
882  DBG << "MediaUnauthorizedException auth hint: '" << auth_hint << "'" << std::endl;
883 
885  url, "Login failed.", _curlError, auth_hint
886  ));
887  }
888 
889  case 502: // bad gateway (bnc #1070851)
890  case 503: // service temporarily unavailable (bnc #462545)
892  case 504: // gateway timeout
894  case 403:
895  {
896  std::string msg403;
897  if ( url.getHost().find(".suse.com") != std::string::npos )
898  msg403 = _("Visit the SUSE Customer Center to check whether your registration is valid and has not expired.");
899  else if (url.asString().find("novell.com") != std::string::npos)
900  msg403 = _("Visit the Novell Customer Center to check whether your registration is valid and has not expired.");
902  }
903  case 404:
904  case 410:
906  }
907 
908  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
910  }
911  else
912  {
913  std::string msg = "Unable to retrieve HTTP response:";
914  DBG << msg << " (URL: " << url.asString() << ")" << std::endl;
916  }
917  }
918  break;
919  case CURLE_FTP_COULDNT_RETR_FILE:
920 #if CURLVERSION_AT_LEAST(7,16,0)
921  case CURLE_REMOTE_FILE_NOT_FOUND:
922 #endif
923  case CURLE_FTP_ACCESS_DENIED:
924  case CURLE_TFTP_NOTFOUND:
925  err = "File not found";
927  break;
928  case CURLE_BAD_PASSWORD_ENTERED:
929  case CURLE_FTP_USER_PASSWORD_INCORRECT:
930  err = "Login failed";
931  break;
932  case CURLE_COULDNT_RESOLVE_PROXY:
933  case CURLE_COULDNT_RESOLVE_HOST:
934  case CURLE_COULDNT_CONNECT:
935  case CURLE_FTP_CANT_GET_HOST:
936  err = "Connection failed";
937  break;
938  case CURLE_WRITE_ERROR:
939  err = "Write error";
940  break;
941  case CURLE_PARTIAL_FILE:
942  case CURLE_OPERATION_TIMEDOUT:
943  timeout_reached = true; // fall though to TimeoutException
944  // fall though...
945  case CURLE_ABORTED_BY_CALLBACK:
946  if( timeout_reached )
947  {
948  err = "Timeout reached";
950  }
951  else
952  {
953  err = "User abort";
954  }
955  break;
956 
957  // Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy
958  case CURLE_HTTP2:
959  case CURLE_HTTP2_STREAM:
960  err = "Curl error " + str::numstring( code );
962  break;
963 
964  default:
965  err = "Curl error " + str::numstring( code );
966  break;
967  }
968 
969  // uhm, no 0 code but unknown curl exception
971  }
972  }
973  else
974  {
975  // actually the code is 0, nothing happened
976  }
977 }
978 
980 
981 bool MediaCurl::doGetDoesFileExist( const Pathname & filename ) const
982 {
983  DBG << filename.asString() << endl;
984 
985  if(!_url.isValid())
987 
988  if(_url.getHost().empty())
990 
991  Url url(getFileUrl(filename));
992 
993  DBG << "URL: " << url.asString() << endl;
994  // Use URL without options and without username and passwd
995  // (some proxies dislike them in the URL).
996  // Curl seems to need the just scheme, hostname and a path;
997  // the rest was already passed as curl options (in attachTo).
998  Url curlUrl( clearQueryString(url) );
999 
1000  //
1001  // See also Bug #154197 and ftp url definition in RFC 1738:
1002  // The url "ftp://user@host/foo/bar/file" contains a path,
1003  // that is relative to the user's home.
1004  // The url "ftp://user@host//foo/bar/file" (or also with
1005  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1006  // contains an absolute path.
1007  //
1008  _lastRedirect.clear();
1009  std::string urlBuffer( curlUrl.asString());
1010  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1011  urlBuffer.c_str() );
1012  if ( ret != 0 ) {
1014  }
1015 
1016  // If no head requests allowed (?head_requests=no):
1017  // Instead of returning no data with NOBODY, we return
1018  // little data, that works with broken servers, and
1019  // works for ftp as well, because retrieving only headers
1020  // ftp will return always OK code ?
1021  // See http://curl.haxx.se/docs/knownbugs.html #58
1023  struct TempSetHeadRequest
1024  {
1025  TempSetHeadRequest( CURL * curl_r, bool doHttpHeadRequest_r )
1026  : _curl { curl_r }
1027  , _doHttpHeadRequest { doHttpHeadRequest_r }
1028  {
1029  if ( _doHttpHeadRequest ) {
1030  curl_easy_setopt( _curl, CURLOPT_NOBODY, 1L );
1031  } else {
1032  curl_easy_setopt( _curl, CURLOPT_RANGE, "0-1" );
1033  }
1034  }
1035  ~TempSetHeadRequest() {
1036  if ( _doHttpHeadRequest ) {
1037  curl_easy_setopt( _curl, CURLOPT_NOBODY, 0L);
1038  /* yes, this is why we never got to get NOBODY working before,
1039  because setting it changes this option too, and we also*
1040  need to reset it
1041  See: http://curl.haxx.se/mail/archive-2005-07/0073.html
1042  */
1043  curl_easy_setopt( _curl, CURLOPT_HTTPGET, 1L );
1044  } else {
1045  curl_easy_setopt( _curl, CURLOPT_RANGE, NULL );
1046  }
1047  }
1048  private:
1049  CURL * _curl;
1050  bool _doHttpHeadRequest;
1051  } _guard( _curl, (_url.getScheme() == "http" || _url.getScheme() == "https") && _settings.headRequestsAllowed() );
1052 
1053 
1054  AutoFILE file { ::fopen( "/dev/null", "w" ) };
1055  if ( !file ) {
1056  ERR << "fopen failed for /dev/null" << endl;
1057  ZYPP_THROW(MediaWriteException("/dev/null"));
1058  }
1059 
1060  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, (*file) );
1061  if ( ret != 0 ) {
1063  }
1064 
1065  CURLcode ok = curl_easy_perform( _curl );
1066  MIL << "perform code: " << ok << " [ " << curl_easy_strerror(ok) << " ]" << endl;
1067 
1068  // as we are not having user interaction, the user can't cancel
1069  // the file existence checking, a callback or timeout return code
1070  // will be always a timeout.
1071  try {
1072  evaluateCurlCode( filename, ok, true /* timeout */);
1073  }
1074  catch ( const MediaFileNotFoundException &e ) {
1075  // if the file did not exist then we can return false
1076  return false;
1077  }
1078  catch ( const MediaException &e ) {
1079  // some error, we are not sure about file existence, rethrw
1080  ZYPP_RETHROW(e);
1081  }
1082  // exists
1083  return ( ok == CURLE_OK );
1084 }
1085 
1087 
1088 void MediaCurl::doGetFileCopy( const OnMediaLocation &srcFile , const Pathname & target, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1089 {
1090  Pathname dest = target.absolutename();
1091  if( assert_dir( dest.dirname() ) )
1092  {
1093  DBG << "assert_dir " << dest.dirname() << " failed" << endl;
1094  ZYPP_THROW( MediaSystemException(getFileUrl(srcFile.filename()), "System error on " + dest.dirname().asString()) );
1095  }
1096 
1097  ManagedFile destNew { target.extend( ".new.zypp.XXXXXX" ) };
1098  AutoFILE file;
1099  {
1100  AutoFREE<char> buf { ::strdup( (*destNew).c_str() ) };
1101  if( ! buf )
1102  {
1103  ERR << "out of memory for temp file name" << endl;
1104  ZYPP_THROW(MediaSystemException(getFileUrl(srcFile.filename()), "out of memory for temp file name"));
1105  }
1106 
1107  AutoFD tmp_fd { ::mkostemp( buf, O_CLOEXEC ) };
1108  if( tmp_fd == -1 )
1109  {
1110  ERR << "mkstemp failed for file '" << destNew << "'" << endl;
1111  ZYPP_THROW(MediaWriteException(destNew));
1112  }
1113  destNew = ManagedFile( (*buf), filesystem::unlink );
1114 
1115  file = ::fdopen( tmp_fd, "we" );
1116  if ( ! file )
1117  {
1118  ERR << "fopen failed for file '" << destNew << "'" << endl;
1119  ZYPP_THROW(MediaWriteException(destNew));
1120  }
1121  tmp_fd.resetDispose(); // don't close it here! ::fdopen moved ownership to file
1122  }
1123 
1124  DBG << "dest: " << dest << endl;
1125  DBG << "temp: " << destNew << endl;
1126 
1127  // set IFMODSINCE time condition (no download if not modified)
1128  if( PathInfo(target).isExist() && !(options & OPTION_NO_IFMODSINCE) )
1129  {
1130  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_IFMODSINCE);
1131  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, (long)PathInfo(target).mtime());
1132  }
1133  else
1134  {
1135  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1136  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1137  }
1138  try
1139  {
1140  doGetFileCopyFile( srcFile, dest, file, report, options);
1141  }
1142  catch (Exception &e)
1143  {
1144  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1145  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1146  ZYPP_RETHROW(e);
1147  }
1148 
1149  long httpReturnCode = 0;
1150  CURLcode infoRet = curl_easy_getinfo(_curl,
1151  CURLINFO_RESPONSE_CODE,
1152  &httpReturnCode);
1153  bool modified = true;
1154  if (infoRet == CURLE_OK)
1155  {
1156  DBG << "HTTP response: " + str::numstring(httpReturnCode);
1157  if ( httpReturnCode == 304
1158  || ( httpReturnCode == 213 && (_url.getScheme() == "ftp" || _url.getScheme() == "tftp") ) ) // not modified
1159  {
1160  DBG << " Not modified.";
1161  modified = false;
1162  }
1163  DBG << endl;
1164  }
1165  else
1166  {
1167  WAR << "Could not get the response code." << endl;
1168  }
1169 
1170  if (modified || infoRet != CURLE_OK)
1171  {
1172  // apply umask
1173  if ( ::fchmod( ::fileno(file), filesystem::applyUmaskTo( 0644 ) ) )
1174  {
1175  ERR << "Failed to chmod file " << destNew << endl;
1176  }
1177 
1178  file.resetDispose(); // we're going to close it manually here
1179  if ( ::fclose( file ) )
1180  {
1181  ERR << "Fclose failed for file '" << destNew << "'" << endl;
1182  ZYPP_THROW(MediaWriteException(destNew));
1183  }
1184 
1185  // move the temp file into dest
1186  if ( rename( destNew, dest ) != 0 ) {
1187  ERR << "Rename failed" << endl;
1189  }
1190  destNew.resetDispose(); // no more need to unlink it
1191  }
1192 
1193  DBG << "done: " << PathInfo(dest) << endl;
1194 }
1195 
1197 
1198 void MediaCurl::doGetFileCopyFile( const OnMediaLocation & srcFile, const Pathname & dest, FILE *file, callback::SendReport<DownloadProgressReport> & report, RequestOptions options ) const
1199 {
1200  DBG << srcFile.filename().asString() << endl;
1201 
1202  if(!_url.isValid())
1204 
1205  if(_url.getHost().empty())
1207 
1208  Url url(getFileUrl(srcFile.filename()));
1209 
1210  DBG << "URL: " << url.asString() << endl;
1211  // Use URL without options and without username and passwd
1212  // (some proxies dislike them in the URL).
1213  // Curl seems to need the just scheme, hostname and a path;
1214  // the rest was already passed as curl options (in attachTo).
1215  Url curlUrl( clearQueryString(url) );
1216 
1217  //
1218  // See also Bug #154197 and ftp url definition in RFC 1738:
1219  // The url "ftp://user@host/foo/bar/file" contains a path,
1220  // that is relative to the user's home.
1221  // The url "ftp://user@host//foo/bar/file" (or also with
1222  // encoded slash as %2f) "ftp://user@host/%2ffoo/bar/file"
1223  // contains an absolute path.
1224  //
1225  _lastRedirect.clear();
1226  std::string urlBuffer( curlUrl.asString());
1227  CURLcode ret = curl_easy_setopt( _curl, CURLOPT_URL,
1228  urlBuffer.c_str() );
1229  if ( ret != 0 ) {
1231  }
1232 
1233  ret = curl_easy_setopt( _curl, CURLOPT_WRITEDATA, file );
1234  if ( ret != 0 ) {
1236  }
1237 
1238  // Set callback and perform.
1239  internal::ProgressData progressData(_curl, _settings.timeout(), url, srcFile.downloadSize(), &report);
1240  if (!(options & OPTION_NO_REPORT_START))
1241  report->start(url, dest);
1242  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, &progressData ) != 0 ) {
1243  WAR << "Can't set CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1244  }
1245 
1246  ret = curl_easy_perform( _curl );
1247 #if CURLVERSION_AT_LEAST(7,19,4)
1248  // bnc#692260: If the client sends a request with an If-Modified-Since header
1249  // with a future date for the server, the server may respond 200 sending a
1250  // zero size file.
1251  // curl-7.19.4 introduces CURLINFO_CONDITION_UNMET to check this condition.
1252  if ( ftell(file) == 0 && ret == 0 )
1253  {
1254  long httpReturnCode = 33;
1255  if ( curl_easy_getinfo( _curl, CURLINFO_RESPONSE_CODE, &httpReturnCode ) == CURLE_OK && httpReturnCode == 200 )
1256  {
1257  long conditionUnmet = 33;
1258  if ( curl_easy_getinfo( _curl, CURLINFO_CONDITION_UNMET, &conditionUnmet ) == CURLE_OK && conditionUnmet )
1259  {
1260  WAR << "TIMECONDITION unmet - retry without." << endl;
1261  curl_easy_setopt(_curl, CURLOPT_TIMECONDITION, CURL_TIMECOND_NONE);
1262  curl_easy_setopt(_curl, CURLOPT_TIMEVALUE, 0L);
1263  ret = curl_easy_perform( _curl );
1264  }
1265  }
1266  }
1267 #endif
1268 
1269  if ( curl_easy_setopt( _curl, CURLOPT_PROGRESSDATA, NULL ) != 0 ) {
1270  WAR << "Can't unset CURLOPT_PROGRESSDATA: " << _curlError << endl;;
1271  }
1272 
1273  if ( ret != 0 )
1274  {
1275  ERR << "curl error: " << ret << ": " << _curlError
1276  << ", temp file size " << ftell(file)
1277  << " bytes." << endl;
1278 
1279  // the timeout is determined by the progress data object
1280  // which holds whether the timeout was reached or not,
1281  // otherwise it would be a user cancel
1282  try {
1283 
1284  if ( progressData.fileSizeExceeded() )
1285  ZYPP_THROW(MediaFileSizeExceededException(url, progressData.expectedFileSize()));
1286 
1287  evaluateCurlCode( srcFile.filename(), ret, progressData.timeoutReached() );
1288  }
1289  catch ( const MediaException &e ) {
1290  // some error, we are not sure about file existence, rethrw
1291  ZYPP_RETHROW(e);
1292  }
1293  }
1294 }
1295 
1297 
1298 void MediaCurl::getDir( const Pathname & dirname, bool recurse_r ) const
1299 {
1300  filesystem::DirContent content;
1301  getDirInfo( content, dirname, /*dots*/false );
1302 
1303  for ( filesystem::DirContent::const_iterator it = content.begin(); it != content.end(); ++it ) {
1304  Pathname filename = dirname + it->name;
1305  int res = 0;
1306 
1307  switch ( it->type ) {
1308  case filesystem::FT_NOT_AVAIL: // old directory.yast contains no typeinfo at all
1309  case filesystem::FT_FILE:
1310  getFile( OnMediaLocation( filename ) );
1311  break;
1312  case filesystem::FT_DIR: // newer directory.yast contain at least directory info
1313  if ( recurse_r ) {
1314  getDir( filename, recurse_r );
1315  } else {
1316  res = assert_dir( localPath( filename ) );
1317  if ( res ) {
1318  WAR << "Ignore error (" << res << ") on creating local directory '" << localPath( filename ) << "'" << endl;
1319  }
1320  }
1321  break;
1322  default:
1323  // don't provide devices, sockets, etc.
1324  break;
1325  }
1326  }
1327 }
1328 
1330 
1331 void MediaCurl::getDirInfo( std::list<std::string> & retlist,
1332  const Pathname & dirname, bool dots ) const
1333 {
1334  getDirectoryYast( retlist, dirname, dots );
1335 }
1336 
1338 
1340  const Pathname & dirname, bool dots ) const
1341 {
1342  getDirectoryYast( retlist, dirname, dots );
1343 }
1344 
1346 //
1347 int MediaCurl::aliveCallback( void *clientp, curl_off_t /*dltotal*/, curl_off_t dlnow, curl_off_t /*ultotal*/, curl_off_t /*ulnow*/ )
1348 {
1349  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1350  if( pdata )
1351  {
1352  // Do not propagate dltotal in alive callbacks. MultiCurl uses this to
1353  // prevent a percentage raise while downloading a metalink file. Download
1354  // activity however is indicated by propagating the download rate (via dlnow).
1355  pdata->updateStats( 0.0, dlnow );
1356  return pdata->reportProgress();
1357  }
1358  return 0;
1359 }
1360 
1361 int MediaCurl::progressCallback( void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow )
1362 {
1363  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>( clientp );
1364  if( pdata )
1365  {
1366  // work around curl bug that gives us old data
1367  long httpReturnCode = 0;
1368  if ( curl_easy_getinfo( pdata->curl(), CURLINFO_RESPONSE_CODE, &httpReturnCode ) != CURLE_OK || httpReturnCode == 0 )
1369  return aliveCallback( clientp, dltotal, dlnow, ultotal, ulnow );
1370 
1371  pdata->updateStats( dltotal, dlnow );
1372  return pdata->reportProgress();
1373  }
1374  return 0;
1375 }
1376 
1378 {
1379  internal::ProgressData *pdata = reinterpret_cast<internal::ProgressData *>(clientp);
1380  return pdata ? pdata->curl() : 0;
1381 }
1382 
1384 
1385 std::string MediaCurl::getAuthHint() const
1386 {
1387  long auth_info = CURLAUTH_NONE;
1388 
1389  CURLcode infoRet =
1390  curl_easy_getinfo(_curl, CURLINFO_HTTPAUTH_AVAIL, &auth_info);
1391 
1392  if(infoRet == CURLE_OK)
1393  {
1394  return CurlAuthData::auth_type_long2str(auth_info);
1395  }
1396 
1397  return "";
1398 }
1399 
1404 void MediaCurl::resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
1405 {
1406  internal::ProgressData *data = reinterpret_cast<internal::ProgressData *>(clientp);
1407  if ( data ) {
1408  data->expectedFileSize( expectedFileSize );
1409  }
1410 }
1411 
1413 
1414 bool MediaCurl::authenticate(const std::string & availAuthTypes, bool firstTry) const
1415 {
1417  CredentialManager cm(CredManagerOptions(ZConfig::instance().repoManagerRoot()));
1418  CurlAuthData_Ptr credentials;
1419 
1420  // get stored credentials
1421  AuthData_Ptr cmcred = cm.getCred(_url);
1422 
1423  if (cmcred && firstTry)
1424  {
1425  credentials.reset(new CurlAuthData(*cmcred));
1426  DBG << "got stored credentials:" << endl << *credentials << endl;
1427  }
1428  // if not found, ask user
1429  else
1430  {
1431 
1432  CurlAuthData_Ptr curlcred;
1433  curlcred.reset(new CurlAuthData());
1435 
1436  // preset the username if present in current url
1437  if (!_url.getUsername().empty() && firstTry)
1438  curlcred->setUsername(_url.getUsername());
1439  // if CM has found some credentials, preset the username from there
1440  else if (cmcred)
1441  curlcred->setUsername(cmcred->username());
1442 
1443  // indicate we have no good credentials from CM
1444  cmcred.reset();
1445 
1446  std::string prompt_msg = str::Format(_("Authentication required for '%s'")) % _url.asString();
1447 
1448  // set available authentication types from the exception
1449  // might be needed in prompt
1450  curlcred->setAuthType(availAuthTypes);
1451 
1452  // ask user
1453  if (auth_report->prompt(_url, prompt_msg, *curlcred))
1454  {
1455  DBG << "callback answer: retry" << endl
1456  << "CurlAuthData: " << *curlcred << endl;
1457 
1458  if (curlcred->valid())
1459  {
1460  credentials = curlcred;
1461  // if (credentials->username() != _url.getUsername())
1462  // _url.setUsername(credentials->username());
1470  }
1471  }
1472  else
1473  {
1474  DBG << "callback answer: cancel" << endl;
1475  }
1476  }
1477 
1478  // set username and password
1479  if (credentials)
1480  {
1481  // HACK, why is this const?
1482  const_cast<MediaCurl*>(this)->_settings.setUsername(credentials->username());
1483  const_cast<MediaCurl*>(this)->_settings.setPassword(credentials->password());
1484 
1485  // set username and password
1486  CURLcode ret = curl_easy_setopt(_curl, CURLOPT_USERPWD, _settings.userPassword().c_str());
1488 
1489  // set available authentication types from the exception
1490  if (credentials->authType() == CURLAUTH_NONE)
1491  credentials->setAuthType(availAuthTypes);
1492 
1493  // set auth type (seems this must be set _after_ setting the userpwd)
1494  if (credentials->authType() != CURLAUTH_NONE)
1495  {
1496  // FIXME: only overwrite if not empty?
1497  const_cast<MediaCurl*>(this)->_settings.setAuthType(credentials->authTypeAsString());
1498  ret = curl_easy_setopt(_curl, CURLOPT_HTTPAUTH, credentials->authType());
1500  }
1501 
1502  if (!cmcred)
1503  {
1504  credentials->setUrl(_url);
1505  cm.addCred(*credentials);
1506  cm.save();
1507  }
1508 
1509  return true;
1510  }
1511 
1512  return false;
1513 }
1514 
1515 //need a out of line definiton, otherwise vtable is emitted for every translation unit
1517 
1518  } // namespace media
1519 } // namespace zypp
1520 //
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
long timeout() const
transfer timeout
void globalInitCurlOnce()
Definition: curlhelper.cc:64
Url getFileUrl(const Pathname &filename) const
concatenate the attach url and the filename to a complete download url
virtual bool checkAttachPoint(const Pathname &apoint) const override
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
Definition: MediaCurl.cc:698
const Pathname & certificateAuthoritiesPath() const
SSL certificate authorities path ( default: /etc/ssl/certs )
int assert_dir(const Pathname &path, unsigned mode)
Like &#39;mkdir -p&#39;.
Definition: PathInfo.cc:319
MediaCurlExceptionMayRetryInternaly(const Url &url_r, const std::string &err_r, const std::string &msg_r)
Definition: MediaCurl.cc:300
std::string targetDistribution() const
This is register.target attribute of the installed base product.
Definition: Target.cc:102
#define SET_OPTION_OFFT(opt, val)
Definition: MediaCurl.cc:330
#define MIL
Definition: Logger.h:96
const Pathname & clientCertificatePath() const
SSL client certificate file.
bool progress(int value_r, const Url &file_r, double dbps_avg_r=-1, double dbps_current_r=-1) override
Download progress.
Definition: MediaCurl.cc:83
void addHeader(std::string &&val_r)
add a header, on the form "Foo: Bar" (trims)
#define _(MSG)
Definition: Gettext.h:37
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:137
ProgressData()
Ctor no range [0,0](0).
Definition: progressdata.h:174
const Pathname & path() const
Return current Pathname.
Definition: PathInfo.h:246
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
Describes a resource file located on a medium.
static ZConfig & instance()
Singleton ctor.
Definition: ZConfig.cc:922
std::string proxyUserPassword() const
returns the proxy user and password as a user:pass string
void checkProtocol(const Url &url) const
check the url is supported by the curl library
Definition: MediaCurl.cc:381
bool authenticate(const std::string &availAuthTypes, bool firstTry) const
Definition: MediaCurl.cc:1414
Implementation class for FTP, HTTP and HTTPS MediaHandler.
Definition: MediaCurl.h:31
void setPassword(const std::string &val_r)
sets the auth password
std::chrono::steady_clock::time_point TimePoint
Definition: MediaCurl.cc:51
ByteCount _expectedFileSize
Definition: MediaCurl.cc:167
virtual bool checkAttachPoint(const Pathname &apoint) const
Verify if the specified directory as attach point (root) as requires by the particular media handler ...
const char * distributionFlavorHeader()
Definition: MediaCurl.cc:265
Store and operate with byte count.
Definition: ByteCount.h:30
long maxDownloadSpeed() const
Maximum download speed (bytes per second)
to not add a IFMODSINCE header if target exists
Definition: MediaCurl.h:43
Holds transfer setting.
void save()
Saves any unsaved credentials added via addUserCred() or addGlobalCred() methods. ...
const std::string & authType() const
get the allowed authentication types
bool verifyHostEnabled() const
Whether to verify host for ssl.
const std::string & proxyUsername() const
proxy auth username
Pathname extend(const std::string &r) const
Append string r to the last component of the path.
Definition: Pathname.h:173
int reportProgress() const
Definition: MediaCurl.cc:240
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:367
void setAttachPoint(const Pathname &path, bool temp)
Set a new attach point.
void start(const Url &file_r, Pathname localfile_r) override
Definition: MediaCurl.cc:71
const char * anonymousIdHeader()
Definition: MediaCurl.cc:251
const char * c_str() const
String representation.
Definition: Pathname.h:110
void addHistory(const std::string &msg_r)
Add some message text to the history.
Definition: Exception.cc:140
bool isUseableAttachPoint(const Pathname &path, bool mtab=true) const
Ask media manager, if the specified path is already used as attach point or if there are another atta...
void setUsername(const std::string &val_r)
sets the auth username
Action problem(const Url &file_r, Error error_r, const std::string &description_r) override
Definition: MediaCurl.cc:102
virtual void getFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename) const override
Definition: MediaCurl.cc:740
time_t _timeNow
Now.
Definition: MediaCurl.cc:173
AuthData_Ptr getCred(const Url &url)
Get credentials for the specified url.
AutoDispose< const Pathname > ManagedFile
A Pathname plus associated cleanup code to be executed when path is no longer needed.
Definition: ManagedFile.h:27
virtual void setupEasy()
initializes the curl easy handle with the data from the url
Definition: MediaCurl.cc:406
bool timeoutReached() const
Definition: MediaCurl.cc:149
Convenient building of std::string with boost::format.
Definition: String.h:252
Structure holding values of curlrc options.
Definition: curlconfig.h:26
std::string form(const char *format,...) __attribute__((format(printf
Printf style construction of std::string.
Definition: String.cc:36
Edition * _value
Definition: SysContent.cc:311
AutoDispose<int> calling ::close
Definition: AutoDispose.h:301
std::string _currentCookieFile
Definition: MediaCurl.h:160
const std::string & password() const
auth password
virtual void getDir(const Pathname &dirname, bool recurse_r) const override
Call concrete handler to provide directory content (not recursive!) below attach point.
Definition: MediaCurl.cc:1298
virtual void getFile(const OnMediaLocation &file) const override
Call concrete handler to provide file below attach point.
Definition: MediaCurl.cc:731
#define ERR
Definition: Logger.h:98
const std::string & username() const
auth username
bool fileSizeExceeded() const
Definition: MediaCurl.cc:152
Pathname localPath(const Pathname &pathname) const
Files provided will be available at &#39;localPath(filename)&#39;.
const Headers & headers() const
returns a list of all added headers (trimmed)
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
static void setCookieFile(const Pathname &)
Definition: MediaCurl.cc:374
virtual void releaseFrom(const std::string &ejectDev) override
Call concrete handler to release the media.
Definition: MediaCurl.cc:724
bool verifyPeerEnabled() const
Whether to verify peer for ssl.
static void resetExpectedFileSize(void *clientp, const ByteCount &expectedFileSize)
MediaMultiCurl needs to reset the expected filesize in case a metalink file is downloaded otherwise t...
Definition: MediaCurl.cc:1404
const std::string & hint() const
comma separated list of available authentication types
bool empty() const
Test for an empty path.
Definition: Pathname.h:114
#define ZYPP_RETHROW(EXCPT)
Drops a logline and rethrows, updating the CodeLocation.
Definition: Exception.h:440
static int parseConfig(CurlConfig &config, const std::string &filename="")
Parse a curlrc file and store the result in the config structure.
Definition: curlconfig.cc:24
int assert_file_mode(const Pathname &path, unsigned mode)
Like assert_file but enforce mode even if the file already exists.
Definition: PathInfo.cc:1202
std::string asString() const
Returns a default string representation of the Url object.
Definition: Url.cc:497
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
Bottleneck filtering all DownloadProgressReport issued from Media[Muli]Curl.
Definition: MediaCurl.cc:49
Url clearQueryString(const Url &url) const
Definition: MediaCurl.cc:369
void setAuthType(const std::string &val_r)
set the allowed authentication types
std::string trim(const std::string &s, const Trim trim_r)
Definition: String.cc:223
void finish(const Url &file_r, Error error_r, const std::string &reason_r) override
Definition: MediaCurl.cc:108
int unlink(const Pathname &path)
Like &#39;unlink&#39;.
Definition: PathInfo.cc:700
const Url _url
Url to handle.
Definition: MediaHandler.h:113
void setMediaSource(const MediaSourceRef &ref)
Set new media source reference.
const std::string & asString() const
String representation.
Definition: Pathname.h:91
int rename(const Pathname &oldpath, const Pathname &newpath)
Like &#39;rename&#39;.
Definition: PathInfo.cc:742
Just inherits Exception to separate media exceptions.
void evaluateCurlCode(const zypp::Pathname &filename, CURLcode code, bool timeout) const
Evaluates a curl return code and throws the right MediaException filename Filename being downloaded c...
Definition: MediaCurl.cc:833
std::string asUserHistory() const
A single (multiline) string composed of asUserString and historyAsString.
Definition: Exception.cc:91
const ByteCount & downloadSize() const
The size of the resource on the server.
void disconnect()
Use concrete handler to isconnect media.
long connectTimeout() const
connection timeout
Pathname dirname() const
Return all but the last component od this path.
Definition: Pathname.h:124
do not send a start ProgressReport
Definition: MediaCurl.h:45
#define WAR
Definition: Logger.h:97
std::list< DirEntry > DirContent
Returned by readdir.
Definition: PathInfo.h:518
void getDirectoryYast(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const
Retrieve and if available scan dirname/directory.yast.
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:177
static int progressCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback reporting download progress.
Definition: MediaCurl.cc:1361
Receiver * _oldRec
std::string proxyuserpwd
Definition: curlconfig.h:49
bool isValid() const
Verifies the Url.
Definition: Url.cc:489
virtual bool doGetDoesFileExist(const Pathname &filename) const
Definition: MediaCurl.cc:981
const Pathname & clientKeyPath() const
SSL client key file.
void expectedFileSize(ByteCount newval_r)
Definition: MediaCurl.cc:158
shared_ptr< CurlAuthData > CurlAuthData_Ptr
Definition: curlauthdata.h:102
const Pathname & filename() const
The path to the resource on the medium.
virtual void attachTo(bool next=false) override
Call concrete handler to attach the media.
Definition: MediaCurl.cc:663
std::string numstring(char n, int w=0)
Definition: String.h:289
Common baseclass for MediaCurl and MediaNetwork.
virtual bool getDoesFileExist(const Pathname &filename) const override
Repeatedly calls doGetDoesFileExist() until it successfully returns, fails unexpectedly, or user cancels the operation.
Definition: MediaCurl.cc:802
const char * agentString()
Definition: MediaCurl.cc:279
void resetDispose()
Set no dispose function.
Definition: AutoDispose.h:180
void doGetFileCopyFile(const OnMediaLocation &srcFile, const Pathname &dest, FILE *file, callback::SendReport< DownloadProgressReport > &report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1198
Media source internally used by MediaManager and MediaHandler.
Definition: MediaSource.h:36
static std::string auth_type_long2str(long auth_type)
Converts a long of ORed CURLAUTH_* identifiers into a string of comma separated list of authenticatio...
long minDownloadSpeed() const
Minimum download speed (bytes per second) until the connection is dropped.
int _dnlPercent
Percent completed or 0 if _dnlTotal is unknown.
Definition: MediaCurl.cc:179
curl_slist * _customHeaders
Definition: MediaCurl.h:168
bool proxyEnabled() const
proxy is enabled
shared_ptr< AuthData > AuthData_Ptr
Definition: authdata.h:79
#define LIBZYPP_VERSION_STRING
Definition: APIConfig.h:15
int rmdir(const Pathname &path)
Like &#39;rmdir&#39;.
Definition: PathInfo.cc:366
#define SET_OPTION(opt, val)
Definition: MediaCurl.cc:323
Pathname absolutename() const
Return this path, adding a leading &#39;/&#39; if relative.
Definition: Pathname.h:139
Base class for Exception.
Definition: Exception.h:145
Pathname attachPoint() const
Return the currently used attach point.
std::string _lastRedirect
to log/report redirections
Definition: MediaCurl.h:163
curl_off_t _dnlNow
Bytes downloaded now.
Definition: MediaCurl.cc:177
Url url() const
Url used.
Definition: MediaHandler.h:503
curl_off_t _dnlTotal
Bytes to download or 0 if unknown.
Definition: MediaCurl.cc:175
double _drateLast
Download rate in last period.
Definition: MediaCurl.cc:182
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
std::string getHost(EEncoding eflag=zypp::url::E_DECODED) const
Returns the hostname or IP from the URL authority.
Definition: Url.cc:588
time_t _timeStart
Start total stats.
Definition: MediaCurl.cc:170
virtual void getDirInfo(std::list< std::string > &retlist, const Pathname &dirname, bool dots=true) const override
Call concrete handler to provide a content list of directory on media via retlist.
Definition: MediaCurl.cc:1331
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
virtual void disconnectFrom() override
Definition: MediaCurl.cc:705
static CURL * progressCallback_getcurl(void *clientp)
Definition: MediaCurl.cc:1377
Typesafe passing of user data via callbacks.
Definition: UserData.h:38
bool strToBool(const C_Str &str, bool default_r)
Parse str into a bool depending on the default value.
Definition: String.h:429
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
void setUserAgentString(std::string &&val_r)
sets the user agent ie: "Mozilla v3" (trims)
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
curl_off_t _dnlLast
Bytes downloaded at period start.
Definition: MediaCurl.cc:176
void report(const UserData &userData_r=UserData()) override
The most generic way of sending/receiving data.
Definition: MediaCurl.cc:67
AutoDispose<FILE*> calling ::fclose
Definition: AutoDispose.h:312
static Pathname _cookieFile
Definition: MediaCurl.h:161
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:325
double _drateTotal
Download rate so far.
Definition: MediaCurl.cc:181
std::string anonymousUniqueId() const
anonymous unique id
Definition: Target.cc:132
mode_t applyUmaskTo(mode_t mode_r)
Modify mode_r according to the current umask ( mode_r & ~getUmask() ).
Definition: PathInfo.h:789
std::string userPassword() const
returns the user and password as a user:pass string
std::string getAuthHint() const
Return a comma separated list of available authentication methods supported by server.
Definition: MediaCurl.cc:1385
time_t _timeRcv
Start of no-data timeout.
Definition: MediaCurl.cc:172
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
Pathname createAttachPoint() const
Try to create a default / temporary attach point.
virtual void doGetFileCopy(const OnMediaLocation &srcFile, const Pathname &targetFilename, callback::SendReport< DownloadProgressReport > &_report, RequestOptions options=OPTION_NONE) const
Definition: MediaCurl.cc:1088
void updateStats(curl_off_t dltotal=0.0, curl_off_t dlnow=0.0)
Definition: MediaCurl.cc:197
void addCred(const AuthData &cred)
Add new credentials with user callbacks.
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
Curl HTTP authentication data.
Definition: curlauthdata.h:22
std::string distributionFlavor() const
This is flavor attribute of the installed base product but does not require the target to be loaded a...
Definition: Target.cc:127
OptionalDownloadProgressReport(bool isOptional=false)
Definition: MediaCurl.cc:53
char _curlError[CURL_ERROR_SIZE]
Definition: MediaCurl.h:167
ByteCount expectedFileSize() const
Definition: MediaCurl.cc:155
const std::string & proxy() const
proxy host
MediaCurlException(const Url &url_r, const std::string &err_r, const std::string &msg_r)
bool optional() const
Whether this is an optional resource.
time_t _timeLast
Start last period(~1sec)
Definition: MediaCurl.cc:171
const char * c_str() const
Definition: IdStringType.h:105
TrueBool _guard
Definition: TargetImpl.cc:1608
static int aliveCallback(void *clientp, curl_off_t dltotal, curl_off_t dlnow, curl_off_t ultotal, curl_off_t ulnow)
Callback sending just an alive trigger to the UI, without stats (e.g.
Definition: MediaCurl.cc:1347
bool userMayRWX() const
Definition: PathInfo.h:353
const std::string & userAgentString() const
user agent string (trimmed)
Url manipulation class.
Definition: Url.h:91
bool headRequestsAllowed() const
whether HEAD requests are allowed
#define DBG
Definition: Logger.h:95
Attempt to work around certain issues by autoretry in MediaCurl::getFileCopy E.g. ...
Definition: MediaCurl.cc:297
zypp::callback::SendReport< zypp::media::DownloadProgressReport > * report
Definition: MediaCurl.cc:168
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572