libzypp  17.31.22
curlhelper.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include "private/curlhelper_p.h"
13 
14 #include <zypp/APIConfig.h>
15 
16 #include <zypp-core/fs/PathInfo.h>
17 #include <zypp-core/Pathname.h>
18 #include <zypp-core/base/LogTools.h>
19 #include <zypp-core/base/String.h>
20 #include <zypp-core/base/StringV.h>
21 #include <zypp-curl/ProxyInfo>
22 #include <zypp-curl/auth/CurlAuthData>
23 #include <zypp-media/MediaException>
24 #include <list>
25 #include <string>
26 
27 #define TRANSFER_TIMEOUT_MAX 60 * 60
28 
29 using std::endl;
30 using namespace zypp;
31 
32 namespace zypp
33 {
34  namespace env
35  {
36  const long & ZYPP_MEDIA_CURL_DEBUG()
37  {
38  static const long ret = [](){
39  const char * env = getenv("ZYPP_MEDIA_CURL_DEBUG");
40  return env && *env ? str::strtonum<ulong>( env ) : 0;
41  }();
42  return ret;
43  }
44 
46  {
47  static int _v = [](){
48  int ret = 0;
49  if ( const char * envp = getenv( "ZYPP_MEDIA_CURL_IPRESOLVE" ) ) {
50  WAR << "env set: $ZYPP_MEDIA_CURL_IPRESOLVE='" << envp << "'" << std::endl;
51  if ( strcmp( envp, "4" ) == 0 ) ret = 4;
52  else if ( strcmp( envp, "6" ) == 0 ) ret = 6;
53  }
54  return ret;
55  }();
56  return _v;
57  }
58  } // namespace env
59 } // namespace zypp
60 
61 namespace internal
62 {
63 
65 {
66  // function-level static <=> std::call_once
67  static bool once __attribute__ ((__unused__)) = ( [] {
68  MIL << "global_init libcurl: " << curl_version_info(CURLVERSION_NOW)->version << endl;
69  if ( curl_global_init( CURL_GLOBAL_ALL ) != 0 )
70  WAR << "curl global init failed" << std::endl;
71  } (), true );
72 }
73 
74 int log_curl( CURL * curl, curl_infotype info, char * ptr, size_t len, void * max_lvl )
75 {
76  if ( max_lvl == nullptr )
77  return 0;
78 
79  long maxlvl = *((long *)max_lvl);
80  const char * pfx = "";
81  bool isContent = true; // otherwise it's data
82  switch( info )
83  {
84  case CURLINFO_TEXT: if ( maxlvl < 1 ) return 0; pfx = "*"; break;
85  case CURLINFO_HEADER_IN: if ( maxlvl < 2 ) return 0; pfx = "<"; break;
86  case CURLINFO_HEADER_OUT: if ( maxlvl < 2 ) return 0; pfx = ">"; break;
87  case CURLINFO_SSL_DATA_IN: if ( maxlvl < 3 ) return 0; isContent = false; pfx = "<[SSL]"; break;
88  case CURLINFO_SSL_DATA_OUT: if ( maxlvl < 3 ) return 0; isContent = false; pfx = ">[SSL]"; break;
89  case CURLINFO_DATA_IN: if ( maxlvl < 3 ) return 0; isContent = false; pfx = "<[DTA]"; break;
90  case CURLINFO_DATA_OUT: if ( maxlvl < 3 ) return 0; isContent = false; pfx = ">[DTA]"; break;
91 
92  default:
93  return 0;
94  }
95 
96  // We'd like to keep all log messages within function `log_curl`
97  // because this tag to grep for is known and communicate to users.
98  if ( isContent ) {
99  std::vector<std::string_view> lines; // don't want log from within the lambda
100  strv::split( std::string_view( ptr, len ), "\n", [&lines]( std::string_view line, unsigned, bool last ) {
101  if ( last ) return; // empty word after final \n
102  line = strv::rtrim( line, "\r" );
103  lines.push_back( line );
104  });
105  for ( const auto & line : lines ) {
106  if ( str::hasPrefix( line, "Authorization:" ) ) {
107  std::string_view::size_type pos { line.find( " ", 15 ) }; // Authorization: <type> <credentials>
108  if ( pos == std::string::npos )
109  pos = 15;
110  DBG << curl << " " << pfx << " " << line.substr( 0, pos ) << " <credentials removed>" << endl;
111  }
112  else
113  DBG << curl << " " << pfx << " " << line << endl;
114  }
115  } else {
116  if ( maxlvl < 4 )
117  DBG << curl << " " << pfx << " " << len << " byte" << endl;
118  else
119  hexdumpOn( DBG << curl << " " << pfx << " ", ptr, len );
120  }
121  return 0;
122 }
123 
124 void setupZYPP_MEDIA_CURL_DEBUG( CURL *curl )
125 {
126  if ( not curl ) {
127  INT << "Got a NULL curl handle" << endl;
128  return;
129  }
130  if ( env::ZYPP_MEDIA_CURL_DEBUG() > 0 ) {
131  curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L );
132  curl_easy_setopt( curl, CURLOPT_DEBUGFUNCTION, log_curl );
133  curl_easy_setopt( curl, CURLOPT_DEBUGDATA, &env::ZYPP_MEDIA_CURL_DEBUG() );
134  }
135 }
136 
137 size_t log_redirects_curl( char *ptr, size_t size, size_t nmemb, void *userdata)
138 {
139  //INT << "got header: " << std::string(ptr, ptr + size*nmemb) << endl;
140 
141  char * lstart = ptr, * lend = ptr;
142  size_t pos = 0;
143  size_t max = size * nmemb;
144  while (pos + 1 < max)
145  {
146  // get line
147  for (lstart = lend; *lend != '\n' && pos < max; ++lend, ++pos);
148 
149  // look for "Location"
150  if ( strncasecmp( lstart, "Location:", 9 ) == 0 )
151  {
152  std::string line { lstart, *(lend-1)=='\r' ? lend-1 : lend };
153  DBG << "redirecting to " << line << std::endl;
154  if ( userdata ) {
155  *reinterpret_cast<std::string *>( userdata ) = line;
156  }
157  return max;
158  }
159 
160  // continue with the next line
161  if (pos + 1 < max)
162  {
163  ++lend;
164  ++pos;
165  }
166  else
167  break;
168  }
169 
170  return max;
171 }
172 
178 {
179  {
180  const std::string & param { url.getQueryParam("timeout") };
181  if( ! param.empty() )
182  {
183  long num = str::strtonum<long>(param);
184  if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX )
185  s.setTimeout( num );
186  }
187  }
188  {
189  std::string param { url.getUsername() };
190  if ( ! param.empty() )
191  {
192  s.setUsername( std::move(param) );
193  param = url.getPassword();
194  if ( ! param.empty() )
195  s.setPassword( std::move(param) );
196  }
197  else
198  {
199  // if there is no username, set anonymous auth
200  if ( ( url.getScheme() == "ftp" || url.getScheme() == "tftp" ) && s.username().empty() )
201  s.setAnonymousAuth();
202  }
203  }
204  if ( url.getScheme() == "https" )
205  {
206  s.setVerifyPeerEnabled( false );
207  s.setVerifyHostEnabled( false );
208 
209  const std::string & verify { url.getQueryParam("ssl_verify") };
210  if( verify.empty() || verify == "yes" )
211  {
212  s.setVerifyPeerEnabled( true );
213  s.setVerifyHostEnabled( true );
214  }
215  else if ( verify == "no" )
216  {
217  s.setVerifyPeerEnabled( false );
218  s.setVerifyHostEnabled( false );
219  }
220  else
221  {
222  std::vector<std::string> flags;
223  str::split( verify, std::back_inserter(flags), "," );
224  for ( const auto & flag : flags )
225  {
226  if ( flag == "host" )
227  s.setVerifyHostEnabled( true );
228  else if ( flag == "peer" )
229  s.setVerifyPeerEnabled( true );
230  else
231  ZYPP_THROW( media::MediaBadUrlException(url, "Unknown ssl_verify flag "+flag) );
232  }
233  }
234  }
235  {
236  Pathname ca_path { url.getQueryParam("ssl_capath") };
237  if( ! ca_path.empty() )
238  {
239  if( ! PathInfo(ca_path).isDir() || ! ca_path.absolute() )
240  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_capath path"));
241  else
242  s.setCertificateAuthoritiesPath( std::move(ca_path) );
243  }
244  }
245  {
246  Pathname client_cert { url.getQueryParam("ssl_clientcert") };
247  if( ! client_cert.empty() )
248  {
249  if( ! PathInfo(client_cert).isFile() || ! client_cert.absolute() )
250  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientcert file"));
251  else
252  s.setClientCertificatePath( std::move(client_cert) );
253  }
254  }
255  {
256  Pathname client_key { url.getQueryParam("ssl_clientkey") };
257  if( ! client_key.empty() )
258  {
259  if( ! PathInfo(client_key).isFile() || ! client_key.absolute() )
260  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientkey file"));
261  else
262  s.setClientKeyPath( std::move(client_key) );
263  }
264  }
265  {
266  std::string param { url.getQueryParam( "proxy" ) };
267  if ( ! param.empty() )
268  {
269  if ( param == EXPLICITLY_NO_PROXY ) {
270  // Workaround TransferSettings shortcoming: With an
271  // empty proxy string, code will continue to look for
272  // valid proxy settings. So set proxy to some non-empty
273  // string, to indicate it has been explicitly disabled.
275  s.setProxyEnabled(false);
276  }
277  else {
278  const std::string & proxyport { url.getQueryParam( "proxyport" ) };
279  if ( ! proxyport.empty() ) {
280  param += ":";
281  param += proxyport;
282  }
283  s.setProxy( std::move(param) );
284  s.setProxyEnabled( true );
285  }
286  }
287  }
288  {
289  std::string param { url.getQueryParam( "proxyuser" ) };
290  if ( ! param.empty() )
291  {
292  s.setProxyUsername( std::move(param) );
293  s.setProxyPassword( url.getQueryParam( "proxypass" ) );
294  }
295  }
296  {
297  // HTTP authentication type
298  std::string param { url.getQueryParam("auth") };
299  if ( ! param.empty() && (url.getScheme() == "http" || url.getScheme() == "https") )
300  {
301  try
302  {
303  media::CurlAuthData::auth_type_str2long (param ); // check if we know it
304  }
305  catch ( const media::MediaException & ex_r )
306  {
307  DBG << "Rethrowing as MediaUnauthorizedException.";
308  ZYPP_THROW(media::MediaUnauthorizedException(url, ex_r.msg(), "", ""));
309  }
310  s.setAuthType( std::move(param) );
311  }
312  }
313  {
314  // workarounds
315  const std::string & param { url.getQueryParam("head_requests") };
316  if( ! param.empty() && param == "no" )
317  s.setHeadRequestsAllowed( false );
318  }
319 }
320 
326 {
327  media::ProxyInfo proxy_info;
328  if ( proxy_info.useProxyFor( url ) )
329  {
330  // We must extract any 'user:pass' from the proxy url
331  // otherwise they won't make it into curl (.curlrc wins).
332  try {
333  Url u( proxy_info.proxy( url ) );
335  // don't overwrite explicit auth settings
336  if ( s.proxyUsername().empty() )
337  {
338  s.setProxyUsername( u.getUsername( url::E_ENCODED ) );
339  s.setProxyPassword( u.getPassword( url::E_ENCODED ) );
340  }
341  s.setProxyEnabled( true );
342  }
343  catch (...) {} // no proxy if URL is malformed
344  }
345 }
346 
347 void curlEscape( std::string & str_r,
348  const char char_r, const std::string & escaped_r ) {
349  for ( std::string::size_type pos = str_r.find( char_r );
350  pos != std::string::npos; pos = str_r.find( char_r, pos ) ) {
351  str_r.replace( pos, 1, escaped_r );
352  }
353 }
354 
355 std::string curlEscapedPath( std::string path_r ) {
356  curlEscape( path_r, ' ', "%20" );
357  return path_r;
358 }
359 
360 std::string curlUnEscape( std::string text_r ) {
361  char * tmp = curl_unescape( text_r.c_str(), 0 );
362  std::string ret( tmp );
363  curl_free( tmp );
364  return ret;
365 }
366 
368 {
369  Url curlUrl (url);
370  curlUrl.setUsername( "" );
371  curlUrl.setPassword( "" );
372  curlUrl.setPathParams( "" );
373  curlUrl.setFragment( "" );
374  curlUrl.delQueryParam("cookies");
375  curlUrl.delQueryParam("proxy");
376  curlUrl.delQueryParam("proxyport");
377  curlUrl.delQueryParam("proxyuser");
378  curlUrl.delQueryParam("proxypass");
379  curlUrl.delQueryParam("ssl_capath");
380  curlUrl.delQueryParam("ssl_verify");
381  curlUrl.delQueryParam("ssl_clientcert");
382  curlUrl.delQueryParam("timeout");
383  curlUrl.delQueryParam("auth");
384  curlUrl.delQueryParam("username");
385  curlUrl.delQueryParam("password");
386  curlUrl.delQueryParam("mediahandler");
387  curlUrl.delQueryParam("credentials");
388  curlUrl.delQueryParam("head_requests");
389  return curlUrl;
390 }
391 
392 // bsc#933839: propagate proxy settings passed in the repo URL
393 // boo#1127591: propagate ssl settings passed in the repo URL
394 zypp::Url propagateQueryParams( zypp::Url url_r, const zypp::Url & template_r )
395 {
396  using namespace std::literals::string_literals;
397  for ( const std::string &param : { "proxy"s, "proxyport"s, "proxyuser"s, "proxypass"s, "ssl_capath"s, "ssl_verify"s } )
398  {
399  const std::string & value( template_r.getQueryParam( param ) );
400  if ( ! value.empty() )
401  url_r.setQueryParam( param, value );
402  }
403  return url_r;
404 }
405 
406 }
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
void setPassword(const std::string &pass, EEncoding eflag=zypp::url::E_DECODED)
Set the password in the URL authority.
Definition: Url.cc:739
void globalInitCurlOnce()
Definition: curlhelper.cc:64
#define MIL
Definition: Logger.h:96
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:137
void setQueryParam(const std::string &param, const std::string &value)
Set or add value for the specified query parameter.
Definition: Url.cc:838
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
Definition: curlhelper.cc:394
Flag to request encoded string(s).
Definition: UrlUtils.h:53
void setPassword(const std::string &val_r)
sets the auth password
Holds transfer setting.
const std::string & proxyUsername() const
proxy auth username
#define INT
Definition: Logger.h:100
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:367
bool useProxyFor(const Url &url_r) const
Return true if enabled and url_r does not match noProxy.
Definition: proxyinfo.cc:55
static const ViewOption WITH_SCHEME
Option to include scheme name in the URL string.
Definition: UrlBase.h:51
static const ViewOption WITH_HOST
Option to include hostname in the URL string.
Definition: UrlBase.h:74
void setPathParams(const std::string &params)
Set the path parameters.
Definition: Url.cc:791
void setUsername(const std::string &val_r)
sets the auth username
std::string curlEscapedPath(std::string path_r)
Definition: curlhelper.cc:355
void setHeadRequestsAllowed(bool allowed)
set whether HEAD requests are allowed
void setUsername(const std::string &user, EEncoding eflag=zypp::url::E_DECODED)
Set the username in the URL authority.
Definition: Url.cc:730
void setFragment(const std::string &fragment, EEncoding eflag=zypp::url::E_DECODED)
Set the fragment string in the URL.
Definition: Url.cc:722
const std::string & username() const
auth username
void setAnonymousAuth()
sets anonymous authentication (ie: for ftp)
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
void curlEscape(std::string &str_r, const char char_r, const std::string &escaped_r)
Definition: curlhelper.cc:347
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
void setAuthType(const std::string &val_r)
set the allowed authentication types
void setProxy(const std::string &val_r)
proxy to use if it is enabled
Just inherits Exception to separate media exceptions.
Provides API related macros.
#define WAR
Definition: Logger.h:97
const long & ZYPP_MEDIA_CURL_DEBUG()
const long& for setting CURLOPT_DEBUGDATA Returns a reference to a static variable, so it&#39;s safe to pass ...
Definition: curlhelper.cc:36
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:177
void setTimeout(long t)
set the transfer timeout
std::string proxy(const Url &url) const
Definition: proxyinfo.cc:43
std::ostream & hexdumpOn(std::ostream &outs, const unsigned char *ptr, size_t size)
hexdump data on stream
Definition: LogTools.h:445
SolvableIdType size_type
Definition: PoolMember.h:126
std::string rtrim(const std::string &s)
Definition: String.h:511
struct zypp::media::MediaBlock __attribute__
void setProxyPassword(const std::string &val_r)
sets the proxy password
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:360
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:124
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
void setClientCertificatePath(const Pathname &val_r)
Sets the SSL client certificate file.
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:325
void setProxyUsername(const std::string &val_r)
sets the proxy user
#define TRANSFER_TIMEOUT_MAX
Definition: curlhelper.cc:27
void setCertificateAuthoritiesPath(const Pathname &val_r)
Sets the SSL certificate authorities path.
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
static const ViewOption WITH_PORT
Option to include port number in the URL string.
Definition: UrlBase.h:81
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
int log_curl(CURL *curl, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: curlhelper.cc:74
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
void setVerifyPeerEnabled(bool enabled)
Sets whether to verify host for ssl.
void setClientKeyPath(const Pathname &val_r)
Sets the SSL client key file.
std::string getPassword(EEncoding eflag=zypp::url::E_DECODED) const
Returns the password from the URL authority.
Definition: Url.cc:580
void setVerifyHostEnabled(bool enabled)
Sets whether to verify host for ssl.
Url manipulation class.
Definition: Url.h:91
void setProxyEnabled(bool enabled)
whether the proxy is used or not
#define DBG
Definition: Logger.h:95
void delQueryParam(const std::string &param)
remove the specified query parameter.
Definition: Url.cc:845
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572
const std::string & msg() const
Return the message string provided to the ctor.
Definition: Exception.h:195