libzypp  17.31.11
curlhelper.cc
Go to the documentation of this file.
1 /*---------------------------------------------------------------------\
2 | ____ _ __ __ ___ |
3 | |__ / \ / / . \ . \ |
4 | / / \ V /| _/ _/ |
5 | / /__ | | | | | | |
6 | /_____||_| |_| |_| |
7 | |
8 \---------------------------------------------------------------------*/
12 #include "private/curlhelper_p.h"
13 
14 #include <zypp/APIConfig.h>
15 
16 #include <zypp-core/fs/PathInfo.h>
17 #include <zypp-core/Pathname.h>
18 #include <zypp-core/base/LogTools.h>
19 #include <zypp-core/base/String.h>
20 #include <zypp-core/base/StringV.h>
21 #include <zypp-curl/ProxyInfo>
22 #include <zypp-curl/auth/CurlAuthData>
23 #include <zypp-media/MediaException>
24 #include <list>
25 #include <string>
26 
27 #define TRANSFER_TIMEOUT_MAX 60 * 60
28 
29 using std::endl;
30 using namespace zypp;
31 
32 namespace zypp
33 {
34  namespace env
35  {
36  const long & ZYPP_MEDIA_CURL_DEBUG()
37  {
38  static const long ret = [](){
39  const char * env = getenv("ZYPP_MEDIA_CURL_DEBUG");
40  return env && *env ? str::strtonum<ulong>( env ) : 0;
41  }();
42  return ret;
43  }
44 
46  {
47  static int _v = [](){
48  int ret = 0;
49  if ( const char * envp = getenv( "ZYPP_MEDIA_CURL_IPRESOLVE" ) ) {
50  WAR << "env set: $ZYPP_MEDIA_CURL_IPRESOLVE='" << envp << "'" << std::endl;
51  if ( strcmp( envp, "4" ) == 0 ) ret = 4;
52  else if ( strcmp( envp, "6" ) == 0 ) ret = 6;
53  }
54  return ret;
55  }();
56  return _v;
57  }
58  } // namespace env
59 } // namespace zypp
60 
61 namespace internal
62 {
63 
65 {
66  // function-level static <=> std::call_once
67  static bool once __attribute__ ((__unused__)) = ( [] {
68  if ( curl_global_init( CURL_GLOBAL_ALL ) != 0 )
69  WAR << "curl global init failed" << std::endl;
70  } (), true );
71 }
72 
73 int log_curl( CURL * curl, curl_infotype info, char * ptr, size_t len, void * max_lvl )
74 {
75  if ( max_lvl == nullptr )
76  return 0;
77 
78  long maxlvl = *((long *)max_lvl);
79  const char * pfx = "";
80  bool isContent = true; // otherwise it's data
81  switch( info )
82  {
83  case CURLINFO_TEXT: if ( maxlvl < 1 ) return 0; pfx = "*"; break;
84  case CURLINFO_HEADER_IN: if ( maxlvl < 2 ) return 0; pfx = "<"; break;
85  case CURLINFO_HEADER_OUT: if ( maxlvl < 2 ) return 0; pfx = ">"; break;
86  case CURLINFO_SSL_DATA_IN: if ( maxlvl < 3 ) return 0; isContent = false; pfx = "<[SSL]"; break;
87  case CURLINFO_SSL_DATA_OUT: if ( maxlvl < 3 ) return 0; isContent = false; pfx = ">[SSL]"; break;
88  case CURLINFO_DATA_IN: if ( maxlvl < 3 ) return 0; isContent = false; pfx = "<[DTA]"; break;
89  case CURLINFO_DATA_OUT: if ( maxlvl < 3 ) return 0; isContent = false; pfx = ">[DTA]"; break;
90 
91  default:
92  return 0;
93  }
94 
95  // We'd like to keep all log messages within function `log_curl`
96  // because this tag to grep for is known and communicate to users.
97  if ( isContent ) {
98  std::vector<std::string_view> lines; // don't want log from within the lambda
99  strv::split( std::string_view( ptr, len ), "\n", [&lines]( std::string_view line, unsigned, bool last ) {
100  if ( last ) return; // empty word after final \n
101  line = strv::rtrim( line, "\r" );
102  lines.push_back( line );
103  });
104  for ( const auto & line : lines ) {
105  if ( str::hasPrefix( line, "Authorization:" ) ) {
106  std::string_view::size_type pos { line.find( " ", 15 ) }; // Authorization: <type> <credentials>
107  if ( pos == std::string::npos )
108  pos = 15;
109  DBG << curl << " " << pfx << " " << line.substr( 0, pos ) << " <credentials removed>" << endl;
110  }
111  else
112  DBG << curl << " " << pfx << " " << line << endl;
113  }
114  } else {
115  if ( maxlvl < 4 )
116  DBG << curl << " " << pfx << " " << len << " byte" << endl;
117  else
118  hexdumpOn( DBG << curl << " " << pfx << " ", ptr, len );
119  }
120  return 0;
121 }
122 
123 void setupZYPP_MEDIA_CURL_DEBUG( CURL *curl )
124 {
125  if ( not curl ) {
126  INT << "Got a NULL curl handle" << endl;
127  return;
128  }
129  if ( env::ZYPP_MEDIA_CURL_DEBUG() > 0 ) {
130  curl_easy_setopt( curl, CURLOPT_VERBOSE, 1L );
131  curl_easy_setopt( curl, CURLOPT_DEBUGFUNCTION, log_curl );
132  curl_easy_setopt( curl, CURLOPT_DEBUGDATA, &env::ZYPP_MEDIA_CURL_DEBUG() );
133  }
134 }
135 
136 size_t log_redirects_curl( char *ptr, size_t size, size_t nmemb, void *userdata)
137 {
138  //INT << "got header: " << std::string(ptr, ptr + size*nmemb) << endl;
139 
140  char * lstart = ptr, * lend = ptr;
141  size_t pos = 0;
142  size_t max = size * nmemb;
143  while (pos + 1 < max)
144  {
145  // get line
146  for (lstart = lend; *lend != '\n' && pos < max; ++lend, ++pos);
147 
148  // look for "Location"
149  if ( strncasecmp( lstart, "Location:", 9 ) == 0 )
150  {
151  std::string line { lstart, *(lend-1)=='\r' ? lend-1 : lend };
152  DBG << "redirecting to " << line << std::endl;
153  if ( userdata ) {
154  *reinterpret_cast<std::string *>( userdata ) = line;
155  }
156  return max;
157  }
158 
159  // continue with the next line
160  if (pos + 1 < max)
161  {
162  ++lend;
163  ++pos;
164  }
165  else
166  break;
167  }
168 
169  return max;
170 }
171 
177 {
178  {
179  const std::string & param { url.getQueryParam("timeout") };
180  if( ! param.empty() )
181  {
182  long num = str::strtonum<long>(param);
183  if( num >= 0 && num <= TRANSFER_TIMEOUT_MAX )
184  s.setTimeout( num );
185  }
186  }
187  {
188  std::string param { url.getUsername() };
189  if ( ! param.empty() )
190  {
191  s.setUsername( std::move(param) );
192  param = url.getPassword();
193  if ( ! param.empty() )
194  s.setPassword( std::move(param) );
195  }
196  else
197  {
198  // if there is no username, set anonymous auth
199  if ( ( url.getScheme() == "ftp" || url.getScheme() == "tftp" ) && s.username().empty() )
200  s.setAnonymousAuth();
201  }
202  }
203  if ( url.getScheme() == "https" )
204  {
205  s.setVerifyPeerEnabled( false );
206  s.setVerifyHostEnabled( false );
207 
208  const std::string & verify { url.getQueryParam("ssl_verify") };
209  if( verify.empty() || verify == "yes" )
210  {
211  s.setVerifyPeerEnabled( true );
212  s.setVerifyHostEnabled( true );
213  }
214  else if ( verify == "no" )
215  {
216  s.setVerifyPeerEnabled( false );
217  s.setVerifyHostEnabled( false );
218  }
219  else
220  {
221  std::vector<std::string> flags;
222  str::split( verify, std::back_inserter(flags), "," );
223  for ( const auto & flag : flags )
224  {
225  if ( flag == "host" )
226  s.setVerifyHostEnabled( true );
227  else if ( flag == "peer" )
228  s.setVerifyPeerEnabled( true );
229  else
230  ZYPP_THROW( media::MediaBadUrlException(url, "Unknown ssl_verify flag "+flag) );
231  }
232  }
233  }
234  {
235  Pathname ca_path { url.getQueryParam("ssl_capath") };
236  if( ! ca_path.empty() )
237  {
238  if( ! PathInfo(ca_path).isDir() || ! ca_path.absolute() )
239  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_capath path"));
240  else
241  s.setCertificateAuthoritiesPath( std::move(ca_path) );
242  }
243  }
244  {
245  Pathname client_cert { url.getQueryParam("ssl_clientcert") };
246  if( ! client_cert.empty() )
247  {
248  if( ! PathInfo(client_cert).isFile() || ! client_cert.absolute() )
249  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientcert file"));
250  else
251  s.setClientCertificatePath( std::move(client_cert) );
252  }
253  }
254  {
255  Pathname client_key { url.getQueryParam("ssl_clientkey") };
256  if( ! client_key.empty() )
257  {
258  if( ! PathInfo(client_key).isFile() || ! client_key.absolute() )
259  ZYPP_THROW(media::MediaBadUrlException(url, "Invalid ssl_clientkey file"));
260  else
261  s.setClientKeyPath( std::move(client_key) );
262  }
263  }
264  {
265  std::string param { url.getQueryParam( "proxy" ) };
266  if ( ! param.empty() )
267  {
268  if ( param == EXPLICITLY_NO_PROXY ) {
269  // Workaround TransferSettings shortcoming: With an
270  // empty proxy string, code will continue to look for
271  // valid proxy settings. So set proxy to some non-empty
272  // string, to indicate it has been explicitly disabled.
274  s.setProxyEnabled(false);
275  }
276  else {
277  const std::string & proxyport { url.getQueryParam( "proxyport" ) };
278  if ( ! proxyport.empty() ) {
279  param += ":";
280  param += proxyport;
281  }
282  s.setProxy( std::move(param) );
283  s.setProxyEnabled( true );
284  }
285  }
286  }
287  {
288  std::string param { url.getQueryParam( "proxyuser" ) };
289  if ( ! param.empty() )
290  {
291  s.setProxyUsername( std::move(param) );
292  s.setProxyPassword( url.getQueryParam( "proxypass" ) );
293  }
294  }
295  {
296  // HTTP authentication type
297  std::string param { url.getQueryParam("auth") };
298  if ( ! param.empty() && (url.getScheme() == "http" || url.getScheme() == "https") )
299  {
300  try
301  {
302  media::CurlAuthData::auth_type_str2long (param ); // check if we know it
303  }
304  catch ( const media::MediaException & ex_r )
305  {
306  DBG << "Rethrowing as MediaUnauthorizedException.";
307  ZYPP_THROW(media::MediaUnauthorizedException(url, ex_r.msg(), "", ""));
308  }
309  s.setAuthType( std::move(param) );
310  }
311  }
312  {
313  // workarounds
314  const std::string & param { url.getQueryParam("head_requests") };
315  if( ! param.empty() && param == "no" )
316  s.setHeadRequestsAllowed( false );
317  }
318 }
319 
325 {
326  media::ProxyInfo proxy_info;
327  if ( proxy_info.useProxyFor( url ) )
328  {
329  // We must extract any 'user:pass' from the proxy url
330  // otherwise they won't make it into curl (.curlrc wins).
331  try {
332  Url u( proxy_info.proxy( url ) );
334  // don't overwrite explicit auth settings
335  if ( s.proxyUsername().empty() )
336  {
337  s.setProxyUsername( u.getUsername( url::E_ENCODED ) );
338  s.setProxyPassword( u.getPassword( url::E_ENCODED ) );
339  }
340  s.setProxyEnabled( true );
341  }
342  catch (...) {} // no proxy if URL is malformed
343  }
344 }
345 
346 void curlEscape( std::string & str_r,
347  const char char_r, const std::string & escaped_r ) {
348  for ( std::string::size_type pos = str_r.find( char_r );
349  pos != std::string::npos; pos = str_r.find( char_r, pos ) ) {
350  str_r.replace( pos, 1, escaped_r );
351  }
352 }
353 
354 std::string curlEscapedPath( std::string path_r ) {
355  curlEscape( path_r, ' ', "%20" );
356  return path_r;
357 }
358 
359 std::string curlUnEscape( std::string text_r ) {
360  char * tmp = curl_unescape( text_r.c_str(), 0 );
361  std::string ret( tmp );
362  curl_free( tmp );
363  return ret;
364 }
365 
367 {
368  Url curlUrl (url);
369  curlUrl.setUsername( "" );
370  curlUrl.setPassword( "" );
371  curlUrl.setPathParams( "" );
372  curlUrl.setFragment( "" );
373  curlUrl.delQueryParam("cookies");
374  curlUrl.delQueryParam("proxy");
375  curlUrl.delQueryParam("proxyport");
376  curlUrl.delQueryParam("proxyuser");
377  curlUrl.delQueryParam("proxypass");
378  curlUrl.delQueryParam("ssl_capath");
379  curlUrl.delQueryParam("ssl_verify");
380  curlUrl.delQueryParam("ssl_clientcert");
381  curlUrl.delQueryParam("timeout");
382  curlUrl.delQueryParam("auth");
383  curlUrl.delQueryParam("username");
384  curlUrl.delQueryParam("password");
385  curlUrl.delQueryParam("mediahandler");
386  curlUrl.delQueryParam("credentials");
387  curlUrl.delQueryParam("head_requests");
388  return curlUrl;
389 }
390 
391 // bsc#933839: propagate proxy settings passed in the repo URL
392 // boo#1127591: propagate ssl settings passed in the repo URL
393 zypp::Url propagateQueryParams( zypp::Url url_r, const zypp::Url & template_r )
394 {
395  using namespace std::literals::string_literals;
396  for ( const std::string &param : { "proxy"s, "proxyport"s, "proxyuser"s, "proxypass"s, "ssl_capath"s, "ssl_verify"s } )
397  {
398  const std::string & value( template_r.getQueryParam( param ) );
399  if ( ! value.empty() )
400  url_r.setQueryParam( param, value );
401  }
402  return url_r;
403 }
404 
405 }
std::string getScheme() const
Returns the scheme name of the URL.
Definition: Url.cc:533
void setPassword(const std::string &pass, EEncoding eflag=zypp::url::E_DECODED)
Set the password in the URL authority.
Definition: Url.cc:739
void globalInitCurlOnce()
Definition: curlhelper.cc:64
size_t log_redirects_curl(char *ptr, size_t size, size_t nmemb, void *userdata)
Definition: curlhelper.cc:136
void setQueryParam(const std::string &param, const std::string &value)
Set or add value for the specified query parameter.
Definition: Url.cc:838
#define ZYPP_THROW(EXCPT)
Drops a logline and throws the Exception.
Definition: Exception.h:428
zypp::Url propagateQueryParams(zypp::Url url_r, const zypp::Url &template_r)
Definition: curlhelper.cc:393
Flag to request encoded string(s).
Definition: UrlUtils.h:53
void setPassword(const std::string &val_r)
sets the auth password
Holds transfer setting.
const std::string & proxyUsername() const
proxy auth username
#define INT
Definition: Logger.h:100
Url clearQueryString(const Url &url)
Definition: curlhelper.cc:366
bool useProxyFor(const Url &url_r) const
Return true if enabled and url_r does not match noProxy.
Definition: proxyinfo.cc:55
static const ViewOption WITH_SCHEME
Option to include scheme name in the URL string.
Definition: UrlBase.h:51
static const ViewOption WITH_HOST
Option to include hostname in the URL string.
Definition: UrlBase.h:74
void setPathParams(const std::string &params)
Set the path parameters.
Definition: Url.cc:791
void setUsername(const std::string &val_r)
sets the auth username
std::string curlEscapedPath(std::string path_r)
Definition: curlhelper.cc:354
void setHeadRequestsAllowed(bool allowed)
set whether HEAD requests are allowed
void setUsername(const std::string &user, EEncoding eflag=zypp::url::E_DECODED)
Set the username in the URL authority.
Definition: Url.cc:730
void setFragment(const std::string &fragment, EEncoding eflag=zypp::url::E_DECODED)
Set the fragment string in the URL.
Definition: Url.cc:722
const std::string & username() const
auth username
void setAnonymousAuth()
sets anonymous authentication (ie: for ftp)
int ZYPP_MEDIA_CURL_IPRESOLVE()
4/6 to force IPv4/v6
Definition: curlhelper.cc:45
void curlEscape(std::string &str_r, const char char_r, const std::string &escaped_r)
Definition: curlhelper.cc:346
unsigned split(const C_Str &line_r, TOutputIterator result_r, const C_Str &sepchars_r=" \, const Trim trim_r=NO_TRIM)
Split line_r into words.
Definition: String.h:531
std::string getQueryParam(const std::string &param, EEncoding eflag=zypp::url::E_DECODED) const
Return the value for the specified query parameter.
Definition: Url.cc:660
void setAuthType(const std::string &val_r)
set the allowed authentication types
void setProxy(const std::string &val_r)
proxy to use if it is enabled
Just inherits Exception to separate media exceptions.
#define WAR
Definition: Logger.h:97
const long & ZYPP_MEDIA_CURL_DEBUG()
const long& for setting CURLOPT_DEBUGDATA Returns a reference to a static variable, so it&#39;s safe to pass ...
Definition: curlhelper.cc:36
void fillSettingsFromUrl(const Url &url, media::TransferSettings &s)
Fills the settings structure using options passed on the url for example ?timeout=x&proxy=foo.
Definition: curlhelper.cc:176
void setTimeout(long t)
set the transfer timeout
std::string proxy(const Url &url) const
Definition: proxyinfo.cc:43
std::ostream & hexdumpOn(std::ostream &outs, const unsigned char *ptr, size_t size)
hexdump data on stream
Definition: LogTools.h:445
SolvableIdType size_type
Definition: PoolMember.h:126
std::string rtrim(const std::string &s)
Definition: String.h:511
struct zypp::media::MediaBlock __attribute__
void setProxyPassword(const std::string &val_r)
sets the proxy password
std::string curlUnEscape(std::string text_r)
Definition: curlhelper.cc:359
void setupZYPP_MEDIA_CURL_DEBUG(CURL *curl)
Setup CURLOPT_VERBOSE and CURLOPT_DEBUGFUNCTION according to env::ZYPP_MEDIA_CURL_DEBUG.
Definition: curlhelper.cc:123
static long auth_type_str2long(std::string &auth_type_str)
Converts a string of comma separated list of authetication type names into a long of ORed CURLAUTH_* ...
Definition: curlauthdata.cc:50
Wrapper class for ::stat/::lstat.
Definition: PathInfo.h:220
void setClientCertificatePath(const Pathname &val_r)
Sets the SSL client certificate file.
void fillSettingsSystemProxy(const Url &url, media::TransferSettings &s)
Reads the system proxy configuration and fills the settings structure proxy information.
Definition: curlhelper.cc:324
void setProxyUsername(const std::string &val_r)
sets the proxy user
#define TRANSFER_TIMEOUT_MAX
Definition: curlhelper.cc:27
void setCertificateAuthoritiesPath(const Pathname &val_r)
Sets the SSL certificate authorities path.
#define EXPLICITLY_NO_PROXY
Definition: curlhelper_p.h:21
static const ViewOption WITH_PORT
Option to include port number in the URL string.
Definition: UrlBase.h:81
Easy-to use interface to the ZYPP dependency resolver.
Definition: CodePitfalls.doc:1
int log_curl(CURL *curl, curl_infotype info, char *ptr, size_t len, void *max_lvl)
Definition: curlhelper.cc:73
bool hasPrefix(const C_Str &str_r, const C_Str &prefix_r)
Return whether str_r has prefix prefix_r.
Definition: String.h:1027
void setVerifyPeerEnabled(bool enabled)
Sets whether to verify host for ssl.
void setClientKeyPath(const Pathname &val_r)
Sets the SSL client key file.
std::string getPassword(EEncoding eflag=zypp::url::E_DECODED) const
Returns the password from the URL authority.
Definition: Url.cc:580
void setVerifyHostEnabled(bool enabled)
Sets whether to verify host for ssl.
Url manipulation class.
Definition: Url.h:91
void setProxyEnabled(bool enabled)
whether the proxy is used or not
#define DBG
Definition: Logger.h:95
void delQueryParam(const std::string &param)
remove the specified query parameter.
Definition: Url.cc:845
std::string getUsername(EEncoding eflag=zypp::url::E_DECODED) const
Returns the username from the URL authority.
Definition: Url.cc:572
const std::string & msg() const
Return the message string provided to the ctor.
Definition: Exception.h:195