XRootD
XrdClUtils.cc
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2012 by European Organization for Nuclear Research (CERN)
3 // Author: Lukasz Janyst <ljanyst@cern.ch>
4 //------------------------------------------------------------------------------
5 // XRootD is free software: you can redistribute it and/or modify
6 // it under the terms of the GNU Lesser General Public License as published by
7 // the Free Software Foundation, either version 3 of the License, or
8 // (at your option) any later version.
9 //
10 // XRootD is distributed in the hope that it will be useful,
11 // but WITHOUT ANY WARRANTY; without even the implied warranty of
12 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
13 // GNU General Public License for more details.
14 //
15 // You should have received a copy of the GNU Lesser General Public License
16 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
17 //------------------------------------------------------------------------------
18 
19 #include "XrdCl/XrdClUtils.hh"
20 #include "XrdCl/XrdClFileSystem.hh"
21 #include "XrdCl/XrdClDefaultEnv.hh"
22 #include "XrdCl/XrdClConstants.hh"
25 #include "XrdCl/XrdClMessage.hh"
26 #include "XrdCl/XrdClOptimizers.hh"
27 #include "XrdNet/XrdNetAddr.hh"
28 
29 #include <algorithm>
30 #include <iomanip>
31 #include <sstream>
32 #include <fstream>
33 #include <functional>
34 #include <cctype>
35 #include <locale>
36 #include <map>
37 #include <string>
38 #include <set>
39 #include <cctype>
40 #include <random>
41 #include <chrono>
42 
43 #include <sys/types.h>
44 #include <dirent.h>
45 
46 #if __cplusplus < 201103L
47 #include <ctime>
48 #endif
49 
50 namespace
51 {
52  bool isNotSpace( char c )
53  {
54  return c != ' ';
55  }
56 
57  //----------------------------------------------------------------------------
58  // Ordering function for sorting IP addresses
59  //----------------------------------------------------------------------------
60  struct PreferIPv6
61  {
62  bool operator() ( const XrdNetAddr &l, const XrdNetAddr &r )
63  {
64  bool rIsIPv4 = false;
65  if( r.isIPType( XrdNetAddrInfo::IPv4 ) ||
66  (r.isIPType( XrdNetAddrInfo::IPv6 ) && r.isMapped()) )
67  rIsIPv4 = true;
68 
69  if( l.isIPType( XrdNetAddrInfo::IPv6 ) && rIsIPv4 )
70  return true;
71  return false;
72  }
73  };
74 }
75 
76 namespace XrdCl
77 {
78  //----------------------------------------------------------------------------
79  // Get a parameter either from the environment or URL
80  //----------------------------------------------------------------------------
81  int Utils::GetIntParameter( const URL &url,
82  const std::string &name,
83  int defaultVal )
84  {
85  Env *env = DefaultEnv::GetEnv();
86  int value = defaultVal;
87  char *endPtr;
88  URL::ParamsMap::const_iterator it;
89 
90  env->GetInt( name, value );
91  it = url.GetParams().find( std::string("XrdCl.") + name );
92  if( it != url.GetParams().end() )
93  {
94  int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 );
95  if( !*endPtr )
96  value = urlValue;
97  }
98  return value;
99  }
100 
101  //----------------------------------------------------------------------------
102  // Get a parameter either from the environment or URL
103  //----------------------------------------------------------------------------
104  std::string Utils::GetStringParameter( const URL &url,
105  const std::string &name,
106  const std::string &defaultVal )
107  {
108  Env *env = DefaultEnv::GetEnv();
109  std::string value = defaultVal;
110  URL::ParamsMap::const_iterator it;
111 
112  env->GetString( name, value );
113  it = url.GetParams().find( std::string("XrdCl.") + name );
114  if( it != url.GetParams().end() )
115  value = it->second;
116 
117  return value;
118  }
119 
120  //----------------------------------------------------------------------------
121  // Interpret a string as address type, default to IPAll
122  //----------------------------------------------------------------------------
123  Utils::AddressType Utils::String2AddressType( const std::string &addressType )
124  {
125  if( addressType == "IPv6" )
126  return IPv6;
127  else if( addressType == "IPv4" )
128  return IPv4;
129  else if( addressType == "IPv4Mapped6" )
130  return IPv4Mapped6;
131  else if( addressType == "IPAll" )
132  return IPAll;
133  else
134  return IPAuto;
135  }
136 
137  //----------------------------------------------------------------------------
138  // Resolve IP addresses
139  //----------------------------------------------------------------------------
140  Status Utils::GetHostAddresses( std::vector<XrdNetAddr> &addresses,
141  const URL &url,
142  Utils::AddressType type )
143  {
144  Log *log = DefaultEnv::GetLog();
145  const char *err = 0;
146  int ordn;
147 
148  //--------------------------------------------------------------------------
149  // Resolve all the addresses
150  //--------------------------------------------------------------------------
151  std::ostringstream o; o << url.GetHostName() << ":" << url.GetPort();
153 
154  if( type == IPv6 ) opts = XrdNetUtils::onlyIPv6;
155  else if( type == IPv4 ) opts = XrdNetUtils::onlyIPv4;
156  else if( type == IPv4Mapped6 ) opts = XrdNetUtils::allV4Map;
157  else if( type == IPAll ) opts = XrdNetUtils::allIPMap;
159 
160  //--------------------------------------------------------------------------
161  // Check what are the preferences IPv6 or IPv4
162  //--------------------------------------------------------------------------
163  int preferIPv4 = DefaultPreferIPv4;
164  DefaultEnv::GetEnv()->GetInt( "PreferIPv4", preferIPv4 );
165 
166  //--------------------------------------------------------------------------
167  // Partition the addresses according to the preferences
168  //
169  // The preferred IP family goes to the back as it is easier to remove
170  // items from the back of the vector
171  //--------------------------------------------------------------------------
172  opts |= (preferIPv4 ? XrdNetUtils::order64 : XrdNetUtils::order46);
173 
174  //--------------------------------------------------------------------------
175  // Now get all of the properly partitioned addresses; ordn will hold the
176  // number of non-preferred addresses at the front of the table.
177  //--------------------------------------------------------------------------
178  err = XrdNetUtils::GetAddrs( o.str(), addresses, &ordn, opts );
179 
180  if( err )
181  {
182  log->Error( UtilityMsg, "Unable to resolve %s: %s", o.str().c_str(),
183  err );
184  return Status( stError, errInvalidAddr );
185  }
186 
187  if( addresses.size() == 0 )
188  {
189  log->Error( UtilityMsg, "No addresses for %s were found",
190  o.str().c_str() );
191  return Status( stError, errInvalidAddr );
192  }
193 
194  //--------------------------------------------------------------------------
195  // Shuffle each partition
196  //--------------------------------------------------------------------------
197 
198  int ipNoShuffle = DefaultIPNoShuffle;
199  Env *env = DefaultEnv::GetEnv();
200  env->GetInt( "IPNoShuffle", ipNoShuffle );
201 
202  if( !ipNoShuffle )
203  {
204 #if __cplusplus < 201103L
205  // initialize the random generator only once
206  static struct only_once_t
207  {
208  only_once_t()
209  {
210  std::srand ( unsigned ( std::time(0) ) );
211  }
212  } only_once;
213 
214  std::random_shuffle( addresses.begin(), addresses.begin() + ordn );
215  std::random_shuffle( addresses.begin() + ordn, addresses.end() );
216 #else
217  static std::default_random_engine rand_engine(
218  std::chrono::system_clock::now().time_since_epoch().count() );
219 
220  std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine );
221  std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine );
222 #endif
223  }
224 
225  //--------------------------------------------------------------------------
226  // Return status as the result is already in the output parameter
227  //--------------------------------------------------------------------------
228  return Status();
229  }
230 
231  //----------------------------------------------------------------------------
232  // Log all the addresses on the list
233  //----------------------------------------------------------------------------
235  uint64_t type,
236  const std::string &hostId,
237  std::vector<XrdNetAddr> &addresses )
238  {
239  std::string addrStr;
240  std::vector<XrdNetAddr>::iterator it;
241  for( it = addresses.begin(); it != addresses.end(); ++it )
242  {
243  char nameBuff[256];
244  it->Format( nameBuff, 256, XrdNetAddrInfo::fmtAdv6 );
245  addrStr += nameBuff;
246  addrStr += ", ";
247  }
248  addrStr.erase( addrStr.length()-2, 2 );
249  log->Debug( type, "[%s] Found %zu address(es): %s",
250  hostId.c_str(), addresses.size(), addrStr.c_str() );
251  }
252 
253  //----------------------------------------------------------------------------
254  // Convert timestamp to a string
255  //----------------------------------------------------------------------------
256  std::string Utils::TimeToString( time_t timestamp )
257  {
258  char now[30];
259  tm tsNow;
260  time_t ttNow = timestamp;
261  localtime_r( &ttNow, &tsNow );
262  strftime( now, 30, "%Y-%m-%d %H:%M:%S %z", &tsNow );
263  return now;
264  }
265 
266  //----------------------------------------------------------------------------
267  // Get the elapsed microseconds between two timevals
268  //----------------------------------------------------------------------------
269  uint64_t Utils::GetElapsedMicroSecs( timeval start, timeval end )
270  {
271  uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec;
272  uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec;
273  return endUSec-startUSec;
274  }
275 
276  //----------------------------------------------------------------------------
277  // Get remote checksum
278  //----------------------------------------------------------------------------
279  XRootDStatus Utils::GetRemoteCheckSum( std::string &checkSum,
280  const std::string &checkSumType,
281  const URL &url )
282  {
283  FileSystem *fs = new FileSystem( url );
284  // add the 'cks.type' cgi tag in order to
285  // select the proper checksum type in case
286  // the server supports more than one checksum
287  size_t pos = url.GetPath().find( '?' );
288  std::string cksPath = url.GetPath() + ( pos == std::string::npos ? '?' : '&' ) + "cks.type=" + checkSumType;
289  Buffer arg; arg.FromString( cksPath );
290  Buffer *cksResponse = 0;
291  XRootDStatus st;
292  Log *log = DefaultEnv::GetLog();
293 
294  st = fs->Query( QueryCode::Checksum, arg, cksResponse );
295  delete fs;
296 
297  if( !st.IsOK() )
298  {
299  std::string msg = st.GetErrorMessage();
300  msg += " Got an error while querying the checksum!";
301  st.SetErrorMessage( msg );
302  return st;
303  }
304 
305  if( !cksResponse )
306  return XRootDStatus( stError, errInternal, 0, "Got invalid response while querying the checksum!" );
307 
308  std::vector<std::string> elems;
309  Utils::splitString( elems, cksResponse->ToString(), " " );
310  delete cksResponse;
311 
312  if( elems.size() != 2 )
313  return XRootDStatus( stError, errInvalidResponse, 0, "Got invalid response while querying the checksum!" );
314 
315  if( elems[0] != checkSumType )
317 
318  checkSum = elems[0] + ":";
319  checkSum += NormalizeChecksum( elems[0], elems[1] );
320 
321  log->Dump( UtilityMsg, "Checksum for %s checksum: %s",
322  url.GetPath().c_str(), checkSum.c_str() );
323 
324  return XRootDStatus();
325  }
326 
327  //------------------------------------------------------------------------
328  // Get a checksum from local file
329  //------------------------------------------------------------------------
330  XRootDStatus Utils::GetLocalCheckSum( std::string &checkSum,
331  const std::string &checkSumType,
332  const std::string &path )
333  {
334  Log *log = DefaultEnv::GetLog();
336 
337  if( !cksMan )
338  {
339  log->Error( UtilityMsg, "Unable to get the checksum manager" );
340  return XRootDStatus( stError, errInternal );
341  }
342 
343  XrdCksData ckSum; ckSum.Set( checkSumType.c_str() );
344  bool status = cksMan->Calculate( ckSum, checkSumType, path.c_str() );
345  if( !status )
346  {
347  log->Error( UtilityMsg, "Error while calculating checksum for %s",
348  path.c_str() );
350  }
351 
352  char *cksBuffer = new char[265];
353  ckSum.Get( cksBuffer, 256 );
354  checkSum = checkSumType + ":";
355  checkSum += NormalizeChecksum( checkSumType, cksBuffer );
356  delete [] cksBuffer;
357 
358  log->Dump( UtilityMsg, "Checksum for %s is: %s", path.c_str(),
359  checkSum.c_str() );
360 
361  return XRootDStatus();
362  }
363 
364  //----------------------------------------------------------------------------
365  // Convert bytes to a human readable string
366  //----------------------------------------------------------------------------
367  std::string Utils::BytesToString( uint64_t bytes )
368  {
369  double final = bytes;
370  int i = 0;
371  char suf[3] = { 'k', 'M', 'G' };
372  for( i = 0; i < 3 && final > 1024; ++i, final /= 1024 ) {};
373  std::ostringstream o;
374  o << std::setprecision(4) << final;
375  if( i > 0 ) o << suf[i-1];
376  return o.str();
377  }
378 
379  //----------------------------------------------------------------------------
380  // Check if peer supports tpc
381  //----------------------------------------------------------------------------
382  XRootDStatus Utils::CheckTPC( const std::string &server, uint16_t timeout )
383  {
384  Log *log = DefaultEnv::GetLog();
385  log->Debug( UtilityMsg, "Checking if the data server %s supports tpc",
386  server.c_str() );
387 
388  FileSystem sourceDSFS( server );
389  Buffer queryArg; queryArg.FromString( "tpc" );
390  Buffer *queryResponse = 0;
391  XRootDStatus st;
392  st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
393  timeout );
394  if( !st.IsOK() )
395  {
396  log->Error( UtilityMsg, "Cannot query source data server %s: %s",
397  server.c_str(), st.ToStr().c_str() );
398  st.status = stFatal;
399  return st;
400  }
401 
402  if( !queryResponse )
403  {
404  log->Error( UtilityMsg, "Cannot query source data server: empty response." );
405  st.status = stFatal;
406  return st;
407  }
408 
409  std::string answer = queryResponse->ToString();
410  delete queryResponse;
411  if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0)
412  {
413  log->Debug( UtilityMsg, "Third party copy not supported at: %s",
414  server.c_str() );
416  }
417  log->Debug( UtilityMsg, "Third party copy supported at: %s",
418  server.c_str() );
419 
420  return XRootDStatus();
421  }
422 
423  //------------------------------------------------------------------------
424  // Check if peer supports tpc / tpc lite
425  //------------------------------------------------------------------------
426  XRootDStatus Utils::CheckTPCLite( const std::string &server, uint16_t timeout )
427  {
428  Log *log = DefaultEnv::GetLog();
429  log->Debug( UtilityMsg, "Checking if the data server %s supports tpc / tpc lite",
430  server.c_str() );
431 
432  FileSystem sourceDSFS( server );
433  Buffer queryArg; queryArg.FromString( "tpc tpcdlg" );
434  Buffer *queryResponse = 0;
435  XRootDStatus st;
436  st = sourceDSFS.Query( QueryCode::Config, queryArg, queryResponse,
437  timeout );
438  if( !st.IsOK() )
439  {
440  log->Error( UtilityMsg, "Cannot query source data server %s: %s",
441  server.c_str(), st.ToStr().c_str() );
442  st.status = stFatal;
443  return st;
444  }
445 
446  if( !queryResponse )
447  {
448  log->Error( UtilityMsg, "Cannot query source data server: empty response." );
449  st.status = stFatal;
450  return st;
451  }
452 
453  std::string answer = queryResponse->ToString();
454  delete queryResponse;
455 
456  if( answer.empty() )
457  {
458  log->Error( UtilityMsg, "Cannot query source data server: empty response." );
459  st.status = stFatal;
460  return st;
461  }
462 
463  std::vector<std::string> resp;
464  Utils::splitString( resp, answer, "\n" );
465 
466  if( resp.empty() || resp[0].empty() ||
467  !isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 )
468  {
469  log->Debug( UtilityMsg, "Third party copy not supported at: %s",
470  server.c_str() );
472  }
473 
474  if( resp.size() == 1 || resp[1] == "tpcdlg" )
475  {
476  log->Debug( UtilityMsg, "TPC lite not supported at: %s",
477  server.c_str() );
478  return XRootDStatus( stOK, suPartial );
479  }
480 
481  log->Debug( UtilityMsg, "TPC lite supported at: %s",
482  server.c_str() );
483 
484  return XRootDStatus();
485  }
486 
487  //----------------------------------------------------------------------------
488  // Convert the fully qualified host name to country code
489  //----------------------------------------------------------------------------
490  std::string Utils::FQDNToCC( const std::string &fqdn )
491  {
492  std::vector<std::string> el;
493  Utils::splitString( el, fqdn, "." );
494  if( el.size() < 2 )
495  return "us";
496 
497  std::string cc = *el.rbegin();
498  if( cc.length() == 2 )
499  return cc;
500  return "us";
501  }
502 
503  //----------------------------------------------------------------------------
504  // Get directory entries
505  //----------------------------------------------------------------------------
506  Status Utils::GetDirectoryEntries( std::vector<std::string> &entries,
507  const std::string &path )
508  {
509  DIR *dp = opendir( path.c_str() );
510  if( !dp )
511  return Status( stError, errOSError, errno );
512 
513  dirent *dirEntry;
514 
515  while( (dirEntry = readdir(dp)) != 0 )
516  {
517  std::string entryName = dirEntry->d_name;
518  if( !entryName.compare( 0, 2, "..") )
519  continue;
520  if( !entryName.compare( 0, 1, ".") )
521  continue;
522 
523  entries.push_back( dirEntry->d_name );
524  }
525 
526  closedir(dp);
527 
528  return Status();
529  }
530 
531  //----------------------------------------------------------------------------
532  // Process a config file and return key-value pairs
533  //----------------------------------------------------------------------------
534  Status Utils::ProcessConfig( std::map<std::string, std::string> &config,
535  const std::string &file )
536  {
537  config.clear();
538  std::ifstream inFile( file.c_str() );
539  if( !inFile.good() )
540  return Status( stError, errOSError, errno );
541 
542  errno = 0;
543  std::string line;
544  while( std::getline( inFile, line ) )
545  {
546  if( line.empty() || line[0] == '#' )
547  continue;
548 
549  std::vector<std::string> elems;
550  splitString( elems, line, "=" );
551  if( elems.size() != 2 )
552  return Status( stError, errConfig );
553  std::string key = elems[0]; Trim( key );
554  std::string value = elems[1]; Trim( value );
555  config[key] = value;
556  }
557 
558  if( errno )
559  return Status( stError, errOSError, errno );
560  return Status();
561  }
562 
563  //------------------------------------------------------------------------
565  //------------------------------------------------------------------------
566  Status Utils::ProcessConfigDir( std::map<std::string, std::string> &config,
567  const std::string &dir )
568  {
569  Log *log = DefaultEnv::GetLog();
570  log->Debug( UtilityMsg, "Processing configuration files in %s...",
571  dir.c_str());
572 
573  std::vector<std::string> entries;
574  Status st = Utils::GetDirectoryEntries( entries, dir );
575  if( !st.IsOK() )
576  {
577  log->Debug( UtilityMsg, "Unable to process directory %s: %s",
578  dir.c_str(), st.ToString().c_str() );
579  return st;
580  }
581 
582  static const std::string suffix = ".conf";
583  for( auto &entry : entries )
584  {
585  std::string confFile = dir + "/" + entry;
586 
587  if( confFile.length() <= suffix.length() )
588  continue;
589  if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) )
590  continue;
591 
592  st = ProcessConfig( config, confFile );
593  if( !st.IsOK() )
594  {
595  log->Debug( UtilityMsg, "Unable to process configuration file %s: %s",
596  confFile.c_str(), st.ToString().c_str() );
597  }
598  }
599 
600  return Status();
601  }
602 
603  //----------------------------------------------------------------------------
604  // Trim a string
605  //----------------------------------------------------------------------------
606  void Utils::Trim( std::string &str )
607  {
608  str.erase( str.begin(),
609  std::find_if( str.begin(), str.end(), isNotSpace ) );
610  str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(),
611  str.end() );
612  }
613 
614  //----------------------------------------------------------------------------
615  // Log property list
616  //----------------------------------------------------------------------------
618  uint64_t topic,
619  const char *format,
620  const PropertyList &list )
621  {
622  if( unlikely(log->GetLevel() >= Log::DumpMsg) ) {
623  PropertyList::PropertyMap::const_iterator it;
624  std::string keyVals;
625  for (it = list.begin(); it != list.end(); ++it)
626  keyVals += "'" + it->first + "' = '" + obfuscateAuth(it->second) + "', ";
627  keyVals.erase(keyVals.length() - 2, 2);
628  log->Dump(topic, format, keyVals.c_str());
629  }
630  }
631 
632  //----------------------------------------------------------------------------
633  // Print a char array as hex
634  //----------------------------------------------------------------------------
635  std::string Utils::Char2Hex( uint8_t *array, uint16_t size )
636  {
637  char *hex = new char[2*size+1];
638  for( uint16_t i = 0; i < size; ++i )
639  snprintf( hex+(2*i), 3, "%02x", (int)array[i] );
640  std::string result = hex;
641  delete [] hex;
642  return result;
643  }
644 
645  //----------------------------------------------------------------------------
646  // Normalize checksum
647  //----------------------------------------------------------------------------
648  std::string Utils::NormalizeChecksum( const std::string &name,
649  const std::string &checksum )
650  {
651  if( name == "adler32" || name == "crc32" )
652  {
653  size_t i;
654  for( i = 0; i < checksum.length(); ++i )
655  if( checksum[i] != '0' )
656  break;
657  return checksum.substr(i);
658  }
659  return checksum;
660  }
661 
662  //----------------------------------------------------------------------------
663  // Get supported checksum types for given URL
664  //----------------------------------------------------------------------------
665  std::vector<std::string> Utils::GetSupportedCheckSums( const XrdCl::URL &url )
666  {
667  std::vector<std::string> ret;
668 
669  FileSystem fs( url );
670  Buffer arg; arg.FromString( "chksum" );
671  Buffer *resp = 0;
672  XRootDStatus st = fs.Query( QueryCode::Config, arg, resp );
673  if( st.IsOK() )
674  {
675  std::string response = resp->ToString();
676  if( response != "chksum" )
677  {
678  // we are expecting a response of format: '0:zcrc32,1:adler32'
679  std::vector<std::string> result;
680  Utils::splitString( result, response, "," );
681 
682  std::vector<std::string>::iterator itr = result.begin();
683  for( ; itr != result.end(); ++itr )
684  {
685  size_t pos = itr->find( ':' );
686  if( pos == std::string::npos ) continue;
687  std::string cksname = itr->substr( pos + 1 );
688  // remove all white spaces
689  cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ),
690  cksname.end() );
691  ret.push_back( std::move( cksname ) );
692  }
693  }
694  }
695 
696  return ret;
697  }
698 
699 
700  //------------------------------------------------------------------------
702  //------------------------------------------------------------------------
703  bool Utils::CheckEC( const Message *req, const URL &url )
704  {
705 #ifdef WITH_XRDEC
706  // make sure that if we will be writing it is a new file
707  ClientRequest *request = (ClientRequest*)req->GetBuffer();
708  uint16_t options = ntohs( request->open.options );
709  bool open_wrt = ( options & kXR_open_updt ) || ( options & kXR_open_wrto );
710  bool open_new = ( options & kXR_new );
711  if( open_wrt && !open_new ) return false;
712 
713  const URL::ParamsMap &params = url.GetParams();
714  // make sure all the xrdec. tokens are present and the values are sane
715  URL::ParamsMap::const_iterator itr = params.find( "xrdec.nbdta" );
716  if( itr == params.end() ) return false;
717  size_t nbdta = std::stoul( itr->second );
718 
719  itr = params.find( "xrdec.nbprt" );
720  if( itr == params.end() ) return false;
721  size_t nbprt = std::stoul( itr->second );
722 
723  itr = params.find( "xrdec.blksz" );
724  if( itr == params.end() ) return false;
725 
726  itr = params.find( "xrdec.plgr" );
727  if( itr == params.end() ) return false;
728  std::vector<std::string> plgr;
729  splitString( plgr, itr->second, "," );
730  if( plgr.size() < nbdta + nbprt ) return false;
731 
732  itr = params.find( "xrdec.objid" );
733  if( itr == params.end() ) return false;
734 
735  itr = params.find( "xrdec.format" );
736  if( itr == params.end() ) return false;
737  size_t format = std::stoul( itr->second );
738  if( format != 1 ) return false; // TODO use constant
739 
740  itr = params.find( "xrdec.dtacgi" );
741  if( itr != params.end() )
742  {
743  std::vector<std::string> dtacgi;
744  splitString( dtacgi, itr->second, "," );
745  if( plgr.size() != dtacgi.size() ) return false;
746  }
747 
748  itr = params.find( "xrdec.mdtacgi" );
749  if( itr != params.end() )
750  {
751  std::vector<std::string> mdtacgi;
752  splitString( mdtacgi, itr->second, "," );
753  if( plgr.size() != mdtacgi.size() ) return false;
754  }
755 
756  itr = params.find( "xrdec.cosc" );
757  if( itr == params.end() ) return false;
758  std::string cosc = itr->second;
759  if( cosc != "true" && cosc != "false" ) return false;
760 
761  return true;
762 #else
763  return false;
764 #endif
765  }
766 
767 
768  //----------------------------------------------------------------------------
770  //----------------------------------------------------------------------------
771  std::string Utils::InferChecksumType( const XrdCl::URL &source,
772  const XrdCl::URL &destination,
773  bool zip)
774  {
775  //--------------------------------------------------------------------------
776  // If both files are local we won't be checksumming at all
777  //--------------------------------------------------------------------------
778  if( source.IsLocalFile() && !source.IsMetalink() && destination.IsLocalFile() ) return std::string();
779 
780  // checksums supported by local files
781  std::set<std::string> local_supported;
782  local_supported.insert( "adler32" );
783  local_supported.insert( "crc32" );
784  local_supported.insert( "md5" );
785  local_supported.insert( "zcrc32" );
786 
787  std::vector<std::string> srccks;
788 
789  if( source.IsMetalink() )
790  {
791  int useMtlnCksum = DefaultZipMtlnCksum;
792  Env *env = DefaultEnv::GetEnv();
793  env->GetInt( "ZipMtlnCksum", useMtlnCksum );
794 
795  //------------------------------------------------------------------------
796  // In case of ZIP use other checksums than zcrc32 only if the user
797  // requested it explicitly.
798  //------------------------------------------------------------------------
799  if( !zip || ( zip && useMtlnCksum ) )
800  {
802  VirtualRedirector *redirector = registry.Get( source );
803  std::vector<std::string> cks = redirector->GetSupportedCheckSums();
804  srccks.insert( srccks.end(), cks.begin(), cks.end() );
805  }
806  }
807 
808  if( zip )
809  {
810  //------------------------------------------------------------------------
811  // In case of ZIP we can always extract the checksum from the archive
812  //------------------------------------------------------------------------
813  srccks.push_back( "zcrc32" );
814  }
815  else if( source.GetProtocol() == "root" || source.GetProtocol() == "xroot" )
816  {
817  //------------------------------------------------------------------------
818  // If the source is a remote endpoint query the supported checksums
819  //------------------------------------------------------------------------
820  std::vector<std::string> cks = GetSupportedCheckSums( source );
821  srccks.insert( srccks.end(), cks.begin(), cks.end() );
822  }
823 
824  std::vector<std::string> dstcks;
825 
826  if( destination.GetProtocol() == "root" ||
827  destination.GetProtocol() == "xroot" )
828  {
829  //------------------------------------------------------------------------
830  // If the destination is a remote endpoint query the supported checksums
831  //------------------------------------------------------------------------
832  std::vector<std::string> cks = GetSupportedCheckSums( destination );
833  dstcks.insert( dstcks.end(), cks.begin(), cks.end() );
834  }
835 
836  //--------------------------------------------------------------------------
837  // Now we have all the information we need, we can infer the right checksum
838  // type!!!
839  //
840  // First check if source is local
841  //--------------------------------------------------------------------------
842  if( source.IsLocalFile() && !source.IsMetalink() )
843  {
844  std::vector<std::string>::iterator itr = dstcks.begin();
845  for( ; itr != dstcks.end(); ++itr )
846  if( local_supported.count( *itr ) ) return *itr;
847  return std::string();
848  }
849 
850  //--------------------------------------------------------------------------
851  // then check if destination is local
852  //--------------------------------------------------------------------------
853  if( destination.IsLocalFile() )
854  {
855  std::vector<std::string>::iterator itr = srccks.begin();
856  for( ; itr != srccks.end(); ++itr )
857  if( local_supported.count( *itr ) ) return *itr;
858  return std::string();
859  }
860 
861  //--------------------------------------------------------------------------
862  // if both source and destination are remote look for a checksum that can
863  // satisfy both
864  //--------------------------------------------------------------------------
865  std::set<std::string> dst_supported( dstcks.begin(), dstcks.end() );
866  std::vector<std::string>::iterator itr = srccks.begin();
867  for( ; itr != srccks.end(); ++itr )
868  if( dst_supported.count( *itr ) ) return *itr;
869  return std::string();
870  }
871 
872  //----------------------------------------------------------------------------
874  //----------------------------------------------------------------------------
875  void Utils::SplitChunks( std::vector<ChunkList> &listsvec,
876  const ChunkList &chunks,
877  const uint32_t maxcs,
878  const size_t maxc )
879  {
880  listsvec.clear();
881  if( !chunks.size() ) return;
882 
883  listsvec.emplace_back();
884  ChunkList *c = &listsvec.back();
885  const size_t cs = chunks.size();
886  size_t idx = 0;
887  size_t nc = 0;
888  ChunkInfo tmpc;
889 
890  c->reserve( cs );
891 
892  while( idx < cs )
893  {
894  if( maxc && nc >= maxc )
895  {
896  listsvec.emplace_back();
897  c = &listsvec.back();
898  c->reserve( cs - idx );
899  nc = 0;
900  }
901 
902  if( tmpc.length == 0 )
903  tmpc = chunks[idx];
904 
905  if( maxcs && tmpc.length > maxcs )
906  {
907  c->emplace_back( tmpc.offset, maxcs, tmpc.buffer );
908  tmpc.offset += maxcs;
909  tmpc.length -= maxcs;
910  tmpc.buffer = static_cast<char*>( tmpc.buffer ) + maxcs;
911  }
912  else
913  {
914  c->emplace_back( tmpc.offset, tmpc.length, tmpc.buffer );
915  tmpc.length = 0;
916  ++idx;
917  }
918  ++nc;
919  }
920  }
921 }
kXR_unt16 options
Definition: XProtocol.hh:481
@ kXR_open_wrto
Definition: XProtocol.hh:469
@ kXR_open_updt
Definition: XProtocol.hh:457
@ kXR_new
Definition: XProtocol.hh:455
struct ClientOpenRequest open
Definition: XProtocol.hh:860
#define unlikely(x)
std::string obfuscateAuth(const std::string &input)
struct dirent * readdir(DIR *dirp)
int closedir(DIR *dirp)
DIR * opendir(const char *path)
void getline(uchar *buff, int blen)
struct myOpts opts
int Set(const char *csName)
Definition: XrdCksData.hh:81
int Get(char *Buff, int Blen)
Definition: XrdCksData.hh:69
Binary blob representation.
Definition: XrdClBuffer.hh:34
void FromString(const std::string str)
Fill the buffer from a string.
Definition: XrdClBuffer.hh:205
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
Definition: XrdClBuffer.hh:72
std::string ToString() const
Convert the buffer to a string.
Definition: XrdClBuffer.hh:215
Manage the checksum calc objects.
bool Calculate(XrdCksData &result, const std::string &algName, const std::string &filePath)
Calculate a checksum of for a given file.
static CheckSumManager * GetCheckSumManager()
Get checksum manager.
static Log * GetLog()
Get default log.
static Env * GetEnv()
Get default client environment.
bool GetString(const std::string &key, std::string &value)
Definition: XrdClEnv.cc:31
bool GetInt(const std::string &key, int &value)
Definition: XrdClEnv.cc:89
Send file/filesystem queries to an XRootD cluster.
XRootDStatus Query(QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
Handle diagnostics.
Definition: XrdClLog.hh:101
@ DumpMsg
print details of the request and responses
Definition: XrdClLog.hh:113
void Error(uint64_t topic, const char *format,...)
Report an error.
Definition: XrdClLog.cc:231
LogLevel GetLevel() const
Get the log level.
Definition: XrdClLog.hh:258
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
Definition: XrdClLog.cc:299
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
Definition: XrdClLog.cc:282
The message representation used throughout the system.
Definition: XrdClMessage.hh:32
A key-value pair map storing both keys and values as strings.
PropertyMap::const_iterator end() const
Get the end iterator.
PropertyMap::const_iterator begin() const
Get the begin iterator.
Singleton access to URL to virtual redirector mapping.
static RedirectorRegistry & Instance()
Returns reference to the single instance.
VirtualRedirector * Get(const URL &url) const
Get a virtual redirector associated with the given URL.
URL representation.
Definition: XrdClURL.hh:31
bool IsMetalink() const
Is it a URL to a metalink.
Definition: XrdClURL.cc:458
const std::string & GetHostName() const
Get the name of the target host.
Definition: XrdClURL.hh:170
std::map< std::string, std::string > ParamsMap
Definition: XrdClURL.hh:33
const std::string & GetProtocol() const
Get the protocol.
Definition: XrdClURL.hh:118
bool IsLocalFile() const
Definition: XrdClURL.cc:467
const ParamsMap & GetParams() const
Get the URL params.
Definition: XrdClURL.hh:244
const std::string & GetPath() const
Get the path.
Definition: XrdClURL.hh:217
int GetPort() const
Get the target port.
Definition: XrdClURL.hh:188
static std::string TimeToString(time_t timestamp)
Convert timestamp to a string.
Definition: XrdClUtils.cc:256
static XRootDStatus CheckTPCLite(const std::string &server, uint16_t timeout=0)
Definition: XrdClUtils.cc:426
static void LogHostAddresses(Log *log, uint64_t type, const std::string &hostId, std::vector< XrdNetAddr > &addresses)
Log all the addresses on the list.
Definition: XrdClUtils.cc:234
static std::string NormalizeChecksum(const std::string &name, const std::string &checksum)
Normalize checksum.
Definition: XrdClUtils.cc:648
static Status ProcessConfig(std::map< std::string, std::string > &config, const std::string &file)
Process a config file and return key-value pairs.
Definition: XrdClUtils.cc:534
static Status ProcessConfigDir(std::map< std::string, std::string > &config, const std::string &dir)
Process a config directory and return key-value pairs.
Definition: XrdClUtils.cc:566
static std::string FQDNToCC(const std::string &fqdn)
Convert the fully qualified host name to country code.
Definition: XrdClUtils.cc:490
static std::string InferChecksumType(const XrdCl::URL &source, const XrdCl::URL &destination, bool zip=false)
Automatically infer the right checksum type.
Definition: XrdClUtils.cc:771
static void LogPropertyList(Log *log, uint64_t topic, const char *format, const PropertyList &list)
Log property list.
Definition: XrdClUtils.cc:617
static std::string Char2Hex(uint8_t *array, uint16_t size)
Print a char array as hex.
Definition: XrdClUtils.cc:635
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
Definition: XrdClUtils.hh:56
static Status GetHostAddresses(std::vector< XrdNetAddr > &addresses, const URL &url, AddressType type)
Resolve IP addresses.
Definition: XrdClUtils.cc:140
static uint64_t GetElapsedMicroSecs(timeval start, timeval end)
Get the elapsed microseconds between two timevals.
Definition: XrdClUtils.cc:269
static std::vector< std::string > GetSupportedCheckSums(const XrdCl::URL &url)
Get supported checksum types for given URL.
Definition: XrdClUtils.cc:665
static AddressType String2AddressType(const std::string &addressType)
Interpret a string as address type, default to IPAll.
Definition: XrdClUtils.cc:123
static int GetIntParameter(const URL &url, const std::string &name, int defaultVal)
Get a parameter either from the environment or URL.
Definition: XrdClUtils.cc:81
static Status GetDirectoryEntries(std::vector< std::string > &entries, const std::string &path)
Get directory entries.
Definition: XrdClUtils.cc:506
static XRootDStatus GetLocalCheckSum(std::string &checkSum, const std::string &checkSumType, const std::string &path)
Get a checksum from local file.
Definition: XrdClUtils.cc:330
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
Definition: XrdClUtils.cc:367
static void Trim(std::string &str)
Trim a string.
Definition: XrdClUtils.cc:606
static bool CheckEC(const Message *req, const URL &url)
Check if this client can support given EC redirect.
Definition: XrdClUtils.cc:703
static XRootDStatus GetRemoteCheckSum(std::string &checkSum, const std::string &checkSumType, const URL &url)
Get a checksum from a remote xrootd server.
Definition: XrdClUtils.cc:279
static std::string GetStringParameter(const URL &url, const std::string &name, const std::string &defaultVal)
Get a parameter either from the environment or URL.
Definition: XrdClUtils.cc:104
static XRootDStatus CheckTPC(const std::string &server, uint16_t timeout=0)
Check if peer supports tpc.
Definition: XrdClUtils.cc:382
AddressType
Address type.
Definition: XrdClUtils.hh:87
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
Definition: XrdClUtils.cc:875
An interface for metadata redirectors.
virtual std::vector< std::string > GetSupportedCheckSums() const =0
void SetErrorMessage(const std::string &message)
Set the error message.
const std::string & GetErrorMessage() const
Get error message.
std::string ToStr() const
Convert to string.
bool isMapped() const
bool isIPType(IPType ipType) const
static const char * GetAddrs(const char *hSpec, XrdNetAddr *aListP[], int &aListN, AddrOpts opts=allIPMap, int pNum=PortInSpec)
Definition: XrdNetUtils.cc:239
const uint16_t suPartial
Definition: XrdClStatus.hh:41
const uint16_t errInvalidAddr
Definition: XrdClStatus.hh:71
const uint16_t stFatal
Fatal error, it's still an error.
Definition: XrdClStatus.hh:33
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint16_t errInternal
Internal error.
Definition: XrdClStatus.hh:56
const uint16_t stOK
Everything went OK.
Definition: XrdClStatus.hh:31
const int DefaultIPNoShuffle
const uint16_t errConfig
System misconfigured.
Definition: XrdClStatus.hh:55
const uint16_t errOSError
Definition: XrdClStatus.hh:61
const uint64_t UtilityMsg
const uint16_t errInvalidResponse
Definition: XrdClStatus.hh:99
std::vector< ChunkInfo > ChunkList
List of chunks.
const uint16_t errNotSupported
Definition: XrdClStatus.hh:62
const int DefaultPreferIPv4
const uint16_t errCheckSumError
Definition: XrdClStatus.hh:101
const int DefaultZipMtlnCksum
@ hex
Definition: XrdSysTrace.hh:42
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ Config
Query server configuration.
@ Checksum
Query file checksum.
Procedure execution status.
Definition: XrdClStatus.hh:115
uint16_t status
Status of the execution.
Definition: XrdClStatus.hh:146
bool IsOK() const
We're fine.
Definition: XrdClStatus.hh:124
std::string ToString() const
Create a string representation.
Definition: XrdClStatus.cc:97