43 #include <sys/types.h>
46 #if __cplusplus < 201103L
52 bool isNotSpace(
char c )
82 const std::string &name,
86 int value = defaultVal;
88 URL::ParamsMap::const_iterator it;
90 env->
GetInt( name, value );
91 it = url.
GetParams().find( std::string(
"XrdCl.") + name );
94 int urlValue = (int)strtol( it->second.c_str(), &endPtr, 0 );
105 const std::string &name,
106 const std::string &defaultVal )
109 std::string value = defaultVal;
110 URL::ParamsMap::const_iterator it;
113 it = url.
GetParams().find( std::string(
"XrdCl.") + name );
125 if( addressType ==
"IPv6" )
127 else if( addressType ==
"IPv4" )
129 else if( addressType ==
"IPv4Mapped6" )
131 else if( addressType ==
"IPAll" )
187 if( addresses.size() == 0 )
200 env->
GetInt(
"IPNoShuffle", ipNoShuffle );
204 #if __cplusplus < 201103L
206 static struct only_once_t
210 std::srand (
unsigned ( std::time(0) ) );
214 std::random_shuffle( addresses.begin(), addresses.begin() + ordn );
215 std::random_shuffle( addresses.begin() + ordn, addresses.end() );
217 static std::default_random_engine rand_engine(
218 std::chrono::system_clock::now().time_since_epoch().count() );
220 std::shuffle( addresses.begin(), addresses.begin() + ordn, rand_engine );
221 std::shuffle( addresses.begin() + ordn, addresses.end(), rand_engine );
236 const std::string &hostId,
237 std::vector<XrdNetAddr> &addresses )
240 std::vector<XrdNetAddr>::iterator it;
241 for( it = addresses.begin(); it != addresses.end(); ++it )
248 addrStr.erase( addrStr.length()-2, 2 );
249 log->
Debug( type,
"[%s] Found %zu address(es): %s",
250 hostId.c_str(), addresses.size(), addrStr.c_str() );
260 time_t ttNow = timestamp;
261 localtime_r( &ttNow, &tsNow );
262 strftime( now, 30,
"%Y-%m-%d %H:%M:%S %z", &tsNow );
271 uint64_t startUSec = start.tv_sec*1000000 + start.tv_usec;
272 uint64_t endUSec = end.tv_sec*1000000 + end.tv_usec;
273 return endUSec-startUSec;
280 const std::string &checkSumType,
287 size_t pos = url.
GetPath().find(
'?' );
288 std::string cksPath = url.
GetPath() + ( pos == std::string::npos ?
'?' :
'&' ) +
"cks.type=" + checkSumType;
300 msg +=
" Got an error while querying the checksum!";
308 std::vector<std::string> elems;
312 if( elems.size() != 2 )
315 if( elems[0] != checkSumType )
318 checkSum = elems[0] +
":";
322 url.
GetPath().c_str(), checkSum.c_str() );
331 const std::string &checkSumType,
332 const std::string &path )
344 bool status = cksMan->
Calculate( ckSum, checkSumType, path.c_str() );
352 char *cksBuffer =
new char[265];
353 ckSum.
Get( cksBuffer, 256 );
354 checkSum = checkSumType +
":";
369 double final = bytes;
371 char suf[3] = {
'k',
'M',
'G' };
372 for( i = 0; i < 3 && final > 1024; ++i,
final /= 1024 ) {};
373 std::ostringstream o;
374 o << std::setprecision(4) <<
final;
375 if( i > 0 ) o << suf[i-1];
390 Buffer *queryResponse = 0;
397 server.c_str(), st.
ToStr().c_str() );
404 log->
Error(
UtilityMsg,
"Cannot query source data server: empty response." );
409 std::string answer = queryResponse->
ToString();
410 delete queryResponse;
411 if( answer.length() == 1 || !isdigit( answer[0] ) || atoi(answer.c_str()) == 0)
429 log->
Debug(
UtilityMsg,
"Checking if the data server %s supports tpc / tpc lite",
434 Buffer *queryResponse = 0;
441 server.c_str(), st.
ToStr().c_str() );
448 log->
Error(
UtilityMsg,
"Cannot query source data server: empty response." );
453 std::string answer = queryResponse->
ToString();
454 delete queryResponse;
458 log->
Error(
UtilityMsg,
"Cannot query source data server: empty response." );
463 std::vector<std::string> resp;
466 if( resp.empty() || resp[0].empty() ||
467 !isdigit( resp[0][0]) || atoi( resp[0].c_str() ) == 0 )
474 if( resp.size() == 1 || resp[1] ==
"tpcdlg" )
492 std::vector<std::string> el;
497 std::string cc = *el.rbegin();
498 if( cc.length() == 2 )
507 const std::string &path )
509 DIR *dp =
opendir( path.c_str() );
515 while( (dirEntry =
readdir(dp)) != 0 )
517 std::string entryName = dirEntry->d_name;
518 if( !entryName.compare( 0, 2,
"..") )
520 if( !entryName.compare( 0, 1,
".") )
523 entries.push_back( dirEntry->d_name );
535 const std::string &file )
538 std::ifstream inFile( file.c_str() );
546 if( line.empty() || line[0] ==
'#' )
549 std::vector<std::string> elems;
551 if( elems.size() != 2 )
553 std::string key = elems[0];
Trim( key );
554 std::string value = elems[1];
Trim( value );
567 const std::string &dir )
573 std::vector<std::string> entries;
578 dir.c_str(), st.
ToString().c_str() );
582 static const std::string suffix =
".conf";
583 for(
auto &entry : entries )
585 std::string confFile = dir +
"/" + entry;
587 if( confFile.length() <= suffix.length() )
589 if( !std::equal( suffix.rbegin(), suffix.rend(), confFile.rbegin() ) )
596 confFile.c_str(), st.
ToString().c_str() );
608 str.erase( str.begin(),
609 std::find_if( str.begin(), str.end(), isNotSpace ) );
610 str.erase( std::find_if( str.rbegin(), str.rend(), isNotSpace ).base(),
623 PropertyList::PropertyMap::const_iterator it;
625 for (it = list.
begin(); it != list.
end(); ++it)
626 keyVals +=
"'" + it->first +
"' = '" +
obfuscateAuth(it->second) +
"', ";
627 keyVals.erase(keyVals.length() - 2, 2);
628 log->
Dump(topic, format, keyVals.c_str());
637 char *
hex =
new char[2*size+1];
638 for( uint16_t i = 0; i < size; ++i )
639 snprintf(
hex+(2*i), 3,
"%02x", (
int)array[i] );
640 std::string result =
hex;
649 const std::string &checksum )
651 if( name ==
"adler32" || name ==
"crc32" )
654 for( i = 0; i < checksum.length(); ++i )
655 if( checksum[i] !=
'0' )
657 return checksum.substr(i);
667 std::vector<std::string> ret;
675 std::string response = resp->
ToString();
676 if( response !=
"chksum" )
679 std::vector<std::string> result;
682 std::vector<std::string>::iterator itr = result.begin();
683 for( ; itr != result.end(); ++itr )
685 size_t pos = itr->find(
':' );
686 if( pos == std::string::npos )
continue;
687 std::string cksname = itr->substr( pos + 1 );
689 cksname.erase( std::remove_if( cksname.begin(), cksname.end(), ::isspace ),
691 ret.push_back( std::move( cksname ) );
710 bool open_new = ( options &
kXR_new );
711 if( open_wrt && !open_new )
return false;
715 URL::ParamsMap::const_iterator itr = params.find(
"xrdec.nbdta" );
716 if( itr == params.end() )
return false;
717 size_t nbdta = std::stoul( itr->second );
719 itr = params.find(
"xrdec.nbprt" );
720 if( itr == params.end() )
return false;
721 size_t nbprt = std::stoul( itr->second );
723 itr = params.find(
"xrdec.blksz" );
724 if( itr == params.end() )
return false;
726 itr = params.find(
"xrdec.plgr" );
727 if( itr == params.end() )
return false;
728 std::vector<std::string> plgr;
730 if( plgr.size() < nbdta + nbprt )
return false;
732 itr = params.find(
"xrdec.objid" );
733 if( itr == params.end() )
return false;
735 itr = params.find(
"xrdec.format" );
736 if( itr == params.end() )
return false;
737 size_t format = std::stoul( itr->second );
738 if( format != 1 )
return false;
740 itr = params.find(
"xrdec.dtacgi" );
741 if( itr != params.end() )
743 std::vector<std::string> dtacgi;
745 if( plgr.size() != dtacgi.size() )
return false;
748 itr = params.find(
"xrdec.mdtacgi" );
749 if( itr != params.end() )
751 std::vector<std::string> mdtacgi;
753 if( plgr.size() != mdtacgi.size() )
return false;
756 itr = params.find(
"xrdec.cosc" );
757 if( itr == params.end() )
return false;
758 std::string cosc = itr->second;
759 if( cosc !=
"true" && cosc !=
"false" )
return false;
781 std::set<std::string> local_supported;
782 local_supported.insert(
"adler32" );
783 local_supported.insert(
"crc32" );
784 local_supported.insert(
"md5" );
785 local_supported.insert(
"zcrc32" );
787 std::vector<std::string> srccks;
793 env->
GetInt(
"ZipMtlnCksum", useMtlnCksum );
799 if( !zip || ( zip && useMtlnCksum ) )
804 srccks.insert( srccks.end(), cks.begin(), cks.end() );
813 srccks.push_back(
"zcrc32" );
821 srccks.insert( srccks.end(), cks.begin(), cks.end() );
824 std::vector<std::string> dstcks;
833 dstcks.insert( dstcks.end(), cks.begin(), cks.end() );
844 std::vector<std::string>::iterator itr = dstcks.begin();
845 for( ; itr != dstcks.end(); ++itr )
846 if( local_supported.count( *itr ) )
return *itr;
847 return std::string();
855 std::vector<std::string>::iterator itr = srccks.begin();
856 for( ; itr != srccks.end(); ++itr )
857 if( local_supported.count( *itr ) )
return *itr;
858 return std::string();
865 std::set<std::string> dst_supported( dstcks.begin(), dstcks.end() );
866 std::vector<std::string>::iterator itr = srccks.begin();
867 for( ; itr != srccks.end(); ++itr )
868 if( dst_supported.count( *itr ) )
return *itr;
869 return std::string();
877 const uint32_t maxcs,
881 if( !chunks.size() )
return;
883 listsvec.emplace_back();
885 const size_t cs = chunks.size();
894 if( maxc && nc >= maxc )
896 listsvec.emplace_back();
897 c = &listsvec.back();
898 c->reserve( cs - idx );
905 if( maxcs && tmpc.
length > maxcs )
910 tmpc.
buffer =
static_cast<char*
>( tmpc.
buffer ) + maxcs;
struct ClientOpenRequest open
std::string obfuscateAuth(const std::string &input)
struct dirent * readdir(DIR *dirp)
DIR * opendir(const char *path)
void getline(uchar *buff, int blen)
int Set(const char *csName)
int Get(char *Buff, int Blen)
Binary blob representation.
void FromString(const std::string str)
Fill the buffer from a string.
const char * GetBuffer(uint32_t offset=0) const
Get the message buffer.
std::string ToString() const
Convert the buffer to a string.
Manage the checksum calc objects.
bool Calculate(XrdCksData &result, const std::string &algName, const std::string &filePath)
Calculate a checksum of for a given file.
static CheckSumManager * GetCheckSumManager()
Get checksum manager.
static Log * GetLog()
Get default log.
static Env * GetEnv()
Get default client environment.
bool GetString(const std::string &key, std::string &value)
bool GetInt(const std::string &key, int &value)
Send file/filesystem queries to an XRootD cluster.
XRootDStatus Query(QueryCode::Code queryCode, const Buffer &arg, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
@ DumpMsg
print details of the request and responses
void Error(uint64_t topic, const char *format,...)
Report an error.
LogLevel GetLevel() const
Get the log level.
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
The message representation used throughout the system.
A key-value pair map storing both keys and values as strings.
PropertyMap::const_iterator end() const
Get the end iterator.
PropertyMap::const_iterator begin() const
Get the begin iterator.
Singleton access to URL to virtual redirector mapping.
static RedirectorRegistry & Instance()
Returns reference to the single instance.
VirtualRedirector * Get(const URL &url) const
Get a virtual redirector associated with the given URL.
bool IsMetalink() const
Is it a URL to a metalink.
const std::string & GetHostName() const
Get the name of the target host.
std::map< std::string, std::string > ParamsMap
const std::string & GetProtocol() const
Get the protocol.
const ParamsMap & GetParams() const
Get the URL params.
const std::string & GetPath() const
Get the path.
int GetPort() const
Get the target port.
static std::string TimeToString(time_t timestamp)
Convert timestamp to a string.
static XRootDStatus CheckTPCLite(const std::string &server, uint16_t timeout=0)
static void LogHostAddresses(Log *log, uint64_t type, const std::string &hostId, std::vector< XrdNetAddr > &addresses)
Log all the addresses on the list.
static std::string NormalizeChecksum(const std::string &name, const std::string &checksum)
Normalize checksum.
static Status ProcessConfig(std::map< std::string, std::string > &config, const std::string &file)
Process a config file and return key-value pairs.
static Status ProcessConfigDir(std::map< std::string, std::string > &config, const std::string &dir)
Process a config directory and return key-value pairs.
static std::string FQDNToCC(const std::string &fqdn)
Convert the fully qualified host name to country code.
static std::string InferChecksumType(const XrdCl::URL &source, const XrdCl::URL &destination, bool zip=false)
Automatically infer the right checksum type.
static void LogPropertyList(Log *log, uint64_t topic, const char *format, const PropertyList &list)
Log property list.
static std::string Char2Hex(uint8_t *array, uint16_t size)
Print a char array as hex.
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
static Status GetHostAddresses(std::vector< XrdNetAddr > &addresses, const URL &url, AddressType type)
Resolve IP addresses.
static uint64_t GetElapsedMicroSecs(timeval start, timeval end)
Get the elapsed microseconds between two timevals.
static std::vector< std::string > GetSupportedCheckSums(const XrdCl::URL &url)
Get supported checksum types for given URL.
static AddressType String2AddressType(const std::string &addressType)
Interpret a string as address type, default to IPAll.
static int GetIntParameter(const URL &url, const std::string &name, int defaultVal)
Get a parameter either from the environment or URL.
static Status GetDirectoryEntries(std::vector< std::string > &entries, const std::string &path)
Get directory entries.
static XRootDStatus GetLocalCheckSum(std::string &checkSum, const std::string &checkSumType, const std::string &path)
Get a checksum from local file.
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
static void Trim(std::string &str)
Trim a string.
static bool CheckEC(const Message *req, const URL &url)
Check if this client can support given EC redirect.
static XRootDStatus GetRemoteCheckSum(std::string &checkSum, const std::string &checkSumType, const URL &url)
Get a checksum from a remote xrootd server.
static std::string GetStringParameter(const URL &url, const std::string &name, const std::string &defaultVal)
Get a parameter either from the environment or URL.
static XRootDStatus CheckTPC(const std::string &server, uint16_t timeout=0)
Check if peer supports tpc.
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
An interface for metadata redirectors.
virtual std::vector< std::string > GetSupportedCheckSums() const =0
void SetErrorMessage(const std::string &message)
Set the error message.
const std::string & GetErrorMessage() const
Get error message.
std::string ToStr() const
Convert to string.
bool isIPType(IPType ipType) const
static const char * GetAddrs(const char *hSpec, XrdNetAddr *aListP[], int &aListN, AddrOpts opts=allIPMap, int pNum=PortInSpec)
const uint16_t errInvalidAddr
const uint16_t stFatal
Fatal error, it's still an error.
const uint16_t stError
An error occurred that could potentially be retried.
const uint16_t errInternal
Internal error.
const uint16_t stOK
Everything went OK.
const int DefaultIPNoShuffle
const uint16_t errConfig
System misconfigured.
const uint16_t errOSError
const uint64_t UtilityMsg
const uint16_t errInvalidResponse
std::vector< ChunkInfo > ChunkList
List of chunks.
const uint16_t errNotSupported
const int DefaultPreferIPv4
const uint16_t errCheckSumError
const int DefaultZipMtlnCksum
Describe a data chunk for vector read.
void * buffer
length of the chunk
uint32_t length
offset in the file
@ Config
Query server configuration.
@ Checksum
Query file checksum.
Procedure execution status.
uint16_t status
Status of the execution.
bool IsOK() const
We're fine.
std::string ToString() const
Create a string representation.