54 pPrintSourceCheckSum(false), pPrintTargetCheckSum(false),
55 pPrintAdditionalCheckSum(false)
67 if( pPrintProgressBar )
71 std::cerr <<
"Job: " << jobNum <<
"/" << jobTotal << std::endl;
72 std::cerr <<
"Source: " << source->
GetURL() << std::endl;
73 std::cerr <<
"Target: " << destination->
GetURL() << std::endl;
81 d.target = destination;
82 pOngoingJobs[jobNum] = d;
92 std::map<uint16_t, JobData>::iterator it = pOngoingJobs.find( jobNum );
93 if( it == pOngoingJobs.end() )
96 JobData &d = it->second;
101 JobProgress( jobNum, d.bytesProcessed, d.bytesTotal );
103 if( pPrintProgressBar )
105 if( pOngoingJobs.size() > 1 )
106 std::cerr <<
"\r" << std::string(70,
' ') <<
"\r";
108 std::cerr << std::endl;
112 results->
Get(
"status", st );
115 pOngoingJobs.erase(it);
119 std::string checkSum;
121 results->
Get(
"size", size );
122 if( pPrintSourceCheckSum )
124 results->
Get(
"sourceCheckSum", checkSum );
128 if( pPrintTargetCheckSum )
130 results->
Get(
"targetCheckSum", checkSum );
134 if( pPrintAdditionalCheckSum )
136 std::vector<std::string> addcksums;
137 results->
Get(
"additionalCkeckSum", addcksums );
138 for(
auto &cks : addcksums )
142 pOngoingJobs.erase(it);
150 JobData &d = pOngoingJobs.begin()->second;
154 speed = d.bytesProcessed/(now-d.started);
156 speed = d.bytesProcessed;
164 prog = (int)((
double)d.bytesProcessed/d.bytesTotal*50);
165 proc = (int)((
double)d.bytesProcessed/d.bytesTotal*100);
172 bar.append( prog,
'=' );
176 std::ostringstream o;
179 o <<
"[" << std::setw(3) << std::right << proc <<
"%]";
180 o <<
"[" << std::setw(50) << std::left;
192 std::map<uint16_t, JobData>::iterator it;
193 std::ostringstream o;
195 for( it = pOngoingJobs.begin(); it != pOngoingJobs.end(); ++it )
197 JobData &d = it->second;
198 uint16_t jobNum = it->first;
202 speed = d.bytesProcessed/(now-d.started);
206 proc = (int)((
double)d.bytesProcessed/d.bytesTotal*100);
210 o <<
"[#" << jobNum <<
": ";
222 uint64_t bytesProcessed,
223 uint64_t bytesTotal )
227 if( pPrintProgressBar )
229 time_t now = time(0);
230 if( (now - pPrevious < 1) && (bytesProcessed != bytesTotal) )
234 std::map<uint16_t, JobData>::iterator it = pOngoingJobs.find( jobNum );
235 if( it == pOngoingJobs.end() )
238 JobData &d = it->second;
240 d.bytesProcessed = bytesProcessed;
241 d.bytesTotal = bytesTotal;
243 std::string progress;
244 if( pOngoingJobs.size() == 1 )
249 std::cerr <<
"\r" << progress << std::flush;
257 const std::string &checkSum,
260 if( checkSum.empty() )
262 std::string::size_type i = checkSum.find(
':' );
263 std::cerr << checkSum.substr( 0, i+1 ) <<
" ";
264 std::cerr << checkSum.substr( i+1, checkSum.length()-i ) <<
" ";
267 std::cerr << url->
GetPath() <<
" ";
271 std::cerr << url->
GetPath() <<
" ";
275 std::cerr << std::endl;
289 JobData(): bytesProcessed(0), bytesTotal(0),
290 started(0), source(0), target(0) {}
291 uint64_t bytesProcessed;
299 bool pPrintProgressBar;
300 bool pPrintSourceCheckSum;
301 bool pPrintTargetCheckSum;
302 bool pPrintAdditionalCheckSum;
303 std::map<uint16_t, JobData> pOngoingJobs;
314 std::cerr <<
"SOCKS Proxies are not yet supported" << std::endl;
326 if( !newCGI || !(*newCGI) )
332 if( url.find(
'?' ) == std::string::npos )
335 if( url.find(
'&' ) == std::string::npos )
352 cursor = cursor->
Next;
359 cursor = cursor->
Next;
376 default:
return "other";
386 for( count = 0; file; file = file->
Next, ++count ) {};
399 if( file->
Doff == 0 )
401 char *slash = file->
Path;
402 for( ; *slash; ++slash ) {};
403 for( ; *slash !=
'/' && slash > file->
Path; --slash ) {};
412 std::string basePath,
415 using namespace XrdCl;
418 log->
Debug(
AppMsg,
"Indexing %s", basePath.c_str() );
425 log->
Info(
AppMsg,
"Failed to get directory listing for %s: %s",
434 for(
auto itr = dirList->
Begin(); itr != dirList->
End(); ++itr )
439 std::string path = basePath +
'/' + e->
GetName();
440 current =
new XrdCpFile( path.c_str(), badUrl );
448 current->
Doff = dirOffset;
463 std::vector<XrdCl::PropertyList *>::iterator it;
464 for( it = results.begin(); it != results.end(); ++it )
471 int main(
int argc,
char **argv )
473 using namespace XrdCl;
503 bool makedir =
false;
505 bool delegate =
false;
506 bool preserveXAttr =
false;
507 bool rmOnBadCksum =
false;
508 bool continue_ =
false;
509 bool recurse =
false;
510 bool zipappend =
false;
511 bool doserver =
false;
512 std::string thirdParty =
"none";
544 if( force && continue_ )
546 std::cerr <<
"Invalid argument combination: continue + force." << std::endl;
553 std::string checkSumType;
554 std::string checkSumPreset;
555 std::string checkSumMode =
"none";
558 checkSumMode =
"end2end";
559 std::vector<std::string> ckSumParams;
561 if( ckSumParams.size() > 1 )
563 if( ckSumParams[1] ==
"print" )
565 checkSumMode =
"target";
569 checkSumPreset = ckSumParams[1];
571 checkSumType = ckSumParams[0];
576 checkSumMode =
"source";
577 std::vector<std::string> ckSumParams;
579 if( ckSumParams.size() == 2 )
581 checkSumMode =
"source";
582 checkSumType = ckSumParams[0];
587 std::cerr <<
"Invalid parameter: " << config.
CksVal << std::endl;
613 nbSources = config.
nSrcs;
627 if( config.
nStrm != 0 )
628 env->
PutInt(
"SubStreamsPerChannel", config.
nStrm + 1 );
630 if( config.
Retry != -1 )
637 env->
PutInt(
"NoTlsOK", 1 );
640 env->
PutInt(
"TlsNoData", 1 );
643 env->
PutInt(
"TlsMetalink", 1 );
646 env->
PutInt(
"ZipMtlnCksum", 1 );
649 env->
GetInt(
"CPChunkSize", chunkSize );
652 env->
GetInt(
"XCpBlockSize", blockSize );
655 env->
GetInt(
"CPParallelChunks", parallelChunks );
656 if( parallelChunks < 1 ||
657 parallelChunks > std::numeric_limits<uint8_t>::max() )
659 std::cerr <<
"Can only handle between 1 and ";
660 std::cerr << (int)std::numeric_limits<uint8_t>::max();
661 std::cerr <<
" chunks in parallel. You asked for " << parallelChunks;
662 std::cerr <<
"." << std::endl;
669 env->
GetInt(
"PreserveXAttrs", val );
670 if( val ) preserveXAttr =
true;
673 log->
Dump(
AppMsg,
"Chunk size: %d, parallel chunks %d, streams: %d",
674 chunkSize, parallelChunks, config.
nStrm + 1 );
679 std::vector<XrdCl::PropertyList*> resultVect;
690 char buf[FILENAME_MAX];
691 char *cwd = getcwd( buf, FILENAME_MAX );
711 bool targetIsDir =
false;
712 bool targetExists =
false;
737 std::cerr << st.
ToStr() << std::endl;
744 if( !targetIsDir && targetExists && !force && !recurse && !zipappend )
749 std::cerr <<
"Run: " << st.
ToStr() << std::endl;
760 std::cerr <<
"Multiple sources were given but target is not a directory.";
761 std::cerr << std::endl;
769 bool remoteSrcIsDir =
false;
780 remoteSrcIsDir =
true;
785 std::string url = source.
GetURL();
789 std::cerr <<
"Error indexing remote directory.";
811 std::string source = sourceFile->
Path;
815 if( source[0] ==
'/' )
816 source =
"file://" + source;
819 char buf[FILENAME_MAX];
820 char *cwd = getcwd( buf, FILENAME_MAX );
827 source =
"file://" + std::string( cwd ) +
'/' + source;
833 std::string sourcePathObf = sourceFile->
Path;
834 std::string destPathObf = dest;
839 log->
Dump(
AppMsg,
"Processing source entry: %s, type %s, target file: %s, logLevel = %d",
841 destPathObf.c_str(), log->
GetLevel() );
846 std::string target = dest;
849 bool srcIsDir =
false;
851 if( strncmp( sourceFile->
ProtName,
"file", 4 ) == 0 )
852 srcIsDir = std::string( sourceFile->
Path ).size() == size_t( sourceFile->
Doff + sourceFile->
Dlen );
855 srcIsDir = remoteSrcIsDir;
860 std::string srcDir( sourceFile->
Path, sourceFile->
Doff );
862 if( srcDir[srcDir.size() - 1] ==
'/' )
863 srcDir = srcDir.substr( 0, srcDir.size() - 1 );
864 size_t diroff = srcDir.rfind(
'/' );
866 if( diroff == std::string::npos ) diroff = 0;
868 target += sourceFile->
Path + diroff;
870 target = target.substr( 0 , target.rfind(
'/') );
874 properties.
Set(
"source", source );
875 properties.
Set(
"target", target );
876 properties.
Set(
"force", force );
877 properties.
Set(
"posc", posc );
878 properties.
Set(
"coerce", coerce );
879 properties.
Set(
"makeDir", makedir );
880 properties.
Set(
"dynamicSource", dynSrc );
881 properties.
Set(
"thirdParty", thirdParty );
882 properties.
Set(
"checkSumMode", checkSumMode );
883 properties.
Set(
"checkSumType", checkSumType );
884 properties.
Set(
"checkSumPreset", checkSumPreset );
885 properties.
Set(
"chunkSize", chunkSize );
886 properties.
Set(
"parallelChunks", parallelChunks );
887 properties.
Set(
"zipArchive", zip );
888 properties.
Set(
"xcp", xcp );
889 properties.
Set(
"xcpBlockSize", blockSize );
890 properties.
Set(
"delegate", delegate );
891 properties.
Set(
"targetIsDir", targetIsDir );
892 properties.
Set(
"preserveXAttr", preserveXAttr );
893 properties.
Set(
"xrate", config.
xRate );
895 properties.
Set(
"rmOnBadCksum", rmOnBadCksum );
896 properties.
Set(
"continue", continue_ );
897 properties.
Set(
"zipAppend", zipappend );
899 properties.
Set(
"doServer", doserver );
902 properties.
Set(
"zipSource", zipFile );
905 properties.
Set(
"nbXcpSources", nbSources );
911 std::cerr <<
"AddJob " << source <<
" -> " << target <<
": ";
912 std::cerr << st.
ToStr() << std::endl;
914 resultVect.push_back( results );
915 sourceFile = sourceFile->
Next;
922 processConfig.
Set(
"jobType",
"configuration" );
924 process.
AddJob( processConfig, 0 );
933 std::cerr <<
"Prepare: " << st.
ToStr() << std::endl;
937 st = process.
Run( &progress );
940 if( resultVect.size() == 1 )
941 std::cerr <<
"Run: " << st.
ToStr() << std::endl;
944 std::vector<XrdCl::PropertyList*>::iterator it;
946 uint16_t jobsRun = 0;
948 for( it = resultVect.begin(); it != resultVect.end(); ++it, ++i )
950 if( !(*it)->HasProperty(
"status" ) )
956 std::cerr <<
"Job #" << i <<
": " << st.
ToStr();
961 std::cerr <<
"Jobs total: " << resultVect.size();
962 std::cerr <<
", run: " << jobsRun;
963 std::cerr <<
", errors: " << errors << std::endl;
bool AllOptionsSupported(XrdCpConfig *config)
const char * FileType2String(XrdCpFile::PType type)
int main(int argc, char **argv)
XrdCpFile * IndexRemote(XrdCl::FileSystem *fs, std::string basePath, long dirOffset)
void ProcessCommandLineEnv(XrdCpConfig *config)
void CleanUpResults(std::vector< XrdCl::PropertyList * > &results)
void AdjustFileInfo(XrdCpFile *file)
uint32_t CountSources(XrdCpFile *file)
void AppendCGI(std::string &url, const char *newCGI)
std::string obfuscateAuth(const std::string &input)
void PrintAdditionalCheckSum(bool print)
void PrintSourceCheckSum(bool print)
virtual void EndJob(uint16_t jobNum, const XrdCl::PropertyList *results)
End job.
void PrintProgressBar(bool print)
void PrintCheckSum(const XrdCl::URL *url, const std::string &checkSum, uint64_t size)
Print the checksum.
virtual void JobProgress(uint16_t jobNum, uint64_t bytesProcessed, uint64_t bytesTotal)
Job progress.
std::string GetProgressBar(time_t now)
Get progress bar.
virtual void BeginJob(uint16_t jobNum, uint16_t jobTotal, const XrdCl::URL *source, const XrdCl::URL *destination)
Begin job.
std::string GetSummaryBar(time_t now)
Get sumary bar.
ProgressDisplay()
Constructor.
void PrintTargetCheckSum(bool print)
static int mapError(int rc)
Copy the data from one point to another.
XRootDStatus Run(CopyProgressHandler *handler)
Run the copy jobs.
XRootDStatus AddJob(const PropertyList &properties, PropertyList *results)
Interface for copy progress notification.
static Log * GetLog()
Get default log.
static PostMaster * GetPostMaster()
Get default post master.
static Env * GetEnv()
Get default client environment.
const std::string & GetName() const
Get file name.
StatInfo * GetStatInfo()
Get the stat info object.
Iterator End()
Get the end iterator.
Iterator Begin()
Get the begin iterator.
void Enable()
Enable delegation in the environment.
static DlgEnv & Instance()
void Disable()
Disable delegation in the environment.
bool PutInt(const std::string &key, int value)
bool PutString(const std::string &key, const std::string &value)
bool GetInt(const std::string &key, int &value)
Send file/filesystem queries to an XRootD cluster.
XRootDStatus DirList(const std::string &path, DirListFlags::Flags flags, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
XRootDStatus Stat(const std::string &path, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
@ DebugMsg
print debug info
@ DumpMsg
print details of the request and responses
void SetLevel(LogLevel level)
Set the level of the messages that should be sent to the destination.
void Error(uint64_t topic, const char *format,...)
Report an error.
LogLevel GetLevel() const
Get the log level.
void Dump(uint64_t topic, const char *format,...)
Print a dump message.
void Info(uint64_t topic, const char *format,...)
Print an info.
void Debug(uint64_t topic, const char *format,...)
Print a debug message.
bool Stop()
Stop the postmaster.
A key-value pair map storing both keys and values as strings.
void Set(const std::string &name, const Item &value)
bool Get(const std::string &name, Item &item) const
bool TestFlags(uint32_t flags) const
Test flags.
@ IsDir
This is a directory.
std::string GetHostId() const
Get the host part of the URL (user:password@host:port)
const std::string & GetProtocol() const
Get the protocol.
std::string GetPathWithParams() const
Get the path with params.
std::string GetURL() const
Get the URL.
const std::string & GetPath() const
Get the path.
static void splitString(Container &result, const std::string &input, const std::string &delimiter)
Split a string.
static std::string BytesToString(uint64_t bytes)
Convert bytes to a human readable string.
const std::string & GetErrorMessage() const
Get error message.
std::string ToStr() const
Convert to string.
const int DefaultCPChunkSize
const uint16_t stError
An error occurred that could potentially be retried.
const uint16_t errInvalidOp
const int DefaultCPParallelChunks
const int DefaultXCpBlockSize
const int DefaultPreserveXAttrs
@ Recursive
Do a recursive listing.
bool IsOK() const
We're fine.
std::string ToString() const
Create a string representation.
uint32_t errNo
Errno, if any.
int GetShellCode() const
Get the status code that may be returned to the shell.
void Config(int argc, char **argv, int Opts=0)
std::vector< std::string > AddCksVal
static const uint64_t DoZipMtlnCksum
static const uint64_t DoNoPbar
static const uint64_t DoCoerce
static const uint64_t DoForce
static const uint64_t DoRmOnBadCksum
static const uint64_t DoNoTlsOK
static const uint64_t DoTpc
static const uint64_t DoCksum
static const uint64_t DoCksrc
static const uint64_t DoTpcDlgt
static const uint64_t DoZip
static const uint64_t DoContinue
static const uint64_t DoRecurse
static const uint64_t DoZipAppend
static const uint64_t DoDynaSrc
static const uint64_t DoSources
static const uint64_t DoXAttr
static const uint64_t DoTlsMLF
static const int optRmtRec
static const uint64_t DoPath
static const uint64_t DoPosc
static const uint64_t DoTpcOnly
static const uint64_t DoTlsNoData
static const uint64_t DoServer