XRootD
XrdCl::ZipArchive Class Reference

#include <XrdClZipArchive.hh>

+ Collaboration diagram for XrdCl::ZipArchive:

Public Member Functions

 ZipArchive (bool enablePlugIns=true)
 Constructor. More...
 
virtual ~ZipArchive ()
 Destructor. More...
 
XRootDStatus AppendFile (const std::string &fn, uint32_t crc32, uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus CloseArchive (ResponseHandler *handler, uint16_t timeout=0)
 Create the central directory at the end of ZIP archive and close it. More...
 
XRootDStatus CloseFile ()
 
XRootDStatus GetCRC32 (const std::string &fn, uint32_t &cksum)
 
FileGetFile ()
 Get the underlying File object. More...
 
XRootDStatus GetOffset (const std::string &fn, uint64_t &offset)
 
bool GetProperty (const std::string &name, std::string &value)
 Get property on the underlying File object. More...
 
bool IsOpen ()
 
bool IsSecure ()
 Check if the underlying file is using an encrypted connection. More...
 
XRootDStatus List (DirectoryList *&list)
 
XRootDStatus OpenArchive (const std::string &url, OpenFlags::Flags flags, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus OpenFile (const std::string &fn, OpenFlags::Flags flags=OpenFlags::None, uint64_t size=0, uint32_t crc32=0)
 
XRootDStatus PgRead (uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus PgReadFrom (const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus Read (uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
XRootDStatus ReadFrom (const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 
bool SetProperty (const std::string &name, const std::string &value)
 Set property on the underlying File object. More...
 
XRootDStatus Stat (const std::string &fn, StatInfo *&info)
 
XRootDStatus Stat (StatInfo *&info)
 
XRootDStatus UpdateMetadata (uint32_t crc32)
 
XRootDStatus Write (uint32_t size, const void *buffer, ResponseHandler *handler, uint16_t timeout=0)
 

Friends

class ::MicroTest
 
class ::XrdEcTests
 
template<typename RSP >
XRootDStatus ReadFromImpl (ZipArchive &, const std::string &, uint64_t, uint32_t, void *, ResponseHandler *, uint16_t)
 
template<bool >
class XrdEc::OpenOnlyImpl
 
class XrdEc::Reader
 
class XrdEc::StrmWriter
 

Detailed Description

Definition at line 60 of file XrdClZipArchive.hh.

Constructor & Destructor Documentation

◆ ZipArchive()

XrdCl::ZipArchive::ZipArchive ( bool  enablePlugIns = true)

Constructor.

Definition at line 208 of file XrdClZipArchive.cc.

208  : archive( enablePlugIns ),
209  archsize( 0 ),
210  cdexists( false ),
211  updated( false ),
212  cdoff( 0 ),
213  orgcdsz( 0 ),
214  orgcdcnt( 0 ),
215  openstage( None ),
216  ckpinit( false )
217  {
218  }

◆ ~ZipArchive()

XrdCl::ZipArchive::~ZipArchive ( )
virtual

Destructor.

Definition at line 223 of file XrdClZipArchive.cc.

224  {
225  }

Member Function Documentation

◆ AppendFile()

XRootDStatus XrdCl::ZipArchive::AppendFile ( const std::string &  fn,
uint32_t  crc32,
uint32_t  size,
const void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)

Create a new file in the ZIP archive and append the data

Parameters
fn: the name of the new file to be created
crc32: the crc32 of the file
size: the size of the file
buffer: the buffer with the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 877 of file XrdClZipArchive.cc.

883  {
884  Log *log = DefaultEnv::GetLog();
885  auto itr = cdmap.find( fn );
886  // check if the file already exists in the archive
887  if( itr != cdmap.end() )
888  {
889  log->Dump( ZipMsg, "[%p] Open failed: file exists %s, cannot append.",
890  this, fn.c_str() );
891  return XRootDStatus( stError, errInvalidOp );
892  }
893 
894  log->Dump( ZipMsg, "[%p] Appending file: %s.", this, fn.c_str() );
895  //-------------------------------------------------------------------------
896  // Create Local File Header record
897  //-------------------------------------------------------------------------
898  lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) );
899  //-------------------------------------------------------------------------
900  // And write it all
901  //-------------------------------------------------------------------------
902  return WriteImpl( size, buffer, handler, timeout );
903  }
static Log * GetLog()
Get default log.
const uint16_t stError
An error occurred that could potentially be retried.
Definition: XrdClStatus.hh:32
const uint64_t ZipMsg
const uint16_t errInvalidOp
Definition: XrdClStatus.hh:51
XrdSysError Log
Definition: XrdConfig.cc:112
A data structure representing ZIP Local File Header.
Definition: XrdZipLFH.hh:42

References XrdCl::Log::Dump(), XrdCl::errInvalidOp, XrdCl::DefaultEnv::GetLog(), XrdCl::stError, and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ CloseArchive()

XRootDStatus XrdCl::ZipArchive::CloseArchive ( ResponseHandler handler,
uint16_t  timeout = 0 
)

Create the central directory at the end of ZIP archive and close it.

Parameters
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 604 of file XrdClZipArchive.cc.

606  {
607  Log *log = DefaultEnv::GetLog();
608 
609  //-------------------------------------------------------------------------
610  // If the file was updated, we need to write the Central Directory before
611  // closing the file.
612  //-------------------------------------------------------------------------
613  if( updated )
614  {
615  ChunkList chunks;
616  std::vector<std::shared_ptr<buffer_t>> wrtbufs;
617  for( auto &p : newfiles )
618  {
619  NewFile &nf = p.second;
620  if( !nf.overwrt ) continue;
621  uint32_t lfhlen = lfh->lfhSize;
622  auto lfhbuf = std::make_shared<buffer_t>();
623  lfhbuf->reserve( lfhlen );
624  nf.lfh->Serialize( *lfhbuf );
625  chunks.emplace_back( nf.offset, lfhbuf->size(), lfhbuf->data() );
626  wrtbufs.emplace_back( std::move( lfhbuf ) );
627  }
628 
629  auto wrtbuff = std::make_shared<buffer_t>( GetCD() );
630  Pipeline p = XrdCl::Write( archive, cdoff,
631  wrtbuff->size(),
632  wrtbuff->data() );
633  wrtbufs.emplace_back( std::move( wrtbuff ) );
634 
635  std::vector<ChunkList> listsvec;
636  XrdCl::Utils::SplitChunks( listsvec, chunks, 262144, 1024 );
637 
638  for(auto itr = listsvec.rbegin(); itr != listsvec.rend(); ++itr)
639  {
640  p = XrdCl::VectorWrite( archive, *itr ) | p;
641  }
642  if( ckpinit )
643  p |= XrdCl::Checkpoint( archive, ChkPtCode::COMMIT );
644  p |= Close( archive ) >>
645  [=]( XRootDStatus &st )
646  {
647  if( st.IsOK() ) Clear();
648  else openstage = Error;
649  }
650  | XrdCl::Final( [=]( const XRootDStatus &st ) mutable
651  {
652  if( st.IsOK() )
653  log->Dump( ZipMsg, "[%p] Successfully closed ZIP archive "
654  "(CD written).", this );
655  else
656  log->Error( ZipMsg, "[%p] Failed to close ZIP archive: %s",
657  this, st.ToString().c_str() );
658  wrtbufs.clear();
659  if( handler ) handler->HandleResponse( make_status( st ), nullptr );
660  } );
661 
662  Async( std::move( p ), timeout );
663  return XRootDStatus();
664  }
665 
666  //-------------------------------------------------------------------------
667  // Otherwise, just close the ZIP archive
668  //-------------------------------------------------------------------------
669  Pipeline p = Close( archive ) >>
670  [=]( XRootDStatus &st )
671  {
672  if( st.IsOK() )
673  {
674  Clear();
675  log->Dump( ZipMsg, "[%p] Successfully closed "
676  "ZIP archive.", this );
677  }
678  else
679  {
680  openstage = Error;
681  log->Error( ZipMsg, "[%p] Failed to close ZIP archive:"
682  " %s", this, st.ToString().c_str() );
683  }
684  if( handler )
685  handler->HandleResponse( make_status( st ), nullptr );
686  };
687  Async( std::move( p ), timeout );
688  return XRootDStatus();
689  }
static void SplitChunks(std::vector< ChunkList > &listsvec, const ChunkList &chunks, const uint32_t maxcs, const size_t maxc)
Split chunks in a ChunkList into one or more ChunkLists.
Definition: XrdClUtils.cc:875
VectorWriteImpl< false > VectorWrite(Ctx< File > file, Arg< ChunkList > chunks, uint16_t timeout=0)
Factory for creating VectorWriteImpl objects.
CheckpointImpl< false > Checkpoint(Ctx< File > file, Arg< ChkPtCode > code, uint16_t timeout=0)
Factory for creating ReadImpl objects.
WriteImpl< false > Write(Ctx< File > file, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< const void * > buffer, uint16_t timeout=0)
Factory for creating WriteImpl objects.
std::future< XRootDStatus > Async(Pipeline pipeline, uint16_t timeout=0)
std::vector< ChunkInfo > ChunkList
List of chunks.
FinalOperation Final
CloseImpl< false > Close(Ctx< File > file, uint16_t timeout=0)
Factory for creating CloseImpl objects.

References XrdCl::Async(), XrdCl::Checkpoint(), XrdCl::Close(), XrdCl::COMMIT, XrdCl::Log::Dump(), XrdCl::Log::Error(), XrdCl::DefaultEnv::GetLog(), XrdCl::ResponseHandler::HandleResponse(), XrdCl::Status::IsOK(), XrdCl::Utils::SplitChunks(), XrdCl::Status::ToString(), XrdCl::VectorWrite(), XrdCl::Write(), and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ CloseFile()

XRootDStatus XrdCl::ZipArchive::CloseFile ( )
inline

Close an open file within the ZIP archive

Returns
: the status of the operation

Definition at line 339 of file XrdClZipArchive.hh.

340  {
341  if( openstage != Done || openfn.empty() )
342  return XRootDStatus( stError, errInvalidOp,
343  0, "Archive not opened." );
344  openfn.clear();
345  lfh.reset();
346  return XRootDStatus();
347  }

References XrdCl::errInvalidOp, and XrdCl::stError.

◆ GetCRC32()

XRootDStatus XrdCl::ZipArchive::GetCRC32 ( const std::string &  fn,
uint32_t &  cksum 
)
inline

Get crc32 for a given file

Parameters
fn: file name
cksum: output parameter
Returns
: the status of the operation

Definition at line 278 of file XrdClZipArchive.hh.

279  { // make sure archive has been opened and CD has been parsed
280  if( openstage != Done )
281  return XRootDStatus( stError, errInvalidOp );
282  // make sure the file is part of the archive
283  auto cditr = cdmap.find( fn );
284  if( cditr == cdmap.end() )
285  return XRootDStatus( stError, errNotFound );
286  cksum = cdvec[cditr->second]->ZCRC32;
287  return XRootDStatus();
288  }
const uint16_t errNotFound
Definition: XrdClStatus.hh:100

References XrdCl::errInvalidOp, XrdCl::errNotFound, and XrdCl::stError.

◆ GetFile()

File& XrdCl::ZipArchive::GetFile ( )
inline

Get the underlying File object.

Definition at line 390 of file XrdClZipArchive.hh.

391  {
392  return archive;
393  }

◆ GetOffset()

XRootDStatus XrdCl::ZipArchive::GetOffset ( const std::string &  fn,
uint64_t &  offset 
)
inline

Definition at line 290 of file XrdClZipArchive.hh.

290  {
291  if( openstage != XrdCl::ZipArchive::Done || !archive.IsOpen() )
293 
294  auto cditr = cdmap.find( fn );
295  if( cditr == cdmap.end() )
297  XrdCl::errNotFound, "File not found." );
298 
299  XrdCl::CDFH *cdfh = cdvec[cditr->second].get();
300 
301  // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression
302  if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED )
304  0, "The compression algorithm is not supported!" );
305 
306  // Now the problem is that at the beginning of our
307  // file there is the Local-file-header, which size
308  // is not known because of the variable size 'extra'
309  // field, so we need to know the offset of the next
310  // record and shift it by the file size.
311  // The next record is either the next LFH (next file)
312  // or the start of the Central-directory.
313  uint64_t cdOffset = zip64eocd ? zip64eocd->cdOffset : eocd->cdOffset;
314  uint64_t nextRecordOffset = ( cditr->second + 1 < cdvec.size() ) ?
315  XrdCl::CDFH::GetOffset( *cdvec[cditr->second + 1] ) : cdOffset;
316  uint64_t filesize = cdfh->compressedSize;
317  if( filesize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
318  filesize = cdfh->extra->compressedSize;
319  uint16_t descsize = cdfh->HasDataDescriptor() ?
321  offset = nextRecordOffset - filesize - descsize;
322  return XrdCl::XRootDStatus();
323  }
bool IsOpen() const
Check if the file is open.
Definition: XrdClFile.cc:846
const uint16_t errNotSupported
Definition: XrdClStatus.hh:62
static uint64_t GetOffset(const CDFH &cdfh)
Definition: XrdZipCDFH.hh:227
std::unique_ptr< Extra > extra
Definition: XrdZipCDFH.hh:345
uint16_t compressionMethod
Definition: XrdZipCDFH.hh:332
bool HasDataDescriptor()
Definition: XrdZipCDFH.hh:324
uint32_t compressedSize
Definition: XrdZipCDFH.hh:335
bool IsZIP64() const
Definition: XrdZipCDFH.hh:316
static uint8_t GetSize(bool zip64)

References XrdZip::CDFH::compressedSize, XrdZip::CDFH::compressionMethod, XrdCl::errInvalidOp, XrdCl::errNotFound, XrdCl::errNotSupported, XrdZip::CDFH::extra, XrdZip::CDFH::GetOffset(), XrdZip::DataDescriptor::GetSize(), XrdZip::CDFH::HasDataDescriptor(), XrdZip::CDFH::IsZIP64(), and XrdCl::stError.

+ Here is the call graph for this function:

◆ GetProperty()

bool XrdCl::ZipArchive::GetProperty ( const std::string &  name,
std::string &  value 
)
inline

Get property on the underlying File object.

Definition at line 382 of file XrdClZipArchive.hh.

383  {
384  return archive.GetProperty( name, value );
385  }
bool GetProperty(const std::string &name, std::string &value) const
Definition: XrdClFile.cc:878

◆ IsOpen()

bool XrdCl::ZipArchive::IsOpen ( )
inline
Returns
: true if ZIP archive has been successfully opened

Definition at line 358 of file XrdClZipArchive.hh.

359  {
360  return openstage == Done;
361  }

Referenced by XrdCl::ZipListHandler::HandleResponse().

+ Here is the caller graph for this function:

◆ IsSecure()

bool XrdCl::ZipArchive::IsSecure ( )
inline

Check if the underlying file is using an encrypted connection.

Definition at line 366 of file XrdClZipArchive.hh.

367  {
368  return archive.IsSecure();
369  }
bool IsSecure() const
Check if the file is using an encrypted connection.
Definition: XrdClFile.cc:857

◆ List()

XRootDStatus XrdCl::ZipArchive::List ( DirectoryList *&  list)

List files in the ZIP archive

Returns
: the status of the operation

Definition at line 720 of file XrdClZipArchive.cc.

721  {
722  if( openstage != Done )
723  return XRootDStatus( stError, errInvalidOp,
724  0, "Archive not opened." );
725 
726  std::string value;
727  archive.GetProperty( "LastURL", value );
728  URL url( value );
729 
730  StatInfo *infoptr = 0;
731  XRootDStatus st = archive.Stat( false, infoptr );
732  std::unique_ptr<StatInfo> info( infoptr );
733 
734  list = new DirectoryList();
735  list->SetParentName( url.GetPath() );
736 
737  auto itr = cdvec.begin();
738  for( ; itr != cdvec.end() ; ++itr )
739  {
740  CDFH *cdfh = itr->get();
741  uint64_t uncompressedSize = cdfh->uncompressedSize;
742  if( uncompressedSize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
743  uncompressedSize = cdfh->extra->uncompressedSize;
744  StatInfo *entry_info = make_stat( *info, uncompressedSize );
745  DirectoryList::ListEntry *entry =
746  new DirectoryList::ListEntry( url.GetHostId(), cdfh->filename, entry_info );
747  list->Add( entry );
748  }
749 
750  return XRootDStatus();
751  }
XRootDStatus Stat(bool force, ResponseHandler *handler, uint16_t timeout=0) XRD_WARN_UNUSED_RESULT
Definition: XrdClFile.cc:177
uint32_t uncompressedSize
Definition: XrdZipCDFH.hh:336
std::string filename
Definition: XrdZipCDFH.hh:344

References XrdCl::DirectoryList::Add(), XrdCl::errInvalidOp, XrdZip::CDFH::extra, XrdZip::CDFH::filename, XrdCl::URL::GetHostId(), XrdCl::URL::GetPath(), XrdCl::File::GetProperty(), XrdCl::DirectoryList::SetParentName(), XrdCl::File::Stat(), XrdCl::stError, and XrdZip::CDFH::uncompressedSize.

Referenced by XrdCl::ZipListHandler::HandleResponse().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ OpenArchive()

XRootDStatus XrdCl::ZipArchive::OpenArchive ( const std::string &  url,
OpenFlags::Flags  flags,
ResponseHandler handler,
uint16_t  timeout = 0 
)

Open ZIP Archive (and parse the Central Directory)

Parameters
url: the URL of the ZIP archive
flags: open flags to be used when openning the file
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 266 of file XrdClZipArchive.cc.

270  {
271  Log *log = DefaultEnv::GetLog();
272  Fwd<uint32_t> rdsize; // number of bytes to be read
273  Fwd<uint64_t> rdoff; // offset for the read request
274  Fwd<void*> rdbuff; // buffer for data to be read
275  uint32_t maxrdsz = EOCD::maxCommentLength + EOCD::eocdBaseSize +
277 
278  Pipeline open_archive = // open the archive
279  XrdCl::Open( archive, url, flags ) >>
280  [=]( XRootDStatus &status, StatInfo &info ) mutable
281  {
282  // check the status is OK
283  if( !status.IsOK() ) return;
284 
285  archsize = info.GetSize();
286  // if it is an empty file (possibly a new file) there's nothing more to do
287  if( archsize == 0 )
288  {
289  cdexists = false;
290  openstage = Done;
291  log->Dump( ZipMsg, "[%p] Opened a ZIP archive (file empty).", this );
292  Pipeline::Stop();
293  }
294  // prepare the arguments for the subsequent read
295  rdsize = ( archsize <= maxrdsz ? archsize : maxrdsz );
296  rdoff = archsize - *rdsize;
297  buffer.reset( new char[*rdsize] );
298  rdbuff = buffer.get();
299  openstage = HaveEocdBlk;
300  log->Dump( ZipMsg, "[%p] Opened a ZIP archive, reading "
301  "Central Directory at offset: %llu.", this, (unsigned long long) *rdoff );
302  }
303  // read the Central Directory (in several stages if necessary)
304  | XrdCl::Read( archive, rdoff, rdsize, rdbuff ) >>
305  [=]( XRootDStatus &status, ChunkInfo &chunk ) mutable
306  {
307  // check the status is OK
308  if( !status.IsOK() ) return;
309 
310  const char *buff = reinterpret_cast<char*>( chunk.buffer );
311  while( true )
312  {
313  switch( openstage )
314  {
315  case HaveEocdBlk:
316  {
317  // Parse the EOCD record
318  const char *eocdBlock = EOCD::Find( buff, chunk.length );
319  if( !eocdBlock )
320  {
321  XRootDStatus error( stError, errDataError, 0,
322  "End-of-central-directory signature not found." );
323  Pipeline::Stop( error );
324  }
325  try{
326  eocd.reset( new EOCD( eocdBlock, chunk.length - uint32_t(eocdBlock - buff) ) );
327  log->Dump( ZipMsg, "[%p] EOCD record parsed: %s", this,
328  eocd->ToString().c_str() );
329  if(eocd->cdOffset > archsize || eocd->cdOffset + eocd->cdSize > archsize)
330  throw bad_data();
331  }
332  catch(const bad_data &ex){
333  XRootDStatus error( stError, errDataError, 0,
334  "End-of-central-directory signature corrupted." );
335  Pipeline::Stop( error );
336  }
337  // Do we have the whole archive?
338  if( chunk.length == archsize )
339  {
340  // If we managed to download the whole archive we don't need to
341  // worry about zip64, it is so small that standard EOCD will do
342  cdoff = eocd->cdOffset;
343  orgcdsz = eocd->cdSize;
344  orgcdcnt = eocd->nbCdRec;
345  buff = buff + cdoff;
346  openstage = HaveCdRecords;
347  continue;
348  }
349 
350  // Let's see if it is ZIP64 (if yes, the EOCD will be preceded with ZIP64 EOCD locator)
351  const char *zip64EocdlBlock = eocdBlock - ZIP64_EOCDL::zip64EocdlSize;
352  // make sure there is enough data to assume there's a ZIP64 EOCD locator
353  if( zip64EocdlBlock > buffer.get() )
354  {
355  uint32_t signature = to<uint32_t>( zip64EocdlBlock );
356  if( signature == ZIP64_EOCDL::zip64EocdlSign )
357  {
358  buff = zip64EocdlBlock;
359  openstage = HaveZip64EocdlBlk;
360  continue;
361  }
362  }
363 
364  // It's not ZIP64, we already know where the CD records are
365  // we need to read more data
366  cdoff = eocd->cdOffset;
367  orgcdsz = eocd->cdSize;
368  orgcdcnt = eocd->nbCdRec;
369  rdoff = eocd->cdOffset;
370  rdsize = eocd->cdSize;
371  buffer.reset( new char[*rdsize] );
372  rdbuff = buffer.get();
373  openstage = HaveCdRecords;
374  log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
375  this, (unsigned long long) *rdoff );
376  Pipeline::Repeat(); break; // the break is really not needed ...
377  }
378 
379  case HaveZip64EocdlBlk:
380  {
381  std::unique_ptr<ZIP64_EOCDL> eocdl( new ZIP64_EOCDL( buff ) );
382  log->Dump( ZipMsg, "[%p] EOCDL record parsed: %s",
383  this, eocdl->ToString().c_str() );
384 
385  if( chunk.offset > eocdl->zip64EocdOffset )
386  {
387  // we need to read more data, adjust the read arguments
388  rdsize = archsize - eocdl->zip64EocdOffset;
389  rdoff = eocdl->zip64EocdOffset;
390  buffer.reset( new char[*rdsize] );
391  rdbuff = buffer.get();
392  openstage = HaveZip64EocdBlk;
393  log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
394  this, (unsigned long long) *rdoff );
396  }
397 
398  buff = buffer.get() + ( eocdl->zip64EocdOffset - chunk.offset );
399  openstage = HaveZip64EocdBlk;
400  continue;
401  }
402 
403  case HaveZip64EocdBlk:
404  {
405  uint32_t signature = to<uint32_t>( buff );
406  if( signature != ZIP64_EOCD::zip64EocdSign )
407  {
408  XRootDStatus error( stError, errDataError, 0,
409  "ZIP64 End-of-central-directory signature not found." );
410  Pipeline::Stop( error );
411  }
412  zip64eocd.reset( new ZIP64_EOCD( buff ) );
413  log->Dump( ZipMsg, "[%p] ZIP64EOCD record parsed: %s",
414  this, zip64eocd->ToString().c_str() );
415 
416  // now we can read the CD records, adjust the read arguments
417  cdoff = zip64eocd->cdOffset;
418  orgcdsz = zip64eocd->cdSize;
419  orgcdcnt = zip64eocd->nbCdRec;
420  rdoff = zip64eocd->cdOffset;
421  rdsize = zip64eocd->cdSize;
422  buffer.reset( new char[*rdsize] );
423  rdbuff = buffer.get();
424  openstage = HaveCdRecords;
425  log->Dump( ZipMsg, "[%p] Reading additional data at offset: %llu.",
426  this, (unsigned long long) *rdoff );
427  Pipeline::Repeat(); break; // the break is really not needed ...
428  }
429 
430  case HaveCdRecords:
431  {
432  // make a copy of the original CDFH records
433  orgcdbuf.reserve( orgcdsz );
434  std::copy( buff, buff + orgcdsz, std::back_inserter( orgcdbuf ) );
435  try
436  {
437  if( zip64eocd )
438  std::tie( cdvec, cdmap ) = CDFH::Parse( buff, zip64eocd->cdSize, zip64eocd->nbCdRec );
439  else
440  std::tie( cdvec, cdmap ) = CDFH::Parse( buff, eocd->cdSize, eocd->nbCdRec );
441  log->Dump( ZipMsg, "[%p] CD records parsed.", this );
442  uint64_t sumCompSize = 0;
443  for (auto it = cdvec.begin(); it != cdvec.end(); it++)
444  {
445  sumCompSize += (*it)->IsZIP64() ? (*it)->extra->compressedSize : (*it)->compressedSize;
446  if ((*it)->offset > archsize || (*it)->offset + (*it)->compressedSize > archsize)
447  throw bad_data();
448  }
449  if (sumCompSize > archsize)
450  throw bad_data();
451  }
452  catch( const bad_data &ex )
453  {
454  XRootDStatus error( stError, errDataError, 0,
455  "ZIP Central Directory corrupted." );
456  Pipeline::Stop( error );
457  }
458  if( chunk.length != archsize ) buffer.reset();
459  openstage = Done;
460  cdexists = true;
461  break;
462  }
463 
464  default: Pipeline::Stop( XRootDStatus( stError, errInvalidOp ) );
465  }
466 
467  break;
468  }
469  }
470  | XrdCl::Final( [=]( const XRootDStatus &status )
471  { // finalize the pipeline by calling the user callback
472  if( status.IsOK() )
473  log->Debug( ZipMsg, "[%p] Opened a ZIP archive (%s): %s",
474  this, url.c_str(), status.ToString().c_str() );
475  else
476  log->Error( ZipMsg, "[%p] Failed to open a ZIP archive (%s): %s",
477  this, url.c_str(), status.ToString().c_str() );
478  if( handler )
479  handler->HandleResponse( make_status( status ), nullptr );
480  } );
481 
482  Async( std::move( open_archive ), timeout );
483  return XRootDStatus();
484  }
static void Repeat()
Repeat current operation.
static void Stop(const XRootDStatus &status=XrdCl::XRootDStatus())
ReadImpl< false > Read(Ctx< File > file, Arg< uint64_t > offset, Arg< uint32_t > size, Arg< void * > buffer, uint16_t timeout=0)
Factory for creating ReadImpl objects.
const uint16_t errDataError
data is corrupted
Definition: XrdClStatus.hh:63
OpenImpl< false > Open(Ctx< File > file, Arg< std::string > url, Arg< OpenFlags::Flags > flags, Arg< Access::Mode > mode=Access::None, uint16_t timeout=0)
Factory for creating ReadImpl objects.
static std::tuple< cdvec_t, cdmap_t > Parse(const char *buffer, uint32_t bufferSize, uint16_t nbCdRecords)
Definition: XrdZipCDFH.hh:75
static const uint16_t eocdBaseSize
Definition: XrdZipEOCD.hh:153
static const uint16_t maxCommentLength
Definition: XrdZipEOCD.hh:154
static const char * Find(const char *buffer, uint64_t size)
Definition: XrdZipEOCD.hh:41
A data structure representing the ZIP64 end of central directory locator.
static const uint32_t zip64EocdlSign
static const uint16_t zip64EocdlSize
static const uint32_t zip64EocdSign

References XrdCl::Async(), XrdCl::Log::Debug(), XrdCl::Log::Dump(), XrdZip::EOCD::eocdBaseSize, XrdCl::errDataError, XrdCl::errInvalidOp, XrdCl::Log::Error(), XrdZip::EOCD::Find(), XrdCl::DefaultEnv::GetLog(), XrdCl::ResponseHandler::HandleResponse(), XrdCl::Status::IsOK(), XrdZip::EOCD::maxCommentLength, XrdCl::Open(), XrdZip::CDFH::Parse(), XrdCl::Read(), XrdCl::Pipeline::Repeat(), XrdCl::stError, XrdCl::Pipeline::Stop(), XrdCl::Status::ToString(), XrdZip::ZIP64_EOCDL::zip64EocdlSign, XrdZip::ZIP64_EOCDL::zip64EocdlSize, XrdZip::ZIP64_EOCD::zip64EocdSign, and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ OpenFile()

XRootDStatus XrdCl::ZipArchive::OpenFile ( const std::string &  fn,
OpenFlags::Flags  flags = OpenFlags::None,
uint64_t  size = 0,
uint32_t  crc32 = 0 
)

Open a file within the ZIP Archive

Parameters
fn: file name to be opened
flags: open flags (either 'Read' or 'New | Write')
size: file size (to be included in the LFH)
crc32: file crc32 (to be included in the LFH)
Returns
: the status of the operation

Definition at line 489 of file XrdClZipArchive.cc.

493  {
494  if( !openfn.empty() || openstage != Done || !archive.IsOpen() )
495  return XRootDStatus( stError, errInvalidOp );
496 
497  Log *log = DefaultEnv::GetLog();
498  auto itr = cdmap.find( fn );
499  if( itr == cdmap.end() )
500  {
501  // the file does not exist in the archive so it only makes sense
502  // if our user is opening for append
503  if( flags & OpenFlags::New )
504  {
505  openfn = fn;
506  lfh.reset( new LFH( fn, crc32, size, time( 0 ) ) );
507  log->Dump( ZipMsg, "[%p] File %s opened for append.",
508  this, fn.c_str() );
509  return XRootDStatus();
510  }
511  log->Dump( ZipMsg, "[%p] Open failed: %s not in the ZIP archive.",
512  this, fn.c_str() );
513  return XRootDStatus( stError, errNotFound );
514  }
515 
516  // the file name exist in the archive but our user wants to append
517  // a file with the same name
518  if( flags & OpenFlags::New )
519  {
520  log->Dump( ZipMsg, "[%p] Open failed: file exists %s, cannot append.",
521  this, fn.c_str() );
522 
523  return XRootDStatus( stError, errInvalidOp, EEXIST, "The file already exists in the ZIP archive." );
524  }
525 
526  openfn = fn;
527  log->Dump( ZipMsg, "[%p] File %s opened for reading.",
528  this, fn.c_str() );
529  return XRootDStatus();
530  }

References XrdCl::Log::Dump(), XrdCl::errInvalidOp, XrdCl::errNotFound, XrdCl::DefaultEnv::GetLog(), XrdCl::File::IsOpen(), XrdCl::OpenFlags::New, XrdCl::stError, and XrdCl::ZipMsg.

+ Here is the call graph for this function:

◆ PgRead()

XRootDStatus XrdCl::ZipArchive::PgRead ( uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)
inline

PgRead data from an open file

Parameters
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 143 of file XrdClZipArchive.hh.

148  {
149  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
150  return PgReadFrom( openfn, offset, size, buffer, handler, timeout );
151  }
XRootDStatus PgReadFrom(const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)

References XrdCl::errInvalidOp, XrdCl::Operation< HasHndl >::handler, XrdCl::stError, and XrdCl::ConcreteOperation< Derived, HasHndl, HdlrFactory, Args >::timeout.

◆ PgReadFrom()

XRootDStatus XrdCl::ZipArchive::PgReadFrom ( const std::string &  fn,
uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)

PgRead data from a given file

Parameters
fn: the name of the file from which we are going to read
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 707 of file XrdClZipArchive.cc.

713  {
714  return ReadFromImpl<PageInfo>( *this, fn, offset, size, buffer, handler, timeout );
715  }

◆ Read()

XRootDStatus XrdCl::ZipArchive::Read ( uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)
inline

Read data from an open file

Parameters
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 122 of file XrdClZipArchive.hh.

127  {
128  if( openfn.empty() ) return XRootDStatus( stError, errInvalidOp );
129  return ReadFrom( openfn, offset, size, buffer, handler, timeout );
130  }
XRootDStatus ReadFrom(const std::string &fn, uint64_t offset, uint32_t size, void *buffer, ResponseHandler *handler, uint16_t timeout=0)

References XrdCl::errInvalidOp, XrdCl::Operation< HasHndl >::handler, XrdCl::ReadFrom(), XrdCl::stError, and XrdCl::ConcreteOperation< Derived, HasHndl, HdlrFactory, Args >::timeout.

+ Here is the call graph for this function:

◆ ReadFrom()

XRootDStatus XrdCl::ZipArchive::ReadFrom ( const std::string &  fn,
uint64_t  offset,
uint32_t  size,
void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)

Read data from a given file

Parameters
fn: the name of the file from which we are going to read
offset: offset within the file to read at
size: number of bytes to be read
buffer: the buffer for the data
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 694 of file XrdClZipArchive.cc.

700  {
701  return ReadFromImpl<ChunkInfo>( *this, fn, offset, size, buffer, handler, timeout );
702  }

◆ SetProperty()

bool XrdCl::ZipArchive::SetProperty ( const std::string &  name,
const std::string &  value 
)
inline

Set property on the underlying File object.

Definition at line 374 of file XrdClZipArchive.hh.

375  {
376  return archive.SetProperty( name, value );
377  }
bool SetProperty(const std::string &name, const std::string &value)
Definition: XrdClFile.cc:867

◆ Stat() [1/2]

XRootDStatus XrdCl::ZipArchive::Stat ( const std::string &  fn,
StatInfo *&  info 
)
inline

Get stat info for given file

Parameters
fn: the name of the file
info: output parameter
Returns
: the status of the operation

Definition at line 242 of file XrdClZipArchive.hh.

243  { // make sure archive has been opened and CD has been parsed
244  if( openstage != Done )
245  return XRootDStatus( stError, errInvalidOp );
246  // make sure the file is part of the archive
247  auto cditr = cdmap.find( fn );
248  if( cditr == cdmap.end() )
249  return XRootDStatus( stError, errNotFound );
250  // create the result
251  info = make_stat( fn );
252  if (info)
253  return XRootDStatus();
254  else // have difficult to access the openned archive.
255  return XRootDStatus( stError, errNotFound );
256  }

References XrdCl::errInvalidOp, XrdCl::errNotFound, and XrdCl::stError.

◆ Stat() [2/2]

XRootDStatus XrdCl::ZipArchive::Stat ( StatInfo *&  info)
inline

Get stat info for an open file

Parameters
info: output parameter
Returns
: the status of the operation

Definition at line 264 of file XrdClZipArchive.hh.

265  {
266  if( openfn.empty() )
267  return XRootDStatus( stError, errInvalidOp );
268  return Stat( openfn, info );
269  }
XRootDStatus Stat(const std::string &fn, StatInfo *&info)

References XrdCl::errInvalidOp, XrdCl::Stat(), and XrdCl::stError.

+ Here is the call graph for this function:

◆ UpdateMetadata()

XRootDStatus XrdCl::ZipArchive::UpdateMetadata ( uint32_t  crc32)

Update the metadata of the currently open file

Parameters
crc32: the crc32 checksum
Returns
: the status of the operation

Definition at line 849 of file XrdClZipArchive.cc.

850  {
851  if( openstage != Done || openfn.empty() )
852  return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
853 
854  //---------------------------------------------------------------------
855  // Firstly, update the crc32 in the central directory
856  //---------------------------------------------------------------------
857  auto itr = cdmap.find( openfn );
858  if( itr == cdmap.end() )
859  return XRootDStatus( stError, errInvalidOp );
860  cdvec[itr->second]->ZCRC32 = crc32;
861 
862  //---------------------------------------------------------------------
863  // Secondly, update the crc32 in the LFH and mark it as needing
864  // overwriting
865  //---------------------------------------------------------------------
866  auto itr2 = newfiles.find( openfn );
867  if( itr2 == newfiles.end() )
868  return XRootDStatus( stError, errInvalidOp );
869  itr2->second.lfh->ZCRC32 = crc32;
870 
871  return XRootDStatus();
872  }

References XrdCl::errInvalidOp, and XrdCl::stError.

◆ Write()

XRootDStatus XrdCl::ZipArchive::Write ( uint32_t  size,
const void *  buffer,
ResponseHandler handler,
uint16_t  timeout = 0 
)
inline

Append data to a new file

Parameters
size: number of bytes to be appended
buffer: the buffer with the data to be appended
handler: user callback
timeout: operation timeout
Returns
: the status of the operation

Definition at line 198 of file XrdClZipArchive.hh.

202  {
203  if( openstage != Done || openfn.empty() )
204  return XRootDStatus( stError, errInvalidOp, 0, "Archive not opened." );
205 
206  return WriteImpl( size, buffer, handler, timeout );
207  }

References XrdCl::errInvalidOp, XrdCl::Operation< HasHndl >::handler, XrdCl::stError, and XrdCl::ConcreteOperation< Derived, HasHndl, HdlrFactory, Args >::timeout.

Friends And Related Function Documentation

◆ ::MicroTest

friend class ::MicroTest
friend

Definition at line 66 of file XrdClZipArchive.hh.

◆ ::XrdEcTests

friend class ::XrdEcTests
friend

Definition at line 67 of file XrdClZipArchive.hh.

◆ ReadFromImpl

template<typename RSP >
XRootDStatus ReadFromImpl ( ZipArchive me,
const std::string &  fn,
uint64_t  relativeOffset,
uint32_t  size,
void *  usrbuff,
ResponseHandler usrHandler,
uint16_t  timeout 
)
friend

Definition at line 44 of file XrdClZipArchive.cc.

51  {
52  if( me.openstage != ZipArchive::Done || !me.archive.IsOpen() )
53  return XRootDStatus( stError, errInvalidOp );
54 
55  Log *log = DefaultEnv::GetLog();
56 
57  auto cditr = me.cdmap.find( fn );
58  if( cditr == me.cdmap.end() )
59  return XRootDStatus( stError, errNotFound,
60  errNotFound, "File not found." );
61 
62  CDFH *cdfh = me.cdvec[cditr->second].get();
63 
64  // check if the file is compressed, for now we only support uncompressed and inflate/deflate compression
65  if( cdfh->compressionMethod != 0 && cdfh->compressionMethod != Z_DEFLATED )
66  return XRootDStatus( stError, errNotSupported,
67  0, "The compression algorithm is not supported!" );
68 
69  // Now the problem is that at the beginning of our
70  // file there is the Local-file-header, which size
71  // is not known because of the variable size 'extra'
72  // field, so we need to know the offset of the next
73  // record and shift it by the file size.
74  // The next record is either the next LFH (next file)
75  // or the start of the Central-directory.
76  uint64_t cdOffset = me.zip64eocd ? me.zip64eocd->cdOffset : me.eocd->cdOffset;
77  uint64_t nextRecordOffset = ( cditr->second + 1 < me.cdvec.size() ) ?
78  CDFH::GetOffset( *me.cdvec[cditr->second + 1] ) : cdOffset;
79  uint64_t filesize = cdfh->compressedSize;
80  if( filesize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
81  filesize = cdfh->extra->compressedSize;
82  uint16_t descsize = cdfh->HasDataDescriptor() ?
83  DataDescriptor::GetSize( cdfh->IsZIP64() ) : 0;
84  uint64_t fileoff = nextRecordOffset - filesize - descsize;
85  uint64_t offset = fileoff + relativeOffset;
86  uint64_t uncompressedSize = cdfh->uncompressedSize;
87  if( uncompressedSize == std::numeric_limits<uint32_t>::max() && cdfh->extra )
88  uncompressedSize = cdfh->extra->uncompressedSize;
89  uint64_t sizeTillEnd = relativeOffset > uncompressedSize ?
90  0 : uncompressedSize - relativeOffset;
91  if( size > sizeTillEnd ) size = sizeTillEnd;
92 
93  // if it is a compressed file use ZIP cache to read from the file
94  if( cdfh->compressionMethod == Z_DEFLATED )
95  {
96  log->Dump( ZipMsg, "[%p] Reading compressed data.", &me );
97  // check if respective ZIP cache exists
98  bool empty = me.zipcache.find( fn ) == me.zipcache.end();
99  // if the entry does not exist, it will be created using
100  // default constructor
101  ZipCache &cache = me.zipcache[fn];
102 
103  if( relativeOffset > uncompressedSize )
104  {
105  // we are reading past the end of file,
106  // we can serve the request right away!
107  RSP *r = new RSP( relativeOffset, 0, usrbuff );
108  AnyObject *rsp = new AnyObject();
109  rsp->Set( r );
110  usrHandler->HandleResponse( new XRootDStatus(), rsp );
111  return XRootDStatus();
112  }
113 
114  uint32_t sizereq = size;
115  if( relativeOffset + size > uncompressedSize )
116  sizereq = uncompressedSize - relativeOffset;
117  cache.QueueReq( relativeOffset, sizereq, usrbuff, usrHandler );
118 
119  // if we have the whole ZIP archive we can populate the cache
120  // straight away
121  if( empty && me.buffer)
122  {
123  auto begin = me.buffer.get() + fileoff;
124  auto end = begin + filesize ;
125  buffer_t buff( begin, end );
126  cache.QueueRsp( XRootDStatus(), 0, std::move( buff ) );
127  return XRootDStatus();
128  }
129 
130  // if we don't have the data we need to issue a remote read
131  if( !me.buffer )
132  {
133  if( relativeOffset > filesize ) return XRootDStatus(); // there's nothing to do,
134  // we already have all the data locally
135  uint32_t rdsize = size;
136  // check if this is the last read (we reached the end of
137  // file from user perspective)
138  if( relativeOffset + size >= uncompressedSize )
139  {
140  // if yes, make sure we readout all the compressed data
141  // Note: In a patological case the compressed size may
142  // be greater than the uncompressed size
143  rdsize = filesize > relativeOffset ?
144  filesize - relativeOffset :
145  0;
146  }
147  // make sure we are not reading past the end of
148  // compressed data
149  if( relativeOffset + size > filesize )
150  rdsize = filesize - relativeOffset;
151 
152 
153  // now read the data ...
154  auto rdbuff = std::make_shared<ZipCache::buffer_t>( rdsize );
155  Pipeline p = XrdCl::RdWithRsp<RSP>( me.archive, offset, rdbuff->size(), rdbuff->data() ) >>
156  [relativeOffset, rdbuff, &cache, &me]( XRootDStatus &st, RSP &rsp )
157  {
158  Log *log = DefaultEnv::GetLog();
159  log->Dump( ZipMsg, "[%p] Read %u bytes of remote data at offset %llu.",
160  &me, rsp.GetLength(), (unsigned long long) rsp.GetOffset() );
161  cache.QueueRsp( st, relativeOffset, std::move( *rdbuff ) );
162  };
163  Async( std::move( p ), timeout );
164  }
165 
166  return XRootDStatus();
167  }
168 
169  // check if we have the whole file in our local buffer
170  if( me.buffer || size == 0 )
171  {
172  if( size )
173  {
174  memcpy( usrbuff, me.buffer.get() + offset, size );
175  log->Dump( ZipMsg, "[%p] Serving read from local cache.", &me );
176  }
177 
178  if( usrHandler )
179  {
180  XRootDStatus *st = ZipArchive::make_status();
181  RSP *rsp = new RSP( relativeOffset, size, usrbuff );
182  ZipArchive::Schedule( usrHandler, st, rsp );
183  }
184  return XRootDStatus();
185  }
186 
187  Pipeline p = XrdCl::RdWithRsp<RSP>( me.archive, offset, size, usrbuff ) >>
188  [=, &me]( XRootDStatus &st, RSP &r )
189  {
190  log->Dump( ZipMsg, "[%p] Read %u bytes of remote data at "
191  "offset %llu.", &me, r.GetLength(), (unsigned long long) r.GetOffset() );
192  if( usrHandler )
193  {
194  XRootDStatus *status = ZipArchive::make_status( st );
195  RSP *rsp = nullptr;
196  if( st.IsOK() )
197  rsp = new RSP( relativeOffset, r.GetLength(), r.GetBuffer() );
198  usrHandler->HandleResponse( status, ZipArchive::PkgRsp( rsp ) );
199  }
200  };
201  Async( std::move( p ), timeout );
202  return XRootDStatus();
203  }
std::vector< char > buffer_t
Definition: XrdZipUtils.hh:56

◆ XrdEc::OpenOnlyImpl

template<bool >
friend class XrdEc::OpenOnlyImpl
friend

Definition at line 65 of file XrdClZipArchive.hh.

◆ XrdEc::Reader

friend class XrdEc::Reader
friend

Definition at line 63 of file XrdClZipArchive.hh.

◆ XrdEc::StrmWriter

friend class XrdEc::StrmWriter
friend

Definition at line 62 of file XrdClZipArchive.hh.


The documentation for this class was generated from the following files: