XRootD
XrdPfc::IOFileBlock Class Reference

Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are downloaded. Handles read requests as they come along. More...

#include <XrdPfcIOFileBlock.hh>

+ Inheritance diagram for XrdPfc::IOFileBlock:
+ Collaboration diagram for XrdPfc::IOFileBlock:

Public Member Functions

 IOFileBlock (XrdOucCacheIO *io, Cache &cache)
 
 ~IOFileBlock ()
 
void DetachFinalize () override
 Abstract virtual method of XrdPfc::IO Called to destruct the IO object after it is no longer used. More...
 
long long FSize () override
 
int Fstat (struct stat &sbuff) override
 
bool ioActive () override
 Abstract virtual method of XrdPfc::IO Called to check if destruction needs to be done in a separate task. More...
 
virtual int Read (char *buff, long long offs, int rlen)=0
 Pass Read request to the corresponding File object. More...
 
int Read (char *Buffer, long long Offset, int Length) override
 
virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen)
 Pass Read request to the corresponding File object. More...
 
void Update (XrdOucCacheIO &iocp) override
 
- Public Member Functions inherited from XrdPfc::IO
 IO (XrdOucCacheIO *io, Cache &cache)
 
virtual XrdOucCacheIOBase ()
 Original data source. More...
 
bool Detach (XrdOucCacheIOCD &iocdP) final
 
XrdOucCacheIOGetInput ()
 
const char * GetLocation ()
 
XrdSysTraceGetTrace ()
 
const char * Path () override
 Original data source URL. More...
 
int Sync () override
 
virtual int Sync ()=0
 
virtual void Sync (XrdOucCacheIOCB &iocb)
 
virtual int Trunc (long long offs)=0
 
int Trunc (long long Offset) override
 
virtual void Trunc (XrdOucCacheIOCB &iocb, long long offs)
 
virtual int Write (char *buff, long long offs, int wlen)=0
 
int Write (char *Buffer, long long Offset, int Length) override
 
virtual void Write (XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen)
 
- Public Member Functions inherited from XrdOucCacheIO
 XrdOucCacheIO ()
 Construct and Destructor. More...
 
virtual const char * Location (bool refresh=false)
 
virtual int pgRead (char *buff, long long offs, int rdlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual void pgRead (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rdlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual int pgWrite (char *buff, long long offs, int wrlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual void pgWrite (XrdOucCacheIOCB &iocb, char *buff, long long offs, int wrlen, std::vector< uint32_t > &csvec, uint64_t opts=0, int *csfix=0)
 
virtual void Preread (aprParms &Parms)
 
virtual void Preread (long long offs, int rlen, int opts=0)
 
virtual void Read (XrdOucCacheIOCB &iocb, char *buff, long long offs, int rlen)
 
virtual int ReadV (const XrdOucIOVec *readV, int rnum)
 
virtual void ReadV (XrdOucCacheIOCB &iocb, const XrdOucIOVec *readV, int rnum)
 
virtual void Sync (XrdOucCacheIOCB &iocb)
 
virtual void Trunc (XrdOucCacheIOCB &iocb, long long offs)
 
virtual void Write (XrdOucCacheIOCB &iocb, char *buff, long long offs, int wlen)
 
virtual int WriteV (const XrdOucIOVec *writV, int wnum)
 
virtual void WriteV (XrdOucCacheIOCB &iocb, const XrdOucIOVec *writV, int wnum)
 

Additional Inherited Members

- Static Public Attributes inherited from XrdOucCacheIO
static const uint64_t forceCS = 0x0000000000000001ULL
 
static const int SingleUse = 0x0001
 Mark pages for single use. More...
 
- Protected Member Functions inherited from XrdPfc::IO
std::string GetFilename ()
 
const char * GetPath ()
 
unsigned short ObtainReadSid ()
 
const char * RefreshLocation ()
 
bool register_block_error (int res)
 
bool register_incomplete_read ()
 
- Protected Member Functions inherited from XrdOucCacheIO
virtual ~XrdOucCacheIO ()
 
- Protected Attributes inherited from XrdPfc::IO
RAtomic_int m_active_read_reqs
 number of active read requests More...
 
Cachem_cache
 reference to Cache object More...
 
std::map< int, int > m_error_counts
 
int m_incomplete_count {0}
 
const char * m_traceID
 

Detailed Description

Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are downloaded. Handles read requests as they come along.

Definition at line 37 of file XrdPfcIOFileBlock.hh.

Constructor & Destructor Documentation

◆ IOFileBlock()

IOFileBlock::IOFileBlock ( XrdOucCacheIO io,
Cache cache 
)

Definition at line 40 of file XrdPfcIOFileBlock.cc.

40  :
41  IO(io, cache), m_localStat(0), m_info(cache.GetTrace(), false), m_info_file(0)
42 {
44  GetBlockSizeFromPath();
45  initLocalStat();
46 }
XrdSysTrace * GetTrace()
Definition: XrdPfc.hh:283
const Configuration & RefConfiguration() const
Reference XrdPfc configuration.
Definition: XrdPfc.hh:204
static Cache & GetInstance()
Singleton access.
Definition: XrdPfc.cc:132
IO(XrdOucCacheIO *io, Cache &cache)
Definition: XrdPfcIO.cc:7
long long m_hdfsbsize
used with m_hdfsmode, default 128MB
Definition: XrdPfc.hh:115

References XrdPfc::Cache::GetInstance(), XrdPfc::Configuration::m_hdfsbsize, and XrdPfc::Cache::RefConfiguration().

+ Here is the call graph for this function:

◆ ~IOFileBlock()

IOFileBlock::~IOFileBlock ( )

Definition at line 49 of file XrdPfcIOFileBlock.cc.

50 {
51  // called from Detach() if no sync is needed or
52  // from Cache's sync thread
53 
54  TRACEIO(Debug, "deleting IOFileBlock");
55 }
#define TRACEIO(act, x)
Definition: XrdPfcTrace.hh:63

References Macaroons::Debug, and TRACEIO.

Member Function Documentation

◆ DetachFinalize()

void IOFileBlock::DetachFinalize ( )
overridevirtual

Abstract virtual method of XrdPfc::IO Called to destruct the IO object after it is no longer used.

Implements XrdPfc::IO.

Definition at line 102 of file XrdPfcIOFileBlock.cc.

103 {
104  // Effectively a destructor.
105 
106  TRACEIO(Info, "DetachFinalize() " << this);
107 
108  CloseInfoFile();
109  {
110  XrdSysMutexHelper lock(&m_mutex);
111  for (std::map<int, File*>::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it)
112  {
113  if (it->second)
114  {
115  it->second->RequestSyncOfDetachStats();
116  m_cache.ReleaseFile(it->second, this);
117  }
118  }
119  }
120 
121  delete this;
122 }
void ReleaseFile(File *, IO *)
Definition: XrdPfc.cc:471
Cache & m_cache
reference to Cache object
Definition: XrdPfcIO.hh:50
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:41

References XrdPfc::IO::m_cache, XrdPfc::Cache::ReleaseFile(), and TRACEIO.

+ Here is the call graph for this function:

◆ FSize()

long long IOFileBlock::FSize ( )
overridevirtual

Obtain size of the file.

Returns
Size of the file in bytes.

Implements XrdOucCacheIO.

Definition at line 207 of file XrdPfcIOFileBlock.cc.

208 {
209  if ( ! m_localStat) return -ENOENT;
210 
211  return m_localStat->st_size;
212 }

Referenced by Read().

+ Here is the caller graph for this function:

◆ Fstat()

int IOFileBlock::Fstat ( struct stat sbuff)
overridevirtual

Perform an fstat() operation (defaults to passthrough).

Parameters
sbuffreference to the stat buffer to be filled in. Only fields st_size, st_blocks, st_mtime (st_atime and st_ctime may be set to st_mtime), st_ino, and st_mode need to be set. All other fields are preset and should not be changed.
Returns
<0 - fstat failed, value is -errno. =0 - fstat succeeded, sbuff holds stat information. >0 - fstat could not be done, forward operation to next level.

Reimplemented from XrdOucCacheIO.

Definition at line 195 of file XrdPfcIOFileBlock.cc.

196 {
197  // local stat is create in constructor. if file was on disk before
198  // attach that the only way stat was not successful is becuse there
199  // were info file read errors
200  if ( ! m_localStat) return -ENOENT;
201 
202  memcpy(&sbuff, m_localStat, sizeof(struct stat));
203  return 0;
204 }
int stat(const char *path, struct stat *buf)

References stat().

+ Here is the call graph for this function:

◆ ioActive()

bool IOFileBlock::ioActive ( )
overridevirtual

Abstract virtual method of XrdPfc::IO Called to check if destruction needs to be done in a separate task.

Implements XrdPfc::IO.

Definition at line 78 of file XrdPfcIOFileBlock.cc.

79 {
80  // Called from XrdPosixFile when local connection is closed.
81 
83 
84  bool active = false;
85  {
86  XrdSysMutexHelper lock(&m_mutex);
87 
88  for (std::map<int, File*>::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it)
89  {
90  // Need to initiate stop on all File / block objects.
91  if (it->second && it->second->ioActive(this))
92  {
93  active = true;
94  }
95  }
96  }
97 
98  return active;
99 }
const char * RefreshLocation()
Definition: XrdPfcIO.hh:55

References XrdPfc::IO::RefreshLocation().

+ Here is the call graph for this function:

◆ Read() [1/3]

virtual int XrdOucCacheIO::Read

Pass Read request to the corresponding File object.

◆ Read() [2/3]

int IOFileBlock::Read ( char *  buff,
long long  offs,
int  rlen 
)
overridevirtual

Perform an synchronous read.

Parameters
buffpointer to the buffer to receive the results. The buffer must remain valid until the callback is invoked.
offsthe offset into the file.
rlenthe number of bytes to read.
Returns
< 0 - Read failed, value is -errno. >=0 - Read succeeded, value is number of bytes read.

Implements XrdOucCacheIO.

Definition at line 287 of file XrdPfcIOFileBlock.cc.

288 {
289  // protect from reads over the file size
290 
291  long long fileSize = FSize();
292 
293  if (off >= fileSize)
294  return 0;
295  if (off < 0)
296  {
297  return -EINVAL;
298  }
299  if (off + size > fileSize)
300  size = fileSize - off;
301 
302  long long off0 = off;
303  int idx_first = off0 / m_blocksize;
304  int idx_last = (off0 + size - 1) / m_blocksize;
305  int bytes_read = 0;
306  TRACEIO(Dump, "Read() "<< off << "@" << size << " block range ["<< idx_first << ", " << idx_last << "]");
307 
308  for (int blockIdx = idx_first; blockIdx <= idx_last; ++blockIdx)
309  {
310  // locate block
311  File *fb;
312  m_mutex.Lock();
313  std::map<int, File*>::iterator it = m_blocks.find(blockIdx);
314  if (it != m_blocks.end())
315  {
316  fb = it->second;
317  }
318  else
319  {
320  size_t pbs = m_blocksize;
321  // check if this is last block
322  int lastIOFileBlock = (fileSize-1)/m_blocksize;
323  if (blockIdx == lastIOFileBlock )
324  {
325  pbs = fileSize - blockIdx*m_blocksize;
326  // TRACEIO(Dump, "Read() last block, change output file size to " << pbs);
327  }
328 
329  // Note: File* can be 0 and stored as 0 if local open fails!
330  fb = newBlockFile(blockIdx*m_blocksize, pbs);
331  m_blocks.insert(std::make_pair(blockIdx, fb));
332  }
333  m_mutex.UnLock();
334 
335  // edit size if read request is reaching more than a block
336  int readBlockSize = size;
337  if (idx_first != idx_last)
338  {
339  if (blockIdx == idx_first)
340  {
341  readBlockSize = (blockIdx + 1) * m_blocksize - off0;
342  TRACEIO(Dump, "Read partially till the end of the block");
343  }
344  else if (blockIdx == idx_last)
345  {
346  readBlockSize = (off0 + size) - blockIdx * m_blocksize;
347  TRACEIO(Dump, "Read partially till the end of the block");
348  }
349  else
350  {
351  readBlockSize = m_blocksize;
352  }
353  }
354 
355  TRACEIO(Dump, "Read() block[ " << blockIdx << "] read-block-size[" << readBlockSize << "], offset[" << readBlockSize << "] off = " << off );
356 
357  int retvalBlock;
358  if (fb != 0)
359  {
360  struct ZHandler : public ReadReqRH
361  { using ReadReqRH::ReadReqRH;
362  XrdSysCondVar m_cond {0};
363  int m_retval {0};
364 
365  void Done(int result) override
366  { m_cond.Lock(); m_retval = result; m_cond.Signal(); m_cond.UnLock(); }
367  };
368 
369  ReadReqRHCond rh(ObtainReadSid(), nullptr);
370 
371  rh.m_cond.Lock();
372  retvalBlock = fb->Read(this, buff, off, readBlockSize, &rh);
373  if (retvalBlock == -EWOULDBLOCK)
374  {
375  rh.m_cond.Wait();
376  retvalBlock = rh.m_retval;
377  }
378  rh.m_cond.UnLock();
379  }
380  else
381  {
382  retvalBlock = GetInput()->Read(buff, off, readBlockSize);
383  }
384 
385  TRACEIO(Dump, "Read() Block read returned " << retvalBlock);
386  if (retvalBlock == readBlockSize)
387  {
388  bytes_read += retvalBlock;
389  buff += retvalBlock;
390  off += retvalBlock;
391  }
392  else if (retvalBlock >= 0)
393  {
394  TRACEIO(Warning, "Read() incomplete read, missing bytes " << readBlockSize-retvalBlock);
395  return -EIO;
396  }
397  else
398  {
399  TRACEIO(Error, "Read() read error, retval" << retvalBlock);
400  return retvalBlock;
401  }
402  }
403 
404  return bytes_read;
405 }
@ Warning
virtual int Read(char *buff, long long offs, int rlen)=0
int Read(IO *io, char *buff, long long offset, int size, ReadReqRH *rh)
Normal read.
Definition: XrdPfcFile.cc:748
long long FSize() override
XrdOucCacheIO * GetInput()
Definition: XrdPfcIO.cc:31
unsigned short ObtainReadSid()
Definition: XrdPfcIO.hh:57
ReadReqRH(unsigned short sid, XrdOucCacheIOCB *iocb)
Definition: XrdPfcFile.hh:56

References XrdPfc::ReadReqRH::ReadReqRH(), Macaroons::Error, FSize(), XrdPfc::IO::GetInput(), XrdSysCondVar::Lock(), XrdSysMutex::Lock(), XrdPfc::IO::ReadReqRHCond::m_cond, XrdPfc::IO::ReadReqRHCond::m_retval, XrdPfc::IO::ObtainReadSid(), XrdOucCacheIO::Read(), XrdPfc::File::Read(), TRACEIO, XrdSysCondVar::UnLock(), XrdSysMutex::UnLock(), XrdSysCondVar::Wait(), and Warning.

+ Here is the call graph for this function:

◆ Read() [3/3]

virtual void XrdOucCacheIO::Read
inline

Pass Read request to the corresponding File object.

Definition at line 333 of file XrdOucCache.hh.

334  {iocb.Done(Read(buff, offs, rlen));}
virtual int Read(char *buff, long long offs, int rlen)=0
Pass Read request to the corresponding File object.

◆ Update()

void IOFileBlock::Update ( XrdOucCacheIO iocp)
overridevirtual

Update the originally passed XrdOucCacheIO object with the object passed. All future uses underlying XrdOucCacheIO object must now use this object. Update() is called when Prepare() indicated that the file should not be physically opened and a file method was invoked in the XrdOucCacheIO passed to Attach(). When this occurs, the file is actually opened and Update() called to replace the original XrdOucCacheIO object with one that uses the newly opened file.

Parameters
iocpreference to the new XrdOucCacheIO object.

Reimplemented from XrdPfc::IO.

Definition at line 63 of file XrdPfcIOFileBlock.cc.

64 {
65  IO::Update(iocp);
66  {
67  XrdSysMutexHelper lock(&m_mutex);
68 
69  for (std::map<int, File*>::iterator it = m_blocks.begin(); it != m_blocks.end(); ++it)
70  {
71  // Need to update all File / block objects.
72  if (it->second) it->second->ioUpdated(this);
73  }
74  }
75 }
void Update(XrdOucCacheIO &iocp) override
Definition: XrdPfcIO.cc:17

References XrdPfc::IO::Update().

+ Here is the call graph for this function:

The documentation for this class was generated from the following files: