XRootD
XrdOssCsiPages Class Reference

#include <XrdOssCsiPages.hh>

+ Collaboration diagram for XrdOssCsiPages:

Public Types

typedef std::pair< off_t, off_t > Sizes_t
 

Public Member Functions

 XrdOssCsiPages (const std::string &fn, std::unique_ptr< XrdOssCsiTagstore > ts, bool wh, bool am, bool dpe, bool dlw, const char *)
 
 ~XrdOssCsiPages ()
 
void BasicConsistencyCheck (XrdOssDF *)
 
int Close ()
 
int FetchRange (XrdOssDF *, const void *, off_t, size_t, uint32_t *, uint64_t, XrdOssCsiRangeGuard &)
 
void Flush ()
 
int Fsync ()
 
bool IsReadOnly () const
 
int LockResetSizes (XrdOssDF *, off_t)
 
void LockTrackinglen (XrdOssCsiRangeGuard &, off_t, off_t, bool)
 
int Open (const char *path, off_t dsize, int flags, XrdOucEnv &envP)
 
int StoreRange (XrdOssDF *, const void *, off_t, size_t, uint32_t *, uint64_t, XrdOssCsiRangeGuard &)
 
void TrackedSizeRelease ()
 
int TrackedSizesGet (Sizes_t &, bool)
 
int truncate (XrdOssDF *, off_t, XrdOssCsiRangeGuard &)
 
int UpdateRange (XrdOssDF *, const void *, off_t, size_t, XrdOssCsiRangeGuard &)
 
int VerificationStatus ()
 
int VerifyRange (XrdOssDF *, const void *, off_t, size_t, XrdOssCsiRangeGuard &)
 

Static Public Member Functions

static void pgDoCalc (const void *, off_t, size_t, uint32_t *)
 
static int pgWritePrelockCheck (const void *, off_t, size_t, const uint32_t *, uint64_t)
 

Protected Member Functions

ssize_t apply_sequential_aligned_modify (const void *, off_t, size_t, const uint32_t *, bool, bool, uint32_t, uint32_t)
 
std::string ByteMismatchError (size_t blen, off_t off, uint8_t user, uint8_t page)
 
std::string CRCMismatchError (size_t blen, off_t pgnum, uint32_t got, uint32_t expected)
 
int FetchRangeAligned (const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
 
int FetchRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
 
int FetchRangeUnaligned_postblock (XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, size_t, uint64_t)
 
int FetchRangeUnaligned_preblock (XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, uint64_t)
 
int LockMakeUnverified ()
 
int LockSetTrackedSize (off_t)
 
int LockTruncateSize (off_t, bool)
 
std::string PageReadError (size_t blen, off_t pgnum, int ret)
 
int StoreRangeAligned (const void *, off_t, size_t, const Sizes_t &, uint32_t *)
 
int StoreRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, const uint32_t *)
 
int StoreRangeUnaligned_postblock (XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
 
int StoreRangeUnaligned_preblock (XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
 
std::string TagsReadError (off_t start, size_t n, int ret)
 
std::string TagsWriteError (off_t start, size_t n, int ret)
 
int UpdateRangeAligned (const void *, off_t, size_t, const Sizes_t &)
 
int UpdateRangeHoleUntilPage (XrdOssDF *, off_t, const Sizes_t &)
 
int UpdateRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)
 
int VerifyRangeAligned (const void *, off_t, size_t, const Sizes_t &)
 
int VerifyRangeUnaligned (XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)
 

Static Protected Member Functions

static ssize_t fullread (XrdOssDF *fd, void *buff, const off_t off, const size_t sz)
 
static ssize_t maxread (XrdOssDF *fd, void *buff, const off_t off, const size_t sz, size_t tg=0)
 

Protected Attributes

bool allowMissingTags_
 
bool checklastpg_
 
bool disablePgExtend_
 
const std::string fn_
 
bool hasMissingTags_
 
off_t lastpgforloose_
 
bool loosewrite_
 
const bool loosewriteConfigured_
 
XrdSysMutex rangeaddmtx_
 
XrdOssCsiRanges ranges_
 
bool rdonly_
 
const char * tident
 
const std::string tident_
 
std::unique_ptr< XrdOssCsiTagstorets_
 
XrdSysCondVar tscond_
 
bool tsforupdate_
 
bool writeHoles_
 

Static Protected Attributes

static const size_t stsize_ = 1024
 

Detailed Description

Definition at line 46 of file XrdOssCsiPages.hh.

Member Typedef Documentation

◆ Sizes_t

typedef std::pair<off_t,off_t> XrdOssCsiPages::Sizes_t

Definition at line 49 of file XrdOssCsiPages.hh.

Constructor & Destructor Documentation

◆ XrdOssCsiPages()

XrdOssCsiPages::XrdOssCsiPages ( const std::string &  fn,
std::unique_ptr< XrdOssCsiTagstore ts,
bool  wh,
bool  am,
bool  dpe,
bool  dlw,
const char *  tid 
)

Definition at line 44 of file XrdOssCsiPages.cc.

44  :
45  ts_(std::move(ts)),
46  writeHoles_(wh),
48  disablePgExtend_(dpe),
49  hasMissingTags_(false),
50  rdonly_(false),
52  loosewrite_(false),
53  tscond_(0),
54  tsforupdate_(false),
55  fn_(fn),
56  tident_(tid),
57  tident(tident_.c_str()),
58  lastpgforloose_(0),
59  checklastpg_(false)
60 {
61  // empty constructor
62 }
XrdSysCondVar tscond_
std::unique_ptr< XrdOssCsiTagstore > ts_
const char * tident
const std::string tident_
const bool loosewriteConfigured_
const std::string fn_

◆ ~XrdOssCsiPages()

XrdOssCsiPages::~XrdOssCsiPages ( )
inline

Definition at line 52 of file XrdOssCsiPages.hh.

52 { (void)Close(); }

References Close().

+ Here is the call graph for this function:

Member Function Documentation

◆ apply_sequential_aligned_modify()

ssize_t XrdOssCsiPages::apply_sequential_aligned_modify ( const void *  buff,
off_t  startp,
size_t  nbytes,
const uint32_t *  csvec,
bool  preblockset,
bool  lastblockset,
uint32_t  cspre,
uint32_t  cslast 
)
protected

Definition at line 288 of file XrdOssCsiPages.cc.

291 {
292  EPNAME("apply_sequential_aligned_modify");
293 
294  if (lastblockset && (nbytes % XrdSys::PageSize)==0)
295  {
296  return -EINVAL;
297  }
298  if (preblockset && startp==0)
299  {
300  return -EINVAL;
301  }
302 
303  uint32_t calcbuf[stsize_];
304  const size_t calcbufsz = sizeof(calcbuf)/sizeof(uint32_t);
305  const uint8_t *const p = (uint8_t*)buff;
306 
307  // will be using calcbuf
308  bool useinternal = true;
309  if (csvec && !preblockset && !lastblockset)
310  {
311  useinternal = false;
312  }
313 
314  bool dopre = preblockset;
315  const off_t sp = preblockset ? startp-1 : startp;
316 
317  size_t blktowrite = ((nbytes+XrdSys::PageSize-1)/XrdSys::PageSize) + (preblockset ? 1 : 0);
318  size_t nblkwritten = 0;
319  size_t calcbytot = 0;
320  while(blktowrite>0)
321  {
322  size_t blkwcnt = blktowrite;
323  if (useinternal)
324  {
325  size_t cidx = 0;
326  size_t calcbycnt = nbytes - calcbytot;
327  if (nblkwritten == 0 && dopre)
328  {
329  calcbycnt = std::min(calcbycnt, (calcbufsz-1)*XrdSys::PageSize);
330  blkwcnt = (calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize;
331  calcbuf[cidx] = cspre;
332  cidx++;
333  blkwcnt++;
334  dopre = false;
335  }
336  else
337  {
338  calcbycnt = std::min(calcbycnt, calcbufsz*XrdSys::PageSize);
339  blkwcnt = (calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize;
340  }
341  if ((calcbycnt % XrdSys::PageSize)!=0 && lastblockset)
342  {
343  const size_t x = calcbycnt / XrdSys::PageSize;
344  calcbycnt = XrdSys::PageSize * x;
345  calcbuf[cidx + x] = cslast;
346  }
347  if (csvec)
348  {
349  memcpy(&calcbuf[cidx], &csvec[calcbytot/XrdSys::PageSize], 4*((calcbycnt+XrdSys::PageSize-1)/XrdSys::PageSize));
350  }
351  else
352  {
353  XrdOucCRC::Calc32C(&p[calcbytot], calcbycnt, &calcbuf[cidx]);
354  }
355  calcbytot += calcbycnt;
356  }
357  const ssize_t wret = ts_->WriteTags(useinternal ? calcbuf : &csvec[nblkwritten], sp+nblkwritten, blkwcnt);
358  if (wret<0)
359  {
360  TRACE(Warn, TagsWriteError(sp+nblkwritten, blkwcnt, wret));
361  return wret;
362  }
363  blktowrite -= blkwcnt;
364  nblkwritten += blkwcnt;
365  }
366  return nblkwritten;
367 }
#define EPNAME(x)
Definition: XrdBwmTrace.hh:56
#define TRACE(act, x)
Definition: XrdTrace.hh:63
std::string TagsWriteError(off_t start, size_t n, int ret)
static const size_t stsize_
static uint32_t Calc32C(const void *data, size_t count, uint32_t prevcs=0)
Definition: XrdOucCRC.cc:190
static const int PageSize

References XrdOucCRC::Calc32C(), EPNAME, XrdSys::PageSize, stsize_, TagsWriteError(), TRACE, and ts_.

Referenced by StoreRangeAligned(), and StoreRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ BasicConsistencyCheck()

void XrdOssCsiPages::BasicConsistencyCheck ( XrdOssDF fd)

Definition at line 820 of file XrdOssCsiPages.cc.

821 {
822  EPNAME("BasicConsistencyCheck");
823 
824  if (!loosewrite_ || rdonly_) return;
825 
826  uint8_t b[XrdSys::PageSize];
827  static const uint8_t bz[XrdSys::PageSize] = {0};
828 
829  const off_t tagsize = ts_->GetTrackedTagSize();
830  const off_t datasize = ts_->GetTrackedDataSize();
831 
832  off_t taglp = 0, datalp = 0;
833  size_t tag_len = 0, data_len = 0;
834 
835  if (tagsize>0)
836  {
837  taglp = (tagsize - 1) / XrdSys::PageSize;
838  tag_len = tagsize % XrdSys::PageSize;
839  tag_len = tag_len ? tag_len : XrdSys::PageSize;
840  }
841  if (datasize>0)
842  {
843  datalp = (datasize - 1) / XrdSys::PageSize;
844  data_len = datasize % XrdSys::PageSize;
845  data_len = data_len ? data_len : XrdSys::PageSize;
846  }
847 
848  lastpgforloose_ = taglp;
849  checklastpg_ = true;
850 
851  if (datasize>0 && taglp > datalp)
852  {
853  ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * datalp, XrdSys::PageSize);
854  if (rlen<0)
855  {
856  TRACE(Warn, PageReadError(XrdSys::PageSize, datalp, rlen));
857  return;
858  }
859 
860  memset(&b[rlen], 0, XrdSys::PageSize-rlen);
861  const uint32_t data_crc = XrdOucCRC::Calc32C(b, data_len, 0u);
862  const uint32_t data_crc_z = XrdOucCRC::Calc32C(b, XrdSys::PageSize, 0u);
863  uint32_t tagv;
864  ssize_t rret = ts_->ReadTags(&tagv, datalp, 1);
865  if (rret<0)
866  {
867  TRACE(Warn, TagsReadError(datalp, 1, rret));
868  return;
869  }
870 
871  if (tagv == data_crc_z)
872  {
873  // expected
874  }
875  else if (tagv == data_crc)
876  {
877  // should set tagv to data_crc_z
878  TRACE(Warn, "Resetting tag for page at " << datalp*XrdSys::PageSize << " to zero-extended");
879  const ssize_t wret = ts_->WriteTags(&data_crc_z, datalp, 1);
880  if (wret < 0)
881  {
882  TRACE(Warn, TagsWriteError(datalp, 1, wret));
883  return;
884  }
885  }
886  else
887  {
888  // something else wrong
889  TRACE(Warn, CRCMismatchError(data_len, datalp, data_crc, tagv) << " (ignoring)");
890  }
891  }
892  else if (tagsize>0 && taglp < datalp)
893  {
894  // datafile has more pages than recorded in the tag file:
895  // the tag file should have a crc corresponding to the relevant data fragment that is tracked in the last page.
896  // If it has the crc for a whole page (and there no non-zero content later in the page) reset it.
897  // This is so that a subsequnt UpdateRangeHoleUntilPage can zero-extend the CRC and get a consistent CRC.
898 
899  ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * taglp, XrdSys::PageSize);
900  if (rlen<0)
901  {
902  TRACE(Warn, PageReadError(XrdSys::PageSize, taglp, rlen));
903  return;
904  }
905 
906  memset(&b[rlen], 0, XrdSys::PageSize-rlen);
907  const uint32_t tag_crc = XrdOucCRC::Calc32C(b, tag_len, 0u);
908  const uint32_t tag_crc_z = XrdOucCRC::Calc32C(b, XrdSys::PageSize, 0u);
909  const uint32_t dp_ext_is_zero = !memcmp(&b[tag_len], bz, XrdSys::PageSize-tag_len);
910  uint32_t tagv;
911  ssize_t rret = ts_->ReadTags(&tagv, taglp, 1);
912  if (rret<0)
913  {
914  TRACE(Warn, TagsReadError(taglp, 1, rret));
915  return;
916  }
917 
918  if (tagv == tag_crc)
919  {
920  // expected
921  }
922  else if (tagv == tag_crc_z && dp_ext_is_zero)
923  {
924  // should set tagv to tag_crc
925  TRACE(Warn, "Resetting tag for page at " << taglp*XrdSys::PageSize << " to not zero-extended");
926  const ssize_t wret = ts_->WriteTags(&tag_crc, taglp, 1);
927  if (wret < 0)
928  {
929  TRACE(Warn, TagsWriteError(taglp, 1, wret));
930  return;
931  }
932  }
933  else
934  {
935  // something else wrong
936  TRACE(Warn, CRCMismatchError(tag_len, taglp, tag_crc, tagv) << " dp_ext_is_zero=" << dp_ext_is_zero << " (ignoring)");
937  }
938  }
939 }
static ssize_t maxread(XrdOssDF *fd, void *buff, const off_t off, const size_t sz, size_t tg=0)
std::string TagsReadError(off_t start, size_t n, int ret)
std::string CRCMismatchError(size_t blen, off_t pgnum, uint32_t got, uint32_t expected)
std::string PageReadError(size_t blen, off_t pgnum, int ret)

References XrdOucCRC::Calc32C(), checklastpg_, CRCMismatchError(), EPNAME, lastpgforloose_, loosewrite_, maxread(), PageReadError(), XrdSys::PageSize, rdonly_, TagsReadError(), TagsWriteError(), TRACE, and ts_.

Referenced by LockResetSizes().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ ByteMismatchError()

std::string XrdOssCsiPages::ByteMismatchError ( size_t  blen,
off_t  off,
uint8_t  user,
uint8_t  page 
)
inlineprotected

Definition at line 161 of file XrdOssCsiPages.hh.

162  {
163  char buf[256],buf2[256];
164  snprintf(buf, sizeof(buf),
165  "unexpected byte mismatch between user-buffer and page/0x%04" PRIx32 " in file ",
166  (uint32_t)blen);
167  snprintf(buf2, sizeof(buf2),
168  " at offset 0x%" PRIx64 ", user-byte 0x%02" PRIx8 ", page-byte 0x%02" PRIx8,
169  (uint64_t)off,
170  user, page);
171  return buf + fn_ + buf2;
172  }

References fn_.

Referenced by FetchRangeUnaligned_postblock(), and FetchRangeUnaligned_preblock().

+ Here is the caller graph for this function:

◆ Close()

int XrdOssCsiPages::Close ( )

Definition at line 88 of file XrdOssCsiPages.cc.

89 {
90  if (hasMissingTags_)
91  {
92  hasMissingTags_ = false;
93  return 0;
94  }
95  return ts_->Close();
96 }

References hasMissingTags_, and ts_.

Referenced by ~XrdOssCsiPages().

+ Here is the caller graph for this function:

◆ CRCMismatchError()

std::string XrdOssCsiPages::CRCMismatchError ( size_t  blen,
off_t  pgnum,
uint32_t  got,
uint32_t  expected 
)
inlineprotected

Definition at line 148 of file XrdOssCsiPages.hh.

149  {
150  char buf[256],buf2[256];
151  snprintf(buf, sizeof(buf),
152  "bad crc32c/0x%04" PRIx32 " checksum in file ",
153  (uint32_t)blen);
154  snprintf(buf2, sizeof(buf2),
155  " at offset 0x%" PRIx64 ", got 0x%08" PRIx32 ", expected 0x%08" PRIx32,
156  (uint64_t)(pgnum*XrdSys::PageSize),
157  got, expected);
158  return buf + fn_ + buf2;
159  }

References fn_, and XrdSys::PageSize.

Referenced by BasicConsistencyCheck(), FetchRangeAligned(), FetchRangeUnaligned(), FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

+ Here is the caller graph for this function:

◆ FetchRange()

int XrdOssCsiPages::FetchRange ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
uint32_t *  csvec,
uint64_t  opts,
XrdOssCsiRangeGuard rg 
)

Definition at line 627 of file XrdOssCsiPages.cc.

630 {
631  EPNAME("FetchRange");
632  if (offset<0)
633  {
634  return -EINVAL;
635  }
636 
637  // if the tag file is missing there is nothing to fetch or verify
638  // but if we should return a list of checksums calculate them from the data
639  if (hasMissingTags_)
640  {
641  if (csvec)
642  {
643  pgDoCalc(buff, offset, blen, csvec);
644  }
645  return 0;
646  }
647 
648  const Sizes_t sizes = rg.getTrackinglens();
649  const off_t trackinglen = sizes.first;
650 
651  if (offset >= trackinglen && blen == 0)
652  {
653  return 0;
654  }
655 
656  if (blen == 0)
657  {
658  // if offset is before the tracked len we should not be requested to verify zero bytes:
659  // the file may have been truncated
660  TRACE(Warn, "Fetch request for zero bytes " << fn_ << ", file may be truncated");
661  return -EDOM;
662  }
663 
664  if (offset+blen > static_cast<size_t>(trackinglen))
665  {
666  TRACE(Warn, "Fetch request for " << (offset+blen-trackinglen) << " bytes from " << fn_ << " beyond tracked length");
667  return -EDOM;
668  }
669 
670  if (csvec == NULL && !(opts & XrdOssDF::Verify))
671  {
672  // if the crc values are not wanted nor checks against data, then
673  // there's nothing more to do here
674  return 0;
675  }
676 
677  int fret;
678  if ((offset % XrdSys::PageSize) != 0 || (offset+blen != static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0))
679  {
680  fret = FetchRangeUnaligned(fd, buff, offset, blen, sizes, csvec, opts);
681  }
682  else
683  {
684  fret = FetchRangeAligned(buff,offset,blen,sizes,csvec,opts);
685  }
686  return fret;
687 }
struct myOpts opts
int FetchRangeAligned(const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
std::pair< off_t, off_t > Sizes_t
int FetchRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, uint32_t *, uint64_t)
static void pgDoCalc(const void *, off_t, size_t, uint32_t *)
const std::pair< off_t, off_t > & getTrackinglens() const
static const uint64_t Verify
all: Verify checksums
Definition: XrdOss.hh:223

References EPNAME, FetchRangeAligned(), FetchRangeUnaligned(), fn_, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, opts, XrdSys::PageSize, pgDoCalc(), TRACE, and XrdOssDF::Verify.

Referenced by XrdOssCsiFileAioJob::DoItRead2(), and XrdOssCsiFile::pgRead().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeAligned()

int XrdOssCsiPages::FetchRangeAligned ( const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t ,
uint32_t *  csvec,
uint64_t  opts 
)
protected

Definition at line 375 of file XrdOssCsiPages.cc.

376 {
377  EPNAME("FetchRangeAligned");
378  uint32_t rdvec[stsize_],vrbuf[stsize_];
379 
380  const off_t p1 = offset / XrdSys::PageSize;
381  const off_t p2 = (offset+blen) / XrdSys::PageSize;
382  const size_t p2_off = (offset+blen) % XrdSys::PageSize;
383  const size_t nfull = p2-p1;
384 
385  uint32_t *rdbuf;
386  size_t rdbufsz;
387  if (csvec == NULL)
388  {
389  // use fixed sized stack buffer
390  rdbuf = rdvec;
391  rdbufsz = sizeof(rdvec)/sizeof(uint32_t);
392  }
393  else
394  {
395  // use supplied buffer, assumed to be large enough
396  rdbuf = csvec;
397  rdbufsz = (p2_off==0) ? nfull : (nfull+1);
398  }
399 
400  // always use stack based, fixed sized buffer for verify
401  const size_t vrbufsz = sizeof(vrbuf)/sizeof(uint32_t);
402 
403  // pointer to data
404  const uint8_t *const p = (uint8_t*)buff;
405 
406  // process full pages + any partial page
407  size_t toread = (p2_off>0) ? nfull+1 : nfull;
408  size_t nread = 0;
409  while(toread>0)
410  {
411  const size_t rcnt = std::min(toread, rdbufsz-(nread%rdbufsz));
412  const ssize_t rret = ts_->ReadTags(&rdbuf[nread%rdbufsz], p1+nread, rcnt);
413  if (rret<0)
414  {
415  TRACE(Warn, TagsReadError(p1+nread, rcnt, rret));
416  return rret;
417  }
418  if ((opts & XrdOssDF::Verify))
419  {
420  size_t toverif = rcnt;
421  size_t nverif = 0;
422  while(toverif>0)
423  {
424  const size_t vcnt = std::min(toverif, vrbufsz);
425  const size_t databytes = (nread+nverif+vcnt <= nfull) ? (vcnt*XrdSys::PageSize) : ((vcnt-1)*XrdSys::PageSize+p2_off);
426  XrdOucCRC::Calc32C(&p[XrdSys::PageSize*(nread+nverif)],databytes,vrbuf);
427  if (memcmp(vrbuf, &rdbuf[(nread+nverif)%rdbufsz], 4*vcnt))
428  {
429  size_t badpg;
430  for(badpg=0;badpg<vcnt;++badpg) { if (memcmp(&vrbuf[badpg],&rdbuf[(nread+nverif+badpg)%rdbufsz],4)) break; }
431  TRACE(Warn, CRCMismatchError( (nread+nverif+badpg<nfull) ? XrdSys::PageSize : p2_off,
432  (p1+nread+nverif+badpg),
433  vrbuf[badpg],
434  rdbuf[(nread+nverif+badpg)%rdbufsz] ));
435  return -EDOM;
436  }
437  toverif -= vcnt;
438  nverif += vcnt;
439  }
440  }
441  toread -= rcnt;
442  nread += rcnt;
443  }
444 
445  return 0;
446 }

References XrdOucCRC::Calc32C(), CRCMismatchError(), EPNAME, opts, XrdSys::PageSize, stsize_, TagsReadError(), TRACE, ts_, and XrdOssDF::Verify.

Referenced by FetchRange(), and VerifyRangeAligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeUnaligned()

int XrdOssCsiPages::FetchRangeUnaligned ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t sizes,
uint32_t *  csvec,
uint64_t  opts 
)
protected

Definition at line 786 of file XrdOssCsiPagesUnaligned.cc.

787 {
788  EPNAME("FetchRangeUnaligned");
789 
790  const off_t p1 = offset / XrdSys::PageSize;
791  const size_t p1_off = offset % XrdSys::PageSize;
792  const off_t p2 = (offset+blen) / XrdSys::PageSize;
793  const size_t p2_off = (offset+blen) % XrdSys::PageSize;
794 
795  const off_t trackinglen = sizes.first;
796 
797  size_t ntagstoread = (p2_off>0) ? p2-p1+1 : p2-p1;
798  size_t ntagsbase = p1;
799  uint32_t tbufint[stsize_], *tbuf=0;
800  size_t tbufsz = 0;
801  if (!csvec)
802  {
803  tbuf = tbufint;
804  tbufsz = sizeof(tbufint)/sizeof(uint32_t);
805  }
806  else
807  {
808  tbuf = csvec;
809  tbufsz = ntagstoread;
810  }
811 
812  size_t tcnt = std::min(ntagstoread, tbufsz);
813  ssize_t rret = ts_->ReadTags(tbuf, ntagsbase, tcnt);
814  if (rret<0)
815  {
816  TRACE(Warn, TagsReadError(ntagsbase, tcnt, rret) << " (first)");
817  return rret;
818  }
819  ntagstoread -= tcnt;
820 
821  // deal with partial first page
822  if ( p1_off>0 || blen < static_cast<size_t>(XrdSys::PageSize) )
823  {
824  const int ret = FetchRangeUnaligned_preblock(fd, buff, offset, blen, trackinglen, tbuf, csvec, opts);
825  if (ret<0)
826  {
827  return ret;
828  }
829  }
830 
831  // first (inclusive) and last (exclusive) full page
832  const off_t fp = (p1_off != 0) ? p1+1 : p1;
833  const off_t lp = p2;
834 
835  // verify full pages if wanted
836  if (fp<lp && (opts & XrdOssDF::Verify))
837  {
838  const uint8_t *const p = (uint8_t*)buff;
839  uint32_t calcbuf[stsize_];
840  const size_t cbufsz = sizeof(calcbuf)/sizeof(uint32_t);
841  size_t toread = lp-fp;
842  size_t nread = 0;
843  while(toread>0)
844  {
845  const size_t ccnt = std::min(toread, cbufsz);
846  XrdOucCRC::Calc32C(&p[(p1_off ? XrdSys::PageSize-p1_off : 0)+XrdSys::PageSize*nread],ccnt*XrdSys::PageSize,calcbuf);
847  size_t tovalid = ccnt;
848  size_t nvalid = 0;
849  while(tovalid>0)
850  {
851  const size_t tidx=fp+nread+nvalid - ntagsbase;
852  const size_t nv = std::min(tovalid, tbufsz-tidx);
853  if (nv == 0)
854  {
855  assert(csvec == NULL);
856  ntagsbase += tbufsz;
857  tcnt = std::min(ntagstoread, tbufsz);
858  rret = ts_->ReadTags(tbuf, ntagsbase, tcnt);
859  if (rret<0)
860  {
861  TRACE(Warn, TagsReadError(ntagsbase, tcnt, rret) << " (mid)");
862  return rret;
863  }
864  ntagstoread -= tcnt;
865  continue;
866  }
867  if (memcmp(&calcbuf[nvalid], &tbuf[tidx], 4*nv))
868  {
869  size_t badpg;
870  for(badpg=0;badpg<nv;badpg++) { if (memcmp(&calcbuf[nvalid+badpg], &tbuf[tidx+badpg],4)) break; }
872  (ntagsbase+tidx+badpg),
873  calcbuf[nvalid+badpg], tbuf[tidx+badpg]));
874  return -EDOM;
875  }
876  tovalid -= nv;
877  nvalid += nv;
878  }
879  toread -= ccnt;
880  nread += ccnt;
881  }
882  }
883 
884  // last partial page
885  if (p2>p1 && p2_off > 0)
886  {
887  // make sure we have last tag;
888  // (should already have all of them if we're returning them in csvec)
889  size_t tidx = p2 - ntagsbase;
890  if (tidx >= tbufsz)
891  {
892  assert(csvec == NULL);
893  tidx = 0;
894  ntagsbase = p2;
895  rret = ts_->ReadTags(tbuf, ntagsbase, 1);
896  if (rret<0)
897  {
898  TRACE(Warn, TagsReadError(ntagsbase, 1, rret) << " (last)");
899  return rret;
900  }
901  ntagstoread = 0;
902  }
903 
904  const int ret = FetchRangeUnaligned_postblock(fd, buff, offset, blen, trackinglen, tbuf, csvec, tidx, opts);
905  if (ret<0)
906  {
907  return ret;
908  }
909  }
910 
911  return 0;
912 }
int FetchRangeUnaligned_preblock(XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, uint64_t)
int FetchRangeUnaligned_postblock(XrdOssDF *, const void *, off_t, size_t, off_t, uint32_t *, uint32_t *, size_t, uint64_t)

References XrdOucCRC::Calc32C(), CRCMismatchError(), EPNAME, FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), opts, XrdSys::PageSize, stsize_, TagsReadError(), TRACE, ts_, and XrdOssDF::Verify.

Referenced by FetchRange(), and VerifyRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeUnaligned_postblock()

int XrdOssCsiPages::FetchRangeUnaligned_postblock ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
off_t  trackinglen,
uint32_t *  tbuf,
uint32_t *  csvec,
size_t  tidx,
uint64_t  opts 
)
protected

Definition at line 710 of file XrdOssCsiPagesUnaligned.cc.

712 {
713  EPNAME("FetchRangeUnaligned_postblock");
714 
715  const off_t p2 = (offset+blen) / XrdSys::PageSize;
716  const size_t p2_off = (offset+blen) % XrdSys::PageSize;
717 
718  // length of data in last (p2) page
719  const size_t bavail = std::min(trackinglen - (XrdSys::PageSize*p2), (off_t)XrdSys::PageSize);
720 
721  // how much of that data is not being returned
722  const size_t bremain = (p2_off < bavail) ? bavail-p2_off : 0;
723  uint8_t b[XrdSys::PageSize];
724  const uint8_t *ub = &((uint8_t*)buff)[blen-p2_off];
725  if (bremain>0)
726  {
727  const ssize_t rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize*p2, bavail);
728  if (rret<0)
729  {
730  TRACE(Warn, PageReadError(bavail, p2, rret));
731  return rret;
732  }
733  // if we're verifying make sure overlapping part of data just read matches user's buffer
734  if ((opts & XrdOssDF::Verify))
735  {
736  const uint8_t *const p = (uint8_t*)buff;
737  if (memcmp(&p[blen-p2_off], b, p2_off))
738  {
739  size_t badoff;
740  for(badoff=0;badoff<p2_off;badoff++) { if (p[blen-p2_off+badoff] != b[badoff]) break; }
741  badoff = (badoff < p2_off) ? badoff : 0; // may be possible with concurrent modification
742  TRACE(Warn, ByteMismatchError(bavail, XrdSys::PageSize*p2+badoff, p[blen-p2_off+badoff], b[badoff]));
743  return -EDOM;
744  }
745  }
746  ub = b;
747  }
748  if ((opts & XrdOssDF::Verify))
749  {
750  const uint32_t crc32calc = XrdOucCRC::Calc32C(ub, bavail, 0U);
751  if (tbuf[tidx] != crc32calc)
752  {
753  TRACE(Warn, CRCMismatchError(bavail, p2, crc32calc, tbuf[tidx]));
754  return -EDOM;
755  }
756  }
757  // if we're returning csvec and user only request part of page
758  // adjust the crc
759  if (csvec && bremain>0)
760  {
761  if ((opts & XrdOssDF::Verify))
762  {
763  // verified; calculate crc based on common part of page.
764  csvec[tidx] = XrdOucCRC::Calc32C(b, p2_off, 0u);
765  }
766  else
767  {
768  // recalculate crc based on recorded checksum and adjusting for part of data not returned.
769  // If either the returned data or the data not included in the user's request are
770  // corrupt the returned checksum and returned data will (probably) mismatch.
771 
772  const uint32_t crc32c = XrdOucCRC::Calc32C(&b[p2_off], bremain, 0u);
773  csvec[tidx] = CrcUtils.crc32c_split1(csvec[tidx], crc32c, bremain);
774  }
775  }
776 
777  return 0;
778 }
static XrdOssCsiCrcUtils CrcUtils
uint32_t crc32c(uint32_t crc, void const *buf, size_t len)
static uint32_t crc32c_split1(uint32_t crctot, uint32_t crc2, size_t len2)
std::string ByteMismatchError(size_t blen, off_t off, uint8_t user, uint8_t page)
static ssize_t fullread(XrdOssDF *fd, void *buff, const off_t off, const size_t sz)

References ByteMismatchError(), XrdOucCRC::Calc32C(), crc32c(), XrdOssCsiCrcUtils::crc32c_split1(), CRCMismatchError(), CrcUtils, EPNAME, fullread(), opts, PageReadError(), XrdSys::PageSize, TRACE, and XrdOssDF::Verify.

Referenced by FetchRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ FetchRangeUnaligned_preblock()

int XrdOssCsiPages::FetchRangeUnaligned_preblock ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
off_t  trackinglen,
uint32_t *  tbuf,
uint32_t *  csvec,
uint64_t  opts 
)
protected

Definition at line 622 of file XrdOssCsiPagesUnaligned.cc.

624 {
625  EPNAME("FetchRangeUnaligned_preblock");
626 
627  const off_t p1 = offset / XrdSys::PageSize;
628  const size_t p1_off = offset % XrdSys::PageSize;
629 
630  // bavail is length of data in this page
631  const size_t bavail = std::min(trackinglen - (XrdSys::PageSize*p1), (off_t)XrdSys::PageSize);
632 
633  // bcommon is length of data in this page that user wants
634  const size_t bcommon = std::min(bavail - p1_off, blen);
635 
636  uint8_t b[XrdSys::PageSize];
637  const uint8_t *ub = (uint8_t*)buff;
638  if (bavail>bcommon)
639  {
640  // will need more data to either verify or return crc of the user's data
641  // (in case of no verify and no csvec FetchRange() returns early)
642  const ssize_t rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize*p1, bavail);
643  if (rret<0)
644  {
645  TRACE(Warn, PageReadError(bavail, p1, rret));
646  return rret;
647  }
648  // if we're going to verify, make sure we just read the same overlapping data as that in the user's buffer
649  if ((opts & XrdOssDF::Verify))
650  {
651  if (memcmp(buff, &b[p1_off], bcommon))
652  {
653  size_t badoff;
654  for(badoff=0;badoff<bcommon;badoff++) { if (((uint8_t*)buff)[badoff] != b[p1_off+badoff]) break; }
655  badoff = (badoff < bcommon) ? badoff : 0; // may be possible with concurrent modification
656  TRACE(Warn, ByteMismatchError(bavail, XrdSys::PageSize*p1+p1_off+badoff, ((uint8_t*)buff)[badoff], b[p1_off+badoff]));
657  return -EDOM;
658  }
659  }
660  ub = b;
661  }
662  // verify; based on whole block, or user's buffer (if it contains the whole block)
663  if ((opts & XrdOssDF::Verify))
664  {
665  const uint32_t crc32calc = XrdOucCRC::Calc32C(ub, bavail, 0U);
666  if (tbuf[0] != crc32calc)
667  {
668  TRACE(Warn, CRCMismatchError(bavail, p1, crc32calc, tbuf[0]));
669  return -EDOM;
670  }
671  }
672 
673  // if we're returning csvec values and this first block
674  // needs adjustment because user requested a subset..
675  if (bavail>bcommon && csvec)
676  {
677  // make sure csvec[0] corresponds to only the data the user wanted, not whole page.
678  // if we have already verified the page + common part matches user's, take checksum of common.
679  // (Use local copy of page, perhaps less chance of accidental concurrent modification than buffer)
680  // Otherwise base on saved checksum.
681  if ((opts & XrdOssDF::Verify))
682  {
683  csvec[0] = XrdOucCRC::Calc32C(&b[p1_off], bcommon, 0u);
684  }
685  else
686  {
687  // calculate expected user checksum based on block's recorded checksum, adjusting
688  // for data not included in user's request. If either the returned data or the
689  // data not included in the user's request are corrupt the returned checksum and
690  // returned data will (probably) mismatch.
691 
692  // remove block data before p1_off from checksum
693  uint32_t crc32c = XrdOucCRC::Calc32C(b, p1_off, 0u);
694  csvec[0] = CrcUtils.crc32c_split2(csvec[0], crc32c, bavail-p1_off);
695 
696  // remove block data after p1_off+bcommon upto bavail
697  crc32c = XrdOucCRC::Calc32C(&b[p1_off+bcommon], bavail-p1_off-bcommon, 0u);
698  csvec[0] = CrcUtils.crc32c_split1(csvec[0], crc32c, bavail-p1_off-bcommon);
699  }
700  }
701  return 0;
702 }
static uint32_t crc32c_split2(uint32_t crctot, uint32_t crc1, size_t len2)

References ByteMismatchError(), XrdOucCRC::Calc32C(), crc32c(), XrdOssCsiCrcUtils::crc32c_split1(), XrdOssCsiCrcUtils::crc32c_split2(), CRCMismatchError(), CrcUtils, EPNAME, fullread(), opts, PageReadError(), XrdSys::PageSize, TRACE, and XrdOssDF::Verify.

Referenced by FetchRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ Flush()

void XrdOssCsiPages::Flush ( )

Definition at line 98 of file XrdOssCsiPages.cc.

99 {
100  if (!hasMissingTags_) ts_->Flush();
101 }

References hasMissingTags_, and ts_.

Referenced by XrdOssCsiFile::Flush().

+ Here is the caller graph for this function:

◆ Fsync()

int XrdOssCsiPages::Fsync ( void  )

Definition at line 103 of file XrdOssCsiPages.cc.

104 {
105  if (hasMissingTags_) return 0;
106  return ts_->Fsync();
107 }

References hasMissingTags_, and ts_.

Referenced by XrdOssCsiFile::Fsync().

+ Here is the caller graph for this function:

◆ fullread()

static ssize_t XrdOssCsiPages::fullread ( XrdOssDF fd,
void *  buff,
const off_t  off,
const size_t  sz 
)
inlinestaticprotected

Definition at line 123 of file XrdOssCsiPages.hh.

124  {
125  ssize_t rret = maxread(fd, buff, off, sz);
126  if (rret<0) return rret;
127  if (static_cast<size_t>(rret) != sz) return -EDOM;
128  return rret;
129  }

References maxread().

Referenced by FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ IsReadOnly()

bool XrdOssCsiPages::IsReadOnly ( ) const
inline

Definition at line 68 of file XrdOssCsiPages.hh.

68 { return rdonly_; }

References rdonly_.

Referenced by XrdOssCsiFile::Open().

+ Here is the caller graph for this function:

◆ LockMakeUnverified()

int XrdOssCsiPages::LockMakeUnverified ( )
protected

Definition at line 152 of file XrdOssCsiPages.cc.

153 {
155  return ts_->SetUnverified();
156 }

References ts_, and tscond_.

Referenced by StoreRange(), and UpdateRange().

+ Here is the caller graph for this function:

◆ LockResetSizes()

int XrdOssCsiPages::LockResetSizes ( XrdOssDF fd,
off_t  sz 
)

Definition at line 134 of file XrdOssCsiPages.cc.

135 {
136  // nothing to do is no tag file
137  if (hasMissingTags_) return 0;
138 
140  const int ret = ts_->ResetSizes(sz);
143  return ret;
144 }
void BasicConsistencyCheck(XrdOssDF *)

References BasicConsistencyCheck(), hasMissingTags_, loosewrite_, loosewriteConfigured_, ts_, and tscond_.

+ Here is the call graph for this function:

◆ LockSetTrackedSize()

int XrdOssCsiPages::LockSetTrackedSize ( off_t  sz)
protected

Definition at line 128 of file XrdOssCsiPages.cc.

129 {
131  return ts_->SetTrackedSize(sz);
132 }

References ts_, and tscond_.

Referenced by StoreRange(), and UpdateRange().

+ Here is the caller graph for this function:

◆ LockTrackinglen()

void XrdOssCsiPages::LockTrackinglen ( XrdOssCsiRangeGuard rg,
off_t  offset,
off_t  offend,
bool  rdonly 
)

Definition at line 499 of file XrdOssCsiPages.cc.

500 {
501  // no need to lock if we don't have a tag file
502  if (hasMissingTags_) return;
503 
504  // in case of empty range the tracking len is not copied
505  if (offset == offend) return;
506 
507  {
509 
510  Sizes_t sizes;
511  (void)TrackedSizesGet(sizes, !rdonly);
512 
513  // tag tracking data filesize, as recorded in the tagfile and for which the tagfile
514  // should be approprately sized, is sizes.first: usually the same as the in
515  // memory "actual" data filesize (sizes.second), but may differ after crashes or write failure.
516  const off_t trackinglen = sizes.first;
517 
518  const off_t p1 = (offset>trackinglen ? trackinglen : offset) / XrdSys::PageSize;
519  bool unlock = false;
520  if (!rdonly && offend <= trackinglen)
521  {
522  unlock = true;
523  }
524 
525  off_t p2 = offend / XrdSys::PageSize;
526  const size_t p2_off = offend % XrdSys::PageSize;
527 
528  // range is exclusive
529  if (p2_off ==0) p2--;
530 
531  ranges_.AddRange(p1, p2, rg, rdonly);
532 
533  if (unlock)
534  {
536  }
537  rg.SetTrackingInfo(this, sizes, (!rdonly && !unlock));
538  }
539 
540  rg.Wait();
541 }
XrdOssCsiRanges ranges_
void TrackedSizeRelease()
XrdSysMutex rangeaddmtx_
int TrackedSizesGet(Sizes_t &, bool)
void SetTrackingInfo(XrdOssCsiPages *p, const std::pair< off_t, off_t > &tsizes, bool locked)
void AddRange(const off_t start, const off_t end, XrdOssCsiRangeGuard &rg, bool rdonly)

References XrdOssCsiRanges::AddRange(), hasMissingTags_, XrdSys::PageSize, rangeaddmtx_, ranges_, XrdOssCsiRangeGuard::SetTrackingInfo(), TrackedSizeRelease(), TrackedSizesGet(), and XrdOssCsiRangeGuard::Wait().

Referenced by XrdOssCsiFileAioJob::DoItRead1(), XrdOssCsiFileAioJob::DoItWrite1(), XrdOssCsiFile::Ftruncate(), XrdOssCsiFile::pgRead(), XrdOssCsiFile::pgWrite(), XrdOssCsiFile::Read(), XrdOssCsiFile::ReadRaw(), XrdOssCsiFile::ReadV(), XrdOssCsiFile::Write(), and XrdOssCsiFile::WriteV().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ LockTruncateSize()

int XrdOssCsiPages::LockTruncateSize ( off_t  sz,
bool  datatoo 
)
protected

Definition at line 146 of file XrdOssCsiPages.cc.

147 {
149  return ts_->Truncate(sz,datatoo);
150 }

References ts_, and tscond_.

Referenced by truncate().

+ Here is the caller graph for this function:

◆ maxread()

static ssize_t XrdOssCsiPages::maxread ( XrdOssDF fd,
void *  buff,
const off_t  off,
const size_t  sz,
size_t  tg = 0 
)
inlinestaticprotected

Definition at line 132 of file XrdOssCsiPages.hh.

133  {
134  size_t toread = sz, nread = 0;
135  uint8_t *p = (uint8_t*)buff;
136  tg = tg ? tg : sz;
137  while(toread>0 && nread<tg)
138  {
139  const ssize_t rret = fd->Read(&p[nread], off+nread, toread);
140  if (rret<0) return rret;
141  if (rret==0) break;
142  toread -= rret;
143  nread += rret;
144  }
145  return nread;
146  }
virtual ssize_t Read(off_t offset, size_t size)
Definition: XrdOss.hh:281

References XrdOssDF::Read().

Referenced by BasicConsistencyCheck(), fullread(), and StoreRangeUnaligned_preblock().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ Open()

int XrdOssCsiPages::Open ( const char *  path,
off_t  dsize,
int  flags,
XrdOucEnv envP 
)

Definition at line 64 of file XrdOssCsiPages.cc.

65 {
66  EPNAME("Pages::Open");
67  hasMissingTags_ = false;
68  rdonly_ = false;
69  int ret = ts_->Open(path, dsize, flags, envP);
70  if (ret == -ENOENT)
71  {
72  // no existing tag
74  {
75  TRACE(Info, "Opening with missing tagfile: " << fn_);
76  hasMissingTags_ = true;
77  return 0;
78  }
79  TRACE(Warn, "Could not open tagfile for " << fn_ << " error " << ret);
80  return -EDOM;
81  }
82  if (ret<0) return ret;
83  if ((flags & O_ACCMODE) == O_RDONLY) rdonly_ = true;
84  loosewrite_ = (dsize==0 && ts_->GetTrackedTagSize()==0) ? false : loosewriteConfigured_;
85  return 0;
86 }
@ Info
XrdOucEnv * envP
Definition: XrdPss.cc:109

References allowMissingTags_, XrdProxy::envP, EPNAME, fn_, hasMissingTags_, Info, loosewrite_, loosewriteConfigured_, rdonly_, TRACE, and ts_.

◆ PageReadError()

std::string XrdOssCsiPages::PageReadError ( size_t  blen,
off_t  pgnum,
int  ret 
)
inlineprotected

Definition at line 174 of file XrdOssCsiPages.hh.

175  {
176  char buf[256],buf2[256];
177  snprintf(buf, sizeof(buf),
178  "error %d while reading page/0x%04" PRIx32 " in file ",
179  ret, (uint32_t)blen);
180  snprintf(buf2, sizeof(buf2),
181  " at offset 0x%" PRIx64,
182  (uint64_t)(pgnum*XrdSys::PageSize));
183  return buf + fn_ + buf2;
184  }

References fn_, and XrdSys::PageSize.

Referenced by BasicConsistencyCheck(), FetchRangeUnaligned_postblock(), FetchRangeUnaligned_preblock(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), and truncate().

+ Here is the caller graph for this function:

◆ pgDoCalc()

void XrdOssCsiPages::pgDoCalc ( const void *  buffer,
off_t  offset,
size_t  wrlen,
uint32_t *  csvec 
)
static

Definition at line 777 of file XrdOssCsiPages.cc.

778 {
779  const size_t p_off = offset % XrdSys::PageSize;
780  const size_t p_alen = (p_off > 0) ? (XrdSys::PageSize - p_off) : wrlen;
781  if (p_alen < wrlen)
782  {
783  XrdOucCRC::Calc32C((uint8_t *)buffer+p_alen, wrlen-p_alen, &csvec[1]);
784  }
785  XrdOucCRC::Calc32C((void*)buffer, std::min(p_alen, wrlen), csvec);
786 }

References XrdOucCRC::Calc32C(), and XrdSys::PageSize.

Referenced by FetchRange(), and StoreRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ pgWritePrelockCheck()

int XrdOssCsiPages::pgWritePrelockCheck ( const void *  buffer,
off_t  offset,
size_t  wrlen,
const uint32_t *  csvec,
uint64_t  opts 
)
static

Definition at line 788 of file XrdOssCsiPages.cc.

789 {
790  // do verify before taking locks to allow for faster fail
791  if (csvec && (opts & XrdOssDF::Verify))
792  {
793  uint32_t valcs;
794  const size_t p_off = offset % XrdSys::PageSize;
795  const size_t p_alen = (p_off > 0) ? (XrdSys::PageSize - p_off) : wrlen;
796  if (p_alen < wrlen)
797  {
798  if (XrdOucCRC::Ver32C((uint8_t *)buffer+p_alen, wrlen-p_alen, &csvec[1], valcs)>=0)
799  {
800  return -EDOM;
801  }
802  }
803  if (XrdOucCRC::Ver32C((void*)buffer, std::min(p_alen, wrlen), csvec, valcs)>=0)
804  {
805  return -EDOM;
806  }
807  }
808 
809  return 0;
810 }
static bool Ver32C(const void *data, size_t count, const uint32_t csval, uint32_t *csbad=0)
Definition: XrdOucCRC.cc:222

References opts, XrdSys::PageSize, XrdOucCRC::Ver32C(), and XrdOssDF::Verify.

Referenced by XrdOssCsiFile::pgWrite().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRange()

int XrdOssCsiPages::StoreRange ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
uint32_t *  csvec,
uint64_t  opts,
XrdOssCsiRangeGuard rg 
)

Definition at line 691 of file XrdOssCsiPages.cc.

692 {
693  if (offset<0)
694  {
695  return -EINVAL;
696  }
697 
698  if (blen == 0)
699  {
700  return 0;
701  }
702 
703  // if the tag file is missing there is nothing to store
704  // but do calculate checksums to return, if requested to do so
705  if (hasMissingTags_)
706  {
707  if (csvec && (opts & XrdOssDF::doCalc))
708  {
709  pgDoCalc(buff, offset, blen, csvec);
710  }
711  return 0;
712  }
713 
714  const Sizes_t sizes = rg.getTrackinglens();
715  const off_t trackinglen = sizes.first;
716 
717  // in the original specification of pgWrite there was the idea of a logical-eof, set by
718  // the a pgWrite with non-page aligned length: We support an option to approximate that
719  // by disallowing pgWrite past the current (non page aligned) eof.
720  if (disablePgExtend_ && (trackinglen % XrdSys::PageSize) !=0 && offset+blen > static_cast<size_t>(trackinglen))
721  {
722  return -ESPIPE;
723  }
724 
725  // if doCalc is set and we have a csvec buffer fill it with calculated values
726  if (csvec && (opts & XrdOssDF::doCalc))
727  {
728  pgDoCalc(buff, offset, blen, csvec);
729  }
730 
731  // if no vector of crc have been given and not specifically requested to calculate,
732  // then mark this file as having unverified checksums
733  if (!csvec && !(opts & XrdOssDF::doCalc))
734  {
736  }
737 
738  if (offset+blen > static_cast<size_t>(trackinglen))
739  {
740  LockSetTrackedSize(offset+blen);
741  rg.unlockTrackinglen();
742  }
743 
744  int ret;
745  if ((offset % XrdSys::PageSize) != 0 ||
746  (offset+blen < static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0) ||
747  ((trackinglen % XrdSys::PageSize) !=0 && offset > trackinglen))
748  {
749  ret = StoreRangeUnaligned(fd,buff,offset,blen,sizes,csvec);
750  }
751  else
752  {
753  ret = StoreRangeAligned(buff,offset,blen,sizes,csvec);
754  }
755 
756  return ret;
757 }
int StoreRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &, const uint32_t *)
int StoreRangeAligned(const void *, off_t, size_t, const Sizes_t &, uint32_t *)
int LockSetTrackedSize(off_t)
static const uint64_t doCalc
pgw: Calculate checksums
Definition: XrdOss.hh:225

References disablePgExtend_, XrdOssDF::doCalc, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockMakeUnverified(), LockSetTrackedSize(), opts, XrdSys::PageSize, pgDoCalc(), StoreRangeAligned(), StoreRangeUnaligned(), and XrdOssCsiRangeGuard::unlockTrackinglen().

Referenced by XrdOssCsiFileAioJob::DoItWrite1(), and XrdOssCsiFile::pgWrite().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeAligned()

int XrdOssCsiPages::StoreRangeAligned ( const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t sizes,
uint32_t *  csvec 
)
protected

Definition at line 453 of file XrdOssCsiPages.cc.

454 {
455  EPNAME("StoreRangeAligned");
456 
457  // if csvec given store those values
458  // if no csvec then calculate against data and store
459 
460  const off_t p1 = offset / XrdSys::PageSize;
461  const off_t trackinglen = sizes.first;
462 
463  if (offset > trackinglen)
464  {
465  const int ret = UpdateRangeHoleUntilPage(NULL, p1, sizes);
466  if (ret<0)
467  {
468  TRACE(Warn, "Error updating tags for holes, error=" << ret);
469  return ret;
470  }
471  }
472 
473  const ssize_t aret = apply_sequential_aligned_modify(buff, p1, blen, csvec, false, false, 0U, 0U);
474  if (aret<0)
475  {
476  TRACE(Warn, "Error updating tags, error=" << aret);
477  return aret;
478  }
479 
480  return 0;
481 }
ssize_t apply_sequential_aligned_modify(const void *, off_t, size_t, const uint32_t *, bool, bool, uint32_t, uint32_t)
int UpdateRangeHoleUntilPage(XrdOssDF *, off_t, const Sizes_t &)

References apply_sequential_aligned_modify(), EPNAME, XrdSys::PageSize, TRACE, and UpdateRangeHoleUntilPage().

Referenced by StoreRange(), and UpdateRangeAligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeUnaligned()

int XrdOssCsiPages::StoreRangeUnaligned ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t sizes,
const uint32_t *  csvec 
)
protected

Definition at line 515 of file XrdOssCsiPagesUnaligned.cc.

516 {
517  EPNAME("StoreRangeUnaligned");
518  const off_t p1 = offset / XrdSys::PageSize;
519 
520  const off_t trackinglen = sizes.first;
521  if (offset > trackinglen)
522  {
523  const int ret = UpdateRangeHoleUntilPage(fd, p1, sizes);
524  if (ret<0)
525  {
526  TRACE(Warn, "Error updating tags for holes, error=" << ret);
527  return ret;
528  }
529  }
530 
531  const size_t p1_off = offset % XrdSys::PageSize;
532  const size_t p2_off = (offset+blen) % XrdSys::PageSize;
533 
534  bool hasprepage = false;
535  uint32_t prepageval;
536 
537  // deal with partial first page
538  if ( p1_off>0 || blen < static_cast<size_t>(XrdSys::PageSize) )
539  {
540  const size_t bavail = (XrdSys::PageSize-p1_off > blen) ? blen : (XrdSys::PageSize-p1_off);
541  const int ret = StoreRangeUnaligned_preblock(fd, buff, bavail, offset, trackinglen, csvec, prepageval);
542  if (ret<0)
543  {
544  return ret;
545  }
546  hasprepage = true;
547  }
548 
549  // next page (if any)
550  const off_t np = hasprepage ? p1+1 : p1;
551  // next page starts at buffer offset
552  const size_t npoff = hasprepage ? (XrdSys::PageSize - p1_off) : 0;
553 
554  // anything in next page?
555  if (blen <= npoff)
556  {
557  // only need to write the first, partial page
558  if (hasprepage)
559  {
560  const ssize_t wret = ts_->WriteTags(&prepageval, p1, 1);
561  if (wret<0)
562  {
563  TRACE(Warn, TagsWriteError(p1, 1, wret));
564  return wret;
565  }
566  }
567  return 0;
568  }
569 
570  const uint8_t *const p = (uint8_t*)buff;
571  const uint32_t *csp = csvec;
572  if (csp && hasprepage) csp++;
573 
574  // see if there will be no old data to account for in the last page
575  if (p2_off == 0 || (offset + blen >= static_cast<size_t>(trackinglen)))
576  {
577  // write any precomputed prepage, then write full pages and last partial page (computing or using supplied csvec)
578  const ssize_t aret = apply_sequential_aligned_modify(&p[npoff], np, blen-npoff, csp, hasprepage, false, prepageval, 0U);
579  if (aret<0)
580  {
581  TRACE(Warn, "Error updating tags, error=" << aret);
582  return aret;
583  }
584  return 0;
585  }
586 
587  // last page contains existing data that has to be read to modify it
588 
589  uint32_t lastpageval;
590  const int ret = StoreRangeUnaligned_postblock(fd, &p[npoff], blen-npoff, offset+npoff, trackinglen, csp, lastpageval);
591  if (ret<0)
592  {
593  return ret;
594  }
595 
596  // write any precomputed prepage, then write full pages (computing or using supplied csvec) and finally write precomputed last page
597  const ssize_t aret = apply_sequential_aligned_modify(&p[npoff], np, blen-npoff, csp, hasprepage, true, prepageval, lastpageval);
598  if (aret<0)
599  {
600  TRACE(Warn, "Error updating tags, error=" << aret);
601  return aret;
602  }
603 
604  return 0;
605 }
int StoreRangeUnaligned_preblock(XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)
int StoreRangeUnaligned_postblock(XrdOssDF *, const void *, size_t, off_t, off_t, const uint32_t *, uint32_t &)

References apply_sequential_aligned_modify(), EPNAME, XrdSys::PageSize, StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), TagsWriteError(), TRACE, ts_, and UpdateRangeHoleUntilPage().

Referenced by StoreRange(), and UpdateRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeUnaligned_postblock()

int XrdOssCsiPages::StoreRangeUnaligned_postblock ( XrdOssDF fd,
const void *  buff,
size_t  blen,
off_t  offset,
off_t  trackinglen,
const uint32_t *  csvec,
uint32_t &  lastpageval 
)
protected

Definition at line 414 of file XrdOssCsiPagesUnaligned.cc.

417 {
418  EPNAME("StoreRangeUnaligned_postblock");
419 
420  const uint8_t *const p = (uint8_t*)buff;
421  const off_t p2 = (offset+blen) / XrdSys::PageSize;
422  const size_t p2_off = (offset+blen) % XrdSys::PageSize;
423 
424  const off_t tracked_page = trackinglen / XrdSys::PageSize;
425  const size_t tracked_off = trackinglen % XrdSys::PageSize;
426 
427  // we should not be called in this case
428  assert(p2_off != 0);
429 
430  // how much existing data this last (p2) page
431  const size_t bavail = (p2==tracked_page) ? tracked_off : XrdSys::PageSize;
432 
433  // how much of that data will not be overwritten
434  const size_t bremain = (p2_off < bavail) ? bavail-p2_off : 0;
435 
436  // we should not be called if it is a complete overwrite
437  assert(bremain>0);
438 
439  // need to use remaining data to calculate the crc of the new p2 page.
440  // read and verify it now.
441 
442  uint32_t crc32v;
443  ssize_t rret = ts_->ReadTags(&crc32v, p2, 1);
444  if (rret<0)
445  {
446  TRACE(Warn, TagsReadError(p2, 1, rret));
447  return rret;
448  }
449 
450  uint8_t b[XrdSys::PageSize];
451  rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize * p2, bavail);
452  if (rret<0)
453  {
454  TRACE(Warn, PageReadError(bavail, p2, rret));
455  return rret;
456  }
457 
458  // calculate crc of new data with remaining data at the end:
459  uint32_t crc32c = 0;
460  if (csvec)
461  {
462  crc32c = csvec[(blen-1)/XrdSys::PageSize];
463  }
464  else
465  {
466  crc32c = XrdOucCRC::Calc32C(&p[blen-p2_off], p2_off, 0U);
467  }
468 
469  const uint32_t cl = XrdOucCRC::Calc32C(&b[p2_off], bremain, 0U);
470  // crc of page with new data
471  crc32c = CrcUtils.crc32c_combine(crc32c, cl, bremain);
472  // crc of current page (before write)
473  const uint32_t crc32prev = XrdOucCRC::Calc32C(b, bavail, 0U);
474 
475  // check(s) to see if remaining data was valid
476 
477  // usual check; unmodified block is consistent with stored crc
478  // for loose write we allow case were the new crc has already been stored in the tagfile
479 
480  // this may be an implicit verification (e.g. pgWrite may return EDOM without Verify requested)
481  // however, it's not clear if there is a meaningful way to crc a mismatching page during a partial overwrite
482  if (crc32v != crc32prev)
483  {
484  if (loosewrite_ && crc32c != crc32prev)
485  {
486  // log that we chceked if the tag was matching the previous data
487  TRACE(Warn, CRCMismatchError(bavail, p2, crc32prev, crc32v) << " (loose match, still trying)");
488  if (crc32c == crc32v)
489  {
490  TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p2) << " of file " << fn_);
491  lastpageval = crc32c;
492  return 0;
493  }
494  TRACE(Warn, CRCMismatchError(bavail, p2, crc32c, crc32v));
495  }
496  else
497  {
498  TRACE(Warn, CRCMismatchError(bavail, p2, crc32prev, crc32v));
499  }
500  return -EDOM;
501  }
502 
503  lastpageval = crc32c;
504  return 0;
505 }
static uint32_t crc32c_combine(uint32_t crc1, uint32_t crc2, size_t len2)

References XrdOucCRC::Calc32C(), crc32c(), XrdOssCsiCrcUtils::crc32c_combine(), CRCMismatchError(), CrcUtils, EPNAME, fn_, fullread(), loosewrite_, PageReadError(), XrdSys::PageSize, TagsReadError(), TRACE, and ts_.

Referenced by StoreRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ StoreRangeUnaligned_preblock()

int XrdOssCsiPages::StoreRangeUnaligned_preblock ( XrdOssDF fd,
const void *  buff,
size_t  blen,
off_t  offset,
off_t  trackinglen,
const uint32_t *  csvec,
uint32_t &  prepageval 
)
protected

Definition at line 131 of file XrdOssCsiPagesUnaligned.cc.

134 {
135  EPNAME("StoreRangeUnaligned_preblock");
136  const off_t p1 = offset / XrdSys::PageSize;
137  const size_t p1_off = offset % XrdSys::PageSize;
138 
139  const off_t tracked_page = trackinglen / XrdSys::PageSize;
140  const size_t tracked_off = trackinglen % XrdSys::PageSize;
141 
142  if (p1 > tracked_page)
143  {
144  // the start of will have a number of implied zero bytes
145  uint32_t crc32c = CrcUtils.crc32c_extendwith_zero(0u, p1_off);
146  if (csvec)
147  {
148  crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
149  }
150  else
151  {
152  crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
153  }
154  prepageval = crc32c;
155  return 0;
156  }
157 
158  // we're appending, or appending within the last page after a gap of zeros
159  if (p1 == tracked_page && p1_off >= tracked_off)
160  {
161  // appending: with or without some implied zeros.
162 
163  // zero initialised value may be used
164  uint32_t crc32v = 0;
165  if (tracked_off > 0)
166  {
167  const ssize_t rret = ts_->ReadTags(&crc32v, p1, 1);
168  if (rret<0)
169  {
170  TRACE(Warn, TagsReadError(p1, 1, rret) << " (append)");
171  return rret;
172  }
173  }
174 
175  uint32_t crc32c = 0;
176 
177  // only do the loosewrite extending check one time for the page which was the
178  // last page according to the trackinglen at time the check was configured (open or size-resync).
179  // don't do the check every time because it needs an extra read compared to the non loose case;
180  // checklastpg_ is checked and modified here, but is protected from concurrent
181  // access because of the condition that p1==lastpgforloose_
182 
184  {
185  checklastpg_ = false;
186  uint8_t b[XrdSys::PageSize];
187 
188  // this will reissue read() until eof, or tracked_off bytes read but accept up to PageSize
189  const ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * p1, XrdSys::PageSize, tracked_off);
190 
191  if (rlen<0)
192  {
193  TRACE(Warn, PageReadError(tracked_off, p1, rlen));
194  return rlen;
195  }
196  memset(&b[rlen], 0, XrdSys::PageSize - rlen);
197 
198  // in the loose-write mode, the new crc is based on the crc of data
199  // read from file up to p1_off, not on the previously stored tag.
200  // However must check if the data read were consistent with stored tag (crc32v)
201 
202  uint32_t crc32x = XrdOucCRC::Calc32C(b, tracked_off, 0u);
203  crc32c = XrdOucCRC::Calc32C(&b[tracked_off], p1_off-tracked_off, crc32x);
204 
205  do
206  {
207  if (static_cast<size_t>(rlen) == tracked_off)
208  {
209  // this is the expected match
210  if (tracked_off==0 || crc32x == crc32v) break;
211  }
212 
213  // any bytes on disk beyond p1_off+blan would not be included in the new crc.
214  // if tracked_off==0 we have no meaningful crc32v value.
215  if ((tracked_off>0 || p1_off==0) && static_cast<size_t>(rlen) <= p1_off+blen)
216  {
217 
218  if (tracked_off != 0)
219  {
220  TRACE(Warn, CRCMismatchError(tracked_off, p1, crc32x, crc32v) << " (loose match, still trying)");
221  }
222 
223  // there was no tag recorded for the page, and we're completely overwriting anything on disk in the page
224  if (tracked_off==0)
225  {
226  TRACE(Warn, "Recovered page with no tag at offset " << (XrdSys::PageSize * p1) <<
227  " of file " << fn_ << " rlen=" << rlen << " (append)");
228  break;
229  }
230 
231  if (static_cast<size_t>(rlen) != tracked_off && rlen>0)
232  {
233  crc32x = XrdOucCRC::Calc32C(b, rlen, 0u);
234  if (crc32x == crc32v)
235  {
236  TRACE(Warn, "Recovered page at offset " << (XrdSys::PageSize * p1)+p1_off << " of file " << fn_ << " (append)");
237  break;
238  }
239  TRACE(Warn, CRCMismatchError(rlen, p1, crc32x, crc32v) << " (loose match, still trying)");
240  }
241 
242  memcpy(&b[p1_off], buff, blen);
243  crc32x = XrdOucCRC::Calc32C(b, p1_off+blen, 0u);
244  if (crc32x == crc32v)
245  {
246  TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p1)+p1_off <<
247  " of file " << fn_ << " (append)");
248  break;
249  }
250  TRACE(Warn, CRCMismatchError(p1_off+blen, p1, crc32x, crc32v) << " (append)");
251  }
252  else
253  {
254  if (tracked_off>0)
255  {
256  TRACE(Warn, CRCMismatchError(tracked_off, p1, crc32x, crc32v) << " (append)");
257  }
258  else
259  {
260  TRACE(Warn, "Unexpected content, write at page at offset " << (XrdSys::PageSize * p1) <<
261  " of file " << fn_ << ", offset-in-page=" << p1_off << " rlen=" << rlen << " (append)");
262  }
263  }
264  return -EDOM;
265  } while(0);
266  }
267  else
268  {
269  // non-loose case;
270  // can recalc crc with new data without re-reading existing partial block's data
271  const size_t nz = p1_off - tracked_off;
272  crc32c = CrcUtils.crc32c_extendwith_zero(crc32v, nz);
273  }
274 
275  // crc32c is crc up to p1_off. Now add the user's data.
276  if (csvec)
277  {
278  crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
279  }
280  else
281  {
282  crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
283  }
284  prepageval = crc32c;
285  return 0;
286  }
287 
288  const size_t bavail = (p1==tracked_page) ? tracked_off : XrdSys::PageSize;
289 
290  // assert we're overwriting some (or all) of the previous data (other case was above)
291  assert(p1_off < bavail);
292 
293  // case p1_off==0 && blen>=bavail is either handled by aligned case (p1==tracked_page)
294  // or not sent to preblock, so will need to read some preexisting data
295  assert(p1_off !=0 || blen<bavail);
296  uint8_t b[XrdSys::PageSize];
297 
298  uint32_t crc32v;
299  ssize_t rret = ts_->ReadTags(&crc32v, p1, 1);
300  if (rret<0)
301  {
302  TRACE(Warn, TagsReadError(p1, 1, rret) << " (overwrite)");
303  return rret;
304  }
305 
306  // in either loosewrite or non-loosewrite a read-modify-write sequence is done and the
307  // final crc is that of the modified block. The difference between loose and non-loose
308  // case if that the looser checks are done on the block.
309  //
310  // in either case there are implicit verification(s) (e.g. pgWrite may return EDOM without Verify requested)
311  // as it's not clear if there is a meaningful way to crc a mismatching page during a partial overwrite
312 
313  if (loosewrite_)
314  {
315  // this will reissue read() until eof, or bavail bytes read but accept up to PageSize
316  const ssize_t rlen = XrdOssCsiPages::maxread(fd, b, XrdSys::PageSize * p1, XrdSys::PageSize, bavail);
317  if (rlen<0)
318  {
319  TRACE(Warn, PageReadError(bavail, p1, rlen));
320  return rlen;
321  }
322  memset(&b[rlen], 0, XrdSys::PageSize - rlen);
323  do
324  {
325  uint32_t crc32c = XrdOucCRC::Calc32C(b, bavail, 0U);
326  // this is the expected case
327  if (static_cast<size_t>(rlen) == bavail && crc32c == crc32v) break;
328 
329  // after this write there will be nothing changed between p1_off+blen
330  // and bavail; if there is nothing on disk in this range it will not
331  // be added by the write. So don't try to match crc with implied zero
332  // in this range. Beyond bavail bytes on disk will not be included
333  // in the new crc.
334  const size_t rmin = (p1_off+blen < bavail) ? bavail : 0;
335  if (static_cast<size_t>(rlen) >= rmin && static_cast<size_t>(rlen)<=bavail)
336  {
337  if (crc32c == crc32v)
338  {
339  TRACE(Warn, "Recovered page at offset " << (XrdSys::PageSize * p1) << " of file " << fn_ << " (overwrite)");
340  break;
341  }
342  TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v) << " (loose match, still trying)");
343 
344  if (static_cast<size_t>(rlen) != bavail && rlen > 0)
345  {
346  crc32c = XrdOucCRC::Calc32C(b, rlen, 0U);
347  if (crc32c == crc32v)
348  {
349  TRACE(Warn, "Recovered page (2) at offset " << (XrdSys::PageSize * p1) << " of file " << fn_ << " (overwrite)");
350  break;
351  }
352  TRACE(Warn, CRCMismatchError(rlen, p1, crc32c, crc32v) << " (loose match, still trying)");
353  }
354 
355  memcpy(&b[p1_off], buff, blen);
356  const size_t vl = std::max(bavail, p1_off+blen);
357  crc32c = XrdOucCRC::Calc32C(b, vl, 0U);
358  if (crc32c == crc32v)
359  {
360  TRACE(Warn, "Recovered matching write at offset " << (XrdSys::PageSize * p1)+p1_off << " of file " << fn_ << " (overwrite)");
361  break;
362  }
363  TRACE(Warn, CRCMismatchError(vl, p1, crc32c, crc32v) << " (overwrite)");
364  }
365  else
366  {
367  TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v) << " (overwrite)");
368  }
369  return -EDOM;
370  } while(0);
371  }
372  else
373  {
374  // non-loose case
375  rret = XrdOssCsiPages::fullread(fd, b, XrdSys::PageSize * p1, bavail);
376  if (rret<0)
377  {
378  TRACE(Warn, PageReadError(bavail, p1, rret));
379  return rret;
380  }
381  const uint32_t crc32c = XrdOucCRC::Calc32C(b, bavail, 0U);
382  if (crc32v != crc32c)
383  {
384  TRACE(Warn, CRCMismatchError(bavail, p1, crc32c, crc32v));
385  return -EDOM;
386  }
387  }
388 
389  uint32_t crc32c = XrdOucCRC::Calc32C(b, p1_off, 0U);
390  if (csvec)
391  {
392  crc32c = CrcUtils.crc32c_combine(crc32c, csvec[0], blen);
393  }
394  else
395  {
396  crc32c = XrdOucCRC::Calc32C(buff, blen, crc32c);
397  }
398  if (p1_off+blen < bavail)
399  {
400  const uint32_t cl = XrdOucCRC::Calc32C(&b[p1_off+blen], bavail-p1_off-blen, 0U);
401  crc32c = CrcUtils.crc32c_combine(crc32c, cl, bavail-p1_off-blen);
402  }
403  prepageval = crc32c;
404  return 0;
405 }
static uint32_t crc32c_extendwith_zero(uint32_t crc, size_t len)

References XrdOucCRC::Calc32C(), checklastpg_, crc32c(), XrdOssCsiCrcUtils::crc32c_combine(), XrdOssCsiCrcUtils::crc32c_extendwith_zero(), CRCMismatchError(), CrcUtils, EPNAME, fn_, fullread(), lastpgforloose_, loosewrite_, maxread(), PageReadError(), XrdSys::PageSize, TagsReadError(), TRACE, and ts_.

Referenced by StoreRangeUnaligned().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ TagsReadError()

std::string XrdOssCsiPages::TagsReadError ( off_t  start,
size_t  n,
int  ret 
)
inlineprotected

Definition at line 186 of file XrdOssCsiPages.hh.

187  {
188  char buf[256];
189  snprintf(buf, sizeof(buf),
190  "error %d while reading crc32c values for pages [0x%" PRIx64":0x%" PRIx64 "] for file ",
191  ret, (uint64_t)start, (uint64_t)(start + n - 1));
192  return buf + fn_;
193  }

References fn_.

Referenced by BasicConsistencyCheck(), FetchRangeAligned(), FetchRangeUnaligned(), StoreRangeUnaligned_postblock(), StoreRangeUnaligned_preblock(), truncate(), and UpdateRangeHoleUntilPage().

+ Here is the caller graph for this function:

◆ TagsWriteError()

std::string XrdOssCsiPages::TagsWriteError ( off_t  start,
size_t  n,
int  ret 
)
inlineprotected

Definition at line 195 of file XrdOssCsiPages.hh.

196  {
197  char buf[256];
198  snprintf(buf, sizeof(buf),
199  "error %d while writing crc32c values for pages [0x%" PRIx64":0x%" PRIx64 "] for file ",
200  ret, (uint64_t)start, (uint64_t)(start + n - 1));
201  return buf + fn_;
202  }

References fn_.

Referenced by apply_sequential_aligned_modify(), BasicConsistencyCheck(), StoreRangeUnaligned(), truncate(), and UpdateRangeHoleUntilPage().

+ Here is the caller graph for this function:

◆ TrackedSizeRelease()

void XrdOssCsiPages::TrackedSizeRelease ( )

Definition at line 158 of file XrdOssCsiPages.cc.

159 {
161  assert(tsforupdate_ == true);
162 
163  tsforupdate_ = false;
164  tscond_.Broadcast();
165 }

References XrdSysCondVar::Broadcast(), tscond_, and tsforupdate_.

Referenced by LockTrackinglen(), and XrdOssCsiRangeGuard::unlockTrackinglen().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ TrackedSizesGet()

int XrdOssCsiPages::TrackedSizesGet ( XrdOssCsiPages::Sizes_t rsizes,
bool  forupdate 
)

Definition at line 109 of file XrdOssCsiPages.cc.

110 {
111  if (hasMissingTags_) return -ENOENT;
112 
114  while (tsforupdate_)
115  {
116  tscond_.Wait();
117  }
118  off_t tagsize = ts_->GetTrackedTagSize();
119  off_t datasize = ts_->GetTrackedDataSize();
120  if (forupdate)
121  {
122  tsforupdate_ = true;
123  }
124  rsizes = std::make_pair(tagsize,datasize);
125  return 0;
126 }

References hasMissingTags_, ts_, tscond_, tsforupdate_, and XrdSysCondVar::Wait().

Referenced by XrdOssCsiFile::Fstat(), and LockTrackinglen().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ truncate()

int XrdOssCsiPages::truncate ( XrdOssDF fd,
off_t  len,
XrdOssCsiRangeGuard rg 
)

Definition at line 543 of file XrdOssCsiPages.cc.

544 {
545  EPNAME("truncate");
546 
547  if (len<0) return -EINVAL;
548 
549  // nothing to truncate if there is no tag file
550  if (hasMissingTags_) return 0;
551 
552  const Sizes_t sizes = rg.getTrackinglens();
553 
554  const off_t trackinglen = sizes.first;
555  const off_t p_until = len / XrdSys::PageSize;
556  const size_t p_off = len % XrdSys::PageSize;
557 
558  if (len>trackinglen)
559  {
560  int ret = UpdateRangeHoleUntilPage(fd,p_until,sizes);
561  if (ret<0)
562  {
563  TRACE(Warn, "Error updating tags for holes, error=" << ret);
564  return ret;
565  }
566  }
567 
568  if (len != trackinglen && p_off != 0)
569  {
570  const off_t tracked_page = trackinglen / XrdSys::PageSize;
571  const size_t tracked_off = trackinglen % XrdSys::PageSize;
572  size_t toread = tracked_off;
573  if (len>trackinglen)
574  {
575  if (p_until != tracked_page) toread = 0;
576  }
577  else
578  {
579  if (p_until != tracked_page) toread = XrdSys::PageSize;
580  }
581  uint8_t b[XrdSys::PageSize];
582  if (toread>0)
583  {
584  ssize_t rret = XrdOssCsiPages::fullread(fd, b, p_until*XrdSys::PageSize, toread);
585  if (rret<0)
586  {
587  TRACE(Warn, PageReadError(toread, p_until, rret));
588  return rret;
589  }
590  const uint32_t crc32c = XrdOucCRC::Calc32C(b, toread, 0U);
591  uint32_t crc32v;
592  rret = ts_->ReadTags(&crc32v, p_until, 1);
593  if (rret<0)
594  {
595  TRACE(Warn, TagsReadError(p_until, 1, rret));
596  return rret;
597  }
598  if (crc32v != crc32c)
599  {
600  TRACE(Warn, CRCMismatchError(toread, p_until, crc32c, crc32v));
601  return -EDOM;
602  }
603  }
604  if (p_off > toread)
605  {
606  memset(&b[toread],0,p_off-toread);
607  }
608  const uint32_t crc32c = XrdOucCRC::Calc32C(b, p_off, 0U);
609  const ssize_t wret = ts_->WriteTags(&crc32c, p_until, 1);
610  if (wret < 0)
611  {
612  TRACE(Warn, TagsWriteError(p_until, 1, wret));
613  return wret;
614  }
615  }
616 
617  LockTruncateSize(len,true);
618  rg.unlockTrackinglen();
619  return 0;
620 }
int LockTruncateSize(off_t, bool)

References XrdOucCRC::Calc32C(), crc32c(), CRCMismatchError(), EPNAME, fullread(), XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockTruncateSize(), PageReadError(), XrdSys::PageSize, TagsReadError(), TagsWriteError(), TRACE, ts_, XrdOssCsiRangeGuard::unlockTrackinglen(), and UpdateRangeHoleUntilPage().

Referenced by XrdOssCsiFile::Ftruncate().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRange()

int XrdOssCsiPages::UpdateRange ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
XrdOssCsiRangeGuard rg 
)

Definition at line 169 of file XrdOssCsiPages.cc.

170 {
171  if (offset<0)
172  {
173  return -EINVAL;
174  }
175 
176  if (blen == 0)
177  {
178  return 0;
179  }
180 
181  // if the tag file is missing we don't need to store anything
182  if (hasMissingTags_)
183  {
184  return 0;
185  }
186 
187  // update of file were checksums are based on the file data suppplied: as there's no separate
188  // source of checksum information mark this file as having unverified checksums
190 
191  const Sizes_t sizes = rg.getTrackinglens();
192 
193  const off_t trackinglen = sizes.first;
194  if (offset+blen > static_cast<size_t>(trackinglen))
195  {
196  LockSetTrackedSize(offset+blen);
197  rg.unlockTrackinglen();
198  }
199 
200  int ret;
201  if ((offset % XrdSys::PageSize) != 0 ||
202  (offset+blen < static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0) ||
203  ((trackinglen % XrdSys::PageSize) !=0 && offset > trackinglen))
204  {
205  ret = UpdateRangeUnaligned(fd, buff, offset, blen, sizes);
206  }
207  else
208  {
209  ret = UpdateRangeAligned(buff, offset, blen, sizes);
210  }
211 
212  return ret;
213 }
int UpdateRangeAligned(const void *, off_t, size_t, const Sizes_t &)
int UpdateRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)

References XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, LockMakeUnverified(), LockSetTrackedSize(), XrdSys::PageSize, XrdOssCsiRangeGuard::unlockTrackinglen(), UpdateRangeAligned(), and UpdateRangeUnaligned().

Referenced by XrdOssCsiFileAioJob::DoItWrite1(), XrdOssCsiFile::Write(), and XrdOssCsiFile::WriteV().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRangeAligned()

int XrdOssCsiPages::UpdateRangeAligned ( const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t sizes 
)
protected

Definition at line 485 of file XrdOssCsiPages.cc.

486 {
487  return StoreRangeAligned(buff, offset, blen, sizes, NULL);
488 }

References StoreRangeAligned().

Referenced by UpdateRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRangeHoleUntilPage()

int XrdOssCsiPages::UpdateRangeHoleUntilPage ( XrdOssDF fd,
off_t  until,
const Sizes_t sizes 
)
protected

Definition at line 52 of file XrdOssCsiPagesUnaligned.cc.

53 {
54  EPNAME("UpdateRangeHoleUntilPage");
55 
56  static const uint32_t crczero = CrcUtils.crc32c_extendwith_zero(0u, XrdSys::PageSize);
57  static const std::vector<uint32_t> crc32Vec(stsize_, crczero);
58 
59  const off_t trackinglen = sizes.first;
60  const off_t tracked_page = trackinglen / XrdSys::PageSize;
61  if (until <= tracked_page) return 0;
62 
63  const size_t tracked_off = trackinglen % XrdSys::PageSize;
64 
65  // if last tracked page is before page "until" extend it
66  if (tracked_off>0)
67  {
68  if (fd == NULL)
69  {
70  TRACE(Warn, "Unexpected partially filled last page " << fn_);
71  return -EDOM;
72  }
73 
74  uint32_t prevtag;
75  const ssize_t rret = ts_->ReadTags(&prevtag, tracked_page, 1);
76  if (rret < 0)
77  {
78  TRACE(Warn, TagsReadError(tracked_page, 1, rret));
79  return rret;
80  }
81 
82  // extend prevtag up to PageSize. If there is a mismatch it will only be
83  // discovered during a later read (but this saves a read now).
84  const uint32_t crc32c = CrcUtils.crc32c_extendwith_zero(prevtag, XrdSys::PageSize - tracked_off);
85  const ssize_t wret = ts_->WriteTags(&crc32c, tracked_page, 1);
86  if (wret < 0)
87  {
88  TRACE(Warn, TagsWriteError(tracked_page, 1, wret) << " (prev)");
89  return wret;
90  }
91  }
92 
93  if (!writeHoles_) return 0;
94 
95  const off_t nAllEmpty = (tracked_off>0) ? (until - tracked_page - 1) : (until - tracked_page);
96  const off_t firstEmpty = (tracked_off>0) ? (tracked_page + 1) : tracked_page;
97 
98  off_t towrite = nAllEmpty;
99  off_t nwritten = 0;
100  while(towrite>0)
101  {
102  const size_t nw = std::min(towrite, (off_t)crc32Vec.size());
103  const ssize_t wret = ts_->WriteTags(&crc32Vec[0], firstEmpty+nwritten, nw);
104  if (wret<0)
105  {
106  TRACE(Warn, TagsWriteError(firstEmpty+nwritten, nw, wret) << " (new)");
107  return wret;
108  }
109  towrite -= wret;
110  nwritten += wret;
111  }
112 
113  return 0;
114 }

References crc32c(), XrdOssCsiCrcUtils::crc32c_extendwith_zero(), CrcUtils, EPNAME, fn_, XrdSys::PageSize, stsize_, TagsReadError(), TagsWriteError(), TRACE, ts_, and writeHoles_.

Referenced by StoreRangeAligned(), StoreRangeUnaligned(), and truncate().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ UpdateRangeUnaligned()

int XrdOssCsiPages::UpdateRangeUnaligned ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t sizes 
)
protected

Definition at line 120 of file XrdOssCsiPagesUnaligned.cc.

121 {
122  return StoreRangeUnaligned(fd, buff, offset, blen, sizes, NULL);
123 }

References StoreRangeUnaligned().

Referenced by UpdateRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ VerificationStatus()

int XrdOssCsiPages::VerificationStatus ( )

Definition at line 759 of file XrdOssCsiPages.cc.

760 {
761  if (hasMissingTags_)
762  {
763  return 0;
764  }
765  bool iv;
766  {
768  iv = ts_->IsVerified();
769  }
770  if (iv)
771  {
772  return XrdOss::PF_csVer;
773  }
774  return XrdOss::PF_csVun;
775 }
static const int PF_csVer
verified file checksums present
Definition: XrdOss.hh:778
static const int PF_csVun
unverified file checksums present
Definition: XrdOss.hh:779

References hasMissingTags_, XrdOss::PF_csVer, XrdOss::PF_csVun, ts_, and tscond_.

Referenced by XrdOssCsiFile::VerificationStatus().

+ Here is the caller graph for this function:

◆ VerifyRange()

int XrdOssCsiPages::VerifyRange ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
XrdOssCsiRangeGuard rg 
)

Definition at line 220 of file XrdOssCsiPages.cc.

221 {
222  EPNAME("VerifyRange");
223 
224  if (offset<0)
225  {
226  return -EINVAL;
227  }
228 
229  // if the tag file is missing we don't verify anything
230  if (hasMissingTags_)
231  {
232  return 0;
233  }
234 
235  const Sizes_t sizes = rg.getTrackinglens();
236  const off_t trackinglen = sizes.first;
237 
238  if (offset >= trackinglen && blen == 0)
239  {
240  return 0;
241  }
242 
243  if (blen == 0)
244  {
245  // if offset is before the tracked len we should not be requested to verify zero bytes:
246  // the file may have been truncated
247  TRACE(Warn, "Verify request for zero bytes " << fn_ << ", file may be truncated");
248  return -EDOM;
249  }
250 
251  if (offset+blen > static_cast<size_t>(trackinglen))
252  {
253  TRACE(Warn, "Verify request for " << (offset+blen-trackinglen) << " bytes from " << fn_ << " beyond tracked length");
254  return -EDOM;
255  }
256 
257  int vret;
258  if ((offset % XrdSys::PageSize) != 0 || (offset+blen != static_cast<size_t>(trackinglen) && (blen % XrdSys::PageSize) != 0))
259  {
260  vret = VerifyRangeUnaligned(fd, buff, offset, blen, sizes);
261  }
262  else
263  {
264  vret = VerifyRangeAligned(buff, offset, blen, sizes);
265  }
266 
267  return vret;
268 }
int VerifyRangeAligned(const void *, off_t, size_t, const Sizes_t &)
int VerifyRangeUnaligned(XrdOssDF *, const void *, off_t, size_t, const Sizes_t &)

References EPNAME, fn_, XrdOssCsiRangeGuard::getTrackinglens(), hasMissingTags_, XrdSys::PageSize, TRACE, VerifyRangeAligned(), and VerifyRangeUnaligned().

Referenced by XrdOssCsiFileAioJob::DoItRead2(), XrdOssCsiFile::Read(), XrdOssCsiFile::ReadRaw(), and XrdOssCsiFile::ReadV().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ VerifyRangeAligned()

int XrdOssCsiPages::VerifyRangeAligned ( const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t sizes 
)
protected

Definition at line 448 of file XrdOssCsiPages.cc.

449 {
450  return FetchRangeAligned(buff,offset,blen,sizes,NULL,XrdOssDF::Verify);
451 }

References FetchRangeAligned(), and XrdOssDF::Verify.

Referenced by VerifyRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ VerifyRangeUnaligned()

int XrdOssCsiPages::VerifyRangeUnaligned ( XrdOssDF fd,
const void *  buff,
off_t  offset,
size_t  blen,
const Sizes_t sizes 
)
protected

Definition at line 611 of file XrdOssCsiPagesUnaligned.cc.

612 {
613  return FetchRangeUnaligned(fd, buff, offset, blen, sizes, NULL, XrdOssDF::Verify);
614 }

References FetchRangeUnaligned(), and XrdOssDF::Verify.

Referenced by VerifyRange().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

Member Data Documentation

◆ allowMissingTags_

bool XrdOssCsiPages::allowMissingTags_
protected

Definition at line 84 of file XrdOssCsiPages.hh.

Referenced by Open().

◆ checklastpg_

bool XrdOssCsiPages::checklastpg_
protected

Definition at line 102 of file XrdOssCsiPages.hh.

Referenced by BasicConsistencyCheck(), and StoreRangeUnaligned_preblock().

◆ disablePgExtend_

bool XrdOssCsiPages::disablePgExtend_
protected

Definition at line 85 of file XrdOssCsiPages.hh.

Referenced by StoreRange().

◆ fn_

◆ hasMissingTags_

bool XrdOssCsiPages::hasMissingTags_
protected

◆ lastpgforloose_

off_t XrdOssCsiPages::lastpgforloose_
protected

Definition at line 101 of file XrdOssCsiPages.hh.

Referenced by BasicConsistencyCheck(), and StoreRangeUnaligned_preblock().

◆ loosewrite_

bool XrdOssCsiPages::loosewrite_
protected

◆ loosewriteConfigured_

const bool XrdOssCsiPages::loosewriteConfigured_
protected

Definition at line 88 of file XrdOssCsiPages.hh.

Referenced by LockResetSizes(), and Open().

◆ rangeaddmtx_

XrdSysMutex XrdOssCsiPages::rangeaddmtx_
protected

Definition at line 81 of file XrdOssCsiPages.hh.

Referenced by LockTrackinglen().

◆ ranges_

XrdOssCsiRanges XrdOssCsiPages::ranges_
protected

Definition at line 82 of file XrdOssCsiPages.hh.

Referenced by LockTrackinglen().

◆ rdonly_

bool XrdOssCsiPages::rdonly_
protected

Definition at line 87 of file XrdOssCsiPages.hh.

Referenced by BasicConsistencyCheck(), IsReadOnly(), and Open().

◆ stsize_

const size_t XrdOssCsiPages::stsize_ = 1024
staticprotected

◆ tident

const char* XrdOssCsiPages::tident
protected

Definition at line 98 of file XrdOssCsiPages.hh.

◆ tident_

const std::string XrdOssCsiPages::tident_
protected

Definition at line 97 of file XrdOssCsiPages.hh.

◆ ts_

◆ tscond_

◆ tsforupdate_

bool XrdOssCsiPages::tsforupdate_
protected

Definition at line 92 of file XrdOssCsiPages.hh.

Referenced by TrackedSizeRelease(), and TrackedSizesGet().

◆ writeHoles_

bool XrdOssCsiPages::writeHoles_
protected

Definition at line 83 of file XrdOssCsiPages.hh.

Referenced by UpdateRangeHoleUntilPage().


The documentation for this class was generated from the following files: