XRootD
XrdPfcInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDPFC_INFO_HH__
2 #define __XRDPFC_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include "XrdPfcTypes.hh"
22 
23 #include <cstdio>
24 #include <ctime>
25 #include <cassert>
26 #include <vector>
27 
28 class XrdOssDF;
29 class XrdCksCalc;
30 class XrdSysTrace;
31 
32 namespace XrdPfc
33 {
34 class Stats;
35 
36 //----------------------------------------------------------------------------
38 //----------------------------------------------------------------------------
39 
40 class Info
41 {
42 public:
43  struct Status {
44  union {
45  struct {
46  int f_cksum_check : 3;
47 
48  int _free_bits_ : 29;
49  };
50  unsigned int _raw_;
51  };
52  Status() : _raw_(0) {}
53  };
54 
56  struct AStat
57  {
58  time_t AttachTime;
59  time_t DetachTime;
60  int NumIos;
61  int Duration;
62  int NumMerged;
63  int Reserved;
64  long long BytesHit;
65  long long BytesMissed;
66  long long BytesBypassed;
67 
68  AStat() :
69  AttachTime(0), DetachTime(0), NumIos(0), Duration(0), NumMerged(0), Reserved(0),
71  {}
72 
73  void MergeWith(const AStat &a);
74  };
75 
76  struct Store
77  {
78  long long m_buffer_size;
79  long long m_file_size;
80  time_t m_creationTime;
81  time_t m_noCkSumTime;
82  size_t m_accessCnt;
85 
86  Store () :
89  {}
90  };
91 
92 
93  //------------------------------------------------------------------------
95  //------------------------------------------------------------------------
96  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
97 
98  //------------------------------------------------------------------------
100  //------------------------------------------------------------------------
101  ~Info();
102 
103  //---------------------------------------------------------------------
105  //---------------------------------------------------------------------
106  void SetBitWritten(int i);
107 
108  //---------------------------------------------------------------------
110  //---------------------------------------------------------------------
111  bool TestBitWritten(int i) const;
112 
113  //---------------------------------------------------------------------
115  //---------------------------------------------------------------------
116  bool TestBitPrefetch(int i) const;
117 
118  //---------------------------------------------------------------------
120  //---------------------------------------------------------------------
121  void SetBitPrefetch(int i);
122 
123  //---------------------------------------------------------------------
125  //---------------------------------------------------------------------
126  void SetBitSynced(int i);
127 
128  //---------------------------------------------------------------------
130  //---------------------------------------------------------------------
131  void SetAllBitsSynced();
132 
133  void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs);
134 
135  //---------------------------------------------------------------------
137  //---------------------------------------------------------------------
138  void ResizeBits();
139 
140  //---------------------------------------------------------------------
146  //---------------------------------------------------------------------
147  bool Read(XrdOssDF* fp, const char *dname, const char *fname = 0);
148 
149  //---------------------------------------------------------------------
155  //---------------------------------------------------------------------
156  bool Write(XrdOssDF* fp, const char *dname, const char *fname = 0);
157 
158  //---------------------------------------------------------------------
160  //---------------------------------------------------------------------
162 
163  //---------------------------------------------------------------------
165  //---------------------------------------------------------------------
166  void ResetAllAccessStats();
167 
168  //---------------------------------------------------------------------
170  //---------------------------------------------------------------------
171  void WriteIOStatAttach();
172 
173  //---------------------------------------------------------------------
175  //---------------------------------------------------------------------
176  void WriteIOStat(Stats& s);
177 
178  //---------------------------------------------------------------------
180  //---------------------------------------------------------------------
181  void WriteIOStatDetach(Stats& s);
182 
183  //---------------------------------------------------------------------
185  //---------------------------------------------------------------------
186  void WriteIOStatSingle(long long bytes_disk);
187 
188  //---------------------------------------------------------------------
190  //---------------------------------------------------------------------
191  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
192 
193  //---------------------------------------------------------------------
195  //---------------------------------------------------------------------
196  int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const;
197 
198  //---------------------------------------------------------------------
200  //---------------------------------------------------------------------
201  int GetBitvecSizeInBytes() const;
202 
203  //---------------------------------------------------------------------
205  //---------------------------------------------------------------------
206  int GetNBlocks() const;
207 
208  //---------------------------------------------------------------------
210  //---------------------------------------------------------------------
211  long long GetFileSize() const;
212 
213  //---------------------------------------------------------------------
215  //---------------------------------------------------------------------
216  bool GetLatestDetachTime(time_t& t) const;
217 
218  //---------------------------------------------------------------------
220  //---------------------------------------------------------------------
221  const AStat* GetLastAccessStats() const;
222 
223  //---------------------------------------------------------------------
225  //---------------------------------------------------------------------
226  long long GetBufferSize() const;
227 
228  //---------------------------------------------------------------------
230  //---------------------------------------------------------------------
231  bool IsComplete() const;
232 
233  //---------------------------------------------------------------------
235  //---------------------------------------------------------------------
236  int GetNDownloadedBlocks() const;
237 
238  //---------------------------------------------------------------------
240  //---------------------------------------------------------------------
241  long long GetNDownloadedBytes() const;
242 
243  //---------------------------------------------------------------------
245  //---------------------------------------------------------------------
246  int GetLastDownloadedBlock() const;
247 
248  //---------------------------------------------------------------------
250  //---------------------------------------------------------------------
251  long long GetExpectedDataFileSize() const;
252 
253  //---------------------------------------------------------------------
255  //---------------------------------------------------------------------
257 
258  //---------------------------------------------------------------------
260  //---------------------------------------------------------------------
261  size_t GetAccessCnt() const { return m_store.m_accessCnt; }
262 
263  //---------------------------------------------------------------------
265  //---------------------------------------------------------------------
266  int GetVersion() { return m_version; }
267 
268  //---------------------------------------------------------------------
270  //---------------------------------------------------------------------
271  const Store& RefStoredData() const { return m_store; }
272  const std::vector<AStat>& RefAStats() const { return m_astats; }
273 
274  //---------------------------------------------------------------------
276  //---------------------------------------------------------------------
277  time_t GetCreationTime() const { return m_store.m_creationTime; }
278 
279  //---------------------------------------------------------------------
281  //---------------------------------------------------------------------
282  uint32_t CalcCksumStore();
283  uint32_t CalcCksumSyncedAndAStats();
284  void CalcCksumMd5(unsigned char* buff, char* digest);
285 
286  CkSumCheck_e GetCkSumState() const { return (CkSumCheck_e) m_store.m_status.f_cksum_check; }
287  const char* GetCkSumStateAsText() const;
288 
289  bool IsCkSumCache() const { return m_store.m_status.f_cksum_check & CSChk_Cache; }
290  bool IsCkSumNet() const { return m_store.m_status.f_cksum_check & CSChk_Net; }
291  bool IsCkSumAny() const { return m_store.m_status.f_cksum_check & CSChk_Both; }
292  bool IsCkSumBoth() const { return (m_store.m_status.f_cksum_check & CSChk_Both) == CSChk_Both; }
293 
294  void SetCkSumState(CkSumCheck_e css) { m_store.m_status.f_cksum_check = css; }
295  void DowngradeCkSumState(CkSumCheck_e css_ref) { m_store.m_status.f_cksum_check &= css_ref; }
296  void ResetCkSumCache();
297  void ResetCkSumNet();
298 
299  bool HasNoCkSumTime() const { return m_store.m_noCkSumTime != 0; }
300  time_t GetNoCkSumTime() const { return m_store.m_noCkSumTime; }
303 
304 #ifdef XRDPFC_CKSUM_TEST
305  static void TestCksumStuff();
306 #endif
307 
308  static const char* m_traceID; // has to be m_ (convention in TRACE macros)
309  static const char* s_infoExtension;
310  static const size_t s_infoExtensionLen;
311  static size_t s_maxNumAccess; // can be set from configuration
312  static const int s_defaultVersion;
313 
314  XrdSysTrace* GetTrace() const {return m_trace; }
315 
316 protected:
318 
320  unsigned char *m_buff_synced;
321  unsigned char *m_buff_written;
322  unsigned char *m_buff_prefetch;
323  std::vector<AStat> m_astats;
324 
328  bool m_complete;
330 
331 private:
332  inline unsigned char cfiBIT(int n) const { return 1 << n; }
333 
334  // Reading functions for older cinfo file formats
335  bool ReadV2(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
336  bool ReadV3(XrdOssDF* fp, off_t off, const char *dname, const char *fname);
337 
338  XrdCksCalc* m_cksCalcMd5;
339 };
340 
341 //------------------------------------------------------------------------------
342 
343 inline bool Info::TestBitWritten(int i) const
344 {
345  const int cn = i/8;
346  assert(cn < GetBitvecSizeInBytes());
347 
348  const int off = i - cn*8;
349  return (m_buff_written[cn] & cfiBIT(off)) != 0;
350 }
351 
352 inline void Info::SetBitWritten(int i)
353 {
354  const int cn = i/8;
355  assert(cn < GetBitvecSizeInBytes());
356 
357  const int off = i - cn*8;
358 
359  m_buff_written[cn] |= cfiBIT(off);
360 
361  if (--m_missingBlocks == 0)
362  m_complete = true;
363 }
364 
365 inline void Info::SetBitPrefetch(int i)
366 {
367  if (!m_buff_prefetch) return;
368 
369  const int cn = i/8;
370  assert(cn < GetBitvecSizeInBytes());
371 
372  const int off = i - cn*8;
373  m_buff_prefetch[cn] |= cfiBIT(off);
374 }
375 
376 inline bool Info::TestBitPrefetch(int i) const
377 {
378  if (!m_buff_prefetch) return false;
379 
380  const int cn = i/8;
381  assert(cn < GetBitvecSizeInBytes());
382 
383  const int off = i - cn*8;
384  return (m_buff_prefetch[cn] & cfiBIT(off)) != 0;
385 }
386 
387 inline void Info::SetBitSynced(int i)
388 {
389  const int cn = i/8;
390  assert(cn < GetBitvecSizeInBytes());
391 
392  const int off = i - cn*8;
393  m_buff_synced[cn] |= cfiBIT(off);
394 }
395 
396 //------------------------------------------------------------------------------
397 
398 inline int Info::GetNDownloadedBlocks() const
399 {
400  int cntd = 0;
401  for (int i = 0; i < m_bitvecSizeInBits; ++i)
402  if (TestBitWritten(i)) cntd++;
403 
404  return cntd;
405 }
406 
407 inline long long Info::GetNDownloadedBytes() const
408 {
410 }
411 
413 {
414  for (int i = m_bitvecSizeInBits - 1; i >= 0; --i)
415  if (TestBitWritten(i)) return i;
416 
417  return -1;
418 }
419 
420 inline long long Info::GetExpectedDataFileSize() const
421 {
422  int last_block = GetLastDownloadedBlock();
423  if (last_block == m_bitvecSizeInBits - 1)
424  return m_store.m_file_size;
425  else
426  return (last_block + 1) * m_store.m_buffer_size;
427 }
428 
429 inline int Info::GetBitvecSizeInBytes() const
430 {
431  if (m_bitvecSizeInBits)
432  return ((m_bitvecSizeInBits - 1)/8 + 1);
433  else
434  return 0;
435 }
436 
437 inline int Info::GetNBlocks() const
438 {
439  return m_bitvecSizeInBits;
440 }
441 
442 inline long long Info::GetFileSize() const
443 {
444  return m_store.m_file_size;
445 }
446 
447 inline bool Info::IsComplete() const
448 {
449  return m_complete;
450 }
451 
452 inline int Info::CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
453 {
454  // TODO rewrite to use full byte comparisons outside of edges ?
455  // Also, it seems to be always called with firstIdx = 0, lastIdx = m_bitvecSizeInBits.
456  int cnt = 0;
457  for (int i = firstIdx; i < lastIdx; ++i)
458  if (! TestBitWritten(i)) ++cnt;
459 
460  return cnt;
461 }
462 
464 {
466  m_complete = (m_missingBlocks == 0);
467 }
468 
469 inline long long Info::GetBufferSize() const
470 {
471  return m_store.m_buffer_size;
472 }
473 
474 }
475 #endif
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:41
void SetBitPrefetch(int i)
Mark block as obtained through prefetch.
Definition: XrdPfcInfo.hh:365
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdPfcInfo.hh:463
bool IsCkSumAny() const
Definition: XrdPfcInfo.hh:291
static const char * s_infoExtension
Definition: XrdPfcInfo.hh:309
uint32_t CalcCksumStore()
Get cksum, MD5 is for backward compatibility with V2 and V3.
Definition: XrdPfcInfo.cc:227
void ResizeBits()
Reserve bit vectors for file_size / buffer_size bytes.
Definition: XrdPfcInfo.cc:172
void ResetCkSumCache()
Definition: XrdPfcInfo.cc:203
static const int s_defaultVersion
Definition: XrdPfcInfo.hh:312
void SetBitSynced(int i)
Mark block as synced to disk.
Definition: XrdPfcInfo.hh:387
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
Definition: XrdPfcInfo.cc:444
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
Definition: XrdPfcInfo.cc:123
int GetBitvecSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdPfcInfo.hh:429
time_t GetNoCkSumTimeForUVKeep() const
Definition: XrdPfcInfo.hh:301
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdPfcInfo.hh:322
static const size_t s_infoExtensionLen
Definition: XrdPfcInfo.hh:310
CkSumCheck_e GetCkSumState() const
Definition: XrdPfcInfo.hh:286
const AStat * GetLastAccessStats() const
Get latest access stats.
Definition: XrdPfcInfo.cc:489
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
Definition: XrdPfcInfo.cc:420
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
Definition: XrdPfcInfo.cc:470
void ResetCkSumNet()
Definition: XrdPfcInfo.cc:213
bool Write(XrdOssDF *fp, const char *dname, const char *fname=0)
Definition: XrdPfcInfo.cc:266
void DowngradeCkSumState(CkSumCheck_e css_ref)
Definition: XrdPfcInfo.hh:295
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdPfcInfo.hh:407
void CompactifyAccessRecords()
Compactify access records to the configured maximum.
Definition: XrdPfcInfo.cc:379
int GetLastDownloadedBlock() const
Get number of the last downloaded block.
Definition: XrdPfcInfo.hh:412
uint32_t CalcCksumSyncedAndAStats()
Definition: XrdPfcInfo.cc:232
~Info()
Destructor.
Definition: XrdPfcInfo.cc:134
time_t GetCreationTime() const
Get file size.
Definition: XrdPfcInfo.hh:277
unsigned char * m_buff_written
download state vector
Definition: XrdPfcInfo.hh:321
bool IsCkSumNet() const
Definition: XrdPfcInfo.hh:290
unsigned char * m_buff_synced
disk written state vector
Definition: XrdPfcInfo.hh:320
bool m_complete
cached; if false, set to true when missingBlocks hit zero
Definition: XrdPfcInfo.hh:328
void ResetAllAccessStats()
Reset IO Stats.
Definition: XrdPfcInfo.cc:359
bool TestBitPrefetch(int i) const
Test if block at the given index has been prefetched.
Definition: XrdPfcInfo.hh:376
bool IsComplete() const
Get complete status.
Definition: XrdPfcInfo.hh:447
const std::vector< AStat > & RefAStats() const
Definition: XrdPfcInfo.hh:272
bool IsCkSumCache() const
Definition: XrdPfcInfo.hh:289
const Store & RefStoredData() const
Get stored data.
Definition: XrdPfcInfo.hh:271
void SetBitWritten(int i)
Mark block as written to disk.
Definition: XrdPfcInfo.hh:352
time_t GetNoCkSumTime() const
Definition: XrdPfcInfo.hh:300
int m_missingBlocks
cached, updated in SetBitWritten()
Definition: XrdPfcInfo.hh:327
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdPfcInfo.hh:469
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
Definition: XrdPfcInfo.cc:429
long long GetExpectedDataFileSize() const
Get expected data file size.
Definition: XrdPfcInfo.hh:420
bool TestBitWritten(int i) const
Test if block at the given index is written to disk.
Definition: XrdPfcInfo.hh:343
int m_bitvecSizeInBits
cached
Definition: XrdPfcInfo.hh:326
long long GetFileSize() const
Get file size.
Definition: XrdPfcInfo.hh:442
XrdSysTrace * m_trace
Definition: XrdPfcInfo.hh:317
std::vector< AStat > m_astats
access records
Definition: XrdPfcInfo.hh:323
void SetAllBitsSynced()
Mark all blocks as synced to disk.
Definition: XrdPfcInfo.cc:144
bool Read(XrdOssDF *fp, const char *dname, const char *fname=0)
Read content of cinfo file into this object.
Definition: XrdPfcInfo.cc:294
XrdSysTrace * GetTrace() const
Definition: XrdPfcInfo.hh:314
void SetCkSumState(CkSumCheck_e css)
Definition: XrdPfcInfo.hh:294
const char * GetCkSumStateAsText() const
Definition: XrdPfcInfo.cc:249
void ResetNoCkSumTime()
Definition: XrdPfcInfo.hh:302
bool IsCkSumBoth() const
Definition: XrdPfcInfo.hh:292
int GetVersion()
Get version.
Definition: XrdPfcInfo.hh:266
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdPfcInfo.hh:398
size_t GetAccessCnt() const
Get number of accesses.
Definition: XrdPfcInfo.hh:261
int CountBlocksNotWrittenInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdPfcInfo.hh:452
void SetBufferSizeFileSizeAndCreationTime(long long bs, long long fs)
Definition: XrdPfcInfo.cc:161
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
Definition: XrdPfcInfo.cc:438
static size_t s_maxNumAccess
Definition: XrdPfcInfo.hh:311
static const char * m_traceID
Definition: XrdPfcInfo.hh:308
void CalcCksumMd5(unsigned char *buff, char *digest)
Definition: XrdPfcInfo.cc:238
int GetNBlocks() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdPfcInfo.hh:437
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdPfcInfo.hh:329
bool HasNoCkSumTime() const
Definition: XrdPfcInfo.hh:299
Statistics of cache utilisation by a File object.
Definition: XrdPfcStats.hh:35
Definition: XrdPfc.hh:41
@ CSChk_Both
Definition: XrdPfcTypes.hh:27
@ CSChk_Net
Definition: XrdPfcTypes.hh:27
@ CSChk_Cache
Definition: XrdPfcTypes.hh:27
XrdPosixStats Stats
Definition: XrdPosixFile.cc:64
Access statistics.
Definition: XrdPfcInfo.hh:57
long long BytesHit
read from cache
Definition: XrdPfcInfo.hh:64
long long BytesBypassed
read from remote and dropped
Definition: XrdPfcInfo.hh:66
void MergeWith(const AStat &a)
Definition: XrdPfcInfo.cc:366
int Duration
total duration of all IOs attached
Definition: XrdPfcInfo.hh:61
int Reserved
reserved / alignment
Definition: XrdPfcInfo.hh:63
int NumIos
number of IO objects attached during this access
Definition: XrdPfcInfo.hh:60
time_t DetachTime
close time
Definition: XrdPfcInfo.hh:59
long long BytesMissed
read from remote and cached
Definition: XrdPfcInfo.hh:65
time_t AttachTime
open time
Definition: XrdPfcInfo.hh:58
int NumMerged
number of times the record has been merged
Definition: XrdPfcInfo.hh:62
int m_astatSize
size of AStat vector
Definition: XrdPfcInfo.hh:84
Status m_status
status information
Definition: XrdPfcInfo.hh:83
long long m_buffer_size
buffer / block size
Definition: XrdPfcInfo.hh:78
time_t m_noCkSumTime
time when first non-cksummed block was detected
Definition: XrdPfcInfo.hh:81
size_t m_accessCnt
total access count for the file
Definition: XrdPfcInfo.hh:82
long long m_file_size
size of file in bytes
Definition: XrdPfcInfo.hh:79
time_t m_creationTime
time the info file was created
Definition: XrdPfcInfo.hh:80