XRootD
XrdPfc Namespace Reference

Classes

class  Block
 
class  BlockResponseHandler
 
class  Cache
 Attaches/creates and detaches/deletes cache-io objects for disk based cache. More...
 
struct  ChunkRequest
 
struct  Configuration
 Contains parameters configurable from the xrootd config file. More...
 
struct  DataFsPurgeshot
 
struct  DataFsSnapshot
 
struct  DataFsState
 
struct  DataFsStateBase
 
class  Decision
 Base class for selecting which files should be cached. More...
 
class  DirectResponseHandler
 
struct  DirPurgeElement
 
struct  DirState
 
struct  DirStateBase
 
struct  DirStateElement
 
class  DirStats
 
struct  DirUsage
 
class  File
 
class  FPurgeState
 
class  FsTraversal
 
class  Info
 Status of cached file. Can be read from and written into a binary file. More...
 
class  IO
 Base cache-io class that implements some XrdOucCacheIO abstract methods. More...
 
class  IOFile
 Downloads original file into a single file on local disk. Handles read requests as they come along. More...
 
class  IOFileBlock
 Downloads original file into multiple files, chunked into blocks. Only blocks that are asked for are downloaded. Handles read requests as they come along. More...
 
struct  MutexHolder
 
struct  PathTokenizer
 
class  Print
 
class  PurgePin
 Base class for reguesting directory space to obtain. More...
 
struct  ReadReqRH
 
struct  ReadRequest
 
class  ResourceMonitor
 
struct  SplitParser
 
class  Stats
 Statistics of cache utilisation by a File object. More...
 
struct  TmpConfiguration
 

Typedefs

using BlockList_i = std::list< Block * >::iterator
 
using BlockList_t = std::list< Block * >
 
using unlink_func = std::function< int(const std::string &)>
 
using vChunkRequest_i = std::vector< ChunkRequest >::iterator
 
using vChunkRequest_t = std::vector< ChunkRequest >
 
typedef std::vector< uint32_t > vCkSum_t
 

Enumerations

enum  CkSumCheck_e {
  CSChk_Unknown = -1 ,
  CSChk_None = 0 ,
  CSChk_Cache = 1 ,
  CSChk_Net = 2 ,
  CSChk_Both = 3 ,
  CSChk_TLS = 4
}
 

Functions

void OldStylePurgeDriver (DataFsPurgeshot &ps)
 
m_NDirectories PFC_DEFINE_TYPE_NON_INTRUSIVE (DirStateElement, m_dir_name, m_stats, m_usage, m_parent, m_daughters_begin, m_daughters_end) PFC_DEFINE_TYPE_NON_INTRUSIVE(DataFsSnapshot
 
 PFC_DEFINE_TYPE_NON_INTRUSIVE (DirStats, m_NumIos, m_Duration, m_BytesHit, m_BytesMissed, m_BytesBypassed, m_BytesWritten, m_StBlocksAdded, m_NCksumErrors, m_StBlocksRemoved, m_NFilesOpened, m_NFilesClosed, m_NFilesCreated, m_NFilesRemoved, m_NDirectoriesCreated, m_NDirectoriesRemoved) PFC_DEFINE_TYPE_NON_INTRUSIVE(DirUsage
 
long long UnlinkPurgeStateFilesInMap (FPurgeState &purgeState, long long bytes_to_remove, const std::string &root_path)
 

Variables

m_NDirectories m_disk_total
 
m_NDirectories m_disk_used
 
m_NDirectories m_file_usage
 
 m_LastCloseTime
 
 m_LastOpenTime
 
m_NDirectories m_meta_total
 
m_NDirectories m_meta_used
 
 m_NFiles
 
 m_NFilesOpen
 
m_NDirectories m_sshot_stats_reset_time
 
 m_StBlocks
 
m_NDirectories m_usage_update_time
 
const char * trace_what_strings [] = {"","error ","warning ","info ","debug ","dump "}
 

Typedef Documentation

◆ BlockList_i

using XrdPfc::BlockList_i = typedef std::list<Block*>::iterator

Definition at line 167 of file XrdPfcFile.hh.

◆ BlockList_t

using XrdPfc::BlockList_t = typedef std::list<Block*>

Definition at line 166 of file XrdPfcFile.hh.

◆ unlink_func

using XrdPfc::unlink_func = typedef std::function<int(const std::string&)>

Definition at line 38 of file XrdPfcDirState.hh.

◆ vChunkRequest_i

using XrdPfc::vChunkRequest_i = typedef std::vector<ChunkRequest>::iterator

Definition at line 102 of file XrdPfcFile.hh.

◆ vChunkRequest_t

using XrdPfc::vChunkRequest_t = typedef std::vector<ChunkRequest>

Definition at line 101 of file XrdPfcFile.hh.

◆ vCkSum_t

typedef std::vector<uint32_t> XrdPfc::vCkSum_t

Definition at line 31 of file XrdPfcTypes.hh.

Enumeration Type Documentation

◆ CkSumCheck_e

Enumerator
CSChk_Unknown 
CSChk_None 
CSChk_Cache 
CSChk_Net 
CSChk_Both 
CSChk_TLS 

Definition at line 27 of file XrdPfcTypes.hh.

27  { CSChk_Unknown = -1, CSChk_None = 0, CSChk_Cache = 1, CSChk_Net = 2, CSChk_Both = 3,
28  CSChk_TLS = 4 // Only used during configuration parsing.
29 };
@ CSChk_Both
Definition: XrdPfcTypes.hh:27
@ CSChk_Net
Definition: XrdPfcTypes.hh:27
@ CSChk_TLS
Definition: XrdPfcTypes.hh:28
@ CSChk_Unknown
Definition: XrdPfcTypes.hh:27
@ CSChk_Cache
Definition: XrdPfcTypes.hh:27
@ CSChk_None
Definition: XrdPfcTypes.hh:27

Function Documentation

◆ OldStylePurgeDriver()

void XrdPfc::OldStylePurgeDriver ( DataFsPurgeshot ps)

PurgePin

Default purge

Definition at line 99 of file XrdPfcPurge.cc.

100 {
101  static const char *trc_pfx = "OldStylePurgeDriver ";
102  const auto &cache = Cache::TheOne();
103  const auto &conf = Cache::Conf();
104  auto &oss = *cache.GetOss();
105 
106  time_t purge_start = time(0);
107 
111  PurgePin *purge_pin = cache.GetPurgePin();
112  long long std_blocks_removed_by_pin = 0;
113  if (purge_pin)
114  {
115  // set dir stat for each path and calculate nBytes to recover for each path
116  // return total bytes to recover within the plugin
117  long long clearVal = purge_pin->GetBytesToRecover(ps);
118  if (clearVal)
119  {
120  TRACE(Debug, "PurgePin remove total " << clearVal << " bytes");
121  PurgePin::list_t &dpl = purge_pin->refDirInfos();
122  // iterate through the plugin paths
123  for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit)
124  {
125  TRACE(Debug, trc_pfx << "PurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes");
126 
127  FPurgeState fps(ppit->nBytesToRecover, oss);
128  bool scan_ok = fps.TraverseNamespace(ppit->path.c_str());
129  if ( ! scan_ok) {
130  TRACE(Warning, trc_pfx << "purge-pin scan of directory failed for " << ppit->path);
131  continue;
132  }
133 
134  fps.MoveListEntriesToMap();
135  std_blocks_removed_by_pin += UnlinkPurgeStateFilesInMap(fps, ppit->nBytesToRecover, ppit->path);
136  }
137  }
138  }
139 
143 
144  // check if the default pargue is still needed after purge pin
145  long long pin_removed_bytes = std_blocks_removed_by_pin * 512ll;
146  long long default_purge_blocks_removed = 0;
147  if (ps.m_bytes_to_remove > pin_removed_bytes)
148  {
149  // init default purge
150  long long bytes_to_remove = ps.m_bytes_to_remove - pin_removed_bytes;
151  FPurgeState purgeState(2 * bytes_to_remove, oss); // prepare twice more volume than required
152 
153  if (ps.m_age_based_purge)
154  {
155  purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge);
156  }
157  if (conf.is_uvkeep_purge_in_effect())
158  {
159  purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep);
160  }
161 
162  // Make a map of file paths, sorted by access time.
163  bool scan_ok = purgeState.TraverseNamespace("/");
164  if (!scan_ok)
165  {
166  TRACE(Error, trc_pfx << "default purge namespace traversal failed at top-directory, this should not happen.");
167  return;
168  }
169 
170  TRACE(Debug, trc_pfx << "default purge usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes.");
171 
172  purgeState.MoveListEntriesToMap();
173  default_purge_blocks_removed = UnlinkPurgeStateFilesInMap(purgeState, bytes_to_remove, "/");
174  }
175 
176  // print the total summary
178  int purge_duration = time(0) - purge_start;
179  long long total_bytes_removed = (default_purge_blocks_removed + std_blocks_removed_by_pin) * 512ll;
180  TRACE(Info, trc_pfx << "Finished, removed total size " << total_bytes_removed << ", purge duration " << purge_duration);
181 }
@ Info
@ Warning
#define TRACE(act, x)
Definition: XrdTrace.hh:63
long long UnlinkPurgeStateFilesInMap(FPurgeState &purgeState, long long bytes_to_remove, const std::string &root_path)
Definition: XrdPfcPurge.cc:24

References XrdPfc::Cache::Conf(), Macaroons::Debug, Macaroons::Error, XrdPfc::PurgePin::GetBytesToRecover(), XrdPfc::FPurgeState::getNBytesTotal(), XrdPfc::DataFsPurgeshot::m_age_based_purge, XrdPfc::DataFsPurgeshot::m_bytes_to_remove, XrdPfc::FPurgeState::MoveListEntriesToMap(), XrdPfc::PurgePin::refDirInfos(), XrdPfc::FPurgeState::setMinTime(), XrdPfc::FPurgeState::setUVKeepMinTime(), XrdPfc::Cache::TheOne(), TRACE, XrdPfc::FPurgeState::TraverseNamespace(), UnlinkPurgeStateFilesInMap(), and Warning.

Referenced by XrdPfc::ResourceMonitor::perform_purge_task().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

◆ PFC_DEFINE_TYPE_NON_INTRUSIVE() [1/2]

m_NDirectories XrdPfc::PFC_DEFINE_TYPE_NON_INTRUSIVE ( DirStateElement  ,
m_dir_name  ,
m_stats  ,
m_usage  ,
m_parent  ,
m_daughters_begin  ,
m_daughters_end   
)

◆ PFC_DEFINE_TYPE_NON_INTRUSIVE() [2/2]

XrdPfc::PFC_DEFINE_TYPE_NON_INTRUSIVE ( DirStats  ,
m_NumIos  ,
m_Duration  ,
m_BytesHit  ,
m_BytesMissed  ,
m_BytesBypassed  ,
m_BytesWritten  ,
m_StBlocksAdded  ,
m_NCksumErrors  ,
m_StBlocksRemoved  ,
m_NFilesOpened  ,
m_NFilesClosed  ,
m_NFilesCreated  ,
m_NFilesRemoved  ,
m_NDirectoriesCreated  ,
m_NDirectoriesRemoved   
)

◆ UnlinkPurgeStateFilesInMap()

long long XrdPfc::UnlinkPurgeStateFilesInMap ( FPurgeState purgeState,
long long  bytes_to_remove,
const std::string &  root_path 
)

Definition at line 24 of file XrdPfcPurge.cc.

25 {
26  static const char *trc_pfx = "UnlinkPurgeStateFilesInMap ";
27 
28  struct stat fstat;
29  int protected_cnt = 0;
30  int deleted_file_count = 0;
31  long long deleted_st_blocks = 0;
32  long long protected_st_blocks = 0;
33  long long st_blocks_to_remove = (bytes_to_remove >> 9) + 1ll;
34 
35 
36  const auto &cache = Cache::TheOne();
37  auto &resmon = Cache::ResMon();
38  auto &oss = *cache.GetOss();
39 
40  TRACE(Info, trc_pfx << "Started, root_path = " << root_path << ", bytes_to_remove = " << bytes_to_remove);
41 
42  // Loop over map and remove files with oldest values of access time.
43  for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it)
44  {
45  // Finish when enough space has been freed but not while age-based purging is in progress.
46  // Those files are marked with time-stamp = 0.
47  if (st_blocks_to_remove <= 0 && it->first != 0)
48  {
49  break;
50  }
51 
52  std::string &infoPath = it->second.path;
53  std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen);
54 
55  if (cache.IsFileActiveOrPurgeProtected(dataPath))
56  {
57  ++protected_cnt;
58  protected_st_blocks += it->second.nStBlocks;
59  TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks);
60  continue;
61  }
62 
63  // remove info file
64  if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK)
65  {
66  oss.Unlink(infoPath.c_str());
67  TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size);
68  }
69  else
70  {
71  TRACE(Error, trc_pfx << "Can't locate file " << dataPath);
72  }
73 
74  // remove data file
75  if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK)
76  {
77  st_blocks_to_remove -= it->second.nStBlocks;
78  deleted_st_blocks += it->second.nStBlocks;
79  ++deleted_file_count;
80 
81  oss.Unlink(dataPath.c_str());
82  TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first);
83 
84  resmon.register_file_purge(dataPath, it->second.nStBlocks);
85  }
86  }
87  if (protected_cnt > 0)
88  {
89  TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks);
90  }
91 
92  TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks)
93 
94  return deleted_st_blocks;
95 }
#define XrdOssOK
Definition: XrdOss.hh:50
int stat(const char *path, struct stat *buf)
int fstat(int fildes, struct stat *buf)

References Macaroons::Debug, Macaroons::Error, fstat(), XrdPfc::FPurgeState::refMap(), XrdPfc::Cache::ResMon(), XrdPfc::Info::s_infoExtensionLen, stat(), XrdPfc::Cache::TheOne(), TRACE, and XrdOssOK.

Referenced by OldStylePurgeDriver().

+ Here is the call graph for this function:
+ Here is the caller graph for this function:

Variable Documentation

◆ m_disk_total

m_NDirectories XrdPfc::m_disk_total

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_disk_used

m_NDirectories XrdPfc::m_disk_used

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_file_usage

m_NDirectories XrdPfc::m_file_usage

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_LastCloseTime

XrdPfc::m_LastCloseTime

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_LastOpenTime

XrdPfc::m_LastOpenTime

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_meta_total

m_NDirectories XrdPfc::m_meta_total

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_meta_used

m_NDirectories XrdPfc::m_meta_used

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_NFiles

XrdPfc::m_NFiles

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_NFilesOpen

XrdPfc::m_NFilesOpen

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_sshot_stats_reset_time

m_NDirectories XrdPfc::m_sshot_stats_reset_time

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ m_StBlocks

XrdPfc::m_StBlocks

Definition at line 38 of file XrdPfcDirStateSnapshot.cc.

◆ m_usage_update_time

m_NDirectories XrdPfc::m_usage_update_time

Definition at line 43 of file XrdPfcDirStateSnapshot.cc.

◆ trace_what_strings

const char * XrdPfc::trace_what_strings = {"","error ","warning ","info ","debug ","dump "}

Definition at line 26 of file XrdPfcConfiguration.cc.