XRootD
XrdPfcPurge.cc
Go to the documentation of this file.
1 #include "XrdPfc.hh"
4 #include "XrdPfcFPurgeState.hh"
5 #include "XrdPfcPurgePin.hh"
6 #include "XrdPfcTrace.hh"
7 
8 #include "XrdOss/XrdOss.hh"
9 
10 #include <sys/time.h>
11 
12 namespace
13 {
14  XrdSysTrace* GetTrace() { return XrdPfc::Cache::GetInstance().GetTrace(); }
15  const char *m_traceID = "ResourceMonitor";
16 }
17 
18 //==============================================================================
19 // OldStylePurgeDriver
20 //==============================================================================
21 namespace XrdPfc
22 {
23 
24 long long UnlinkPurgeStateFilesInMap(FPurgeState& purgeState, long long bytes_to_remove, const std::string& root_path)
25 {
26  static const char *trc_pfx = "UnlinkPurgeStateFilesInMap ";
27 
28  struct stat fstat;
29  int protected_cnt = 0;
30  int deleted_file_count = 0;
31  long long deleted_st_blocks = 0;
32  long long protected_st_blocks = 0;
33  long long st_blocks_to_remove = (bytes_to_remove >> 9) + 1ll;
34 
35 
36  const auto &cache = Cache::TheOne();
37  auto &resmon = Cache::ResMon();
38  auto &oss = *cache.GetOss();
39 
40  TRACE(Info, trc_pfx << "Started, root_path = " << root_path << ", bytes_to_remove = " << bytes_to_remove);
41 
42  // Loop over map and remove files with oldest values of access time.
43  for (FPurgeState::map_i it = purgeState.refMap().begin(); it != purgeState.refMap().end(); ++it)
44  {
45  // Finish when enough space has been freed but not while age-based purging is in progress.
46  // Those files are marked with time-stamp = 0.
47  if (st_blocks_to_remove <= 0 && it->first != 0)
48  {
49  break;
50  }
51 
52  std::string &infoPath = it->second.path;
53  std::string dataPath = infoPath.substr(0, infoPath.size() - Info::s_infoExtensionLen);
54 
55  if (cache.IsFileActiveOrPurgeProtected(dataPath))
56  {
57  ++protected_cnt;
58  protected_st_blocks += it->second.nStBlocks;
59  TRACE(Debug, trc_pfx << "File is active or purge-protected: " << dataPath << " size: " << 512ll * it->second.nStBlocks);
60  continue;
61  }
62 
63  // remove info file
64  if (oss.Stat(infoPath.c_str(), &fstat) == XrdOssOK)
65  {
66  oss.Unlink(infoPath.c_str());
67  TRACE(Dump, trc_pfx << "Removed file: '" << infoPath << "' size: " << 512ll * fstat.st_size);
68  }
69  else
70  {
71  TRACE(Error, trc_pfx << "Can't locate file " << dataPath);
72  }
73 
74  // remove data file
75  if (oss.Stat(dataPath.c_str(), &fstat) == XrdOssOK)
76  {
77  st_blocks_to_remove -= it->second.nStBlocks;
78  deleted_st_blocks += it->second.nStBlocks;
79  ++deleted_file_count;
80 
81  oss.Unlink(dataPath.c_str());
82  TRACE(Dump, trc_pfx << "Removed file: '" << dataPath << "' size: " << 512ll * it->second.nStBlocks << ", time: " << it->first);
83 
84  resmon.register_file_purge(dataPath, it->second.nStBlocks);
85  }
86  }
87  if (protected_cnt > 0)
88  {
89  TRACE(Info, trc_pfx << "Encountered " << protected_cnt << " protected files, sum of their size: " << 512ll * protected_st_blocks);
90  }
91 
92  TRACE(Info, trc_pfx << "Finished, removed " << deleted_file_count << " data files, removed total size " << 512ll * deleted_st_blocks)
93 
94  return deleted_st_blocks;
95 }
96 
97 // -------------------------------------------------------------------------------------
98 
100 {
101  static const char *trc_pfx = "OldStylePurgeDriver ";
102  const auto &cache = Cache::TheOne();
103  const auto &conf = Cache::Conf();
104  auto &oss = *cache.GetOss();
105 
106  time_t purge_start = time(0);
107 
111  PurgePin *purge_pin = cache.GetPurgePin();
112  long long std_blocks_removed_by_pin = 0;
113  if (purge_pin)
114  {
115  // set dir stat for each path and calculate nBytes to recover for each path
116  // return total bytes to recover within the plugin
117  long long clearVal = purge_pin->GetBytesToRecover(ps);
118  if (clearVal)
119  {
120  TRACE(Debug, "PurgePin remove total " << clearVal << " bytes");
121  PurgePin::list_t &dpl = purge_pin->refDirInfos();
122  // iterate through the plugin paths
123  for (PurgePin::list_i ppit = dpl.begin(); ppit != dpl.end(); ++ppit)
124  {
125  TRACE(Debug, trc_pfx << "PurgePin scanning dir " << ppit->path.c_str() << " to remove " << ppit->nBytesToRecover << " bytes");
126 
127  FPurgeState fps(ppit->nBytesToRecover, oss);
128  bool scan_ok = fps.TraverseNamespace(ppit->path.c_str());
129  if ( ! scan_ok) {
130  TRACE(Warning, trc_pfx << "purge-pin scan of directory failed for " << ppit->path);
131  continue;
132  }
133 
134  fps.MoveListEntriesToMap();
135  std_blocks_removed_by_pin += UnlinkPurgeStateFilesInMap(fps, ppit->nBytesToRecover, ppit->path);
136  }
137  }
138  }
139 
143 
144  // check if the default pargue is still needed after purge pin
145  long long pin_removed_bytes = std_blocks_removed_by_pin * 512ll;
146  long long default_purge_blocks_removed = 0;
147  if (ps.m_bytes_to_remove > pin_removed_bytes)
148  {
149  // init default purge
150  long long bytes_to_remove = ps.m_bytes_to_remove - pin_removed_bytes;
151  FPurgeState purgeState(2 * bytes_to_remove, oss); // prepare twice more volume than required
152 
153  if (ps.m_age_based_purge)
154  {
155  purgeState.setMinTime(time(0) - conf.m_purgeColdFilesAge);
156  }
157  if (conf.is_uvkeep_purge_in_effect())
158  {
159  purgeState.setUVKeepMinTime(time(0) - conf.m_cs_UVKeep);
160  }
161 
162  // Make a map of file paths, sorted by access time.
163  bool scan_ok = purgeState.TraverseNamespace("/");
164  if (!scan_ok)
165  {
166  TRACE(Error, trc_pfx << "default purge namespace traversal failed at top-directory, this should not happen.");
167  return;
168  }
169 
170  TRACE(Debug, trc_pfx << "default purge usage measured from cinfo files " << purgeState.getNBytesTotal() << " bytes.");
171 
172  purgeState.MoveListEntriesToMap();
173  default_purge_blocks_removed = UnlinkPurgeStateFilesInMap(purgeState, bytes_to_remove, "/");
174  }
175 
176  // print the total summary
178  int purge_duration = time(0) - purge_start;
179  long long total_bytes_removed = (default_purge_blocks_removed + std_blocks_removed_by_pin) * 512ll;
180  TRACE(Info, trc_pfx << "Finished, removed total size " << total_bytes_removed << ", purge duration " << purge_duration);
181 }
182 
183 } // end namespace XrdPfc
@ Warning
#define XrdOssOK
Definition: XrdOss.hh:50
int stat(const char *path, struct stat *buf)
int fstat(int fildes, struct stat *buf)
#define TRACE(act, x)
Definition: XrdTrace.hh:63
static const Configuration & Conf()
Definition: XrdPfc.cc:134
XrdSysTrace * GetTrace()
Definition: XrdPfc.hh:283
static ResourceMonitor & ResMon()
Definition: XrdPfc.cc:135
static Cache & GetInstance()
Singleton access.
Definition: XrdPfc.cc:132
static const Cache & TheOne()
Definition: XrdPfc.cc:133
map_t::iterator map_i
void setUVKeepMinTime(time_t min_time)
void setMinTime(time_t min_time)
bool TraverseNamespace(const char *root_path)
long long getNBytesTotal() const
Status of cached file. Can be read from and written into a binary file.
Definition: XrdPfcInfo.hh:41
static const size_t s_infoExtensionLen
Definition: XrdPfcInfo.hh:310
Base class for reguesting directory space to obtain.
virtual long long GetBytesToRecover(const DataFsPurgeshot &)=0
std::vector< DirInfo > list_t
list_t::iterator list_i
list_t & refDirInfos()
Definition: XrdPfc.hh:41
void OldStylePurgeDriver(DataFsPurgeshot &ps)
Definition: XrdPfcPurge.cc:99
long long UnlinkPurgeStateFilesInMap(FPurgeState &purgeState, long long bytes_to_remove, const std::string &root_path)
Definition: XrdPfcPurge.cc:24