XRootD
XrdFrmPurge.cc
Go to the documentation of this file.
1 /******************************************************************************/
2 /* */
3 /* X r d F r m P u r g e . c c */
4 /* */
5 /* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */
6 /* All Rights Reserved */
7 /* Produced by Andrew Hanushevsky for Stanford University under contract */
8 /* DE-AC02-76-SFO0515 with the Department of Energy */
9 /* */
10 /* This file is part of the XRootD software suite. */
11 /* */
12 /* XRootD is free software: you can redistribute it and/or modify it under */
13 /* the terms of the GNU Lesser General Public License as published by the */
14 /* Free Software Foundation, either version 3 of the License, or (at your */
15 /* option) any later version. */
16 /* */
17 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20 /* License for more details. */
21 /* */
22 /* You should have received a copy of the GNU Lesser General Public License */
23 /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24 /* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25 /* */
26 /* The copyright holder's institutional names and contributor's names may not */
27 /* be used to endorse or promote products derived from this software without */
28 /* specific prior written permission of the institution or contributor. */
29 /******************************************************************************/
30 
31 #include <cstdio>
32 #include <cstring>
33 #include <strings.h>
34 #include <utime.h>
35 #include <sys/param.h>
36 #include <sys/types.h>
37 
39 #include "XrdOss/XrdOss.hh"
40 #include "XrdOss/XrdOssPath.hh"
41 #include "XrdOuc/XrdOucNSWalk.hh"
42 #include "XrdOuc/XrdOucTList.hh"
43 #include "XrdOuc/XrdOucProg.hh"
44 #include "XrdOuc/XrdOucStream.hh"
45 #include "XrdOuc/XrdOucUtils.hh"
46 #include "XrdFrc/XrdFrcTrace.hh"
47 #include "XrdFrm/XrdFrmFiles.hh"
48 #include "XrdFrm/XrdFrmCns.hh"
49 #include "XrdFrm/XrdFrmConfig.hh"
50 #include "XrdFrm/XrdFrmMonitor.hh"
51 #include "XrdFrm/XrdFrmPurge.hh"
52 #include "XrdSys/XrdSysPlatform.hh"
53 
54 using namespace XrdFrc;
55 using namespace XrdFrm;
56 
57 /******************************************************************************/
58 /* L o c a l C l a s s e s */
59 /******************************************************************************/
60 /******************************************************************************/
61 /* C l a s s X r d F r m P u r g e D i r */
62 /******************************************************************************/
63 
65 {
66 public:
67 
68 void isEmpty(struct stat *dStat, const char *dPath, const char *lkFN);
69 
70 void Reset(time_t dExp)
71  {expDirTime = dExp; lowDirTime = 0; numRMD = numEMD = 0;}
72 
73 time_t expDirTime;
74 time_t lowDirTime;
75 int numRMD;
76 int numEMD;
77 
80 };
81 
82 /******************************************************************************/
83 /* i s E m p t y */
84 /******************************************************************************/
85 
86 void XrdFrmPurgeDir::isEmpty(struct stat *dStat, const char *dPath,
87  const char *lkFN)
88 {
89  static const int ossOpts = XRDOSS_isPFN | XRDOSS_resonly;
90  static const char *What = (Config.Test ? "Zorch " : "Purged ");
91  struct stat pStat;
92  struct utimbuf times;
93  char Parent[MAXPATHLEN+1], *Slash;
94  int n, rc;
95 
96 // Check if this directory is still considered active
97 //
98  numEMD++;
99  if (dStat->st_mtime > expDirTime)
100  {if (!lowDirTime || lowDirTime > dStat->st_mtime)
101  lowDirTime = dStat->st_mtime;
102  return;
103  }
104 
105 // We can expire the directory. However, we need to get the parent mtime
106 // because removing this directory should not change the parent's mtime.
107 //
108  strcpy(Parent, dPath);
109  n = strlen(Parent);
110  if (Parent[n-1] == '/') Parent[--n] = '\0';
111  if ((Slash = rindex(Parent, '/')))
112  {*Slash = '\0';
113  if (stat(Parent, &pStat)) Slash = 0;
114  }
115 
116 // Delete the directory
117 //
118  if (Config.Test) rc = 0;
119  else if (!(rc = Config.ossFS->Remdir(dPath, ossOpts)) && Slash)
120  {times.actime = pStat.st_atime;
121  times.modtime = pStat.st_mtime;
122  utime(Parent, &times);
123  XrdFrmCns::Rmd(dPath);
124  }
125 
126 // Report if successful
127 //
128  if (!rc)
129  {numRMD++;
130  if (Config.Verbose)
131  {char sbuff[64];
132  struct tm tNow;
133  localtime_r(&(dStat->st_mtime), &tNow);
134  sprintf(sbuff, "%02d%02d%02d %02d:%02d:%02d ",
135  tNow.tm_year-100, tNow.tm_mon+1, tNow.tm_mday,
136  tNow.tm_hour, tNow.tm_min, tNow.tm_sec);
137  Say.Say(What, "empty dir ", sbuff, dPath);
138  }
139  }
140 }
141 
142 /******************************************************************************/
143 /* C l a s s X r d F r m P u r g e */
144 /******************************************************************************/
145 /******************************************************************************/
146 /* S t a t i c M e m b e r s */
147 /******************************************************************************/
148 
149 XrdFrmPurge *XrdFrmPurge::First = 0;
150 XrdFrmPurge *XrdFrmPurge::Default = 0;
151 
152 XrdOucProg *XrdFrmPurge::PolProg = 0;
153 XrdOucStream *XrdFrmPurge::PolStream = 0;
154 
155 int XrdFrmPurge::Left2Do = 0;
156 
157 time_t XrdFrmPurge::lastReset = 0;
158 time_t XrdFrmPurge::nextReset = 0;
159 
160 /******************************************************************************/
161 /* C o n s t r u c t o r */
162 /******************************************************************************/
163 
164 XrdFrmPurge::XrdFrmPurge(const char *snp, XrdFrmPurge *spp) : FSTab(1)
165 {
166  strncpy(SName, snp, sizeof(SName)-1); SName[sizeof(SName)-1] = '\0';
167  Next = spp;
168  freeSpace = 0;
169  usedSpace =-1;
170  pmaxSpace = 0;
171  totlSpace = 0;
172  contSpace = 0;
173  minFSpace = 0;
174  maxFSpace = 0;
175  Enabled = 0;
176  Stop = 0;
177  SNlen = strlen(SName);
178  memset(DeferQ, 0, sizeof(DeferQ));
179  Clear();
180 }
181 
182 /******************************************************************************/
183 /* Private: A d d */
184 /******************************************************************************/
185 
186 void XrdFrmPurge::Add(XrdFrmFileset *sP)
187 {
188  EPNAME("Add");
189  XrdOucNSWalk::NSEnt *baseFile = sP->baseFile();
190  XrdFrmPurge *psP = Default;
191  const char *Why;
192  time_t xTime;
193 
194 // First, get the space name associated with the base file
195 //
196  if ((baseFile->Link))
197  {char snBuff[XrdOssSpace::minSNbsz];
198  XrdOssPath::getCname(0, snBuff, baseFile->Link, baseFile->Lksz);
199  if (!(psP = Find(snBuff))) psP = Default;
200  }
201 
202 // Ignore the file is the space is not enabled for purging
203 //
204  if (!(psP->Enabled)) {delete sP; return;}
205  psP->numFiles++;
206 
207 // Check to see if the file is really eligible for purging
208 //
209  if ((Why = psP->Eligible(sP, xTime)))
210  {DEBUG(sP->basePath() <<"cannot be purged; " <<Why);
211  delete sP;
212  return;
213  }
214 
215 // Add the file to the purge table or the defer queue based on access time
216 //
217  if (xTime >= psP->Hold) psP->FSTab.Add(sP);
218  else psP->Defer(sP, xTime);
219 }
220 
221 /******************************************************************************/
222 /* Private: A d v a n c e */
223 /******************************************************************************/
224 
225 XrdFrmFileset *XrdFrmPurge::Advance()
226 {
227  XrdFrmFileset *fP, *xP;
228  int n;
229 
230 // Find a defer queue entry that meets the hold threshold
231 //
232  for (n = DeferQsz-1; n >= 0 && !DeferQ[n]; n--) {}
233  if (n < 0) return 0;
234  if (time(0) - DeferT[n] > Hold) return 0;
235  fP = DeferQ[n]; DeferQ[n] = 0; DeferT[n] = 0;
236 
237 // Try to re-add everything in this queue
238 //
239  while((xP = fP))
240  {fP = fP->Next;
241  if (xP->Refresh(0,0)) Add(xP);
242  else delete xP;
243  }
244 
245 // Return based on whether something now exists in the purge table
246 //
247  return FSTab.Oldest();
248 }
249 
250 /******************************************************************************/
251 /* Private: C l e a r */
252 /******************************************************************************/
253 
254 void XrdFrmPurge::Clear()
255 {
256  XrdFrmFileset *fP;
257  int n;
258 
259 // Zero out the defer queue
260 //
261  for (n = 0; n < DeferQsz; n++)
262  while ((fP = DeferQ[n])) {DeferQ[n] = fP->Next; delete fP;}
263  memset(DeferT, 0, sizeof(DeferT));
264 
265 // Purge the eligible file table
266 //
267  FSTab.Purge();
268 
269 // Clear counters
270 //
271  numFiles = 0; prgFiles = 0; purgBytes = 0;
272 }
273 
274 /******************************************************************************/
275 /* Private: D e f e r */
276 /******************************************************************************/
277 
278 void XrdFrmPurge::Defer(XrdFrmFileset *sP, time_t xTime)
279 {
280  time_t aTime = sP->baseFile()->Stat.st_atime;
281  int n = xTime/DeferQsz;
282 
283 // Slot the entry into the defer queue vector
284 //
285  if (n >= DeferQsz) n = DeferQsz-1;
286  if (!DeferQ[n] || aTime < DeferT[n]) DeferT[n] = aTime;
287  sP->Next = DeferQ[n];
288  DeferQ[n] = sP;
289 }
290 
291 /******************************************************************************/
292 /* D i s p l a y */
293 /******************************************************************************/
294 
296 {
297  XrdFrmConfig::VPInfo *vP = Config.pathList;
298  XrdFrmPurge *spP = First;
299  XrdOucTList *tP;
300  const char *isExt;
301  char buff[1024], minfsp[32], maxfsp[32];
302 
303 // Type header
304 //
305  Say.Say("=====> ", "Purge configuration:");
306 
307 // Display what we will scan
308 //
309  while(vP)
310  {Say.Say("=====> ", "Scanning ", (vP->Val?"r/w: ":"r/o: "), vP->Name);
311  tP = vP->Dir;
312  while(tP) {Say.Say("=====> ", "Excluded ", tP->text); tP = tP->next;}
313  vP = vP->Next;
314  }
315 
316 // Display directory hold value
317 //
318  if (Config.dirHold < 0) strcpy(buff, "forever");
319  else sprintf(buff, "%d", Config.dirHold);
320  Say.Say("=====> ", "Directory hold: ", buff);
321 
322 // Run through all of the policies, displaying each one
323 //
324  spP = First;
325  while(spP)
326  {if (spP->Enabled)
327  {XrdOucUtils::fmtBytes(spP->minFSpace, minfsp, sizeof(minfsp));
328  XrdOucUtils::fmtBytes(spP->maxFSpace, maxfsp, sizeof(maxfsp));
329  isExt = spP->Ext ? " polprog" : "";
330  sprintf(buff, "policy %s min %s max %s free; hold: %d%s",
331  spP->SName, minfsp, maxfsp, spP->Hold, isExt);
332  } else sprintf(buff, "policy %s nopurge", spP->SName);
333  Say.Say("=====> ", buff);
334  spP = spP->Next;
335  }
336 }
337 
338 /******************************************************************************/
339 /* Private: E l i g i b l e */
340 /******************************************************************************/
341 
342 const char *XrdFrmPurge::Eligible(XrdFrmFileset *sP, time_t &xTime, int hTime)
343 {
344  XrdOucNSWalk::NSEnt *baseFile = sP->baseFile();
345  time_t aTime, mTime, nowTime = time(0);
346 
347 // Get the acess time and modification time
348 //
349  aTime = baseFile->Stat.st_atime;
350  mTime = baseFile->Stat.st_mtime;
351 
352 // File is not eligible if it's been accessed too recently
353 //
354  xTime = static_cast<int>(nowTime - aTime);
355  if (hTime && xTime <= hTime) return "is in hold";
356 
357 // File is ineligible if it has a fail file
358 //
359  if (sP->failFile()) return "has fail file";
360 
361 // If there is a lock file and the file has not migrated, then ineligible
362 // Note that entries were pre-screened for copy file requirements.
363 //
364  if (sP->cpyInfo.Attr.cpyTime
365  && sP->cpyInfo.Attr.cpyTime < static_cast<long long>(mTime))
366  return "not migrated";
367 
368 // If there is no pin info, then it is eligible subject to external policy
369 //
370  if (!(sP->pinInfo.Attr.Flags)) return 0;
371 
372 // See if pin is permanent
373 //
375  return "is perm pinned";
376 
377 // See if the file is pinned until a certain time
378 //
380  && sP->pinInfo.Attr.pinTime > static_cast<long long>(nowTime))
381  return "is time pinned";
382 
383 // Check if the file can only be unpinned after going idle
384 //
386  && sP->pinInfo.Attr.pinTime > static_cast<long long>(xTime))
387  return "is pin deferred";
388  return 0;
389 }
390 
391 /******************************************************************************/
392 /* Private: F i n d */
393 /******************************************************************************/
394 
395 XrdFrmPurge *XrdFrmPurge::Find(const char *snp)
396 {
397  XrdFrmPurge *spP = First;
398 
399 // See if we already have this space defined
400 //
401  while(spP && strcmp(snp, spP->SName)) spP = spP->Next;
402  return spP;
403 }
404 
405 /******************************************************************************/
406 /* I n i t */
407 /******************************************************************************/
408 
409 int XrdFrmPurge::Init(XrdOucTList *sP, long long minV, int hVal)
410 {
411  static char pVec[] = {char(XrdFrmConfig::PP_sname),
412  char(XrdFrmConfig::PP_pfn),
416  };
417 
419  XrdFrmPurge *xP, *ppP = 0, *spP = First;
420  XrdOucTList *tP;
421  char xBuff[32];
422  int setIt, rc, haveExt = 0;
423 
424 // The first step is to remove any defined policies for which there is no space
425 //
426  while(spP)
427  {vP = Config.VPList;
428  while(vP && strcmp(spP->SName, vP->Name)) vP = vP->Next;
429  if (!vP && strcmp("public", spP->SName))
430  {Say.Emsg("Init", "Purge policy", spP->SName,
431  "deleted; space not defined.");
432  if (ppP) ppP->Next = spP->Next;
433  else First = spP->Next;
434  xP = spP; spP = spP->Next; delete xP;
435  } else {ppP = spP; spP = spP->Next;}
436  }
437 
438 // For each space enable it and optionally over-ride policy
439 //
440  spP = First;
441  while(spP)
442  {setIt = 1;
443  if ((tP = sP))
444  {while(tP && strcmp(tP->text, spP->SName)) tP = tP->next;
445  if (!tP) setIt = 0;
446  }
447  if (setIt)
448  {if (minV) spP->minFSpace = spP->maxFSpace = minV;
449  if (hVal >= 0) {spP->Hold = hVal; spP->Hold2x = hVal*2;}
450  if (spP->minFSpace && spP->Hold >= 0)
451  {spP->Enabled = 1; haveExt |= spP->Ext;}
452  }
453  spP = spP->Next;
454  }
455 
456 // Go through each space policy getting the actual space and calculating
457 // the targets based on the policy (we need to do this only once)
458 //
459  spP = First; ppP = 0;
460  while(spP)
461  {XrdOssVSInfo vsInfo;
462  if ((rc = Config.ossFS->StatVS(&vsInfo, spP->SName, 1)))
463  {Say.Emsg("Init", rc, "calculate space for", spP->SName);
464  if (ppP) ppP->Next = spP->Next;
465  else First = spP->Next;
466  xP = spP; spP = spP->Next; delete xP; continue;
467  }
468  spP->totlSpace = vsInfo.Total;
469  spP->spaceTLen = sprintf(xBuff, "%lld", vsInfo.Total);
470  spP->spaceTotl = strdup(xBuff);
471  spP->pmaxSpace = vsInfo.Large;
472  spP->spaceTLep = sprintf(xBuff, "%lld", vsInfo.Large);
473  spP->spaceTotP = strdup(xBuff);
474  if (spP->minFSpace < 0)
475  {spP->minFSpace = vsInfo.Total * XRDABS(spP->minFSpace) / 100LL;
476  spP->maxFSpace = vsInfo.Total * XRDABS(spP->maxFSpace) / 100LL;
477  } else if (vsInfo.Total < spP->minFSpace
478  || vsInfo.Total < spP->maxFSpace)
479  Say.Emsg("Init", "Warning: ", spP->SName, " min/max free "
480  "space policy exceeds total available!");
481  ppP = spP; spP = spP->Next;
482  }
483 
484 // Make sure "public" still exists. While this should not happen, we check for
485 // this possibility anyway.
486 //
487  if (!(Default = Find("public")))
488  {Say.Emsg("Init", "Unable to start purge; no public policy found.");
489  return 0;
490  }
491 
492 // If a policy program is present, then we need to verify it
493 //
494  if (Config.pProg && haveExt)
495  {PolProg = new XrdOucProg(&Say);
496  if (PolProg->Setup(Config.pProg) || PolProg->Start()) return 0;
497  PolStream = PolProg->getStream();
498  if (!Config.pVecNum)
499  {memcpy(Config.pVec, pVec, sizeof(pVec));
500  Config.pVecNum = sizeof(pVec);
501  }
502  }
503 
504 // All went well
505 //
506  return 1;
507 }
508 
509 /******************************************************************************/
510 /* Private: L o w O n S p a c e */
511 /******************************************************************************/
512 
513 // This method *must* be called prior to Purge() and returns:
514 // =0 -> Purge not needed.
516 
517 int XrdFrmPurge::LowOnSpace()
518 {
519  XrdFrmPurge *psP = First;
520  time_t eNow;
521 
522 // Recalculate free space and set initial status
523 //
524  Left2Do = 0;
525  while(psP)
526  {if (psP->Enabled)
527  {XrdOssVSInfo VSInfo;
528  if (Config.ossFS->StatVS(&VSInfo, psP->SName, 1)) psP->Stop = 1;
529  else {psP->freeSpace = VSInfo.Free;
530  psP->contSpace = VSInfo.LFree;
531  psP->usedSpace = VSInfo.Usage;
532  if (psP->freeSpace >= psP->minFSpace) psP->Stop = 1;
533  else {Left2Do++; psP->Stop = 0;}
534  }
535  } else psP->Stop = 1;
536  psP = psP->Next;
537  }
538 
539 // If enough space then indicate no purge is needed
540 //
541  if (!Left2Do) return 0;
542 
543 // Reset all policies to prepare for purging
544 //
545  psP = First;
546  while(psP)
547  {psP->Clear();
548  psP = psP->Next;
549  }
550 
551 // We must check whether or not a full name space scan is required. This is
552 // based on the last time we did one and whether or not a space needs one now.
553 //
554  eNow = time(0);
555  if (eNow >= nextReset) {lastReset = eNow; nextReset = 0; Scan();}
556  return 1;
557 }
558 
559 /******************************************************************************/
560 /* P o l i c y */
561 /******************************************************************************/
562 
563 XrdFrmPurge *XrdFrmPurge::Policy(const char *sname, long long minV,
564  long long maxV, int hVal, int xVal)
565 {
566  XrdFrmPurge *psP;
567 
568 // Find or create a new policy
569 //
570  if (!(psP = Find(sname))) First = psP = new XrdFrmPurge(sname, First);
571 
572 // Fill out the policy
573 //
574  psP->minFSpace = minV;
575  psP->maxFSpace = maxV;
576  psP->Hold = hVal;
577  psP->Hold2x = hVal*2;
578  psP->Ext = xVal;
579  return psP;
580 }
581 
582 /******************************************************************************/
583 /* P u r g e */
584 /******************************************************************************/
585 
587 {
588  XrdFrmPurge *psP = First;
589 
590 // Check if are low on space, if not, ignore the call
591 //
592  if (!LowOnSpace())
593  {Say.Emsg("Purge", "Purge cycle skipped; all policies met.");
594  return;
595  }
596 
597 // Report data at the start of the purge cycle
598 //
599  Say.Emsg("Purge", "Purge cycle started.");
600  Stats(0);
601 
602 // Cycle through each space until we no longer can cycle
603 //
604 do{psP = First;
605  while(psP && Left2Do)
606  {if (!(psP->Stop) && (psP->Stop = psP->PurgeFile())) Left2Do--;
607  psP = psP->Next;
608  }
609  } while(Left2Do);
610 
611 // Report data at the end of the purge cycle
612 //
613  Stats(1);
614  Say.Emsg("Purge", "Purge cycle ended.");
615 }
616 
617 /******************************************************************************/
618 /* Private: P u r g e F i l e */
619 /******************************************************************************/
620 
621 int XrdFrmPurge::PurgeFile()
622 {
623  EPNAME("PurgeFile");
624  XrdFrmFileset *fP;
625  const char *fn, *Why;
626  time_t xTime;
627  int rc, FilePurged = 0;
628 
629 // If we have don't have a file, see if we can grab some from the defer queue
630 //
631 do{if (!(fP = FSTab.Oldest()) && !(fP = Advance()))
632  {time_t nextScan = time(0)+Hold;
633  if (!nextReset || nextScan < nextReset) nextReset = nextScan;
634  return 1;
635  }
636  Why = "file in use";
637  if (fP->Refresh() && !(Why = Eligible(fP, xTime, Hold))
638  && (!Ext || !(Why = XPolOK(fP))))
639  {fn = fP->basePath();
640  rc = (Config.Test ? 0 : PurgeFile(fP, fn));
641  if (!rc) {prgFiles++; FilePurged = 1;
642  freeSpace += fP->baseFile()->Stat.st_size;
643  purgBytes += fP->baseFile()->Stat.st_size;
644  if (Config.Verbose) Track(fP);
645  }
646  } else {DEBUG("Purge " <<SName <<": keeping " <<fP->basePath() <<"; " <<Why);}
647  delete fP;
648  } while(!FilePurged && !Stop);
649 
650 // All done, indicate whether we should stop now
651 //
652  return freeSpace >= maxFSpace || Stop;
653 }
654 
655 /******************************************************************************/
656 
657 int XrdFrmPurge::PurgeFile(XrdFrmFileset *fP, const char *pFN)
658 {
659  static const char *Me = "frm_purged:0.0@localhost";
660  static const int unOpts = XRDOSS_isPFN|XRDOSS_isMIG;
661  XrdOucNSWalk::NSEnt *bfP;
662  char mBuff[MAXPATHLEN+1024];
663  int n, rc, isLFN = 0;
664 
665 // First try to unlink the file
666 //
667  if ((rc = Config.ossFS->Unlink(pFN, unOpts))) return rc;
668 
669 // Now convert pfn to lfn
670 //
671  if (!(isLFN = Config.LogicalPath(pFN, mBuff, sizeof(mBuff))))
672  strcpy(mBuff,pFN);
673 
674 // Notify the cmsd and the cnsd
675 //
676  if (Config.cmsPath) Config.cmsPath->Gone(mBuff, !isLFN);
677  XrdFrmCns::Rm(mBuff, isLFN);
678 
679 // Monitor this event
680 //
682  {n = strlen(mBuff);
683  bfP = fP->baseFile();
684  snprintf(mBuff+n, sizeof(mBuff)-n,
685  "\n&tod=%lld&sz=%lld&at=%lld&ct=%lld&mt=%lld&fn=%c",
686  static_cast<long long>(time(0)),
687  static_cast<long long>(bfP->Stat.st_size),
688  static_cast<long long>(bfP->Stat.st_atime),
689  static_cast<long long>(bfP->Stat.st_ctime),
690  static_cast<long long>(bfP->Stat.st_mtime),
691  (isLFN ? 'l' : 'p'));
693  }
694 
695 // All done
696 //
697  return 0;
698 }
699 
700 /******************************************************************************/
701 /* Private: S c a n */
702 /******************************************************************************/
703 
704 void XrdFrmPurge::Scan()
705 {
708  static time_t lastHP = time(0), nextDP = 0, nowT = time(0);
709  static XrdFrmPurgeDir purgeDir;
710  static XrdOucNSWalk::CallBack *cbP;
711 
712  XrdFrmConfig::VPInfo *vP = Config.pathList;
713  XrdFrmFileset *sP;
714  XrdFrmFiles *fP;
715  const char *Extra;
716  char buff[128];
717  int needLF, ec = 0, Bad = 0, aFiles = 0, bFiles = 0;
718 
719 // Purge that bad file table evey 24 hours to keep complaints down
720 //
721  if (nowT - lastHP >= 86400) {XrdFrmFileset::Purge(); lastHP = nowT;}
722 
723 // Determine if we need to do an empty directory trim
724 //
725  if (Config.dirHold < 0 || nowT < nextDP) {cbP = 0; Extra = 0;}
726  else {nextDP = nowT + Config.dirHold;
727  purgeDir.Reset(nowT - Config.dirHold);
728  cbP = (XrdOucNSWalk::CallBack *)&purgeDir;
729  Extra = "and empty directory";
730  }
731 
732 // Indicate scan started
733 //
734  VMSG("Scan", "Name space", Extra, "scan started. . .");
735 
736 // Process each directory
737 //
738  do {fP = new XrdFrmFiles(vP->Name, Opts, vP->Dir, cbP);
739  needLF = vP->Val;
740  while((sP = fP->Get(ec,1)))
741  {aFiles++;
742  if (sP->Screen(needLF)) Add(sP);
743  else {delete sP; bFiles++;}
744  }
745  if (ec) Bad = 1;
746  delete fP;
747  } while((vP = vP->Next));
748 
749 // If we did a directory purge, schedule the next one and say what we did
750 //
751  if (cbP)
752  {if ((purgeDir.numEMD - purgeDir.numRMD) > 0
753  && purgeDir.lowDirTime + Config.dirHold < nextDP)
754  nextDP = purgeDir.lowDirTime + Config.dirHold;
755  sprintf(buff, "%d of %d empty dir%s removed", purgeDir.numRMD,
756  purgeDir.numEMD, (purgeDir.numEMD != 1 ? "s":""));
757  VMSG("Scan", "Empty directory space scan ended;", buff);
758  }
759 
760 // Indicate scan ended
761 //
762  sprintf(buff, "%d file%s with %d error%s", aFiles, (aFiles != 1 ? "s":""),
763  bFiles, (bFiles != 1 ? "s":""));
764  VMSG("Scan", "Name space scan ended;", buff);
765 
766 // Issue warning if we encountered errors
767 //
768  if (Bad) Say.Emsg("Scan", "Errors encountered while scanning for "
769  "purgeable files.");
770 }
771 
772 /******************************************************************************/
773 /* Private: S t a t s */
774 /******************************************************************************/
775 
776 void XrdFrmPurge::Stats(int Final)
777 {
778  XrdFrmPurge *xsP, *psP = First;
779  long long pVal, xBytes, zBytes;
780  const char *xWhat, *nWhat, *zWhat;
781  char fBuff[64], uBuff[80], sBuff[512], xBuff[64], zBuff[64];
782  int nFiles;
783 
784 // Report data for each enabled space
785 //
786  while((xsP = psP))
787  {psP = psP->Next;
788  if (!(xsP->Enabled)) continue;
789  if (xsP->usedSpace >= 0)
790  {if (Final) xsP->usedSpace -= xsP->purgBytes;
791  XrdOucUtils::fmtBytes(xsP->usedSpace, fBuff, sizeof(fBuff));
792  pVal = xsP->usedSpace*100/xsP->totlSpace;
793  sprintf(uBuff, "used %s (%lld%%) ", fBuff, pVal);
794  } else *uBuff = '\0';
795  XrdOucUtils::fmtBytes(xsP->freeSpace, fBuff, sizeof(fBuff));
796  pVal = xsP->freeSpace*100/xsP->totlSpace;
797  if (Final)
798  {xBytes = xsP->purgBytes; xWhat = "freed";
799  if ((zBytes = xsP->maxFSpace - xsP->freeSpace) > 0)
800  {XrdOucUtils::fmtBytes(zBytes, zBuff, sizeof(zBuff));
801  zWhat = " deficit";
802  } else {*zBuff = '\0'; zWhat = "need met";}
803  nFiles = xsP->prgFiles; nWhat = "prgd";
804  } else {
805  xBytes = (xsP->freeSpace < xsP->minFSpace
806  ? xsP->maxFSpace - xsP->freeSpace : 0);
807  nFiles = xsP->FSTab.Count();
808  xWhat = "needed"; nWhat = "idle"; *zBuff = '\0'; zWhat = "";
809  }
810  XrdOucUtils::fmtBytes(xBytes, xBuff, sizeof(xBuff));
811  sprintf(sBuff, " %sfree %s (%lld%%) %d files %d %s; %s %s %s%s",
812  uBuff,fBuff,pVal,xsP->numFiles,nFiles,nWhat,
813  xBuff,xWhat,zBuff,zWhat);
814  Say.Say("++++++ ", xsP->SName, sBuff);
815  }
816 }
817 
818 /******************************************************************************/
819 /* Private: T r a c k */
820 /******************************************************************************/
821 
822 void XrdFrmPurge::Track(XrdFrmFileset *sP)
823 {
824  XrdOucNSWalk::NSEnt *fP = sP->baseFile();
825  const char *What = (Config.Test ? "Zorch " : "Purged ");
826  char sbuff[128], fszbf[16];
827  struct tm tNow;
828 
829 // Format the size
830 //
831  XrdOucUtils::fmtBytes(static_cast<long long>(fP->Stat.st_size),
832  fszbf, sizeof(fszbf));
833 
834 // Format the information and display it
835 //
836  localtime_r(&(fP->Stat.st_atime), &tNow);
837  sprintf(sbuff, " %8s %02d%02d%02d %02d:%02d:%02d ", fszbf,
838  tNow.tm_year-100, tNow.tm_mon+1, tNow.tm_mday,
839  tNow.tm_hour, tNow.tm_min, tNow.tm_sec);
840 
841  Say.Say(What, SName, sbuff, sP->basePath());
842 }
843 
844 /******************************************************************************/
845 /* Private: X P o l O K */
846 /******************************************************************************/
847 
848 const char *XrdFrmPurge::XPolOK(XrdFrmFileset *fsP)
849 {
850  static char neg1[] = {'-','1','\0'};
851  XrdOucNSWalk::NSEnt *fP = fsP->baseFile();
852  char *Data[sizeof(Config.pVec)*2+2];
853  int Dlen[sizeof(Config.pVec)*2+2];
854  char atBuff[32], ctBuff[32], mtBuff[32], fsBuff[32], spBuff[32], usBuff[32];
855  char *Resp;
856  int i, k = 0;
857 
858 // Construct the data to be sent (not mt here)
859 //
860  for (i = 0; i < Config.pVecNum; i++)
861  {switch(Config.pVec[i])
863  Data[k] = atBuff;
864  Dlen[k] = sprintf(atBuff, "%lld",
865  static_cast<long long>(fP->Stat.st_atime));
866  break;
868  Data[k] = ctBuff;
869  Dlen[k] = sprintf(ctBuff, "%lld",
870  static_cast<long long>(fP->Stat.st_ctime));
871  break;
873  Data[k] = fP->File; Dlen[k] = strlen(fP->File);
874  break;
876  Data[k] = fsBuff;
877  Dlen[k] = sprintf(fsBuff, "%lld",
878  static_cast<long long>(fP->Stat.st_size));
879  break;
881  Data[k] = spBuff;
882  Dlen[k] = sprintf(spBuff, "%lld", freeSpace);
883  break;
885  Data[k] = mtBuff;
886  Dlen[k] = sprintf(mtBuff, "%lld",
887  static_cast<long long>(fP->Stat.st_mtime));
888  break;
890  Data[k] = (char *)fsP->basePath();
891  Dlen[k] = strlen(Data[k]);
892  break;
894  Data[k] = SName; Dlen[k] = SNlen;
895  break;
897  Data[k] = spaceTotl; Dlen[k] = spaceTLen;
898  break;
900  if (usedSpace < 0) {Data[k] = neg1; Dlen[k]=2;}
901  else {Dlen[k] = sprintf(usBuff, "%lld",
902  usedSpace - purgBytes);
903  Data[k] = usBuff;
904  }
905  break;
906  default: break;
907  }
908  Data[++k] = (char *)" "; Dlen[k] = 1; k++;
909  }
910 
911 // Now finish up the vector
912 //
913  Data[k-1] = (char *)"\n"; Data[k] = 0; Dlen[k] = 0;
914 
915 // Feed the program this information get the response
916 //
917  if (PolProg->Feed((const char **)Data, Dlen) || !(Resp=PolStream->GetLine()))
918  {Stop = 1; return "external policy failed";}
919 
920 // Decode the response (single line with a charcater y|n|a)
921 //
922  if (*Resp == 'y') return 0;
923  if (*Resp == 'n') return "external policy reject";
924  Stop = 1;
925  return "external policy stop";
926 }
#define DEBUG(x)
Definition: XrdBwmTrace.hh:54
#define EPNAME(x)
Definition: XrdBwmTrace.hh:56
#define VMSG(a,...)
Definition: XrdFrcTrace.hh:66
#define XRDOSS_isPFN
Definition: XrdOss.hh:469
#define XRDOSS_isMIG
Definition: XrdOss.hh:470
#define XRDOSS_resonly
Definition: XrdOss.hh:486
int stat(const char *path, struct stat *buf)
@ Default
#define XRDABS(x)
const kXR_char XROOTD_MON_MAPPURG
XrdOss * ossFS
long long cpyTime
Definition: XrdFrcXAttr.hh:55
static const char pinKeep
Definition: XrdFrcXAttr.hh:135
static const char pinPerm
Definition: XrdFrcXAttr.hh:133
static const char pinIdle
Definition: XrdFrcXAttr.hh:134
long long pinTime
Definition: XrdFrcXAttr.hh:127
static void Rmd(const char *Path, int islfn=0)
Definition: XrdFrmCns.hh:56
static void Rm(const char *Path, int islfn=0)
Definition: XrdFrmCns.hh:53
static const int CompressD
Definition: XrdFrmFiles.hh:122
static const int NoAutoDel
Definition: XrdFrmFiles.hh:123
XrdFrmFileset * Get(int &rc, int noBase=0)
Definition: XrdFrmFiles.cc:340
static const int Recursive
Definition: XrdFrmFiles.hh:121
int Refresh(int isMig=0, int doLock=1)
Definition: XrdFrmFiles.cc:118
XrdOucXAttr< XrdFrcXAttrCpy > cpyInfo
Definition: XrdFrmFiles.hh:55
int Screen(int needLF=1)
Definition: XrdFrmFiles.cc:170
XrdOucXAttr< XrdFrcXAttrPin > pinInfo
Definition: XrdFrmFiles.hh:56
XrdFrmFileset * Next
Definition: XrdFrmFiles.hh:90
XrdOucNSWalk::NSEnt * baseFile()
Definition: XrdFrmFiles.hh:60
const char * basePath()
Definition: XrdFrmFiles.hh:61
static void Purge()
Definition: XrdFrmFiles.hh:77
XrdOucNSWalk::NSEnt * failFile()
Definition: XrdFrmFiles.hh:62
static char monPURGE
static kXR_unt32 Map(char code, const char *uname, const char *path)
void isEmpty(struct stat *dStat, const char *dPath, const char *lkFN)
Definition: XrdFrmPurge.cc:86
void Reset(time_t dExp)
Definition: XrdFrmPurge.cc:70
time_t expDirTime
Definition: XrdFrmPurge.cc:73
time_t lowDirTime
Definition: XrdFrmPurge.cc:74
static void Purge()
Definition: XrdFrmPurge.cc:586
static void Display()
Definition: XrdFrmPurge.cc:295
static int Init(XrdOucTList *sP=0, long long minV=-1, int hVal=-1)
Definition: XrdFrmPurge.cc:409
XrdFrmPurge(const char *snp, XrdFrmPurge *spp=0)
Definition: XrdFrmPurge.cc:164
static XrdFrmPurge * Policy(const char *sname)
Definition: XrdFrmPurge.hh:52
void Purge()
Definition: XrdFrmTSort.cc:143
XrdFrmFileset * Oldest()
Definition: XrdFrmTSort.cc:113
int Add(XrdFrmFileset *fsp)
Definition: XrdFrmTSort.cc:39
static int getCname(const char *path, char *Cache, char *lbuf=0, int lbsz=0)
Definition: XrdOssPath.cc:241
static const int minSNbsz
Definition: XrdOssSpace.hh:45
long long LFree
Definition: XrdOssVS.hh:93
long long Usage
Definition: XrdOssVS.hh:94
long long Large
Definition: XrdOssVS.hh:92
long long Total
Definition: XrdOssVS.hh:90
long long Free
Definition: XrdOssVS.hh:91
virtual int Remdir(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
virtual int StatVS(XrdOssVSInfo *vsP, const char *sname=0, int updt=0)
Definition: XrdOss.cc:117
virtual int Unlink(const char *path, int Opts=0, XrdOucEnv *envP=0)=0
int Start(void)
Definition: XrdOucProg.cc:349
XrdOucStream * getStream() const
Definition: XrdOucProg.hh:68
int Feed(const char *data[], const int dlen[])
Definition: XrdOucProg.cc:63
int Setup(const char *prog, XrdSysError *errP=0, int(*Proc)(XrdOucStream *, char **, int)=0)
Definition: XrdOucProg.cc:296
char * GetLine()
XrdOucTList * next
Definition: XrdOucTList.hh:45
char * text
Definition: XrdOucTList.hh:46
static int fmtBytes(long long val, char *buff, int bsz)
Definition: XrdOucUtils.cc:390
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:95
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
Definition: XrdSysError.cc:141
FinalOperation Final
XrdSysError Say
XrdCmsConfig Config
XrdSysError Say
int Opts
Definition: XrdMpxStats.cc:58
XrdPosixStats Stats
Definition: XrdPosixFile.cc:64