XRootD
XrdFrmFiles.cc
Go to the documentation of this file.
1 /******************************************************************************/
2 /* */
3 /* X r d F r m F i l e s . c c */
4 /* */
5 /* (c) 2009 by the Board of Trustees of the Leland Stanford, Jr., University */
6 /* All Rights Reserved */
7 /* Produced by Andrew Hanushevsky for Stanford University under contract */
8 /* DE-AC02-76-SFO0515 with the Department of Energy */
9 /* */
10 /* This file is part of the XRootD software suite. */
11 /* */
12 /* XRootD is free software: you can redistribute it and/or modify it under */
13 /* the terms of the GNU Lesser General Public License as published by the */
14 /* Free Software Foundation, either version 3 of the License, or (at your */
15 /* option) any later version. */
16 /* */
17 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20 /* License for more details. */
21 /* */
22 /* You should have received a copy of the GNU Lesser General Public License */
23 /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24 /* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25 /* */
26 /* The copyright holder's institutional names and contributor's names may not */
27 /* be used to endorse or promote products derived from this software without */
28 /* specific prior written permission of the institution or contributor. */
29 /******************************************************************************/
30 
31 #include <cerrno>
32 #include <cstring>
33 #include <strings.h>
34 #include <unistd.h>
35 #include <fcntl.h>
36 #include <sys/param.h>
37 #include <sys/stat.h>
38 #include <sys/types.h>
39 
40 #include "XrdFrc/XrdFrcTrace.hh"
41 #include "XrdFrm/XrdFrmConfig.hh"
42 #include "XrdFrm/XrdFrmFiles.hh"
43 #include "XrdOuc/XrdOucTList.hh"
44 #include "XrdSys/XrdSysPlatform.hh"
45 
46 using namespace XrdFrc;
47 using namespace XrdFrm;
48 
49 /******************************************************************************/
50 /* C l a s s X r d F r m F i l e s e t */
51 /******************************************************************************/
52 /******************************************************************************/
53 /* S t a t i c O b j e c t s */
54 /******************************************************************************/
55 
56 XrdOucHash<char> XrdFrmFileset::BadFiles;
57 
58 /******************************************************************************/
59 /* C o n s t r u c t o r */
60 /******************************************************************************/
61 
63  : Next(sP), dInfo(diP)
64 { memset(File, 0, sizeof(File));
65  if (diP) diP->ival[dRef]++;
66 }
67 
68 /******************************************************************************/
69 /* D e s t r u c t o r */
70 /******************************************************************************/
71 
73 {
74  int i;
75 
76 // Delete all the table entries
77 //
78  for (i = 0; i < XrdOssPath::sfxNum; i++) if(File[i]) delete File[i];
79 
80 // If there is a shared directory buffer, decrease reference count, delete if 0
81 //
82  if (dInfo && ((dInfo->ival[dRef] -= 1) <= 0)) delete dInfo;
83 }
84 
85 /******************************************************************************/
86 /* d i r P a t h */
87 /******************************************************************************/
88 
89 int XrdFrmFileset::dirPath(char *dBuff, int dBlen)
90 {
91  char *dP = 0;
92  int dN = 0, i;
93 
94 // If we have a shared directory pointer, use that as directory information
95 // Otherwise, get it from one of the files in the fileset.
96 //
97  if (dInfo) {dP = dInfo->text; dN = dInfo->ival[dLen];}
98  else {for (i = 0; i < XrdOssPath::sfxNum; i++)
99  if (File[i])
100  {dP = File[i]->Path;
101  dN = File[i]->File - File[i]->Path;
102  break;
103  }
104  }
105 
106 // Copy out the directory path
107 //
108  if (dBlen > dN && dP) strncpy(dBuff, dP, dN);
109  else dN = 0;
110  *(dBuff+dN) = '\0';
111  return dN;
112 }
113 
114 /******************************************************************************/
115 /* R e f r e s h */
116 /******************************************************************************/
117 
118 int XrdFrmFileset::Refresh(int isMig, int doLock)
119 {
120  class fdClose
121  {public:
122  int Num;
123  fdClose() : Num(-1) {}
124  ~fdClose() {if (Num >= 0) close(Num);}
125  } fnFD;
126  XrdOucNSWalk::NSEnt *lP, *bP = baseFile();
127  char pBuff[MAXPATHLEN+1], *fnP, *pnP = pBuff;
128  int n, lkFD = -1;
129 
130 // Get the directory path for this entry
131 //
132  if (!(n = dirPath(pBuff, sizeof(pBuff)-1))) return 0;
133  fnP = pBuff+n;
134 
135 // If we need to lock the entry, do so. We also check if file is in use
136 //
137  if (doLock && bP)
138  {strcpy(fnP, baseFile()->File);
139  if (!(lkFD = chkLock(pBuff))) return 0;
140  fnFD.Num = lkFD;
141  }
142 
143 // Do a new stat call on each relevant file (pin file excluded for isMig)
144 //
145  if (bP)
146  {if (bP->Link) pnP = bP->Link;
147  else strcpy(fnP, bP->File);
148  if (stat(pnP, &(bP->Stat)))
149  {Say.Emsg("Refresh", errno, "stat", pnP); return 0;}
150  }
151 
152  if (!isMig) pinInfo.Get(pnP, lkFD);
153 
154  if ((lP = lockFile()))
155  {strcpy(fnP, lP->File);
156  if (stat(pBuff, &(lP->Stat)))
157  {Say.Emsg("Refresh", errno, "stat", pBuff); return 0;}
158  cpyInfo.Attr.cpyTime = static_cast<long long>(lP->Stat.st_mtime);
159  } else if (cpyInfo.Get(pnP, lkFD) <= 0) cpyInfo.Attr.cpyTime = 0;
160 
161 // All done
162 //
163  return 1;
164 }
165 
166 /******************************************************************************/
167 /* S c r e e n */
168 /******************************************************************************/
169 
170 int XrdFrmFileset::Screen(int needLF)
171 {
172  const char *What = 0, *badFN = 0;
173 
174 // Verify that we have all the relevant files (old mode only)
175 //
176  if (!Config.runNew && !baseFile())
177  {if (Config.Fix)
178  {if (lockFile()) Remfix("Lock", lockPath());
179  if ( pinFile()) Remfix("Pin", pinPath());
180  return 0;
181  }
182  if (lockFile()) badFN = lockPath();
183  else if ( pinFile()) badFN = pinPath();
184  else return 0;
185  What = "No base file for";
186  }
187 
188 // If no errors from above, try to get the copy time for this file
189 //
190  if (!What)
191  {if (!needLF || setCpyTime()) return 1;
192  What = Config.runNew ? "no copy time xattr for" : "no lock file for";
193  badFN = basePath();
194  }
195 
196 // Issue message if we haven't issued one before
197 //
198  if (!BadFiles.Add(badFN, 0, 0, Hash_data_is_key))
199  Say.Emsg("Screen", What, badFN);
200  return 0;
201 }
202 
203 /******************************************************************************/
204 /* s e t C p y T i m e */
205 /******************************************************************************/
206 
208 {
210 
211 // In new run mode the copy time comes from the extended attributes
212 //
213  if (Config.runNew) return cpyInfo.Get(basePath()) > 0;
214 
215 // If there is no lock file, indicate so
216 //
217  if (!(lP = lockFile())) return 0;
218 
219 // Use the lock file as the source of information
220 //
221  if (Refresh && stat(lockPath(), &(lP->Stat)))
222  {Say.Emsg("setCpyTime", errno, "stat", lockPath()); return 0;}
223  cpyInfo.Attr.cpyTime = static_cast<long long>(lP->Stat.st_mtime);
224  return 1;
225 }
226 
227 /******************************************************************************/
228 /* P r i v a t e M e t h o d s */
229 /******************************************************************************/
230 /******************************************************************************/
231 /* c h k L o c k */
232 /******************************************************************************/
233 
234 // Returns 0 if lock exists or an error occurred, o/w returns fd for the file.
235 
236 int XrdFrmFileset::chkLock(const char *Path)
237 {
238  FLOCK_t lock_args;
239  int rc, lokFD;
240 
241 // Open the file appropriately
242 //
243  if ((lokFD = open(Path, O_RDONLY)) < 0)
244  {Say.Emsg("chkLock", errno, "open", Path); return 0;}
245 
246 // Initialize the lock arguments
247 //
248  bzero(&lock_args, sizeof(lock_args));
249  lock_args.l_type = F_WRLCK;
250 
251 // Now check if the lock can be obtained
252 //
253  do {rc = fcntl(lokFD, F_GETLK, &lock_args);} while(rc < 0 && errno == EINTR);
254 
255 // Determine the result
256 //
257  if (!rc) return lokFD;
258  Say.Emsg("chkLock", errno, "lock", Path);
259  close(lokFD);
260  return 0;
261 }
262 
263 /******************************************************************************/
264 /* M k f n */
265 /******************************************************************************/
266 
267 const char *XrdFrmFileset::Mkfn(XrdOucNSWalk::NSEnt *fP)
268 {
269 
270 // If we have no file for this, return the null string
271 //
272  if (!fP) return "";
273 
274 // If we have no shared directory pointer, return the full path
275 //
276  if (!dInfo) return fP->Path;
277 
278 // Construct the name in a non-renterant way (this is documented)
279 //
280  strcpy(dInfo->text+dInfo->ival[dLen], fP->File);
281  return dInfo->text;
282 }
283 
284 /******************************************************************************/
285 /* R e m f i x */
286 /******************************************************************************/
287 
288 void XrdFrmFileset::Remfix(const char *fType, const char *fPath)
289 {
290 
291 // Remove the offending file
292 //
293  if (unlink(fPath)) Say.Emsg("Remfix", errno, "remove orphan", fPath);
294  Say.Emsg("Remfix", fType, "file orphan fixed; removed", fPath);
295 }
296 
297 /******************************************************************************/
298 /* C l a s s X r d F r m F i l e s */
299 /******************************************************************************/
300 /******************************************************************************/
301 /* C o n s t r u c t o r */
302 /******************************************************************************/
303 
304 XrdFrmFiles::XrdFrmFiles(const char *dname, int opts,
305  XrdOucTList *XList, XrdOucNSWalk::CallBack *cbP)
306  : nsObj(&Say, dname, 0,
307  XrdOucNSWalk::retFile | XrdOucNSWalk::retLink
308  |XrdOucNSWalk::retStat | XrdOucNSWalk::skpErrs
309  |XrdOucNSWalk::retIILO
310  | (opts & CompressD ? XrdOucNSWalk::noPath : 0)
311  | (opts & Recursive ? XrdOucNSWalk::Recurse : 0), XList),
312  fsList(0), manMem(opts & NoAutoDel ? Hash_keep : Hash_default),
313  shareD(opts & CompressD), getCPT(opts & GetCpyTim)
314 {
315 
316 // Set Call Back method
317 //
318  nsObj.setCallBack(cbP);
319 }
320 
321 /******************************************************************************/
322 /* D e s t r u c t o r */
323 /******************************************************************************/
324 
326 {
327  XrdFrmFileset *fsetP;
328 
329 // If manual memory is wante then we must delete any unreturned objects
330 //
331  if (manMem)
332  while((fsetP = fsList))
333  {fsList = fsetP->Next; fsetP->Next = 0; delete fsetP;}
334 }
335 
336 /******************************************************************************/
337 /* G e t */
338 /******************************************************************************/
339 
340 XrdFrmFileset *XrdFrmFiles::Get(int &rc, int noBase)
341 {
343  XrdFrmFileset *fsetP;
344  const char *dPath;
345 
346 // Check if we have something to return
347 //
348 do{while ((fsetP = fsList))
349  {fsList = fsetP->Next; fsetP->Next = 0;
350  if (fsetP->File[XrdOssPath::isBase])
351  {if (getCPT) fsetP->setCpyTime();
352  rc = 0; return fsetP;
353  }
354  else if (noBase) {rc = 0; return fsetP;}
355  else if (manMem) delete fsetP;
356  }
357 
358 // Start with next directory (we return when no directories left).
359 //
360  do {if (!(nP = nsObj.Index(rc, &dPath))) return 0;
361  fsTab.Purge(); fsList = 0;
362  } while(!Process(nP, dPath));
363 
364  } while(1);
365 
366 // To keep the compiler happy
367 //
368  return 0;
369 }
370 
371 /******************************************************************************/
372 /* P r i v a t e M e t h o d s */
373 /******************************************************************************/
374 /******************************************************************************/
375 /* C o m p l a i n */
376 /******************************************************************************/
377 
378 void XrdFrmFiles::Complain(const char *dPath)
379 {
380  static const int OneDay = 24*60*60;
381  static XrdOucHash<char> dTab;
382 
383 // We want to complain about old=style directories only once every 24 hours
384 //
385  if (dTab.Add(dPath, 0, OneDay, Hash_data_is_key)) return;
386 
387 // Complain about this directory
388 //
389  Say.Emsg("Complain","Found old-style files in directory", dPath);
390  Say.Emsg("Complain","In new run mode, migrate & purge will skip them.");
391 }
392 
393 /******************************************************************************/
394 /* o l d F i l e */
395 /******************************************************************************/
396 
397 int XrdFrmFiles::oldFile(XrdOucNSWalk::NSEnt *fP, XrdOucTList *dP, int fType)
398 {
399  char pBuff[MAXPATHLEN+8], *pnP = pBuff, *fnP;
400 
401 // Ignore (for now): '.anew', '.fail', or '.pfn'
402 //
403  if (fType == XrdOssPath::isAnew
404  || fType == XrdOssPath::isFail
405  || fType == XrdOssPath::isPfn) return 0;
406 
407 // If this is not a directory lock file, indicate we should complain
408 //
409  if (fType >= 0) return 1;
410 
411 // This is a directory lock file, quietly remove it (we no longer use them)
412 //
413  if (!dP) pnP = fP->Path;
414  else {strcpy(pBuff, dP->text);
415  fnP = pBuff + dP->ival[XrdFrmFileset::dLen];
416  *fnP++ = '/';
417  strcpy(fnP, fP->File);
418  }
419  unlink(pnP);
420  return 0;
421 }
422 
423 /******************************************************************************/
424 /* P r o c e s s */
425 /******************************************************************************/
426 
427 int XrdFrmFiles::Process(XrdOucNSWalk::NSEnt *nP, const char *dPath)
428 {
430  XrdFrmFileset *sP;
431  XrdOucTList *dP = 0;
432  char *dotP;
433  int fType, noDLKF = 1, runOldFault = 0;
434 
435 // If compressed directories wanted, then setup a shared directory buffer
436 // Warning! We use a hard-coded value for maximum filename length instead of
437 // constantly calling pathconf().
438 //
439  if (shareD)
440  {int n = strlen(dPath);
441  char *dBuff = (char *)malloc(n+264);
442  strcpy(dBuff, dPath);
443  dP = new XrdOucTList;
444  dP->text = dBuff;
445  dP->ival[XrdFrmFileset::dLen] = n;
446  dP->ival[XrdFrmFileset::dRef] = 0;
447  }
448 
449 // Process the file list
450 //
451  while((fP = nP))
452  {nP = fP->Next; fP->Next = 0;
453  if (noDLKF && !strcmp(fP->File, Config.lockFN))
454  {oldFile(fP, dP, -1); delete fP; noDLKF = 0; continue;}
455  if (!(fType = (int)XrdOssPath::pathType(fP->File))
456  || !(dotP = rindex(fP->File, '.'))) dotP = 0;
457  else {if (Config.runNew)
458  {runOldFault |= oldFile(fP, dP, fType);
459  delete fP; continue;
460  }
461  *dotP = '\0';
462  }
463  if (!(sP = fsTab.Find(fP->File)))
464  {sP = fsList = new XrdFrmFileset(fsList, dP);
465  fsTab.Add(fP->File, sP, 0, manMem);
466  }
467  if (dotP) *dotP = '.';
468  sP->File[fType] = fP;
469  }
470 
471 // If we found on old-style file while in runNew, complain
472 //
473  if (runOldFault) Complain(dPath);
474 
475 // Indicate whether we have anything here
476 //
477  if (fsList) return 1;
478  if (dP) delete dP;
479  return 0;
480 }
@ Hash_data_is_key
Definition: XrdOucHash.hh:52
@ Hash_default
Definition: XrdOucHash.hh:51
@ Hash_keep
Definition: XrdOucHash.hh:55
int stat(const char *path, struct stat *buf)
int open(const char *path, int oflag,...)
int fcntl(int fd, int cmd,...)
int unlink(const char *path)
#define close(a)
Definition: XrdPosix.hh:43
XrdOucString Path
struct myOpts opts
#define FLOCK_t
long long cpyTime
Definition: XrdFrcXAttr.hh:55
XrdFrmFiles(const char *dname, int opts=Recursive, XrdOucTList *XList=0, XrdOucNSWalk::CallBack *cbP=0)
Definition: XrdFrmFiles.cc:304
XrdFrmFileset * Get(int &rc, int noBase=0)
Definition: XrdFrmFiles.cc:340
int setCpyTime(int Refresh=0)
Definition: XrdFrmFiles.cc:207
int Refresh(int isMig=0, int doLock=1)
Definition: XrdFrmFiles.cc:118
XrdOucXAttr< XrdFrcXAttrCpy > cpyInfo
Definition: XrdFrmFiles.hh:55
int Screen(int needLF=1)
Definition: XrdFrmFiles.cc:170
int dirPath(char *dBuff, int dBlen)
Definition: XrdFrmFiles.cc:89
XrdOucXAttr< XrdFrcXAttrPin > pinInfo
Definition: XrdFrmFiles.hh:56
XrdOucNSWalk::NSEnt * pinFile()
Definition: XrdFrmFiles.hh:68
XrdFrmFileset * Next
Definition: XrdFrmFiles.hh:90
XrdOucNSWalk::NSEnt * baseFile()
Definition: XrdFrmFiles.hh:60
const char * lockPath()
Definition: XrdFrmFiles.hh:65
const char * basePath()
Definition: XrdFrmFiles.hh:61
XrdFrmFileset(XrdFrmFileset *sP=0, XrdOucTList *diP=0)
Definition: XrdFrmFiles.cc:62
XrdOucNSWalk::NSEnt * lockFile()
Definition: XrdFrmFiles.hh:64
const char * pinPath()
Definition: XrdFrmFiles.hh:69
static theSfx pathType(const char *Path, int chkWhat=chkAll)
Definition: XrdOssPath.cc:278
void Purge()
Definition: XrdOucHash.icc:193
T * Add(const char *KeyVal, T *KeyData, const int LifeTime=0, XrdOucHash_Options opt=Hash_default)
Definition: XrdOucHash.icc:61
T * Find(const char *KeyVal, time_t *KeyTime=0)
Definition: XrdOucHash.icc:160
void setCallBack(CallBack *cbP=0)
Definition: XrdOucNSWalk.hh:96
NSEnt * Index(int &rc, const char **dPath=0)
Definition: XrdOucNSWalk.cc:93
char * text
Definition: XrdOucTList.hh:46
int Get(const char *Path, int fd=-1)
Definition: XrdOucXAttr.hh:128
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:95
XrdSysError Say
XrdCmsConfig Config
void * Refresh(void *parg)
struct NSEnt * Next
Definition: XrdOucNSWalk.hh:48