XRootD
XrdOssCache.cc
Go to the documentation of this file.
1 /******************************************************************************/
2 /* */
3 /* X r d O s s C a c h e . c c */
4 /* */
5 /* (c) 2003 by the Board of Trustees of the Leland Stanford, Jr., University */
6 /* All Rights Reserved */
7 /* Produced by Andrew Hanushevsky for Stanford University under contract */
8 /* DE-AC02-76-SFO0515 with the Department of Energy */
9 /* */
10 /* This file is part of the XRootD software suite. */
11 /* */
12 /* XRootD is free software: you can redistribute it and/or modify it under */
13 /* the terms of the GNU Lesser General Public License as published by the */
14 /* Free Software Foundation, either version 3 of the License, or (at your */
15 /* option) any later version. */
16 /* */
17 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20 /* License for more details. */
21 /* */
22 /* You should have received a copy of the GNU Lesser General Public License */
23 /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24 /* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25 /* */
26 /* The copyright holder's institutional names and contributor's names may not */
27 /* be used to endorse or promote products derived from this software without */
28 /* specific prior written permission of the institution or contributor. */
29 /******************************************************************************/
30 
31 #include <unistd.h>
32 #include <dirent.h>
33 #include <cerrno>
34 #include <fcntl.h>
35 #include <map>
36 #include <cstdio>
37 #include <string>
38 #include <strings.h>
39 #include <ctime>
40 #include <sys/param.h>
41 #include <sys/types.h>
42 #include <sys/stat.h>
43 
44 #ifdef __linux__
45 #include <sys/sysmacros.h>
46 #endif
47 
48 #include "XrdOss/XrdOssCache.hh"
49 #include "XrdOss/XrdOssOpaque.hh"
50 #include "XrdOss/XrdOssPath.hh"
51 #include "XrdOss/XrdOssSpace.hh"
52 #include "XrdOss/XrdOssTrace.hh"
53 #include "XrdOuc/XrdOucStream.hh"
54 #include "XrdSys/XrdSysHeaders.hh"
55 #include "XrdSys/XrdSysPlatform.hh"
56 
57 /******************************************************************************/
58 /* G l o b a l s a n d S t a t i c M e m b e r s */
59 /******************************************************************************/
60 
61 extern XrdSysError OssEroute;
62 
63 extern XrdSysTrace OssTrace;
64 
67 
68 long long XrdOssCache_Group::PubQuota = -1;
69 
71 long long XrdOssCache::fsTotal = 0;
72 long long XrdOssCache::fsLarge = 0;
73 long long XrdOssCache::fsTotFr = 0;
74 long long XrdOssCache::fsFree = 0;
75 long long XrdOssCache::fsSize = 0;
79 double XrdOssCache::fuzAlloc= 0.0;
80 long long XrdOssCache::minAlloc= 0;
81 int XrdOssCache::fsCount = 0;
82 int XrdOssCache::ovhAlloc= 0;
83 int XrdOssCache::Quotas = 0;
84 int XrdOssCache::Usage = 0;
85 
86 namespace XrdOssCacheDevs
87 {
88 struct devID
89  {int bdevID;
90  int partID;
91  char nDev[16];
92  };
93 
94 std::map<dev_t, devID> dev2ID;
95 
96 int devNMax = 1;
97 int prtNMax = 1;
98 }
99 using namespace XrdOssCacheDevs;
100 
101 /******************************************************************************/
102 /* X r d O s s C a c h e _ F S D a t a M e t h o d s */
103 /******************************************************************************/
104 
106  STATFS_t &fsbuff,
107  dev_t fsID)
108 {
109  path = strdup(fsp);
110  if (!(pact= realpath(fsp,0))) pact = path;
111  size = static_cast<long long>(fsbuff.f_blocks)
112  * static_cast<long long>(fsbuff.FS_BLKSZ);
113  frsz = static_cast<long long>(fsbuff.f_bavail)
114  * static_cast<long long>(fsbuff.FS_BLKSZ);
115  XrdOssCache::fsTotal += size;
116  XrdOssCache::fsTotFr += frsz;
118  if (size > XrdOssCache::fsLarge) XrdOssCache::fsLarge= size;
119  if (frsz > XrdOssCache::fsFree) XrdOssCache::fsFree = frsz;
120  fsid = fsID;
121  updt = time(0);
122  next = 0;
123  stat = 0;
124 
125 // This is created only for new partitions!
126 //
127  std::map<dev_t, devID>::iterator it = dev2ID.find(fsID);
128  if (it != dev2ID.end())
129  {bdevID = static_cast<unsigned short>(it->second.bdevID);
130  if (it->second.partID == 0) it->second.partID = prtNMax++;
131  partID = static_cast<unsigned short>(it->second.partID);
132  devN = it->second.nDev;
133  } else {
134  bdevID = 0;
135  partID = static_cast<unsigned short>(prtNMax++);
136  devN = "dev";
137  }
138 }
139 
140 /******************************************************************************/
141 /* X r d O s s C a c h e _ F S C o n s t r u c t o r */
142 /******************************************************************************/
143 
144 // Cache_FS objects are only created during configuration. No locks are needed.
145 
147  const char *fsGrp,
148  const char *fsPath,
149  FSOpts fsOpts)
150 {
151  static const mode_t theMode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH;
152  STATFS_t fsbuff;
153  struct stat sfbuff;
154  XrdOssCache_FSData *fdp;
155  XrdOssCache_FS *fsp;
156  int n;
157 
158 // Prefill in case of failure
159 //
160  path = group = 0;
161 
162 // Verify that this is not a duplicate
163 //
164  fsp = XrdOssCache::fsfirst;
165  while(fsp && (strcmp(fsp->path,fsPath)||strcmp(fsp->fsgroup->group,fsGrp)))
166  if ((fsp = fsp->next) == XrdOssCache::fsfirst) {fsp = 0; break;}
167  if (fsp) {retc = EEXIST; return;}
168 
169 // Set the groupname and the path which is the supplied path/group name
170 //
171  if (!(fsOpts & isXA)) path = strdup(fsPath);
172  else {path = XrdOssPath::genPath(fsPath, fsGrp, suffix);
173  if (mkdir(path, theMode) && errno != EEXIST) {retc=errno; return;}
174  }
175  plen = strlen(path);
176  group = strdup(fsGrp);
177  fsgroup= 0;
178  opts = fsOpts;
179  retc = ENOMEM;
180 
181 // Find the filesystem for this object
182 //
183  if (FS_Stat(fsPath, &fsbuff) || stat(fsPath, &sfbuff)) {retc=errno; return;}
184 
185 // Find the matching filesystem data
186 //
187  fdp = XrdOssCache::fsdata;
188  while(fdp) {if (fdp->fsid == sfbuff.st_dev) break; fdp = fdp->next;}
189 
190 // If we didn't find the filesystem, then create one
191 //
192  if (!fdp)
193  {if (!(fdp = new XrdOssCache_FSData(fsPath,fsbuff,sfbuff.st_dev))) return;
194  else {fdp->next = XrdOssCache::fsdata; XrdOssCache::fsdata = fdp;}
195  }
196 
197 // Complete the filesystem block (failure now is not an option)
198 //
199  fsdata = fdp;
200  retc = 0;
201 
202 // Link this filesystem into the filesystem chain
203 //
204  if (!XrdOssCache::fsfirst) {next = this;
205  XrdOssCache::fsfirst = this;
206  XrdOssCache::fslast = this;
207  }
208  else {next = XrdOssCache::fslast->next;
209  XrdOssCache::fslast->next = this;
210  XrdOssCache::fslast = this;
211  }
212 
213 // Check if this is the first group allocation
214 //
215  fsgroup = XrdOssCache_Group::fsgroups;
216  while(fsgroup && strcmp(group, fsgroup->group)) fsgroup = fsgroup->next;
217  if (!fsgroup && (fsgroup = new XrdOssCache_Group(group, this)))
218  {fsgroup->next = XrdOssCache_Group::fsgroups;
220  }
221 
222 // Add a filesystem to this group but only if it is a new one
223 //
224  XrdOssCache_FSAP *fsAP;
225  for (n = 0; n < fsgroup->fsNum && fdp != fsgroup->fsVec[n].fsP; n++);
226  if (n >= fsgroup->fsNum)
227  {n= (fsgroup->fsNum + 1) * sizeof(XrdOssCache_FSAP);
228  fsgroup->fsVec = (XrdOssCache_FSAP *)realloc((void *)fsgroup->fsVec,n);
229  fsAP = &(fsgroup->fsVec[fsgroup->fsNum++]);
230  fsAP->fsP = fdp;
231  fsAP->apVec = 0;
232  fsAP->apNum = 0;
233  } else fsAP = &(fsgroup->fsVec[n]);
234 
235 // Add the allocation path to this partition
236 //
237  n = (fsAP->apNum + 2) * sizeof(char *);
238  fsAP->apVec = (const char **)realloc((void *)fsAP->apVec, n);
239  fsAP->apVec[fsAP->apNum++] = path;
240  fsAP->apVec[fsAP->apNum] = 0;
241 }
242 
243 /******************************************************************************/
244 /* A d d */
245 /******************************************************************************/
246 
247 // Add is only called during configuration. No locks are needed. It merely
248 // adds an unnamed file system partition. This allows us to track space.
249 
250 int XrdOssCache_FS::Add(const char *fsPath)
251 {
252  STATFS_t fsbuff;
253  struct stat sfbuff;
254  XrdOssCache_FSData *fdp;
255 
256 // Find the filesystem for this object
257 //
258  if (FS_Stat(fsPath, &fsbuff) || stat(fsPath, &sfbuff)) return -errno;
259 
260 // Find the matching filesystem data
261 //
262  fdp = XrdOssCache::fsdata;
263  while(fdp) {if (fdp->fsid == sfbuff.st_dev) break; fdp = fdp->next;}
264  if (fdp) return 0;
265 
266 // Create new filesystem data that will not be linked to any filesystem
267 //
268  if (!(fdp = new XrdOssCache_FSData(fsPath,fsbuff,sfbuff.st_dev)))
269  return -ENOMEM;
270  fdp->next = XrdOssCache::fsdata;
271  XrdOssCache::fsdata = fdp;
272  return 0;
273 }
274 
275 /******************************************************************************/
276 /* f r e e S p a c e */
277 /******************************************************************************/
278 
279 long long XrdOssCache_FS::freeSpace(long long &Size, const char *path)
280 {
281  STATFS_t fsbuff;
282  long long fSpace;
283 
284 // Free space for a specific path
285 //
286  if (path)
287  {if (FS_Stat(path, &fsbuff)) return -1;
288  Size = static_cast<long long>(fsbuff.f_blocks)
289  * static_cast<long long>(fsbuff.FS_BLKSZ);
290  return static_cast<long long>(fsbuff.f_bavail)
291  * static_cast<long long>(fsbuff.FS_BLKSZ);
292  }
293 
294 // Free space for the whole system
295 //
297  fSpace = XrdOssCache::fsFree;
298  Size = XrdOssCache::fsSize;
300  return fSpace;
301 }
302 
303 /******************************************************************************/
304 
305 long long XrdOssCache_FS::freeSpace(XrdOssCache_Space &Space, const char *path)
306 {
307  STATFS_t fsbuff;
308 
309 // Free space for a specific path
310 //
311  if (!path || FS_Stat(path, &fsbuff)) return -1;
312 
313  Space.Total = static_cast<long long>(fsbuff.f_blocks)
314  * static_cast<long long>(fsbuff.FS_BLKSZ);
315  Space.Free = static_cast<long long>(fsbuff.f_bavail)
316  * static_cast<long long>(fsbuff.FS_BLKSZ);
317  Space.Inodes= static_cast<long long>(fsbuff.f_files);
318  Space.Inleft= static_cast<long long>(fsbuff.FS_FFREE);
319 
320  return Space.Free;
321 }
322 
323 /******************************************************************************/
324 /* g e t S p a c e */
325 /******************************************************************************/
326 
327 int XrdOssCache_FS::getSpace(XrdOssCache_Space &Space, const char *sname,
328  XrdOssVSPart **vsPart)
329 {
331 
332 // Try to find the space group name
333 //
334  while(fsg && strcmp(sname, fsg->group)) fsg = fsg->next;
335  if (!fsg) return 0;
336 
337 // Return the accumulated result
338 //
339  return getSpace(Space, fsg, vsPart);
340 }
341 
342 /******************************************************************************/
343 
345  XrdOssVSPart **vsPart)
346 {
347  XrdOssVSPart *pVec;
348  XrdOssCache_FSData *fsd;
349 
350 // If there is no partition table then we really have no space to report
351 //
352  if (fsg->fsNum < 1 || !fsg->fsVec) return 0;
353 
354 // If partion information is wanted, allocate a partition table
355 //
356  if (vsPart) *vsPart = pVec = new XrdOssVSPart[fsg->fsNum];
357  else pVec = 0;
358 
359 // Prevent any changes
360 //
362 
363 // Get overall values
364 //
365  Space.Quota = fsg->Quota;
366  Space.Usage = fsg->Usage;
367 
368 // Accumulate the stats.
369 //
370  for (int i = 0; i < fsg->fsNum; i++)
371  {fsd = fsg->fsVec[i].fsP;
372  Space.Total += fsd->size;
373  Space.Free += fsd->frsz;
374  if (fsd->frsz > Space.Maxfree) Space.Maxfree = fsd->frsz;
375  if (fsd->size > Space.Largest) Space.Largest = fsd->size;
376 
377  if (pVec)
378  {pVec[i].pPath = fsd->path;
379  pVec[i].aPath = fsg->fsVec[i].apVec;
380  pVec[i].Total = fsd->size;
381  pVec[i].Free = fsd->frsz;
382  pVec[i].bdevID= fsd->bdevID;
383  pVec[i].partID= fsd->partID;
384  }
385  }
387 
388 // All done
389 //
390  return fsg->fsNum;
391 }
392 
393 /******************************************************************************/
394 /* A d j u s t */
395 /******************************************************************************/
396 
397 void XrdOssCache::Adjust(dev_t devid, off_t size)
398 {
399  EPNAME("Adjust")
400  XrdOssCache_FSData *fsdp;
401 
402 // Search for matching filesystem
403 //
404  fsdp = XrdOssCache::fsdata;
405  while(fsdp && fsdp->fsid != devid) fsdp = fsdp->next;
406 
407 // Adjust file system free space
408 //
409  Mutex.Lock();
410  if (fsdp)
411  {DEBUG("free=" <<fsdp->frsz <<'-' <<size <<" path=" <<fsdp->path);
412  if ((fsdp->frsz -= size) < 0) fsdp->frsz = 0;
413  fsdp->stat |= XrdOssFSData_ADJUSTED;
414  } else {
415  DEBUG("dev " <<devid <<" not found.");
416  }
417 
418 // Adjust group usage
419 //
421  {DEBUG("usage=" <<XrdOssCache_Group::PubGroup->Usage <<'+' <<size
422  <<" space=" <<XrdOssCache_Group::PubGroup->group);
423  if ((XrdOssCache_Group::PubGroup->Usage += size) < 0)
426  }
427 
428 // All done
429 //
430  Mutex.UnLock();
431 }
432 
433 /******************************************************************************/
434 
435 void XrdOssCache::Adjust(const char *Path, off_t size, struct stat *buf)
436 {
437  EPNAME("Adjust")
438  XrdOssCache_FS *fsp;
439 
440 // If we have a struct then we need to do some more work
441 //
442  if (buf)
443  {if ((buf->st_mode & S_IFMT) != S_IFLNK) Adjust(buf->st_dev, size);
444  else {char lnkbuff[MAXPATHLEN+64];
445  int lnklen = readlink(Path, lnkbuff, sizeof(lnkbuff)-1);
446  if (lnklen > 0)
447  {XrdOssPath::Trim2Base(lnkbuff+lnklen-1);
448  Adjust(lnkbuff, size);
449  }
450  }
451  return;
452  }
453 
454 // Search for matching logical partition
455 //
456  fsp = fsfirst;
457  while(fsp && strcmp(fsp->path, Path))
458  if ((fsp = fsp->next) == fsfirst) {fsp = 0; break;}
459 
460 // Process the result
461 //
462  if (fsp) Adjust(fsp, size);
463  else {DEBUG("cache path " <<Path <<" not found.");}
464 }
465 
466 /******************************************************************************/
467 
468 void XrdOssCache::Adjust(XrdOssCache_FS *fsp, off_t size)
469 {
470  EPNAME("Adjust")
471  XrdOssCache_FSData *fsdp;
472 
473 // Process the result
474 //
475  if (fsp)
476  {fsdp = fsp->fsdata;
477  DEBUG("used=" <<fsp->fsgroup->Usage <<'+' <<size <<" path=" <<fsp->path);
478  DEBUG("free=" <<fsdp->frsz <<'-' <<size <<" path=" <<fsdp->path);
479  Mutex.Lock();
480  if ((fsp->fsgroup->Usage += size) < 0) fsp->fsgroup->Usage = 0;
481  if ( (fsdp->frsz -= size) < 0) fsdp->frsz = 0;
482  fsdp->stat |= XrdOssFSData_ADJUSTED;
483  if (Usage) XrdOssSpace::Adjust(fsp->fsgroup->GRPid, size);
484  Mutex.UnLock();
485  }
486 }
487 
488 /******************************************************************************/
489 /* A l l o c */
490 /******************************************************************************/
491 
493 {
494  EPNAME("Alloc");
495  static const mode_t theMode = S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH;
496  XrdSysMutexHelper myMutex(&Mutex);
497  double diffree;
499  XrdOssCache_FS *fsp, *fspend, *fsp_sel;
500  XrdOssCache_Group *cgp = 0;
501  long long size, maxfree, curfree;
502  int rc, madeDir, datfd = 0;
503 
504 // Compute appropriate allocation size
505 //
506  if (!aInfo.cgSize
507  || (size=aInfo.cgSize*ovhAlloc/100+aInfo.cgSize) < minAlloc)
508  aInfo.cgSize = size = minAlloc;
509 
510 // Find the corresponding cache group
511 //
513  while(cgp && strcmp(aInfo.cgName, cgp->group)) cgp = cgp->next;
514  if (!cgp) return -ENOENT;
515 
516 // Find a cache that will fit this allocation request. We start with the next
517 // entry past the last one we selected and go full round looking for a
518 // compatable entry (enough space and in the right space group).
519 //
520  fsp_sel = 0; maxfree = 0;
521  fsp = cgp->curr->next; fspend = fsp; // End when we hit the start again
522  do {
523  if (strcmp(aInfo.cgName, fsp->group)
524  || (aInfo.cgPath && (aInfo.cgPlen > fsp->plen
525  || strncmp(aInfo.cgPath,fsp->path,aInfo.cgPlen)))) continue;
526  curfree = fsp->fsdata->frsz;
527  if (size > curfree) continue;
528 
529  if (fuzAlloc > 0.999) {fsp_sel = fsp; break;}
530  else if (!fuzAlloc || !fsp_sel)
531  {if (curfree > maxfree) {fsp_sel = fsp; maxfree = curfree;}}
532  else {diffree = (!(curfree + maxfree) ? 0.0
533  : static_cast<double>(XRDABS(maxfree - curfree)) /
534  static_cast<double>( maxfree + curfree));
535  if (diffree > fuzAlloc) {fsp_sel = fsp; maxfree = curfree;}
536  }
537  } while((fsp = fsp->next) != fspend);
538 
539 // Check if we can realy fit this file. If so, update current scan pointer
540 //
541  if (!fsp_sel) return -ENOSPC;
542  cgp->curr = fsp_sel;
543 
544 // Construct the target filename
545 //
546  Info.Path = fsp_sel->path;
547  Info.Plen = fsp_sel->plen;
548  Info.Sfx = fsp_sel->suffix;
549  aInfo.cgPsfx = XrdOssPath::genPFN(Info, aInfo.cgPFbf, aInfo.cgPFsz,
550  (fsp_sel->opts & XrdOssCache_FS::isXA ? 0 : aInfo.Path));
551 
552 // Verify that target name was constructed
553 //
554  if (!(*aInfo.cgPFbf)) return -ENAMETOOLONG;
555 
556 // Simply open the file in the local filesystem, creating it if need be.
557 //
558  if (aInfo.aMode)
559  {madeDir = 0;
560  do {do {datfd = open(aInfo.cgPFbf,O_CREAT|O_TRUNC|O_WRONLY,aInfo.aMode);}
561  while(datfd < 0 && errno == EINTR);
562  if (datfd >= 0 || errno != ENOENT || madeDir) break;
563  *Info.Slash='\0'; rc=mkdir(aInfo.cgPFbf,theMode); *Info.Slash='/';
564  madeDir = 1;
565  } while(!rc);
566  if (datfd < 0) return (errno ? -errno : -EFAULT);
567  }
568 
569 // All done (temporarily adjust down the free space)x
570 //
571  DEBUG("free=" <<fsp_sel->fsdata->frsz <<'-' <<size <<" path="
572  <<fsp_sel->fsdata->path);
573  fsp_sel->fsdata->frsz -= size;
574  fsp_sel->fsdata->stat |= XrdOssFSData_REFRESH;
575  aInfo.cgFSp = fsp_sel;
576  return datfd;
577 }
578 
579 /******************************************************************************/
580 /* D e v I n f o */
581 /******************************************************************************/
582 
583 void XrdOssCache::DevInfo(struct stat &buf, bool limits)
584 {
585 
586 // Check if only the maximum values ae to be returned
587 //
588  if (limits)
589  {memset(&buf, 0, sizeof(struct stat));
590  buf.st_dev = static_cast<dev_t>(XrdOssCacheDevs::prtNMax);
591  buf.st_rdev = static_cast<dev_t>(XrdOssCacheDevs::devNMax);
592  return;
593  }
594 
595 // Look up the device info
596 //
597  std::map<dev_t, devID>::iterator it = dev2ID.find(buf.st_dev);
598  if (it != dev2ID.end())
599  {buf.st_rdev = static_cast<dev_t>(it->second.bdevID);
600  buf.st_dev = static_cast<dev_t>(it->second.partID);
601  } else {
602  buf.st_rdev = 0;
603  buf.st_dev = 0;
604  }
605 }
606 
607 /******************************************************************************/
608 /* F i n d */
609 /******************************************************************************/
610 
611 XrdOssCache_FS *XrdOssCache::Find(const char *Path, int lnklen)
612 {
613  XrdOssCache_FS *fsp;
614  char lnkbuff[MAXPATHLEN+64];
615  struct stat sfbuff;
616 
617 // First see if this is a symlink that refers to a new style cache
618 //
619  if (lnklen)
620  {if (strlcpy(lnkbuff,Path,sizeof(lnkbuff)) >= sizeof(lnkbuff)) return 0;}
621  else if (lstat(Path, &sfbuff)
622  || (sfbuff.st_mode & S_IFMT) != S_IFLNK
623  || (lnklen = readlink(Path,lnkbuff,sizeof(lnkbuff)-1)) <= 0)
624  return 0;
625 
626 // Trim the link to the base name
627 //
628  XrdOssPath::Trim2Base(lnkbuff+lnklen-1);
629 
630 // Search for matching logical partition
631 //
632  fsp = fsfirst;
633  while(fsp && strcmp(fsp->path, lnkbuff))
634  if ((fsp = fsp->next) == fsfirst) {fsp = 0; break;}
635  return fsp;
636 }
637 
638 /******************************************************************************/
639 /* I n i t */
640 /******************************************************************************/
641 
642 // Init() is only called during configuration and no locks are needed.
643 
644 int XrdOssCache::Init(const char *UPath, const char *Qfile, int isSOL, int us)
645 {
646  XrdOssCache_Group *cgp;
647  long long bytesUsed;
648 
649 // If usage directory or quota file was passed then we initialize space handling
650 // We need to create a space object to track usage across failures.
651 //
652  if ((UPath || Qfile) && !XrdOssSpace::Init(UPath,Qfile,isSOL,us)) return 1;
653  if (Qfile) Quotas = !isSOL;
654  if (UPath) Usage = 1;
655 
656 // If we will be saving space information then we need to assign each group
657 // to a save set. If there is no space object then skip all of this.
658 //
659  if (UPath && (cgp = XrdOssCache_Group::fsgroups))
660  do {cgp->GRPid = XrdOssSpace::Assign(cgp->group, bytesUsed);
661  cgp->Usage = bytesUsed;
662  } while((cgp = cgp->next));
663  return 0;
664 }
665 
666 /******************************************************************************/
667 
668 int XrdOssCache::Init(long long aMin, int ovhd, int aFuzz)
669 {
670 // Set values
671 //
672  minAlloc = aMin;
673  ovhAlloc = ovhd;
674  fuzAlloc = static_cast<double>(aFuzz)/100.0;
675  return 0;
676 }
677 
678 /******************************************************************************/
679 /* L i s t */
680 /******************************************************************************/
681 
682 void XrdOssCache::List(const char *lname, XrdSysError &Eroute)
683 {
684  XrdOssCache_FS *fsp;
685  const char *theCmd, *rpath;
686  char *pP, buff[4096];
687 
688  if ((fsp = fsfirst)) do
689  {if (fsp->opts & XrdOssCache_FS::isXA)
690  {pP = (char *)fsp->path + fsp->plen - 1;
691  do {pP--;} while(*pP != '/');
692  *pP = '\0'; theCmd = "space";
693  } else {pP=0; theCmd = "cache";}
694  rpath = (strcmp(fsp->fsdata->path, fsp->fsdata->pact)
695  ? fsp->fsdata->pact : "");
696  snprintf(buff, sizeof(buff), "%s%s %s %s -> %s[%d:%d] %s",
697  lname, theCmd, fsp->group, fsp->path, fsp->fsdata->devN,
698  fsp->fsdata->bdevID, fsp->fsdata->partID, rpath);
699  if (pP) *pP = '/';
700  Eroute.Say(buff);
701  fsp = fsp->next;
702  } while(fsp != fsfirst);
703 }
704 
705 /******************************************************************************/
706 /* M a p D e v s */
707 /******************************************************************************/
708 
709 void XrdOssCache::MapDevs(bool dBug)
710 {
711 #ifdef __linux__
712  const char *pPart = "/proc/partitions";
713  std::map<std::string, int> dn2id;
714  XrdOucStream strm;
715  std::string sOrg;
716  char *line, *sMaj, *sMin, *sBlk, *sDev, sAlt[16], sDN[sizeof(devID::nDev)];
717  dev_t dNode;
718  int dNum, n, fd, vMaj, vMin;
719 
720 // Our first step is to find all of the block devices on this machine and
721 // map them to device numbers (our own as well as the st_dev values).
722 //
723  if ((fd = open(pPart, O_RDONLY)) < 0) return;
724  strm.Attach(fd);
725 
726 // Read through the table until the end getting information we need
727 //
728  while((line = strm.GetLine()))
729  {if (!(sMaj = strm.GetToken()) || !isdigit(*sMaj)) continue;
730  if (!(vMaj = atoi(sMaj))) continue;
731  if (!(sMin = strm.GetToken()) || !isdigit(*sMin)) continue;
732  vMin = atoi(sMin);
733  if (!(sBlk = strm.GetToken()) || !isdigit(*sBlk)) continue;
734  if (!(sDev = strm.GetToken())) continue;
735 
736  // Preprocess LVM devices
737  //
738  if (!strncmp(sDev, "dm-", 3))
739  {if (!MapDM(sDev, sAlt, sizeof(sAlt)))
740  {if (dBug) std::cerr <<"Config " <<sDev <<'[' <<vMaj <<':'
741  <<vMin <<"] -> dev[0]" <<std::endl;
742  continue;
743  }
744  sOrg = sDev;
745  sDev = sAlt;
746  } else sOrg = sDev;
747 
748  // We are only concerned about normal block devices
749  //
750  if (sDev[1] != 'd' || (*sDev != 's' && *sDev != 'h')) continue;
751  strlcpy(sDN, sDev, sizeof(sDN));
752  sDN[sizeof(sDN)-1] = 0;
753 
754  // Trim off any numbers from the id
755  //
756  n = strlen(sDev)-1;
757  while(isdigit(sDev[n])) sDev[n--] = 0;
758 
759  // Generate the device number (existing or new)
760  //
761  std::map<std::string,int>::iterator it = dn2id.find(std::string(sDev));
762  if (it != dn2id.end()) dNum = it->second;
763  else {dNum = devNMax++;
764  dn2id[std::string(sDev)] = dNum;
765  }
766 
767  // Add the device to out map
768  //
769  dNode = makedev(vMaj, vMin);
770  devID theID = {dNum, 0, {0}};
771  strcpy(theID.nDev, sDN);
772  dev2ID[dNode] = theID;
773 
774  // Print result if so wanted
775  //
776  if (dBug) std::cerr <<"Config " <<sOrg <<'[' <<vMaj <<':' <<vMin
777  <<"] -> " <<sDev <<'[' <<dNum <<']' <<std::endl;
778  }
779 #endif
780 }
781 
782 /******************************************************************************/
783 /* Private: M a p D M */
784 /******************************************************************************/
785 
786 bool XrdOssCache::MapDM(const char *ldm, char *buff, int blen)
787 {
788  const char *dmInfo1 = "/sys/devices/virtual/block/", *dmInfo2 = "/slaves";
789  struct dirent *dP;
790  bool aOK = false;
791 
792  std::string dmPath = dmInfo1;
793  dmPath += ldm;
794  dmPath += dmInfo2;
795 
796  DIR *slaves = opendir(dmPath.c_str());
797  if (!slaves) return 0;
798  while((dP = readdir(slaves)))
799  {if (dP->d_type == DT_LNK && dP->d_name[1] == 'd'
800  && (dP->d_name[0] == 's' || dP->d_name[0] == 'h'))
801  {if ((int)strlen(dP->d_name) < blen)
802  {strcpy(buff, dP->d_name); aOK = true; break;}
803  }
804  }
805 
806  closedir(slaves);
807  return aOK;
808 }
809 
810 /******************************************************************************/
811 /* P a r s e */
812 /******************************************************************************/
813 
814 char *XrdOssCache::Parse(const char *token, char *cbuff, int cblen)
815 {
816  char *Path;
817 
818 // Check for default
819 //
820  if (!token || *token == ':')
821  {strlcpy(cbuff, OSS_CGROUP_DEFAULT, cblen);
822  return 0;
823  }
824 
825 // Get the correct cache group and partition path
826 //
827  if (!(Path = (char *) index(token, ':'))) strlcpy(cbuff, token, cblen);
828  else {int n = Path - token;
829  if (n >= cblen) n = cblen-1;
830  strncpy(cbuff, token, n); cbuff[n] = '\0';
831  Path++;
832  }
833 
834 // All done
835 //
836  return Path;
837 }
838 
839 /******************************************************************************/
840 /* S c a n */
841 /******************************************************************************/
842 
843 void *XrdOssCache::Scan(int cscanint)
844 {
845  EPNAME("CacheScan")
846  XrdOssCache_FSData *fsdp;
847  XrdOssCache_Group *fsgp;
848  const struct timespec naptime = {cscanint, 0};
849  long long frsz, llT; // llT is a dummy temporary
850  int retc, dbgMsg, dbgNoMsg, dbgDoMsg;
851 
852 // Try to prevent floodingthe log with scan messages
853 //
854  if (cscanint > 60) dbgMsg = cscanint/60;
855  else dbgMsg = 1;
856  dbgNoMsg = dbgMsg;
857 
858 // Loop scanning the cache
859 //
860  while(1)
861  {if (cscanint > 0) nanosleep(&naptime, 0);
862  dbgDoMsg = !dbgNoMsg--;
863  if (dbgDoMsg) dbgNoMsg = dbgMsg;
864 
865  // Get the cache context lock
866  //
867  Mutex.Lock();
868 
869  // Scan through all filesystems skip filesystem that have been
870  // recently adjusted to avoid fs statstics latency problems.
871  //
872  fsSize = 0;
873  fsTotFr= 0;
874  fsFree = 0;
875  fsdp = fsdata;
876  while(fsdp)
877  {retc = 0;
878  if ((fsdp->stat & XrdOssFSData_REFRESH)
879  || !(fsdp->stat & XrdOssFSData_ADJUSTED) || cscanint <= 0)
880  {frsz = XrdOssCache_FS::freeSpace(llT,fsdp->path);
881  if (frsz < 0) OssEroute.Emsg("CacheScan", errno ,
882  "state file system ",(char *)fsdp->path);
883  else {fsdp->frsz = frsz;
884  fsdp->stat &= ~(XrdOssFSData_REFRESH |
886  if (dbgDoMsg)
887  {DEBUG("New free=" <<fsdp->frsz <<" path=" <<fsdp->path);}
888  }
889  } else fsdp->stat |= XrdOssFSData_REFRESH;
890  if (!retc)
891  {if (fsdp->frsz > fsFree)
892  {fsFree = fsdp->frsz; fsSize = fsdp->size;}
893  fsTotFr += fsdp->frsz;
894  }
895  fsdp = fsdp->next;
896  }
897 
898  // Unlock the cache and if we have quotas check them out
899  //
900  Mutex.UnLock();
901  if (cscanint <= 0) return (void *)0;
902  if (Quotas) XrdOssSpace::Quotas();
903 
904  // Update usage information if we are keeping track of it
905  if (Usage && XrdOssSpace::Readjust())
907  Mutex.Lock();
908  while(fsgp)
909  {fsgp->Usage = XrdOssSpace::Usage(fsgp->GRPid);
910  fsgp = fsgp->next;
911  }
912  Mutex.UnLock();
913  }
914  }
915 
916 // Keep the compiler happy
917 //
918  return (void *)0;
919 }
void Usage(const char *msg)
Definition: XrdAccTest.cc:105
#define DEBUG(x)
Definition: XrdBwmTrace.hh:54
#define EPNAME(x)
Definition: XrdBwmTrace.hh:56
XrdSysTrace OssTrace
XrdSysError OssEroute
#define XrdOssFSData_REFRESH
Definition: XrdOssCache.hh:104
const char ** apVec
Definition: XrdOssCache.hh:172
#define XrdOssFSData_ADJUSTED
Definition: XrdOssCache.hh:103
XrdOssCache_FSData * fsP
Definition: XrdOssCache.hh:171
#define OSS_CGROUP_DEFAULT
Definition: XrdOssOpaque.hh:41
@ Info
int stat(const char *path, struct stat *buf)
struct dirent * readdir(DIR *dirp)
int open(const char *path, int oflag,...)
int lstat(const char *path, struct stat *buf)
int mkdir(const char *path, mode_t mode)
int closedir(DIR *dirp)
DIR * opendir(const char *path)
XrdOucString Path
struct myOpts opts
size_t strlcpy(char *dst, const char *src, size_t sz)
#define XRDABS(x)
const char * path
Definition: XrdOssCache.hh:114
XrdOssCache_FSData(const char *, STATFS_t &, dev_t)
Definition: XrdOssCache.cc:105
const char * devN
Definition: XrdOssCache.hh:116
const char * pact
Definition: XrdOssCache.hh:115
unsigned short partID
Definition: XrdOssCache.hh:120
unsigned short bdevID
Definition: XrdOssCache.hh:119
XrdOssCache_FSData * next
Definition: XrdOssCache.hh:110
const char * group
Definition: XrdOssCache.hh:139
static int getSpace(XrdOssCache_Space &Space, const char *sname, XrdOssVSPart **vsPart=0)
Definition: XrdOssCache.cc:327
XrdOssCache_Group * fsgroup
Definition: XrdOssCache.hh:145
static long long freeSpace(long long &Size, const char *path=0)
Definition: XrdOssCache.cc:279
const char * path
Definition: XrdOssCache.hh:140
XrdOssCache_FS * next
Definition: XrdOssCache.hh:138
static int Add(const char *Path)
Definition: XrdOssCache.cc:250
XrdOssCache_FSData * fsdata
Definition: XrdOssCache.hh:144
XrdOssCache_FS(int &retc, const char *fsg, const char *fsp, FSOpts opt)
Definition: XrdOssCache.cc:146
static long long PubQuota
Definition: XrdOssCache.hh:197
static XrdOssCache_Group * fsgroups
Definition: XrdOssCache.hh:199
XrdOssCache_FS * curr
Definition: XrdOssCache.hh:188
XrdOssCache_FSAP * fsVec
Definition: XrdOssCache.hh:189
XrdOssCache_Group * next
Definition: XrdOssCache.hh:186
static XrdOssCache_Group * PubGroup
Definition: XrdOssCache.hh:196
long long Inodes
Definition: XrdOssCache.hh:86
long long Largest
Definition: XrdOssCache.hh:85
long long Inleft
Definition: XrdOssCache.hh:87
long long Maxfree
Definition: XrdOssCache.hh:84
static int Init(const char *UDir, const char *Qfile, int isSOL, int usync=0)
Definition: XrdOssCache.cc:644
static XrdOssCache_FSData * fsdata
Definition: XrdOssCache.hh:271
static void * Scan(int cscanint)
Definition: XrdOssCache.cc:843
static long long fsSize
Definition: XrdOssCache.hh:268
static long long fsLarge
Definition: XrdOssCache.hh:265
static long long fsTotal
Definition: XrdOssCache.hh:264
static int Alloc(allocInfo &aInfo)
Definition: XrdOssCache.cc:492
static void DevInfo(struct stat &buf, bool limits=false)
Definition: XrdOssCache.cc:583
static long long fsTotFr
Definition: XrdOssCache.hh:266
static char * Parse(const char *token, char *cbuff, int cblen)
Definition: XrdOssCache.cc:814
static int fsCount
Definition: XrdOssCache.hh:272
static XrdOssCache_FS * fsfirst
Definition: XrdOssCache.hh:269
static void List(const char *lname, XrdSysError &Eroute)
Definition: XrdOssCache.cc:682
static long long fsFree
Definition: XrdOssCache.hh:267
static XrdOssCache_FS * fslast
Definition: XrdOssCache.hh:270
static void MapDevs(bool dBug=false)
Definition: XrdOssCache.cc:709
static XrdOssCache_FS * Find(const char *Path, int lklen=0)
Definition: XrdOssCache.cc:611
static void Adjust(dev_t devid, off_t size)
Definition: XrdOssCache.cc:397
static XrdSysMutex Mutex
Definition: XrdOssCache.hh:262
static char * genPFN(fnInfo &Info, char *buff, int blen, const char *Path=0)
Definition: XrdOssPath.cc:172
static void Trim2Base(char *eP)
Definition: XrdOssPath.cc:304
static char * genPath(const char *inPath, const char *cgrp, char *sfx)
Definition: XrdOssPath.cc:134
static long long Usage(int gent)
Definition: XrdOssSpace.cc:521
static int Init()
Definition: XrdOssSpace.cc:224
static void Adjust(int Gent, off_t Space, sType=Serv)
Definition: XrdOssSpace.cc:81
static int Quotas()
Definition: XrdOssSpace.cc:345
unsigned short partID
Definition: XrdOssVS.hh:71
long long Total
Definition: XrdOssVS.hh:68
const char * pPath
Definition: XrdOssVS.hh:66
const char ** aPath
Definition: XrdOssVS.hh:67
long long Free
Definition: XrdOssVS.hh:69
unsigned short bdevID
Definition: XrdOssVS.hh:70
char * GetLine()
int Attach(int FileDescriptor, int bsz=2047)
char * GetToken(int lowcase=0)
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:95
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
Definition: XrdSysError.cc:141
std::map< dev_t, devID > dev2ID
Definition: XrdOssCache.cc:94
XrdOssCache_FS * cgFSp
Definition: XrdOssCache.hh:231