XRootD
XrdOssMio.cc
Go to the documentation of this file.
1 /******************************************************************************/
2 /* */
3 /* X r d O s s M i o . c c */
4 /* */
5 /* (c) 2005 by the Board of Trustees of the Leland Stanford, Jr., University */
6 /* All Rights Reserved */
7 /* Produced by Andrew Hanushevsky for Stanford University under contract */
8 /* DE-AC02-76-SFO0515 with the Department of Energy */
9 /* */
10 /* This file is part of the XRootD software suite. */
11 /* */
12 /* XRootD is free software: you can redistribute it and/or modify it under */
13 /* the terms of the GNU Lesser General Public License as published by the */
14 /* Free Software Foundation, either version 3 of the License, or (at your */
15 /* option) any later version. */
16 /* */
17 /* XRootD is distributed in the hope that it will be useful, but WITHOUT */
18 /* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or */
19 /* FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public */
20 /* License for more details. */
21 /* */
22 /* You should have received a copy of the GNU Lesser General Public License */
23 /* along with XRootD in a file called COPYING.LESSER (LGPL license) and file */
24 /* COPYING (GPL license). If not, see <http://www.gnu.org/licenses/>. */
25 /* */
26 /* The copyright holder's institutional names and contributor's names may not */
27 /* be used to endorse or promote products derived from this software without */
28 /* specific prior written permission of the institution or contributor. */
29 /******************************************************************************/
30 
31 #include <unistd.h>
32 #include <cstdio>
33 #include <sys/param.h>
34 #include <sys/types.h>
35 #include <sys/stat.h>
36 
37 #if defined(_POSIX_MAPPED_FILES)
38 #include <sys/mman.h>
39 #endif
40 
41 #include "XrdSys/XrdSysPthread.hh"
42 #include "XrdOss/XrdOssMio.hh"
43 #include "XrdOss/XrdOssMioFile.hh"
44 #include "XrdOss/XrdOssTrace.hh"
45 #include "XrdOuc/XrdOucUtils.hh"
46 
47 /******************************************************************************/
48 /* S t a t i c V a r i a b l e s */
49 /******************************************************************************/
50 
51 XrdOucHash<XrdOssMioFile> XrdOssMio::MM_Hash;
52 
53 XrdSysMutex XrdOssMio::MM_Mutex;
54 
55 XrdOssMioFile *XrdOssMio::MM_Perm = 0;
56 XrdOssMioFile *XrdOssMio::MM_Idle = 0;
57 XrdOssMioFile *XrdOssMio::MM_IdleLast = 0;
58 
59 char XrdOssMio::MM_on = 1;
60 char XrdOssMio::MM_chk = 0;
61 char XrdOssMio::MM_okmlock = 1;
62 char XrdOssMio::MM_preld = 0;
63 long long XrdOssMio::MM_pagsz = (long long)sysconf(_SC_PAGESIZE);
64 #ifdef __APPLE__
65 long long XrdOssMio::MM_pages = 1024*1024*1024;
66 #else
67 long long XrdOssMio::MM_pages = (long long)sysconf(_SC_PHYS_PAGES);
68 #endif
69 long long XrdOssMio::MM_max = MM_pagsz*MM_pages/2;
70 long long XrdOssMio::MM_inuse = 0;
71 
72 extern XrdSysError OssEroute;
73 
74 extern XrdSysTrace OssTrace;
75 
76 /******************************************************************************/
77 /* D i s p l a y */
78 /******************************************************************************/
79 
81 {
82  char buff[1080];
83  snprintf(buff, sizeof(buff), " oss.memfile %s%s%s max %lld",
84  (MM_on ? "" : "off "),
85  (MM_preld ? "preload" : ""),
86  (MM_chk ? "check xattr" : ""), MM_max);
87  Eroute.Say(buff);
88 }
89 
90 /******************************************************************************/
91 /* M a p */
92 /******************************************************************************/
93 
94 XrdOssMioFile *XrdOssMio::Map(char *path, int fd, int opts)
95 {
96 #if defined(_POSIX_MAPPED_FILES)
97  EPNAME("MioMap");
98  XrdSysMutexHelper mapMutex;
99  struct stat statb;
100  XrdOssMioFile *mp;
101  void *thefile;
102  char hashname[64];
103 
104 // Get the size of the file
105 //
106  if (fstat(fd, &statb))
107  {OssEroute.Emsg("Mio", errno, "fstat file", path);
108  return 0;
109  }
110 
111 // Develop hash name for this file
112 //
113  int st_devSZ = sizeof(statb.st_dev);
114  XrdOucUtils::bin2hex((char *)&statb.st_dev, st_devSZ,
115  hashname, sizeof(hashname), false);
116  st_devSZ <<= 1;
117  XrdOucUtils::bin2hex((char *)&statb.st_ino, int(sizeof(statb.st_ino)),
118  hashname+st_devSZ, sizeof(hashname) - st_devSZ, false);
119 
120 // Because of potntial race conditions, we must serialize execution
121 //
122  mapMutex.Lock(&MM_Mutex);
123 
124 // Check if we already have this mapping
125 //
126  if ((mp = MM_Hash.Find(hashname)))
127  {DEBUG("Reusing mmap; usecnt=" <<mp->inUse <<" path=" <<path);
128  if (!(mp->Status & OSSMIO_MPRM) && !mp->inUse) Reclaim(mp);
129  mp->inUse++;
130  return mp;
131  }
132 
133 // Check if memory will be over committed
134 //
135  if (MM_inuse + statb.st_size > MM_max)
136  {if (!Reclaim(statb.st_size))
137  {OssEroute.Emsg("Mio", "Unable to reclaim enough storage to mmap",path);
138  return 0;
139  }
140  }
141  MM_inuse += statb.st_size;
142 
143 // Memory map the file
144 //
145  if ((thefile = mmap(0,statb.st_size,PROT_READ,MAP_PRIVATE,fd,0))==MAP_FAILED)
146  {OssEroute.Emsg("Mio", errno, "mmap file", path);
147  return 0;
148  } else {DEBUG("mmap " <<statb.st_size <<" bytes for " <<path);}
149 
150 // Lock the file, if need be. Turn off locking if we don't have privs
151 //
152  if (MM_okmlock && (opts & OSSMIO_MLOK))
153  {if (mlock((char *)thefile, statb.st_size))
154  { if (errno == ENOSYS || errno == ENOTSUP)
155  {OssEroute.Emsg("Mio","mlock() not supported; feature disabled.");
156  MM_okmlock = 0;
157  }
158  else if (errno == EPERM)
159  {OssEroute.Emsg("Mio","Not privileged for mlock(); feature disabled.");
160  MM_okmlock = 0;
161  }
162  else OssEroute.Emsg("Mio", errno, "mlock file", path);
163  } else {DEBUG("Locked " <<statb.st_size <<" bytes for " <<path);}
164  }
165 
166 // get a new file object
167 //
168  if (!(mp = new XrdOssMioFile(hashname)))
169  {OssEroute.Emsg("Mio", "Unable to allocate mmap file object for", path);
170  munmap((char *)thefile, statb.st_size);
171  return 0;
172  }
173 
174 // Complete the object here
175 //
176  mp->Base = thefile;
177  mp->Size = statb.st_size;
178  mp->Dev = statb.st_dev;
179  mp->Ino = statb.st_ino;
180  mp->Status = opts;
181 
182 // Add the mapping to our hash table
183 //
184  if (MM_Hash.Add(hashname, mp))
185  {OssEroute.Emsg("Mio", "Hash add failed for", path);
186  munmap((char *)thefile, statb.st_size);
187  delete mp;
188  return 0;
189  }
190 
191 // If this is a permanent file, place it on the permanent queue
192 //
193  if (opts & OSSMIO_MPRM)
194  {mp->Next = MM_Perm; MM_Perm = mp;
195  DEBUG("Placed file on permanent queue " <<path);
196  }
197 
198 // If this file is to be preloaded, start it now
199 //
200  if (MM_preld && mp->inUse == 1)
201  {pthread_t tid;
202  int retc;
203  mp->inUse++;
204  if ((retc = XrdSysThread::Run(&tid, preLoad, (void *)mp)) < 0)
205  {OssEroute.Emsg("Mio", retc, "creating mmap preload thread");
206  mp->inUse--;
207  }
208  else DEBUG("started mmap preload thread; tid=" <<(unsigned long)tid);
209  }
210 
211 // All done
212 //
213  return mp;
214 #else
215  return 0;
216 #endif
217 }
218 
219 /******************************************************************************/
220 /* p r e L o a d */
221 /******************************************************************************/
222 
223 void *XrdOssMio::preLoad(void *arg)
224 {
225  XrdOssMioFile *mp = (XrdOssMioFile *)arg;
226  char *Base = (char *)(mp->Base);
227  char *Bend = Base + mp->Size;
228  long long MY_pagsz = MM_pagsz;
229 
230 // Reference each page until we are done. This is somewhat obtuse but we
231 // are trying to keep the compiler from optimizing out the code.
232 //
233  while(Base < Bend) Base += (*Base ? MY_pagsz : MM_pagsz);
234 
235 // All done
236 //
237  Recycle(mp);
238  return (void *)0;
239 }
240 
241 /******************************************************************************/
242 /* R e c l a i m */
243 /******************************************************************************/
244 
245 // Reclaim() can only be called if the caller has the MM_Mutex lock!
246 //
247 int XrdOssMio::Reclaim(off_t amount)
248 {
249  EPNAME("MioReclaim");
250  XrdOssMioFile *mp;
251  DEBUG("Trying to reclaim " <<amount <<" bytes.");
252 
253 // Try to reclaim memory
254 //
255  while((mp = MM_Idle) && amount > 0)
256  {MM_Idle = mp->Next;
257  MM_inuse -= mp->Size;
258  amount -= mp->Size;
259  MM_Hash.Del(mp->HashName); // This will delete the object
260  }
261 
262 // Indicate whether we cleared enough
263 //
264  return amount <= 0;
265 }
266 
267 /******************************************************************************/
268 
269 int XrdOssMio::Reclaim(XrdOssMioFile *mp)
270 {
271  EPNAME("MioReclaim");
272  XrdOssMioFile *pmp = 0, *cmp = MM_Idle;
273 
274 // Try to find the mapping
275 //
276  while(cmp && mp != cmp) {pmp = cmp; cmp = cmp->Next;}
277 
278 // Remove mapping from the idle list
279 //
280  if (cmp)
281  {if (pmp) pmp->Next = mp->Next;
282  else MM_Idle = mp->Next;
283  if (MM_IdleLast == cmp) MM_IdleLast = pmp;
284  }
285  else {DEBUG("Cannot find mapping for " <<mp->Dev <<':' <<mp->Ino);}
286 
287  return (cmp != 0);
288 }
289 
290 /******************************************************************************/
291 /* R e c y c l e */
292 /******************************************************************************/
293 
295 {
296  XrdSysMutexHelper mmMutex(&MM_Mutex);
297 
298 // Decrement the use count
299 //
300  mp->inUse--;
301  if (mp->inUse < 0)
302  {OssEroute.Emsg("Mio", "MM usecount underflow for ", mp->HashName);
303  mp->inUse = 0;
304  } else if (mp->inUse > 0) return;
305 
306 // If this is not a kept mapping, put it on the reclaim list
307 //
308  if (!(mp->Status & OSSMIO_MPRM))
309  {if (MM_IdleLast) MM_IdleLast->Next = mp;
310  else MM_Idle = mp;
311  MM_IdleLast = mp;
312  mp->Next = 0;
313  }
314 }
315 
316 /******************************************************************************/
317 /* S e t */
318 /******************************************************************************/
319 
320 void XrdOssMio::Set(int V_on, int V_preld, int V_check)
321 {
322  if (V_on >= 0) MM_on = (char)V_on;
323  if (V_preld >= 0) MM_preld = (char)V_preld;
324  if (V_check >= 0) MM_chk = (char)V_check;
325 }
326 
327 void XrdOssMio::Set(long long V_max)
328 {
329  if (V_max > 0) MM_max = V_max;
330  else if (V_max < 0) MM_max = MM_pagsz*MM_pages*(-V_max)/100;
331 }
332 
333 /******************************************************************************/
334 /* X r d O s s d M i o F i l e D e s t r u c t o r */
335 /******************************************************************************/
336 
338 {
339 #if defined(_POSIX_MAPPED_FILES)
340  munmap((char *)Base, Size);
341 #endif
342 }
#define DEBUG(x)
Definition: XrdBwmTrace.hh:54
#define EPNAME(x)
Definition: XrdBwmTrace.hh:56
XrdSysTrace OssTrace
XrdSysError OssEroute
#define OSSMIO_MLOK
Definition: XrdOssMio.hh:40
#define OSSMIO_MPRM
Definition: XrdOssMio.hh:42
int stat(const char *path, struct stat *buf)
int fstat(int fildes, struct stat *buf)
struct myOpts opts
static XrdOssMioFile * Map(char *path, int fd, int opts)
Definition: XrdOssMio.cc:94
static void Recycle(XrdOssMioFile *mp)
Definition: XrdOssMio.cc:294
static void * preLoad(void *arg)
Definition: XrdOssMio.cc:223
static void Set(int V_off, int V_preld, int V_check)
Definition: XrdOssMio.cc:320
static void Display(XrdSysError &Eroute)
Definition: XrdOssMio.cc:80
int Del(const char *KeyVal, XrdOucHash_Options opt=Hash_default)
Definition: XrdOucHash.icc:136
T * Add(const char *KeyVal, T *KeyData, const int LifeTime=0, XrdOucHash_Options opt=Hash_default)
Definition: XrdOucHash.icc:61
T * Find(const char *KeyVal, time_t *KeyTime=0)
Definition: XrdOucHash.icc:160
static char * bin2hex(char *inbuff, int dlen, char *buff, int blen, bool sep=true)
Definition: XrdOucUtils.cc:163
int Emsg(const char *esfx, int ecode, const char *text1, const char *text2=0)
Definition: XrdSysError.cc:95
void Say(const char *text1, const char *text2=0, const char *txt3=0, const char *text4=0, const char *text5=0, const char *txt6=0)
Definition: XrdSysError.cc:141
void Lock(XrdSysMutex *Mutex)
static int Run(pthread_t *, void *(*proc)(void *), void *arg, int opts=0, const char *desc=0)