XRootD
XrdClActionMetrics.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2021 by European Organization for Nuclear Research (CERN)
3 // Author: Andreas-Joachim Peters <andreas.joachim.peters@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #include "XrdClAction.hh"
26 #include <fstream>
27 #include <vector>
28 #include <tuple>
29 #include <unordered_map>
30 #include <chrono>
31 #include <iostream>
32 #include <iomanip>
33 #include <atomic>
34 #include <stdarg.h>
35 #include <getopt.h>
36 #include <map>
37 #include <vector>
38 #include <numeric>
39 #include <regex>
40 
41 namespace XrdCl
42 {
43 //------------------------------------------------------------------------------
45 //------------------------------------------------------------------------------
47 {
49  {
50  std::string op[] = { "OpenR", "OpenW", "Open", "Read", "Write", "Stat", "Close",
51  "PgRead", "PgWrite", "Truncate", "Sync", "VectorRead", "VectorWrite" };
52  for (auto& i : op)
53  {
54  std::string gain = i + "::tgain"; // time early for IO submission
55  std::string loss = i + "::tloss"; // time late for IO submission
56  std::string nomi = i + "::tnomi"; // nominal IO time
57  std::string exec = i + "::texec"; // time to submit IO
58  std::string meas = i + "::tmeas"; // time measured for IO to complete
59 
60  delays[gain] = 0;
61  delays[loss] = 0;
62  delays[nomi] = 0;
63  delays[exec] = 0;
64  delays[meas] = 0;
65 
66  std::string cnt = i + "::n"; // IOPS
67  std::string vol = i + "::b"; // number of bytes
68  std::string err = i + "::e"; // number of unsuccessful IOs
69  std::string off = i + "::o"; // maximum offset seen
70 
71  ios[cnt] = 0;
72  ios[vol] = 0;
73  ios[err] = 0;
74  ios[off] = 0;
75  }
76 
77  ios["All::e"] = 0; // Error counter for summing over files
78  synchronicity = 0.0;
79  errors = 0;
80  }
81 
82  std::string Dump(bool json) const
83  {
84  std::stringstream ss;
85  if (!json)
86  {
87  ss << "# -----------------------------------------------------------------" << std::endl;
88  if (fname != "")
89  {
90  ss << "# File: " << fname << std::endl;
91  ss << "# Sync: " << std::fixed << std::setprecision(2) << synchronicity << "%" << std::endl;
92  ss << "# Errs: " << std::fixed << errors << std::endl;
93  }
94  else
95  {
96  ss << "# Summary" << std::endl;
97  }
98  ss << "# -----------------------------------------------------------------" << std::endl;
99  for (auto& i : delays)
100  {
101  std::string key = i.first;
102  std::transform(key.begin(), key.end(), key.begin(), ::tolower);
103  if (i.second)
104  {
105  ss << "# " << std::setw(16) << key << " : " << std::setw(16) << std::fixed << i.second
106  << " s" << std::endl;
107  }
108  }
109  for (auto& i : ios)
110  {
111  std::string key = i.first;
112  std::transform(key.begin(), key.end(), key.begin(), ::tolower);
113  if (i.second)
114  {
115  ss << "# " << std::setw(16) << key << " : " << std::setw(16) << i.second << std::endl;
116  }
117  }
118  }
119  else
120  {
121  std::string name = fname;
122  if (fname.empty())
123  name = "_files_summary_";
124 
125  ss << " {" << std::endl;
126  ss << " \"name\":"
127  << "\"" << name << "\"," << std::endl;
128  ss << " \"synchronicity\": " << synchronicity << "," << std::endl;
129  ss << " \"errors\": " << errors << "," << std::endl;
130  for (auto& i : delays)
131  {
132  std::string key = i.first;
133  std::transform(key.begin(), key.end(), key.begin(), ::tolower);
134  if (i.second)
135  {
136  ss << " \"" << key << "\": " << i.second << "," << std::endl;
137  }
138  }
139  for (auto& i : ios)
140  {
141  std::string key = i.first;
142  std::transform(key.begin(), key.end(), key.begin(), ::tolower);
143  if (i.second)
144  {
145  ss << " \"" << key << "\": " << i.second << "," << std::endl;
146  }
147  }
148  ss.seekp(-2, std::ios_base::end);
149  ss << "\n";
150  if (fname.empty())
151  ss << " }" << std::endl;
152  else
153  ss << " }," << std::endl;
154  }
155  return ss.str();
156  }
157 
158  size_t getIopsRead() const
159  {
160  auto v1 = ios.find("Read::n");
161  auto v2 = ios.find("PgRead::n");
162  auto v3 = ios.find("VectorRead::n");
163  return (v1->second + v2->second + v3->second);
164  }
165 
166  size_t getIopsWrite() const
167  {
168  auto v1 = ios.find("Write::n");
169  auto v2 = ios.find("PgWrite::n");
170  auto v3 = ios.find("VectorWrite::n");
171  return (v1->second + v2->second + v3->second);
172  }
173 
174  size_t getBytesRead() const
175  {
176  auto v1 = ios.find("Read::b");
177  auto v2 = ios.find("PgRead::b");
178  auto v3 = ios.find("VectorRead::b");
179  return (v1->second + v2->second + v3->second);
180  }
181 
182  size_t getBytesWritten() const
183  {
184  auto v1 = ios.find("Write::b");
185  auto v2 = ios.find("PgWrite::b");
186  auto v3 = ios.find("VectorWrite::b");
187  return (v1->second + v2->second + v3->second);
188  }
189 
190  void addDelays(const std::string& action, const std::string& field, double value)
191  {
192  // function called from callbacks requires a guard
193  std::unique_lock<std::mutex> guard(mtx);
194  delays[action + "::" + field] += value;
195  }
196 
197  void addIos(const std::string& action, const std::string& field, double value)
198  {
199  // function called from callbacks requires a guard
200  std::unique_lock<std::mutex> guard(mtx);
201  ios[action + "::" + field] += value;
202  if (field == "e")
203  {
204  ios["All::e"] += value;
205  }
206  }
207 
208  void add(const ActionMetrics& other)
209  {
210  for (auto& k : other.ios)
211  {
212  ios[k.first] += k.second;
213  }
214  for (auto& k : other.delays)
215  {
216  delays[k.first] += k.second;
217  }
218  errors += other.errors;
219 
220  auto w1 = other.ios.find("Write::b");
221  auto w2 = other.ios.find("PgWrite::b");
222  auto w3 = other.ios.find("VectorWrite::b");
223 
224  if (((w1 != other.ios.end()) && w1->second) || ((w2 != other.ios.end()) && w2->second)
225  || ((w3 != other.ios.end()) && w3->second))
226  {
227  // count as EGRES
229  }
230  else
231  {
232  // count es INGRES
234  }
235  }
236 
237  static std::string humanreadable(uint64_t insize)
238  {
239  const uint64_t KB = 1000ll;
240  const uint64_t MB = 1000ll * KB;
241  const uint64_t GB = 1000ll * MB;
242  const uint64_t TB = 1000ll * GB;
243  const uint64_t PB = 1000ll * TB;
244  const uint64_t EB = 1000ll * PB;
245  std::stringstream ss;
246  if (insize >= (10 * KB))
247  {
248  if (insize >= MB)
249  {
250  if (insize >= GB)
251  {
252  if (insize >= TB)
253  {
254  if (insize >= PB)
255  {
256  if (insize >= EB)
257  {
258  // EB
259  ss << std::fixed << std::setprecision(2) << (insize * 1.0 / EB) << " EB";
260  }
261  else
262  {
263  // PB
264  ss << std::fixed << std::setprecision(2) << (insize * 1.0 / PB) << " PB";
265  }
266  }
267  else
268  {
269  // TB
270  ss << std::fixed << std::setprecision(2) << (insize * 1.0 / TB) << " TB";
271  }
272  }
273  else
274  {
275  // GB
276  ss << std::fixed << std::setprecision(2) << (insize * 1.0 / GB) << " GB";
277  }
278  }
279  else
280  {
281  // MB
282  ss << std::fixed << std::setprecision(2) << (insize * 1.0 / MB) << " MB";
283  }
284  }
285  else
286  {
287  // KB
288  ss << std::fixed << std::setprecision(2) << (insize * 1.0 / KB) << " KB";
289  }
290  }
291  else
292  {
293  ss << std::fixed << insize << " B";
294  }
295  return ss.str();
296  }
297 
298 
299  std::string fname;
300  std::string url;
301  double synchronicity; // sync->0: async sync->100.: sync IO
302  size_t errors; //< number of responses != SUCCESS out of the CVS file
303 
305  {
306  std::vector<double> reads;
307  std::vector<double> writes;
308 
309  double ReadSynchronicity() const
310  {
311  if (reads.size())
312  {
313  return accumulate(reads.begin(), reads.end(), 0.0) / reads.size();
314  }
315  else
316  {
317  return 0;
318  }
319  }
320 
321  double WriteSynchronicity() const
322  {
323  if (writes.size())
324  {
325  return accumulate(writes.begin(), writes.end(), 0.0) / writes.size();
326  }
327  else
328  {
329  return 0;
330  }
331  }
332  };
333 
335 
336  std::map<std::string, uint64_t> ios;
337  std::map<std::string, double> delays;
338  std::mutex mtx; // only required for async callbacks
339 };
340 }
nlohmann::json json
Metrics struct storing all timing and IO information of an action.
synchronicity_t aggregated_synchronicity
void add(const ActionMetrics &other)
std::map< std::string, uint64_t > ios
void addIos(const std::string &action, const std::string &field, double value)
std::map< std::string, double > delays
size_t getBytesWritten() const
void addDelays(const std::string &action, const std::string &field, double value)
std::string Dump(bool json) const
static std::string humanreadable(uint64_t insize)