XRootD
XrdEcReader.hh
Go to the documentation of this file.
1 //------------------------------------------------------------------------------
2 // Copyright (c) 2011-2014 by European Organization for Nuclear Research (CERN)
3 // Author: Michal Simon <michal.simon@cern.ch>
4 //------------------------------------------------------------------------------
5 // This file is part of the XRootD software suite.
6 //
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //
20 // In applying this licence, CERN does not waive the privileges and immunities
21 // granted to it by virtue of its status as an Intergovernmental Organization
22 // or submit itself to any jurisdiction.
23 //------------------------------------------------------------------------------
24 
25 #ifndef SRC_XRDEC_XRDECREADER_HH_
26 #define SRC_XRDEC_XRDECREADER_HH_
27 
28 #include "XrdEc/XrdEcObjCfg.hh"
29 
30 #include "XrdCl/XrdClZipArchive.hh"
31 #include "XrdCl/XrdClOperations.hh"
32 
33 #include <string>
34 #include <unordered_map>
35 #include <unordered_set>
36 
37 class MicroTest;
38 class XrdEcTests;
39 
40 namespace XrdEc
41 {
42  //---------------------------------------------------------------------------
43  // Forward declaration for the internal cache
44  //---------------------------------------------------------------------------
45  struct block_t;
46  //---------------------------------------------------------------------------
47  // Buffer for a single chunk of data
48  //---------------------------------------------------------------------------
49  typedef std::vector<char> buffer_t;
50  //---------------------------------------------------------------------------
51  // Read callback, to be called with status and number of bytes read
52  //---------------------------------------------------------------------------
53  typedef std::function<void( const XrdCl::XRootDStatus&, uint32_t )> callback_t;
54 
55  //---------------------------------------------------------------------------
56  // Reader object for reading erasure coded and striped data
57  //---------------------------------------------------------------------------
58  class Reader
59  {
60  friend class ::MicroTest;
61  friend class ::XrdEcTests;
62  friend struct block_t;
63 
64  public:
65  //-----------------------------------------------------------------------
70  //-----------------------------------------------------------------------
71  Reader( ObjCfg &objcfg ) : objcfg( objcfg ), lstblk( 0 ), filesize( 0 )
72  {
73  }
74 
75  //-----------------------------------------------------------------------
76  // Destructor
77  //-----------------------------------------------------------------------
78  virtual ~Reader();
79 
80  //-----------------------------------------------------------------------
84  //-----------------------------------------------------------------------
85  void Open( XrdCl::ResponseHandler *handler, uint16_t timeout = 0 );
86 
87  //-----------------------------------------------------------------------
94  //-----------------------------------------------------------------------
95  void Read( uint64_t offset,
96  uint32_t length,
97  void *buffer,
98  XrdCl::ResponseHandler *handler,
99  uint16_t timeout );
100 
101  /*
102  * Read multiple locations and lengths of data
103  * internally remapped to stripes
104  *
105  * @param chunks : list of offsets, lengths and separate buffers
106  * @param buffer : optional full buffer for all data
107  */
108  void VectorRead( const XrdCl::ChunkList &chunks,
109  void *buffer,
110  XrdCl::ResponseHandler *handler,
111  uint16_t timeout);
112 
113  //-----------------------------------------------------------------------
115  //-----------------------------------------------------------------------
116  void Close( XrdCl::ResponseHandler *handler, uint16_t timeout = 0 );
117 
118  //-----------------------------------------------------------------------
120  //-----------------------------------------------------------------------
121  inline uint64_t GetSize()
122  {
123  return filesize;
124  }
125 
126  private:
127 
128  //-----------------------------------------------------------------------
136  //-----------------------------------------------------------------------
137  void Read( size_t blknb, size_t strpnb, buffer_t &buffer, callback_t cb, uint16_t timeout = 0 );
138 
139  //-----------------------------------------------------------------------
143  //-----------------------------------------------------------------------
144  XrdCl::Pipeline ReadMetadata( size_t index );
145 
146  //-----------------------------------------------------------------------
150  //-----------------------------------------------------------------------
151  XrdCl::Pipeline ReadSize( size_t index );
152 
153  //-----------------------------------------------------------------------
157  //-----------------------------------------------------------------------
158  bool ParseMetadata( XrdCl::ChunkInfo &ch );
159 
160  //-----------------------------------------------------------------------
164  //-----------------------------------------------------------------------
165  void AddMissing( const buffer_t &cdbuff );
166 
167  //-----------------------------------------------------------------------
169  //-----------------------------------------------------------------------
170  bool IsMissing( const std::string &fn );
171 
172  inline static callback_t ErrorCorrected(Reader *reader, std::shared_ptr<block_t> &self, size_t blkid, size_t strpid);
173 
174  void MissingVectorRead(std::shared_ptr<block_t> &block, size_t blkid, size_t strpid, uint16_t timeout = 0);
175 
176  typedef std::unordered_map<std::string, std::shared_ptr<XrdCl::ZipArchive>> dataarchs_t;
177  typedef std::unordered_map<std::string, buffer_t> metadata_t;
178  typedef std::unordered_map<std::string, std::string> urlmap_t;
179  typedef std::unordered_set<std::string> missing_t;
180 
181  ObjCfg &objcfg;
182  dataarchs_t dataarchs; //> map URL to ZipArchive object
183  metadata_t metadata; //> map URL to CD metadata
184  urlmap_t urlmap; //> map blknb/strpnb (data chunk) to URL
185  missing_t missing; //> set of missing stripes
186  std::shared_ptr<block_t> block; //> cache for the block we are reading from
187  std::mutex blkmtx; //> mutex guarding the block from parallel access
188  size_t lstblk; //> last block number
189  uint64_t filesize; //> file size (obtained from xattr)
190  std::map<std::string, size_t> archiveIndices;
191 
192  std::mutex missingChunksMutex;
193  std::vector<std::tuple<size_t, size_t>> missingChunksVectorRead;
194  std::condition_variable waitMissing;
195  };
196 
197 } /* namespace XrdEc */
198 
199 #endif /* SRC_XRDEC_XRDECREADER_HH_ */
Handle an async response.
uint64_t GetSize()
Definition: XrdEcReader.hh:121
void Close(XrdCl::ResponseHandler *handler, uint16_t timeout=0)
Close the data object.
Definition: XrdEcReader.cc:588
virtual ~Reader()
Definition: XrdEcReader.cc:427
void Read(uint64_t offset, uint32_t length, void *buffer, XrdCl::ResponseHandler *handler, uint16_t timeout)
Definition: XrdEcReader.cc:490
void Open(XrdCl::ResponseHandler *handler, uint16_t timeout=0)
Definition: XrdEcReader.cc:434
void VectorRead(const XrdCl::ChunkList &chunks, void *buffer, XrdCl::ResponseHandler *handler, uint16_t timeout)
Definition: XrdEcReader.cc:868
Reader(ObjCfg &objcfg)
Definition: XrdEcReader.hh:71
std::vector< ChunkInfo > ChunkList
List of chunks.
std::function< void(const XrdCl::XRootDStatus &, uint32_t)> callback_t
Definition: XrdEcReader.hh:53
std::vector< char > buffer_t
a buffer type
Definition: XrdEcReader.hh:45
Describe a data chunk for vector read.