xrootd
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros
XrdFileCacheInfo.hh
Go to the documentation of this file.
1 #ifndef __XRDFILECACHE_INFO_HH__
2 #define __XRDFILECACHE_INFO_HH__
3 //----------------------------------------------------------------------------------
4 // Copyright (c) 2014 by Board of Trustees of the Leland Stanford, Jr., University
5 // Author: Alja Mrak-Tadel, Matevz Tadel, Brian Bockelman
6 //----------------------------------------------------------------------------------
7 // XRootD is free software: you can redistribute it and/or modify
8 // it under the terms of the GNU Lesser General Public License as published by
9 // the Free Software Foundation, either version 3 of the License, or
10 // (at your option) any later version.
11 //
12 // XRootD is distributed in the hope that it will be useful,
13 // but WITHOUT ANY WARRANTY; without even the implied warranty of
14 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
15 // GNU General Public License for more details.
16 //
17 // You should have received a copy of the GNU Lesser General Public License
18 // along with XRootD. If not, see <http://www.gnu.org/licenses/>.
19 //----------------------------------------------------------------------------------
20 
21 #include <stdio.h>
22 #include <time.h>
23 #include <assert.h>
24 #include <vector>
25 
26 #include "XrdSys/XrdSysPthread.hh"
27 #include "XrdCl/XrdClConstants.hh"
28 #include "XrdCl/XrdClDefaultEnv.hh"
29 
30 class XrdOssDF;
31 class XrdCksCalc;
32 class XrdSysTrace;
33 
34 
35 namespace XrdCl
36 {
37 class Log;
38 }
39 
40 namespace XrdFileCache
41 {
42 class Stats;
43 
44 //----------------------------------------------------------------------------
46 //----------------------------------------------------------------------------
47 
48 class Info
49 {
50 public:
51  // !Access statistics
52  struct AStat
53  {
54  time_t AttachTime;
55  time_t DetachTime;
56  long long BytesDisk;
57  long long BytesRam;
58  long long BytesMissed;
59 
61  };
62 
63  struct Store {
64  int m_version;
65  long long m_bufferSize;
66  long long m_fileSize;
67  unsigned char *m_buff_synced;
68  char m_cksum[16];
69  time_t m_creationTime;
70  size_t m_accessCnt;
71  std::vector<AStat> m_astats;
72 
74  };
75 
76 
77  //------------------------------------------------------------------------
79  //------------------------------------------------------------------------
80  Info(XrdSysTrace* trace, bool prefetchBuffer = false);
81 
82  //------------------------------------------------------------------------
84  //------------------------------------------------------------------------
85  ~Info();
86 
87  //---------------------------------------------------------------------
91  //---------------------------------------------------------------------
92  void SetBitWritten(int i);
93 
94  //---------------------------------------------------------------------
98  //---------------------------------------------------------------------
99  void SetBitSynced(int i);
100 
101  //---------------------------------------------------------------------
103  //---------------------------------------------------------------------
104  void SetAllBitsSynced();
105 
106  //---------------------------------------------------------------------
110  //---------------------------------------------------------------------
111  void SetBitPrefetch(int i);
112 
113  void SetBufferSize(long long);
114 
115  void SetFileSize(long long);
116 
117  //---------------------------------------------------------------------
121  //---------------------------------------------------------------------
122  void ResizeBits(int n);
123 
124  //---------------------------------------------------------------------
131  //---------------------------------------------------------------------
132  bool Read(XrdOssDF* fp, const std::string &fname = "<unknown>");
133 
134  //---------------------------------------------------------------------
137  //---------------------------------------------------------------------
138  bool Write(XrdOssDF* fp, const std::string &fname = "<unknown>");
139 
140  //---------------------------------------------------------------------
142  //---------------------------------------------------------------------
143  void DisableDownloadStatus();
144 
145  //---------------------------------------------------------------------
147  //---------------------------------------------------------------------
148  void WriteIOStatAttach();
149 
151  //---------------------------------------------------------------------
152  void WriteIOStat(Stats& s);
153 
154  //---------------------------------------------------------------------
156  //---------------------------------------------------------------------
157  void WriteIOStatDetach(Stats& s);
158 
159  //---------------------------------------------------------------------
161  //---------------------------------------------------------------------
162  void WriteIOStatSingle(long long bytes_disk);
163 
164  //---------------------------------------------------------------------
166  //---------------------------------------------------------------------
167  void WriteIOStatSingle(long long bytes_disk, time_t att, time_t dtc);
168 
169  //---------------------------------------------------------------------
171  //---------------------------------------------------------------------
172  bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const;
173 
174  //---------------------------------------------------------------------
176  //---------------------------------------------------------------------
177  int GetSizeInBytes() const;
178 
179  //---------------------------------------------------------------------
181  //---------------------------------------------------------------------
182  int GetSizeInBits() const;
183 
184  //---------------------------------------------------------------------
186  //---------------------------------------------------------------------
187  long long GetFileSize() const;
188 
189  //---------------------------------------------------------------------
191  //---------------------------------------------------------------------
192  bool GetLatestDetachTime(time_t& t) const;
193 
194  //---------------------------------------------------------------------
196  //---------------------------------------------------------------------
197  long long GetBufferSize() const;
198 
199  //---------------------------------------------------------------------
201  //---------------------------------------------------------------------
202  bool TestBit(int i) const;
203 
204  //---------------------------------------------------------------------
206  //---------------------------------------------------------------------
207  bool TestPrefetchBit(int i) const;
208 
209  //---------------------------------------------------------------------
211  //---------------------------------------------------------------------
212  bool IsComplete() const;
213 
214  //---------------------------------------------------------------------
216  //---------------------------------------------------------------------
217  int GetNDownloadedBlocks() const;
218 
219  //---------------------------------------------------------------------
221  //---------------------------------------------------------------------
222  long long GetNDownloadedBytes() const;
223 
224  //---------------------------------------------------------------------
226  //---------------------------------------------------------------------
228 
229  //---------------------------------------------------------------------
231  //---------------------------------------------------------------------
232  size_t GetAccessCnt() { return m_store.m_accessCnt; }
233 
234  //---------------------------------------------------------------------
236  //---------------------------------------------------------------------
237  int GetVersion() { return m_store.m_version; }
238 
239  //---------------------------------------------------------------------
241  //---------------------------------------------------------------------
242  const Store& RefStoredData() const { return m_store; }
243 
244  //---------------------------------------------------------------------
246  //---------------------------------------------------------------------
247  void GetCksum( unsigned char* buff, char* digest);
248 
249  const static char* m_infoExtension;
250  const static char* m_traceID;
251  const static int m_defaultVersion;
252  const static size_t m_maxNumAccess;
253 
254  XrdSysTrace* GetTrace() const {return m_trace; }
255 
256  static size_t GetMaxNumAccess() { return m_maxNumAccess; }
257 
258 protected:
260 
263  unsigned char *m_buff_written;
264  unsigned char *m_buff_prefetch;
265 
267  bool m_complete;
268 
269 private:
270  inline unsigned char cfiBIT(int n) const { return 1 << n; }
271 
272  // split reading for V1
273  bool ReadV1(XrdOssDF* fp, const std::string &fname);
275 };
276 
277 //------------------------------------------------------------------------------
278 
279 inline bool Info::TestBit(int i) const
280 {
281  const int cn = i/8;
282  assert(cn < GetSizeInBytes());
283 
284  const int off = i - cn*8;
285  return (m_buff_written[cn] & cfiBIT(off)) == cfiBIT(off);
286 }
287 
288 inline bool Info::TestPrefetchBit(int i) const
289 {
290  if (!m_buff_prefetch) return false;
291 
292  const int cn = i/8;
293  assert(cn < GetSizeInBytes());
294 
295  const int off = i - cn*8;
296  return (m_buff_prefetch[cn] & cfiBIT(off)) == cfiBIT(off);
297 }
298 
299 inline int Info::GetNDownloadedBlocks() const
300 {
301  int cntd = 0;
302  for (int i = 0; i < m_sizeInBits; ++i)
303  if (TestBit(i)) cntd++;
304 
305  return cntd;
306 }
307 
308 inline long long Info::GetNDownloadedBytes() const
309 {
311 }
312 
313 inline int Info::GetSizeInBytes() const
314 {
315  if (m_sizeInBits)
316  return ((m_sizeInBits - 1)/8 + 1);
317  else
318  return 0;
319 }
320 
321 inline int Info::GetSizeInBits() const
322 {
323  return m_sizeInBits;
324 }
325 
326 inline long long Info::GetFileSize() const
327 {
328  return m_store.m_fileSize;
329 }
330 
331 inline bool Info::IsComplete() const
332 {
333  return m_complete;
334 }
335 
336 inline bool Info::IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
337 {
338  // TODO rewrite to use full byte comparisons outside of edges ?
339  // Also, it is always called with fisrtsdx = 0, lastIdx = m_sizeInBits.
340  for (int i = firstIdx; i < lastIdx; ++i)
341  if (! TestBit(i)) return true;
342 
343  return false;
344 }
345 
347 {
349 }
350 
351 inline void Info::SetBitSynced(int i)
352 {
353  const int cn = i/8;
354  assert(cn < GetSizeInBytes());
355 
356  const int off = i - cn*8;
357  m_store.m_buff_synced[cn] |= cfiBIT(off);
358 }
359 
360 inline void Info::SetBitWritten(int i)
361 {
362  const int cn = i/8;
363  assert(cn < GetSizeInBytes());
364 
365  const int off = i - cn*8;
366  m_buff_written[cn] |= cfiBIT(off);
367 }
368 
369 inline void Info::SetBitPrefetch(int i)
370 {
371  if (!m_buff_prefetch) return;
372 
373  const int cn = i/8;
374  assert(cn < GetSizeInBytes());
375 
376  const int off = i - cn*8;
377  m_buff_prefetch[cn] |= cfiBIT(off);
378 }
379 
380 inline long long Info::GetBufferSize() const
381 {
382  return m_store.m_bufferSize;
383 }
384 
385 }
386 #endif
int m_version
info version
Definition: XrdFileCacheInfo.hh:64
bool Read(XrdOssDF *fp, const std::string &fname="<unknown>")
Rea load content from cinfo file into this object.
unsigned char * m_buff_written
download state vector
Definition: XrdFileCacheInfo.hh:263
void WriteIOStat(Stats &s)
Write bytes missed, hits, and disk.
XrdCksCalc * m_cksCalc
Definition: XrdFileCacheInfo.hh:274
void SetBitWritten(int i)
Mark block as downloaded.
Definition: XrdFileCacheInfo.hh:360
Statistics of disk cache utilisation.
Definition: XrdFileCacheStats.hh:30
bool ReadV1(XrdOssDF *fp, const std::string &fname)
time_t AttachTime
Definition: XrdFileCacheInfo.hh:54
int GetVersion()
Get version.
Definition: XrdFileCacheInfo.hh:237
AStat()
read remote client
Definition: XrdFileCacheInfo.hh:60
static const char * m_infoExtension
Definition: XrdFileCacheInfo.hh:249
time_t m_creationTime
time the info file was created
Definition: XrdFileCacheInfo.hh:69
Store m_store
Definition: XrdFileCacheInfo.hh:261
unsigned char * m_buff_prefetch
prefetch statistics
Definition: XrdFileCacheInfo.hh:264
Store()
Definition: XrdFileCacheInfo.hh:73
bool m_hasPrefetchBuffer
constains current prefetch score
Definition: XrdFileCacheInfo.hh:262
static const size_t m_maxNumAccess
Definition: XrdFileCacheInfo.hh:252
long long GetBufferSize() const
Get prefetch buffer size.
Definition: XrdFileCacheInfo.hh:380
long long GetNDownloadedBytes() const
Get number of downloaded bytes.
Definition: XrdFileCacheInfo.hh:308
int GetSizeInBits() const
Get number of blocks represented in download-state bit-vector.
Definition: XrdFileCacheInfo.hh:321
XrdSysTrace * GetTrace() const
Definition: XrdFileCacheInfo.hh:254
time_t DetachTime
open time
Definition: XrdFileCacheInfo.hh:55
void SetBitSynced(int i)
Mark block as disk written.
Definition: XrdFileCacheInfo.hh:351
bool IsComplete() const
Get complete status.
Definition: XrdFileCacheInfo.hh:331
long long BytesDisk
close time
Definition: XrdFileCacheInfo.hh:56
Status of cached file. Can be read from and written into a binary file.
Definition: XrdFileCacheInfo.hh:48
void ResizeBits(int n)
Reserve buffer for fileSize/bufferSize bytes.
const Store & RefStoredData() const
Get stored data.
Definition: XrdFileCacheInfo.hh:242
long long GetFileSize() const
Get file size.
Definition: XrdFileCacheInfo.hh:326
bool TestPrefetchBit(int i) const
Test if block at the given index is prewritten.
Definition: XrdFileCacheInfo.hh:288
Definition: XrdSysTrace.hh:45
void WriteIOStatSingle(long long bytes_disk)
Write single open/close time for given bytes read from disk.
unsigned char * m_buff_synced
disk written state vector
Definition: XrdFileCacheInfo.hh:67
bool Write(XrdOssDF *fp, const std::string &fname="<unknown>")
void UpdateDownloadCompleteStatus()
Update complete status.
Definition: XrdFileCacheInfo.hh:346
Definition: XrdCksCalc.hh:39
size_t m_accessCnt
number of written AStat structs
Definition: XrdFileCacheInfo.hh:70
Definition: XrdFileCacheInfo.hh:52
bool IsAnythingEmptyInRng(int firstIdx, int lastIdx) const
Check download status in given block range.
Definition: XrdFileCacheInfo.hh:336
size_t GetAccessCnt()
Get number of accesses.
Definition: XrdFileCacheInfo.hh:232
XrdSysTrace * m_trace
Definition: XrdFileCacheInfo.hh:259
Info(XrdSysTrace *trace, bool prefetchBuffer=false)
Constructor.
char m_cksum[16]
cksum of downloaded information
Definition: XrdFileCacheInfo.hh:68
static size_t GetMaxNumAccess()
Definition: XrdFileCacheInfo.hh:256
void WriteIOStatDetach(Stats &s)
Write close time together with bytes missed, hits, and disk.
bool m_complete
cached
Definition: XrdFileCacheInfo.hh:267
std::vector< AStat > m_astats
number of last m_maxAcessCnts
Definition: XrdFileCacheInfo.hh:71
void SetFileSize(long long)
void SetBufferSize(long long)
int m_sizeInBits
cached
Definition: XrdFileCacheInfo.hh:266
void SetBitPrefetch(int i)
Mark block as written from prefetchxs.
Definition: XrdFileCacheInfo.hh:369
static const char * m_traceID
Definition: XrdFileCacheInfo.hh:250
int GetSizeInBytes() const
Get size of download-state bit-vector in bytes.
Definition: XrdFileCacheInfo.hh:313
long long BytesMissed
read from ram
Definition: XrdFileCacheInfo.hh:58
int GetNDownloadedBlocks() const
Get number of downloaded blocks.
Definition: XrdFileCacheInfo.hh:299
Definition: XrdOss.hh:59
void SetAllBitsSynced()
Mark all blocks as writte.
long long BytesRam
read from disk
Definition: XrdFileCacheInfo.hh:57
~Info()
Destructor.
void GetCksum(unsigned char *buff, char *digest)
Get md5 cksum.
Definition: XrdFileCacheInfo.hh:63
void DisableDownloadStatus()
Disable allocating, writing, and reading of downlaod status.
long long m_bufferSize
prefetch buffer size
Definition: XrdFileCacheInfo.hh:65
bool GetLatestDetachTime(time_t &t) const
Get latest detach time.
bool TestBit(int i) const
Test if block at the given index is downlaoded.
Definition: XrdFileCacheInfo.hh:279
unsigned char cfiBIT(int n) const
Definition: XrdFileCacheInfo.hh:270
void WriteIOStatAttach()
Write open time in the last entry of access statistics.
long long m_fileSize
number of file blocks
Definition: XrdFileCacheInfo.hh:66
static const int m_defaultVersion
Definition: XrdFileCacheInfo.hh:251