AliRoot Core  edcc906 (edcc906)
AliCDBGrid.cxx
Go to the documentation of this file.
1 /**************************************************************************
2  * Copyright(c) 1998-1999, ALICE Experiment at CERN, All rights reserved. *
3  * *
4  * Author: The ALICE Off-line Project. *
5  * Contributors are mentioned in the code where appropriate. *
6  * *
7  * Permission to use, copy, modify and distribute this software and its *
8  * documentation strictly for non-commercial purposes is hereby granted *
9  * without fee, provided that the above copyright notice appears in all *
10  * copies and that both the copyright notice and this permission notice *
11  * appear in the supporting documentation. The authors make no claims *
12  * about the suitability of this software for any purpose. It is *
13  * provided "as is" without express or implied warranty. *
14  **************************************************************************/
15 
17 // //
18 // AliCDBGrid //
19 // access class to a DataBase in an AliEn storage //
20 // //
22 
23 #include <cstdlib>
24 #include <TGrid.h>
25 #include <TGridResult.h>
26 #include <TFile.h>
27 #include <TKey.h>
28 #include <TROOT.h>
29 #include <TList.h>
30 #include <TObjArray.h>
31 #include <TObjString.h>
32 #include <TMath.h>
33 #include <TRegexp.h>
34 #include <stdexcept>
35 
36 #include "AliLog.h"
37 #include "AliCDBEntry.h"
38 #include "AliCDBGrid.h"
39 #include "AliCDBManager.h"
40 
41 
42 ClassImp(AliCDBGrid)
43 
44 Bool_t AliCDBGrid::fgForbidGRPOverride = kTRUE;
45 
46 //_____________________________________________________________________________
47 AliCDBGrid::AliCDBGrid(const char *gridUrl, const char *user, const char *dbFolder,
48  const char *se, const char* cacheFolder, Bool_t operateDisconnected,
49  Long64_t cacheSize, Long_t cleanupInterval) :
50  AliCDBStorage(),
51  fGridUrl(gridUrl),
52  fUser(user),
53  fDBFolder(dbFolder),
54  fSE(se),
55  fMirrorSEs(""),
56  fCacheFolder(cacheFolder),
57  fOperateDisconnected(operateDisconnected),
58  fCacheSize(cacheSize),
59  fCleanupInterval(cleanupInterval)
60 {
61 // constructor //
62 
63  // if the same Grid is alreay active, skip connection
64  if (!gGrid || fGridUrl != gGrid->GridUrl()
65  || (( fUser != "" ) && ( fUser != gGrid->GetUser() )) ) {
66  // connection to the Grid
67  AliInfo("Connection to the Grid...");
68  if(gGrid){
69  AliInfo(Form("gGrid = %p; fGridUrl = %s; gGrid->GridUrl() = %s",gGrid,fGridUrl.Data(), gGrid->GridUrl()));
70  AliInfo(Form("fUser = %s; gGrid->GetUser() = %s",fUser.Data(), gGrid->GetUser()));
71  }
72  TGrid::Connect(fGridUrl.Data(),fUser.Data());
73  }
74 
75  if(!gGrid) {
76  AliError("Connection failed!");
77  return;
78  }
79 
80  TString initDir(gGrid->Pwd(0));
81  if (fDBFolder[0] != '/') {
82  fDBFolder.Prepend(initDir);
83  }
84 
85  // check DBFolder: trying to cd to DBFolder; if it does not exist, create it
86  if(!gGrid->Cd(fDBFolder.Data(),0)){
87  AliDebug(2,Form("Creating new folder <%s> ...",fDBFolder.Data()));
88  TGridResult* res = gGrid->Command(Form("mkdir -p %s",fDBFolder.Data()));
89  TString result = res->GetKey(0,"__result__");
90  if(result == "0"){
91  AliFatal(Form("Cannot create folder <%s> !",fDBFolder.Data()));
92  return;
93  }
94  } else {
95  AliDebug(2,Form("Folder <%s> found",fDBFolder.Data()));
96  }
97 
98  // removes any '/' at the end of path, then append one '/'
99  while(fDBFolder.EndsWith("/")) fDBFolder.Remove(fDBFolder.Last('/'));
100  fDBFolder+="/";
101 
102  fType="alien";
103  fBaseFolder = fDBFolder;
104 
105  // Setting the cache
106 
107  // Check if local cache folder is already defined
108  TString origCache(TFile::GetCacheFileDir());
109  if(fCacheFolder.Length() > 0) {
110  if(origCache.Length() == 0) {
111  AliInfo(Form("Setting local cache to: %s", fCacheFolder.Data()));
112  } else if(fCacheFolder != origCache) {
113  AliWarning(Form("Local cache folder was already defined, changing it to: %s",
114  fCacheFolder.Data()));
115  }
116 
117  // default settings are: operateDisconnected=kTRUE, forceCacheread = kFALSE
118  if(!TFile::SetCacheFileDir(fCacheFolder.Data(), fOperateDisconnected)) {
119  AliError(Form("Could not set cache folder %s !", fCacheFolder.Data()));
120  fCacheFolder = "";
121  } else {
122  // reset fCacheFolder because the function may have
123  // slightly changed the folder name (e.g. '/' added)
124  fCacheFolder = TFile::GetCacheFileDir();
125  }
126 
127  // default settings are: cacheSize=1GB, cleanupInterval = 0
128  if(!TFile::ShrinkCacheFileDir(fCacheSize, fCleanupInterval)) {
129  AliError(Form("Could not set following values "
130  "to ShrinkCacheFileDir: cacheSize = %lld, cleanupInterval = %ld !",
131  fCacheSize, fCleanupInterval));
132  }
133  }
134 
135  // return to the initial directory
136  gGrid->Cd(initDir.Data(),0);
137 
138  fNretry = 3; // default
139  fInitRetrySeconds = 5; // default
140 }
141 
142 //_____________________________________________________________________________
144 // destructor
145  delete gGrid; gGrid=0;
146 
147 }
148 
149 //_____________________________________________________________________________
150 Bool_t AliCDBGrid::FilenameToId(TString& filename, AliCDBId& id) {
151 // build AliCDBId from full path filename (fDBFolder/path/Run#x_#y_v#z_s0.root)
152 
153  if(filename.Contains(fDBFolder)){
154  filename = filename(fDBFolder.Length(),filename.Length()-fDBFolder.Length());
155  }
156 
157  TString idPath = filename(0,filename.Last('/'));
158  id.SetPath(idPath);
159  if(!id.IsValid()) return kFALSE;
160 
161  filename=filename(idPath.Length()+1,filename.Length()-idPath.Length());
162 
163  Ssiz_t mSize;
164  // valid filename: Run#firstRun_#lastRun_v#version_s0.root
165  TRegexp keyPattern("^Run[0-9]+_[0-9]+_v[0-9]+_s0.root$");
166  keyPattern.Index(filename, &mSize);
167  if (!mSize) {
168 
169  // TODO backward compatibility ... maybe remove later!
170  Ssiz_t oldmSize;
171  TRegexp oldKeyPattern("^Run[0-9]+_[0-9]+_v[0-9]+.root$");
172  oldKeyPattern.Index(filename, &oldmSize);
173  if(!oldmSize) {
174  AliDebug(2,Form("Bad filename <%s>.", filename.Data()));
175  return kFALSE;
176  } else {
177  AliDebug(2,Form("Old filename format <%s>.", filename.Data()));
178  id.SetSubVersion(-11); // TODO trick to ensure backward compatibility
179  }
180 
181  } else {
182  id.SetSubVersion(-1); // TODO trick to ensure backward compatibility
183  }
184 
185  filename.Resize(filename.Length() - sizeof(".root") + 1);
186 
187  TObjArray* strArray = (TObjArray*) filename.Tokenize("_");
188 
189  TString firstRunString(((TObjString*) strArray->At(0))->GetString());
190  id.SetFirstRun(atoi(firstRunString.Data() + 3));
191  id.SetLastRun(atoi(((TObjString*) strArray->At(1))->GetString()));
192 
193  TString verString(((TObjString*) strArray->At(2))->GetString());
194  id.SetVersion(atoi(verString.Data() + 1));
195 
196  delete strArray;
197 
198  return kTRUE;
199 }
200 
201 //_____________________________________________________________________________
202 Bool_t AliCDBGrid::IdToFilename(const AliCDBId& id, TString& filename) const {
203 // build file name from AliCDBId (path, run range, version) and fDBFolder
204 
205  if (!id.GetAliCDBRunRange().IsValid()) {
206  AliDebug(2,Form("Invalid run range <%d, %d>.",
207  id.GetFirstRun(), id.GetLastRun()));
208  return kFALSE;
209  }
210 
211  if (id.GetVersion() < 0) {
212  AliDebug(2,Form("Invalid version <%d>.", id.GetVersion()));
213  return kFALSE;
214  }
215 
216  filename = Form("Run%d_%d_v%d",
217  id.GetFirstRun(),
218  id.GetLastRun(),
219  id.GetVersion());
220 
221  if (id.GetSubVersion() != -11) filename += "_s0"; // TODO to ensure backward compatibility
222  filename += ".root";
223 
224  filename.Prepend(fDBFolder + id.GetPath() + '/');
225 
226  return kTRUE;
227 }
228 
229 //_____________________________________________________________________________
230 void AliCDBGrid::SetRetry(Int_t nretry, Int_t initsec) {
231 
232  // Function to set the exponential retry for putting entries in the OCDB
233 
234  AliWarning("WARNING!!! You are changing the exponential retry times and delay: this function should be used by experts!");
235  fNretry = nretry;
236  fInitRetrySeconds = initsec;
237  AliDebug(2,Form("fNretry = %d, fInitRetrySeconds = %d", fNretry, fInitRetrySeconds));
238 }
239 
240 
241 //_____________________________________________________________________________
243 // prepare id (version) of the object that will be stored (called by PutEntry)
244 
245  TString initDir(gGrid->Pwd(0));
246 
247  TString dirName(fDBFolder);
248 
249  Bool_t dirExist=kFALSE;
250 
251 
252 
253  // go to the path; if directory does not exist, create it
254  for(int i=0;i<3;i++){
255  //TString cmd("find -d ");
256  //cmd += Form("%s ",dirName);
257  //cmd +=
258  //gGrid->Command(cmd.Data());
259  dirName+=Form("%s/",id.GetPathLevel(i).Data());
260  dirExist=gGrid->Cd(dirName,0);
261  if (!dirExist) {
262  AliDebug(2,Form("Creating new folder <%s> ...",dirName.Data()));
263  if(!gGrid->Mkdir(dirName,"",0)){
264  AliError(Form("Cannot create directory <%s> !",dirName.Data()));
265  gGrid->Cd(initDir.Data());
266  return kFALSE;
267  }
268 
269  // if folders are new add tags to them
270  if(i == 1) {
271  // TODO Currently disabled
272  // add short lived tag!
273  // AliInfo("Tagging level 1 folder with \"ShortLived\" tag");
274  // if(!AddTag(dirName,"ShortLived_try")){
275  // AliError(Form("Could not tag folder %s !", dirName.Data()));
276  // if(!gGrid->Rmdir(dirName.Data())){
277  // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
278  // }
279  // return 0;
280  //}
281 
282  } else if(i == 2) {
283  AliDebug(2,"Tagging level 2 folder with \"CDB\" and \"CDB_MD\" tag");
284  if(!AddTag(dirName,"CDB")){
285  AliError(Form("Could not tag folder %s !", dirName.Data()));
286  if(!gGrid->Rmdir(dirName.Data())){
287  AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
288  }
289  return 0;
290  }
291  if(!AddTag(dirName,"CDB_MD")){
292  AliError(Form("Could not tag folder %s !", dirName.Data()));
293  if(!gGrid->Rmdir(dirName.Data())){
294  AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
295  }
296  return 0;
297  }
298 
299  // TODO Currently disabled
300  // add short lived tag!
301  // TString path=id.GetPath();
302  // if(AliCDBManager::Instance()->IsShortLived(path.Data())) {
303  // AliInfo(Form("Tagging %s as short lived", dirName.Data()));
304  // if(!TagShortLived(dirName, kTRUE)){
305  // AliError(Form("Could not tag folder %s !", dirName.Data()));
306  // if(!gGrid->Rmdir(dirName.Data())){
307  // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
308  // }
309  // return 0;
310  // }
311  // } else {
312  // AliInfo(Form("Tagging %s as long lived", dirName.Data()));
313  // if(!TagShortLived(dirName, kFALSE)){
314  // AliError(Form("Could not tag folder %s !", dirName.Data()));
315  // if(!gGrid->Rmdir(dirName.Data())){
316  // AliError(Form("Unexpected: could not remove %s directory!", dirName.Data()));
317  // }
318  // return 0;
319  // }
320  // }
321  }
322  }
323  }
324  gGrid->Cd(initDir,0);
325 
326  TString filename;
327  AliCDBId anId; // the id got from filename
328  AliCDBRunRange lastRunRange(-1,-1); // highest runRange found
329  Int_t lastVersion=0; // highest version found
330 
331  TGridResult *res = gGrid->Ls(dirName);
332 
333  //loop on the files in the directory, look for highest version
334  for(int i=0; i < res->GetEntries(); i++){
335  filename=res->GetFileNamePath(i);
336  if (!FilenameToId(filename, anId)) continue;
337  if (anId.GetAliCDBRunRange().Overlaps(id.GetAliCDBRunRange()) && anId.GetVersion() > lastVersion) {
338  lastVersion = anId.GetVersion();
339  lastRunRange = anId.GetAliCDBRunRange();
340  }
341 
342  }
343  delete res;
344 
345  // GRP entries with explicitly set version escape default incremental versioning
346  if(id.GetPath().Contains("GRP") && id.HasVersion() && lastVersion!=0)
347  {
348  if (GetForbidGRPOverride()) {
349  AliDebug(5,Form("Entry %s won't be put in the destination OCDB", id.ToString().Data()));
350  return kFALSE;
351  }
352  else {
353  AliInfo("ATTENTION: Interdiction to override GRP* objects was explicitly disabled");
354  }
355  }
356 
357  id.SetVersion(lastVersion + 1);
358  id.SetSubVersion(0);
359 
360  TString lastStorage = id.GetLastStorage();
361  if(lastStorage.Contains(TString("new"), TString::kIgnoreCase) && id.GetVersion() > 1 ){
362  AliDebug(2, Form("A NEW object is being stored with version %d",
363  id.GetVersion()));
364  AliDebug(2, Form("and it will hide previously stored object with version %d!",
365  id.GetVersion()-1));
366  }
367 
368  if(!lastRunRange.IsAnyRange() && !(lastRunRange.IsEqual(&id.GetAliCDBRunRange())))
369  AliWarning(Form("Run range modified w.r.t. previous version (Run%d_%d_v%d)",
370  lastRunRange.GetFirstRun(), lastRunRange.GetLastRun(), id.GetVersion()));
371 
372  return kTRUE;
373 }
374 
375 //_____________________________________________________________________________
376 AliCDBId* AliCDBGrid::GetId(const TObjArray& validFileIds, const AliCDBId& query) {
377 // look for the Id that matches query's requests (highest or exact version)
378 
379  if(validFileIds.GetEntriesFast() < 1)
380  return NULL;
381 
382  TIter iter(&validFileIds);
383 
384  AliCDBId *anIdPtr=0;
385  AliCDBId* result=0;
386 
387  while((anIdPtr = dynamic_cast<AliCDBId*> (iter.Next()))){
388  if(anIdPtr->GetPath() != query.GetPath()) continue;
389 
390  //if(!CheckVersion(query, anIdPtr, result)) return NULL;
391 
392  if (!query.HasVersion()){ // look for highest version
393  if(result && result->GetVersion() > anIdPtr->GetVersion()) continue;
394  if(result && result->GetVersion() == anIdPtr->GetVersion()) {
395  AliError(Form("More than one object valid for run %d, version %d!",
396  query.GetFirstRun(), anIdPtr->GetVersion()));
397  return NULL;
398  }
399  result = new AliCDBId(*anIdPtr);
400  } else { // look for specified version
401  if(query.GetVersion() != anIdPtr->GetVersion()) continue;
402  if(result && result->GetVersion() == anIdPtr->GetVersion()){
403  AliError(Form("More than one object valid for run %d, version %d!",
404  query.GetFirstRun(), anIdPtr->GetVersion()));
405  return NULL;
406  }
407  result = new AliCDBId(*anIdPtr);
408  }
409 
410  }
411 
412  return result;
413 }
414 
415 //_____________________________________________________________________________
417 // get AliCDBId from the database
418 // User must delete returned object
419 
420  AliCDBId* dataId=0;
421 
422  AliCDBId selectedId(queryId);
423  if (!selectedId.HasVersion()) {
424  // if version is not specified, first check the selection criteria list
425  GetSelection(&selectedId);
426  }
427 
428  TObjArray validFileIds;
429  validFileIds.SetOwner(1);
430 
431  // look for file matching query requests (path, runRange, version)
432  if(selectedId.GetFirstRun() == fRun && fPathFilter.Comprises(selectedId.GetAliCDBPath()) &&
433  fVersion == selectedId.GetVersion() && !fMetaDataFilter){
434  // look into list of valid files previously loaded with AliCDBStorage::FillValidFileIds()
435  AliDebug(2, Form("List of files valid for run %d was loaded. Looking there for fileids valid for path %s!",
436  selectedId.GetFirstRun(), selectedId.GetPath().Data()));
437  dataId = GetId(fValidFileIds, selectedId);
438 
439  } else {
440  // List of files valid for reqested run was not loaded. Looking directly into CDB
441  AliDebug(2, Form("List of files valid for run %d and version %d was not loaded. Looking directly into CDB for fileids valid for path %s!",
442  selectedId.GetFirstRun(), selectedId.GetVersion(), selectedId.GetPath().Data()));
443 
444  TString filter;
445  MakeQueryFilter(selectedId.GetFirstRun(), selectedId.GetLastRun(), 0, filter);
446 
447  TString pattern = ".root";
448  TString optionQuery = "-y -m";
449  if(selectedId.GetVersion() >= 0) {
450  pattern.Prepend(Form("_v%d_s0",selectedId.GetVersion()));
451  optionQuery = "";
452  }
453 
454  TString folderCopy(Form("%s%s/Run",fDBFolder.Data(),selectedId.GetPath().Data()));
455 
456  if (optionQuery.Contains("-y")){
457  AliInfo("Only latest version will be returned");
458  }
459 
460  AliDebug(2,Form("** fDBFolder = %s, pattern = %s, filter = %s",folderCopy.Data(), pattern.Data(), filter.Data()));
461  TGridResult *res = gGrid->Query(folderCopy, pattern, filter, optionQuery.Data());
462  if (res) {
463  for(int i=0; i<res->GetEntries(); i++){
464  AliCDBId *validFileId = new AliCDBId();
465  TString filename = res->GetKey(i, "lfn");
466  if(filename == "") continue;
467  if(FilenameToId(filename, *validFileId))
468  validFileIds.AddLast(validFileId);
469  }
470  delete res;
471  }else{
472  return 0; // this should be only in case of file catalogue glitch
473  }
474 
475  dataId = GetId(validFileIds, selectedId);
476  }
477 
478  return dataId;
479 }
480 
481 //_____________________________________________________________________________
483 // get AliCDBEntry from the database
484 
485  AliCDBId* dataId = GetEntryId(queryId);
486 
487  TString errMessage(TString::Format("No valid CDB object found! request was: %s", queryId.ToString().Data()));
488 
489  if (!dataId){
490  AliError(Form("No file found matching this id!"));
491  throw std::runtime_error(errMessage.Data());
492  return NULL;
493  }
494 
495  TString filename;
496  if (!IdToFilename(*dataId, filename)) {
497  AliDebug(2,Form("Bad data ID encountered! Subnormal error!"));
498  delete dataId;
499  throw std::runtime_error(errMessage.Data());
500  return NULL;
501  }
502 
503  AliCDBEntry* anEntry = GetEntryFromFile(filename, dataId);
504 
505  delete dataId;
506  if(!anEntry) {
507  AliError(Form("Error getting CDB Entry from file"));
508  throw std::runtime_error(errMessage.Data());
509  return NULL;
510  }
511 
512  return anEntry;
513 }
514 
515 //_____________________________________________________________________________
516 AliCDBEntry* AliCDBGrid::GetEntryFromFile(TString& filename, AliCDBId* dataId){
517 // Get AliCBEntry object from file "filename"
518 
519  AliDebug(2,Form("Opening file: %s",filename.Data()));
520 
521  filename.Prepend("/alien");
522 
523  // if option="CACHEREAD" TFile will use the local caching facility!
524  TString option="READ";
525  if(fCacheFolder != ""){
526 
527  // Check if local cache folder was changed in the meanwhile
528  TString origCache(TFile::GetCacheFileDir());
529  if(fCacheFolder != origCache) {
530  AliWarning(Form("Local cache folder has been overwritten!! fCacheFolder = %s origCache = %s",
531  fCacheFolder.Data(), origCache.Data()));
532  TFile::SetCacheFileDir(fCacheFolder.Data(), fOperateDisconnected);
533  TFile::ShrinkCacheFileDir(fCacheSize, fCleanupInterval);
534  }
535 
536  option.Prepend("CACHE");
537  }
538 
539  AliDebug(2, Form("Option: %s", option.Data()));
540 
541  TFile *file = TFile::Open(filename, option);
542  if (!file) {
543  AliDebug(2,Form("Can't open file <%s>!", filename.Data()));
544  return NULL;
545  }
546 
547  // get the only AliCDBEntry object from the file
548  // the object in the file is an AliCDBEntry entry named "AliCDBEntry"
549 
550  AliCDBEntry* anEntry = dynamic_cast<AliCDBEntry*> (file->Get("AliCDBEntry"));
551 
552  if (!anEntry) {
553  AliDebug(2,Form("Bad storage data: file does not contain an AliCDBEntry object!"));
554  file->Close();
555  return NULL;
556  }
557 
558  // The object's Id is not reset during storage
559  // If object's Id runRange or version do not match with filename,
560  // it means that someone renamed file by hand. In this case a warning msg is issued.
561 
562  if(anEntry){
563  AliCDBId entryId = anEntry->GetId();
564  Int_t tmpSubVersion = dataId->GetSubVersion();
565  dataId->SetSubVersion(entryId.GetSubVersion()); // otherwise filename and id may mismatch
566  if(!entryId.IsEqual(dataId)){
567  AliWarning(Form("Mismatch between file name and object's Id!"));
568  AliWarning(Form("File name: %s", dataId->ToString().Data()));
569  AliWarning(Form("Object's Id: %s", entryId.ToString().Data()));
570  }
571  dataId->SetSubVersion(tmpSubVersion);
572  }
573 
574  anEntry->SetLastStorage("grid");
575 
576  // Check whether entry contains a TTree. In case load the tree in memory!
577  LoadTreeFromFile(anEntry);
578 
579  // close file, return retieved entry
580  file->Close(); delete file; file=0;
581 
582  return anEntry;
583 }
584 
585 //_____________________________________________________________________________
586 TList* AliCDBGrid::GetEntries(const AliCDBId& queryId) {
587 // multiple request (AliCDBStorage::GetAll)
588 
589  TList* result = new TList();
590  result->SetOwner();
591 
592  TObjArray validFileIds;
593  validFileIds.SetOwner(1);
594 
595  Bool_t alreadyLoaded = kFALSE;
596 
597  // look for file matching query requests (path, runRange)
598  if(queryId.GetFirstRun() == fRun &&
600  // look into list of valid files previously loaded with AliCDBStorage::FillValidFileIds()
601  AliDebug(2,Form("List of files valid for run %d and for path %s was loaded. Looking there!",
602  queryId.GetFirstRun(), queryId.GetPath().Data()));
603 
604  alreadyLoaded = kTRUE;
605 
606  } else {
607  // List of files valid for reqested run was not loaded. Looking directly into CDB
608  AliDebug(2,Form("List of files valid for run %d and for path %s was not loaded. Looking directly into CDB!",
609  queryId.GetFirstRun(), queryId.GetPath().Data()));
610 
611  TString filter;
612  MakeQueryFilter(queryId.GetFirstRun(), queryId.GetLastRun(), 0, filter);
613 
614  TString path = queryId.GetPath();
615 
616  TString pattern = "Run*.root";
617  TString optionQuery = "-y";
618 
619  TString addFolder = "";
620  if (!path.Contains("*")){
621  if (!path.BeginsWith("/")) addFolder += "/";
622  addFolder += path;
623  }
624  else{
625  if (path.BeginsWith("/")) path.Remove(0,1);
626  if (path.EndsWith("/")) path.Remove(path.Length()-1,1);
627  TObjArray* tokenArr = path.Tokenize("/");
628  if (tokenArr->GetEntries() != 3) {
629  AliError("Not a 3 level path! Keeping old query...");
630  pattern.Prepend(path+"/");
631  }
632  else{
633  TString str0 = ((TObjString*)tokenArr->At(0))->String();
634  TString str1 = ((TObjString*)tokenArr->At(1))->String();
635  TString str2 = ((TObjString*)tokenArr->At(2))->String();
636  if (str0 != "*" && str1 != "*" && str2 == "*"){
637  // e.g. "ITS/Calib/*"
638  addFolder = "/"+str0+"/"+str1;
639  }
640  else if (str0 != "*" && str1 == "*" && str2 == "*"){
641  // e.g. "ITS/*/*"
642  addFolder = "/"+str0;
643  }
644  else if (str0 == "*" && str1 == "*" && str2 == "*"){
645  // e.g. "*/*/*"
646  // do nothing: addFolder is already an empty string;
647  }
648  else{
649  // e.g. "ITS/*/RecoParam"
650  pattern.Prepend(path+"/");
651  }
652  }
653  delete tokenArr; tokenArr=0;
654  }
655 
656  TString folderCopy(Form("%s%s",fDBFolder.Data(),addFolder.Data()));
657 
658  AliDebug(2,Form("fDBFolder = %s, pattern = %s, filter = %s",folderCopy.Data(), pattern.Data(), filter.Data()));
659 
660  TGridResult *res = gGrid->Query(folderCopy, pattern, filter, optionQuery.Data());
661 
662  if (!res) {
663  AliError("Grid query failed");
664  return 0;
665  }
666 
667  for(int i=0; i<res->GetEntries(); i++){
668  AliCDBId *validFileId = new AliCDBId();
669  TString filename = res->GetKey(i, "lfn");
670  if(filename == "") continue;
671  if(FilenameToId(filename, *validFileId))
672  validFileIds.AddLast(validFileId);
673  }
674  delete res;
675  }
676 
677  TIter *iter=0;
678  if(alreadyLoaded){
679  iter = new TIter(&fValidFileIds);
680  } else {
681  iter = new TIter(&validFileIds);
682  }
683 
684  TObjArray selectedIds;
685  selectedIds.SetOwner(1);
686 
687  // loop on list of valid Ids to select the right version to get.
688  // According to query and to the selection criteria list, version can be the highest or exact
689  AliCDBPath pathCopy;
690  AliCDBId* anIdPtr=0;
691  AliCDBId* dataId=0;
692  AliCDBPath queryPath = queryId.GetAliCDBPath();
693  while((anIdPtr = dynamic_cast<AliCDBId*> (iter->Next()))){
694  AliCDBPath thisCDBPath = anIdPtr->GetAliCDBPath();
695  if(!(queryPath.Comprises(thisCDBPath)) || pathCopy.GetPath() == thisCDBPath.GetPath()) continue;
696  pathCopy = thisCDBPath;
697 
698  // check the selection criteria list for this query
699  AliCDBId thisId(*anIdPtr);
700  thisId.SetVersion(queryId.GetVersion());
701  if(!thisId.HasVersion()) GetSelection(&thisId);
702 
703  if(alreadyLoaded){
704  dataId = GetId(fValidFileIds, thisId);
705  } else {
706  dataId = GetId(validFileIds, thisId);
707  }
708  if(dataId) selectedIds.Add(dataId);
709  }
710 
711  delete iter; iter=0;
712 
713  // selectedIds contains the Ids of the files matching all requests of query!
714  // All the objects are now ready to be retrieved
715  iter = new TIter(&selectedIds);
716  while((anIdPtr = dynamic_cast<AliCDBId*> (iter->Next()))){
717  TString filename;
718  if (!IdToFilename(*anIdPtr, filename)) {
719  AliDebug(2,Form("Bad data ID encountered! Subnormal error!"));
720  continue;
721  }
722 
723  AliCDBEntry* anEntry = GetEntryFromFile(filename, anIdPtr);
724 
725  if(anEntry) result->Add(anEntry);
726 
727  }
728  delete iter; iter=0;
729 
730  return result;
731 }
732 
733 //_____________________________________________________________________________
734 Bool_t AliCDBGrid::PutEntry(AliCDBEntry* entry, const char* mirrors) {
735 // put an AliCDBEntry object into the database
736 
737  AliCDBId& id = entry->GetId();
738 
739  // set version for the entry to be stored
740  if (!PrepareId(id)) return kFALSE;
741 
742  // build filename from entry's id
743  TString filename;
744  if (!IdToFilename(id, filename)) {
745  AliError("Bad ID encountered, cannot make a file name out of it!");
746  return kFALSE;
747  }
748 
749  TString folderToTag = Form("%s%s",
750  fDBFolder.Data(),
751  id.GetPath().Data());
752 
753  TDirectory* saveDir = gDirectory;
754 
755  TString fullFilename = Form("/alien%s", filename.Data());
756  TString seMirrors(mirrors);
757  if(seMirrors.IsNull() || seMirrors.IsWhitespace()) seMirrors=GetMirrorSEs();
758  // specify SE to filename
759  // if a list of SEs was passed to this method or set via SetMirrorSEs, set the first as SE for opening the file.
760  // The other SEs will be used in cascade in case of failure in opening the file.
761  // The remaining SEs will be used to create replicas.
762  TObjArray *arraySEs = seMirrors.Tokenize(',');
763  Int_t nSEs = arraySEs->GetEntries();
764  Int_t remainingSEs = 1;
765  if(nSEs == 0){
766  if (fSE != "default") fullFilename += Form("?se=%s",fSE.Data());
767  }else{
768  remainingSEs = nSEs;
769  }
770 
771  // open file
772  TFile *file = 0;
773  TFile *reopenedFile = 0;
774  AliDebug(2, Form("fNretry = %d, fInitRetrySeconds = %d",fNretry,fInitRetrySeconds));
775  TString targetSE("");
776 
777  Bool_t result = kFALSE;
778  Bool_t reOpenResult = kFALSE;
779  Int_t reOpenAttempts=0;
780  while( !reOpenResult && reOpenAttempts<2 ) { //loop to check the file after closing it, to catch the unlikely but possible case when the file
781  // is cleaned up by alien just before closing as a consequence of a network disconnection while writing
782 
783  while( !file && remainingSEs>0 ) {
784  if(nSEs!=0){
785  TObjString *target = (TObjString*) arraySEs->At(nSEs-remainingSEs);
786  targetSE=target->String();
787  if ( !(targetSE.BeginsWith("ALICE::") && targetSE.CountChar(':')==4) ) {
788  AliError( Form("\"%s\" is an invalid storage element identifier.",targetSE.Data()) );
789  continue;
790  }
791  if ( fullFilename.Contains('?')) fullFilename.Remove(fullFilename.Last('?') );
792  fullFilename += Form("?se=%s",targetSE.Data());
793  }
794  Int_t remainingAttempts=fNretry;
795  Int_t nsleep = fInitRetrySeconds; // number of seconds between attempts. We let it increase exponentially
796  AliDebug(2, Form("Uploading file into SE #%d: %s",nSEs-remainingSEs+1,targetSE.Data()));
797  while(remainingAttempts > 0) {
798  AliDebug(2, Form("Uploading file into OCDB at %s - Attempt #%d",targetSE.Data(),fNretry-remainingAttempts+1));
799  remainingAttempts--;
800  file = TFile::Open(fullFilename,"CREATE");
801  if(!file || !file->IsWritable()){
802  if(file) { // file is not writable
803  file->Close(); delete file; file=0;
804  }
805  TString message(TString::Format("Attempt %d failed.",fNretry-remainingAttempts));
806  if(remainingAttempts>0) {
807  message += " Sleeping for "; message += nsleep; message += " seconds";
808  }else{
809  if(remainingSEs>0) message += " Trying to upload at next SE";
810  }
811  AliDebug(2, message.Data());
812  if(remainingAttempts>0) sleep(nsleep);
813  }else{
814  remainingAttempts=0;
815  }
816  nsleep*=fInitRetrySeconds;
817  }
818  remainingSEs--;
819  }
820  if(!file){
821  AliError(Form("All %d attempts have failed on all %d SEs. Returning...",fNretry,nSEs));
822  return kFALSE;
823  }
824 
825  file->cd();
826 
827  //SetTreeToFile(entry, file);
828  entry->SetVersion(id.GetVersion());
829 
830  // write object (key name: "AliCDBEntry")
831  result = (file->WriteTObject(entry, "AliCDBEntry") != 0);
832  file->Close();
833  if (!result) {
834  AliError(Form("Can't write entry to file <%s>!", filename.Data()));
835  } else {
836  AliDebug(2, Form("Reopening file %s for checking its correctness",fullFilename.Data()));
837  reopenedFile = TFile::Open(fullFilename.Data(),"READ");
838  if(!reopenedFile){
839  reOpenResult = kFALSE;
840  AliInfo(Form("The file %s was closed successfully but cannot be reopened. Trying now to regenerate it (regeneration attempt number %d)",
841  fullFilename.Data(),++reOpenAttempts));
842  delete file; file=0;
843  AliDebug(2, Form("Removing file %s", filename.Data()));
844  if(!gGrid->Rm(filename.Data()))
845  AliError("Can't delete file!");
846  remainingSEs++;
847  }else{
848  reOpenResult = kTRUE;
849  if ( ! AliCDBManager::Instance()->IsOCDBUploadMode() ) {
850  reopenedFile->Close();
851  delete reopenedFile; reopenedFile=0;
852  }
853  }
854  }
855  }
856 
857  if (saveDir) saveDir->cd(); else gROOT->cd();
858  delete file; file=0;
859 
860  if(result && reOpenResult) {
861 
862  if(!TagFileId(filename, &id)){
863  AliInfo(Form("CDB tagging failed. Deleting file %s!",filename.Data()));
864  if(!gGrid->Rm(filename.Data()))
865  AliError("Can't delete file!");
866  return kFALSE;
867  }
868 
869  TagFileMetaData(filename, entry->GetMetaData());
870  }else{
871  AliError("The file could not be opened or the object could not be written.");
872  if(!gGrid->Rm(filename.Data()))
873  AliError("Can't delete file!");
874  return kFALSE;
875  }
876 
877  AliInfo(Form("CDB object stored into file %s", filename.Data()));
878  if(nSEs==0)
879  AliInfo(Form("Storage Element: %s", fSE.Data()));
880  else
881  AliInfo(Form("Storage Element: %s", targetSE.Data()));
882 
883  //In case of other SEs specified by the user, mirror the file to the remaining SEs
884  for(Int_t i=0; i<nSEs; i++){
885  if(i==nSEs-remainingSEs-1) continue; // skip mirroring to the SE where the file was saved
886  TString mirrorCmd("mirror ");
887  mirrorCmd += filename;
888  mirrorCmd += " ";
889  TObjString *target = (TObjString*) arraySEs->At(i);
890  TString mirrorSE(target->String());
891  mirrorCmd += mirrorSE;
892  AliDebug(5,Form("mirror command: \"%s\"",mirrorCmd.Data()));
893  AliInfo(Form("Mirroring to storage element: %s", mirrorSE.Data()));
894  gGrid->Command(mirrorCmd.Data());
895  }
896  arraySEs->Delete(); arraySEs=0;
897 
898  if ( AliCDBManager::Instance()->IsOCDBUploadMode() ) { // if uploading to OCDBs, add to cvmfs too
899  if ( !filename.BeginsWith("/alice/data") && !filename.BeginsWith("/alice/simulation/2008/v4-15-Release") ) {
900  AliError ( Form ( "Cannot upload to CVMFS OCDBs a non official CDB object: \"%s\"!", filename.Data() ) );
901  } else {
902  if ( !PutInCvmfs( filename, reopenedFile) )
903  AliError( Form( "Could not upload AliEn file \"%s\" to CVMFS OCDB!", filename.Data() ) );
904  }
905  reopenedFile->Close();
906  delete reopenedFile; reopenedFile=0;
907  }
908 
909  return kTRUE;
910 }
911 
912 //_____________________________________________________________________________
913 Bool_t AliCDBGrid::PutInCvmfs( TString& filename, TFile* cdbFile ) const
914 {
915  // Add the CDB object to cvmfs OCDB
916 
917  TString cvmfsFilename( filename );
918  // cvmfsFilename.Remove(TString::kTrailing, '/');
919  TString basename = ( cvmfsFilename( cvmfsFilename.Last( '/' ) + 1, cvmfsFilename.Length() ) );
920  TString cvmfsDirname = cvmfsFilename.Remove( cvmfsFilename.Last( '/' ), cvmfsFilename.Length() );
921  TRegexp threeLevelsRE( "[^/]+/[^/]+/[^/]+$" );
922  TString threeLevels = cvmfsDirname( threeLevelsRE );
923 
924  TRegexp re_RawFolder("^/alice/data/20[0-9]+/OCDB");
925  TRegexp re_MCFolder("^/alice/simulation/2008/v4-15-Release");
926  TString rawFolder = cvmfsDirname(re_RawFolder);
927  TString mcFolder = cvmfsDirname(re_MCFolder);
928  if ( !rawFolder.IsNull() ) {
929  cvmfsDirname.Replace(0, 6, "/cvmfs/alice-ocdb.cern.ch/calibration");
930  } else if ( !mcFolder.IsNull() ){
931  cvmfsDirname.Replace(0,36,"/cvmfs/alice-ocdb.cern.ch/calibration/MC");
932  } else {
933  AliError(Form("OCDB folder set for an invalid OCDB storage:\n %s", cvmfsDirname.Data()));
934  return kFALSE;
935  }
936  // now cvmfsDirname is the full dirname in cvmfs
937  AliDebug(3, Form("Publishing \"%s\" in \"%s\"", basename.Data(), cvmfsDirname.Data()));
938 
939  // Tar the file with the right prefix path. Include the directory structure in the tarball
940  // to cover the case of a containing directory being new in cvmfs, plus a container directory
941  // to avoid clashing with stuff present in the local directory
942  TString firstLevel(threeLevels(0, threeLevels.First('/')));
943  TString tempDir("tmpToCvmfsOcdbs");
944  gSystem->Exec( Form("rm -r %s > /dev/null 2>&1", tempDir.Data()) ); //to be sure not to publish other stuff in cvmfs
945  Int_t result = gSystem->Exec( Form("mkdir -p %s/%s", tempDir.Data(), threeLevels.Data()) );
946  if ( result != 0 ) {
947  AliError ( Form ( "Could not create the directory \"%s/%s\"", tempDir.Data(), threeLevels.Data() ) );
948  return kFALSE;
949  }
950  cdbFile->Cp(Form("%s/%s/%s", tempDir.Data(), threeLevels.Data(), basename.Data() ));
951  TString tarFileName("cdbObjectToAdd.tar.gz");
952  TString cvmfsBaseFolder(cvmfsDirname(0, cvmfsDirname.Last('/')));
953  cvmfsBaseFolder = cvmfsBaseFolder(0, cvmfsBaseFolder.Last('/'));
954  cvmfsBaseFolder = cvmfsBaseFolder(0, cvmfsBaseFolder.Last('/'));
955  // tarCommand should be e.g.: tar --transform 's,^,/cvmfs/alice-ocdb.cern.ch/calibration/data/2010/OCDB/,S' -cvzf objecttoadd.tar.gz basename
956  result = gSystem->Exec ( Form( "tar --transform 's,^%s,%s,S' -cvzf %s %s", tempDir.Data(), cvmfsBaseFolder.Data(), tarFileName.Data(), tempDir.Data() ) );
957  if ( result != 0 ) {
958  AliError ( Form ( "Could not create the tarball for the object \"%s\"", filename.Data() ) );
959  return kFALSE;
960  }
961 
962  // Copy the file to cvmfs (requires to have the executable in the path and access to the server)
963  result = gSystem->Exec( Form( "ocdb-cvmfs %s", tarFileName.Data() ) );
964  if ( result != 0 ) {
965  AliError ( Form ( "Could not execute \"ocdb-cvmfs %s\"", filename.Data() ) );
966  return kFALSE;
967  }
968 
969  // Remove the local file and the tar-file
970  gSystem->Exec( Form( "rm -r %s", tempDir.Data() ) );
971  gSystem->Exec( Form( "rm %s", tarFileName.Data() ) );
972 
973  return kTRUE;
974 }
975 
976 //_____________________________________________________________________________
977 Bool_t AliCDBGrid::AddTag(TString& folderToTag, const char* tagname){
978 // add "tagname" tag (CDB or CDB_MD) to folder where object will be stored
979 
980  Bool_t result = kTRUE;
981  AliDebug(2, Form("adding %s tag to folder %s", tagname, folderToTag.Data()));
982  TString addTag = Form("addTag %s %s", folderToTag.Data(), tagname);
983  TGridResult *gridres = gGrid->Command(addTag.Data());
984  const char* resCode = gridres->GetKey(0,"__result__"); // '1' if success
985  if(resCode[0] != '1') {
986  AliError(Form("Couldn't add %s tags to folder %s !",
987  tagname, folderToTag.Data()));
988  result = kFALSE;
989  }
990  delete gridres;
991  return result;
992 }
993 
994 //_____________________________________________________________________________
995 Bool_t AliCDBGrid::TagFileId(TString& filename, const AliCDBId* id){
996 // tag stored object in CDB table using object Id's parameters
997 
998 
999  TString dirname(filename);
1000  Int_t dirNumber = gGrid->Mkdir(dirname.Remove(dirname.Last('/')),"-d");
1001 
1002  TString addTagValue1 = Form("addTagValue %s CDB ", filename.Data());
1003  TString addTagValue2 = Form("first_run=%d last_run=%d version=%d ",
1004  id->GetFirstRun(),
1005  id->GetLastRun(),
1006  id->GetVersion());
1007  TString addTagValue3 = Form("path_level_0=\"%s\" path_level_1=\"%s\" path_level_2=\"%s\" ",
1008  id->GetPathLevel(0).Data(),
1009  id->GetPathLevel(1).Data(),
1010  id->GetPathLevel(2).Data());
1011  //TString addTagValue4 = Form("version_path=\"%s\" dir_number=%d",Form("%d_%s",id->GetVersion(),filename.Data()),dirNumber);
1012  TString addTagValue4 = Form("version_path=\"%09d%s\" dir_number=%d",id->GetVersion(),filename.Data(),dirNumber);
1013  TString addTagValue = Form("%s%s%s%s",
1014  addTagValue1.Data(),
1015  addTagValue2.Data(),
1016  addTagValue3.Data(),
1017  addTagValue4.Data());
1018 
1019  Bool_t result = kFALSE;
1020  AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data()));
1021  TGridResult* res = gGrid->Command(addTagValue.Data());
1022  const char* resCode = res->GetKey(0,"__result__"); // '1' if success
1023  if(resCode[0] != '1') {
1024  AliError(Form("Couldn't add CDB tag value to file %s !",
1025  filename.Data()));
1026  result = kFALSE;
1027  } else {
1028  AliDebug(2, "Object successfully tagged.");
1029  result = kTRUE;
1030  }
1031  delete res;
1032  return result;
1033 
1034 }
1035 
1036 //_____________________________________________________________________________
1037 Bool_t AliCDBGrid::TagShortLived(TString& filename, Bool_t value){
1038 // tag folder with ShortLived tag
1039 
1040  TString addTagValue = Form("addTagValue %s ShortLived_try value=%d", filename.Data(), value);
1041 
1042  Bool_t result = kFALSE;
1043  AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data()));
1044  TGridResult* res = gGrid->Command(addTagValue.Data());
1045  const char* resCode = res->GetKey(0,"__result__"); // '1' if success
1046  if(resCode[0] != '1') {
1047  AliError(Form("Couldn't add ShortLived tag value to file %s !", filename.Data()));
1048  result = kFALSE;
1049  } else {
1050  AliDebug(2,"Object successfully tagged.");
1051  result = kTRUE;
1052  }
1053  delete res;
1054  return result;
1055 
1056 }
1057 
1058 //_____________________________________________________________________________
1059 Bool_t AliCDBGrid::TagFileMetaData(TString& filename, const AliCDBMetaData* md){
1060 // tag stored object in CDB table using object Id's parameters
1061 
1062  TString addTagValue1 = Form("addTagValue %s CDB_MD ", filename.Data());
1063  TString addTagValue2 = Form("object_classname=\"%s\" responsible=\"%s\" beam_period=%d ",
1064  md->GetObjectClassName(),
1065  md->GetResponsible(),
1066  md->GetBeamPeriod());
1067  TString addTagValue3 = Form("aliroot_version=\"%s\" comment=\"%s\"",
1068  md->GetAliRootVersion(),
1069  md->GetComment());
1070  TString addTagValue = Form("%s%s%s",
1071  addTagValue1.Data(),
1072  addTagValue2.Data(),
1073  addTagValue3.Data());
1074 
1075  Bool_t result = kFALSE;
1076  AliDebug(2, Form("Tagging file. Tag command: %s", addTagValue.Data()));
1077  TGridResult* res = gGrid->Command(addTagValue.Data());
1078  const char* resCode = res->GetKey(0,"__result__"); // '1' if success
1079  if(resCode[0] != '1') {
1080  AliWarning(Form("Couldn't add CDB_MD tag value to file %s !",
1081  filename.Data()));
1082  result = kFALSE;
1083  } else {
1084  AliDebug(2,"Object successfully tagged.");
1085  result = kTRUE;
1086  }
1087  return result;
1088 }
1089 
1090 //_____________________________________________________________________________
1092 
1093  TString turl(fileName);
1094  turl.Prepend("/alien" + fDBFolder);
1095  turl += "?se="; turl += fSE.Data();
1096  TFile *file = TFile::Open(turl);
1097  if (!file) {
1098  AliError(Form("Can't open selection file <%s>!", turl.Data()));
1099  return NULL;
1100  }
1101 
1102  TList *list = new TList();
1103  list->SetOwner();
1104  int i=0;
1105  TString keycycle;
1106 
1107  AliCDBId *id;
1108  while(1){
1109  i++;
1110  keycycle = "AliCDBId;";
1111  keycycle+=i;
1112 
1113  id = (AliCDBId*) file->Get(keycycle);
1114  if(!id) break;
1115  list->AddFirst(id);
1116  }
1117  file->Close(); delete file; file=0;
1118 
1119  return list;
1120 
1121 
1122 }
1123 
1124 //_____________________________________________________________________________
1125 Bool_t AliCDBGrid::Contains(const char* path) const{
1126 // check for path in storage's DBFolder
1127 
1128  TString initDir(gGrid->Pwd(0));
1129  TString dirName(fDBFolder);
1130  dirName += path; // dirName = fDBFolder/path
1131  Bool_t result=kFALSE;
1132  if (gGrid->Cd(dirName,0)) result=kTRUE;
1133  gGrid->Cd(initDir.Data(),0);
1134  return result;
1135 }
1136 
1137 //_____________________________________________________________________________
1139 {
1140  // Query the CDB for files valid for AliCDBStorage::fRun
1141  // Fills list fValidFileIds with AliCDBId objects extracted from CDB files
1142  // selected from AliEn metadata.
1143  // If fVersion was not set, fValidFileIds is filled with highest versions.
1144 
1145  TString filter;
1147 
1148  TString path = fPathFilter.GetPath();
1149 
1150  TString pattern = "Run*";
1151  TString optionQuery = "-y";
1152  if(fMaxDate) {
1153  optionQuery = ""; // AliEn find's `-y` returns no guid in fields :-(
1154  AliInfoF("Returning only latest versions as they were at %s UTC", TTimeStamp(fMaxDate).AsString("s"));
1155  }
1156  if(fVersion >= 0) {
1157  pattern += Form("_v%d_s0", fVersion);
1158  optionQuery = "";
1159  }
1160  pattern += ".root";
1161  AliDebug(2,Form("pattern: %s", pattern.Data()));
1162 
1163  TString addFolder = "";
1164  if (!path.Contains("*")){
1165  if (!path.BeginsWith("/")) addFolder += "/";
1166  addFolder += path;
1167  }
1168  else{
1169  if (path.BeginsWith("/")) path.Remove(0,1);
1170  if (path.EndsWith("/")) path.Remove(path.Length()-1,1);
1171  TObjArray* tokenArr = path.Tokenize("/");
1172  if (tokenArr->GetEntries() != 3) {
1173  AliError("Not a 3 level path! Keeping old query...");
1174  pattern.Prepend(path+"/");
1175  }
1176  else{
1177  TString str0 = ((TObjString*)tokenArr->At(0))->String();
1178  TString str1 = ((TObjString*)tokenArr->At(1))->String();
1179  TString str2 = ((TObjString*)tokenArr->At(2))->String();
1180  if (str0 != "*" && str1 != "*" && str2 == "*"){
1181  // e.g. "ITS/Calib/*"
1182  addFolder = "/"+str0+"/"+str1;
1183  }
1184  else if (str0 != "*" && str1 == "*" && str2 == "*"){
1185  // e.g. "ITS/*/*"
1186  addFolder = "/"+str0;
1187  }
1188  else if (str0 == "*" && str1 == "*" && str2 == "*"){
1189  // e.g. "*/*/*"
1190  // do nothing: addFolder is already an empty string;
1191  }
1192  else{
1193  // e.g. "ITS/*/RecoParam"
1194  pattern.Prepend(path+"/");
1195  }
1196  }
1197  delete tokenArr; tokenArr=0;
1198  }
1199 
1200  TString folderCopy(Form("%s%s",fDBFolder.Data(),addFolder.Data()));
1201 
1202  AliDebug(2,Form("fDBFolder = %s, pattern = %s, filter = %s",folderCopy.Data(), pattern.Data(), filter.Data()));
1203 
1204  if (optionQuery == "-y"){
1205  AliInfo("Only latest version will be returned");
1206  }
1207 
1208  TGridResult *res = gGrid->Query(folderCopy, pattern, filter, optionQuery.Data());
1209 
1210  if (!res) {
1211  AliError("Grid query failed");
1212  return;
1213  }
1214 
1215  TIter next(res);
1216  TMap *map;
1217  while ((map = (TMap*)next())) {
1218  TObjString *entry;
1219  if ((entry = (TObjString *)(map->GetValue("lfn")))) {
1220  TString& filename = entry->String();
1221  if(filename.IsNull()) continue;
1222  AliDebug(2,Form("Found valid file: %s", filename.Data()));
1223  AliCDBId *validFileId = new AliCDBId();
1224  Bool_t result = FilenameToId(filename, *validFileId);
1225  if(result) {
1226  if (fMaxDate) {
1227  // Filter out files whose creation date is greater than fMaxDate.
1228  // Reimplement the logic of selecting the latest version normally
1229  // provided by AliEn find's `-y`.
1230  TObjString *o_guid = (TObjString *)map->GetValue("guid");
1231  if (!o_guid) AliFatal("guid not found in AliEn find query result!");
1232  if ( GuidToCreationTimestamp(o_guid->String()) > fMaxDate ) {
1233  AliDebugF(3, "Apply time filter: skip file %s", filename.Data());
1234  delete validFileId;
1235  validFileId = NULL;
1236  }
1237  else {
1238  TIter iterIds(&fValidFileIds);
1239  AliCDBId *curId;
1240  while (( curId = (AliCDBId *)iterIds() )) {
1241  if (curId->GetPath() == validFileId->GetPath()) {
1242  if (curId->GetVersion() < validFileId->GetVersion()) {
1243  fValidFileIds.Remove(curId); // remove older version first
1244  delete curId;
1245  }
1246  else {
1247  delete validFileId; // version in list is more recent: skip
1248  validFileId = NULL;
1249  }
1250  break;
1251  }
1252  }
1253  }
1254  }
1255  if (validFileId) fValidFileIds.AddLast(validFileId);
1256  }
1257  else {
1258  delete validFileId;
1259  }
1260  }
1261  }
1262  delete res;
1263 
1264 }
1265 
1266 //_____________________________________________________________________________
1267 void AliCDBGrid::MakeQueryFilter(Int_t firstRun, Int_t lastRun,
1268  const AliCDBMetaData* md, TString& result) const
1269 {
1270  // create filter for file query
1271 
1272  result = Form("CDB:first_run<=%d and CDB:last_run>=%d", firstRun, lastRun);
1273 
1274  // if(version >= 0) {
1275  // result += Form(" and CDB:version=%d", version);
1276  // }
1277  // if(pathFilter.GetLevel0() != "*") {
1278  // result += Form(" and CDB:path_level_0=\"%s\"", pathFilter.GetLevel0().Data());
1279  // }
1280  // if(pathFilter.GetLevel1() != "*") {
1281  // result += Form(" and CDB:path_level_1=\"%s\"", pathFilter.GetLevel1().Data());
1282  // }
1283  // if(pathFilter.GetLevel2() != "*") {
1284  // result += Form(" and CDB:path_level_2=\"%s\"", pathFilter.GetLevel2().Data());
1285  // }
1286 
1287  if(md){
1288  if(md->GetObjectClassName()[0] != '\0') {
1289  result += Form(" and CDB_MD:object_classname=\"%s\"", md->GetObjectClassName());
1290  }
1291  if(md->GetResponsible()[0] != '\0') {
1292  result += Form(" and CDB_MD:responsible=\"%s\"", md->GetResponsible());
1293  }
1294  if(md->GetBeamPeriod() != 0) {
1295  result += Form(" and CDB_MD:beam_period=%d", md->GetBeamPeriod());
1296  }
1297  if(md->GetAliRootVersion()[0] != '\0') {
1298  result += Form(" and CDB_MD:aliroot_version=\"%s\"", md->GetAliRootVersion());
1299  }
1300  if(md->GetComment()[0] != '\0') {
1301  result += Form(" and CDB_MD:comment=\"%s\"", md->GetComment());
1302  }
1303  }
1304  AliDebug(2, Form("filter: %s",result.Data()));
1305 
1306 }
1307 
1308 
1310 // //
1311 // AliCDBGrid factory //
1312 // //
1314 
1315 ClassImp(AliCDBGridFactory)
1316 
1317  //_____________________________________________________________________________
1318  Bool_t AliCDBGridFactory::Validate(const char* gridString) {
1319  // check if the string is valid Grid URI
1320 
1321  TRegexp gridPattern("^alien://.+$");
1322 
1323  return TString(gridString).Contains(gridPattern);
1324  }
1325 
1326 //_____________________________________________________________________________
1328  // create AliCDBGridParam class from the URI string
1329 
1330  if (!Validate(gridString)) {
1331  return NULL;
1332  }
1333 
1334  TString buffer(gridString);
1335 
1336  TString gridUrl = "alien://";
1337  TString user = "";
1338  TString dbFolder = "";
1339  TString se = "default";
1340  TString cacheFolder = "";
1341  Bool_t operateDisconnected = kTRUE;
1342  Long64_t cacheSize = (UInt_t) 1024*1024*1024; // 1GB
1343  Long_t cleanupInterval = 0;
1344 
1345  TObjArray *arr = buffer.Tokenize('?');
1346  TIter iter(arr);
1347  TObjString *str = 0;
1348 
1349  while((str = (TObjString*) iter.Next())){
1350  TString entry(str->String());
1351  Int_t indeq = entry.Index('=');
1352  if(indeq == -1) {
1353  if(entry.BeginsWith("alien://")) { // maybe it's a gridUrl!
1354  gridUrl = entry;
1355  continue;
1356  } else {
1357  AliError(Form("Invalid entry! %s",entry.Data()));
1358  continue;
1359  }
1360  }
1361 
1362  TString key = entry(0,indeq);
1363  TString value = entry(indeq+1,entry.Length()-indeq);
1364 
1365  if(key.Contains("grid",TString::kIgnoreCase)) {
1366  gridUrl += value;
1367  }
1368  else if (key.Contains("user",TString::kIgnoreCase)){
1369  user = value;
1370  }
1371  else if (key.Contains("se",TString::kIgnoreCase)){
1372  se = value;
1373  }
1374  else if (key.Contains("cacheF",TString::kIgnoreCase)){
1375  cacheFolder = value;
1376  if (!cacheFolder.IsNull() && !cacheFolder.EndsWith("/"))
1377  cacheFolder += "/";
1378  }
1379  else if (key.Contains("folder",TString::kIgnoreCase)){
1380  dbFolder = value;
1381  }
1382  else if (key.Contains("operateDisc",TString::kIgnoreCase)){
1383  if(value == "kTRUE") {
1384  operateDisconnected = kTRUE;
1385  } else if (value == "kFALSE") {
1386  operateDisconnected = kFALSE;
1387  } else if (value == "0" || value == "1") {
1388  operateDisconnected = (Bool_t) value.Atoi();
1389  } else {
1390  AliError(Form("Invalid entry! %s",entry.Data()));
1391  return NULL;
1392  }
1393  }
1394  else if (key.Contains("cacheS",TString::kIgnoreCase)){
1395  if(value.IsDigit()) {
1396  cacheSize = value.Atoi();
1397  } else {
1398  AliError(Form("Invalid entry! %s",entry.Data()));
1399  return NULL;
1400  }
1401  }
1402  else if (key.Contains("cleanupInt",TString::kIgnoreCase)){
1403  if(value.IsDigit()) {
1404  cleanupInterval = value.Atoi();
1405  } else {
1406  AliError(Form("Invalid entry! %s",entry.Data()));
1407  return NULL;
1408  }
1409  }
1410  else{
1411  AliError(Form("Invalid entry! %s",entry.Data()));
1412  return NULL;
1413  }
1414  }
1415  delete arr; arr=0;
1416 
1417  AliDebug(2, Form("gridUrl: %s", gridUrl.Data()));
1418  AliDebug(2, Form("user: %s", user.Data()));
1419  AliDebug(2, Form("dbFolder: %s", dbFolder.Data()));
1420  AliDebug(2, Form("s.e.: %s", se.Data()));
1421  AliDebug(2, Form("local cache folder: %s", cacheFolder.Data()));
1422  AliDebug(2, Form("local cache operate disconnected: %d", operateDisconnected));
1423  AliDebug(2, Form("local cache size: %lld", cacheSize));
1424  AliDebug(2, Form("local cache cleanup interval: %ld", cleanupInterval));
1425 
1426  if(dbFolder == ""){
1427  AliError("The CDB base folder must be specified by preceding it with the \"folder=\" key!");
1428  AliError("Example: \"alien://folder=/alice/data/2015/OCDB\"");
1429  return NULL;
1430  }
1431 
1432  return new AliCDBGridParam(gridUrl.Data(), user.Data(),
1433  dbFolder.Data(), se.Data(), cacheFolder.Data(),
1434  operateDisconnected, cacheSize, cleanupInterval);
1435 }
1436 
1437 //_____________________________________________________________________________
1439  // create AliCDBGrid storage instance from parameters
1440 
1441  AliCDBGrid *grid = 0;
1442  if (AliCDBGridParam::Class() == param->IsA()) {
1443 
1444  const AliCDBGridParam* gridParam = (const AliCDBGridParam*) param;
1445  grid = new AliCDBGrid(gridParam->GridUrl().Data(),
1446  gridParam->GetUser().Data(),
1447  gridParam->GetDBFolder().Data(),
1448  gridParam->GetSE().Data(),
1449  gridParam->GetCacheFolder().Data(),
1450  gridParam->GetOperateDisconnected(),
1451  gridParam->GetCacheSize(),
1452  gridParam->GetCleanupInterval());
1453 
1454  }
1455 
1456  if(!gGrid && grid) {
1457  delete grid; grid=0;
1458  }
1459 
1460  return grid;
1461 }
1462 
1464 // //
1465 // AliCDBGrid Parameter class // //
1466 // //
1468 
1469 ClassImp(AliCDBGridParam)
1470 
1471  //_____________________________________________________________________________
1473  AliCDBParam(),
1474  fGridUrl(),
1475  fUser(),
1476  fDBFolder(),
1477  fSE(),
1478  fCacheFolder(),
1480  fCacheSize(),
1482 
1483 {
1484  // default constructor
1485 
1486 }
1487 
1488 //_____________________________________________________________________________
1489 AliCDBGridParam::AliCDBGridParam(const char* gridUrl, const char* user, const char* dbFolder,
1490  const char* se, const char* cacheFolder, Bool_t operateDisconnected,
1491  Long64_t cacheSize, Long_t cleanupInterval):
1492  AliCDBParam(),
1493  fGridUrl(gridUrl),
1494  fUser(user),
1495  fDBFolder(dbFolder),
1496  fSE(se),
1497  fCacheFolder(cacheFolder),
1498  fOperateDisconnected(operateDisconnected),
1499  fCacheSize(cacheSize),
1500  fCleanupInterval(cleanupInterval)
1501 {
1502  // constructor
1503 
1504  SetType("alien");
1505 
1506  TString uri = Form("%s?User=%s?DBFolder=%s?SE=%s?CacheFolder=%s"
1507  "?OperateDisconnected=%d?CacheSize=%lld?CleanupInterval=%ld",
1508  fGridUrl.Data(), fUser.Data(),
1509  fDBFolder.Data(), fSE.Data(), fCacheFolder.Data(),
1511 
1512  SetURI(uri.Data());
1513 }
1514 
1515 //_____________________________________________________________________________
1517  // destructor
1518 
1519 }
1520 
1521 //_____________________________________________________________________________
1523  // clone parameter
1524 
1525  return new AliCDBGridParam(fGridUrl.Data(), fUser.Data(),
1526  fDBFolder.Data(), fSE.Data(), fCacheFolder.Data(),
1528 }
1529 
1530 //_____________________________________________________________________________
1531 ULong_t AliCDBGridParam::Hash() const {
1532  // return Hash function
1533 
1534  return fGridUrl.Hash()+fUser.Hash()+fDBFolder.Hash()+fSE.Hash()+fCacheFolder.Hash();
1535 }
1536 
1537 //_____________________________________________________________________________
1538 Bool_t AliCDBGridParam::IsEqual(const TObject* obj) const {
1539  // check if this object is equal to AliCDBParam obj
1540 
1541  if (this == obj) {
1542  return kTRUE;
1543  }
1544 
1545  if (AliCDBGridParam::Class() != obj->IsA()) {
1546  return kFALSE;
1547  }
1548 
1549  AliCDBGridParam* other = (AliCDBGridParam*) obj;
1550 
1551  if(fGridUrl != other->fGridUrl) return kFALSE;
1552  if(fUser != other->fUser) return kFALSE;
1553  if(fDBFolder != other->fDBFolder) return kFALSE;
1554  if(fSE != other->fSE) return kFALSE;
1555  if(fCacheFolder != other->fCacheFolder) return kFALSE;
1556  if(fOperateDisconnected != other->fOperateDisconnected) return kFALSE;
1557  if(fCacheSize != other->fCacheSize) return kFALSE;
1558  if(fCleanupInterval != other->fCleanupInterval) return kFALSE;
1559  return kTRUE;
1560 }
1561 
void SetURI(const char *uri)
virtual TList * GetEntries(const AliCDBId &queryId)
Definition: AliCDBGrid.cxx:586
TObjArray fValidFileIds
TFile * Open(const char *filename, Long64_t &nevents)
const TString & GetPath() const
Definition: AliCDBPath.h:38
void SetVersion(Int_t version)
Definition: AliCDBEntry.h:67
Bool_t Comprises(const AliCDBPath &other) const
Definition: AliCDBPath.cxx:250
#define TObjArray
TString fCacheFolder
Definition: AliCDBGrid.h:141
const TString & GetSE() const
Definition: AliCDBGrid.h:125
const char * path
time_t GuidToCreationTimestamp(const TString &guid) const
static Bool_t GetForbidGRPOverride()
Definition: AliCDBGrid.h:32
Bool_t Overlaps(const AliCDBRunRange &other) const
void SetSubVersion(Int_t subVersion)
Definition: AliCDBId.h:62
AliCDBEntry * GetEntryFromFile(TString &filename, AliCDBId *dataId)
Definition: AliCDBGrid.cxx:516
TROOT * gROOT
UInt_t GetBeamPeriod() const
TString fUser
Definition: AliCDBGrid.h:71
const char * GetComment() const
virtual AliCDBParam * CloneParam() const
const AliCDBPath & GetAliCDBPath() const
Definition: AliCDBId.h:39
Short_t fInitRetrySeconds
TString fCacheFolder
Definition: AliCDBGrid.h:75
virtual Bool_t IsEqual(const TObject *obj) const
Int_t GetVersion() const
Definition: AliCDBId.h:59
Int_t GetLastRun() const
Definition: AliCDBId.h:49
Bool_t FilenameToId(TString &filename, AliCDBId &id)
Definition: AliCDBGrid.cxx:150
#define AliInfoF(message,...)
Definition: AliLog.h:499
const AliCDBRunRange & GetAliCDBRunRange() const
Definition: AliCDBId.h:46
TString fileName(const char *dir, int runNumber, const char *da, int i, const char *type)
#define AliWarning(message)
Definition: AliLog.h:541
virtual Bool_t Contains(const char *path) const
Long64_t GetCacheSize() const
Definition: AliCDBGrid.h:128
Long64_t fCacheSize
Definition: AliCDBGrid.h:77
Bool_t fOperateDisconnected
Definition: AliCDBGrid.h:142
virtual ~AliCDBGridParam()
virtual TList * GetIdListFromFile(const char *fileName)
virtual void QueryValidFiles()
Bool_t TagFileId(TString &filename, const AliCDBId *id)
Definition: AliCDBGrid.cxx:995
void SetVersion(Int_t version)
Definition: AliCDBId.h:61
virtual Bool_t IsEqual(const TObject *obj) const
Definition: AliCDBId.cxx:147
const char * GetObjectClassName() const
const TString & GridUrl() const
Definition: AliCDBGrid.h:122
#define AliInfo(message)
Definition: AliLog.h:484
const TString & GetUser() const
Definition: AliCDBGrid.h:123
void GetSelection(AliCDBId *id)
virtual ~AliCDBGrid()
Definition: AliCDBGrid.cxx:143
virtual AliCDBEntry * GetEntry(const AliCDBId &queryId)
Definition: AliCDBGrid.cxx:482
Int_t GetFirstRun() const
Long64_t fCacheSize
Definition: AliCDBGrid.h:143
const TString GetPathLevel(Int_t i) const
Definition: AliCDBId.h:41
Int_t GetLastRun() const
AliCDBPath fPathFilter
TString fGridUrl
Definition: AliCDBGrid.h:137
Definition: AliCDBEntry.h:18
#define AliFatal(message)
Definition: AliLog.h:640
TString ToString() const
Definition: AliCDBId.cxx:163
TString fGridUrl
Definition: AliCDBGrid.h:70
Bool_t IsAnyRange() const
Long_t GetCleanupInterval() const
Definition: AliCDBGrid.h:129
void MakeQueryFilter(Int_t firstRun, Int_t lastRun, const AliCDBMetaData *md, TString &result) const
const TString & GetCacheFolder() const
Definition: AliCDBGrid.h:126
Bool_t PutInCvmfs(TString &fullFilename, TFile *cdbFile) const
Definition: AliCDBGrid.cxx:913
const char * GetResponsible() const
void LoadTreeFromFile(AliCDBEntry *entry) const
#define AliDebug(logLevel, message)
Definition: AliLog.h:300
AliCDBMetaData * GetMetaData()
Definition: AliCDBEntry.h:60
void SetType(const char *type)
Bool_t HasVersion() const
Definition: AliCDBId.h:70
Bool_t TagFileMetaData(TString &filename, const AliCDBMetaData *md)
virtual Bool_t IsEqual(const TObject *obj) const
virtual Bool_t IdToFilename(const AliCDBId &id, TString &filename) const
Definition: AliCDBGrid.cxx:202
Bool_t TagShortLived(TString &filename, Bool_t value)
virtual const char * GetMirrorSEs() const
Definition: AliCDBGrid.h:29
void SetLastStorage(TString lastStorage)
Definition: AliCDBEntry.h:71
AliCDBId * GetId(const TObjArray &validFileIds, const AliCDBId &query)
Definition: AliCDBGrid.cxx:376
TString fDBFolder
Definition: AliCDBGrid.h:72
const TString & GetPath() const
Definition: AliCDBId.h:40
virtual AliCDBParam * CreateParameter(const char *gridString)
Bool_t PrepareId(AliCDBId &id)
Definition: AliCDBGrid.cxx:242
virtual Bool_t PutEntry(AliCDBEntry *entry, const char *mirrors="")
Definition: AliCDBGrid.cxx:734
virtual void SetRetry(Int_t nretry, Int_t initsec)
Definition: AliCDBGrid.cxx:230
#define AliError(message)
Definition: AliLog.h:591
virtual AliCDBId * GetEntryId(const AliCDBId &queryId)
Definition: AliCDBGrid.cxx:416
AliCDBId & GetId()
Definition: AliCDBEntry.h:51
static AliCDBManager * Instance(TMap *entryCache=NULL, Int_t run=-1)
void res(Char_t i)
Definition: Resolution.C:2
Bool_t AddTag(TString &foldername, const char *tagname)
Definition: AliCDBGrid.cxx:977
const char * GetAliRootVersion() const
Long_t fCleanupInterval
Definition: AliCDBGrid.h:144
#define AliDebugF(logLevel, format,...)
Definition: AliLog.h:338
AliCDBMetaData * fMetaDataFilter
Bool_t fOperateDisconnected
Definition: AliCDBGrid.h:76
Short_t fNretry
Local, Grid: base folder name - Dump: file name.
Int_t GetFirstRun() const
Definition: AliCDBId.h:48
AliCDBGrid(const char *gridUrl, const char *user, const char *dbFolder, const char *se, const char *cacheFolder, Bool_t operateDisconnected, Long64_t cacheSize, Long_t cleanupInterval)
Definition: AliCDBGrid.cxx:47
virtual AliCDBStorage * Create(const AliCDBParam *param)
virtual ULong_t Hash() const
Bool_t GetOperateDisconnected() const
Definition: AliCDBGrid.h:127
Long_t fCleanupInterval
Definition: AliCDBGrid.h:78
TString fSE
Definition: AliCDBGrid.h:73
const TString & GetDBFolder() const
Definition: AliCDBGrid.h:124
TString fDBFolder
Definition: AliCDBGrid.h:139
Int_t GetSubVersion() const
Definition: AliCDBId.h:60