FileSystemStore.cc

Go to the documentation of this file.
00001 /*
00002  *    Copyright 2004-2006 Intel Corporation
00003  * 
00004  *    Licensed under the Apache License, Version 2.0 (the "License");
00005  *    you may not use this file except in compliance with the License.
00006  *    You may obtain a copy of the License at
00007  * 
00008  *        http://www.apache.org/licenses/LICENSE-2.0
00009  * 
00010  *    Unless required by applicable law or agreed to in writing, software
00011  *    distributed under the License is distributed on an "AS IS" BASIS,
00012  *    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
00013  *    See the License for the specific language governing permissions and
00014  *    limitations under the License.
00015  */
00016 
00017 
00018 #include "FileSystemStore.h"
00019 #include "StorageConfig.h"
00020 
00021 #include <sys/types.h>
00022 #include <sys/stat.h>
00023 #include <unistd.h>
00024 #include <dirent.h>
00025 #include <fcntl.h>
00026 #include <errno.h>
00027 
00028 #include "../util/ExpandableBuffer.h"
00029 #include "../serialize/KeySerialize.h"
00030 #include "../serialize/TypeCollection.h"
00031 #include "../io/FileUtils.h"
00032 #include "../io/IO.h"
00033 
00034 
00035 namespace oasys {
00036 
00037 //----------------------------------------------------------------------------
00038 FileSystemStore::FileSystemStore(const char* logpath)
00039     : DurableStoreImpl("FileSystemStore", logpath),
00040       db_dir_("INVALID"),
00041       tables_dir_("INVALID"),
00042       default_perm_(S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP),
00043       fd_cache_(0)
00044 {}
00045 
00046 //----------------------------------------------------------------------------
00047 FileSystemStore::~FileSystemStore()
00048 {}
00049 
00050 //----------------------------------------------------------------------------
00051 int 
00052 FileSystemStore::init(const StorageConfig& cfg)
00053 {
00054     if (cfg.dbdir_ == "") {
00055         return -1;
00056     }
00057 
00058     if (cfg.dbname_ == "") {
00059         return -1;
00060     }
00061     
00062     db_dir_ = cfg.dbdir_;
00063     FileUtils::abspath(&db_dir_);
00064 
00065     tables_dir_ = db_dir_ + "/" + cfg.dbname_;
00066 
00067     bool tidy = cfg.tidy_;
00068     bool init = cfg.init_;
00069 
00070     // Always regenerate the directories if we are going to be
00071     // deleting them anyways
00072     if (tidy) {
00073         init = true;
00074     }
00075 
00076     if (init && tidy) 
00077     {
00078         if (check_database() == 0) {
00079             tidy_database();
00080         }
00081         int err = init_database();
00082         if (err != 0) {
00083             return -1;
00084         }
00085     }
00086     else if (init && ! tidy) 
00087     {
00088         if (check_database() == -2) {
00089             int err = init_database();
00090             if (err != 0) {
00091                 return -1;
00092             }
00093         }
00094     }
00095     else 
00096     {
00097         if (check_database() != 0) {
00098             log_err("Database directory not found");
00099             return -1;
00100         }
00101     }
00102 
00103     if (cfg.fs_fd_cache_size_ > 0) {
00104         fd_cache_ = new FdCache(logpath_, cfg.fs_fd_cache_size_);
00105     }
00106 
00107     log_info("init() done");
00108     init_ = true;
00109 
00110     return 0;
00111 }
00112 
00113 //----------------------------------------------------------------------------
00114 int 
00115 FileSystemStore::get_table(DurableTableImpl** table,
00116                            const std::string& name,
00117                            int                flags,
00118                            PrototypeVector&   prototypes)
00119 {
00120     (void)prototypes;
00121     
00122     ASSERT(init_);
00123 
00124     std::string dir_path =  tables_dir_;
00125     dir_path.append("/");
00126     dir_path.append(name);
00127 
00128     struct stat st;
00129     int err = stat(dir_path.c_str(), &st);
00130     
00131     // Already existing directory
00132     if (err != 0 && errno == ENOENT) {
00133         if ( ! (flags & DS_CREATE)) {
00134             return DS_NOTFOUND;
00135         }
00136         
00137         int err = mkdir(dir_path.c_str(), default_perm_);
00138         if (err != 0) {
00139             err = errno;
00140             log_err("Couldn't mkdir: %s", strerror(err));
00141             return DS_ERR;
00142         }
00143     } else if (err != 0) { 
00144         return DS_ERR;
00145     } else if (err == 0 && (flags & DS_EXCL)) {
00146         return DS_EXISTS;
00147     }
00148     
00149     FileSystemTable* table_ptr =
00150         new FileSystemTable(logpath_, 
00151                             name, 
00152                             dir_path, 
00153                             flags & DS_MULTITYPE, 
00154                             fd_cache_);
00155     ASSERT(table_ptr);
00156     
00157     *table = table_ptr;
00158 
00159     return 0;
00160 }
00161 
00162 //----------------------------------------------------------------------------
00163 int 
00164 FileSystemStore::del_table(const std::string& name)
00165 {
00166     // XXX/bowei -- don't handle table sharing for now
00167     ASSERT(init_);
00168 
00169     std::string dir_path = tables_dir_;
00170     dir_path.append("/");
00171     dir_path.append(name);
00172     
00173     FileUtils::rm_all_from_dir(dir_path.c_str());
00174 
00175     // clean out the directory
00176     int err;
00177     err = rmdir(dir_path.c_str());
00178     if (err != 0) {
00179         log_warn("couldn't remove directory, %s", strerror(errno));
00180         return -1;
00181     }
00182 
00183     return 0; 
00184 }
00185 
00186 //----------------------------------------------------------------------------
00187 int 
00188 FileSystemStore::get_table_names(StringVector* names)
00189 {
00190     names->clear();
00191     
00192     DIR* dir = opendir(tables_dir_.c_str());
00193     if (dir == 0) {
00194         log_err("Can't get table names from directory");
00195         return DS_ERR;
00196     }
00197 
00198     struct dirent* ent = readdir(dir);
00199     while (ent != 0) {
00200         names->push_back(ent->d_name);
00201         ent = readdir(dir);
00202     }
00203 
00204     closedir(dir);
00205 
00206     return 0;
00207 }
00208 
00209 //----------------------------------------------------------------------------
00210 std::string 
00211 FileSystemStore::get_info() const
00212 {
00213     StringBuffer desc;
00214 
00215     return "FileSystemStore";
00216 }
00217 
00218 //----------------------------------------------------------------------------
00219 int 
00220 FileSystemStore::check_database()
00221 {
00222     DIR* dir = opendir(tables_dir_.c_str());
00223     if (dir == 0) {
00224         if (errno == ENOENT) {
00225             return -2;
00226         } else {
00227             return -1;
00228         }
00229     }
00230     closedir(dir);
00231 
00232     return 0;
00233 }
00234 
00235 //----------------------------------------------------------------------------
00236 int 
00237 FileSystemStore::init_database()
00238 {
00239     log_notice("init database (tables dir '%s'", tables_dir_.c_str());
00240 
00241     int err = mkdir(db_dir_.c_str(), default_perm_);
00242     if (err != 0 && errno != EEXIST) {
00243         log_warn("init() failed: %s", strerror(errno));
00244         return -1;
00245     }
00246 
00247     err = mkdir(tables_dir_.c_str(), default_perm_);
00248     if (err != 0 && errno != EEXIST) {
00249         log_warn("init() failed: %s", strerror(errno));
00250         return -1;
00251     }
00252 
00253     return 0;
00254 }
00255 
00256 //----------------------------------------------------------------------------
00257 void
00258 FileSystemStore::tidy_database()
00259 {
00260     log_notice("Tidy() database, rm -rf %s", db_dir_.c_str());
00261     
00262     char cmd[256];
00263     int cc = snprintf(cmd, 256, "rm -rf %s", db_dir_.c_str());
00264     ASSERT(cc < 256);
00265     system(cmd);
00266 }
00267 
00268 //----------------------------------------------------------------------------
00269 FileSystemTable::FileSystemTable(const char*               logpath,
00270                                  const std::string&        table_name,
00271                                  const std::string&        path,
00272                                  bool                      multitype,
00273                                  FileSystemStore::FdCache* cache)
00274     : DurableTableImpl(table_name, multitype),
00275       Logger("FileSystemTable", "%s/%s", logpath, table_name.c_str()),
00276       path_(path),
00277       cache_(cache)
00278 {}
00279 
00280 //----------------------------------------------------------------------
00281 FileSystemTable::~FileSystemTable()
00282 {}
00283 
00284 //----------------------------------------------------------------------------
00285 int 
00286 FileSystemTable::get(const SerializableObject& key,
00287                      SerializableObject*       data)
00288 {
00289     ASSERTF(!multitype_, "single-type get called for multi-type table");
00290 
00291     ScratchBuffer<u_char*, 4096> buf;
00292     int err = get_common(key, &buf);
00293     if (err != 0) {
00294         return err;
00295     }
00296     
00297     Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00298     err = um.action(data);
00299     if (err != 0) {
00300         return DS_ERR;
00301     }
00302 
00303     return 0;
00304 }
00305     
00306 //----------------------------------------------------------------------------
00307 int 
00308 FileSystemTable::get(const SerializableObject&   key,
00309                      SerializableObject**        data,
00310                      TypeCollection::Allocator_t allocator)
00311 {
00312     ASSERTF(multitype_, "multi-type get called for single-type table");
00313 
00314     ScratchBuffer<u_char*, 4096> buf;
00315     int err = get_common(key, &buf);
00316     if (err != 0) {
00317         return err;
00318     }
00319     
00320     Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00321 
00322     TypeCollection::TypeCode_t typecode;
00323     um.process("typecode", &typecode);
00324     
00325     err = allocator(typecode, data);
00326     if (err != 0) {
00327         return DS_ERR;
00328     }
00329     err = um.action(*data);
00330     if (err != 0) {
00331         return DS_ERR;
00332     }
00333 
00334     return 0;
00335 }
00336     
00337 //----------------------------------------------------------------------------
00338 int 
00339 FileSystemTable::put(const SerializableObject&  key,
00340                      TypeCollection::TypeCode_t typecode,
00341                      const SerializableObject*  data,
00342                      int flags)
00343 {
00344     ScratchBuffer<char*, 512> key_str;
00345     KeyMarshal s_key(&key_str, "-");
00346 
00347     if (s_key.action(&key) != 0) {
00348         log_err("Can't get key");
00349         return DS_ERR;
00350     }
00351     
00352     ScratchBuffer<u_char*, 4096> scratch;
00353     Marshal m(Serialize::CONTEXT_LOCAL, &scratch);
00354     
00355     if (multitype_) {
00356         m.process("typecode", &typecode);
00357     }
00358     if (m.action(data) != 0) {
00359         log_warn("can't marshal data");
00360         return DS_ERR;
00361     }
00362 
00363     std::string filename = path_ + "/" + key_str.buf();
00364     int data_elt_fd      = -1;
00365     int open_flags       = O_TRUNC | O_RDWR;
00366 
00367     if (flags & DS_EXCL) {
00368         open_flags |= O_EXCL;       
00369     }
00370     
00371     if (flags & DS_CREATE) {
00372         open_flags |= O_CREAT;
00373     }
00374     
00375     log_debug("opening file %s", filename.c_str());
00376     
00377     if (cache_) 
00378     {
00379         data_elt_fd = cache_->get_and_pin(filename);
00380     }
00381 
00382     if (data_elt_fd == -1) 
00383     {
00384         data_elt_fd = open(filename.c_str(), open_flags, 
00385                            S_IRUSR | S_IWUSR | S_IRGRP);
00386         if (data_elt_fd == -1)
00387         {
00388             if (errno == ENOENT) 
00389             {
00390                 ASSERT(! (flags      & DS_CREATE));
00391                 ASSERT(! (open_flags & O_CREAT));
00392                 log_debug("file not found and DS_CREATE not specified");
00393                 return DS_NOTFOUND;
00394             } 
00395             else if (errno == EEXIST) 
00396             {
00397                 ASSERT(open_flags & O_EXCL);
00398                 log_debug("file found and DS_EXCL specified");
00399                 return DS_EXISTS;
00400             } 
00401             else 
00402             {
00403                 log_warn("can't open %s: %s", 
00404                          filename.c_str(), strerror(errno));
00405                 return DS_ERR;
00406             }
00407         }
00408 
00409         if (cache_) 
00410         {
00411             int old_fd = data_elt_fd;
00412             data_elt_fd = cache_->put_and_pin(filename, data_elt_fd);
00413             if (old_fd != data_elt_fd) 
00414             {
00415                 IO::close(old_fd);
00416             }
00417         }
00418     } 
00419     else if (cache_ && (flags & DS_EXCL))
00420     {
00421         cache_->unpin(filename);
00422         return DS_EXISTS;
00423     }
00424 
00425     log_debug("created file %s, fd = %d", 
00426               filename.c_str(), data_elt_fd);
00427     
00428     if (cache_) 
00429     {
00430         int cc = IO::lseek(data_elt_fd, 0, SEEK_SET);
00431         ASSERT(cc == 0);
00432     }
00433 
00434     int cc = IO::writeall(data_elt_fd, 
00435                           reinterpret_cast<char*>(scratch.buf()), 
00436                           scratch.len());
00437     if (cc != static_cast<int>(scratch.len())) 
00438     {
00439         log_warn("put() - errors writing to file %s, %d: %s",
00440                  filename.c_str(), cc, strerror(errno));
00441         if (cache_) 
00442         {
00443             cache_->unpin(filename);
00444         }
00445         return DS_ERR;
00446     }
00447     
00448     if (cache_)
00449     {
00450         // close(data_elt_fd); XXX/bowei -- fsync?
00451         cache_->unpin(filename);
00452     }
00453     else
00454     {
00455         IO::close(data_elt_fd);
00456     }
00457 
00458     return 0;
00459 }
00460     
00461 //----------------------------------------------------------------------------
00462 int 
00463 FileSystemTable::del(const SerializableObject& key)
00464 {
00465     ScratchBuffer<char*, 512> key_str;
00466     KeyMarshal s_key(&key_str, "-");
00467 
00468     if (s_key.action(&key) != 0) {
00469         log_err("Can't get key");
00470         return DS_ERR;
00471     }
00472     
00473     std::string filename = path_ + "/" + key_str.buf();
00474 
00475     if (cache_)
00476     {
00477         cache_->close(filename);
00478     }
00479 
00480     int err = unlink(filename.c_str());
00481     if (err == -1) 
00482     {
00483         if (errno == ENOENT) {
00484             return DS_NOTFOUND;
00485         }
00486         
00487         log_warn("can't unlink file %s - %s", filename.c_str(),
00488                  strerror(errno));
00489         return DS_ERR;
00490     }
00491     
00492     return 0;
00493 }
00494 
00495 //----------------------------------------------------------------------------
00496 size_t 
00497 FileSystemTable::size() const
00498 {
00499     // XXX/bowei -- be inefficient for now
00500     DIR* dir = opendir(path_.c_str());
00501     ASSERT(dir != 0);
00502     
00503     size_t count;
00504     struct dirent* ent;
00505 
00506     for (count = 0, ent = readdir(dir); 
00507          ent != 0; ent = readdir(dir))
00508     {
00509         ASSERT(ent != 0);
00510         ++count;
00511     }
00512 
00513     closedir(dir);
00514 
00515     // count always includes '.' and '..'
00516     log_debug("table size = %zu", count - 2);
00517 
00518     return count - 2; 
00519 }
00520     
00521 //----------------------------------------------------------------------------
00522 DurableIterator* 
00523 FileSystemTable::itr()
00524 {
00525     return new FileSystemIterator(path_);
00526 }
00527 
00528 //----------------------------------------------------------------------------
00529 int 
00530 FileSystemTable::get_common(const SerializableObject& key,
00531                             ExpandableBuffer*         buf)
00532 {
00533     ScratchBuffer<char*, 512> key_str;
00534     KeyMarshal serialize(&key_str, "-");
00535     if (serialize.action(&key) != 0) {
00536         log_err("Can't get key");
00537         return DS_ERR;
00538     }
00539     
00540     std::string file_name(key_str.at(0));
00541     std::string file_path = path_ + "/" + file_name;
00542     log_debug("opening file %s", file_path.c_str());
00543 
00544     
00545     int fd = -1;
00546     if (cache_) 
00547     {
00548         fd = cache_->get_and_pin(file_path);
00549     }
00550 
00551     if (fd == -1) {
00552         fd = open(file_path.c_str(), O_RDWR);
00553         if (fd == -1) 
00554         {
00555             log_debug("error opening file %s: %s",
00556                       file_path.c_str(), strerror(errno));
00557             if (errno == ENOENT) 
00558             {
00559                 return DS_NOTFOUND;
00560             }
00561             
00562             return DS_ERR;
00563         }
00564 
00565         if (cache_) 
00566         {
00567             int old_fd = fd;
00568             fd = cache_->put_and_pin(file_path, fd);
00569             if (old_fd != fd) 
00570             {
00571                 IO::close(old_fd);
00572             }
00573         }
00574     }
00575     
00576     if (cache_) 
00577     {
00578         int cc = IO::lseek(fd, 0, SEEK_SET);
00579         ASSERT(cc == 0);
00580     }
00581     
00582     int cc;
00583     do {
00584         buf->reserve(buf->len() + 4096);
00585         cc = IO::read(fd, buf->end(), 4096, 0);
00586         ASSERTF(cc >= 0, "read failed %s", strerror(errno));
00587         buf->set_len(buf->len() + cc);
00588     } while (cc > 0);
00589 
00590     if (cache_) 
00591     {
00592         cache_->unpin(file_path);
00593     }
00594     else
00595     {
00596         IO::close(fd);
00597     }
00598 
00599     return 0;
00600 }
00601 
00602 //----------------------------------------------------------------------------
00603 FileSystemIterator::FileSystemIterator(const std::string& path)
00604     : ent_(0)
00605 {
00606     dir_ = opendir(path.c_str());
00607     ASSERT(dir_ != 0);
00608 }
00609 
00610 //----------------------------------------------------------------------------
00611 FileSystemIterator::~FileSystemIterator()
00612 {
00613     closedir(dir_);
00614 }
00615     
00616 //----------------------------------------------------------------------------
00617 int 
00618 FileSystemIterator::next()
00619 {
00620   skip_dots:
00621     ent_ = readdir(dir_);
00622 
00623     if (ent_ != 0 && (strcmp(ent_->d_name, ".") == 0 ||
00624                       strcmp(ent_->d_name, "..") == 0))
00625     {
00626         goto skip_dots;
00627     }
00628 
00629     if (ent_ == 0) 
00630     {
00631         if (errno == EBADF) 
00632         {
00633             return DS_ERR;
00634         } 
00635         else 
00636         {
00637             return DS_NOTFOUND;
00638         }
00639     }
00640 
00641     return 0;
00642 }
00643 
00644 //----------------------------------------------------------------------------
00645 int 
00646 FileSystemIterator::get_key(SerializableObject* key)
00647 {
00648     ASSERT(ent_ != 0);
00649     
00650     KeyUnmarshal um(ent_->d_name, strlen(ent_->d_name), "-");
00651     int err = um.action(key);
00652     if (err != 0) {
00653         return DS_ERR;
00654     }
00655 
00656     return 0;
00657 }
00658 
00659 } // namespace oasys

Generated on Thu Jun 7 12:54:27 2007 for DTN Reference Implementation by  doxygen 1.5.1