FileSystemStore.cc

Go to the documentation of this file.
00001 /*
00002  * IMPORTANT: READ BEFORE DOWNLOADING, COPYING, INSTALLING OR USING. By
00003  * downloading, copying, installing or using the software you agree to
00004  * this license. If you do not agree to this license, do not download,
00005  * install, copy or use the software.
00006  * 
00007  * Intel Open Source License 
00008  * 
00009  * Copyright (c) 2004 Intel Corporation. All rights reserved. 
00010  * 
00011  * Redistribution and use in source and binary forms, with or without
00012  * modification, are permitted provided that the following conditions are
00013  * met:
00014  * 
00015  *   Redistributions of source code must retain the above copyright
00016  *   notice, this list of conditions and the following disclaimer.
00017  * 
00018  *   Redistributions in binary form must reproduce the above copyright
00019  *   notice, this list of conditions and the following disclaimer in the
00020  *   documentation and/or other materials provided with the distribution.
00021  * 
00022  *   Neither the name of the Intel Corporation nor the names of its
00023  *   contributors may be used to endorse or promote products derived from
00024  *   this software without specific prior written permission.
00025  *  
00026  * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
00027  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
00028  * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
00029  * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE INTEL OR
00030  * ITS CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
00031  * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
00032  * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
00033  * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
00034  * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
00035  * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
00036  * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
00037  */
00038 
00039 #include "FileSystemStore.h"
00040 #include "StorageConfig.h"
00041 
00042 #include <sys/types.h>
00043 #include <sys/stat.h>
00044 #include <unistd.h>
00045 #include <dirent.h>
00046 #include <fcntl.h>
00047 #include <errno.h>
00048 
00049 #include "../util/ExpandableBuffer.h"
00050 #include "../serialize/KeySerialize.h"
00051 #include "../serialize/TypeCollection.h"
00052 #include "../io/FileUtils.h"
00053 #include "../io/IO.h"
00054 
00055 
00056 namespace oasys {
00057 
00058 //----------------------------------------------------------------------------
00059 FileSystemStore::FileSystemStore(const char* logpath)
00060     : DurableStoreImpl("FileSystemStore", logpath),
00061       db_dir_("INVALID"),
00062       tables_dir_("INVALID"),
00063       default_perm_(S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP),
00064       fd_cache_(0)
00065 {}
00066 
00067 //----------------------------------------------------------------------------
00068 FileSystemStore::~FileSystemStore()
00069 {}
00070 
00071 //----------------------------------------------------------------------------
00072 int 
00073 FileSystemStore::init(const StorageConfig& cfg)
00074 {
00075     if (cfg.dbdir_ == "") {
00076         return -1;
00077     }
00078 
00079     if (cfg.dbname_ == "") {
00080         return -1;
00081     }
00082     
00083     db_dir_ = cfg.dbdir_;
00084     FileUtils::abspath(&db_dir_);
00085 
00086     tables_dir_ = db_dir_ + "/" + cfg.dbname_;
00087 
00088     bool tidy = cfg.tidy_;
00089     bool init = cfg.init_;
00090 
00091     // Always regenerate the directories if we are going to be
00092     // deleting them anyways
00093     if (tidy) {
00094         init = true;
00095     }
00096 
00097     if (init && tidy) 
00098     {
00099         if (check_database() == 0) {
00100             tidy_database();
00101         }
00102         int err = init_database();
00103         if (err != 0) {
00104             return -1;
00105         }
00106     }
00107     else if (init && ! tidy) 
00108     {
00109         if (check_database() == -2) {
00110             int err = init_database();
00111             if (err != 0) {
00112                 return -1;
00113             }
00114         }
00115     }
00116     else 
00117     {
00118         if (check_database() != 0) {
00119             log_err("Database directory not found");
00120             return -1;
00121         }
00122     }
00123 
00124     if (cfg.fs_fd_cache_size_ > 0) {
00125         fd_cache_ = new FdCache(logpath_, cfg.fs_fd_cache_size_);
00126     }
00127 
00128     log_info("init() done");
00129     init_ = true;
00130 
00131     return 0;
00132 }
00133 
00134 //----------------------------------------------------------------------------
00135 int 
00136 FileSystemStore::get_table(DurableTableImpl** table,
00137                            const std::string& name,
00138                            int                flags,
00139                            PrototypeVector&   prototypes)
00140 {
00141     (void)prototypes;
00142     
00143     ASSERT(init_);
00144 
00145     std::string dir_path =  tables_dir_;
00146     dir_path.append("/");
00147     dir_path.append(name);
00148 
00149     struct stat st;
00150     int err = stat(dir_path.c_str(), &st);
00151     
00152     // Already existing directory
00153     if (err != 0 && errno == ENOENT) {
00154         if ( ! (flags & DS_CREATE)) {
00155             return DS_NOTFOUND;
00156         }
00157         
00158         int err = mkdir(dir_path.c_str(), default_perm_);
00159         if (err != 0) {
00160             err = errno;
00161             log_err("Couldn't mkdir: %s", strerror(err));
00162             return DS_ERR;
00163         }
00164     } else if (err != 0) { 
00165         return DS_ERR;
00166     } else if (err == 0 && (flags & DS_EXCL)) {
00167         return DS_EXISTS;
00168     }
00169     
00170     FileSystemTable* table_ptr =
00171         new FileSystemTable(logpath_, 
00172                             name, 
00173                             dir_path, 
00174                             flags & DS_MULTITYPE, 
00175                             fd_cache_);
00176     ASSERT(table_ptr);
00177     
00178     *table = table_ptr;
00179 
00180     return 0;
00181 }
00182 
00183 //----------------------------------------------------------------------------
00184 int 
00185 FileSystemStore::del_table(const std::string& name)
00186 {
00187     // XXX/bowei -- don't handle table sharing for now
00188     ASSERT(init_);
00189 
00190     std::string dir_path = tables_dir_;
00191     dir_path.append("/");
00192     dir_path.append(name);
00193     
00194     FileUtils::rm_all_from_dir(dir_path.c_str());
00195 
00196     // clean out the directory
00197     int err;
00198     err = rmdir(dir_path.c_str());
00199     if (err != 0) {
00200         log_warn("couldn't remove directory, %s", strerror(errno));
00201         return -1;
00202     }
00203 
00204     return 0; 
00205 }
00206 
00207 //----------------------------------------------------------------------------
00208 int 
00209 FileSystemStore::get_table_names(StringVector* names)
00210 {
00211     names->clear();
00212     
00213     DIR* dir = opendir(tables_dir_.c_str());
00214     if (dir == 0) {
00215         log_err("Can't get table names from directory");
00216         return DS_ERR;
00217     }
00218 
00219     struct dirent* ent = readdir(dir);
00220     while (ent != 0) {
00221         names->push_back(ent->d_name);
00222         ent = readdir(dir);
00223     }
00224 
00225     closedir(dir);
00226 
00227     return 0;
00228 }
00229 
00230 //----------------------------------------------------------------------------
00231 int 
00232 FileSystemStore::check_database()
00233 {
00234     DIR* dir = opendir(tables_dir_.c_str());
00235     if (dir == 0) {
00236         if (errno == ENOENT) {
00237             return -2;
00238         } else {
00239             return -1;
00240         }
00241     }
00242     closedir(dir);
00243 
00244     return 0;
00245 }
00246 
00247 //----------------------------------------------------------------------------
00248 int 
00249 FileSystemStore::init_database()
00250 {
00251     log_notice("init database (tables dir '%s'", tables_dir_.c_str());
00252 
00253     int err = mkdir(db_dir_.c_str(), default_perm_);
00254     if (err != 0 && errno != EEXIST) {
00255         log_warn("init() failed: %s", strerror(errno));
00256         return -1;
00257     }
00258 
00259     err = mkdir(tables_dir_.c_str(), default_perm_);
00260     if (err != 0 && errno != EEXIST) {
00261         log_warn("init() failed: %s", strerror(errno));
00262         return -1;
00263     }
00264 
00265     return 0;
00266 }
00267 
00268 //----------------------------------------------------------------------------
00269 void
00270 FileSystemStore::tidy_database()
00271 {
00272     log_notice("Tidy() database, rm -rf %s", db_dir_.c_str());
00273     
00274     char cmd[256];
00275     int cc = snprintf(cmd, 256, "rm -rf %s", db_dir_.c_str());
00276     ASSERT(cc < 256);
00277     system(cmd);
00278 }
00279 
00280 //----------------------------------------------------------------------------
00281 FileSystemTable::FileSystemTable(const char*               logpath,
00282                                  const std::string&        table_name,
00283                                  const std::string&        path,
00284                                  bool                      multitype,
00285                                  FileSystemStore::FdCache* cache)
00286     : DurableTableImpl(table_name, multitype),
00287       Logger("FileSystemTable", "%s/%s", logpath, table_name.c_str()),
00288       path_(path),
00289       cache_(cache)
00290 {}
00291 
00292 //----------------------------------------------------------------------
00293 FileSystemTable::~FileSystemTable()
00294 {}
00295 
00296 //----------------------------------------------------------------------------
00297 int 
00298 FileSystemTable::get(const SerializableObject& key,
00299                      SerializableObject*       data)
00300 {
00301     ASSERTF(!multitype_, "single-type get called for multi-type table");
00302 
00303     ScratchBuffer<u_char*, 4096> buf;
00304     int err = get_common(key, &buf);
00305     if (err != 0) {
00306         return err;
00307     }
00308     
00309     Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00310     err = um.action(data);
00311     if (err != 0) {
00312         return DS_ERR;
00313     }
00314 
00315     return 0;
00316 }
00317     
00318 //----------------------------------------------------------------------------
00319 int 
00320 FileSystemTable::get(const SerializableObject&   key,
00321                      SerializableObject**        data,
00322                      TypeCollection::Allocator_t allocator)
00323 {
00324     ASSERTF(multitype_, "multi-type get called for single-type table");
00325 
00326     ScratchBuffer<u_char*, 4096> buf;
00327     int err = get_common(key, &buf);
00328     if (err != 0) {
00329         return err;
00330     }
00331     
00332     Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00333 
00334     TypeCollection::TypeCode_t typecode;
00335     um.process("typecode", &typecode);
00336     
00337     err = allocator(typecode, data);
00338     if (err != 0) {
00339         return DS_ERR;
00340     }
00341     err = um.action(*data);
00342     if (err != 0) {
00343         return DS_ERR;
00344     }
00345 
00346     return 0;
00347 }
00348     
00349 //----------------------------------------------------------------------------
00350 int 
00351 FileSystemTable::put(const SerializableObject&  key,
00352                      TypeCollection::TypeCode_t typecode,
00353                      const SerializableObject*  data,
00354                      int flags)
00355 {
00356     ScratchBuffer<char*, 512> key_str;
00357     KeyMarshal s_key(&key_str, "-");
00358 
00359     if (s_key.action(&key) != 0) {
00360         log_err("Can't get key");
00361         return DS_ERR;
00362     }
00363     
00364     ScratchBuffer<u_char*, 4096> scratch;
00365     Marshal m(Serialize::CONTEXT_LOCAL, &scratch);
00366     
00367     if (multitype_) {
00368         m.process("typecode", &typecode);
00369     }
00370     if (m.action(data) != 0) {
00371         log_warn("can't marshal data");
00372         return DS_ERR;
00373     }
00374 
00375     std::string filename = path_ + "/" + key_str.buf();
00376     int data_elt_fd      = -1;
00377     int open_flags       = O_TRUNC | O_RDWR;
00378 
00379     if (flags & DS_EXCL) {
00380         open_flags |= O_EXCL;       
00381     }
00382     
00383     if (flags & DS_CREATE) {
00384         open_flags |= O_CREAT;
00385     }
00386     
00387     log_debug("opening file %s", filename.c_str());
00388     
00389     if (cache_) 
00390     {
00391         data_elt_fd = cache_->get_and_pin(filename);
00392     }
00393 
00394     if (data_elt_fd == -1) 
00395     {
00396         data_elt_fd = open(filename.c_str(), open_flags, 
00397                            S_IRUSR | S_IWUSR | S_IRGRP);
00398         if (data_elt_fd == -1)
00399         {
00400             if (errno == ENOENT) {
00401                 log_debug("file not found and DS_CREATE not specified");
00402                 return DS_NOTFOUND;
00403             } else if (errno == EEXIST) {
00404                 log_debug("file found and DS_EXCL specified");
00405                 return DS_EXISTS;
00406             } else {
00407                 log_warn("can't open %s: %s", 
00408                          filename.c_str(), strerror(errno));
00409                 return DS_ERR;
00410             }
00411         }
00412 
00413         if (cache_) 
00414         {
00415             int old_fd = data_elt_fd;
00416             data_elt_fd = cache_->put_and_pin(filename, data_elt_fd);
00417             if (old_fd != data_elt_fd) 
00418             {
00419                 IO::close(old_fd);
00420             }
00421         }
00422     } 
00423     else if (cache_ && (flags & DS_EXCL))
00424     {
00425         cache_->unpin(filename);
00426         return DS_EXISTS;
00427     }
00428 
00429     log_debug("created file %s, fd = %d", 
00430               filename.c_str(), data_elt_fd);
00431     
00432     if (cache_) 
00433     {
00434         int cc = IO::lseek(data_elt_fd, 0, SEEK_SET);
00435         ASSERT(cc == 0);
00436     }
00437 
00438     int cc = IO::writeall(data_elt_fd, 
00439                           reinterpret_cast<char*>(scratch.buf()), 
00440                           scratch.len());
00441     if (cc != static_cast<int>(scratch.len())) 
00442     {
00443         log_warn("put() - errors writing to file %s, %d: %s",
00444                  filename.c_str(), cc, strerror(errno));
00445         if (cache_) 
00446         {
00447             cache_->unpin(filename);
00448         }
00449         return DS_ERR;
00450     }
00451     
00452     if (cache_)
00453     {
00454         // close(data_elt_fd); XXX/bowei -- fsync?
00455         cache_->unpin(filename);
00456     }
00457     else
00458     {
00459         IO::close(data_elt_fd);
00460     }
00461 
00462     return 0;
00463 }
00464     
00465 //----------------------------------------------------------------------------
00466 int 
00467 FileSystemTable::del(const SerializableObject& key)
00468 {
00469     ScratchBuffer<char*, 512> key_str;
00470     KeyMarshal s_key(&key_str, "-");
00471 
00472     if (s_key.action(&key) != 0) {
00473         log_err("Can't get key");
00474         return DS_ERR;
00475     }
00476     
00477     std::string filename = path_ + "/" + key_str.buf();
00478 
00479     if (cache_)
00480     {
00481         cache_->close(filename);
00482     }
00483 
00484     int err = unlink(filename.c_str());
00485     if (err == -1) 
00486     {
00487         if (errno == ENOENT) {
00488             return DS_NOTFOUND;
00489         }
00490         
00491         log_warn("can't unlink file %s - %s", filename.c_str(),
00492                  strerror(errno));
00493         return DS_ERR;
00494     }
00495     
00496     return 0;
00497 }
00498 
00499 //----------------------------------------------------------------------------
00500 size_t 
00501 FileSystemTable::size() const
00502 {
00503     // XXX/bowei -- be inefficient for now
00504     DIR* dir = opendir(path_.c_str());
00505     ASSERT(dir != 0);
00506     
00507     size_t count;
00508     struct dirent* ent;
00509 
00510     for (count = 0, ent = readdir(dir); 
00511          ent != 0; ent = readdir(dir))
00512     {
00513         ASSERT(ent != 0);
00514         ++count;
00515     }
00516 
00517     closedir(dir);
00518 
00519     // count always includes '.' and '..'
00520     log_debug("table size = %zu", count - 2);
00521 
00522     return count - 2; 
00523 }
00524     
00525 //----------------------------------------------------------------------------
00526 DurableIterator* 
00527 FileSystemTable::itr()
00528 {
00529     return new FileSystemIterator(path_);
00530 }
00531 
00532 //----------------------------------------------------------------------------
00533 int 
00534 FileSystemTable::get_common(const SerializableObject& key,
00535                             ExpandableBuffer*         buf)
00536 {
00537     ScratchBuffer<char*, 512> key_str;
00538     KeyMarshal serialize(&key_str, "-");
00539     if (serialize.action(&key) != 0) {
00540         log_err("Can't get key");
00541         return DS_ERR;
00542     }
00543     
00544     std::string file_name(key_str.at(0));
00545     std::string file_path = path_ + "/" + file_name;
00546     log_debug("opening file %s", file_path.c_str());
00547 
00548     
00549     int fd = -1;
00550     if (cache_) 
00551     {
00552         fd = cache_->get_and_pin(file_path);
00553     }
00554 
00555     if (fd == -1) {
00556         fd = open(file_path.c_str(), O_RDWR);
00557         if (fd == -1) 
00558         {
00559             if (errno == ENOENT) 
00560             {
00561                 return DS_NOTFOUND;
00562             }
00563             
00564             return DS_ERR;
00565         }
00566 
00567         if (cache_) 
00568         {
00569             int old_fd = fd;
00570             fd = cache_->put_and_pin(file_path, fd);
00571             if (old_fd != fd) 
00572             {
00573                 IO::close(old_fd);
00574             }
00575         }
00576     }
00577     
00578     if (cache_) 
00579     {
00580         int cc = IO::lseek(fd, 0, SEEK_SET);
00581         ASSERT(cc == 0);
00582     }
00583     
00584     int cc;
00585     do {
00586         buf->reserve(buf->len() + 4096);
00587         cc = IO::read(fd, buf->end(), 4096, 0);
00588         ASSERTF(cc >= 0, "read failed %s", strerror(errno));
00589         buf->set_len(buf->len() + cc);
00590     } while (cc > 0);
00591 
00592     if (cache_) 
00593     {
00594         cache_->unpin(file_path);
00595     }
00596     else
00597     {
00598         IO::close(fd);
00599     }
00600 
00601     return 0;
00602 }
00603 
00604 //----------------------------------------------------------------------------
00605 FileSystemIterator::FileSystemIterator(const std::string& path)
00606     : ent_(0)
00607 {
00608     dir_ = opendir(path.c_str());
00609     ASSERT(dir_ != 0);
00610 }
00611 
00612 //----------------------------------------------------------------------------
00613 FileSystemIterator::~FileSystemIterator()
00614 {
00615     closedir(dir_);
00616 }
00617     
00618 //----------------------------------------------------------------------------
00619 int 
00620 FileSystemIterator::next()
00621 {
00622   skip_dots:
00623     ent_ = readdir(dir_);
00624 
00625     if (ent_ != 0 && (strcmp(ent_->d_name, ".") == 0 ||
00626                       strcmp(ent_->d_name, "..") == 0))
00627     {
00628         goto skip_dots;
00629     }
00630 
00631     if (ent_ == 0) 
00632     {
00633         if (errno == EBADF) 
00634         {
00635             return DS_ERR;
00636         } 
00637         else 
00638         {
00639             return DS_NOTFOUND;
00640         }
00641     }
00642 
00643     return 0;
00644 }
00645 
00646 //----------------------------------------------------------------------------
00647 int 
00648 FileSystemIterator::get_key(SerializableObject* key)
00649 {
00650     ASSERT(ent_ != 0);
00651     
00652     KeyUnmarshal um(ent_->d_name, strlen(ent_->d_name), "-");
00653     int err = um.action(key);
00654     if (err != 0) {
00655         return DS_ERR;
00656     }
00657 
00658     return 0;
00659 }
00660 
00661 } // namespace oasys

Generated on Fri Dec 22 14:47:59 2006 for DTN Reference Implementation by  doxygen 1.5.1