00001
00002
00003
00004
00005
00006
00007
00008
00009
00010
00011
00012
00013
00014
00015
00016
00017
00018
00019
00020
00021
00022
00023
00024
00025
00026
00027
00028
00029
00030
00031
00032
00033
00034
00035
00036
00037
00038
00039 #include "FileSystemStore.h"
00040 #include "StorageConfig.h"
00041
00042 #include <sys/types.h>
00043 #include <sys/stat.h>
00044 #include <unistd.h>
00045 #include <dirent.h>
00046 #include <fcntl.h>
00047 #include <errno.h>
00048
00049 #include "../util/ExpandableBuffer.h"
00050 #include "../serialize/KeySerialize.h"
00051 #include "../serialize/TypeCollection.h"
00052 #include "../io/FileUtils.h"
00053 #include "../io/IO.h"
00054
00055
00056 namespace oasys {
00057
00058
00059 FileSystemStore::FileSystemStore(const char* logpath)
00060 : DurableStoreImpl("FileSystemStore", logpath),
00061 db_dir_("INVALID"),
00062 tables_dir_("INVALID"),
00063 default_perm_(S_IRUSR | S_IWUSR | S_IXUSR | S_IRGRP | S_IXGRP),
00064 fd_cache_(0)
00065 {}
00066
00067
00068 FileSystemStore::~FileSystemStore()
00069 {}
00070
00071
00072 int
00073 FileSystemStore::init(const StorageConfig& cfg)
00074 {
00075 if (cfg.dbdir_ == "") {
00076 return -1;
00077 }
00078
00079 if (cfg.dbname_ == "") {
00080 return -1;
00081 }
00082
00083 db_dir_ = cfg.dbdir_;
00084 FileUtils::abspath(&db_dir_);
00085
00086 tables_dir_ = db_dir_ + "/" + cfg.dbname_;
00087
00088 bool tidy = cfg.tidy_;
00089 bool init = cfg.init_;
00090
00091
00092
00093 if (tidy) {
00094 init = true;
00095 }
00096
00097 if (init && tidy)
00098 {
00099 if (check_database() == 0) {
00100 tidy_database();
00101 }
00102 int err = init_database();
00103 if (err != 0) {
00104 return -1;
00105 }
00106 }
00107 else if (init && ! tidy)
00108 {
00109 if (check_database() == -2) {
00110 int err = init_database();
00111 if (err != 0) {
00112 return -1;
00113 }
00114 }
00115 }
00116 else
00117 {
00118 if (check_database() != 0) {
00119 log_err("Database directory not found");
00120 return -1;
00121 }
00122 }
00123
00124 if (cfg.fs_fd_cache_size_ > 0) {
00125 fd_cache_ = new FdCache(logpath_, cfg.fs_fd_cache_size_);
00126 }
00127
00128 log_info("init() done");
00129 init_ = true;
00130
00131 return 0;
00132 }
00133
00134
00135 int
00136 FileSystemStore::get_table(DurableTableImpl** table,
00137 const std::string& name,
00138 int flags,
00139 PrototypeVector& prototypes)
00140 {
00141 (void)prototypes;
00142
00143 ASSERT(init_);
00144
00145 std::string dir_path = tables_dir_;
00146 dir_path.append("/");
00147 dir_path.append(name);
00148
00149 struct stat st;
00150 int err = stat(dir_path.c_str(), &st);
00151
00152
00153 if (err != 0 && errno == ENOENT) {
00154 if ( ! (flags & DS_CREATE)) {
00155 return DS_NOTFOUND;
00156 }
00157
00158 int err = mkdir(dir_path.c_str(), default_perm_);
00159 if (err != 0) {
00160 err = errno;
00161 log_err("Couldn't mkdir: %s", strerror(err));
00162 return DS_ERR;
00163 }
00164 } else if (err != 0) {
00165 return DS_ERR;
00166 } else if (err == 0 && (flags & DS_EXCL)) {
00167 return DS_EXISTS;
00168 }
00169
00170 FileSystemTable* table_ptr =
00171 new FileSystemTable(logpath_,
00172 name,
00173 dir_path,
00174 flags & DS_MULTITYPE,
00175 fd_cache_);
00176 ASSERT(table_ptr);
00177
00178 *table = table_ptr;
00179
00180 return 0;
00181 }
00182
00183
00184 int
00185 FileSystemStore::del_table(const std::string& name)
00186 {
00187
00188 ASSERT(init_);
00189
00190 std::string dir_path = tables_dir_;
00191 dir_path.append("/");
00192 dir_path.append(name);
00193
00194 FileUtils::rm_all_from_dir(dir_path.c_str());
00195
00196
00197 int err;
00198 err = rmdir(dir_path.c_str());
00199 if (err != 0) {
00200 log_warn("couldn't remove directory, %s", strerror(errno));
00201 return -1;
00202 }
00203
00204 return 0;
00205 }
00206
00207
00208 int
00209 FileSystemStore::get_table_names(StringVector* names)
00210 {
00211 names->clear();
00212
00213 DIR* dir = opendir(tables_dir_.c_str());
00214 if (dir == 0) {
00215 log_err("Can't get table names from directory");
00216 return DS_ERR;
00217 }
00218
00219 struct dirent* ent = readdir(dir);
00220 while (ent != 0) {
00221 names->push_back(ent->d_name);
00222 ent = readdir(dir);
00223 }
00224
00225 closedir(dir);
00226
00227 return 0;
00228 }
00229
00230
00231 int
00232 FileSystemStore::check_database()
00233 {
00234 DIR* dir = opendir(tables_dir_.c_str());
00235 if (dir == 0) {
00236 if (errno == ENOENT) {
00237 return -2;
00238 } else {
00239 return -1;
00240 }
00241 }
00242 closedir(dir);
00243
00244 return 0;
00245 }
00246
00247
00248 int
00249 FileSystemStore::init_database()
00250 {
00251 log_notice("init database (tables dir '%s'", tables_dir_.c_str());
00252
00253 int err = mkdir(db_dir_.c_str(), default_perm_);
00254 if (err != 0 && errno != EEXIST) {
00255 log_warn("init() failed: %s", strerror(errno));
00256 return -1;
00257 }
00258
00259 err = mkdir(tables_dir_.c_str(), default_perm_);
00260 if (err != 0 && errno != EEXIST) {
00261 log_warn("init() failed: %s", strerror(errno));
00262 return -1;
00263 }
00264
00265 return 0;
00266 }
00267
00268
00269 void
00270 FileSystemStore::tidy_database()
00271 {
00272 log_notice("Tidy() database, rm -rf %s", db_dir_.c_str());
00273
00274 char cmd[256];
00275 int cc = snprintf(cmd, 256, "rm -rf %s", db_dir_.c_str());
00276 ASSERT(cc < 256);
00277 system(cmd);
00278 }
00279
00280
00281 FileSystemTable::FileSystemTable(const char* logpath,
00282 const std::string& table_name,
00283 const std::string& path,
00284 bool multitype,
00285 FileSystemStore::FdCache* cache)
00286 : DurableTableImpl(table_name, multitype),
00287 Logger("FileSystemTable", "%s/%s", logpath, table_name.c_str()),
00288 path_(path),
00289 cache_(cache)
00290 {}
00291
00292
00293 FileSystemTable::~FileSystemTable()
00294 {}
00295
00296
00297 int
00298 FileSystemTable::get(const SerializableObject& key,
00299 SerializableObject* data)
00300 {
00301 ASSERTF(!multitype_, "single-type get called for multi-type table");
00302
00303 ScratchBuffer<u_char*, 4096> buf;
00304 int err = get_common(key, &buf);
00305 if (err != 0) {
00306 return err;
00307 }
00308
00309 Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00310 err = um.action(data);
00311 if (err != 0) {
00312 return DS_ERR;
00313 }
00314
00315 return 0;
00316 }
00317
00318
00319 int
00320 FileSystemTable::get(const SerializableObject& key,
00321 SerializableObject** data,
00322 TypeCollection::Allocator_t allocator)
00323 {
00324 ASSERTF(multitype_, "multi-type get called for single-type table");
00325
00326 ScratchBuffer<u_char*, 4096> buf;
00327 int err = get_common(key, &buf);
00328 if (err != 0) {
00329 return err;
00330 }
00331
00332 Unmarshal um(Serialize::CONTEXT_LOCAL, buf.buf(), buf.len());
00333
00334 TypeCollection::TypeCode_t typecode;
00335 um.process("typecode", &typecode);
00336
00337 err = allocator(typecode, data);
00338 if (err != 0) {
00339 return DS_ERR;
00340 }
00341 err = um.action(*data);
00342 if (err != 0) {
00343 return DS_ERR;
00344 }
00345
00346 return 0;
00347 }
00348
00349
00350 int
00351 FileSystemTable::put(const SerializableObject& key,
00352 TypeCollection::TypeCode_t typecode,
00353 const SerializableObject* data,
00354 int flags)
00355 {
00356 ScratchBuffer<char*, 512> key_str;
00357 KeyMarshal s_key(&key_str, "-");
00358
00359 if (s_key.action(&key) != 0) {
00360 log_err("Can't get key");
00361 return DS_ERR;
00362 }
00363
00364 ScratchBuffer<u_char*, 4096> scratch;
00365 Marshal m(Serialize::CONTEXT_LOCAL, &scratch);
00366
00367 if (multitype_) {
00368 m.process("typecode", &typecode);
00369 }
00370 if (m.action(data) != 0) {
00371 log_warn("can't marshal data");
00372 return DS_ERR;
00373 }
00374
00375 std::string filename = path_ + "/" + key_str.buf();
00376 int data_elt_fd = -1;
00377 int open_flags = O_TRUNC | O_RDWR;
00378
00379 if (flags & DS_EXCL) {
00380 open_flags |= O_EXCL;
00381 }
00382
00383 if (flags & DS_CREATE) {
00384 open_flags |= O_CREAT;
00385 }
00386
00387 log_debug("opening file %s", filename.c_str());
00388
00389 if (cache_)
00390 {
00391 data_elt_fd = cache_->get_and_pin(filename);
00392 }
00393
00394 if (data_elt_fd == -1)
00395 {
00396 data_elt_fd = open(filename.c_str(), open_flags,
00397 S_IRUSR | S_IWUSR | S_IRGRP);
00398 if (data_elt_fd == -1)
00399 {
00400 if (errno == ENOENT) {
00401 log_debug("file not found and DS_CREATE not specified");
00402 return DS_NOTFOUND;
00403 } else if (errno == EEXIST) {
00404 log_debug("file found and DS_EXCL specified");
00405 return DS_EXISTS;
00406 } else {
00407 log_warn("can't open %s: %s",
00408 filename.c_str(), strerror(errno));
00409 return DS_ERR;
00410 }
00411 }
00412
00413 if (cache_)
00414 {
00415 int old_fd = data_elt_fd;
00416 data_elt_fd = cache_->put_and_pin(filename, data_elt_fd);
00417 if (old_fd != data_elt_fd)
00418 {
00419 IO::close(old_fd);
00420 }
00421 }
00422 }
00423 else if (cache_ && (flags & DS_EXCL))
00424 {
00425 cache_->unpin(filename);
00426 return DS_EXISTS;
00427 }
00428
00429 log_debug("created file %s, fd = %d",
00430 filename.c_str(), data_elt_fd);
00431
00432 if (cache_)
00433 {
00434 int cc = IO::lseek(data_elt_fd, 0, SEEK_SET);
00435 ASSERT(cc == 0);
00436 }
00437
00438 int cc = IO::writeall(data_elt_fd,
00439 reinterpret_cast<char*>(scratch.buf()),
00440 scratch.len());
00441 if (cc != static_cast<int>(scratch.len()))
00442 {
00443 log_warn("put() - errors writing to file %s, %d: %s",
00444 filename.c_str(), cc, strerror(errno));
00445 if (cache_)
00446 {
00447 cache_->unpin(filename);
00448 }
00449 return DS_ERR;
00450 }
00451
00452 if (cache_)
00453 {
00454
00455 cache_->unpin(filename);
00456 }
00457 else
00458 {
00459 IO::close(data_elt_fd);
00460 }
00461
00462 return 0;
00463 }
00464
00465
00466 int
00467 FileSystemTable::del(const SerializableObject& key)
00468 {
00469 ScratchBuffer<char*, 512> key_str;
00470 KeyMarshal s_key(&key_str, "-");
00471
00472 if (s_key.action(&key) != 0) {
00473 log_err("Can't get key");
00474 return DS_ERR;
00475 }
00476
00477 std::string filename = path_ + "/" + key_str.buf();
00478
00479 if (cache_)
00480 {
00481 cache_->close(filename);
00482 }
00483
00484 int err = unlink(filename.c_str());
00485 if (err == -1)
00486 {
00487 if (errno == ENOENT) {
00488 return DS_NOTFOUND;
00489 }
00490
00491 log_warn("can't unlink file %s - %s", filename.c_str(),
00492 strerror(errno));
00493 return DS_ERR;
00494 }
00495
00496 return 0;
00497 }
00498
00499
00500 size_t
00501 FileSystemTable::size() const
00502 {
00503
00504 DIR* dir = opendir(path_.c_str());
00505 ASSERT(dir != 0);
00506
00507 size_t count;
00508 struct dirent* ent;
00509
00510 for (count = 0, ent = readdir(dir);
00511 ent != 0; ent = readdir(dir))
00512 {
00513 ASSERT(ent != 0);
00514 ++count;
00515 }
00516
00517 closedir(dir);
00518
00519
00520 log_debug("table size = %zu", count - 2);
00521
00522 return count - 2;
00523 }
00524
00525
00526 DurableIterator*
00527 FileSystemTable::itr()
00528 {
00529 return new FileSystemIterator(path_);
00530 }
00531
00532
00533 int
00534 FileSystemTable::get_common(const SerializableObject& key,
00535 ExpandableBuffer* buf)
00536 {
00537 ScratchBuffer<char*, 512> key_str;
00538 KeyMarshal serialize(&key_str, "-");
00539 if (serialize.action(&key) != 0) {
00540 log_err("Can't get key");
00541 return DS_ERR;
00542 }
00543
00544 std::string file_name(key_str.at(0));
00545 std::string file_path = path_ + "/" + file_name;
00546 log_debug("opening file %s", file_path.c_str());
00547
00548
00549 int fd = -1;
00550 if (cache_)
00551 {
00552 fd = cache_->get_and_pin(file_path);
00553 }
00554
00555 if (fd == -1) {
00556 fd = open(file_path.c_str(), O_RDWR);
00557 if (fd == -1)
00558 {
00559 if (errno == ENOENT)
00560 {
00561 return DS_NOTFOUND;
00562 }
00563
00564 return DS_ERR;
00565 }
00566
00567 if (cache_)
00568 {
00569 int old_fd = fd;
00570 fd = cache_->put_and_pin(file_path, fd);
00571 if (old_fd != fd)
00572 {
00573 IO::close(old_fd);
00574 }
00575 }
00576 }
00577
00578 if (cache_)
00579 {
00580 int cc = IO::lseek(fd, 0, SEEK_SET);
00581 ASSERT(cc == 0);
00582 }
00583
00584 int cc;
00585 do {
00586 buf->reserve(buf->len() + 4096);
00587 cc = IO::read(fd, buf->end(), 4096, 0);
00588 ASSERTF(cc >= 0, "read failed %s", strerror(errno));
00589 buf->set_len(buf->len() + cc);
00590 } while (cc > 0);
00591
00592 if (cache_)
00593 {
00594 cache_->unpin(file_path);
00595 }
00596 else
00597 {
00598 IO::close(fd);
00599 }
00600
00601 return 0;
00602 }
00603
00604
00605 FileSystemIterator::FileSystemIterator(const std::string& path)
00606 : ent_(0)
00607 {
00608 dir_ = opendir(path.c_str());
00609 ASSERT(dir_ != 0);
00610 }
00611
00612
00613 FileSystemIterator::~FileSystemIterator()
00614 {
00615 closedir(dir_);
00616 }
00617
00618
00619 int
00620 FileSystemIterator::next()
00621 {
00622 skip_dots:
00623 ent_ = readdir(dir_);
00624
00625 if (ent_ != 0 && (strcmp(ent_->d_name, ".") == 0 ||
00626 strcmp(ent_->d_name, "..") == 0))
00627 {
00628 goto skip_dots;
00629 }
00630
00631 if (ent_ == 0)
00632 {
00633 if (errno == EBADF)
00634 {
00635 return DS_ERR;
00636 }
00637 else
00638 {
00639 return DS_NOTFOUND;
00640 }
00641 }
00642
00643 return 0;
00644 }
00645
00646
00647 int
00648 FileSystemIterator::get_key(SerializableObject* key)
00649 {
00650 ASSERT(ent_ != 0);
00651
00652 KeyUnmarshal um(ent_->d_name, strlen(ent_->d_name), "-");
00653 int err = um.action(key);
00654 if (err != 0) {
00655 return DS_ERR;
00656 }
00657
00658 return 0;
00659 }
00660
00661 }