Osmium  0.1
include/osmium/storage/byid.hpp
Go to the documentation of this file.
00001 #ifndef OSMIUM_HANDLER_STORE_HPP
00002 #define OSMIUM_HANDLER_STORE_HPP
00003 
00004 /*
00005 
00006 Copyright 2011 Jochen Topf <jochen@topf.org> and others (see README).
00007 
00008 This file is part of Osmium (https://github.com/joto/osmium).
00009 
00010 Osmium is free software: you can redistribute it and/or modify it under the
00011 terms of the GNU Lesser General Public License or (at your option) the GNU
00012 General Public License as published by the Free Software Foundation, either
00013 version 3 of the Licenses, or (at your option) any later version.
00014 
00015 Osmium is distributed in the hope that it will be useful, but WITHOUT ANY
00016 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
00017 PARTICULAR PURPOSE. See the GNU Lesser General Public License and the GNU
00018 General Public License for more details.
00019 
00020 You should have received a copy of the Licenses along with Osmium. If not, see
00021 <http://www.gnu.org/licenses/>.
00022 
00023 */
00024 
00025 #include <stdexcept>
00026 #include <cstdlib>
00027 #include <google/sparsetable>
00028 #include <sys/mman.h>
00029 #include <sys/types.h>
00030 #include <unistd.h>
00031 #include <sys/stat.h>
00032 #include <fcntl.h>
00033 #include <boost/utility.hpp>
00034 
00035 namespace Osmium {
00036 
00040     namespace Storage {
00041 
00057         template <typename TValue>
00058         class ById : boost::noncopyable {
00059 
00060         public:
00061 
00062             virtual ~ById() {
00063             }
00064 
00066             typedef TValue value_type;
00067 
00069             virtual void set(uint64_t id, TValue value) = 0;
00070 
00072             virtual const TValue& operator[](uint64_t id) const = 0;
00073 
00080             virtual uint64_t size() const = 0;
00081 
00089             virtual uint64_t used_memory() const = 0;
00090 
00095             virtual void clear() = 0;
00096 
00097         };
00098 
00118         template <typename TValue>
00119         class FixedArray : public ById<TValue> {
00120 
00121         public:
00122 
00129             FixedArray(const uint64_t max_id) : ById<TValue>(), m_size(max_id) {
00130                 m_items = (TValue*) malloc(sizeof(TValue) * max_id);
00131                 if (!m_items) {
00132                     throw std::bad_alloc();
00133                 }
00134             }
00135 
00136             ~FixedArray() {
00137                 clear();
00138             }
00139 
00140             void set(uint64_t id, TValue value) {
00141                 m_items[id] = value;
00142             }
00143 
00144             const TValue& operator[](uint64_t id) const {
00145                 return m_items[id];
00146             }
00147 
00148             uint64_t size() const {
00149                 return m_size;
00150             }
00151 
00152             uint64_t used_memory() const {
00153                 return m_size * sizeof(TValue);
00154             }
00155 
00156             void clear() {
00157                 free(m_items);
00158                 m_items = NULL;
00159             }
00160 
00161         private:
00162 
00163             TValue* m_items;
00164 
00165             uint64_t m_size;
00166 
00167         }; // class FixedArray
00168 
00178         template <typename TValue>
00179         class SparseTable : public ById<TValue> {
00180 
00181         public:
00182 
00190             SparseTable(const uint64_t grow_size=10000)
00191                 : ById<TValue>(),
00192                   m_grow_size(grow_size),
00193                   m_items(grow_size) {
00194             }
00195 
00196             ~SparseTable() {
00197             }
00198 
00199             void set(uint64_t id, TValue value) {
00200                 if (id >= m_items.size()) {
00201                     m_items.resize(id + m_grow_size);
00202                 }
00203                 m_items[id] = value;
00204             }
00205 
00206             const TValue& operator[](uint64_t id) const {
00207                 return m_items[id];
00208             }
00209 
00210             uint64_t size() const {
00211                 return m_items.size();
00212             }
00213 
00214             uint64_t used_memory() const {
00215                 // unused items use 1 bit, used items sizeof(TValue) bytes
00216                 // http://google-sparsehash.googlecode.com/svn/trunk/doc/sparsetable.html
00217                 return (m_items.size() / 8) + (m_items.num_nonempty() * sizeof(TValue));
00218             }
00219 
00220             void clear() {
00221                 m_items.clear();
00222             }
00223 
00224         private:
00225 
00226             uint64_t m_grow_size;
00227 
00228             google::sparsetable<TValue> m_items;
00229 
00230         }; // class SparseTable
00231 
00246         template <typename TValue>
00247         class Mmap : public ById<TValue> {
00248 
00249         public:
00250 
00251             static const uint64_t size_increment = 10 * 1024 * 1024;
00252 
00257             Mmap() : ById<TValue>(), m_size(size_increment), m_fd(-1) {
00258                 m_items = (TValue*) mmap(NULL, sizeof(TValue) * m_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0);
00259                 if (m_items == MAP_FAILED) {
00260                     throw std::bad_alloc();
00261                 }
00262             }
00263 
00272             Mmap(std::string& filename, bool remove=true) : ById<TValue>(), m_size(1) {
00273                 if (filename == "") {
00274                     FILE* file = tmpfile();
00275                     if (!file) {
00276                         throw std::bad_alloc();
00277                     }
00278                     m_fd = fileno(file);
00279                 } else {
00280                     m_fd = open(filename.c_str(), O_RDWR | O_CREAT, 0600);
00281                 }
00282 
00283                 if (m_fd < 0) {
00284                     throw std::bad_alloc();
00285                 }
00286 
00287                 // now that the file is open we can immediately remove it
00288                 // (temporary files are always removed)
00289                 if (remove && filename != "") {
00290                     if (unlink(filename.c_str()) < 0) {
00291                         // XXX what to do here?
00292                     }
00293                 }
00294 
00295                 // make sure the file is at least as large as the initial size
00296                 if (get_file_size() < sizeof(TValue) * m_size) {
00297                     if (ftruncate(m_fd, sizeof(TValue) * m_size) < 0) {
00298                         throw std::bad_alloc();
00299                     }
00300                 }
00301 
00302                 m_items = (TValue*) mmap(NULL, sizeof(TValue) * m_size, PROT_READ|PROT_WRITE, MAP_SHARED, m_fd, 0);
00303                 if (m_items == MAP_FAILED) {
00304                     throw std::bad_alloc();
00305                 }
00306             }
00307 
00308             ~Mmap() {
00309                 clear();
00310             }
00311 
00312             void set(uint64_t id, TValue value) {
00313                 if (id >= m_size) {
00314                     uint64_t new_size = id + size_increment;
00315 
00316                     // if there is a file backing this mmap and its smaller than needed, increase its size
00317                     if (m_fd >= 0 && get_file_size() < sizeof(TValue) * new_size) {
00318                         if (ftruncate(m_fd, sizeof(TValue) * new_size) < 0) {
00319                             throw std::bad_alloc();
00320                         }
00321                     }
00322 
00323                     m_items = (TValue*) mremap(m_items, sizeof(TValue) * m_size, sizeof(TValue) * new_size, MREMAP_MAYMOVE);
00324                     if (m_items == MAP_FAILED) {
00325                         throw std::bad_alloc();
00326                     }
00327                     m_size = new_size;
00328                 }
00329                 m_items[id] = value;
00330             }
00331 
00332             const TValue& operator[](uint64_t id) const {
00333                 return m_items[id];
00334             }
00335 
00336             uint64_t size() const {
00337                 return m_size;
00338             }
00339 
00340             uint64_t used_memory() const {
00341                 return m_size * sizeof(TValue);
00342             }
00343 
00344             void clear() {
00345                 munmap(m_items, sizeof(TValue) * m_size);
00346             }
00347 
00348         private:
00349 
00350             uint64_t m_size;
00351             int m_fd;
00352             TValue* m_items;
00353 
00355             uint64_t get_file_size() {
00356                 struct stat s;
00357                 if (fstat(m_fd, &s) < 0) {
00358                     throw std::bad_alloc();
00359                 }
00360                 return s.st_size;
00361             }
00362 
00363         }; // class Mmap
00364 
00365     } // namespace Storage
00366 
00367 } // namespace Osmium
00368 
00369 #endif // OSMIUM_HANDLER_STORE_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines