Osmium
0.1
|
00001 #ifndef OSMIUM_HANDLER_STORE_HPP 00002 #define OSMIUM_HANDLER_STORE_HPP 00003 00004 /* 00005 00006 Copyright 2011 Jochen Topf <jochen@topf.org> and others (see README). 00007 00008 This file is part of Osmium (https://github.com/joto/osmium). 00009 00010 Osmium is free software: you can redistribute it and/or modify it under the 00011 terms of the GNU Lesser General Public License or (at your option) the GNU 00012 General Public License as published by the Free Software Foundation, either 00013 version 3 of the Licenses, or (at your option) any later version. 00014 00015 Osmium is distributed in the hope that it will be useful, but WITHOUT ANY 00016 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 00017 PARTICULAR PURPOSE. See the GNU Lesser General Public License and the GNU 00018 General Public License for more details. 00019 00020 You should have received a copy of the Licenses along with Osmium. If not, see 00021 <http://www.gnu.org/licenses/>. 00022 00023 */ 00024 00025 #include <stdexcept> 00026 #include <cstdlib> 00027 #include <google/sparsetable> 00028 #include <sys/mman.h> 00029 #include <sys/types.h> 00030 #include <unistd.h> 00031 #include <sys/stat.h> 00032 #include <fcntl.h> 00033 #include <boost/utility.hpp> 00034 00035 namespace Osmium { 00036 00040 namespace Storage { 00041 00057 template <typename TValue> 00058 class ById : boost::noncopyable { 00059 00060 public: 00061 00062 virtual ~ById() { 00063 } 00064 00066 typedef TValue value_type; 00067 00069 virtual void set(uint64_t id, TValue value) = 0; 00070 00072 virtual const TValue& operator[](uint64_t id) const = 0; 00073 00080 virtual uint64_t size() const = 0; 00081 00089 virtual uint64_t used_memory() const = 0; 00090 00095 virtual void clear() = 0; 00096 00097 }; 00098 00118 template <typename TValue> 00119 class FixedArray : public ById<TValue> { 00120 00121 public: 00122 00129 FixedArray(const uint64_t max_id) : ById<TValue>(), m_size(max_id) { 00130 m_items = (TValue*) malloc(sizeof(TValue) * max_id); 00131 if (!m_items) { 00132 throw std::bad_alloc(); 00133 } 00134 } 00135 00136 ~FixedArray() { 00137 clear(); 00138 } 00139 00140 void set(uint64_t id, TValue value) { 00141 m_items[id] = value; 00142 } 00143 00144 const TValue& operator[](uint64_t id) const { 00145 return m_items[id]; 00146 } 00147 00148 uint64_t size() const { 00149 return m_size; 00150 } 00151 00152 uint64_t used_memory() const { 00153 return m_size * sizeof(TValue); 00154 } 00155 00156 void clear() { 00157 free(m_items); 00158 m_items = NULL; 00159 } 00160 00161 private: 00162 00163 TValue* m_items; 00164 00165 uint64_t m_size; 00166 00167 }; // class FixedArray 00168 00178 template <typename TValue> 00179 class SparseTable : public ById<TValue> { 00180 00181 public: 00182 00190 SparseTable(const uint64_t grow_size=10000) 00191 : ById<TValue>(), 00192 m_grow_size(grow_size), 00193 m_items(grow_size) { 00194 } 00195 00196 ~SparseTable() { 00197 } 00198 00199 void set(uint64_t id, TValue value) { 00200 if (id >= m_items.size()) { 00201 m_items.resize(id + m_grow_size); 00202 } 00203 m_items[id] = value; 00204 } 00205 00206 const TValue& operator[](uint64_t id) const { 00207 return m_items[id]; 00208 } 00209 00210 uint64_t size() const { 00211 return m_items.size(); 00212 } 00213 00214 uint64_t used_memory() const { 00215 // unused items use 1 bit, used items sizeof(TValue) bytes 00216 // http://google-sparsehash.googlecode.com/svn/trunk/doc/sparsetable.html 00217 return (m_items.size() / 8) + (m_items.num_nonempty() * sizeof(TValue)); 00218 } 00219 00220 void clear() { 00221 m_items.clear(); 00222 } 00223 00224 private: 00225 00226 uint64_t m_grow_size; 00227 00228 google::sparsetable<TValue> m_items; 00229 00230 }; // class SparseTable 00231 00246 template <typename TValue> 00247 class Mmap : public ById<TValue> { 00248 00249 public: 00250 00251 static const uint64_t size_increment = 10 * 1024 * 1024; 00252 00257 Mmap() : ById<TValue>(), m_size(size_increment), m_fd(-1) { 00258 m_items = (TValue*) mmap(NULL, sizeof(TValue) * m_size, PROT_READ|PROT_WRITE, MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); 00259 if (m_items == MAP_FAILED) { 00260 throw std::bad_alloc(); 00261 } 00262 } 00263 00272 Mmap(std::string& filename, bool remove=true) : ById<TValue>(), m_size(1) { 00273 if (filename == "") { 00274 FILE* file = tmpfile(); 00275 if (!file) { 00276 throw std::bad_alloc(); 00277 } 00278 m_fd = fileno(file); 00279 } else { 00280 m_fd = open(filename.c_str(), O_RDWR | O_CREAT, 0600); 00281 } 00282 00283 if (m_fd < 0) { 00284 throw std::bad_alloc(); 00285 } 00286 00287 // now that the file is open we can immediately remove it 00288 // (temporary files are always removed) 00289 if (remove && filename != "") { 00290 if (unlink(filename.c_str()) < 0) { 00291 // XXX what to do here? 00292 } 00293 } 00294 00295 // make sure the file is at least as large as the initial size 00296 if (get_file_size() < sizeof(TValue) * m_size) { 00297 if (ftruncate(m_fd, sizeof(TValue) * m_size) < 0) { 00298 throw std::bad_alloc(); 00299 } 00300 } 00301 00302 m_items = (TValue*) mmap(NULL, sizeof(TValue) * m_size, PROT_READ|PROT_WRITE, MAP_SHARED, m_fd, 0); 00303 if (m_items == MAP_FAILED) { 00304 throw std::bad_alloc(); 00305 } 00306 } 00307 00308 ~Mmap() { 00309 clear(); 00310 } 00311 00312 void set(uint64_t id, TValue value) { 00313 if (id >= m_size) { 00314 uint64_t new_size = id + size_increment; 00315 00316 // if there is a file backing this mmap and its smaller than needed, increase its size 00317 if (m_fd >= 0 && get_file_size() < sizeof(TValue) * new_size) { 00318 if (ftruncate(m_fd, sizeof(TValue) * new_size) < 0) { 00319 throw std::bad_alloc(); 00320 } 00321 } 00322 00323 m_items = (TValue*) mremap(m_items, sizeof(TValue) * m_size, sizeof(TValue) * new_size, MREMAP_MAYMOVE); 00324 if (m_items == MAP_FAILED) { 00325 throw std::bad_alloc(); 00326 } 00327 m_size = new_size; 00328 } 00329 m_items[id] = value; 00330 } 00331 00332 const TValue& operator[](uint64_t id) const { 00333 return m_items[id]; 00334 } 00335 00336 uint64_t size() const { 00337 return m_size; 00338 } 00339 00340 uint64_t used_memory() const { 00341 return m_size * sizeof(TValue); 00342 } 00343 00344 void clear() { 00345 munmap(m_items, sizeof(TValue) * m_size); 00346 } 00347 00348 private: 00349 00350 uint64_t m_size; 00351 int m_fd; 00352 TValue* m_items; 00353 00355 uint64_t get_file_size() { 00356 struct stat s; 00357 if (fstat(m_fd, &s) < 0) { 00358 throw std::bad_alloc(); 00359 } 00360 return s.st_size; 00361 } 00362 00363 }; // class Mmap 00364 00365 } // namespace Storage 00366 00367 } // namespace Osmium 00368 00369 #endif // OSMIUM_HANDLER_STORE_HPP