Osmium
0.1
|
00001 #ifndef OSMIUM_INPUT_PBF_HPP 00002 #define OSMIUM_INPUT_PBF_HPP 00003 00004 /* 00005 00006 Copyright 2011 Jochen Topf <jochen@topf.org> and others (see README). 00007 00008 This file is part of Osmium (https://github.com/joto/osmium). 00009 00010 Osmium is free software: you can redistribute it and/or modify it under the 00011 terms of the GNU Lesser General Public License or (at your option) the GNU 00012 General Public License as published by the Free Software Foundation, either 00013 version 3 of the Licenses, or (at your option) any later version. 00014 00015 Osmium is distributed in the hope that it will be useful, but WITHOUT ANY 00016 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A 00017 PARTICULAR PURPOSE. See the GNU Lesser General Public License and the GNU 00018 General Public License for more details. 00019 00020 You should have received a copy of the Licenses along with Osmium. If not, see 00021 <http://www.gnu.org/licenses/>. 00022 00023 */ 00024 00025 #include <string> 00026 #include <zlib.h> 00027 00028 #include <osmpbf/osmpbf.h> 00029 00030 namespace Osmium { 00031 00032 namespace Input { 00033 00042 template <class THandler> 00043 class PBF : public Base<THandler> { 00044 00045 typedef std::pair<const void*, size_t> array_t; 00046 00047 unsigned char m_input_buffer[OSMPBF::max_uncompressed_blob_size]; 00048 unsigned char m_unpack_buffer[OSMPBF::max_uncompressed_blob_size]; 00049 00050 OSMPBF::Blob m_pbf_blob; 00051 OSMPBF::BlobHeader m_pbf_blob_header; 00052 OSMPBF::PrimitiveBlock m_pbf_primitive_block; 00053 00054 int64_t m_date_factor; 00055 00056 public: 00057 00064 PBF(OSMFile& file, THandler& handler) : Base<THandler>(file, handler) { 00065 GOOGLE_PROTOBUF_VERIFY_VERSION; 00066 } 00067 00074 void parse() { 00075 try { 00076 while (read_blob_header()) { 00077 const array_t a = read_blob(m_pbf_blob_header.datasize()); 00078 00079 if (m_pbf_blob_header.type() == "OSMData") { 00080 if (!m_pbf_primitive_block.ParseFromArray(a.first, a.second)) { 00081 throw std::runtime_error("Failed to parse PrimitiveBlock."); 00082 } 00083 const OSMPBF::StringTable& stringtable = m_pbf_primitive_block.stringtable(); 00084 m_date_factor = m_pbf_primitive_block.date_granularity() / 1000; 00085 for (int i=0; i < m_pbf_primitive_block.primitivegroup_size(); ++i) { 00086 parse_group(m_pbf_primitive_block.primitivegroup(i), stringtable); 00087 } 00088 } else if (m_pbf_blob_header.type() == "OSMHeader") { 00089 OSMPBF::HeaderBlock pbf_header_block; 00090 if (!pbf_header_block.ParseFromArray(a.first, a.second)) { 00091 throw std::runtime_error("Failed to parse HeaderBlock."); 00092 } 00093 00094 bool has_historical_information_feature = false; 00095 for (int i=0; i < pbf_header_block.required_features_size(); ++i) { 00096 const std::string& feature = pbf_header_block.required_features(i); 00097 00098 if (feature == "OsmSchema-V0.6") continue; 00099 if (feature == "DenseNodes") continue; 00100 if (feature == "HistoricalInformation") { 00101 has_historical_information_feature = true; 00102 continue; 00103 } 00104 00105 std::ostringstream errmsg; 00106 errmsg << "Required feature not supported: " << feature; 00107 throw std::runtime_error(errmsg.str()); 00108 } 00109 00110 const Osmium::OSMFile::FileType* expected_file_type = this->get_file().get_type(); 00111 if (expected_file_type == Osmium::OSMFile::FileType::OSM() && has_historical_information_feature) { 00112 throw Osmium::OSMFile::FileTypeOSMExpected(); 00113 } 00114 if (expected_file_type == Osmium::OSMFile::FileType::History() && !has_historical_information_feature) { 00115 throw Osmium::OSMFile::FileTypeHistoryExpected(); 00116 } 00117 00118 if (pbf_header_block.has_bbox()) { 00119 const OSMPBF::HeaderBBox& bbox = pbf_header_block.bbox(); 00120 this->meta().bounds().extend(Osmium::OSM::Position((double)bbox.left() / OSMPBF::lonlat_resolution, (double)bbox.bottom() / OSMPBF::lonlat_resolution)); 00121 this->meta().bounds().extend(Osmium::OSM::Position((double)bbox.right() / OSMPBF::lonlat_resolution, (double)bbox.top() / OSMPBF::lonlat_resolution)); 00122 } 00123 } else { 00124 if (Osmium::debug()) { 00125 std::cerr << "Ignoring unknown blob type (" << m_pbf_blob_header.type().data() << ").\n"; 00126 } 00127 } 00128 } 00129 this->call_after_and_before_on_handler(UNKNOWN); 00130 } catch (Osmium::Input::StopReading) { 00131 // if a handler says to stop reading, we do 00132 } 00133 this->call_final_on_handler(); 00134 } 00135 00136 private: 00137 00147 void parse_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable) { 00148 if (group.has_dense()) { 00149 this->call_after_and_before_on_handler(NODE); 00150 parse_dense_node_group(group, stringtable, &THandler::node); 00151 } else if (group.ways_size() != 0) { 00152 this->call_after_and_before_on_handler(WAY); 00153 parse_way_group(group, stringtable, &THandler::way); 00154 } else if (group.relations_size() != 0) { 00155 this->call_after_and_before_on_handler(RELATION); 00156 parse_relation_group(group, stringtable, &THandler::relation); 00157 } else if (group.nodes_size() != 0) { 00158 this->call_after_and_before_on_handler(NODE); 00159 parse_node_group(group, stringtable, &THandler::node); 00160 } else { 00161 throw std::runtime_error("Group of unknown type."); 00162 } 00163 } 00164 00165 // empty specialization to optimize the case where the node() method on the handler is empty 00166 void parse_node_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/, 00167 void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Node const>&) const) { 00168 } 00169 00170 template <typename T> 00171 void parse_node_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) { 00172 int max_entity = group.nodes_size(); 00173 for (int entity=0; entity < max_entity; ++entity) { 00174 Osmium::OSM::Node& node = this->prepare_node(); 00175 00176 const OSMPBF::Node& pbf_node = group.nodes(entity); 00177 00178 node.id(pbf_node.id()); 00179 if (pbf_node.has_info()) { 00180 node.version(pbf_node.info().version()) 00181 .changeset(pbf_node.info().changeset()) 00182 .timestamp(pbf_node.info().timestamp() * m_date_factor) 00183 .uid(pbf_node.info().uid()) 00184 .user(stringtable.s(pbf_node.info().user_sid()).data()); 00185 if (pbf_node.info().has_visible()) { 00186 node.visible(pbf_node.info().visible()); 00187 } 00188 } 00189 00190 Osmium::OSM::TagList& tags = node.tags(); 00191 for (int tag=0; tag < pbf_node.keys_size(); ++tag) { 00192 tags.add(stringtable.s( pbf_node.keys( tag ) ).data(), 00193 stringtable.s( pbf_node.vals( tag ) ).data()); 00194 } 00195 00196 node.position(Osmium::OSM::Position( 00197 ( (double) pbf_node.lon() * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lon_offset() ) / OSMPBF::lonlat_resolution, 00198 ( (double) pbf_node.lat() * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lat_offset() ) / OSMPBF::lonlat_resolution)); 00199 this->call_node_on_handler(); 00200 } 00201 } 00202 00203 // empty specialization to optimize the case where the way() method on the handler is empty 00204 void parse_way_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/, 00205 void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Way const>&) const) { 00206 } 00207 00208 template <typename T> 00209 void parse_way_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) { 00210 int max_entity = group.ways_size(); 00211 for (int entity=0; entity < max_entity; ++entity) { 00212 Osmium::OSM::Way& way = this->prepare_way(); 00213 00214 const OSMPBF::Way& pbf_way = group.ways(entity); 00215 00216 way.id(pbf_way.id()); 00217 if (pbf_way.has_info()) { 00218 way.version(pbf_way.info().version()) 00219 .changeset(pbf_way.info().changeset()) 00220 .timestamp(pbf_way.info().timestamp() * m_date_factor) 00221 .uid(pbf_way.info().uid()) 00222 .user(stringtable.s(pbf_way.info().user_sid()).data()); 00223 if (pbf_way.info().has_visible()) { 00224 way.visible(pbf_way.info().visible()); 00225 } 00226 } 00227 00228 Osmium::OSM::TagList& tags = way.tags(); 00229 for (int tag=0; tag < pbf_way.keys_size(); ++tag) { 00230 tags.add(stringtable.s( pbf_way.keys( tag ) ).data(), 00231 stringtable.s( pbf_way.vals( tag ) ).data()); 00232 } 00233 00234 uint64_t ref = 0; 00235 for (int i=0; i < pbf_way.refs_size(); ++i) { 00236 ref += pbf_way.refs(i); 00237 way.add_node(ref); 00238 } 00239 00240 this->call_way_on_handler(); 00241 } 00242 } 00243 00244 // empty specialization to optimize the case where the relation() method on the handler is empty 00245 void parse_relation_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/, 00246 void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Relation const>&) const) { 00247 } 00248 00249 template <typename T> 00250 void parse_relation_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) { 00251 int max_entity = group.relations_size(); 00252 for (int entity=0; entity < max_entity; ++entity) { 00253 Osmium::OSM::Relation& relation = this->prepare_relation(); 00254 00255 const OSMPBF::Relation& pbf_relation = group.relations(entity); 00256 00257 relation.id(pbf_relation.id()); 00258 if (pbf_relation.has_info()) { 00259 relation.version(pbf_relation.info().version()) 00260 .changeset(pbf_relation.info().changeset()) 00261 .timestamp(pbf_relation.info().timestamp() * m_date_factor) 00262 .uid(pbf_relation.info().uid()) 00263 .user(stringtable.s(pbf_relation.info().user_sid()).data()); 00264 if (pbf_relation.info().has_visible()) { 00265 relation.visible(pbf_relation.info().visible()); 00266 } 00267 } 00268 00269 Osmium::OSM::TagList& tags = relation.tags(); 00270 for (int tag=0; tag < pbf_relation.keys_size(); ++tag) { 00271 tags.add(stringtable.s( pbf_relation.keys(tag) ).data(), 00272 stringtable.s( pbf_relation.vals(tag) ).data()); 00273 } 00274 00275 uint64_t ref = 0; 00276 for (int i=0; i < pbf_relation.types_size(); ++i) { 00277 char type = 'x'; 00278 switch (pbf_relation.types(i)) { 00279 case OSMPBF::Relation::NODE: 00280 type = 'n'; 00281 break; 00282 case OSMPBF::Relation::WAY: 00283 type = 'w'; 00284 break; 00285 case OSMPBF::Relation::RELATION: 00286 type = 'r'; 00287 break; 00288 } 00289 ref += pbf_relation.memids(i); 00290 relation.add_member(type, ref, stringtable.s( pbf_relation.roles_sid( i ) ).data()); 00291 } 00292 00293 this->call_relation_on_handler(); 00294 } 00295 } 00296 00297 // empty specialization to optimize the case where the node() method on the handler is empty 00298 void parse_dense_node_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/, 00299 void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Node const>&) const) { 00300 } 00301 00302 template <typename T> 00303 void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) { 00304 int64_t last_dense_id = 0; 00305 int64_t last_dense_latitude = 0; 00306 int64_t last_dense_longitude = 0; 00307 int64_t last_dense_uid = 0; 00308 int64_t last_dense_user_sid = 0; 00309 int64_t last_dense_changeset = 0; 00310 int64_t last_dense_timestamp = 0; 00311 int last_dense_tag = 0; 00312 00313 const OSMPBF::DenseNodes& dense = group.dense(); 00314 int max_entity = dense.id_size(); 00315 for (int entity=0; entity < max_entity; ++entity) { 00316 Osmium::OSM::Node& node = this->prepare_node(); 00317 00318 last_dense_id += dense.id(entity); 00319 node.id(last_dense_id); 00320 00321 if (dense.has_denseinfo()) { 00322 last_dense_changeset += dense.denseinfo().changeset(entity); 00323 last_dense_timestamp += dense.denseinfo().timestamp(entity); 00324 last_dense_uid += dense.denseinfo().uid(entity); 00325 last_dense_user_sid += dense.denseinfo().user_sid(entity); 00326 00327 node.version(dense.denseinfo().version(entity)); 00328 node.changeset(last_dense_changeset); 00329 node.timestamp(last_dense_timestamp * m_date_factor); 00330 node.uid(last_dense_uid); 00331 node.user(stringtable.s(last_dense_user_sid).data()); 00332 00333 if (dense.denseinfo().visible_size() > 0) { 00334 node.visible(dense.denseinfo().visible(entity)); 00335 } 00336 } 00337 00338 last_dense_latitude += dense.lat(entity); 00339 last_dense_longitude += dense.lon(entity); 00340 node.position(Osmium::OSM::Position( 00341 ( (double) last_dense_longitude * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lon_offset() ) / OSMPBF::lonlat_resolution, 00342 ( (double) last_dense_latitude * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lat_offset() ) / OSMPBF::lonlat_resolution)); 00343 00344 while (last_dense_tag < dense.keys_vals_size()) { 00345 int tag_key_pos = dense.keys_vals(last_dense_tag); 00346 00347 if (tag_key_pos == 0) { 00348 last_dense_tag++; 00349 break; 00350 } 00351 00352 Osmium::OSM::TagList& tags = node.tags(); 00353 tags.add(stringtable.s(tag_key_pos).data(), 00354 stringtable.s(dense.keys_vals(last_dense_tag+1)).data()); 00355 00356 last_dense_tag += 2; 00357 } 00358 00359 this->call_node_on_handler(); 00360 } 00361 } 00362 00366 int convert_from_network_byte_order(unsigned char data[4]) { 00367 return (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3]; 00368 } 00369 00375 bool read_blob_header() { 00376 unsigned char size_in_network_byte_order[4]; 00377 int offset = 0; 00378 while (offset < static_cast<int>(sizeof(size_in_network_byte_order))) { 00379 int nread = read(this->get_fd(), size_in_network_byte_order + offset, sizeof(size_in_network_byte_order) - offset); 00380 if (nread < 0) { 00381 throw std::runtime_error("read error"); 00382 } else if (nread == 0) { 00383 return false; // EOF 00384 } 00385 offset += nread; 00386 } 00387 00388 const int size = convert_from_network_byte_order(size_in_network_byte_order); 00389 if (size > OSMPBF::max_blob_header_size || size < 0) { 00390 std::ostringstream errmsg; 00391 errmsg << "BlobHeader size invalid:" << size; 00392 throw std::runtime_error(errmsg.str()); 00393 } 00394 00395 offset = 0; 00396 while (offset < size) { 00397 int nread = read(this->get_fd(), m_input_buffer + offset, size - offset); 00398 if (nread < 1) { 00399 throw std::runtime_error("failed to read BlobHeader"); 00400 } 00401 offset += nread; 00402 } 00403 00404 if (!m_pbf_blob_header.ParseFromArray(m_input_buffer, size)) { 00405 throw std::runtime_error("failed to parse BlobHeader"); 00406 } 00407 return true; 00408 } 00409 00413 array_t read_blob(const int size) { 00414 if (size < 0 || size > OSMPBF::max_uncompressed_blob_size) { 00415 std::ostringstream errmsg; 00416 errmsg << "invalid blob size: " << size; 00417 throw std::runtime_error(errmsg.str()); 00418 } 00419 int offset = 0; 00420 while (offset < size) { 00421 int nread = read(this->get_fd(), m_input_buffer + offset, size - offset); 00422 if (nread < 1) { 00423 throw std::runtime_error("failed to read blob"); 00424 } 00425 offset += nread; 00426 } 00427 if (!m_pbf_blob.ParseFromArray(m_input_buffer, size)) { 00428 throw std::runtime_error("failed to parse blob"); 00429 } 00430 00431 if (m_pbf_blob.has_raw()) { 00432 return array_t(m_pbf_blob.raw().data(), m_pbf_blob.raw().size()); 00433 } else if (m_pbf_blob.has_zlib_data()) { 00434 unsigned long raw_size = m_pbf_blob.raw_size(); 00435 assert(raw_size <= static_cast<unsigned long>(OSMPBF::max_uncompressed_blob_size)); 00436 if (uncompress(m_unpack_buffer, &raw_size, reinterpret_cast<const unsigned char*>(m_pbf_blob.zlib_data().data()), m_pbf_blob.zlib_data().size()) != Z_OK || m_pbf_blob.raw_size() != static_cast<long>(raw_size)) { 00437 throw std::runtime_error("zlib error"); 00438 } 00439 return array_t(m_unpack_buffer, raw_size); 00440 } else if (m_pbf_blob.has_lzma_data()) { 00441 throw std::runtime_error("lzma blobs not implemented"); 00442 } else { 00443 throw std::runtime_error("Blob contains no data"); 00444 } 00445 } 00446 00447 }; // class PBF 00448 00449 } // namespace Input 00450 00451 } // namespace Osmium 00452 00453 #endif // OSMIUM_INPUT_PBF_HPP