Osmium  0.1
include/osmium/input/pbf.hpp
Go to the documentation of this file.
00001 #ifndef OSMIUM_INPUT_PBF_HPP
00002 #define OSMIUM_INPUT_PBF_HPP
00003 
00004 /*
00005 
00006 Copyright 2011 Jochen Topf <jochen@topf.org> and others (see README).
00007 
00008 This file is part of Osmium (https://github.com/joto/osmium).
00009 
00010 Osmium is free software: you can redistribute it and/or modify it under the
00011 terms of the GNU Lesser General Public License or (at your option) the GNU
00012 General Public License as published by the Free Software Foundation, either
00013 version 3 of the Licenses, or (at your option) any later version.
00014 
00015 Osmium is distributed in the hope that it will be useful, but WITHOUT ANY
00016 WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
00017 PARTICULAR PURPOSE. See the GNU Lesser General Public License and the GNU
00018 General Public License for more details.
00019 
00020 You should have received a copy of the Licenses along with Osmium. If not, see
00021 <http://www.gnu.org/licenses/>.
00022 
00023 */
00024 
00025 #include <string>
00026 #include <zlib.h>
00027 
00028 #include <osmpbf/osmpbf.h>
00029 
00030 namespace Osmium {
00031 
00032     namespace Input {
00033 
00042         template <class THandler>
00043         class PBF : public Base<THandler> {
00044 
00045             typedef std::pair<const void*, size_t> array_t;
00046 
00047             unsigned char m_input_buffer[OSMPBF::max_uncompressed_blob_size];
00048             unsigned char m_unpack_buffer[OSMPBF::max_uncompressed_blob_size];
00049 
00050             OSMPBF::Blob           m_pbf_blob;
00051             OSMPBF::BlobHeader     m_pbf_blob_header;
00052             OSMPBF::PrimitiveBlock m_pbf_primitive_block;
00053 
00054             int64_t m_date_factor;
00055 
00056         public:
00057 
00064             PBF(OSMFile& file, THandler& handler) : Base<THandler>(file, handler) {
00065                 GOOGLE_PROTOBUF_VERIFY_VERSION;
00066             }
00067 
00074             void parse() {
00075                 try {
00076                     while (read_blob_header()) {
00077                         const array_t a = read_blob(m_pbf_blob_header.datasize());
00078 
00079                         if (m_pbf_blob_header.type() == "OSMData") {
00080                             if (!m_pbf_primitive_block.ParseFromArray(a.first, a.second)) {
00081                                 throw std::runtime_error("Failed to parse PrimitiveBlock.");
00082                             }
00083                             const OSMPBF::StringTable& stringtable = m_pbf_primitive_block.stringtable();
00084                             m_date_factor = m_pbf_primitive_block.date_granularity() / 1000;
00085                             for (int i=0; i < m_pbf_primitive_block.primitivegroup_size(); ++i) {
00086                                 parse_group(m_pbf_primitive_block.primitivegroup(i), stringtable);
00087                             }
00088                         } else if (m_pbf_blob_header.type() == "OSMHeader") {
00089                             OSMPBF::HeaderBlock pbf_header_block;
00090                             if (!pbf_header_block.ParseFromArray(a.first, a.second)) {
00091                                 throw std::runtime_error("Failed to parse HeaderBlock.");
00092                             }
00093 
00094                             bool has_historical_information_feature = false;
00095                             for (int i=0; i < pbf_header_block.required_features_size(); ++i) {
00096                                 const std::string& feature = pbf_header_block.required_features(i);
00097 
00098                                 if (feature == "OsmSchema-V0.6") continue;
00099                                 if (feature == "DenseNodes") continue;
00100                                 if (feature == "HistoricalInformation") {
00101                                     has_historical_information_feature = true;
00102                                     continue;
00103                                 }
00104 
00105                                 std::ostringstream errmsg;
00106                                 errmsg << "Required feature not supported: " << feature;
00107                                 throw std::runtime_error(errmsg.str());
00108                             }
00109 
00110                             const Osmium::OSMFile::FileType* expected_file_type = this->get_file().get_type();
00111                             if (expected_file_type == Osmium::OSMFile::FileType::OSM() && has_historical_information_feature) {
00112                                 throw Osmium::OSMFile::FileTypeOSMExpected();
00113                             }
00114                             if (expected_file_type == Osmium::OSMFile::FileType::History() && !has_historical_information_feature) {
00115                                 throw Osmium::OSMFile::FileTypeHistoryExpected();
00116                             }
00117 
00118                             if (pbf_header_block.has_bbox()) {
00119                                 const OSMPBF::HeaderBBox& bbox = pbf_header_block.bbox();
00120                                 this->meta().bounds().extend(Osmium::OSM::Position((double)bbox.left()  / OSMPBF::lonlat_resolution, (double)bbox.bottom() / OSMPBF::lonlat_resolution));
00121                                 this->meta().bounds().extend(Osmium::OSM::Position((double)bbox.right() / OSMPBF::lonlat_resolution, (double)bbox.top()    / OSMPBF::lonlat_resolution));
00122                             }
00123                         } else {
00124                             if (Osmium::debug()) {
00125                                 std::cerr << "Ignoring unknown blob type (" << m_pbf_blob_header.type().data() << ").\n";
00126                             }
00127                         }
00128                     }
00129                     this->call_after_and_before_on_handler(UNKNOWN);
00130                 } catch (Osmium::Input::StopReading) {
00131                     // if a handler says to stop reading, we do
00132                 }
00133                 this->call_final_on_handler();
00134             }
00135 
00136         private:
00137 
00147             void parse_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable) {
00148                 if (group.has_dense())  {
00149                     this->call_after_and_before_on_handler(NODE);
00150                     parse_dense_node_group(group, stringtable, &THandler::node);
00151                 } else if (group.ways_size() != 0) {
00152                     this->call_after_and_before_on_handler(WAY);
00153                     parse_way_group(group, stringtable, &THandler::way);
00154                 } else if (group.relations_size() != 0) {
00155                     this->call_after_and_before_on_handler(RELATION);
00156                     parse_relation_group(group, stringtable, &THandler::relation);
00157                 } else if (group.nodes_size() != 0) {
00158                     this->call_after_and_before_on_handler(NODE);
00159                     parse_node_group(group, stringtable, &THandler::node);
00160                 } else {
00161                     throw std::runtime_error("Group of unknown type.");
00162                 }
00163             }
00164 
00165             // empty specialization to optimize the case where the node() method on the handler is empty
00166             void parse_node_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/,
00167                                   void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Node const>&) const) {
00168             }
00169 
00170             template <typename T>
00171             void parse_node_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) {
00172                 int max_entity = group.nodes_size();
00173                 for (int entity=0; entity < max_entity; ++entity) {
00174                     Osmium::OSM::Node& node = this->prepare_node();
00175 
00176                     const OSMPBF::Node& pbf_node = group.nodes(entity);
00177 
00178                     node.id(pbf_node.id());
00179                     if (pbf_node.has_info()) {
00180                         node.version(pbf_node.info().version())
00181                         .changeset(pbf_node.info().changeset())
00182                         .timestamp(pbf_node.info().timestamp() * m_date_factor)
00183                         .uid(pbf_node.info().uid())
00184                         .user(stringtable.s(pbf_node.info().user_sid()).data());
00185                         if (pbf_node.info().has_visible()) {
00186                             node.visible(pbf_node.info().visible());
00187                         }
00188                     }
00189 
00190                     Osmium::OSM::TagList& tags = node.tags();
00191                     for (int tag=0; tag < pbf_node.keys_size(); ++tag) {
00192                         tags.add(stringtable.s( pbf_node.keys( tag ) ).data(),
00193                                  stringtable.s( pbf_node.vals( tag ) ).data());
00194                     }
00195 
00196                     node.position(Osmium::OSM::Position(
00197                                       ( (double) pbf_node.lon() * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lon_offset() ) / OSMPBF::lonlat_resolution,
00198                                       ( (double) pbf_node.lat() * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lat_offset() ) / OSMPBF::lonlat_resolution));
00199                     this->call_node_on_handler();
00200                 }
00201             }
00202 
00203             // empty specialization to optimize the case where the way() method on the handler is empty
00204             void parse_way_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/,
00205                                  void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Way const>&) const) {
00206             }
00207 
00208             template <typename T>
00209             void parse_way_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) {
00210                 int max_entity = group.ways_size();
00211                 for (int entity=0; entity < max_entity; ++entity) {
00212                     Osmium::OSM::Way& way = this->prepare_way();
00213 
00214                     const OSMPBF::Way& pbf_way = group.ways(entity);
00215 
00216                     way.id(pbf_way.id());
00217                     if (pbf_way.has_info()) {
00218                         way.version(pbf_way.info().version())
00219                         .changeset(pbf_way.info().changeset())
00220                         .timestamp(pbf_way.info().timestamp() * m_date_factor)
00221                         .uid(pbf_way.info().uid())
00222                         .user(stringtable.s(pbf_way.info().user_sid()).data());
00223                         if (pbf_way.info().has_visible()) {
00224                             way.visible(pbf_way.info().visible());
00225                         }
00226                     }
00227 
00228                     Osmium::OSM::TagList& tags = way.tags();
00229                     for (int tag=0; tag < pbf_way.keys_size(); ++tag) {
00230                         tags.add(stringtable.s( pbf_way.keys( tag ) ).data(),
00231                                  stringtable.s( pbf_way.vals( tag ) ).data());
00232                     }
00233 
00234                     uint64_t ref = 0;
00235                     for (int i=0; i < pbf_way.refs_size(); ++i) {
00236                         ref += pbf_way.refs(i);
00237                         way.add_node(ref);
00238                     }
00239 
00240                     this->call_way_on_handler();
00241                 }
00242             }
00243 
00244             // empty specialization to optimize the case where the relation() method on the handler is empty
00245             void parse_relation_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/,
00246                                       void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Relation const>&) const) {
00247             }
00248 
00249             template <typename T>
00250             void parse_relation_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) {
00251                 int max_entity = group.relations_size();
00252                 for (int entity=0; entity < max_entity; ++entity) {
00253                     Osmium::OSM::Relation& relation = this->prepare_relation();
00254 
00255                     const OSMPBF::Relation& pbf_relation = group.relations(entity);
00256 
00257                     relation.id(pbf_relation.id());
00258                     if (pbf_relation.has_info()) {
00259                         relation.version(pbf_relation.info().version())
00260                         .changeset(pbf_relation.info().changeset())
00261                         .timestamp(pbf_relation.info().timestamp() * m_date_factor)
00262                         .uid(pbf_relation.info().uid())
00263                         .user(stringtable.s(pbf_relation.info().user_sid()).data());
00264                         if (pbf_relation.info().has_visible()) {
00265                             relation.visible(pbf_relation.info().visible());
00266                         }
00267                     }
00268 
00269                     Osmium::OSM::TagList& tags = relation.tags();
00270                     for (int tag=0; tag < pbf_relation.keys_size(); ++tag) {
00271                         tags.add(stringtable.s( pbf_relation.keys(tag) ).data(),
00272                                  stringtable.s( pbf_relation.vals(tag) ).data());
00273                     }
00274 
00275                     uint64_t ref = 0;
00276                     for (int i=0; i < pbf_relation.types_size(); ++i) {
00277                         char type = 'x';
00278                         switch (pbf_relation.types(i)) {
00279                             case OSMPBF::Relation::NODE:
00280                                 type = 'n';
00281                                 break;
00282                             case OSMPBF::Relation::WAY:
00283                                 type = 'w';
00284                                 break;
00285                             case OSMPBF::Relation::RELATION:
00286                                 type = 'r';
00287                                 break;
00288                         }
00289                         ref += pbf_relation.memids(i);
00290                         relation.add_member(type, ref, stringtable.s( pbf_relation.roles_sid( i ) ).data());
00291                     }
00292 
00293                     this->call_relation_on_handler();
00294                 }
00295             }
00296 
00297             // empty specialization to optimize the case where the node() method on the handler is empty
00298             void parse_dense_node_group(const OSMPBF::PrimitiveGroup& /*group*/, const OSMPBF::StringTable& /*stringtable*/,
00299                                         void (Osmium::Handler::Base::*)(const shared_ptr<Osmium::OSM::Node const>&) const) {
00300             }
00301 
00302             template <typename T>
00303             void parse_dense_node_group(const OSMPBF::PrimitiveGroup& group, const OSMPBF::StringTable& stringtable, T) {
00304                 int64_t last_dense_id        = 0;
00305                 int64_t last_dense_latitude  = 0;
00306                 int64_t last_dense_longitude = 0;
00307                 int64_t last_dense_uid       = 0;
00308                 int64_t last_dense_user_sid  = 0;
00309                 int64_t last_dense_changeset = 0;
00310                 int64_t last_dense_timestamp = 0;
00311                 int     last_dense_tag       = 0;
00312 
00313                 const OSMPBF::DenseNodes& dense = group.dense();
00314                 int max_entity = dense.id_size();
00315                 for (int entity=0; entity < max_entity; ++entity) {
00316                     Osmium::OSM::Node& node = this->prepare_node();
00317 
00318                     last_dense_id += dense.id(entity);
00319                     node.id(last_dense_id);
00320 
00321                     if (dense.has_denseinfo()) {
00322                         last_dense_changeset += dense.denseinfo().changeset(entity);
00323                         last_dense_timestamp += dense.denseinfo().timestamp(entity);
00324                         last_dense_uid       += dense.denseinfo().uid(entity);
00325                         last_dense_user_sid  += dense.denseinfo().user_sid(entity);
00326 
00327                         node.version(dense.denseinfo().version(entity));
00328                         node.changeset(last_dense_changeset);
00329                         node.timestamp(last_dense_timestamp * m_date_factor);
00330                         node.uid(last_dense_uid);
00331                         node.user(stringtable.s(last_dense_user_sid).data());
00332 
00333                         if (dense.denseinfo().visible_size() > 0) {
00334                             node.visible(dense.denseinfo().visible(entity));
00335                         }
00336                     }
00337 
00338                     last_dense_latitude  += dense.lat(entity);
00339                     last_dense_longitude += dense.lon(entity);
00340                     node.position(Osmium::OSM::Position(
00341                                       ( (double) last_dense_longitude * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lon_offset() ) / OSMPBF::lonlat_resolution,
00342                                       ( (double) last_dense_latitude  * m_pbf_primitive_block.granularity() + m_pbf_primitive_block.lat_offset() ) / OSMPBF::lonlat_resolution));
00343 
00344                     while (last_dense_tag < dense.keys_vals_size()) {
00345                         int tag_key_pos = dense.keys_vals(last_dense_tag);
00346 
00347                         if (tag_key_pos == 0) {
00348                             last_dense_tag++;
00349                             break;
00350                         }
00351 
00352                         Osmium::OSM::TagList& tags = node.tags();
00353                         tags.add(stringtable.s(tag_key_pos).data(),
00354                                  stringtable.s(dense.keys_vals(last_dense_tag+1)).data());
00355 
00356                         last_dense_tag += 2;
00357                     }
00358 
00359                     this->call_node_on_handler();
00360                 }
00361             }
00362 
00366             int convert_from_network_byte_order(unsigned char data[4]) {
00367                 return (data[0] << 24) | (data[1] << 16) | (data[2] << 8) | data[3];
00368             }
00369 
00375             bool read_blob_header() {
00376                 unsigned char size_in_network_byte_order[4];
00377                 int offset = 0;
00378                 while (offset < static_cast<int>(sizeof(size_in_network_byte_order))) {
00379                     int nread = read(this->get_fd(), size_in_network_byte_order + offset, sizeof(size_in_network_byte_order) - offset);
00380                     if (nread < 0) {
00381                         throw std::runtime_error("read error");
00382                     } else if (nread == 0) {
00383                         return false; // EOF
00384                     }
00385                     offset += nread;
00386                 }
00387 
00388                 const int size = convert_from_network_byte_order(size_in_network_byte_order);
00389                 if (size > OSMPBF::max_blob_header_size || size < 0) {
00390                     std::ostringstream errmsg;
00391                     errmsg << "BlobHeader size invalid:" << size;
00392                     throw std::runtime_error(errmsg.str());
00393                 }
00394 
00395                 offset = 0;
00396                 while (offset < size) {
00397                     int nread = read(this->get_fd(), m_input_buffer + offset, size - offset);
00398                     if (nread < 1) {
00399                         throw std::runtime_error("failed to read BlobHeader");
00400                     }
00401                     offset += nread;
00402                 }
00403 
00404                 if (!m_pbf_blob_header.ParseFromArray(m_input_buffer, size)) {
00405                     throw std::runtime_error("failed to parse BlobHeader");
00406                 }
00407                 return true;
00408             }
00409 
00413             array_t read_blob(const int size) {
00414                 if (size < 0 || size > OSMPBF::max_uncompressed_blob_size) {
00415                     std::ostringstream errmsg;
00416                     errmsg << "invalid blob size: " << size;
00417                     throw std::runtime_error(errmsg.str());
00418                 }
00419                 int offset = 0;
00420                 while (offset < size) {
00421                     int nread = read(this->get_fd(), m_input_buffer + offset, size - offset);
00422                     if (nread < 1) {
00423                         throw std::runtime_error("failed to read blob");
00424                     }
00425                     offset += nread;
00426                 }
00427                 if (!m_pbf_blob.ParseFromArray(m_input_buffer, size)) {
00428                     throw std::runtime_error("failed to parse blob");
00429                 }
00430 
00431                 if (m_pbf_blob.has_raw()) {
00432                     return array_t(m_pbf_blob.raw().data(), m_pbf_blob.raw().size());
00433                 } else if (m_pbf_blob.has_zlib_data()) {
00434                     unsigned long raw_size = m_pbf_blob.raw_size();
00435                     assert(raw_size <= static_cast<unsigned long>(OSMPBF::max_uncompressed_blob_size));
00436                     if (uncompress(m_unpack_buffer, &raw_size, reinterpret_cast<const unsigned char*>(m_pbf_blob.zlib_data().data()), m_pbf_blob.zlib_data().size()) != Z_OK || m_pbf_blob.raw_size() != static_cast<long>(raw_size)) {
00437                         throw std::runtime_error("zlib error");
00438                     }
00439                     return array_t(m_unpack_buffer, raw_size);
00440                 } else if (m_pbf_blob.has_lzma_data()) {
00441                     throw std::runtime_error("lzma blobs not implemented");
00442                 } else {
00443                     throw std::runtime_error("Blob contains no data");
00444                 }
00445             }
00446 
00447         }; // class PBF
00448 
00449     } // namespace Input
00450 
00451 } // namespace Osmium
00452 
00453 #endif // OSMIUM_INPUT_PBF_HPP
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Defines