json-flat: support relations

.dockerignore

@@ -0,0 +1,8 @@
+.git
+example
+npm
+LICENSE
+*.md
+*.pbf
+*.mask
+*.js

Dockerfile

@@ -0,0 +1,36 @@
+# base image
+FROM ubuntu:latest
+
+# configure env
+ENV DEBIAN_FRONTEND 'noninteractive'
+
+# update apt, install core apt dependencies and delete the apt-cache
+# note: this is done in one command in order to keep down the size of intermediate containers
+RUN apt update && \
+    apt install -y locales git-core sqlite3 libsqlite3-mod-spatialite golang && \
+    rm -rf /var/lib/apt/lists/*
+
+# configure locale
+RUN locale-gen 'en_US.UTF-8'
+ENV LANG 'en_US.UTF-8'
+ENV LANGUAGE 'en_US:en'
+ENV LC_ALL 'en_US.UTF-8'
+
+# configure git
+RUN git config --global 'user.email' 'null@null.com'
+RUN git config --global 'user.name' 'Missinglink PBF'
+
+# set GOPATH
+ENV GOPATH='/tmp/go'
+
+# change working dir
+WORKDIR "$GOPATH/src/github.com/missinglink/pbf"
+
+# copy files
+COPY . "$GOPATH/src/github.com/missinglink/pbf"
+
+# fetch dependencies
+RUN go get
+
+# build binary
+RUN go build

command/bitmask_custom.go

@@ -11,6 +11,11 @@ import (
 	"github.com/codegangsta/cli"
 )
 
+// @todo: depending on which elements are defined in the config, we can
+// skip over large chunks of the file. eg: if the config only contains
+// relation patterns then it will be much faster if we skip over the
+// nodes and ways on the first pass
+
 // BitmaskCustom cli command
 func BitmaskCustom(c *cli.Context) error {
 
@@ -22,7 +27,7 @@ func BitmaskCustom(c *cli.Context) error {
 	}
 
 	// create parser
-	parser := parser.NewParser(c.Args()[0])
+	p := parser.NewParser(c.Args()[0])
 
 	// don't clobber existing bitmask file
 	if _, err := os.Stat(c.Args()[1]); err == nil {
@@ -60,7 +65,32 @@ func BitmaskCustom(c *cli.Context) error {
 	defer handle.Masks.WriteToFile(c.Args()[1])
 
 	// Parse will block until it is done or an error occurs.
-	parser.Parse(handle)
+	p.Parse(handle)
+
+	// --- second pass ---
+	// run parser a second time, skipping the nodes
+	// @todo: skip relations on the second pass too
+
+	// if we are not interested in relations, exit now
+	if 0 == len(config.RelationPatterns) {
+		return nil
+	}
+
+	// disable indexing
+	os.Unsetenv("INDEXING")
+
+	// create a new parser
+	p2 := parser.NewParser(c.Args()[0])
+
+	// find first way offset
+	offset, err := p2.GetDecoder().Index.FirstOffsetOfType("way")
+	if nil != err {
+		log.Printf("target type: %s not found in file\n", "way")
+		os.Exit(1)
+	}
+
+	// Parse will block until it is done or an error occurs.
+	p2.ParseFrom(handle, offset)
 
 	return nil
 }

command/json_flat.go

@@ -9,6 +9,7 @@ import (
 	"github.com/missinglink/pbf/lib"
 	"github.com/missinglink/pbf/parser"
 	"github.com/missinglink/pbf/proxy"
+	"github.com/missinglink/pbf/spatialite"
 
 	"github.com/codegangsta/cli"
 )
@@ -50,6 +51,7 @@ func JSONFlat(c *cli.Context) error {
 	var handle = &handler.DenormalizedJSON{
 		Conn:            conn,
 		Writer:          lib.NewBufferedWriter(),
+		Spatialite:      &spatialite.Connection{},
 		ComputeCentroid: c.BoolT("centroid"),
 		ComputeGeohash:  c.Bool("geohash"),
 		ExportLatLons:   c.Bool("vertices"),
@@ -58,17 +60,26 @@ func JSONFlat(c *cli.Context) error {
 	// close the writer routine and flush
 	defer handle.Writer.Close()
 
+	// open the spatialite connection
+	handle.Spatialite.Open(":memory:")
+
+	// close the spatialite connection
+	defer handle.Spatialite.Close()
+
 	// create db writer routine
-	writer := leveldb.NewCoordWriter(conn)
+	writer := leveldb.NewWriter(conn)
 
 	// ensure all node refs are written to disk before starting on the ways
 	dec := p.GetDecoder()
 	dec.Triggers = []func(int, uint64){
 		func(i int, offset uint64) {
-			if 0 == i {
-				log.Println("writer close")
-				writer.Close()
-				log.Println("writer closed")
+			switch i {
+			case 0:
+				writer.NodeQueue.Close()
+				log.Println("nodes written")
+			case 1:
+				writer.WayQueue.Close()
+				log.Println("ways written")
 			}
 		},
 	}

command/store_noderefs.go

@@ -46,16 +46,19 @@ func StoreNodeRefs(c *cli.Context) error {
 	defer conn.Close()
 
 	// create db writer routine
-	writer := leveldb.NewCoordWriter(conn)
+	writer := leveldb.NewWriter(conn)
 
 	// ensure all node refs are written to disk before starting on the ways
 	dec := parser.GetDecoder()
 	dec.Triggers = []func(int, uint64){
 		func(i int, offset uint64) {
-			if 0 == i {
-				log.Println("writer close")
-				writer.Close()
-				log.Println("writer closed")
+			switch i {
+			case 0:
+				writer.NodeQueue.Close()
+				log.Println("nodes written")
+			case 1:
+				writer.WayQueue.Close()
+				log.Println("ways written")
 			}
 		},
 	}

handler/bitmask_custom.go

@@ -14,30 +14,97 @@ type BitmaskCustom struct {
 
 // ReadNode - called once per node
 func (b *BitmaskCustom) ReadNode(item gosmparse.Node) {
+
+	// nodes in feature list
 	if b.Features.MatchNode(item) {
 		b.Masks.Nodes.Insert(item.ID)
 	}
 }
 
 // ReadWay - called once per way
 func (b *BitmaskCustom) ReadWay(item gosmparse.Way) {
+
+	// ways in feature list
 	if b.Features.MatchWay(item) {
+
 		b.Masks.Ways.Insert(item.ID)
 
 		// insert dependents in mask
 		for _, ref := range item.NodeIDs {
 			b.Masks.WayRefs.Insert(ref)
 		}
 	}
+
+	// ways belonging to a relation
+	if b.Masks.RelWays.Has(item.ID) {
+
+		// insert dependents in mask
+		for _, ref := range item.NodeIDs {
+			b.Masks.RelNodes.Insert(ref)
+		}
+	}
 }
 
 // ReadRelation - called once per relation
 func (b *BitmaskCustom) ReadRelation(item gosmparse.Relation) {
-	// @todo: relations currently not supported
-	// due to requiring a 'second-pass' to gather the node ids for
-	// each member way
+	if b.Features.MatchRelation(item) {
+
+		// we currently only support the 'multipolygon' and 'boundary' types
+		// see: http://wiki.openstreetmap.org/wiki/Types_of_relation
+		if val, ok := item.Tags["type"]; ok && (val == "multipolygon" || val == "boundary") {
+
+			// detect relation class
+			var isSuperRelation = false
+			var hasNodeCentroid = false
 
-	// if b.Features.MatchRelation(item) {
-	// 	b.Masks.Relations.Insert(item.ID)
-	// }
+			// iterate members once to try to classify the relation
+			for _, member := range item.Members {
+				switch member.Type {
+				case gosmparse.RelationType:
+					isSuperRelation = true
+				case gosmparse.NodeType:
+					switch member.Role {
+					case "label":
+						hasNodeCentroid = true
+					case "admin_centre":
+						hasNodeCentroid = true
+					}
+				}
+			}
+
+			// super relations are relations containing other relations
+			// we currently do not support these due to their complexity
+			if isSuperRelation {
+				return
+			}
+
+			// iterate over relation members
+			for _, member := range item.Members {
+
+				switch member.Type {
+				case gosmparse.NodeType:
+
+					// only store nodes if they are for 'label' or 'admin_centre'
+					if member.Role == "label" || member.Role == "admin_centre" {
+						b.Masks.RelNodes.Insert(member.ID)
+					}
+
+				case gosmparse.WayType:
+
+					// only store ways if we don't have a node centroid
+					if !hasNodeCentroid {
+
+						// skip cyclic references to parent (subarea) and other junk roles
+						if member.Role == "outer" || member.Role == "inner" || member.Role == "" {
+
+							b.Masks.RelWays.Insert(member.ID)
+						}
+					}
+				}
+			}
+
+			// insert relation in mask
+			b.Masks.Relations.Insert(item.ID)
+		}
+	}
 }