From d24c96f243588860246e039471788c410d13a02e Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Fri, 11 Nov 2016 16:13:56 +0100 Subject: [PATCH 1/5] [wip] refactoring --- src/DemoConsoleApp/program.cs | 16 +- src/PbfReader/PbfReader.cs | 35 +- src/VectorTileReader/VectorTile.cs | 11 + src/VectorTileReader/VectorTileReader.cs | 506 ++++++++++++++++------- 4 files changed, 402 insertions(+), 166 deletions(-) diff --git a/src/DemoConsoleApp/program.cs b/src/DemoConsoleApp/program.cs index 02d348f..4232002 100644 --- a/src/DemoConsoleApp/program.cs +++ b/src/DemoConsoleApp/program.cs @@ -22,6 +22,15 @@ public static int Main(string[] args) return 1; } + + var bufferedData = File.ReadAllBytes(vtIn); + + VectorTileReader vtr = new VectorTileReader(bufferedData); + foreach (var lyrName in vtr.LayerNames()) + { + Console.WriteLine(lyrName); + } + ulong zoom; ulong tileCol; ulong tileRow; @@ -31,10 +40,9 @@ public static int Main(string[] args) return 1; } - var bufferedData = File.ReadAllBytes(vtIn); - - VectorTile tile = VectorTileReader.Decode( -(ulong)zoom + + VectorTile tile = vtr.Decode( + (ulong)zoom , (ulong)tileCol , (ulong)tileRow , (byte[])bufferedData diff --git a/src/PbfReader/PbfReader.cs b/src/PbfReader/PbfReader.cs index c6e5cee..84252aa 100644 --- a/src/PbfReader/PbfReader.cs +++ b/src/PbfReader/PbfReader.cs @@ -8,6 +8,12 @@ namespace Mapbox.VectorTile { + public struct DataView + { + public ulong start; + public ulong end; + } + public class PbfReader { @@ -61,12 +67,14 @@ public byte[] View() { throw new Exception("not of type string, bytes or message"); } + ulong tmpPos = Pos; ulong skipBytes = Varint(); SkipBytes(skipBytes); byte[] buf = new byte[skipBytes]; Array.Copy(_buffer, (int)Pos - (int)skipBytes, buf, 0, (int)skipBytes); + return buf; } @@ -135,19 +143,20 @@ public bool NextByte() public void SkipVarint() { - while (0 == (_buffer[Pos] & 0x80)) - { - Pos++; - if (Pos >= _length) - { - throw new Exception("Truncated message."); - } - } - - if (Pos > _length) - { - throw new Exception("Truncated message."); - } + Varint(); + //while (0 == (_buffer[Pos] & 0x80)) + //{ + // Pos++; + // if (Pos >= _length) + // { + // throw new Exception("Truncated message."); + // } + //} + + //if (Pos > _length) + //{ + // throw new Exception("Truncated message."); + //} } diff --git a/src/VectorTileReader/VectorTile.cs b/src/VectorTileReader/VectorTile.cs index e1e764a..366140a 100644 --- a/src/VectorTileReader/VectorTile.cs +++ b/src/VectorTileReader/VectorTile.cs @@ -199,6 +199,17 @@ public bool Validate() return true; } + + + public static VectorTile DecodeFully( + ulong zoom + , ulong tileCol + , ulong tileRow + , byte[] data + ) + { + return VectorTileReader.Decode(zoom, tileCol, tileRow, data); + } } diff --git a/src/VectorTileReader/VectorTileReader.cs b/src/VectorTileReader/VectorTileReader.cs index 18ccd1e..ce7b3e8 100644 --- a/src/VectorTileReader/VectorTileReader.cs +++ b/src/VectorTileReader/VectorTileReader.cs @@ -4,6 +4,7 @@ using System.Linq; using Mapbox.VectorTile.Geometry; using System.Globalization; +using System.Collections.ObjectModel; namespace Mapbox.VectorTile { @@ -12,174 +13,381 @@ namespace Mapbox.VectorTile public class VectorTileReader { - public static VectorTile Decode( - ulong zoom - , ulong tileCol - , ulong tileRow - , byte[] bufferedData - ) + public VectorTileReader(byte[] data) { - - if (bufferedData[0] == 0x1f && bufferedData[1] == 0x8b) + if (data[0] == 0x1f && data[1] == 0x8b) { throw new Exception("tile data is zipped"); } - var tileReader = new PbfReader(bufferedData); - VectorTile tile = new VectorTile(zoom, tileCol, tileRow); + layers(data); + } + + private Dictionary _Layers = new Dictionary(); + + private void layers(byte[] data) + { + PbfReader tileReader = new PbfReader(data); while (tileReader.NextByte()) { - if (tileReader.Tag != 3) - { - throw new Exception("unknown tile tag"); - } if (tileReader.Tag == (int)TileType.Layers) { - VectorTileLayer layer = new VectorTileLayer(); - byte[] layerBuffer = tileReader.View(); - PbfReader layerReader = new PbfReader(layerBuffer); - while (layerReader.NextByte()) + string name = null; + byte[] layerMessage = tileReader.View(); + PbfReader layerView = new PbfReader(layerMessage); + while (layerView.NextByte()) { - switch ((LayerType)layerReader.Tag) + if (layerView.Tag == (int)LayerType.Name) { - case LayerType.Version: - ulong version = layerReader.Varint(); - layer.Version = version; - break; - case LayerType.Name: - ulong strLength = layerReader.Varint(); - layer.Name = layerReader.GetString(strLength); - break; - case LayerType.Extent: - layer.Extent = layerReader.Varint(); - break; - case LayerType.Keys: - byte[] keyBuffer = layerReader.View(); - string key = Encoding.UTF8.GetString(keyBuffer); - layer.Keys.Add(key); - break; - case LayerType.Values: - byte[] valueBuffer = layerReader.View(); - PbfReader valReader = new PbfReader(valueBuffer); - while (valReader.NextByte()) - { - switch ((ValueType)valReader.Tag) + ulong strLen = layerView.Varint(); + name = layerView.GetString(strLen); + } else + { + layerView.Skip(); + } + } + if (string.IsNullOrEmpty(name)) + { + throw new Exception("Layer missing name"); + } + _Layers.Add(name, layerMessage); + } else + { + tileReader.Skip(); + } + } + } + + + public ReadOnlyCollection LayerNames() + { + return _Layers.Keys.ToList().AsReadOnly(); + } + + public VectorTileLayer GetLayer(string name) + { + if (!_Layers.ContainsKey(name)) + { + return null; + } + + return getLayer(_Layers[name]); + } + + + private VectorTileLayer getLayer(byte[] data) + { + VectorTileLayer layer = new VectorTileLayer(); + PbfReader layerReader = new PbfReader(data); + while (layerReader.NextByte()) + { + switch ((LayerType)layerReader.Tag) + { + case LayerType.Version: + ulong version = layerReader.Varint(); + layer.Version = version; + break; + case LayerType.Name: + ulong strLength = layerReader.Varint(); + layer.Name = layerReader.GetString(strLength); + break; + case LayerType.Extent: + layer.Extent = layerReader.Varint(); + break; + case LayerType.Keys: + byte[] keyBuffer = layerReader.View(); + string key = Encoding.UTF8.GetString(keyBuffer); + layer.Keys.Add(key); + break; + case LayerType.Values: + byte[] valueBuffer = layerReader.View(); + PbfReader valReader = new PbfReader(valueBuffer); + while (valReader.NextByte()) + { + switch ((ValueType)valReader.Tag) + { + case ValueType.String: + byte[] stringBuffer = valReader.View(); + string value = Encoding.UTF8.GetString(stringBuffer); + layer.Values.Add(value); + break; + case ValueType.Float: + float snglVal = valReader.GetFloat(); + layer.Values.Add(snglVal); + break; + case ValueType.Double: + double dblVal = valReader.GetDouble(); + layer.Values.Add(dblVal); + break; + case ValueType.Int: + ulong i64 = valReader.Varint(); + layer.Values.Add(i64); + break; + case ValueType.UInt: + ulong u64 = valReader.Varint(); + layer.Values.Add(u64); + break; + case ValueType.SInt: + ulong s64 = valReader.Varint(); + layer.Values.Add(s64); + break; + case ValueType.Bool: + ulong b = valReader.Varint(); + layer.Values.Add(b == 1); + break; + default: + throw new Exception(string.Format( + NumberFormatInfo.InvariantInfo + , "NOT IMPLEMENTED valueReader.Tag:{0} valueReader.WireType:{1}" + , valReader.Tag + , valReader.WireType + )); + //uncomment the following lines when not throwing!! + //valReader.Skip(); + //break; + } + } + break; + case LayerType.Features: + byte[] featureBuffer = layerReader.View(); + PbfReader featureReader = new PbfReader(featureBuffer); + VectorTileFeature feat = new VectorTileFeature(layer); + bool geomTypeSet = false; + while (featureReader.NextByte()) + { + switch ((FeatureType)featureReader.Tag) + { + case FeatureType.Id: + feat.Id = featureReader.Varint(); + break; + case FeatureType.Tags: + List tags = featureReader.GetPackedUnit32().Select(t => (int)t).ToList(); + feat.Tags = tags; + break; + case FeatureType.Type: + int geomType = (int)featureReader.Varint(); + if (!Enum.IsDefined(typeof(GeomType), geomType)) { - case ValueType.String: - byte[] stringBuffer = valReader.View(); - string value = Encoding.UTF8.GetString(stringBuffer); - layer.Values.Add(value); - break; - case ValueType.Float: - float snglVal = valReader.GetFloat(); - layer.Values.Add(snglVal); - break; - case ValueType.Double: - double dblVal = valReader.GetDouble(); - layer.Values.Add(dblVal); - break; - case ValueType.Int: - ulong i64 = valReader.Varint(); - layer.Values.Add(i64); - break; - case ValueType.UInt: - ulong u64 = valReader.Varint(); - layer.Values.Add(u64); - break; - case ValueType.SInt: - ulong s64 = valReader.Varint(); - layer.Values.Add(s64); - break; - case ValueType.Bool: - ulong b = valReader.Varint(); - layer.Values.Add(b == 1); - break; - default: - throw new Exception(string.Format( - NumberFormatInfo.InvariantInfo - , "NOT IMPLEMENTED valueReader.Tag:{0} valueReader.WireType:{1}" - , valReader.Tag - , valReader.WireType - )); - //uncomment the following lines when not throwing!! - //valReader.Skip(); - //break; + throw new Exception("Unknown geometry type tag"); } - } - break; - case LayerType.Features: - byte[] featureBuffer = layerReader.View(); - PbfReader featureReader = new PbfReader(featureBuffer); - VectorTileFeature feat = new VectorTileFeature(layer); - bool geomTypeSet = false; - while (featureReader.NextByte()) - { - switch ((FeatureType)featureReader.Tag) + feat.GeometryType = (GeomType)geomType; + geomTypeSet = true; + break; + case FeatureType.Geometry: + if (null != feat.GeometryOnTile) { - case FeatureType.Id: - feat.Id = featureReader.Varint(); - break; - case FeatureType.Tags: - List tags = featureReader.GetPackedUnit32().Select(t => (int)t).ToList(); - feat.Tags = tags; - break; - case FeatureType.Type: - int geomType = (int)featureReader.Varint(); - if (!Enum.IsDefined(typeof(GeomType), geomType)) - { - throw new Exception("Unknown geometry type tag"); - } - feat.GeometryType = (GeomType)geomType; - geomTypeSet = true; - break; - case FeatureType.Geometry: - if (null != feat.GeometryOnTile) - { - throw new Exception("feature already has a geometry"); - } - //get raw array of commands and coordinates - List geometry = featureReader.GetPackedUnit32(); - //decode commands and coordinates - List> geom = DecodeGeometry.GetGeometry( - layer.Extent - , tile.Zoom - , tile.TileColumn - , tile.TileRow - , feat.GeometryType - , geometry - ); - feat.GeometryOnTile = geom; - //convert tile coordinates to LatLnt - List> geomAsLatLng = new List>(); - foreach (var part in geom) - { - geomAsLatLng.Add( - part.Select(g => g.ToLngLat(zoom, tileCol, tileRow, layer.Extent)).ToList() - ); - } - feat.Geometry.AddRange(geomAsLatLng); - break; - default: - featureReader.Skip(); - break; + throw new Exception("feature already has a geometry"); } - } - - if (!geomTypeSet) - { - throw new Exception("feature missing geometry type"); - } - - layer.Features.Add(feat); - break; - default: - layerReader.Skip(); - break; + //get raw array of commands and coordinates + List geometry = featureReader.GetPackedUnit32(); + //decode commands and coordinates + List> geom = DecodeGeometry.GetGeometry( + layer.Extent + , tile.Zoom + , tile.TileColumn + , tile.TileRow + , feat.GeometryType + , geometry + ); + feat.GeometryOnTile = geom; + //convert tile coordinates to LatLnt + List> geomAsLatLng = new List>(); + foreach (var part in geom) + { + geomAsLatLng.Add( + part.Select(g => g.ToLngLat(zoom, tileCol, tileRow, layer.Extent)).ToList() + ); + } + feat.Geometry.AddRange(geomAsLatLng); + break; + default: + featureReader.Skip(); + break; + } } - } + if (!geomTypeSet) + { + throw new Exception("feature missing geometry type"); + } + + layer.Features.Add(feat); + break; + default: + layerReader.Skip(); + break; + } + } + return layer; + } + + + public VectorTile Decode( + ulong zoom + , ulong tileCol + , ulong tileRow + , byte[] _Data + ) + { + + + var tileReader = new PbfReader(_Data); + VectorTile tile = new VectorTile(zoom, tileCol, tileRow); + + while (tileReader.NextByte()) + { + if (tileReader.Tag != 3) + { + throw new Exception("unknown tile tag"); + } + if (tileReader.Tag == (int)TileType.Layers) + { + byte[] layerBuffer = tileReader.View(); + VectorTileLayer layer = getLayer(layerBuffer); tile.Layers.Add(layer); + + //PbfReader layerReader = new PbfReader(layerBuffer); + //while (layerReader.NextByte()) + //{ + // switch ((LayerType)layerReader.Tag) + // { + // case LayerType.Version: + // ulong version = layerReader.Varint(); + // layer.Version = version; + // break; + // case LayerType.Name: + // ulong strLength = layerReader.Varint(); + // layer.Name = layerReader.GetString(strLength); + // break; + // case LayerType.Extent: + // layer.Extent = layerReader.Varint(); + // break; + // case LayerType.Keys: + // byte[] keyBuffer = layerReader.View(); + // string key = Encoding.UTF8.GetString(keyBuffer); + // layer.Keys.Add(key); + // break; + // case LayerType.Values: + // byte[] valueBuffer = layerReader.View(); + // PbfReader valReader = new PbfReader(valueBuffer); + // while (valReader.NextByte()) + // { + // switch ((ValueType)valReader.Tag) + // { + // case ValueType.String: + // byte[] stringBuffer = valReader.View(); + // string value = Encoding.UTF8.GetString(stringBuffer); + // layer.Values.Add(value); + // break; + // case ValueType.Float: + // float snglVal = valReader.GetFloat(); + // layer.Values.Add(snglVal); + // break; + // case ValueType.Double: + // double dblVal = valReader.GetDouble(); + // layer.Values.Add(dblVal); + // break; + // case ValueType.Int: + // ulong i64 = valReader.Varint(); + // layer.Values.Add(i64); + // break; + // case ValueType.UInt: + // ulong u64 = valReader.Varint(); + // layer.Values.Add(u64); + // break; + // case ValueType.SInt: + // ulong s64 = valReader.Varint(); + // layer.Values.Add(s64); + // break; + // case ValueType.Bool: + // ulong b = valReader.Varint(); + // layer.Values.Add(b == 1); + // break; + // default: + // throw new Exception(string.Format( + // NumberFormatInfo.InvariantInfo + // , "NOT IMPLEMENTED valueReader.Tag:{0} valueReader.WireType:{1}" + // , valReader.Tag + // , valReader.WireType + // )); + // //uncomment the following lines when not throwing!! + // //valReader.Skip(); + // //break; + // } + // } + // break; + // case LayerType.Features: + // byte[] featureBuffer = layerReader.View(); + // PbfReader featureReader = new PbfReader(featureBuffer); + // VectorTileFeature feat = new VectorTileFeature(layer); + // bool geomTypeSet = false; + // while (featureReader.NextByte()) + // { + // switch ((FeatureType)featureReader.Tag) + // { + // case FeatureType.Id: + // feat.Id = featureReader.Varint(); + // break; + // case FeatureType.Tags: + // List tags = featureReader.GetPackedUnit32().Select(t => (int)t).ToList(); + // feat.Tags = tags; + // break; + // case FeatureType.Type: + // int geomType = (int)featureReader.Varint(); + // if (!Enum.IsDefined(typeof(GeomType), geomType)) + // { + // throw new Exception("Unknown geometry type tag"); + // } + // feat.GeometryType = (GeomType)geomType; + // geomTypeSet = true; + // break; + // case FeatureType.Geometry: + // if (null != feat.GeometryOnTile) + // { + // throw new Exception("feature already has a geometry"); + // } + // //get raw array of commands and coordinates + // List geometry = featureReader.GetPackedUnit32(); + // //decode commands and coordinates + // List> geom = DecodeGeometry.GetGeometry( + // layer.Extent + // , tile.Zoom + // , tile.TileColumn + // , tile.TileRow + // , feat.GeometryType + // , geometry + // ); + // feat.GeometryOnTile = geom; + // //convert tile coordinates to LatLnt + // List> geomAsLatLng = new List>(); + // foreach (var part in geom) + // { + // geomAsLatLng.Add( + // part.Select(g => g.ToLngLat(zoom, tileCol, tileRow, layer.Extent)).ToList() + // ); + // } + // feat.Geometry.AddRange(geomAsLatLng); + // break; + // default: + // featureReader.Skip(); + // break; + // } + // } + + // if (!geomTypeSet) + // { + // throw new Exception("feature missing geometry type"); + // } + + // layer.Features.Add(feat); + // break; + // default: + // layerReader.Skip(); + // break; + // } + //} + } else { tileReader.Skip(); From 132489063f33ebb78cbfbbd4b558ddb7608dc3d4 Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Mon, 14 Nov 2016 15:33:23 +0100 Subject: [PATCH 2/5] first iteration of refactoring --- src/Bench/Program.cs | 3 +- src/DemoConsoleApp/program.cs | 13 +- src/Geometry/DecodeGeometry.cs | 6 - src/VectorTileReader/VectorTile.cs | 58 +-- src/VectorTileReader/VectorTileFeature.cs | 30 +- src/VectorTileReader/VectorTileLayer.cs | 22 ++ src/VectorTileReader/VectorTileReader.cs | 330 +++++++----------- src/VectorTiles.Tests/TestInvalidMvtInBulk.cs | 5 +- src/VectorTiles.Tests/TestMvtInBulk.cs | 7 +- src/VectorTiles.Tests/TestMvtSingle.cs | 3 +- 10 files changed, 199 insertions(+), 278 deletions(-) diff --git a/src/Bench/Program.cs b/src/Bench/Program.cs index a96ccd5..a588e1d 100644 --- a/src/Bench/Program.cs +++ b/src/Bench/Program.cs @@ -67,7 +67,8 @@ static int Main(string[] args) stopWatch.Start(); foreach (var tile in tiles) { - VectorTileReader.Decode(tile.zoom, tile.col, tile.row, tile.pbf); + VectorTileReader vtr = new VectorTileReader(); + vtr.Decode(tile.zoom, tile.col, tile.row, tile.pbf); } stopWatch.Stop(); //skip first run diff --git a/src/DemoConsoleApp/program.cs b/src/DemoConsoleApp/program.cs index 4232002..f756edb 100644 --- a/src/DemoConsoleApp/program.cs +++ b/src/DemoConsoleApp/program.cs @@ -29,6 +29,17 @@ public static int Main(string[] args) foreach (var lyrName in vtr.LayerNames()) { Console.WriteLine(lyrName); + VectorTileLayer layer = vtr.GetLayer(lyrName); + Console.WriteLine("features: {0}", layer.FeaturesData.Count); + for (int i = 0; i < layer.FeatureCount(); i++) + { + VectorTileFeature feat = layer.GetFeature(i); + Console.WriteLine(feat.Id); + foreach (var prop in feat.GetProperties()) + { + Console.WriteLine("{0}: {1}", prop.Key, prop.Value); + } + } } ulong zoom; @@ -40,7 +51,7 @@ public static int Main(string[] args) return 1; } - + VectorTile tile = vtr.Decode( (ulong)zoom , (ulong)tileCol diff --git a/src/Geometry/DecodeGeometry.cs b/src/Geometry/DecodeGeometry.cs index 3949907..312e73f 100644 --- a/src/Geometry/DecodeGeometry.cs +++ b/src/Geometry/DecodeGeometry.cs @@ -25,17 +25,11 @@ public static class DecodeGeometry /// Each child list contains the corrdinates of this part. /// /// Tile extent - /// Zoom level - /// Tile column - /// Tile row /// Geometry type /// /// List> public static List> GetGeometry( ulong extent - , ulong zoom - , ulong tileColumn - , ulong tileRow , GeomType geomType , List geometry ) diff --git a/src/VectorTileReader/VectorTile.cs b/src/VectorTileReader/VectorTile.cs index 366140a..12923e2 100644 --- a/src/VectorTileReader/VectorTile.cs +++ b/src/VectorTileReader/VectorTile.cs @@ -73,7 +73,8 @@ public string ToGeoJson() string geomType = feat.GeometryType.Description(); //multipart - if (feat.Geometry.Count > 1) + List> geomWgs84 = feat.GeometryAsWgs84(this); + if (geomWgs84.Count > 1) { switch (feat.GeometryType) { @@ -81,7 +82,7 @@ public string ToGeoJson() geomType = "MultiPoint"; geojsonCoords = string.Join( "," - , feat.Geometry + , geomWgs84 .SelectMany((List g) => g) .Select(g => string.Format(NumberFormatInfo.InvariantInfo, "[{0},{1}]", g.Lng, g.Lat)).ToArray() ); @@ -89,7 +90,7 @@ public string ToGeoJson() case GeomType.LINESTRING: geomType = "MultiLineString"; List parts = new List(); - foreach (var part in feat.Geometry) + foreach (var part in geomWgs84) { parts.Add("[" + string.Join( "," @@ -101,7 +102,7 @@ public string ToGeoJson() case GeomType.POLYGON: geomType = "MultiPolygon"; List partsMP = new List(); - foreach (var part in feat.Geometry) + foreach (var part in geomWgs84) { partsMP.Add("[" + string.Join( "," @@ -118,18 +119,18 @@ public string ToGeoJson() switch (feat.GeometryType) { case GeomType.POINT: - geojsonCoords = string.Format(NumberFormatInfo.InvariantInfo, "{0},{1}", feat.Geometry[0][0].Lng, feat.Geometry[0][0].Lat); + geojsonCoords = string.Format(NumberFormatInfo.InvariantInfo, "{0},{1}", geomWgs84[0][0].Lng, geomWgs84[0][0].Lat); break; case GeomType.LINESTRING: geojsonCoords = string.Join( "," - , feat.Geometry[0].Select(g => string.Format(NumberFormatInfo.InvariantInfo, "[{0},{1}]", g.Lng, g.Lat)).ToArray() + , geomWgs84[0].Select(g => string.Format(NumberFormatInfo.InvariantInfo, "[{0},{1}]", g.Lng, g.Lat)).ToArray() ); break; case GeomType.POLYGON: geojsonCoords = "[" + string.Join( "," - , feat.Geometry[0].Select(g => string.Format(NumberFormatInfo.InvariantInfo, "[{0},{1}]", g.Lng, g.Lat)).ToArray() + , geomWgs84[0].Select(g => string.Format(NumberFormatInfo.InvariantInfo, "[{0},{1}]", g.Lng, g.Lat)).ToArray() ) + "]"; break; default: @@ -159,46 +160,6 @@ public string ToGeoJson() } - public bool Validate() - { - - List layerNames = Layers.Select(l => l.Name).Distinct().ToList(); - if (layerNames.Count() != Layers.Count()) - { - throw new Exception("Duplicate layer names."); - } - if (0 < Layers.Where(l => l.Name.Length == 0).Count()) - { - throw new Exception("Layer with no name"); - } - if (0 < Layers.Where(l => l.Version != 2).Count()) - { - throw new Exception("invalid layer version"); - } - if (0 < Layers.Where(l => l.Extent == 0).Count()) - { - throw new Exception("layer without extent"); - } - if (0 < Layers.Where(l => l.Features.Count() == 0).Count()) - { - throw new Exception("layer without features"); - } - if (0 < Layers.SelectMany(l => l.Features).Where(f => null == f.GeometryOnTile).Count()) - { - throw new Exception("features with no geometry"); - } - //if (0 < Layers.SelectMany(l => l.Features).Where(f => f.GeometryType == GeomType.UNKNOWN).Count()) - //{ - // throw new Exception("features with unknown geometry type"); - //} - - if (0 < Layers.Where(l => l.Keys.Count != l.Values.Count).Count()) - { - throw new Exception("number of keys and values doesn't match"); - } - - return true; - } public static VectorTile DecodeFully( @@ -208,7 +169,8 @@ ulong zoom , byte[] data ) { - return VectorTileReader.Decode(zoom, tileCol, tileRow, data); + VectorTileReader vtr = new VectorTileReader(data); + return vtr.Decode(zoom, tileCol, tileRow, data); } } diff --git a/src/VectorTileReader/VectorTileFeature.cs b/src/VectorTileReader/VectorTileFeature.cs index 08385c9..717e6f4 100644 --- a/src/VectorTileReader/VectorTileFeature.cs +++ b/src/VectorTileReader/VectorTileFeature.cs @@ -1,6 +1,7 @@ using System.Collections.Generic; using Mapbox.VectorTile.Geometry; using System.Linq; +using System; namespace Mapbox.VectorTile { @@ -11,10 +12,8 @@ public class VectorTileFeature /// public ulong Id { get; set; } public GeomType GeometryType { get; set; } - /// Geometry in LatLng Coordinates - public List> Geometry { get; set; } /// Geometry in Tile Coordinates - public List> GeometryOnTile { get; set; } + public List> Geometry { get; set; } /// Tags to resolve Properties public List Tags { get; set; } @@ -25,7 +24,6 @@ public class VectorTileFeature public VectorTileFeature(VectorTileLayer layer) { _Layer = layer; - Geometry = new List>(); Tags = new List(); } @@ -68,6 +66,30 @@ public object GetValue(string key) } return null; } + + + private List> _GeometryAsWgs84 = null; + [Obsolete("This is a convenience method during early development and will be deprecated. Future clients will have to convert themselves.")] + /// Geometry in LatLng Coordinates + public List> GeometryAsWgs84(VectorTile vt) + { + + if (null != _GeometryAsWgs84) + { + return _GeometryAsWgs84; + } + + _GeometryAsWgs84 = new List>(); + foreach (var part in Geometry) + { + _GeometryAsWgs84.Add( + part.Select(g => g.ToLngLat(vt.Zoom, vt.TileColumn, vt.TileRow, _Layer.Extent)).ToList() + ); + } + + return _GeometryAsWgs84; + } + } diff --git a/src/VectorTileReader/VectorTileLayer.cs b/src/VectorTileReader/VectorTileLayer.cs index 079fe1e..b4ad748 100644 --- a/src/VectorTileReader/VectorTileLayer.cs +++ b/src/VectorTileReader/VectorTileLayer.cs @@ -10,16 +10,38 @@ public class VectorTileLayer public VectorTileLayer() { Features = new List(); + FeaturesData = new List(); Keys = new List(); Values = new List(); } + public VectorTileLayer(byte[] data) : this() + { + Data = data; + } + + public byte[] Data { get; private set; } + + public int FeatureCount() + { + return FeaturesData.Count; + } + + public VectorTileFeature GetFeature(int feature) + { + VectorTileReader vtr = new VectorTileReader(null); + return vtr.GetFeature(this, feature); + } + public string Name { get; set; } public ulong Version { get; set; } public ulong Extent { get; set; } public List Features { get; set; } + public List FeaturesData { get; set; } public List Values { get; set; } public List Keys { get; set; } + + } } diff --git a/src/VectorTileReader/VectorTileReader.cs b/src/VectorTileReader/VectorTileReader.cs index ce7b3e8..e54adf2 100644 --- a/src/VectorTileReader/VectorTileReader.cs +++ b/src/VectorTileReader/VectorTileReader.cs @@ -13,14 +13,17 @@ namespace Mapbox.VectorTile public class VectorTileReader { - public VectorTileReader(byte[] data) + public VectorTileReader(byte[] data = null) { - if (data[0] == 0x1f && data[1] == 0x8b) + if (null != data) { - throw new Exception("tile data is zipped"); - } + if (data[0] == 0x1f && data[1] == 0x8b) + { + throw new Exception("Tile data is zipped"); + } - layers(data); + layers(data); + } } @@ -51,6 +54,10 @@ private void layers(byte[] data) { throw new Exception("Layer missing name"); } + if (_Layers.ContainsKey(name)) + { + throw new Exception("Duplicate layer names: " + name); + } _Layers.Add(name, layerMessage); } else { @@ -78,11 +85,16 @@ public VectorTileLayer GetLayer(string name) private VectorTileLayer getLayer(byte[] data) { - VectorTileLayer layer = new VectorTileLayer(); - PbfReader layerReader = new PbfReader(data); + VectorTileLayer layer = new VectorTileLayer(data); + PbfReader layerReader = new PbfReader(layer.Data); while (layerReader.NextByte()) { - switch ((LayerType)layerReader.Tag) + int layerType = layerReader.Tag; + if (!Enum.IsDefined(typeof(LayerType), layerType)) + { + throw new Exception("Unknown layer type: " + layerType); + } + switch ((LayerType)layerType) { case LayerType.Version: ulong version = layerReader.Varint(); @@ -150,76 +162,107 @@ private VectorTileLayer getLayer(byte[] data) } break; case LayerType.Features: - byte[] featureBuffer = layerReader.View(); - PbfReader featureReader = new PbfReader(featureBuffer); - VectorTileFeature feat = new VectorTileFeature(layer); - bool geomTypeSet = false; - while (featureReader.NextByte()) + layer.FeaturesData.Add(layerReader.View()); + break; + default: + layerReader.Skip(); + break; + } + } + + if (string.IsNullOrEmpty(layer.Name)) + { + throw new Exception("Layer has no name"); + } + if (0 == layer.Version) + { + throw new Exception("Layer has no version: " + layer.Name); + } + if (2 != layer.Version) + { + throw new Exception("Layer has invalid version: " + layer.Name); + } + if (0 == layer.Extent) + { + throw new Exception("Layer has no extent: " + layer.Name); + } + if (0 == layer.FeatureCount()) + { + throw new Exception("Layer has no features: " + layer.Name); + } + if (layer.Keys.Count != layer.Values.Count) + { + throw new Exception("Number of keys and values does not match, layer " + layer.Name); + } + + return layer; + } + + + public VectorTileFeature GetFeature(VectorTileLayer layer, int idxFeature) + { + + byte[] data = layer.FeaturesData[idxFeature]; + + PbfReader featureReader = new PbfReader(data); + VectorTileFeature feat = new VectorTileFeature(layer); + bool geomTypeSet = false; + while (featureReader.NextByte()) + { + int featureType = featureReader.Tag; + if (!Enum.IsDefined(typeof(FeatureType), featureType)) + { + throw new Exception(layer.Name + ", unknown feature type: " + featureType); + } + switch ((FeatureType)featureType) + { + case FeatureType.Id: + feat.Id = featureReader.Varint(); + break; + case FeatureType.Tags: + List tags = featureReader.GetPackedUnit32().Select(t => (int)t).ToList(); + feat.Tags = tags; + break; + case FeatureType.Type: + int geomType = (int)featureReader.Varint(); + if (!Enum.IsDefined(typeof(GeomType), geomType)) { - switch ((FeatureType)featureReader.Tag) - { - case FeatureType.Id: - feat.Id = featureReader.Varint(); - break; - case FeatureType.Tags: - List tags = featureReader.GetPackedUnit32().Select(t => (int)t).ToList(); - feat.Tags = tags; - break; - case FeatureType.Type: - int geomType = (int)featureReader.Varint(); - if (!Enum.IsDefined(typeof(GeomType), geomType)) - { - throw new Exception("Unknown geometry type tag"); - } - feat.GeometryType = (GeomType)geomType; - geomTypeSet = true; - break; - case FeatureType.Geometry: - if (null != feat.GeometryOnTile) - { - throw new Exception("feature already has a geometry"); - } - //get raw array of commands and coordinates - List geometry = featureReader.GetPackedUnit32(); - //decode commands and coordinates - List> geom = DecodeGeometry.GetGeometry( - layer.Extent - , tile.Zoom - , tile.TileColumn - , tile.TileRow - , feat.GeometryType - , geometry - ); - feat.GeometryOnTile = geom; - //convert tile coordinates to LatLnt - List> geomAsLatLng = new List>(); - foreach (var part in geom) - { - geomAsLatLng.Add( - part.Select(g => g.ToLngLat(zoom, tileCol, tileRow, layer.Extent)).ToList() - ); - } - feat.Geometry.AddRange(geomAsLatLng); - break; - default: - featureReader.Skip(); - break; - } + throw new Exception(layer.Name + ", unknown geometry type tag " + geomType); } - - if (!geomTypeSet) + feat.GeometryType = (GeomType)geomType; + geomTypeSet = true; + break; + case FeatureType.Geometry: + if (null != feat.Geometry) { - throw new Exception("feature missing geometry type"); + throw new Exception(layer.Name + ", feature already has a geometry"); } - - layer.Features.Add(feat); + //get raw array of commands and coordinates + List geometry = featureReader.GetPackedUnit32(); + //decode commands and coordinates + List> geom = DecodeGeometry.GetGeometry( + layer.Extent + , feat.GeometryType + , geometry + ); + feat.Geometry = geom; break; default: - layerReader.Skip(); + featureReader.Skip(); break; } } - return layer; + + if (!geomTypeSet) + { + throw new Exception(layer.Name + ", feature missing geometry type"); + } + if (null == feat.Geometry) + { + throw new Exception(layer.Name + ", feature has no geometry"); + } + + return feat; } @@ -239,155 +282,18 @@ ulong zoom { if (tileReader.Tag != 3) { - throw new Exception("unknown tile tag"); + throw new Exception("Unknown tile tag"); } if (tileReader.Tag == (int)TileType.Layers) { byte[] layerBuffer = tileReader.View(); VectorTileLayer layer = getLayer(layerBuffer); + for (int i = 0; i < layer.FeatureCount(); i++) + { + VectorTileFeature feat = GetFeature(layer, i); + layer.Features.Add(feat); + } tile.Layers.Add(layer); - - //PbfReader layerReader = new PbfReader(layerBuffer); - //while (layerReader.NextByte()) - //{ - // switch ((LayerType)layerReader.Tag) - // { - // case LayerType.Version: - // ulong version = layerReader.Varint(); - // layer.Version = version; - // break; - // case LayerType.Name: - // ulong strLength = layerReader.Varint(); - // layer.Name = layerReader.GetString(strLength); - // break; - // case LayerType.Extent: - // layer.Extent = layerReader.Varint(); - // break; - // case LayerType.Keys: - // byte[] keyBuffer = layerReader.View(); - // string key = Encoding.UTF8.GetString(keyBuffer); - // layer.Keys.Add(key); - // break; - // case LayerType.Values: - // byte[] valueBuffer = layerReader.View(); - // PbfReader valReader = new PbfReader(valueBuffer); - // while (valReader.NextByte()) - // { - // switch ((ValueType)valReader.Tag) - // { - // case ValueType.String: - // byte[] stringBuffer = valReader.View(); - // string value = Encoding.UTF8.GetString(stringBuffer); - // layer.Values.Add(value); - // break; - // case ValueType.Float: - // float snglVal = valReader.GetFloat(); - // layer.Values.Add(snglVal); - // break; - // case ValueType.Double: - // double dblVal = valReader.GetDouble(); - // layer.Values.Add(dblVal); - // break; - // case ValueType.Int: - // ulong i64 = valReader.Varint(); - // layer.Values.Add(i64); - // break; - // case ValueType.UInt: - // ulong u64 = valReader.Varint(); - // layer.Values.Add(u64); - // break; - // case ValueType.SInt: - // ulong s64 = valReader.Varint(); - // layer.Values.Add(s64); - // break; - // case ValueType.Bool: - // ulong b = valReader.Varint(); - // layer.Values.Add(b == 1); - // break; - // default: - // throw new Exception(string.Format( - // NumberFormatInfo.InvariantInfo - // , "NOT IMPLEMENTED valueReader.Tag:{0} valueReader.WireType:{1}" - // , valReader.Tag - // , valReader.WireType - // )); - // //uncomment the following lines when not throwing!! - // //valReader.Skip(); - // //break; - // } - // } - // break; - // case LayerType.Features: - // byte[] featureBuffer = layerReader.View(); - // PbfReader featureReader = new PbfReader(featureBuffer); - // VectorTileFeature feat = new VectorTileFeature(layer); - // bool geomTypeSet = false; - // while (featureReader.NextByte()) - // { - // switch ((FeatureType)featureReader.Tag) - // { - // case FeatureType.Id: - // feat.Id = featureReader.Varint(); - // break; - // case FeatureType.Tags: - // List tags = featureReader.GetPackedUnit32().Select(t => (int)t).ToList(); - // feat.Tags = tags; - // break; - // case FeatureType.Type: - // int geomType = (int)featureReader.Varint(); - // if (!Enum.IsDefined(typeof(GeomType), geomType)) - // { - // throw new Exception("Unknown geometry type tag"); - // } - // feat.GeometryType = (GeomType)geomType; - // geomTypeSet = true; - // break; - // case FeatureType.Geometry: - // if (null != feat.GeometryOnTile) - // { - // throw new Exception("feature already has a geometry"); - // } - // //get raw array of commands and coordinates - // List geometry = featureReader.GetPackedUnit32(); - // //decode commands and coordinates - // List> geom = DecodeGeometry.GetGeometry( - // layer.Extent - // , tile.Zoom - // , tile.TileColumn - // , tile.TileRow - // , feat.GeometryType - // , geometry - // ); - // feat.GeometryOnTile = geom; - // //convert tile coordinates to LatLnt - // List> geomAsLatLng = new List>(); - // foreach (var part in geom) - // { - // geomAsLatLng.Add( - // part.Select(g => g.ToLngLat(zoom, tileCol, tileRow, layer.Extent)).ToList() - // ); - // } - // feat.Geometry.AddRange(geomAsLatLng); - // break; - // default: - // featureReader.Skip(); - // break; - // } - // } - - // if (!geomTypeSet) - // { - // throw new Exception("feature missing geometry type"); - // } - - // layer.Features.Add(feat); - // break; - // default: - // layerReader.Skip(); - // break; - // } - //} - } else { tileReader.Skip(); diff --git a/src/VectorTiles.Tests/TestInvalidMvtInBulk.cs b/src/VectorTiles.Tests/TestInvalidMvtInBulk.cs index 630db44..ccacf4d 100644 --- a/src/VectorTiles.Tests/TestInvalidMvtInBulk.cs +++ b/src/VectorTiles.Tests/TestInvalidMvtInBulk.cs @@ -37,7 +37,8 @@ public void Validate(string fileName) byte[] data = File.ReadAllBytes(fullFileName); Assert.Throws(Is.InstanceOf(), () => { - VectorTile vt = VectorTileReader.Decode(0, 0, 0, data); + VectorTileReader vtr = new VectorTileReader(data); + VectorTile vt = vtr.Decode(0, 0, 0, data); foreach (var lyr in vt.Layers) { foreach (var feat in lyr.Features) @@ -45,7 +46,7 @@ public void Validate(string fileName) feat.GetProperties(); } } - vt.Validate(); + //vt.Validate(); }); } } diff --git a/src/VectorTiles.Tests/TestMvtInBulk.cs b/src/VectorTiles.Tests/TestMvtInBulk.cs index fd047d3..97276e4 100644 --- a/src/VectorTiles.Tests/TestMvtInBulk.cs +++ b/src/VectorTiles.Tests/TestMvtInBulk.cs @@ -35,7 +35,8 @@ public void AtLeastOneLayer(string fileName) string fullFileName = Path.Combine(fixturesPath, fileName); Assert.True(File.Exists(fullFileName), "Vector tile exists"); byte[] data = File.ReadAllBytes(fullFileName); - VectorTile vt = VectorTileReader.Decode(0, 0, 0, data); + VectorTileReader vtr = new VectorTileReader(data); + VectorTile vt = vtr.Decode(0, 0, 0, data); Assert.GreaterOrEqual(vt.Layers.Count, 1, "At least one layer"); string geojson = vt.ToGeoJson(); Assert.GreaterOrEqual(geojson.Length, 30, "geojson >= 30 chars"); @@ -48,7 +49,8 @@ public void IterateAllProperties(string fileName) string fullFileName = Path.Combine(fixturesPath, fileName); Assert.True(File.Exists(fullFileName), "Vector tile exists"); byte[] data = File.ReadAllBytes(fullFileName); - VectorTile vt = VectorTileReader.Decode(0, 0, 0, data); + VectorTileReader vtr = new VectorTileReader(data); + VectorTile vt = vtr.Decode(0, 0, 0, data); foreach (var lyr in vt.Layers) { foreach (var feat in lyr.Features) @@ -60,7 +62,6 @@ public void IterateAllProperties(string fileName) } } } - Assert.IsTrue(vt.Validate(), "VectorTile Validation"); } } diff --git a/src/VectorTiles.Tests/TestMvtSingle.cs b/src/VectorTiles.Tests/TestMvtSingle.cs index 0fcd083..ab3f808 100644 --- a/src/VectorTiles.Tests/TestMvtSingle.cs +++ b/src/VectorTiles.Tests/TestMvtSingle.cs @@ -25,7 +25,8 @@ public void FeatureSinglePoint() { byte[] data = File.ReadAllBytes(Path.Combine(fixturesPath, "Feature-single-point.mvt")); - VectorTile vt = VectorTileReader.Decode(0, 0, 0, data); + VectorTileReader vtr = new VectorTileReader(data); + VectorTile vt = vtr.Decode(0, 0, 0, data); Assert.AreEqual(1, vt.Layers.Count, "one layer"); VectorTileLayer lyr = vt.Layers[0]; Assert.AreEqual("layer_name", lyr.Name, "Layer name"); From a28782b5d736cdacc38dfabe25cb5adac72f9c44 Mon Sep 17 00:00:00 2001 From: bergwerkgis Date: Tue, 15 Nov 2016 19:06:38 +0100 Subject: [PATCH 3/5] still [wip] first working implementation of lazy loading --- src/Bench/Program.cs | 17 +++- src/DemoConsoleApp/program.cs | 11 +-- src/Geometry/DecodeGeometry.cs | 46 ++++++++-- src/Geometry/Geometry.csproj | 7 ++ src/Geometry/packages.config | 4 + src/VectorTileReader/VectorTile.cs | 90 ++++++++++++++----- src/VectorTileReader/VectorTileFeature.cs | 4 +- src/VectorTileReader/VectorTileLayer.cs | 18 +++- src/VectorTileReader/VectorTileReader.cs | 62 ++++++------- src/VectorTileReader/VectorTileReader.csproj | 7 ++ src/VectorTileReader/packages.config | 4 + src/VectorTiles.Tests/TestInvalidMvtInBulk.cs | 3 +- src/VectorTiles.Tests/TestMvtInBulk.cs | 31 +++++-- src/VectorTiles.Tests/TestMvtSingle.cs | 3 +- 14 files changed, 214 insertions(+), 93 deletions(-) create mode 100644 src/Geometry/packages.config create mode 100644 src/VectorTileReader/packages.config diff --git a/src/Bench/Program.cs b/src/Bench/Program.cs index a588e1d..96b2919 100644 --- a/src/Bench/Program.cs +++ b/src/Bench/Program.cs @@ -2,12 +2,13 @@ using System; using System.Collections.Generic; using System.Diagnostics; +using System.Globalization; using System.IO; using System.Linq; using System.Net; using System.Text; -namespace ProfileDecoding +namespace Bench { class Program { @@ -47,7 +48,8 @@ static int Main(string[] args) return 1; } else { - tiles.Add(new TileData() { + tiles.Add(new TileData() + { zoom = zoom, col = col, row = row, @@ -67,8 +69,14 @@ static int Main(string[] args) stopWatch.Start(); foreach (var tile in tiles) { - VectorTileReader vtr = new VectorTileReader(); - vtr.Decode(tile.zoom, tile.col, tile.row, tile.pbf); + VectorTile vt = VectorTile.DecodeFully(tile.pbf); + foreach (var lyr in vt.Layers) + { + foreach (var feat in lyr.Features) + { + var props = feat.GetProperties(); + } + } } stopWatch.Stop(); //skip first run @@ -77,6 +85,7 @@ static int Main(string[] args) elapsed.Add(stopWatch.ElapsedMilliseconds); } stopWatch.Reset(); + elapsed.Add(stopWatch.ElapsedMilliseconds); } diff --git a/src/DemoConsoleApp/program.cs b/src/DemoConsoleApp/program.cs index f756edb..fb2ac01 100644 --- a/src/DemoConsoleApp/program.cs +++ b/src/DemoConsoleApp/program.cs @@ -30,7 +30,7 @@ public static int Main(string[] args) { Console.WriteLine(lyrName); VectorTileLayer layer = vtr.GetLayer(lyrName); - Console.WriteLine("features: {0}", layer.FeaturesData.Count); + Console.WriteLine("features: {0}", layer.FeatureCount()); for (int i = 0; i < layer.FeatureCount(); i++) { VectorTileFeature feat = layer.GetFeature(i); @@ -52,14 +52,9 @@ public static int Main(string[] args) } - VectorTile tile = vtr.Decode( - (ulong)zoom - , (ulong)tileCol - , (ulong)tileRow - , (byte[])bufferedData - ); + VectorTile tile = VectorTile.DecodeFully(bufferedData); - Console.WriteLine(tile.ToGeoJson()); + Console.WriteLine(tile.ToGeoJson(zoom, tileCol, tileRow)); return 0; } diff --git a/src/Geometry/DecodeGeometry.cs b/src/Geometry/DecodeGeometry.cs index 312e73f..599bfc9 100644 --- a/src/Geometry/DecodeGeometry.cs +++ b/src/Geometry/DecodeGeometry.cs @@ -1,7 +1,5 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; +using System.Collections.Generic; +using Mapbox.Geometry; namespace Mapbox.VectorTile.Geometry { @@ -26,12 +24,13 @@ public static class DecodeGeometry /// /// Tile extent /// Geometry type - /// + /// /// List> public static List> GetGeometry( ulong extent , GeomType geomType - , List geometry + , List geometryCommands + , float scale = 1.0f ) { @@ -40,10 +39,10 @@ ulong extent long cursorX = 0; long cursorY = 0; - for (int i = 0; i < geometry.Count; i++) + for (int i = 0; i < geometryCommands.Count; i++) { - uint g = geometry[i]; + uint g = geometryCommands[i]; Commands cmd = (Commands)(g & 0x7); uint cmdCount = g >> 3; @@ -51,7 +50,7 @@ ulong extent { for (int j = 0; j < cmdCount; j++) { - Point2d delta = zigzagDecode(geometry[i + 1], geometry[i + 2]); + Point2d delta = zigzagDecode(geometryCommands[i + 1], geometryCommands[i + 2]); cursorX += delta.X; cursorY += delta.Y; i += 2; @@ -79,6 +78,35 @@ ulong extent geomOut.Add(geomTmp); } + //IGeometryBase geomOut2; + //switch (geomType) + //{ + // case GeomType.UNKNOWN: + // throw new System.Exception("Geometry type unknown"); + // case GeomType.POINT: + // if (geomOut.Count == 1) + // { + // geomOut2 = new Point(geomOut[0][0].X, geomOut[0][0].Y); + // } else + // { + // geomOut2 = new MultiPoint(); + // foreach (var part in geomOut) + // { + // ((MultiPoint)geomOut2).Add(new Point(part[0].X, part[0].Y)); + // } + // } + // break; + // case GeomType.LINESTRING: + // geomOut2 = new LinearRing(); + // break; + // case GeomType.POLYGON: + // geomOut2 = new Polygon(); + // break; + // default: + // throw new System.Exception("Geometry type invalid"); + //} + + return geomOut; } diff --git a/src/Geometry/Geometry.csproj b/src/Geometry/Geometry.csproj index 9d39de1..391dad8 100644 --- a/src/Geometry/Geometry.csproj +++ b/src/Geometry/Geometry.csproj @@ -54,6 +54,10 @@ false + + ..\..\packages\Mapbox.Geometry.1.0.0-alpha1\lib\net35\Mapbox.Geometry.dll + True + @@ -76,6 +80,9 @@ Util + + +