fix: perform get tile with known format to reduce requests(MAPCO-8749)

MergerCli/Process.cs

@@ -54,10 +54,10 @@ public void Start(IData baseData, IData newData, BatchStatusManager batchStatusM
  }
 
  this._logger.LogInformation($"[{MethodBase.GetCurrentMethod()?.Name}] Total amount of tiles to merge: {totalTileCount - tileProgressCount}");
-
+
  ParallelRun(baseData, newData, batchStatusManager,
  tileProgressCount, totalTileCount, resumeBatchIdentifier, resumeMode, pollForBatch);
-
+
  batchStatusManager.CompleteLayer(newData.Path);
  newData.Reset();
  // base data wrap up is in program as the same base data object is used in multiple calls 
@@ -79,7 +79,7 @@ public void Start(IData baseData, IData newData, BatchStatusManager batchStatusM
  newData.setBatchIdentifier(resumeBatchIdentifier);
  }
  }
-
+
  var incompleteBatch = batchStatusManager.GetFirstIncompleteBatch(newData.Path);
  if (incompleteBatch is not null)
  {
@@ -88,21 +88,21 @@ public void Start(IData baseData, IData newData, BatchStatusManager batchStatusM
  }
 
  List<Tile> newTiles = newData.GetNextBatch(out string? currentBatchIdentifier, out string? nextBatchIdentifier, totalTileCount);
-
+
  if (!resumeMode && newTiles.Count != 0)
  {
  batchStatusManager.AssignBatch(newData.Path, currentBatchIdentifier);
  batchStatusManager.SetCurrentBatch(newData.Path, nextBatchIdentifier);
  }
-
+
  return (newTiles, currentBatchIdentifier);
  }
  }
-
- private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProgressCount, long totalTileCount,ref bool pollForBatch)
+
+ private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProgressCount, long totalTileCount,ref bool pollForBatch)
  {
  ConcurrentBag<Tile> tiles = new ConcurrentBag<Tile>();
-
+
  if (newTiles.Count == 0)
  {
  pollForBatch = false;
@@ -118,7 +118,7 @@ private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProg
  var targetCoords = newTile.GetCoord();
  List<CorrespondingTileBuilder> correspondingTileBuilders = new List<CorrespondingTileBuilder>()
  {
- () => baseData.GetCorrespondingTile(targetCoords, shouldUpscale),
+ () => baseData.GetCorrespondingTile(targetCoords, null, shouldUpscale),
  () => newTile
  };
 
@@ -137,7 +137,7 @@ private void ProcessBatch(IData baseData, List<Tile> newTiles, ref long tileProg
  }
 
  private void ParallelRun(IData baseData, IData newData,
- BatchStatusManager batchStatusManager, long tileProgressCount, long totalTileCount, string? resumeBatchIdentifier, bool resumeMode,bool pollForBatch)
+ BatchStatusManager batchStatusManager, long tileProgressCount, long totalTileCount, string? resumeBatchIdentifier, bool resumeMode,bool pollForBatch)
  {
  var numOfThreads = this._configManager.GetConfiguration<int>("GENERAL", "parallel", "numOfThreads");
  Parallel.For(0, numOfThreads, new ParallelOptions { MaxDegreeOfParallelism = -1 }, _ =>
@@ -151,7 +151,7 @@ private void ParallelRun(IData baseData, IData newData,
  }
  });
  }
-
+
  public void Validate(IData baseData, IData newData, string? incompleteBatchIdentifier)
  {
  List<Tile> newTiles;
@@ -170,7 +170,7 @@ public void Validate(IData baseData, IData newData, string? incompleteBatchIdent
  for (int i = 0; i < newTiles.Count; i++)
  {
  Tile newTile = newTiles[i];
- bool baseTileExists = baseData.TileExists(newTile.GetCoord());
+ bool baseTileExists = baseData.TileExists(newTile.GetCoord(), null);
 
  if (baseTileExists)
  {

MergerLogic/Clients/FileClient.cs

@@ -1,22 +1,28 @@
 using MergerLogic.Batching;
 using System.IO.Abstractions;
 using MergerLogic.Utils;
+using MergerLogic.ImageProcessing;
 
 namespace MergerLogic.Clients;
 
 public class FileClient : DataUtils, IFileClient
 {
  private readonly IFileSystem _fileSystem;
 
- public FileClient(string path, IGeoUtils geoUtils, IFileSystem fileSystem)
+ public FileClient(string path, IGeoUtils geoUtils, IFileSystem fileSystem)
  : base(path, geoUtils)
  {
  this._fileSystem = fileSystem;
  }
 
- public override Tile? GetTile(int z, int x, int y)
+ public override Tile? GetTile(int z, int x, int y, TileFormat? format)
  {
- var tilePath = this.GetTilePath(z, x, y);
+ if (format is null)
+ {
+ throw new ArgumentNullException(nameof(format));
+ }
+
+ var tilePath = this.GetTilePath(z, x, y, format.Value);
 
  if (tilePath != null)
  {
@@ -29,23 +35,23 @@ public FileClient(string path, IGeoUtils geoUtils, IFileSystem fileSystem)
  }
  }
 
- public override bool TileExists(int z, int x, int y)
+ public override bool TileExists(int z, int x, int y, TileFormat? format)
  {
- return this.GetTilePath(z,x,y) != null;
+ if (format is null)
+ {
+ throw new ArgumentNullException(nameof(format));
+ }
+ return this.GetTilePath(z, x, y, format.Value) != null;
  }
 
- private string? GetTilePath(int z, int x, int y)
+ private string? GetTilePath(int z, int x, int y, TileFormat format)
  {
  var tilePath = this._fileSystem.Path.Join(z.ToString(), x.ToString(), y.ToString());
- try
- {
- //this may or may not be faster then checking specific files of every supported type depending on the used file system
- return this._fileSystem.Directory
- .EnumerateFiles(this.path, $"{tilePath}.*", SearchOption.TopDirectoryOnly).FirstOrDefault();
- }
- catch (DirectoryNotFoundException)
+ string fullPath = this._fileSystem.Path.Join(this.path, tilePath, ".", format.ToString().ToLower());
+ if (this._fileSystem.File.Exists(fullPath))
  {
- return null;
+ return fullPath;
  }
+ return null;
  }
 }

MergerLogic/Clients/GpkgClient.cs

@@ -1,5 +1,6 @@
 using MergerLogic.Batching;
 using MergerLogic.Extensions;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Utils;
 using Microsoft.Extensions.Logging;
 using System.Data.SQLite;
@@ -124,7 +125,7 @@ public void UpdateExtent(Extent extent)
  }
  }
 
- public override Tile? GetTile(int z, int x, int y)
+ public override Tile? GetTile(int z, int x, int y, TileFormat? format = null)
  {
  byte[]? blob = null;
 
@@ -146,7 +147,7 @@ public void UpdateExtent(Extent extent)
  return this.CreateTile(z, x, y, blob);
  }
 
- public override bool TileExists(int z, int x, int y)
+ public override bool TileExists(int z, int x, int y, TileFormat? format = null)
  {
  using (var connection = new SQLiteConnection($"Data Source={this.path}"))
  {

MergerLogic/Clients/HttpSourceClient.cs

@@ -1,4 +1,5 @@
 using MergerLogic.Batching;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Utils;
 
 namespace MergerLogic.Clients
@@ -15,14 +16,14 @@ public HttpSourceClient(string path, IHttpRequestUtils httpClient, IPathPatternU
  this._pathPatternUtils = pathPatternUtils;
  }
 
- public override Tile? GetTile(int z, int x, int y)
+ public override Tile? GetTile(int z, int x, int y, TileFormat? format = null)
  {
  string url = this._pathPatternUtils.RenderUrlTemplate(x, y, z);
  byte[]? data = this._httpClient.GetData(url, true);
  return this.CreateTile(z, x, y, data);
  }
 
- public override bool TileExists(int z, int x, int y)
+ public override bool TileExists(int z, int x, int y, TileFormat? format = null)
  {
  return this.GetTile(z, x, y) is not null;
  }

MergerLogic/Clients/S3Client.cs

@@ -39,7 +39,7 @@
  {
  return en.ErrorCode == "NoSuchKey";
  }
-
+
  return false;
  }
 
@@ -79,21 +79,20 @@
  }
  }
 
- public override Tile? GetTile(int z, int x, int y)
+ public override Tile? GetTile(int z, int x, int y, TileFormat? format)
  {
  string methodName = MethodBase.GetCurrentMethod().Name;
  this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}");
- string keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, TileFormat.Jpeg, true);
+ if (format is null)
+ {
+ throw new ArgumentNullException(nameof(format));
+ }
+ string keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, format.Value, true);
 
  byte[]? imageBytes = this.GetImageBytes(keyPrefix);
  if (imageBytes == null)
  {
- keyPrefix = this._pathUtils.GetTilePath(this.path, z, x, y, TileFormat.Png, true);
- imageBytes = this.GetImageBytes(keyPrefix);
- if (imageBytes == null)
- {
- return null;
- }
+ return null;
  }
 
  this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}");
@@ -102,22 +101,22 @@
 
  public Tile? GetTile(string key)
  {
  string methodName = MethodBase.GetCurrentMethod().Name;
  this._logger.LogDebug($"[{methodName}] start key: {key}");
  byte[]? imageBytes = this.GetImageBytes(key);
  if (imageBytes == null)
  {
  return null;
  }
-
+
  this._logger.LogDebug($"[{methodName}] end key: {key}");
  Coord coords = this._pathUtils.FromPath(key, true);
  return this.CreateTile(coords, imageBytes);
  }
 
- public override bool TileExists(int z, int x, int y)
+ public override bool TileExists(int z, int x, int y, TileFormat? format = null)
  {
  string methodName = MethodBase.GetCurrentMethod().Name;
  this._logger.LogDebug($"[{methodName}] start z: {z}, x: {x}, y: {y}");
  bool exists = this.GetTileKey(z, x, y) != null;
  this._logger.LogDebug($"[{methodName}] end z: {z}, x: {x}, y: {y}");
@@ -126,7 +125,7 @@
 
  public void UpdateTile(Tile tile)
  {
  string methodName = MethodBase.GetCurrentMethod().Name;
  this._logger.LogDebug($"[{methodName}] start {tile.ToString()}");
  string key = this._pathUtils.GetTilePath(this.path, tile, true);
 
@@ -151,7 +150,7 @@
 
  private string? GetTileKey(int z, int x, int y)
  {
  string methodName = MethodBase.GetCurrentMethod().Name;
  string keyPrefix = this._pathUtils.GetTilePathWithoutExtension(this.path, z, x, y, true);
 
  try

MergerLogic/DataTypes/Data.cs

@@ -1,4 +1,6 @@
+using Amazon.S3.Model;
 using MergerLogic.Batching;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Monitoring.Metrics;
 using MergerLogic.Utils;
 using Microsoft.Extensions.DependencyInjection;
@@ -41,9 +43,9 @@ public abstract class Data<TUtilsType> : IData where TUtilsType : IDataUtils
  public const int MaxZoomRead = 25;
 
  protected delegate int ValFromCoordFunction(Coord coord);
- protected delegate Tile? GetTileFromXYZFunction(int z, int x, int y);
+ protected delegate Tile? GetTileFromXYZFunction(int z, int x, int y, TileFormat? format);
  protected delegate Coord? GetCoordFromCoordFunction(Coord coord);
- protected delegate Tile? GetTileFromCoordFunction(Coord coord);
+ protected delegate Tile? GetTileFromCoordFunction(Coord coord, TileFormat? format);
  protected delegate Tile TileConvertorFunction(Tile tile);
  protected delegate Tile? NullableTileConvertorFunction(Tile tile);
  protected IServiceProvider _container;
@@ -125,10 +127,10 @@ protected Data(IServiceProvider container, DataType type, string path, int batch
  {
  return this.GeoUtils.FlipY(coord);
  };
- this.GetTile = (z, x, y) =>
+ this.GetTile = (z, x, y, format) =>
  {
  int newY = this.GeoUtils.FlipY(z, y);
- Tile? tile = fixedGridGetTileFunction(z, x, newY);
+ Tile? tile = fixedGridGetTileFunction(z, x, newY, format);
  //set cords to current origin
  tile?.SetCoords(z, x, y);
  return tile;
@@ -197,7 +199,7 @@ protected virtual Extent GetExtent()
 
  public abstract void Reset();
 
- protected virtual Tile? InternalGetLastExistingTile(Coord coords)
+ protected virtual Tile? InternalGetLastExistingTile(Coord coords, TileFormat? format)
  {
  this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] started for coord: z:{coords.Z}, x:{coords.X}, y:{coords.Y}");
  // get tiles coordinates
@@ -224,7 +226,7 @@ await Parallel.ForEachAsync(coordsArray, async (coord, cancellationToken) =>
  {
  await Task.Run(() =>
  {
- Tile? tile = this.Utils.GetTile(coord.Z, coord.X, coord.Y);
+ Tile? tile = this.Utils.GetTile(coord.Z, coord.X, coord.Y, format);
  if (tile != null)
  {
  zOrderToTileDictionary.TryAdd(coord.Z, tile);
@@ -247,12 +249,12 @@ await Task.Run(() =>
  return lastTile;
  }
 
- public bool TileExists(Tile tile)
+ public bool TileExists(Tile tile, TileFormat? format)
  {
- return this.TileExists(tile.GetCoord());
+ return this.TileExists(tile.GetCoord(), format);
  }
 
- public bool TileExists(Coord coord)
+ public bool TileExists(Coord coord, TileFormat? format)
  {
  coord.Y = this.ConvertOriginCoord(coord);
  Coord? newCoord = this.FromCurrentGridCoord(coord);
@@ -262,43 +264,43 @@ public bool TileExists(Coord coord)
  return false;
  }
 
- return this.Utils.TileExists(newCoord.Z, newCoord.X, newCoord.Y);
+ return this.Utils.TileExists(newCoord.Z, newCoord.X, newCoord.Y, format);
  }
 
  //TODO: move to util after IOC
- protected Tile? GetLastOneXOneExistingTile(Coord coords)
+ protected Tile? GetLastOneXOneExistingTile(Coord coords, TileFormat? format)
  {
  Coord? newCoords = this.FromCurrentGridCoord(coords);
  if (newCoords is null)
  {
  return null;
  }
- Tile? tile = this.InternalGetLastExistingTile(newCoords);
+ Tile? tile = this.InternalGetLastExistingTile(newCoords, format);
  return tile != null ? this.OneXOneConvertor?.ToTwoXOne(tile) : null;
  }
 
- protected virtual Tile? GetOneXOneTile(int z, int x, int y)
+ protected virtual Tile? GetOneXOneTile(int z, int x, int y, TileFormat? format)
  {
  Coord? oneXoneBaseCoords = this.OneXOneConvertor?.TryFromTwoXOne(z, x, y);
  if (oneXoneBaseCoords == null)
  {
  return null;
  }
- Tile? tile = this.Utils.GetTile(oneXoneBaseCoords);
+ Tile? tile = this.Utils.GetTile(oneXoneBaseCoords, format);
  return tile != null ? this.OneXOneConvertor?.ToTwoXOne(tile) : null;
  }
 
  public abstract List<Tile> GetNextBatch(out string batchIdentifier, out string? nextBatchIdentifier, long? totalTilesCount);
 
- public Tile? GetCorrespondingTile(Coord coords, bool upscale)
+ public Tile? GetCorrespondingTile(Coord coords, TileFormat? format, bool upscale)
  {
  this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] start for coord: z:{coords.Z}, x:{coords.X}, y:{coords.Y}, upscale: {upscale}");
  Stopwatch stopwatch = Stopwatch.StartNew();
- Tile? correspondingTile = this.GetTile(coords.Z, coords.X, coords.Y);
+ Tile? correspondingTile = this.GetTile(coords.Z, coords.X, coords.Y, format);
 
  if (upscale && correspondingTile == null)
  {
- correspondingTile = this.GetLastExistingTile(coords);
+ correspondingTile = this.GetLastExistingTile(coords, format);
  }
  stopwatch.Stop();
  this._metricsProvider.TotalFetchTimePerTileHistogram(stopwatch.Elapsed.TotalMilliseconds);

MergerLogic/DataTypes/Fs.cs

@@ -94,7 +94,7 @@ private IEnumerator<Tile> GetTiles()
  .Where(file => this._supportedFileExtensions.Any(x => file.EndsWith(x, System.StringComparison.OrdinalIgnoreCase))))
  {
  Coord coord = this._pathUtils.FromPath(filePath);
- Tile? tile = this.Utils.GetTile(coord);
+ Tile? tile = this.Utils.GetTile(coord, null);
  if (tile is null)
  {
  continue;

MergerLogic/DataTypes/Gpkg.cs

@@ -1,5 +1,6 @@
 using MergerLogic.Batching;
 using MergerLogic.Clients;
+using MergerLogic.ImageProcessing;
 using MergerLogic.Utils;
 using Microsoft.Extensions.Logging;
 using System.Reflection;
@@ -126,7 +127,7 @@ public override void setBatchIdentifier(string batchIdentifier)
  }
  }
 
- protected override Tile? InternalGetLastExistingTile(Coord baseCoords)
+ protected override Tile? InternalGetLastExistingTile(Coord baseCoords, TileFormat? format = null)
  {
  this._logger.LogDebug($"[{MethodBase.GetCurrentMethod()?.Name}] started for coord: z:{baseCoords.Z}, x:{baseCoords.X}, y:{baseCoords.Y}");
  int cordsLength = baseCoords.Z << 1;