[refactor] get constants from env variables, handle possible exceptions on tiles download/merge/crop

src/__init__.py

@@ -1,5 +1,7 @@
 """Get machine learning predictions from geodata raster images"""
 from aws_lambda_powertools import Logger
+# not used here but contextily_tile is imported in src.io.tms2geotiff
+from contextily import tile as contextily_tile
 from pathlib import Path
 
 from src.utilities.constants import SERVICE_NAME

src/io/tms2geotiff.py

@@ -1,14 +1,22 @@
+import os
 from numpy import ndarray
 
 from src import app_logger
-from src.utilities.constants import OUTPUT_CRS_STRING, DRIVER_RASTERIO_GTIFF, OSM_MAX_RETRIES, OSM_N_CONNECTIONS, \
-    OSM_WAIT, OSM_ZOOM_AUTO, OSM_USE_CACHE
+from src.utilities.constants import (OUTPUT_CRS_STRING, DRIVER_RASTERIO_GTIFF, N_MAX_RETRIES, N_CONNECTION, N_WAIT,
+                                     ZOOM_AUTO, BOOL_USE_CACHE)
 from src.utilities.type_hints import tuple_ndarray_transform, tuple_float
 
 
-def download_extent(w: float, s: float, e: float, n: float, zoom: int or str = OSM_ZOOM_AUTO, source: str = None,
-                    wait: int = OSM_WAIT, max_retries: int = OSM_MAX_RETRIES, n_connections: int = OSM_N_CONNECTIONS,
-                    use_cache: bool = OSM_USE_CACHE) -> tuple_ndarray_transform:
+bool_use_cache = int(os.getenv("BOOL_USE_CACHE", BOOL_USE_CACHE))
+n_connection = int(os.getenv("N_CONNECTION", N_CONNECTION))
+n_max_retries = int(os.getenv("N_MAX_RETRIES", N_MAX_RETRIES))
+n_wait = int(os.getenv("N_WAIT", N_WAIT))
+zoom_auto_string = os.getenv("ZOOM_AUTO", ZOOM_AUTO)
+
+
+def download_extent(w: float, s: float, e: float, n: float, zoom: int or str = zoom_auto_string, source: str = None,
+                    wait: int = n_wait, max_retries: int = n_max_retries, n_connections: int = n_connection,
+                    use_cache: bool = bool_use_cache) -> tuple_ndarray_transform:
     """
     Download, merge and crop a list of tiles into a single geo-referenced image or a raster geodata
 
@@ -42,17 +50,24 @@ def download_extent(w: float, s: float, e: float, n: float, zoom: int or str = O
     Returns:
         parsed request input
     """
-    from contextily.tile import bounds2img
-    from src.io.coordinates_pixel_conversion import _from4326_to3857
-
-    app_logger.debug(f"download raster from source:{source} with bounding box w:{w}, s:{s}, e:{e}, n:{n}.")
-    downloaded_raster, bbox_raster = bounds2img(
-        w, s, e, n, zoom=zoom, source=source, ll=True, wait=wait, max_retries=max_retries, n_connections=n_connections,
-        use_cache=use_cache)
-    xp0, yp0 = _from4326_to3857(n, e)
-    xp1, yp1 = _from4326_to3857(s, w)
-    cropped_image_ndarray, cropped_transform = crop_raster(yp1, xp1, yp0, xp0, downloaded_raster, bbox_raster)
-    return cropped_image_ndarray, cropped_transform
+    try:
+        from src import contextily_tile
+        from src.io.coordinates_pixel_conversion import _from4326_to3857
+
+        app_logger.info(f"connection number:{n_connections}, type:{type(n_connections)}.")
+        app_logger.info(f"zoom:{zoom}, type:{type(zoom)}.")
+        app_logger.debug(f"download raster from source:{source} with bounding box w:{w}, s:{s}, e:{e}, n:{n}.")
+        app_logger.debug(f"types w:{type(w)}, s:{type(s)}, e:{type(e)}, n:{type(n)}.")
+        downloaded_raster, bbox_raster = contextily_tile.bounds2img(
+            w, s, e, n, zoom=zoom, source=source, ll=True, wait=wait, max_retries=max_retries, n_connections=n_connections,
+            use_cache=use_cache)
+        xp0, yp0 = _from4326_to3857(n, e)
+        xp1, yp1 = _from4326_to3857(s, w)
+        cropped_image_ndarray, cropped_transform = crop_raster(yp1, xp1, yp0, xp0, downloaded_raster, bbox_raster)
+        return cropped_image_ndarray, cropped_transform
+    except Exception as e_download_extent:
+        app_logger.exception(f"e_download_extent:{e_download_extent}.", exc_info=True)
+        raise e_download_extent
 
 
 def crop_raster(w: float, s: float, e: float, n: float, raster: ndarray, raster_bbox: tuple_float,
@@ -77,38 +92,42 @@ def crop_raster(w: float, s: float, e: float, n: float, raster: ndarray, raster_
     Returns:
         cropped raster with its Affine transform
     """
-    from rasterio.io import MemoryFile
-    from rasterio.plot import reshape_as_image
-    from rasterio.mask import mask as rio_mask
-    from shapely.geometry import Polygon
-    from geopandas import GeoSeries
-
-    app_logger.debug(f"raster: type {type(raster)}, raster_ext:{type(raster_bbox)}, {raster_bbox}.")
-    img_to_save, transform = get_transform_raster(raster, raster_bbox)
-    img_height, img_width, number_bands = img_to_save.shape
-    # https://rasterio.readthedocs.io/en/latest/topics/memory-files.html
-    with MemoryFile() as rio_mem_file:
-        app_logger.debug("writing raster in-memory to crop it with rasterio.mask.mask()")
-        with rio_mem_file.open(
-                driver=driver,
-                height=img_height,
-                width=img_width,
-                count=number_bands,
-                dtype=str(img_to_save.dtype.name),
-                crs=crs,
-                transform=transform,
-        ) as src_raster_rw:
-            for band in range(number_bands):
-                src_raster_rw.write(img_to_save[:, :, band], band + 1)
-        app_logger.debug("cropping raster in-memory with rasterio.mask.mask()")
-        with rio_mem_file.open() as src_raster_ro:
-            shapes_crop_polygon = Polygon([(n, e), (s, e), (s, w), (n, w), (n, e)])
-            shapes_crop = GeoSeries([shapes_crop_polygon])
-            app_logger.debug(f"cropping with polygon::{shapes_crop_polygon}.")
-            cropped_image, cropped_transform = rio_mask(src_raster_ro, shapes=shapes_crop, crop=True)
-            cropped_image_ndarray = reshape_as_image(cropped_image)
-    app_logger.info(f"cropped image::{cropped_image_ndarray.shape}.")
-    return cropped_image_ndarray, cropped_transform
+    try:
+        from rasterio.io import MemoryFile
+        from rasterio.mask import mask as rio_mask
+        from shapely.geometry import Polygon
+        from geopandas import GeoSeries
+
+        app_logger.debug(f"raster: type {type(raster)}, raster_ext:{type(raster_bbox)}, {raster_bbox}.")
+        img_to_save, transform = get_transform_raster(raster, raster_bbox)
+        img_height, img_width, number_bands = img_to_save.shape
+        # https://rasterio.readthedocs.io/en/latest/topics/memory-files.html
+        with MemoryFile() as rio_mem_file:
+            app_logger.debug("writing raster in-memory to crop it with rasterio.mask.mask()")
+            with rio_mem_file.open(
+                    driver=driver,
+                    height=img_height,
+                    width=img_width,
+                    count=number_bands,
+                    dtype=str(img_to_save.dtype.name),
+                    crs=crs,
+                    transform=transform,
+            ) as src_raster_rw:
+                for band in range(number_bands):
+                    src_raster_rw.write(img_to_save[:, :, band], band + 1)
+            app_logger.debug("cropping raster in-memory with rasterio.mask.mask()")
+            with rio_mem_file.open() as src_raster_ro:
+                shapes_crop_polygon = Polygon([(n, e), (s, e), (s, w), (n, w), (n, e)])
+                shapes_crop = GeoSeries([shapes_crop_polygon])
+                app_logger.debug(f"cropping with polygon::{shapes_crop_polygon}.")
+                cropped_image, cropped_transform = rio_mask(src_raster_ro, shapes=shapes_crop, crop=True)
+                cropped_image_ndarray = reshape_as_image(cropped_image)
+        app_logger.info(f"cropped image::{cropped_image_ndarray.shape}.")
+        return cropped_image_ndarray, cropped_transform
+    except Exception as e_crop_raster:
+        app_logger.exception(f"arguments raster: {type(raster)}, {raster}.")
+        app_logger.exception(f"e_crop_raster:{e_crop_raster}.", exc_info=True)
+        raise e_crop_raster
 
 
 def get_transform_raster(raster: ndarray, raster_bbox: tuple_float) -> tuple_ndarray_transform:
@@ -122,20 +141,37 @@ def get_transform_raster(raster: ndarray, raster_bbox: tuple_float) -> tuple_nda
     Returns:
         rgb raster image and its Affine transform
     """
-    from rasterio.transform import from_origin
-    from numpy import array as np_array, linspace as np_linspace, uint8 as np_uint8
-    from PIL.Image import fromarray
-
-    app_logger.debug(f"raster: type {type(raster)}, raster_ext:{type(raster_bbox)}, {raster_bbox}.")
-    rgb = fromarray(np_uint8(raster)).convert('RGB')
-    np_rgb = np_array(rgb)
-    img_height, img_width, _ = np_rgb.shape
-
-    min_x, max_x, min_y, max_y = raster_bbox
-    app_logger.debug(f"raster rgb shape:{np_rgb.shape}, raster rgb bbox {raster_bbox}.")
-    x = np_linspace(min_x, max_x, img_width)
-    y = np_linspace(min_y, max_y, img_height)
-    res_x = (x[-1] - x[0]) / img_width
-    res_y = (y[-1] - y[0]) / img_height
-    transform = from_origin(x[0] - res_x / 2, y[-1] + res_y / 2, res_x, res_y)
-    return np_rgb, transform
+    try:
+        from rasterio.transform import from_origin
+        from numpy import array as np_array, linspace as np_linspace, uint8 as np_uint8
+        from PIL.Image import fromarray
+
+        app_logger.debug(f"raster: type {type(raster)}, raster_ext:{type(raster_bbox)}, {raster_bbox}.")
+        rgb = fromarray(np_uint8(raster)).convert('RGB')
+        np_rgb = np_array(rgb)
+        img_height, img_width, _ = np_rgb.shape
+
+        min_x, max_x, min_y, max_y = raster_bbox
+        app_logger.debug(f"raster rgb shape:{np_rgb.shape}, raster rgb bbox {raster_bbox}.")
+        x = np_linspace(min_x, max_x, img_width)
+        y = np_linspace(min_y, max_y, img_height)
+        res_x = (x[-1] - x[0]) / img_width
+        res_y = (y[-1] - y[0]) / img_height
+        transform = from_origin(x[0] - res_x / 2, y[-1] + res_y / 2, res_x, res_y)
+        return np_rgb, transform
+    except Exception as e_get_transform_raster:
+        app_logger.exception(f"arguments raster: {type(raster)}, {raster}.")
+        app_logger.exception(f"arguments raster_bbox: {type(raster_bbox)}, {raster_bbox}.")
+        app_logger.exception(f"e_get_transform_raster:{e_get_transform_raster}.")
+        raise e_get_transform_raster
+
+
+def reshape_as_image(arr):
+    try:
+        from numpy import swapaxes
+
+        return swapaxes(swapaxes(arr, 0, 2), 0, 1)
+    except Exception as e_reshape_as_image:
+        app_logger.exception(f"arguments: {type(arr)}, {arr}.")
+        app_logger.exception(f"e_reshape_as_image:{e_reshape_as_image}.", exc_info=True)
+        raise e_reshape_as_image

src/utilities/constants.py

@@ -26,8 +26,8 @@ DEFAULT_LOG_LEVEL = 'INFO'
 RETRY_DOWNLOAD = 3
 TIMEOUT_DOWNLOAD = 60
 CALLBACK_INTERVAL_DOWNLOAD = 0.05
-OSM_USE_CACHE = True
-OSM_WAIT = 0
-OSM_MAX_RETRIES = 2
-OSM_N_CONNECTIONS = 1
-OSM_ZOOM_AUTO = "auto"
+BOOL_USE_CACHE = True
+N_WAIT = 0
+N_MAX_RETRIES = 2
+N_CONNECTION = 2
+ZOOM_AUTO = "auto"