[refactor] download raster tiles with contextily.tile.bounds2img()
Browse files- .idea/.gitignore +0 -3
- requirements.txt +1 -0
- requirements_dockerfile.txt +1 -0
- src/io/coordinates_pixel_conversion.py +20 -3
- src/io/geo_helpers.py +3 -4
- src/io/tms2geotiff.py +131 -236
- src/prediction_api/predictors.py +10 -14
- src/utilities/constants.py +6 -0
- src/utilities/type_hints.py +2 -0
.idea/.gitignore
DELETED
@@ -1,3 +0,0 @@
|
|
1 |
-
# Default ignored files
|
2 |
-
/shelf/
|
3 |
-
/workspace.xml
|
|
|
|
|
|
|
|
requirements.txt
CHANGED
@@ -1,6 +1,7 @@
|
|
1 |
aws-lambda-powertools
|
2 |
awslambdaric
|
3 |
bson
|
|
|
4 |
geopandas
|
5 |
jmespath
|
6 |
myst-parser
|
|
|
1 |
aws-lambda-powertools
|
2 |
awslambdaric
|
3 |
bson
|
4 |
+
contextily
|
5 |
geopandas
|
6 |
jmespath
|
7 |
myst-parser
|
requirements_dockerfile.txt
CHANGED
@@ -1,6 +1,7 @@
|
|
1 |
aws-lambda-powertools
|
2 |
awslambdaric
|
3 |
bson
|
|
|
4 |
geopandas
|
5 |
jmespath
|
6 |
numpy
|
|
|
1 |
aws-lambda-powertools
|
2 |
awslambdaric
|
3 |
bson
|
4 |
+
contextily
|
5 |
geopandas
|
6 |
jmespath
|
7 |
numpy
|
src/io/coordinates_pixel_conversion.py
CHANGED
@@ -1,7 +1,7 @@
|
|
1 |
"""functions useful to convert to/from latitude-longitude coordinates to pixel image coordinates"""
|
2 |
from src import app_logger
|
3 |
-
from src.utilities.constants import TILE_SIZE
|
4 |
-
from src.utilities.type_hints import ImagePixelCoordinates
|
5 |
from src.utilities.type_hints import LatLngDict
|
6 |
|
7 |
|
@@ -58,7 +58,7 @@ def get_latlng_to_pixel_coordinates(
|
|
58 |
latlng_origin_ne: NE latitude-longitude origin point
|
59 |
latlng_origin_sw: SW latitude-longitude origin point
|
60 |
latlng_current_point: latitude-longitude prompt point
|
61 |
-
zoom:
|
62 |
k: prompt type
|
63 |
|
64 |
Returns:
|
@@ -74,3 +74,20 @@ def get_latlng_to_pixel_coordinates(
|
|
74 |
point = ImagePixelCoordinates(x=diff_coord_x, y=diff_coord_y)
|
75 |
app_logger.debug(f"point type - {k}: {point}.")
|
76 |
return point
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
"""functions useful to convert to/from latitude-longitude coordinates to pixel image coordinates"""
|
2 |
from src import app_logger
|
3 |
+
from src.utilities.constants import TILE_SIZE, EARTH_EQUATORIAL_RADIUS
|
4 |
+
from src.utilities.type_hints import ImagePixelCoordinates, tuple_float, tuple_float_any
|
5 |
from src.utilities.type_hints import LatLngDict
|
6 |
|
7 |
|
|
|
58 |
latlng_origin_ne: NE latitude-longitude origin point
|
59 |
latlng_origin_sw: SW latitude-longitude origin point
|
60 |
latlng_current_point: latitude-longitude prompt point
|
61 |
+
zoom: Level of detail
|
62 |
k: prompt type
|
63 |
|
64 |
Returns:
|
|
|
74 |
point = ImagePixelCoordinates(x=diff_coord_x, y=diff_coord_y)
|
75 |
app_logger.debug(f"point type - {k}: {point}.")
|
76 |
return point
|
77 |
+
|
78 |
+
|
79 |
+
def _from4326_to3857(lat: float, lon: float) -> tuple_float or tuple_float_any:
|
80 |
+
from math import radians, log, tan
|
81 |
+
|
82 |
+
x_tile: float = radians(lon) * EARTH_EQUATORIAL_RADIUS
|
83 |
+
y_tile: float = log(tan(radians(45 + lat / 2.0))) * EARTH_EQUATORIAL_RADIUS
|
84 |
+
return x_tile, y_tile
|
85 |
+
|
86 |
+
|
87 |
+
def _deg2num(lat: float, lon: float, zoom: int):
|
88 |
+
from math import radians, pi, asinh, tan
|
89 |
+
|
90 |
+
n = 2 ** zoom
|
91 |
+
x_tile = ((lon + 180) / 360 * n)
|
92 |
+
y_tile = (1 - asinh(tan(radians(lat))) / pi) * n / 2
|
93 |
+
return x_tile, y_tile
|
src/io/geo_helpers.py
CHANGED
@@ -42,13 +42,13 @@ def get_affine_transform_from_gdal(matrix_source_coefficients: list_float or tup
|
|
42 |
return Affine.from_gdal(*matrix_source_coefficients)
|
43 |
|
44 |
|
45 |
-
def get_vectorized_raster_as_geojson(mask: np_ndarray,
|
46 |
"""
|
47 |
Get shapes and values of connected regions in a dataset or array
|
48 |
|
49 |
Args:
|
50 |
mask: numpy mask
|
51 |
-
|
52 |
|
53 |
Returns:
|
54 |
dict containing the output geojson and the predictions number
|
@@ -57,8 +57,7 @@ def get_vectorized_raster_as_geojson(mask: np_ndarray, matrix: tuple_float) -> d
|
|
57 |
from rasterio.features import shapes
|
58 |
from geopandas import GeoDataFrame
|
59 |
|
60 |
-
|
61 |
-
app_logger.info(f"transform to consume with rasterio.shapes: {type(transform)}, {transform}.")
|
62 |
|
63 |
# old value for mask => band != 0
|
64 |
shapes_generator = ({
|
|
|
42 |
return Affine.from_gdal(*matrix_source_coefficients)
|
43 |
|
44 |
|
45 |
+
def get_vectorized_raster_as_geojson(mask: np_ndarray, transform: tuple_float) -> dict_str_int:
|
46 |
"""
|
47 |
Get shapes and values of connected regions in a dataset or array
|
48 |
|
49 |
Args:
|
50 |
mask: numpy mask
|
51 |
+
transform: tuple of float to transform into an Affine transform
|
52 |
|
53 |
Returns:
|
54 |
dict containing the output geojson and the predictions number
|
|
|
57 |
from rasterio.features import shapes
|
58 |
from geopandas import GeoDataFrame
|
59 |
|
60 |
+
app_logger.debug(f"matrix to consume with rasterio.shapes: {type(transform)}, {transform}.")
|
|
|
61 |
|
62 |
# old value for mask => band != 0
|
63 |
shapes_generator = ({
|
src/io/tms2geotiff.py
CHANGED
@@ -1,246 +1,141 @@
|
|
1 |
-
|
2 |
-
Download geo-referenced raster tiles images.
|
3 |
-
Modified from https://github.com/gumblex/tms2geotiff/
|
4 |
-
|
5 |
-
BSD 2-Clause License
|
6 |
-
|
7 |
-
Copyright (c) 2019, Dingyuan Wang
|
8 |
-
All rights reserved.
|
9 |
-
|
10 |
-
Redistribution and use in source and binary forms, with or without
|
11 |
-
modification, are permitted provided that the following conditions are met:
|
12 |
-
|
13 |
-
* Redistributions of source code must retain the above copyright notice, this
|
14 |
-
list of conditions and the following disclaimer.
|
15 |
-
|
16 |
-
* Redistributions in binary form must reproduce the above copyright notice,
|
17 |
-
this list of conditions and the following disclaimer in the documentation
|
18 |
-
and/or other materials provided with the distribution.
|
19 |
-
|
20 |
-
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
21 |
-
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
22 |
-
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
23 |
-
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
24 |
-
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
25 |
-
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
26 |
-
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
27 |
-
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
28 |
-
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
29 |
-
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
30 |
-
"""
|
31 |
-
import concurrent.futures
|
32 |
-
import io
|
33 |
-
import itertools
|
34 |
-
import math
|
35 |
-
import re
|
36 |
-
import time
|
37 |
-
from typing import Tuple, Callable
|
38 |
-
from PIL import Image
|
39 |
|
40 |
from src import app_logger
|
41 |
-
from src.utilities.constants import
|
42 |
-
|
43 |
-
from src.utilities.type_hints import
|
44 |
-
|
45 |
-
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
SESSION = httpx.Client()
|
51 |
-
except ImportError:
|
52 |
-
import requests
|
53 |
-
|
54 |
-
SESSION = requests.Session()
|
55 |
-
|
56 |
-
SESSION.headers.update({
|
57 |
-
"Accept": "*/*",
|
58 |
-
"Accept-Encoding": "gzip, deflate",
|
59 |
-
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; rv:91.0) Gecko/20100101 Firefox/91.0",
|
60 |
-
})
|
61 |
-
|
62 |
-
re_coords_split = re.compile('[ ,;]+')
|
63 |
-
|
64 |
-
|
65 |
-
def _from4326_to3857(lat: float, lon: float) -> tuple_float or tuple_float_any:
|
66 |
-
x_tile: float = math.radians(lon) * EARTH_EQUATORIAL_RADIUS
|
67 |
-
y_tile: float = math.log(math.tan(math.radians(45 + lat / 2.0))) * EARTH_EQUATORIAL_RADIUS
|
68 |
-
return x_tile, y_tile
|
69 |
-
|
70 |
-
|
71 |
-
def _deg2num(lat: float, lon: float, zoom: int):
|
72 |
-
n = 2 ** zoom
|
73 |
-
x_tile = ((lon + 180) / 360 * n)
|
74 |
-
y_tile = (1 - math.asinh(math.tan(math.radians(lat))) / math.pi) * n / 2
|
75 |
-
return x_tile, y_tile
|
76 |
-
|
77 |
-
|
78 |
-
def _is_empty(image):
|
79 |
-
extrema = image.getextrema()
|
80 |
-
if len(extrema) >= 3:
|
81 |
-
if len(extrema) > 3 and extrema[-1] == (0, 0):
|
82 |
-
return True
|
83 |
-
for ext in extrema[:3]:
|
84 |
-
if ext != (0, 0):
|
85 |
-
return False
|
86 |
-
return True
|
87 |
-
return extrema[0] == (0, 0)
|
88 |
-
|
89 |
-
|
90 |
-
def _paste_tile(big_image: PIL_Image or None, base_size: list_int, tile: bytes, corner_xy: tuple_int, bbox: tuple_int):
|
91 |
-
if tile is None:
|
92 |
-
return big_image
|
93 |
-
with Image.open(io.BytesIO(tile)) as tmp_image:
|
94 |
-
mode = 'RGB' if tmp_image.mode == 'RGB' else 'RGBA'
|
95 |
-
size = tmp_image.size
|
96 |
-
new_image = big_image
|
97 |
-
if big_image is None:
|
98 |
-
base_size[0] = size[0]
|
99 |
-
base_size[1] = size[1]
|
100 |
-
new_image = Image.new(mode, (
|
101 |
-
size[0] * (bbox[2] - bbox[0]), size[1] * (bbox[3] - bbox[1])))
|
102 |
-
|
103 |
-
dx = abs(corner_xy[0] - bbox[0])
|
104 |
-
dy = abs(corner_xy[1] - bbox[1])
|
105 |
-
xy0 = (size[0] * dx, size[1] * dy)
|
106 |
-
if mode == 'RGB':
|
107 |
-
new_image.paste(tmp_image, xy0)
|
108 |
-
else:
|
109 |
-
if tmp_image.mode != mode:
|
110 |
-
tmp_image = tmp_image.convert(mode)
|
111 |
-
if not _is_empty(tmp_image):
|
112 |
-
new_image.paste(tmp_image, xy0)
|
113 |
-
return new_image
|
114 |
-
|
115 |
-
|
116 |
-
def _get_tile(url: str) -> bytes or None:
|
117 |
-
retry = RETRY_DOWNLOAD
|
118 |
-
while 1:
|
119 |
-
try:
|
120 |
-
app_logger.debug(f"image tile url to download: {url}.")
|
121 |
-
r = SESSION.get(url, timeout=TIMEOUT_DOWNLOAD)
|
122 |
-
break
|
123 |
-
except Exception as request_tile_exception:
|
124 |
-
app_logger.error(f"retry {retry}, request_tile_exception:{request_tile_exception}.")
|
125 |
-
retry -= 1
|
126 |
-
if not retry:
|
127 |
-
raise
|
128 |
-
if r.status_code == 404 or not r.content:
|
129 |
-
return None
|
130 |
-
r.raise_for_status()
|
131 |
-
return r.content
|
132 |
-
|
133 |
-
|
134 |
-
def log_progress(progress, total, done=False):
|
135 |
-
"""log the progress download"""
|
136 |
-
if done:
|
137 |
-
app_logger.info('Downloaded image %d/%d, %.2f%%' % (progress, total, progress * 100 / total))
|
138 |
-
|
139 |
-
|
140 |
-
def download_extent(
|
141 |
-
source: str, lat0: float, lon0: float, lat1: float, lon1: float, zoom: int,
|
142 |
-
save_image: bool = True, progress_callback: Callable = log_progress,
|
143 |
-
callback_interval: float = CALLBACK_INTERVAL_DOWNLOAD
|
144 |
-
) -> Tuple[PIL_Image, Tuple[float]] or Tuple[None]:
|
145 |
"""
|
146 |
Download, merge and crop a list of tiles into a single geo-referenced image or a raster geodata
|
147 |
|
148 |
Args:
|
149 |
-
|
150 |
-
|
151 |
-
|
152 |
-
|
153 |
-
|
154 |
-
|
155 |
-
|
156 |
-
|
157 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
158 |
|
159 |
Returns:
|
160 |
parsed request input
|
161 |
"""
|
162 |
-
|
163 |
-
|
164 |
-
|
165 |
-
|
166 |
-
|
167 |
-
|
168 |
-
|
169 |
-
|
170 |
-
|
171 |
-
|
172 |
-
|
173 |
-
|
174 |
-
|
175 |
-
|
176 |
-
|
177 |
-
|
178 |
-
|
179 |
-
|
180 |
-
|
181 |
-
|
182 |
-
|
183 |
-
|
184 |
-
|
185 |
-
|
186 |
-
|
187 |
-
|
188 |
-
|
189 |
-
|
190 |
-
|
191 |
-
|
192 |
-
|
193 |
-
|
194 |
-
|
195 |
-
|
196 |
-
|
197 |
-
|
198 |
-
|
199 |
-
|
200 |
-
|
201 |
-
|
202 |
-
|
203 |
-
|
204 |
-
|
205 |
-
|
206 |
-
|
207 |
-
|
208 |
-
|
209 |
-
|
210 |
-
|
211 |
-
|
212 |
-
|
213 |
-
|
214 |
-
|
215 |
-
|
216 |
-
|
217 |
-
|
218 |
-
):
|
219 |
-
|
220 |
-
|
221 |
-
|
222 |
-
|
223 |
-
|
224 |
-
|
225 |
-
|
226 |
-
|
227 |
-
|
228 |
-
|
229 |
-
|
230 |
-
|
231 |
-
|
232 |
-
|
233 |
-
|
234 |
-
|
235 |
-
|
236 |
-
|
237 |
-
|
238 |
-
|
239 |
-
|
240 |
-
|
241 |
-
|
242 |
-
|
243 |
-
|
244 |
-
|
245 |
-
|
246 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
from numpy import ndarray
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
2 |
|
3 |
from src import app_logger
|
4 |
+
from src.utilities.constants import OUTPUT_CRS_STRING, DRIVER_RASTERIO_GTIFF, OSM_MAX_RETRIES, OSM_N_CONNECTIONS, \
|
5 |
+
OSM_WAIT, OSM_ZOOM_AUTO, OSM_USE_CACHE
|
6 |
+
from src.utilities.type_hints import tuple_ndarray_transform, tuple_float
|
7 |
+
|
8 |
+
|
9 |
+
def download_extent(w: float, s: float, e: float, n: float, zoom: int or str = OSM_ZOOM_AUTO, source: str = None,
|
10 |
+
wait: int = OSM_WAIT, max_retries: int = OSM_MAX_RETRIES, n_connections: int = OSM_N_CONNECTIONS,
|
11 |
+
use_cache: bool = OSM_USE_CACHE) -> tuple_ndarray_transform:
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
"""
|
13 |
Download, merge and crop a list of tiles into a single geo-referenced image or a raster geodata
|
14 |
|
15 |
Args:
|
16 |
+
w: West edge
|
17 |
+
s: South edge
|
18 |
+
e: East edge
|
19 |
+
n: North edge
|
20 |
+
zoom: Level of detail
|
21 |
+
source: xyzservices.TileProvider object or str
|
22 |
+
[Optional. Default: OpenStreetMap Humanitarian web tiles]
|
23 |
+
The tile source: web tile provider or path to local file. The web tile
|
24 |
+
provider can be in the form of a :class:`xyzservices.TileProvider` object or a
|
25 |
+
URL. The placeholders for the XYZ in the URL need to be `{x}`, `{y}`,
|
26 |
+
`{z}`, respectively. For local file paths, the file is read with
|
27 |
+
`rasterio` and all bands are loaded into the basemap.
|
28 |
+
IMPORTANT: tiles are assumed to be in the Spherical Mercator
|
29 |
+
projection (EPSG:3857), unless the `crs` keyword is specified.
|
30 |
+
wait: if the tile API is rate-limited, the number of seconds to wait
|
31 |
+
between a failed request and the next try
|
32 |
+
max_retries: total number of rejected requests allowed before contextily will stop trying to fetch more tiles
|
33 |
+
from a rate-limited API.
|
34 |
+
n_connections: Number of connections for downloading tiles in parallel. Be careful not to overload the tile
|
35 |
+
server and to check the tile provider's terms of use before increasing this value. E.g., OpenStreetMap has
|
36 |
+
a max. value of 2 (https://operations.osmfoundation.org/policies/tiles/). If allowed to download in
|
37 |
+
parallel, a recommended value for n_connections is 16, and should never be larger than 64.
|
38 |
+
use_cache: If False, caching of the downloaded tiles will be disabled. This can be useful in resource
|
39 |
+
constrained environments, especially when using n_connections > 1, or when a tile provider's terms of use
|
40 |
+
don't allow caching.
|
41 |
|
42 |
Returns:
|
43 |
parsed request input
|
44 |
"""
|
45 |
+
from contextily.tile import bounds2img
|
46 |
+
from src.io.coordinates_pixel_conversion import _from4326_to3857
|
47 |
+
|
48 |
+
app_logger.debug(f"download raster from source:{source} with bounding box w:{w}, s:{s}, e:{e}, n:{n}.")
|
49 |
+
downloaded_raster, bbox_raster = bounds2img(
|
50 |
+
w, s, e, n, zoom=zoom, source=source, ll=True, wait=wait, max_retries=max_retries, n_connections=n_connections,
|
51 |
+
use_cache=use_cache)
|
52 |
+
xp0, yp0 = _from4326_to3857(n, e)
|
53 |
+
xp1, yp1 = _from4326_to3857(s, w)
|
54 |
+
cropped_image_ndarray, cropped_transform = crop_raster(yp1, xp1, yp0, xp0, downloaded_raster, bbox_raster)
|
55 |
+
return cropped_image_ndarray, cropped_transform
|
56 |
+
|
57 |
+
|
58 |
+
def crop_raster(w: float, s: float, e: float, n: float, raster: ndarray, raster_bbox: tuple_float,
|
59 |
+
crs: str = OUTPUT_CRS_STRING, driver: str = DRIVER_RASTERIO_GTIFF) -> tuple_ndarray_transform:
|
60 |
+
"""
|
61 |
+
Crop a raster using given bounding box (w, s, e, n) values
|
62 |
+
|
63 |
+
Args:
|
64 |
+
w: cropping west edge
|
65 |
+
s: cropping south edge
|
66 |
+
e: cropping east edge
|
67 |
+
n: cropping north edge
|
68 |
+
raster: raster image to crop
|
69 |
+
raster_bbox: bounding box of raster to crop
|
70 |
+
crs: The coordinate reference system. Required in 'w' or 'w+' modes, it is ignored in 'r' or 'r+' modes.
|
71 |
+
driver: A short format driver name (e.g. "GTiff" or "JPEG") or a list of such names (see GDAL docs at
|
72 |
+
https://gdal.org/drivers/raster/index.html ). In 'w' or 'w+' modes a single name is required. In 'r' or 'r+'
|
73 |
+
modes the driver can usually be omitted. Registered drivers will be tried sequentially until a match is
|
74 |
+
found. When multiple drivers are available for a format such as JPEG2000, one of them can be selected by
|
75 |
+
using this keyword argument.
|
76 |
+
|
77 |
+
Returns:
|
78 |
+
cropped raster with its Affine transform
|
79 |
+
"""
|
80 |
+
from rasterio.io import MemoryFile
|
81 |
+
from rasterio.plot import reshape_as_image
|
82 |
+
from rasterio.mask import mask as rio_mask
|
83 |
+
from shapely.geometry import Polygon
|
84 |
+
from geopandas import GeoSeries
|
85 |
+
|
86 |
+
app_logger.debug(f"raster: type {type(raster)}, raster_ext:{type(raster_bbox)}, {raster_bbox}.")
|
87 |
+
img_to_save, transform = get_transform_raster(raster, raster_bbox)
|
88 |
+
img_height, img_width, number_bands = img_to_save.shape
|
89 |
+
# https://rasterio.readthedocs.io/en/latest/topics/memory-files.html
|
90 |
+
with MemoryFile() as rio_mem_file:
|
91 |
+
app_logger.debug("writing raster in-memory to crop it with rasterio.mask.mask()")
|
92 |
+
with rio_mem_file.open(
|
93 |
+
driver=driver,
|
94 |
+
height=img_height,
|
95 |
+
width=img_width,
|
96 |
+
count=number_bands,
|
97 |
+
dtype=str(img_to_save.dtype.name),
|
98 |
+
crs=crs,
|
99 |
+
transform=transform,
|
100 |
+
) as src_raster_rw:
|
101 |
+
for band in range(number_bands):
|
102 |
+
src_raster_rw.write(img_to_save[:, :, band], band + 1)
|
103 |
+
app_logger.debug("cropping raster in-memory with rasterio.mask.mask()")
|
104 |
+
with rio_mem_file.open() as src_raster_ro:
|
105 |
+
shapes_crop_polygon = Polygon([(n, e), (s, e), (s, w), (n, w), (n, e)])
|
106 |
+
shapes_crop = GeoSeries([shapes_crop_polygon])
|
107 |
+
app_logger.debug(f"cropping with polygon::{shapes_crop_polygon}.")
|
108 |
+
cropped_image, cropped_transform = rio_mask(src_raster_ro, shapes=shapes_crop, crop=True)
|
109 |
+
cropped_image_ndarray = reshape_as_image(cropped_image)
|
110 |
+
app_logger.info(f"cropped image::{cropped_image_ndarray.shape}.")
|
111 |
+
return cropped_image_ndarray, cropped_transform
|
112 |
+
|
113 |
+
|
114 |
+
def get_transform_raster(raster: ndarray, raster_bbox: tuple_float) -> tuple_ndarray_transform:
|
115 |
+
"""
|
116 |
+
Convert the input raster image to RGB and extract the Affine
|
117 |
+
|
118 |
+
Args:
|
119 |
+
raster: raster image to geo-reference
|
120 |
+
raster_bbox: bounding box of raster to crop
|
121 |
+
|
122 |
+
Returns:
|
123 |
+
rgb raster image and its Affine transform
|
124 |
+
"""
|
125 |
+
from rasterio.transform import from_origin
|
126 |
+
from numpy import array as np_array, linspace as np_linspace, uint8 as np_uint8
|
127 |
+
from PIL.Image import fromarray
|
128 |
+
|
129 |
+
app_logger.debug(f"raster: type {type(raster)}, raster_ext:{type(raster_bbox)}, {raster_bbox}.")
|
130 |
+
rgb = fromarray(np_uint8(raster)).convert('RGB')
|
131 |
+
np_rgb = np_array(rgb)
|
132 |
+
img_height, img_width, _ = np_rgb.shape
|
133 |
+
|
134 |
+
min_x, max_x, min_y, max_y = raster_bbox
|
135 |
+
app_logger.debug(f"raster rgb shape:{np_rgb.shape}, raster rgb bbox {raster_bbox}.")
|
136 |
+
x = np_linspace(min_x, max_x, img_width)
|
137 |
+
y = np_linspace(min_y, max_y, img_height)
|
138 |
+
res_x = (x[-1] - x[0]) / img_width
|
139 |
+
res_y = (y[-1] - y[0]) / img_height
|
140 |
+
transform = from_origin(x[0] - res_x / 2, y[-1] + res_y / 2, res_x, res_y)
|
141 |
+
return np_rgb, transform
|
src/prediction_api/predictors.py
CHANGED
@@ -1,21 +1,19 @@
|
|
1 |
"""functions using machine learning instance model(s)"""
|
2 |
-
from
|
3 |
-
from numpy import array as np_array, uint8, zeros
|
4 |
|
5 |
from src import app_logger, MODEL_FOLDER
|
6 |
-
from src.io.geo_helpers import get_vectorized_raster_as_geojson
|
7 |
from src.io.tms2geotiff import download_extent
|
8 |
from src.prediction_api.sam_onnx import SegmentAnythingONNX
|
9 |
from src.utilities.constants import MODEL_ENCODER_NAME, MODEL_DECODER_NAME, DEFAULT_TMS
|
10 |
-
from src.utilities.type_hints import llist_float, dict_str_int, list_dict, tuple_ndarr_int
|
11 |
-
|
12 |
|
13 |
models_dict = {"fastsam": {"instance": None}}
|
14 |
|
15 |
|
16 |
def samexporter_predict(
|
17 |
bbox: llist_float,
|
18 |
-
prompt:
|
19 |
zoom: float,
|
20 |
model_name: str = "fastsam",
|
21 |
url_tile: str = DEFAULT_TMS
|
@@ -31,7 +29,7 @@ def samexporter_predict(
|
|
31 |
Args:
|
32 |
bbox: coordinates bounding box
|
33 |
prompt: machine learning input prompt
|
34 |
-
zoom:
|
35 |
model_name: machine learning model name
|
36 |
url_tile: server url tile
|
37 |
|
@@ -51,22 +49,20 @@ def samexporter_predict(
|
|
51 |
app_logger.info(f'tile_source: {url_tile}!')
|
52 |
pt0, pt1 = bbox
|
53 |
app_logger.info(f"downloading geo-referenced raster with bbox {bbox}, zoom {zoom}.")
|
54 |
-
img,
|
55 |
-
app_logger.info(
|
56 |
-
|
57 |
-
transform = get_affine_transform_from_gdal(matrix)
|
58 |
-
app_logger.debug(f"transform to consume with rasterio.shapes: {type(transform)}, {transform}.")
|
59 |
|
60 |
mask, n_predictions = get_raster_inference(img, prompt, models_instance, model_name)
|
61 |
app_logger.info(f"created {n_predictions} masks, preparing conversion to geojson...")
|
62 |
return {
|
63 |
"n_predictions": n_predictions,
|
64 |
-
**get_vectorized_raster_as_geojson(mask,
|
65 |
}
|
66 |
|
67 |
|
68 |
def get_raster_inference(
|
69 |
-
img:
|
70 |
) -> tuple_ndarr_int:
|
71 |
"""
|
72 |
Wrapper for rasterio Affine from_gdal method
|
|
|
1 |
"""functions using machine learning instance model(s)"""
|
2 |
+
from numpy import array as np_array, uint8, zeros, ndarray
|
|
|
3 |
|
4 |
from src import app_logger, MODEL_FOLDER
|
5 |
+
from src.io.geo_helpers import get_vectorized_raster_as_geojson
|
6 |
from src.io.tms2geotiff import download_extent
|
7 |
from src.prediction_api.sam_onnx import SegmentAnythingONNX
|
8 |
from src.utilities.constants import MODEL_ENCODER_NAME, MODEL_DECODER_NAME, DEFAULT_TMS
|
9 |
+
from src.utilities.type_hints import llist_float, dict_str_int, list_dict, tuple_ndarr_int, PIL_Image
|
|
|
10 |
|
11 |
models_dict = {"fastsam": {"instance": None}}
|
12 |
|
13 |
|
14 |
def samexporter_predict(
|
15 |
bbox: llist_float,
|
16 |
+
prompt: list_dict,
|
17 |
zoom: float,
|
18 |
model_name: str = "fastsam",
|
19 |
url_tile: str = DEFAULT_TMS
|
|
|
29 |
Args:
|
30 |
bbox: coordinates bounding box
|
31 |
prompt: machine learning input prompt
|
32 |
+
zoom: Level of detail
|
33 |
model_name: machine learning model name
|
34 |
url_tile: server url tile
|
35 |
|
|
|
49 |
app_logger.info(f'tile_source: {url_tile}!')
|
50 |
pt0, pt1 = bbox
|
51 |
app_logger.info(f"downloading geo-referenced raster with bbox {bbox}, zoom {zoom}.")
|
52 |
+
img, transform = download_extent(w=pt1[1], s=pt1[0], e=pt0[1], n=pt0[0], zoom=zoom, source=url_tile)
|
53 |
+
app_logger.info(
|
54 |
+
f"img type {type(img)} with shape/size:{img.size}, transform type: {type(transform)}, transform:{transform}.")
|
|
|
|
|
55 |
|
56 |
mask, n_predictions = get_raster_inference(img, prompt, models_instance, model_name)
|
57 |
app_logger.info(f"created {n_predictions} masks, preparing conversion to geojson...")
|
58 |
return {
|
59 |
"n_predictions": n_predictions,
|
60 |
+
**get_vectorized_raster_as_geojson(mask, transform)
|
61 |
}
|
62 |
|
63 |
|
64 |
def get_raster_inference(
|
65 |
+
img: PIL_Image or ndarray, prompt: list_dict, models_instance: SegmentAnythingONNX, model_name: str
|
66 |
) -> tuple_ndarr_int:
|
67 |
"""
|
68 |
Wrapper for rasterio Affine from_gdal method
|
src/utilities/constants.py
CHANGED
@@ -1,6 +1,7 @@
|
|
1 |
"""Project constants"""
|
2 |
INPUT_CRS_STRING = "EPSG:4326"
|
3 |
OUTPUT_CRS_STRING = "EPSG:3857"
|
|
|
4 |
ROOT = "/tmp"
|
5 |
CUSTOM_RESPONSE_MESSAGES = {
|
6 |
200: "ok",
|
@@ -25,3 +26,8 @@ DEFAULT_LOG_LEVEL = 'INFO'
|
|
25 |
RETRY_DOWNLOAD = 3
|
26 |
TIMEOUT_DOWNLOAD = 60
|
27 |
CALLBACK_INTERVAL_DOWNLOAD = 0.05
|
|
|
|
|
|
|
|
|
|
|
|
1 |
"""Project constants"""
|
2 |
INPUT_CRS_STRING = "EPSG:4326"
|
3 |
OUTPUT_CRS_STRING = "EPSG:3857"
|
4 |
+
DRIVER_RASTERIO_GTIFF = "GTiff"
|
5 |
ROOT = "/tmp"
|
6 |
CUSTOM_RESPONSE_MESSAGES = {
|
7 |
200: "ok",
|
|
|
26 |
RETRY_DOWNLOAD = 3
|
27 |
TIMEOUT_DOWNLOAD = 60
|
28 |
CALLBACK_INTERVAL_DOWNLOAD = 0.05
|
29 |
+
OSM_USE_CACHE = True
|
30 |
+
OSM_WAIT = 0
|
31 |
+
OSM_MAX_RETRIES = 2
|
32 |
+
OSM_N_CONNECTIONS = 1
|
33 |
+
OSM_ZOOM_AUTO = "auto"
|
src/utilities/type_hints.py
CHANGED
@@ -3,6 +3,7 @@ from enum import Enum
|
|
3 |
from typing import TypedDict
|
4 |
|
5 |
from PIL.Image import Image
|
|
|
6 |
from numpy import ndarray
|
7 |
from pydantic import BaseModel
|
8 |
|
@@ -21,6 +22,7 @@ llist_float = list[list_float]
|
|
21 |
tuple_float = tuple[float]
|
22 |
tuple_float_any = tuple[float, any]
|
23 |
PIL_Image = Image
|
|
|
24 |
|
25 |
|
26 |
class LatLngDict(BaseModel):
|
|
|
3 |
from typing import TypedDict
|
4 |
|
5 |
from PIL.Image import Image
|
6 |
+
from affine import Affine
|
7 |
from numpy import ndarray
|
8 |
from pydantic import BaseModel
|
9 |
|
|
|
22 |
tuple_float = tuple[float]
|
23 |
tuple_float_any = tuple[float, any]
|
24 |
PIL_Image = Image
|
25 |
+
tuple_ndarray_transform = tuple[ndarray, Affine]
|
26 |
|
27 |
|
28 |
class LatLngDict(BaseModel):
|