aletrn commited on
Commit
3209d49
·
1 Parent(s): d071ba9

[doc] update some docstrings

Browse files
src/io/coordinates_pixel_conversion.py CHANGED
@@ -3,8 +3,8 @@ import math
3
 
4
  from src import app_logger
5
  from src.utilities.constants import TILE_SIZE
6
- from src.utilities.type_hints import LatLngDict
7
  from src.utilities.type_hints import ImagePixelCoordinates
 
8
 
9
 
10
  def _get_latlng2pixel_projection(latlng: LatLngDict) -> ImagePixelCoordinates:
@@ -50,7 +50,7 @@ def get_latlng_to_pixel_coordinates(
50
  k: str
51
  ) -> ImagePixelCoordinates:
52
  """
53
- Parse the input request lambda event.
54
 
55
  Args:
56
  latlng_origin_ne: NE latitude-longitude origin point
 
3
 
4
  from src import app_logger
5
  from src.utilities.constants import TILE_SIZE
 
6
  from src.utilities.type_hints import ImagePixelCoordinates
7
+ from src.utilities.type_hints import LatLngDict
8
 
9
 
10
  def _get_latlng2pixel_projection(latlng: LatLngDict) -> ImagePixelCoordinates:
 
50
  k: str
51
  ) -> ImagePixelCoordinates:
52
  """
53
+ Parse the input request lambda event
54
 
55
  Args:
56
  latlng_origin_ne: NE latitude-longitude origin point
src/io/geo_helpers.py CHANGED
@@ -2,28 +2,29 @@
2
  from pathlib import Path
3
  from typing import List, Tuple, Dict
4
 
5
- from affine import Affine
6
  import numpy as np
 
7
 
8
  from src import app_logger, PROJECT_ROOT_FOLDER
9
 
10
 
11
- def load_affine_transformation_from_matrix(matrix_source_coeffs: List[float]) -> Affine:
12
- """wrapper for rasterio Affine from_gdal method
 
13
 
14
  Args:
15
- matrix_source_coeffs: 6 floats ordered by GDAL.
16
 
17
  Returns:
18
- Affine: Affine transform
19
  """
20
 
21
- if len(matrix_source_coeffs) != 6:
22
- raise ValueError(f"Expected 6 coefficients, found {len(matrix_source_coeffs)}; "
23
- f"argument type: {type(matrix_source_coeffs)}.")
24
 
25
  try:
26
- a, d, b, e, c, f = (float(x) for x in matrix_source_coeffs)
27
  center = tuple.__new__(Affine, [a, b, c, d, e, f, 0.0, 0.0, 1.0])
28
  return center * Affine.translation(-0.5, -0.5)
29
  except Exception as e:
@@ -31,28 +32,28 @@ def load_affine_transformation_from_matrix(matrix_source_coeffs: List[float]) ->
31
  raise e
32
 
33
 
34
- def get_affine_transform_from_gdal(matrix_source_coeffs: List[float] or Tuple[float]) -> Affine:
35
  """wrapper for rasterio Affine from_gdal method
36
 
37
  Args:
38
- matrix_source_coeffs: 6 floats ordered by GDAL.
39
 
40
  Returns:
41
- Affine: Affine transform
42
  """
43
- return Affine.from_gdal(*matrix_source_coeffs)
44
 
45
 
46
  def get_vectorized_raster_as_geojson(mask: np.ndarray, matrix: Tuple[float]) -> Dict[str, int]:
47
  """
48
- Parse the input request lambda event.
49
 
50
  Args:
51
  mask: numpy mask
52
  matrix: tuple of float to transform into an Affine transform
53
 
54
  Returns:
55
- Dict: dict containing the output geojson and the predictions number
56
  """
57
  try:
58
  from rasterio.features import shapes
@@ -65,7 +66,7 @@ def get_vectorized_raster_as_geojson(mask: np.ndarray, matrix: Tuple[float]) ->
65
  shapes_generator = ({
66
  'properties': {'raster_val': v}, 'geometry': s}
67
  for i, (s, v)
68
- # in enumerate(shapes(mask, mask=(band != 0), transform=rio_src.transform))
69
  # use mask=None to avoid using source
70
  in enumerate(shapes(mask, mask=None, transform=transform))
71
  )
 
2
  from pathlib import Path
3
  from typing import List, Tuple, Dict
4
 
 
5
  import numpy as np
6
+ from affine import Affine
7
 
8
  from src import app_logger, PROJECT_ROOT_FOLDER
9
 
10
 
11
+ def load_affine_transformation_from_matrix(matrix_source_coefficients: List[float]) -> Affine:
12
+ """
13
+ Wrapper for rasterio.Affine.from_gdal() method
14
 
15
  Args:
16
+ matrix_source_coefficients: 6 floats ordered by GDAL.
17
 
18
  Returns:
19
+ Affine transform
20
  """
21
 
22
+ if len(matrix_source_coefficients) != 6:
23
+ raise ValueError(f"Expected 6 coefficients, found {len(matrix_source_coefficients)}; "
24
+ f"argument type: {type(matrix_source_coefficients)}.")
25
 
26
  try:
27
+ a, d, b, e, c, f = (float(x) for x in matrix_source_coefficients)
28
  center = tuple.__new__(Affine, [a, b, c, d, e, f, 0.0, 0.0, 1.0])
29
  return center * Affine.translation(-0.5, -0.5)
30
  except Exception as e:
 
32
  raise e
33
 
34
 
35
+ def get_affine_transform_from_gdal(matrix_source_coefficients: List[float] or Tuple[float]) -> Affine:
36
  """wrapper for rasterio Affine from_gdal method
37
 
38
  Args:
39
+ matrix_source_coefficients: 6 floats ordered by GDAL.
40
 
41
  Returns:
42
+ Affine transform
43
  """
44
+ return Affine.from_gdal(*matrix_source_coefficients)
45
 
46
 
47
  def get_vectorized_raster_as_geojson(mask: np.ndarray, matrix: Tuple[float]) -> Dict[str, int]:
48
  """
49
+ Get shapes and values of connected regions in a dataset or array
50
 
51
  Args:
52
  mask: numpy mask
53
  matrix: tuple of float to transform into an Affine transform
54
 
55
  Returns:
56
+ dict containing the output geojson and the predictions number
57
  """
58
  try:
59
  from rasterio.features import shapes
 
66
  shapes_generator = ({
67
  'properties': {'raster_val': v}, 'geometry': s}
68
  for i, (s, v)
69
+ # instead of `enumerate(shapes(mask, mask=(band != 0), transform=rio_src.transform))`
70
  # use mask=None to avoid using source
71
  in enumerate(shapes(mask, mask=None, transform=transform))
72
  )
src/io/lambda_helpers.py CHANGED
@@ -2,7 +2,7 @@
2
  import json
3
  import logging
4
  import time
5
- from typing import Dict, List
6
  from aws_lambda_powertools.event_handler import content_types
7
 
8
  from src import app_logger
@@ -14,7 +14,7 @@ from src.utilities.utilities import base64_decode
14
 
15
  def get_response(status: int, start_time: float, request_id: str, response_body: Dict = None) -> str:
16
  """
17
- Return a response for frontend clients.
18
 
19
  Args:
20
  status: status response
@@ -23,7 +23,7 @@ def get_response(status: int, start_time: float, request_id: str, response_body:
23
  response_body: dict we embed into our response
24
 
25
  Returns:
26
- str: json response
27
 
28
  """
29
  app_logger.debug(f"response_body:{response_body}.")
@@ -43,13 +43,13 @@ def get_response(status: int, start_time: float, request_id: str, response_body:
43
 
44
  def get_parsed_bbox_points(request_input: RawRequestInput) -> Dict:
45
  """
46
- Format the bbox and prompt request input
47
 
48
  Args:
49
  request_input: input dict
50
 
51
  Returns:
52
- Dict:
53
  """
54
  app_logger.info(f"try to parsing input request {request_input}...")
55
 
@@ -88,15 +88,15 @@ def get_parsed_bbox_points(request_input: RawRequestInput) -> Dict:
88
  }
89
 
90
 
91
- def get_parsed_request_body(event: Dict):
92
  """
93
- Parse the input request lambda event.
94
 
95
  Args:
96
  event: input dict
97
 
98
  Returns:
99
- RawRequestInput: parsed request input
100
  """
101
  app_logger.info(f"event:{json.dumps(event)}...")
102
  try:
 
2
  import json
3
  import logging
4
  import time
5
+ from typing import Dict
6
  from aws_lambda_powertools.event_handler import content_types
7
 
8
  from src import app_logger
 
14
 
15
  def get_response(status: int, start_time: float, request_id: str, response_body: Dict = None) -> str:
16
  """
17
+ Response composer
18
 
19
  Args:
20
  status: status response
 
23
  response_body: dict we embed into our response
24
 
25
  Returns:
26
+ json response
27
 
28
  """
29
  app_logger.debug(f"response_body:{response_body}.")
 
43
 
44
  def get_parsed_bbox_points(request_input: RawRequestInput) -> Dict:
45
  """
46
+ Parse the raw input request into bbox, prompt and zoom
47
 
48
  Args:
49
  request_input: input dict
50
 
51
  Returns:
52
+ dict with bounding box, prompt and zoom
53
  """
54
  app_logger.info(f"try to parsing input request {request_input}...")
55
 
 
88
  }
89
 
90
 
91
+ def get_parsed_request_body(event: Dict) -> RawRequestInput:
92
  """
93
+ Validator for the raw input request lambda event
94
 
95
  Args:
96
  event: input dict
97
 
98
  Returns:
99
+ parsed request input
100
  """
101
  app_logger.info(f"event:{json.dumps(event)}...")
102
  try:
src/io/tms2geotiff.py CHANGED
@@ -28,19 +28,20 @@ CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
28
  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
  """
31
-
32
  import concurrent.futures
33
  import io
34
  import itertools
35
  import math
36
  import re
37
  import time
38
-
 
39
  from PIL import Image
40
 
41
  from src import app_logger
42
  from src.utilities.constants import EARTH_EQUATORIAL_RADIUS, RETRY_DOWNLOAD, TIMEOUT_DOWNLOAD, TILE_SIZE, \
43
  CALLBACK_INTERVAL_DOWNLOAD
 
44
 
45
  Image.MAX_IMAGE_PIXELS = None
46
 
@@ -140,10 +141,27 @@ def print_progress(progress, total, done=False):
140
 
141
 
142
  def download_extent(
143
- source, lat0, lon0, lat1, lon1, zoom,
144
- save_image=True, progress_callback=print_progress,
145
- callback_interval=CALLBACK_INTERVAL_DOWNLOAD
146
- ):
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
147
  x0, y0 = deg2num(lat0, lon0, zoom)
148
  x1, y1 = deg2num(lat1, lon1, zoom)
149
  if x0 > x1:
@@ -185,16 +203,16 @@ def download_extent(
185
  y2 = round(base_size[1] * y_frac)
186
  img_w = round(base_size[0] * (x1 - x0))
187
  img_h = round(base_size[1] * (y1 - y0))
188
- ret_im = big_im.crop((x2, y2, x2 + img_w, y2 + img_h))
189
- if ret_im.mode == 'RGBA' and ret_im.getextrema()[3] == (255, 255):
190
- ret_im = ret_im.convert('RGB')
191
  big_im.close()
192
  xp0, yp0 = from4326_to3857(lat0, lon0)
193
  xp1, yp1 = from4326_to3857(lat1, lon1)
194
- p_width = abs(xp1 - xp0) / ret_im.size[0]
195
- p_height = abs(yp1 - yp0) / ret_im.size[1]
196
- matrix = (min(xp0, xp1), p_width, 0, max(yp0, yp1), 0, -p_height)
197
- return ret_im, matrix
198
 
199
 
200
  def run_future_tile_download(
 
28
  OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29
  OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
  """
 
31
  import concurrent.futures
32
  import io
33
  import itertools
34
  import math
35
  import re
36
  import time
37
+ from typing import Tuple, Callable
38
+ import PIL
39
  from PIL import Image
40
 
41
  from src import app_logger
42
  from src.utilities.constants import EARTH_EQUATORIAL_RADIUS, RETRY_DOWNLOAD, TIMEOUT_DOWNLOAD, TILE_SIZE, \
43
  CALLBACK_INTERVAL_DOWNLOAD
44
+ from src.utilities.type_hints import PIL_Image
45
 
46
  Image.MAX_IMAGE_PIXELS = None
47
 
 
141
 
142
 
143
  def download_extent(
144
+ source: str, lat0: float, lon0: float, lat1: float, lon1: float, zoom: int,
145
+ save_image: bool = True, progress_callback: Callable = print_progress,
146
+ callback_interval: float = CALLBACK_INTERVAL_DOWNLOAD
147
+ ) -> Tuple[PIL_Image, Tuple[float]] or Tuple[None]:
148
+ """
149
+ Download, merge and crop a list of tiles into a single geo-referenced image or a raster geodata
150
+
151
+ Args:
152
+ source: remote url tile
153
+ lat0: point0 bounding box latitude
154
+ lat1: point0 bounding box longitude
155
+ lon0: point1 bounding box latitude
156
+ lon1: point1 bounding box longitude
157
+ zoom: bounding box zoom
158
+ save_image: boolean to choose if save the image
159
+ progress_callback: callback function
160
+ callback_interval: process callback interval time
161
+
162
+ Returns:
163
+ parsed request input
164
+ """
165
  x0, y0 = deg2num(lat0, lon0, zoom)
166
  x1, y1 = deg2num(lat1, lon1, zoom)
167
  if x0 > x1:
 
203
  y2 = round(base_size[1] * y_frac)
204
  img_w = round(base_size[0] * (x1 - x0))
205
  img_h = round(base_size[1] * (y1 - y0))
206
+ final_image = big_im.crop((x2, y2, x2 + img_w, y2 + img_h))
207
+ if final_image.mode == 'RGBA' and final_image.getextrema()[3] == (255, 255):
208
+ final_image = final_image.convert('RGB')
209
  big_im.close()
210
  xp0, yp0 = from4326_to3857(lat0, lon0)
211
  xp1, yp1 = from4326_to3857(lat1, lon1)
212
+ p_width = abs(xp1 - xp0) / final_image.size[0]
213
+ p_height = abs(yp1 - yp0) / final_image.size[1]
214
+ matrix = min(xp0, xp1), p_width, 0, max(yp0, yp1), 0, -p_height
215
+ return final_image, matrix
216
 
217
 
218
  def run_future_tile_download(
src/utilities/type_hints.py CHANGED
@@ -1,50 +1,54 @@
1
  """custom type hints"""
2
  from enum import Enum
 
3
 
 
4
  from pydantic import BaseModel
5
- from typing import TypedDict
6
 
7
  from src.utilities.constants import DEFAULT_TMS
8
 
 
9
  ts_dict_str2 = dict[str, str]
10
  ts_dict_str3 = dict[str, str, any]
11
  ts_ddict1 = dict[str, dict[str, any], dict, dict, any]
12
  list_float = list[float]
13
  llist_float = list[list_float]
 
14
 
15
 
16
  class LatLngDict(BaseModel):
17
- """A latitude-longitude type"""
18
  lat: float
19
  lng: float
20
 
21
 
22
  class PromptType(str, Enum):
23
  """Segment Anything enumeration prompt type"""
24
- point = "point"
25
  # rectangle = "rectangle"
 
26
 
27
 
28
  class ImagePixelCoordinates(TypedDict):
 
29
  x: int
30
  y: int
31
 
32
 
33
  class RawBBox(BaseModel):
34
- """Input lambda bbox request - not parsed"""
35
  ne: LatLngDict
36
  sw: LatLngDict
37
 
38
 
39
  class RawPrompt(BaseModel):
40
- """Input lambda prompt request - not parsed"""
41
  type: PromptType
42
  data: LatLngDict
43
  label: int = 0
44
 
45
 
46
  class RawRequestInput(BaseModel):
47
- """Input lambda request - not parsed"""
48
  bbox: RawBBox
49
  prompt: list[RawPrompt]
50
  zoom: int | float
 
1
  """custom type hints"""
2
  from enum import Enum
3
+ from typing import TypedDict
4
 
5
+ from PIL.Image import Image
6
  from pydantic import BaseModel
 
7
 
8
  from src.utilities.constants import DEFAULT_TMS
9
 
10
+
11
  ts_dict_str2 = dict[str, str]
12
  ts_dict_str3 = dict[str, str, any]
13
  ts_ddict1 = dict[str, dict[str, any], dict, dict, any]
14
  list_float = list[float]
15
  llist_float = list[list_float]
16
+ PIL_Image = Image
17
 
18
 
19
  class LatLngDict(BaseModel):
20
+ """Generic geographic latitude-longitude type"""
21
  lat: float
22
  lng: float
23
 
24
 
25
  class PromptType(str, Enum):
26
  """Segment Anything enumeration prompt type"""
 
27
  # rectangle = "rectangle"
28
+ point = "point"
29
 
30
 
31
  class ImagePixelCoordinates(TypedDict):
32
+ """Image pixel coordinates type"""
33
  x: int
34
  y: int
35
 
36
 
37
  class RawBBox(BaseModel):
38
+ """Input lambda bbox request type (not parsed)"""
39
  ne: LatLngDict
40
  sw: LatLngDict
41
 
42
 
43
  class RawPrompt(BaseModel):
44
+ """Input lambda prompt request type (not parsed)"""
45
  type: PromptType
46
  data: LatLngDict
47
  label: int = 0
48
 
49
 
50
  class RawRequestInput(BaseModel):
51
+ """Input lambda request validator type (not parsed)"""
52
  bbox: RawBBox
53
  prompt: list[RawPrompt]
54
  zoom: int | float
tests/test_app.py CHANGED
@@ -14,7 +14,7 @@ class TestAppFailures(unittest.TestCase):
14
  @patch.object(app, "samexporter_predict")
15
  @patch.object(app, "get_parsed_bbox_points")
16
  @patch.object(app, "get_parsed_request_body")
17
- def test_lambda_handler_400(
18
  self,
19
  get_parsed_request_body_mocked,
20
  get_parsed_bbox_points_mocked,
@@ -35,16 +35,15 @@ class TestAppFailures(unittest.TestCase):
35
  cognito_identity=None,
36
  epoch_deadline_time_in_ms=time.time()
37
  )
38
- expected_response_400 = '{"statusCode": 400, "header": {"Content-Type": "application/json"}, '
39
- expected_response_400 += '"body": "{\\"duration_run\\": 0, \\"message\\": \\"Bad Request\\", '
40
- expected_response_400 += '\\"request_id\\": \\"test_invoke_id\\"}", "isBase64Encoded": false}'
41
 
42
- response_400 = lambda_handler(event, lambda_context)
43
- assert response_400 == expected_response_400
44
 
45
  @patch.object(time, "time")
46
  @patch.object(app, "get_parsed_request_body")
47
- def test_lambda_handler_500(self, get_parsed_request_body_mocked, time_mocked):
48
  from src.app import lambda_handler
49
 
50
  time_mocked.return_value = 0
@@ -58,14 +57,10 @@ class TestAppFailures(unittest.TestCase):
58
  epoch_deadline_time_in_ms=time.time()
59
  )
60
 
61
- response_500 = lambda_handler(event, lambda_context)
62
- check_500 = response_500 == (
63
- '{"statusCode": 500, "header": {"Content-Type": "application/json"}, '
64
- '"body": "{\\"duration_run\\": 0, \\"message\\": \\"Internal server error\\", '
65
  '\\"request_id\\": \\"test_invoke_id\\"}", "isBase64Encoded": false}')
66
- print(f"test_lambda_handler_422:{check_500}.")
67
- assert check_500
68
- print("check_500")
69
 
70
  @patch.object(time, "time")
71
  def test_lambda_handler_422(self, time_mocked):
 
14
  @patch.object(app, "samexporter_predict")
15
  @patch.object(app, "get_parsed_bbox_points")
16
  @patch.object(app, "get_parsed_request_body")
17
+ def test_lambda_handler_500(
18
  self,
19
  get_parsed_request_body_mocked,
20
  get_parsed_bbox_points_mocked,
 
35
  cognito_identity=None,
36
  epoch_deadline_time_in_ms=time.time()
37
  )
38
+ expected_response_500 = '{"statusCode": 500, "header": {"Content-Type": "application/json"}, '
39
+ expected_response_500 += '"body": "{\\"duration_run\\": 0, \\"message\\": \\"Internal server error\\", '
40
+ expected_response_500 += '\\"request_id\\": \\"test_invoke_id\\"}", "isBase64Encoded": false}'
41
 
42
+ assert lambda_handler(event, lambda_context) == expected_response_500
 
43
 
44
  @patch.object(time, "time")
45
  @patch.object(app, "get_parsed_request_body")
46
+ def test_lambda_handler_400(self, get_parsed_request_body_mocked, time_mocked):
47
  from src.app import lambda_handler
48
 
49
  time_mocked.return_value = 0
 
57
  epoch_deadline_time_in_ms=time.time()
58
  )
59
 
60
+ assert lambda_handler(event, lambda_context) == (
61
+ '{"statusCode": 400, "header": {"Content-Type": "application/json"}, '
62
+ '"body": "{\\"duration_run\\": 0, \\"message\\": \\"Bad Request\\", '
 
63
  '\\"request_id\\": \\"test_invoke_id\\"}", "isBase64Encoded": false}')
 
 
 
64
 
65
  @patch.object(time, "time")
66
  def test_lambda_handler_422(self, time_mocked):