[feat] revert to aws lambda - wip

.dockerignore

@@ -3,4 +3,5 @@ venv/
 __cache__
 .idea
 tmp/
-.env*
+.env*
+__pycache__

Dockerfile

@@ -1,38 +1,46 @@
-FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.2
+# inspired by https://dev.to/gaborschulz/running-python-311-on-aws-lambda-1i7p
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.2 as build-image
 
-WORKDIR /code
-COPY ./requirements.txt /code/requirements.txt
-COPY ./requirements_pip.txt /code/requirements_pip.txt
+LABEL maintainer="alessandro trinca <alessandro@trinca.tornidor.com>"
 
-RUN apt update && apt install -y g++ make cmake unzip libcurl4-openssl-dev python3-pip
+# Include global arg in this stage of the build
+ARG LAMBDA_TASK_ROOT="/var/task"
+ARG PYTHONPATH="${LAMBDA_TASK_ROOT}:${PYTHONPATH}:/usr/local/lib/python3/dist-packages"
 
-# avoid segment-geospatial exception caused by missing libGL.so.1 library
-RUN apt install -y libgl1 curl
-RUN ls -ld /usr/lib/x86_64-linux-gnu/libGL.so* || echo "libGL.so* not found..."
+RUN mkdir -p ${LAMBDA_TASK_ROOT}
+
+# Install aws-lambda-cpp build dependencies
+RUN apt update && \
+  apt install -y g++ make cmake unzip libcurl4-openssl-dev python3-pip
 
-RUN which python
-RUN python --version
-RUN python -m pip install --no-cache-dir --upgrade -r /code/requirements_pip.txt
-RUN python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
-RUN python -m pip install --no-cache-dir -r /code/requirements.txt
+# install required packages
+COPY requirements_pip.txt ${LAMBDA_TASK_ROOT}/
+RUN python -m pip install --target ${LAMBDA_TASK_ROOT} --upgrade -r ${LAMBDA_TASK_ROOT}/requirements_pip.txt
+RUN python -m pip install torch torchvision --target ${LAMBDA_TASK_ROOT} --index-url https://download.pytorch.org/whl/cpu
+COPY requirements.txt ${LAMBDA_TASK_ROOT}/
+RUN python -m pip install --target ${LAMBDA_TASK_ROOT} -r ${LAMBDA_TASK_ROOT}/requirements.txt
 
-RUN useradd -m -u 1000 user
+FROM osgeo/gdal:ubuntu-small-3.7.2
 
-USER user
+# Include global arg in this stage of the build
+ARG LAMBDA_TASK_ROOT="/var/task"
+ARG PYTHONPATH="${LAMBDA_TASK_ROOT}:${PYTHONPATH}:/usr/local/lib/python3/dist-packages"
+ARG RIE="https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie"
 
-ENV HOME=/home/user \
-	PATH=/home/user/.local/bin:$PATH
+# Set working directory to function root directory
+WORKDIR ${LAMBDA_TASK_ROOT}
 
-WORKDIR $HOME/app
+RUN apt update && apt install -y libgl1 curl
+RUN ls -ld /usr/lib/x86_64-linux-gnu/libGL.so* || echo "libGL.so* not found..."
 
-RUN curl -o ${HOME}/sam_vit_h_4b8939.pth https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth
-RUN ls -l ${HOME}/
-COPY --chown=user . $HOME/app
+# Copy in the built dependencies
+COPY --from=build-image ${LAMBDA_TASK_ROOT} ${LAMBDA_TASK_ROOT}
 
-RUN echo $HOME/app
-RUN echo $HOME/
+RUN curl -Lo /usr/local/bin/aws-lambda-rie ${RIE}
+RUN chmod +x /usr/local/bin/aws-lambda-rie
 
-RUN ls -l $HOME/app
-RUN ls -l $HOME/
+COPY ./scripts/lambda-entrypoint.sh /lambda-entrypoint.sh
+RUN chmod +x /lambda-entrypoint.sh
+RUN ls -l /lambda-entrypoint.sh
 
-CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "7860"]
+ENTRYPOINT  ["/lambda-entrypoint.sh"]

dockerfiles/dockerfile-base-webserver

@@ -1,22 +1,24 @@
-FROM python:3.11
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.2
 
-WORKDIR /code
+# Include global arg in this stage of the build
+ARG LAMBDA_TASK_ROOT="/var/task"
+ARG PYTHONPATH="${LAMBDA_TASK_ROOT}:${PYTHONPATH}:/usr/local/lib/python3/dist-packages"
+ARG RIE="https://github.com/aws/aws-lambda-runtime-interface-emulator/releases/latest/download/aws-lambda-rie"
 
-RUN which python
-RUN python --version
-RUN python -m pip install --no-cache-dir fastapi uvicorn loguru
+# Set working directory to function root directory
+WORKDIR ${LAMBDA_TASK_ROOT}
 
-RUN useradd -m -u 1000 user
+RUN curl -Lo /usr/local/bin/aws-lambda-rie ${RIE}
+RUN chmod +x /usr/local/bin/aws-lambda-rie
 
-USER user
+COPY ./scripts/lambda-entrypoint.sh /lambda-entrypoint.sh
+RUN chmod +x /lambda-entrypoint.sh
+RUN ls -l /lambda-entrypoint.sh
 
-ENV HOME=/home/user \
-	PATH=/home/user/.local/bin:$PATH
+COPY ./src ${LAMBDA_TASK_ROOT}/src
 
-WORKDIR $HOME/app
+RUN apt update && apt install -y python3-pip
+RUN python -m pip install awslambdaric
 
-RUN ls -l ${HOME}/
-COPY --chown=user src $HOME/app/src
-COPY --chown=user static $HOME/app/static
-
-CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "7860"]
+ENTRYPOINT  ["/lambda-entrypoint.sh"]
+CMD [ "src.app.lambda_handler" ]

dockerfiles/dockerfile-fastapi-samgeo

@@ -0,0 +1,38 @@
+FROM ghcr.io/osgeo/gdal:ubuntu-small-3.7.2
+
+WORKDIR /code
+COPY ./requirements.txt /code/requirements.txt
+COPY ./requirements_pip.txt /code/requirements_pip.txt
+
+RUN apt update && apt install -y g++ make cmake unzip libcurl4-openssl-dev python3-pip
+
+# avoid segment-geospatial exception caused by missing libGL.so.1 library
+RUN apt install -y libgl1 curl
+RUN ls -ld /usr/lib/x86_64-linux-gnu/libGL.so* || echo "libGL.so* not found..."
+
+RUN which python
+RUN python --version
+RUN python -m pip install --no-cache-dir --upgrade -r /code/requirements_pip.txt
+RUN python -m pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
+RUN python -m pip install --no-cache-dir -r /code/requirements.txt
+
+RUN useradd -m -u 1000 user
+
+USER user
+
+ENV HOME=/home/user \
+	PATH=/home/user/.local/bin:$PATH
+
+WORKDIR $HOME/app
+
+RUN curl -o ${HOME}/sam_vit_h_4b8939.pth https://dl.fbaipublicfiles.com/segment_anything/sam_vit_h_4b8939.pth
+RUN ls -l ${HOME}/
+COPY --chown=user . $HOME/app
+
+RUN echo $HOME/app
+RUN echo $HOME/
+
+RUN ls -l $HOME/app
+RUN ls -l $HOME/
+
+CMD ["uvicorn", "src.main:app", "--host", "0.0.0.0", "--port", "7860"]

dockerfiles/dockerfile-samgeo-api

@@ -0,0 +1,31 @@
+FROM 686901913580.dkr.ecr.eu-west-1.amazonaws.com/lambda-gdal-runner:latest
+
+ARG LAMBDA_TASK_ROOT="/var/task"
+ARG PYTHONPATH="${LAMBDA_TASK_ROOT}:${PYTHONPATH}:/usr/local/lib/python3/dist-packages"
+
+COPY ./src ${LAMBDA_TASK_ROOT}/src
+
+COPY .env ${LAMBDA_TASK_ROOT}
+RUN ls -l ${LAMBDA_TASK_ROOT}/.env
+RUN . ${LAMBDA_TASK_ROOT}/.env
+
+RUN ls -l /usr/bin/which
+RUN /usr/bin/which python
+RUN python -v
+RUN echo "PYTHONPATH: ${PYTHONPATH}."
+RUN echo "PATH: ${PATH}."
+RUN echo "LAMBDA_TASK_ROOT: ${LAMBDA_TASK_ROOT}."
+RUN ls -l ${LAMBDA_TASK_ROOT}
+RUN ls -ld ${LAMBDA_TASK_ROOT}
+RUN python -c "import sys; print(sys.path)"
+RUN python -c "import osgeo"
+RUN python -c "import rasterio"
+RUN python -c "import awslambdaric"
+RUN python -m pip list
+RUN python -m pip freeze
+RUN df -h
+
+RUN echo "$(date "+%Y%m%d_%H%M%S")" > ${LAMBDA_TASK_ROOT}/buildtime.txt
+RUN python -m pip freeze > ${LAMBDA_TASK_ROOT}/pythonpackages.txt
+
+CMD [ "src.surferdtm_prediction_api.app.lambda_handler" ]

requirements.txt

@@ -1,6 +1,3 @@
-fastapi
 bson
-loguru
 python-dotenv
 segment-geospatial
-uvicorn[standard]

scripts/copy_folder_to_host.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+echo "options:"
+echo "\$1: container folder we copy from"
+echo "\$2: container folder we copy to (could also be an host folder)"
+
+cp -r "$1" "$2"
+echo "copied folder $1 to folder $2!"
+ls -ld "$2"
+ls -l "$2"
+
+exit 0

scripts/lambda-entrypoint.sh

@@ -0,0 +1,7 @@
+#!/bin/sh
+
+if [ -z "${AWS_LAMBDA_RUNTIME_API}" ]; then
+  exec /usr/local/bin/aws-lambda-rie /usr/bin/python -m awslambdaric "$@"
+else
+  exec /usr/bin/python -m awslambdaric "$@"
+fi

src/__init__.py

@@ -1,4 +0,0 @@
-from src.utilities.utilities import setup_logging
-
-
-app_logger = setup_logging(debug=True)

src/app.py

@@ -0,0 +1,8 @@
+import json
+import logging
+
+
+def lambda_handler(event, context):
+    logging.debug(f"event:{event}...")
+    logging.debug(f"context:{context}...")
+    return {"msg": f"ciao{json.dumps(event)}."}

src/utilities/utilities.py

@@ -26,14 +26,12 @@ def setup_logging(debug: bool = False, formatter: str = "{time} - {level} - ({ex
     return logger
 
 
-def get_constants(event: dict, root: str = ROOT, dotenv_filename: str = ".env", debug=False) -> dict:
+def get_constants(event: dict, debug=False) -> dict:
     """
     Return constants we need to use from event, context and environment variables (both production and test).
-    
+
     Args:
         event: request event
-        root: path containing the dotenv file
-        dotenv_filename: dotenv filename
         debug: logging debug argument
 
     Returns:
@@ -41,10 +39,7 @@ def get_constants(event: dict, root: str = ROOT, dotenv_filename: str = ".env",
 
     """
     import json
-    import os
-    # from dotenv import dotenv_values
 
-    from src.utilities.constants import SKIP_CONDITIONS_LIST
     local_logger = setup_logging(debug)
     try:
         body = event["body"]
@@ -64,24 +59,10 @@ def get_constants(event: dict, root: str = ROOT, dotenv_filename: str = ".env",
     local_logger.debug(f"constants debug:{debug}, log_level:{local_logger.level}, body:{body}.")
 
     try:
-        dotenv_file_path = os.path.join(root, dotenv_filename)
-        local_logger.info(f"root_path:{root}, dotenv file:{dotenv_file_path}.")
-        # secrets = dotenv_values(dotenv_file_path)
-
-        try:
-            skip_conditions_list = body["skip_conditions_list"]
-            local_logger.info(f"found skip_conditions_list, using it: {skip_conditions_list}.")
-        except KeyError:
-            skip_conditions_list = SKIP_CONDITIONS_LIST
-
         return {
-            "bounding_box": body["bounding_box"],
-            "zoom": body["zoom"],
-            "debug": debug,
-            "slope_cellsize": body["slope_cellsize"],
-            "model_project_name": body["model_project_name"],
-            "model_version": body["model_version"],
-            "skip_conditions_list": skip_conditions_list
+            "bbox": body["bbox"],
+            "point": body["point"],
+            "debug": debug
         }
     except KeyError as e_key_constants2:
         local_logger.error(f"e_key_constants2:{e_key_constants2}.")